From 9fec37759cb83ebd267ba8094a94d30900189d89 Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Sun, 3 May 2026 20:35:08 -0400 Subject: [PATCH 01/20] Add repository manifest identity Add the .build-eips.repo.toml schema, loader, validation rules, and manifest tests for active proposal repositories and declared sibling repositories. Introduce ActiveRepoIdentity so later workspace lifecycle and execution layers can select manifest-backed repository metadata while the legacy EIPs/ERCs fallback continues to operate. --- Cargo.lock | 43 +++- Cargo.toml | 3 + src/config.rs | 547 +++++++++++++++++++++++++++++++++++++++++++++++- src/identity.rs | 70 +++++++ src/main.rs | 1 + 5 files changed, 661 insertions(+), 3 deletions(-) create mode 100644 src/identity.rs diff --git a/Cargo.lock b/Cargo.lock index dbca5a3..aa9d0aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -260,6 +260,7 @@ dependencies = [ "serde_json", "sha3", "snafu", + "tempfile", "tokio", "toml 0.9.11+spec-1.1.0", "toml_datetime 0.7.5+spec-1.1.0", @@ -785,6 +786,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + [[package]] name = "figment" version = "0.10.19" @@ -1341,9 +1348,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.180" +version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" [[package]] name = "libgit2-sys" @@ -1395,6 +1402,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.1" @@ -2007,6 +2020,19 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -2424,6 +2450,19 @@ dependencies = [ "libc", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "tendril" version = "0.4.3" diff --git a/Cargo.toml b/Cargo.toml index d87e3e7..4529c21 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,3 +53,6 @@ iref = "3.2.2" [features] backtrace = [ "snafu/backtrace", "eipw-lint/backtrace" ] + +[dev-dependencies] +tempfile = "3.23.0" diff --git a/src/config.rs b/src/config.rs index 26c4ad4..10f2098 100644 --- a/src/config.rs +++ b/src/config.rs @@ -4,11 +4,324 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -use std::collections::HashMap; +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + path::{Path, PathBuf}, +}; use serde::{Deserialize, Serialize}; +use snafu::{Backtrace, IntoError, OptionExt, ResultExt, Snafu}; use url::Url; +pub const REPO_MANIFEST_FILE: &str = ".build-eips.repo.toml"; +const RESERVED_REPO_IDS: &[&str] = &["theme", "preprocessor", "eipw"]; + +#[derive(Debug, Snafu)] +pub enum RepoManifestError { + #[snafu(display("i/o error while accessing `{}`", path.to_string_lossy()))] + Io { + path: PathBuf, + source: std::io::Error, + backtrace: Backtrace, + }, + + #[snafu(display( + "unable to parse repo manifest `{}`", + manifest_path.to_string_lossy() + ))] + Parse { + manifest_path: PathBuf, + #[snafu(source(from(toml::de::Error, Box::new)))] + source: Box, + backtrace: Backtrace, + }, + + #[snafu(display( + "repo manifest `{}` is invalid: {reason}", + manifest_path.to_string_lossy() + ))] + Invalid { + manifest_path: PathBuf, + reason: String, + backtrace: Backtrace, + }, +} + +/// Environment-specific repository metadata for an active proposal repo or sibling repo. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RepositoryEndpoint { + /// Git repository to fetch proposal content from. + pub repository: Url, + + /// Base URL where rendered HTML and assets for this repository are served. + pub base_url: Url, +} + +/// Tracked active-repo manifest loaded from `.build-eips.repo.toml`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RepoManifest { + /// Stable machine key for workspace directory names, build roots, and sibling references. + pub repo_id: String, + + /// Production repository and base URL for this active repo. + pub production: RepositoryEndpoint, + + /// Staging repository and base URL for this active repo. + pub staging: RepositoryEndpoint, + + /// Directional sibling content repos used by this active repo. + #[serde(default)] + pub siblings: BTreeMap, +} + +impl RepoManifest { + fn from_raw(raw: RawRepoManifest, manifest_path: &Path) -> Result { + let repo_id = Self::required_value(manifest_path, "repo_id", raw.repo_id)?; + let production = Self::required_value(manifest_path, "production", raw.production)?; + let staging = Self::required_value(manifest_path, "staging", raw.staging)?; + let siblings = raw + .siblings + .into_iter() + .map(|(repo_id, sibling)| { + let production = Self::required_value( + manifest_path, + &format!("siblings.{repo_id}.production"), + sibling.production, + )?; + let staging = Self::required_value( + manifest_path, + &format!("siblings.{repo_id}.staging"), + sibling.staging, + )?; + + Ok(( + repo_id, + RepoManifestSibling { + production, + staging, + }, + )) + }) + .collect::>()?; + + let manifest = Self { + repo_id, + production, + staging, + siblings, + }; + manifest.validate(manifest_path)?; + Ok(manifest) + } + + fn validate(&self, manifest_path: &Path) -> Result<(), RepoManifestError> { + Self::validate_repo_key(manifest_path, "repo_id", &self.repo_id)?; + + if self.siblings.contains_key(&self.repo_id) { + return InvalidSnafu { + manifest_path: manifest_path.to_path_buf(), + reason: format!( + "repo_id `{}` cannot also be declared as a sibling", + self.repo_id + ), + } + .fail(); + } + + for sibling_id in self.siblings.keys() { + Self::validate_repo_key(manifest_path, "sibling key", sibling_id)?; + } + + Self::validate_unique_sibling_repositories( + manifest_path, + "production", + self.siblings + .iter() + .map(|(id, sibling)| (id.as_str(), sibling.production.repository.as_str())), + )?; + Self::validate_unique_sibling_repositories( + manifest_path, + "staging", + self.siblings + .iter() + .map(|(id, sibling)| (id.as_str(), sibling.staging.repository.as_str())), + )?; + + Ok(()) + } + + pub fn active_endpoint(&self, staging: bool) -> &RepositoryEndpoint { + if staging { + &self.staging + } else { + &self.production + } + } + + pub fn sibling_repositories(&self, staging: bool) -> BTreeMap { + self.siblings + .iter() + .map(|(repo_id, sibling)| { + let endpoint = if staging { + &sibling.staging + } else { + &sibling.production + }; + (repo_id.clone(), endpoint.repository.clone()) + }) + .collect() + } + + fn required_value( + manifest_path: &Path, + field: &str, + value: Option, + ) -> Result { + value.with_context(|| InvalidSnafu { + manifest_path: manifest_path.to_path_buf(), + reason: format!("missing required `{field}` entry"), + }) + } + + fn validate_repo_key( + manifest_path: &Path, + label: &str, + key: &str, + ) -> Result<(), RepoManifestError> { + let invalid_reason = if key.is_empty() { + Some("must not be empty") + } else if matches!(key, "." | "..") { + Some("must not be `.` or `..`") + } else if key.contains('/') || key.contains('\\') { + Some("must be a single safe path component") + } else if RESERVED_REPO_IDS.contains(&key) { + Some("collides with a reserved workspace/platform directory name") + } else { + None + }; + + if let Some(reason) = invalid_reason { + return InvalidSnafu { + manifest_path: manifest_path.to_path_buf(), + reason: format!("{label} `{key}` {reason}"), + } + .fail(); + } + + Ok(()) + } + + fn validate_unique_sibling_repositories<'a>( + manifest_path: &Path, + environment: &str, + siblings: impl Iterator, + ) -> Result<(), RepoManifestError> { + let mut seen = HashSet::new(); + for (repo_id, repository) in siblings { + if !seen.insert(repository) { + return InvalidSnafu { + manifest_path: manifest_path.to_path_buf(), + reason: format!( + "duplicate {environment} sibling repository declaration `{repository}` under sibling key `{repo_id}`" + ), + } + .fail(); + } + } + + Ok(()) + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct RawRepoManifest { + repo_id: Option, + production: Option, + staging: Option, + #[serde(default)] + siblings: BTreeMap, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct RawRepoManifestSibling { + production: Option, + staging: Option, +} + +/// Environment-specific metadata for one declared sibling content repo. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RepoManifestSibling { + /// Production repository and base URL for this sibling repo. + pub production: RepositoryEndpoint, + + /// Staging repository and base URL for this sibling repo. + pub staging: RepositoryEndpoint, +} + +#[derive(Debug, Clone)] +pub struct LoadedRepoManifest { + manifest_path: PathBuf, + manifest: RepoManifest, +} + +impl LoadedRepoManifest { + pub fn load(repo_root: &Path) -> Result, RepoManifestError> { + let manifest_path = repo_root.join(REPO_MANIFEST_FILE); + match std::fs::read_to_string(&manifest_path) { + Ok(contents) => Self::from_contents(manifest_path, &contents).map(Some), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => + { + Ok(None) + } + Err(error) => Err(IoSnafu { + path: manifest_path, + } + .into_error(error)), + } + } + + #[cfg(test)] + pub fn from_path(path: &Path) -> Result { + let manifest_path = path.canonicalize().with_context(|_| IoSnafu { + path: path.to_path_buf(), + })?; + let contents = std::fs::read_to_string(&manifest_path).with_context(|_| IoSnafu { + path: manifest_path.clone(), + })?; + Self::from_contents(manifest_path, &contents) + } + + fn from_contents(manifest_path: PathBuf, contents: &str) -> Result { + let manifest = + toml::from_str::(contents).with_context(|_| ParseSnafu { + manifest_path: manifest_path.clone(), + })?; + let manifest = RepoManifest::from_raw(manifest, &manifest_path)?; + + Ok(Self { + manifest_path, + manifest, + }) + } + + pub fn manifest_path(&self) -> &Path { + &self.manifest_path + } + + pub fn manifest(&self) -> &RepoManifest { + &self.manifest + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Theme { /// Where to fetch the theme from. @@ -106,3 +419,235 @@ impl Config { } } } + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use tempfile::TempDir; + + use super::{LoadedRepoManifest, RepoManifestError, REPO_MANIFEST_FILE}; + + struct TestRepo { + tempdir: TempDir, + } + + impl TestRepo { + fn new() -> Self { + Self { + tempdir: TempDir::new().unwrap(), + } + } + + fn root(&self) -> &Path { + self.tempdir.path() + } + + fn path(&self, relative: impl AsRef) -> PathBuf { + self.root().join(relative) + } + + fn write_file(&self, relative: impl AsRef, contents: &str) -> PathBuf { + let path = self.path(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(&path, contents).unwrap(); + path + } + } + + fn manifest_text(repo_id: &str, siblings: &str) -> String { + format!( + r#" +repo_id = "{repo_id}" + +[production] +repository = "https://example.test/{repo_id}.git" +base_url = "https://example.test/{repo_id}/" + +[staging] +repository = "https://staging.example.test/{repo_id}.git" +base_url = "https://staging.example.test/{repo_id}/" + +{siblings} +"# + ) + } + + fn manifest_invalid_reason(error: RepoManifestError) -> String { + match error { + RepoManifestError::Invalid { reason, .. } => reason, + other => panic!("expected invalid repo manifest, got {other:?}"), + } + } + + #[test] + fn missing_repo_manifest_loads_as_none() { + let repo = TestRepo::new(); + + assert!(LoadedRepoManifest::load(repo.root()).unwrap().is_none()); + } + + #[test] + fn malformed_repo_manifest_reports_parse_error() { + let repo = TestRepo::new(); + let manifest_path = repo.write_file(REPO_MANIFEST_FILE, "repo_id = ["); + + let error = LoadedRepoManifest::from_path(&manifest_path).unwrap_err(); + + assert!(matches!(error, RepoManifestError::Parse { .. })); + } + + #[test] + fn parses_repo_manifest_with_directional_siblings() { + let repo = TestRepo::new(); + let manifest_path = repo.write_file( + REPO_MANIFEST_FILE, + &manifest_text( + "Core", + r#" +[siblings.EIPs.production] +repository = "https://example.test/EIPs.git" +base_url = "https://example.test/EIPs/" + +[siblings.EIPs.staging] +repository = "https://staging.example.test/EIPs.git" +base_url = "https://staging.example.test/EIPs/" +"#, + ), + ); + + let manifest = LoadedRepoManifest::from_path(&manifest_path).unwrap(); + + assert_eq!(manifest.manifest().repo_id, "Core"); + assert_eq!(manifest.manifest().siblings.len(), 1); + assert!(manifest.manifest().siblings.contains_key("EIPs")); + } + + #[test] + fn repo_manifest_requires_identity_and_environments() { + let repo = TestRepo::new(); + let manifest_path = repo.write_file( + REPO_MANIFEST_FILE, + r#" +[production] +repository = "https://example.test/Core.git" +base_url = "https://example.test/Core/" +"#, + ); + + let reason = + manifest_invalid_reason(LoadedRepoManifest::from_path(&manifest_path).unwrap_err()); + + assert!(reason.contains("missing required `repo_id` entry")); + + let manifest_path = repo.write_file( + REPO_MANIFEST_FILE, + r#" +repo_id = "Core" + +[production] +repository = "https://example.test/Core.git" +base_url = "https://example.test/Core/" +"#, + ); + let reason = + manifest_invalid_reason(LoadedRepoManifest::from_path(&manifest_path).unwrap_err()); + + assert!(reason.contains("missing required `staging` entry")); + + let manifest_path = repo.write_file( + REPO_MANIFEST_FILE, + r#" +repo_id = "Core" + +[staging] +repository = "https://staging.example.test/Core.git" +base_url = "https://staging.example.test/Core/" +"#, + ); + let reason = + manifest_invalid_reason(LoadedRepoManifest::from_path(&manifest_path).unwrap_err()); + + assert!(reason.contains("missing required `production` entry")); + } + + #[test] + fn repo_manifest_rejects_unsafe_and_reserved_keys() { + let repo = TestRepo::new(); + let manifest_path = repo.write_file(REPO_MANIFEST_FILE, &manifest_text("theme", "")); + + let reason = + manifest_invalid_reason(LoadedRepoManifest::from_path(&manifest_path).unwrap_err()); + + assert!(reason.contains("repo_id `theme`")); + assert!(reason.contains("reserved")); + + let manifest_path = repo.write_file(REPO_MANIFEST_FILE, &manifest_text("Core/Meta", "")); + let reason = + manifest_invalid_reason(LoadedRepoManifest::from_path(&manifest_path).unwrap_err()); + + assert!(reason.contains("repo_id `Core/Meta`")); + assert!(reason.contains("single safe path component")); + } + + #[test] + fn repo_manifest_rejects_self_sibling() { + let repo = TestRepo::new(); + let manifest_path = repo.write_file( + REPO_MANIFEST_FILE, + &manifest_text( + "Core", + r#" +[siblings.Core.production] +repository = "https://example.test/Core.git" +base_url = "https://example.test/Core/" + +[siblings.Core.staging] +repository = "https://staging.example.test/Core.git" +base_url = "https://staging.example.test/Core/" +"#, + ), + ); + + let reason = + manifest_invalid_reason(LoadedRepoManifest::from_path(&manifest_path).unwrap_err()); + + assert!(reason.contains("cannot also be declared as a sibling")); + } + + #[test] + fn repo_manifest_rejects_duplicate_sibling_repositories() { + let repo = TestRepo::new(); + let manifest_path = repo.write_file( + REPO_MANIFEST_FILE, + &manifest_text( + "Core", + r#" +[siblings.EIPs.production] +repository = "https://example.test/shared.git" +base_url = "https://example.test/EIPs/" + +[siblings.EIPs.staging] +repository = "https://staging.example.test/EIPs.git" +base_url = "https://staging.example.test/EIPs/" + +[siblings.ERCs.production] +repository = "https://example.test/shared.git" +base_url = "https://example.test/ERCs/" + +[siblings.ERCs.staging] +repository = "https://staging.example.test/ERCs.git" +base_url = "https://staging.example.test/ERCs/" +"#, + ), + ); + + let reason = + manifest_invalid_reason(LoadedRepoManifest::from_path(&manifest_path).unwrap_err()); + + assert!(reason.contains("duplicate production sibling repository")); + assert!(reason.contains("https://example.test/shared.git")); + } +} diff --git a/src/identity.rs b/src/identity.rs new file mode 100644 index 0000000..cbb73c9 --- /dev/null +++ b/src/identity.rs @@ -0,0 +1,70 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Active repository identity selection. + +use std::path::Path; + +use snafu::{ResultExt, Whatever}; + +use crate::{ + config::{self, Config, LoadedRepoManifest}, + git, +}; + +#[derive(Debug, Clone)] +pub(crate) enum ActiveRepoIdentity { + Manifest(Box), + Legacy { repo_id: String }, +} + +impl ActiveRepoIdentity { + pub(crate) fn load(root_path: &Path) -> Result { + if let Some(manifest) = + LoadedRepoManifest::load(root_path).whatever_context("unable to load repo manifest")? + { + return Ok(Self::Manifest(Box::new(manifest))); + } + + match Config::production() + .locations + .identify_repository(root_path) + { + Ok(repository_use) => Ok(Self::Legacy { + repo_id: repository_use.title, + }), + Err(git::Error::NoIdentify { .. }) => { + snafu::whatever!( + "active repository `{}` does not carry `{}` and does not match the legacy EIPs/ERCs identity fallback", + root_path.to_string_lossy(), + config::REPO_MANIFEST_FILE + ) + } + Err(error) => Err(error).whatever_context("cannot identify legacy repository use"), + } + } + + pub(crate) fn repo_id(&self) -> &str { + match self { + Self::Manifest(manifest) => &manifest.manifest().repo_id, + Self::Legacy { repo_id } => repo_id, + } + } + + pub(crate) fn source_description(&self) -> &'static str { + match self { + Self::Manifest(_) => "repo manifest", + Self::Legacy { .. } => "legacy EIPs/ERCs fallback", + } + } + + pub(crate) fn manifest(&self) -> Option<&LoadedRepoManifest> { + match self { + Self::Manifest(manifest) => Some(manifest.as_ref()), + Self::Legacy { .. } => None, + } + } +} diff --git a/src/main.rs b/src/main.rs index bc01379..bc0449b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,6 +12,7 @@ mod context; mod find_root; mod git; mod github; +mod identity; mod layout; mod lint; mod markdown; From 8c9b119489cd0d50bfcc91bd16521c838910ef59 Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Sun, 3 May 2026 20:37:51 -0400 Subject: [PATCH 02/20] Add workspace configuration discovery Add the .build-eips.toml schema, starter config text, upward discovery, and loaded workspace config accessors. Define server/site defaults, workspace build-root paths, local theme and repo paths, and strict parsing for unsupported config fields. Leave init, doctor, runtime consumption, and render-only filtering to the later workspace, execution, and targeted rendering PRs. --- src/config.rs | 500 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 497 insertions(+), 3 deletions(-) diff --git a/src/config.rs b/src/config.rs index 10f2098..271e797 100644 --- a/src/config.rs +++ b/src/config.rs @@ -6,14 +6,21 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, + fmt, path::{Path, PathBuf}, }; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use snafu::{Backtrace, IntoError, OptionExt, ResultExt, Snafu}; -use url::Url; +use url::{Position, Url}; +pub const LOCAL_CONFIG_FILE: &str = ".build-eips.toml"; pub const REPO_MANIFEST_FILE: &str = ".build-eips.repo.toml"; +pub const DEFAULT_BUILD_ROOT_BASE: &str = ".local-build"; +pub const DEFAULT_THEME_DIR: &str = "theme"; +pub const DEFAULT_SERVER_HOST: &str = "127.0.0.1"; +pub const DEFAULT_SERVER_PORT: u16 = 1111; +pub const DEFAULT_SITE_BASE_URL: &str = "http://127.0.0.1:1111"; const RESERVED_REPO_IDS: &[&str] = &["theme", "preprocessor", "eipw"]; #[derive(Debug, Snafu)] @@ -47,6 +54,27 @@ pub enum RepoManifestError { }, } +#[derive(Debug, Snafu)] +pub enum WorkspaceError { + #[snafu(display("i/o error while accessing `{}`", path.to_string_lossy()))] + Fs { + path: PathBuf, + source: std::io::Error, + backtrace: Backtrace, + }, + + #[snafu(display( + "unable to parse workspace config `{}`", + config_path.to_string_lossy() + ))] + Parse { + config_path: PathBuf, + #[snafu(source(from(toml::de::Error, Box::new)))] + source: Box, + backtrace: Backtrace, + }, +} + /// Environment-specific repository metadata for an active proposal repo or sibling repo. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -420,13 +448,238 @@ impl Config { } } +/// Workspace-local configuration loaded from `.build-eips.toml`. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct WorkspaceConfig { + /// Local server defaults for `build-eips serve` and `build-eips preview`. + #[serde(default)] + pub server: ServerSettings, + + /// Local rendered-site URL defaults for build and serve commands. + #[serde(default)] + pub site: SiteSettings, +} + +impl WorkspaceConfig { + fn starter() -> Self { + Self { + server: ServerSettings::default(), + site: SiteSettings::starter(), + } + } +} + +/// Workspace-local bind address defaults for local server commands. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct ServerSettings { + /// Host or interface address used by `serve` and `preview`. + pub host: String, + + /// TCP port used by `serve` and `preview`. + pub port: u16, +} + +impl Default for ServerSettings { + fn default() -> Self { + Self { + host: DEFAULT_SERVER_HOST.to_owned(), + port: DEFAULT_SERVER_PORT, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServerBinding { + pub host: String, + pub port: u16, +} + +impl Default for ServerBinding { + fn default() -> Self { + ServerSettings::default().into() + } +} + +impl From for ServerBinding { + fn from(settings: ServerSettings) -> Self { + Self { + host: settings.host, + port: settings.port, + } + } +} + +impl From<&ServerSettings> for ServerBinding { + fn from(settings: &ServerSettings) -> Self { + settings.clone().into() + } +} + +impl fmt::Display for ServerBinding { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}:{}", self.host, self.port) + } +} + +/// Workspace-local rendered-site URL defaults for build and serve commands. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct SiteSettings { + /// Base URL written into rendered HTML, feeds, canonical links, and sitemaps. + #[serde( + default, + serialize_with = "serialize_optional_base_url", + deserialize_with = "deserialize_optional_base_url" + )] + pub base_url: Option, +} + +impl SiteSettings { + fn starter() -> Self { + Self { + base_url: Some( + DEFAULT_SITE_BASE_URL + .parse() + .expect("default site base URL should parse"), + ), + } + } +} + +fn serialize_optional_base_url(base_url: &Option, serializer: S) -> Result +where + S: Serializer, +{ + match base_url { + Some(base_url) => serializer.serialize_some(&format_base_url(base_url)), + None => serializer.serialize_none(), + } +} + +fn deserialize_optional_base_url<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + Option::::deserialize(deserializer) +} + +fn format_base_url(base_url: &Url) -> String { + if base_url.path() == "/" && base_url.query().is_none() && base_url.fragment().is_none() { + base_url[..Position::BeforePath].to_owned() + } else { + base_url.as_str().to_owned() + } +} + +#[derive(Debug, Clone)] +pub struct LoadedWorkspaceConfig { + config_path: PathBuf, + workspace_root: PathBuf, + config: WorkspaceConfig, +} + +impl LoadedWorkspaceConfig { + pub fn from_path(path: &Path) -> Result { + let config_path = path.canonicalize().with_context(|_| FsSnafu { + path: path.to_path_buf(), + })?; + let contents = std::fs::read_to_string(&config_path).with_context(|_| FsSnafu { + path: config_path.clone(), + })?; + let config = toml::from_str::(&contents).with_context(|_| ParseSnafu { + config_path: config_path.clone(), + })?; + + let workspace_root = config_path + .parent() + .expect("workspace config should always have a parent") + .to_path_buf(); + + Ok(Self { + config_path, + workspace_root, + config, + }) + } + + pub fn discover(start: &Path) -> Result, WorkspaceError> { + match discover_path(start) { + Some(path) => Self::from_path(&path).map(Some), + None => Ok(None), + } + } + + pub fn config_path(&self) -> &Path { + &self.config_path + } + + pub fn workspace_root(&self) -> &Path { + &self.workspace_root + } + + pub fn workspace_build_root(&self, repo_id: &str) -> PathBuf { + self.workspace_root + .join(DEFAULT_BUILD_ROOT_BASE) + .join(repo_id) + } + + pub fn server_settings(&self) -> &ServerSettings { + &self.config.server + } + + pub fn site_settings(&self) -> &SiteSettings { + &self.config.site + } + + pub fn local_theme_path(&self) -> PathBuf { + self.workspace_root.join(DEFAULT_THEME_DIR) + } + + pub fn local_repo_path(&self, repo_id: &str) -> PathBuf { + self.workspace_root.join(repo_id) + } +} + +pub fn discover_path(start: &Path) -> Option { + let mut current = Some(start); + + while let Some(candidate) = current { + let config_path = candidate.join(LOCAL_CONFIG_FILE); + match std::fs::File::open(&config_path) { + Ok(_) => return Some(config_path), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => + { + current = candidate.parent(); + } + Err(_) => return Some(config_path), + } + } + + None +} + +pub fn default_workspace_config_text() -> String { + toml::to_string_pretty(&WorkspaceConfig::starter()) + .expect("workspace starter config should serialize") +} + #[cfg(test)] mod tests { use std::path::{Path, PathBuf}; use tempfile::TempDir; - use super::{LoadedRepoManifest, RepoManifestError, REPO_MANIFEST_FILE}; + use super::{ + default_workspace_config_text, discover_path, LoadedRepoManifest, LoadedWorkspaceConfig, + RepoManifestError, ServerBinding, ServerSettings, WorkspaceError, DEFAULT_SERVER_HOST, + DEFAULT_SERVER_PORT, DEFAULT_SITE_BASE_URL, LOCAL_CONFIG_FILE, REPO_MANIFEST_FILE, + }; struct TestRepo { tempdir: TempDir, @@ -650,4 +903,245 @@ base_url = "https://staging.example.test/ERCs/" assert!(reason.contains("duplicate production sibling repository")); assert!(reason.contains("https://example.test/shared.git")); } + + #[test] + fn parses_default_workspace_config() { + let repo = TestRepo::new(); + let config_path = repo.write_file(LOCAL_CONFIG_FILE, &default_workspace_config_text()); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!(config.workspace_root(), repo.root()); + assert_eq!(config.server_settings(), &ServerSettings::default()); + assert_eq!( + config.site_settings().base_url.as_ref().unwrap().as_str(), + "http://127.0.0.1:1111/" + ); + } + + #[test] + fn starter_workspace_config_roundtrips_stably() { + let original = default_workspace_config_text(); + let parsed = toml::from_str::(&original).unwrap(); + let reparsed = toml::to_string_pretty(&parsed).unwrap(); + + assert_eq!(reparsed, original); + assert!(!original.contains("build_root_base")); + assert!(original.contains("[server]")); + assert!(original.contains("host = \"127.0.0.1\"")); + assert!(original.contains("port = 1111")); + assert!(original.contains("[site]")); + assert!(original.contains(&format!("base_url = \"{DEFAULT_SITE_BASE_URL}\""))); + assert!(!original.contains("default_profile")); + assert!(!original.contains("[profiles")); + assert!(!original.contains("[render]")); + } + + #[test] + fn parses_workspace_config_server_settings() { + let repo = TestRepo::new(); + let config_path = repo.write_file( + LOCAL_CONFIG_FILE, + r#" +[server] +host = "0.0.0.0" +port = 8080 +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!( + config.server_settings(), + &ServerSettings { + host: "0.0.0.0".to_owned(), + port: 8080, + } + ); + } + + #[test] + fn missing_server_settings_use_default_binding() { + let repo = TestRepo::new(); + let config_path = repo.write_file( + LOCAL_CONFIG_FILE, + r#" +[site] +base_url = "http://localhost:4000" +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + let binding = ServerBinding::from(config.server_settings()); + + assert_eq!(binding.host, DEFAULT_SERVER_HOST); + assert_eq!(binding.port, DEFAULT_SERVER_PORT); + assert_eq!(binding.to_string(), "127.0.0.1:1111"); + } + + #[test] + fn parses_workspace_config_site_settings() { + let repo = TestRepo::new(); + let config_path = repo.write_file( + LOCAL_CONFIG_FILE, + r#" +[site] +base_url = "http://localhost:4000" +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!( + config.site_settings().base_url.as_ref().unwrap().as_str(), + "http://localhost:4000/" + ); + } + + #[test] + fn invalid_workspace_config_site_base_url_errors() { + let repo = TestRepo::new(); + let config_path = repo.write_file( + LOCAL_CONFIG_FILE, + r#" +[site] +base_url = "not a url" +"#, + ); + let error = LoadedWorkspaceConfig::from_path(&config_path).unwrap_err(); + + assert!(error + .to_string() + .contains("unable to parse workspace config")); + } + + #[test] + fn missing_site_settings_preserve_no_base_url_override() { + let repo = TestRepo::new(); + let config_path = repo.write_file( + LOCAL_CONFIG_FILE, + r#" +[server] +host = "127.0.0.1" +port = 1111 +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert!(config.site_settings().base_url.is_none()); + } + + #[test] + fn minimal_workspace_config_parses() { + let repo = TestRepo::new(); + let config_path = repo.write_file( + LOCAL_CONFIG_FILE, + r#" +[server] +host = "127.0.0.1" +port = 1111 + +[site] +base_url = "http://127.0.0.1:1111" +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!(config.server_settings(), &ServerSettings::default()); + assert_eq!( + config.site_settings().base_url.as_ref().unwrap().as_str(), + "http://127.0.0.1:1111/" + ); + } + + #[test] + fn empty_workspace_config_uses_defaults() { + let repo = TestRepo::new(); + let config_path = repo.write_file(LOCAL_CONFIG_FILE, " \n"); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!(config.server_settings(), &ServerSettings::default()); + assert!(config.site_settings().base_url.is_none()); + } + + #[test] + fn removed_workspace_config_fields_use_strict_parse_errors() { + let removed_theme_ref_field = concat!("co", "mmit"); + let cases = vec![ + ( + "build_root_base".to_owned(), + r#"build_root_base = ".local-build""#.to_owned(), + ), + ( + "default_profile".to_owned(), + r#"default_profile = "local""#.to_owned(), + ), + ( + "profiles".to_owned(), + r#" +[profiles.local] +staging = true +"# + .to_owned(), + ), + ( + "theme".to_owned(), + r#" +[theme] +repository = "https://github.com/eips-wg/theme.git" +"# + .to_owned(), + ), + ( + format!("theme.{removed_theme_ref_field}"), + format!( + r#" +[theme] +{removed_theme_ref_field} = "3a597d4cd68ec82d36f01c01335492cfa59501ae" +"# + ), + ), + ]; + + for (field, contents) in cases { + let repo = TestRepo::new(); + let config_path = repo.write_file(LOCAL_CONFIG_FILE, &contents); + let error = LoadedWorkspaceConfig::from_path(&config_path).unwrap_err(); + + assert!( + matches!(error, WorkspaceError::Parse { .. }), + "expected strict parse error for removed field `{field}`, got {error:?}" + ); + } + } + + #[test] + fn discover_path_walks_upward() { + let repo = TestRepo::new(); + let config_path = repo.write_file(LOCAL_CONFIG_FILE, &default_workspace_config_text()); + let nested = repo.path("EIPs/content"); + std::fs::create_dir_all(&nested).unwrap(); + + assert_eq!(discover_path(&nested).unwrap(), config_path); + assert_eq!( + LoadedWorkspaceConfig::discover(&nested) + .unwrap() + .unwrap() + .config_path(), + config_path + ); + } + + #[test] + fn missing_workspace_config_is_not_discovered() { + let repo = TestRepo::new(); + let nested = repo.path("EIPs/content"); + std::fs::create_dir_all(&nested).unwrap(); + + assert!(discover_path(&nested).is_none()); + assert!(LoadedWorkspaceConfig::discover(&nested).unwrap().is_none()); + } } From 4b525953a04ef0e9a9372b0260020a4ab14393f2 Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Sun, 3 May 2026 20:40:29 -0400 Subject: [PATCH 03/20] Add source materialization layer Add source materialization modes, reshape RepositoryUse around resolved repository endpoints, and pass source mode explicitly into Fresh. Add dirty working-tree copying, tracked-path sync, dirty rejection errors, and sibling merge behavior that follows local file sibling HEADs. Route existing build and changed-file flows through clean source materialization so current behavior stays unchanged. --- src/changed.rs | 25 +- src/config.rs | 27 +- src/git.rs | 938 ++++++++++++++++++++++++++++++++++++++++++++---- src/identity.rs | 6 +- src/main.rs | 35 +- 5 files changed, 932 insertions(+), 99 deletions(-) diff --git a/src/changed.rs b/src/changed.rs index 533ac26..62b6a9b 100644 --- a/src/changed.rs +++ b/src/changed.rs @@ -61,12 +61,25 @@ pub(crate) fn run( ) -> Result<(), Whatever> { let repo_path = build_path.join(REPO_DIR); - let both = git::Fresh::new(root_path, &repo_path, &config.locations) - .whatever_context("initializing build repo")? - .clone_src() - .whatever_context("cloning source repo")? - .fetch_upstream() - .whatever_context("fetching upstream repo")?; + let repo_id = config + .locations + .identify_repository_title(root_path) + .whatever_context("cannot identify repository use")?; + let Some(repository_use) = config.locations.repository_use_for_title(&repo_id) else { + snafu::whatever!("repository metadata for `{repo_id}` is unavailable"); + }; + + let both = git::Fresh::new( + root_path, + &repo_path, + repository_use, + git::SourceMaterialization::Clean, + ) + .whatever_context("initializing build repo")? + .clone_src() + .whatever_context("cloning source repo")? + .fetch_upstream() + .whatever_context("fetching upstream repo")?; let changed_files: Vec<_> = both .changed_files() diff --git a/src/config.rs b/src/config.rs index 271e797..c9aa8bc 100644 --- a/src/config.rs +++ b/src/config.rs @@ -360,7 +360,7 @@ pub struct Theme { } #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Location { +pub struct LegacyLocation { /// Git repository to fetch proposals from. pub repository: Url, @@ -374,14 +374,23 @@ pub struct Location { pub identifying_commit: String, } +impl LegacyLocation { + pub fn endpoint(&self) -> RepositoryEndpoint { + RepositoryEndpoint { + repository: self.repository.clone(), + base_url: self.base_url.clone(), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(transparent)] -pub struct Locations(pub HashMap); +pub struct LegacyLocations(pub HashMap); #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Config { pub theme: Theme, - pub locations: Locations, + pub locations: LegacyLocations, } impl Config { @@ -390,7 +399,7 @@ impl Config { locations.insert( "EIPs".into(), - Location { + LegacyLocation { repository: "https://github.com/ethereum/EIPs.git".try_into().unwrap(), base_url: "https://eips.ethereum.org/".try_into().unwrap(), identifying_commit: "0f44e2b94df4e504bb7b912f56ebd712db2ad396".into(), @@ -399,7 +408,7 @@ impl Config { locations.insert( "ERCs".into(), - Location { + LegacyLocation { repository: "https://github.com/ethereum/ERCs.git".try_into().unwrap(), base_url: "https://ercs.ethereum.org/".try_into().unwrap(), identifying_commit: "8dd085d159cb123f545c272c0d871a5339550e79".into(), @@ -413,7 +422,7 @@ impl Config { .unwrap(), commit: "0ddac35da36d311a8401c6cfb79c9991f78b647d".into(), }, - locations: Locations(locations), + locations: LegacyLocations(locations), } } @@ -422,7 +431,7 @@ impl Config { locations.insert( "EIPs".into(), - Location { + LegacyLocation { repository: "https://github.com/eips-wg/EIPs.git".try_into().unwrap(), base_url: "https://eips-wg.github.io/EIPs/".try_into().unwrap(), identifying_commit: "0f44e2b94df4e504bb7b912f56ebd712db2ad396".into(), @@ -431,7 +440,7 @@ impl Config { locations.insert( "ERCs".into(), - Location { + LegacyLocation { repository: "https://github.com/eips-wg/ERCs.git".try_into().unwrap(), base_url: "https://eips-wg.github.io/ERCs/".try_into().unwrap(), identifying_commit: "8dd085d159cb123f545c272c0d871a5339550e79".into(), @@ -443,7 +452,7 @@ impl Config { repository: "https://github.com/eips-wg/theme.git".try_into().unwrap(), commit: "0ddac35da36d311a8401c6cfb79c9991f78b647d".into(), }, - locations: Locations(locations), + locations: LegacyLocations(locations), } } } diff --git a/src/git.rs b/src/git.rs index 357743f..1d35379 100644 --- a/src/git.rs +++ b/src/git.rs @@ -5,25 +5,28 @@ */ use std::{ - collections::HashMap, + collections::{BTreeMap, BTreeSet}, ffi::OsStr, path::{absolute, Path, PathBuf}, }; use crate::{ cache::Cache, - config::{Location, Locations}, + config::{LegacyLocations, RepositoryEndpoint}, + layout::{BUILD_DIR, CONTENT_DIR}, progress::{Git, ProgressIteratorExt}, }; use git2::{ build::{CheckoutBuilder, TreeUpdateBuilder}, - BranchType, Commit, FetchOptions, FileMode, ObjectType, Oid, RepositoryOpenFlags, Signature, + Commit, FetchOptions, FileMode, ObjectType, Oid, RepositoryOpenFlags, Signature, Status, StatusOptions, Tree, TreeEntry, TreeWalkResult, }; use log::{debug, info}; use snafu::{ensure, Backtrace, IntoError, OptionExt, ResultExt, Snafu}; use url::Url; +const DIRTY_PATH_DISPLAY_LIMIT: usize = 10; + #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("cannot convert path into URL (`{}`)", path.to_string_lossy()))] @@ -47,8 +50,21 @@ pub enum Error { }, #[snafu(display("unable to determine which repository is being built (none match)"))] NoIdentify { backtrace: Backtrace }, - #[snafu(display("working tree or index has uncommitted modifications"))] - Dirty { backtrace: Backtrace }, + #[snafu(display("{message}"))] + Dirty { + message: String, + backtrace: Backtrace, + }, + #[snafu(display( + "dirty mode cannot materialize conflicted path `{}`; resolve the conflict and try again", + path.to_string_lossy() + ))] + DirtyConflict { path: PathBuf, backtrace: Backtrace }, + #[snafu(display( + "dirty mode cannot materialize `{}` because it is not a tracked file or symlink in the working tree", + path.to_string_lossy() + ))] + DirtyUnsupportedPath { path: PathBuf, backtrace: Backtrace }, #[snafu(display("unable to update tree ({msg})"))] UpdateTree { msg: String, backtrace: Backtrace }, #[snafu(context(false))] @@ -58,15 +74,25 @@ pub enum Error { }, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceMaterialization { + Clean, + Dirty, +} + #[derive(Debug, Clone)] pub struct RepositoryUse { pub title: String, - pub location: Location, - pub other_repos: HashMap, + pub location: RepositoryEndpoint, + pub other_repos: BTreeMap, } -impl Locations { - pub fn identify_repository(&self, path: &Path) -> Result { +pub fn repository_available(path: &Path) -> bool { + git2::Repository::open(path).is_ok() +} + +impl LegacyLocations { + pub fn identify_repository_title(&self, path: &Path) -> Result { let repo = git2::Repository::open_ext(path, RepositoryOpenFlags::NO_SEARCH, &[] as &[&OsStr]) .context(GitSnafu { @@ -94,9 +120,13 @@ impl Locations { ); ensure!(containing_locations.len() == 1, NoIdentifySnafu); - let (title, location) = containing_locations[0]; + let (title, _) = containing_locations[0]; + + Ok(title.clone()) + } - // TODO: this is a bit weird, and is a leftover from the previous architecture. + pub fn repository_use_for_title(&self, title: &str) -> Option { + let location = self.0.get(title)?; let other_repos = self .0 .iter() @@ -109,35 +139,555 @@ impl Locations { }) .collect(); - Ok(RepositoryUse { - title: title.clone(), - location: location.clone(), + Some(RepositoryUse { + title: title.to_owned(), + location: location.endpoint(), other_repos, }) } } +pub fn clone_missing_repo(url: &str, destination: &Path) -> Result<(), Error> { + match git2::Repository::open(destination) { + Ok(_) => { + info!( + "using existing workspace repo `{}`", + destination.to_string_lossy() + ); + return Ok(()); + } + Err(error) if error.code() == git2::ErrorCode::NotFound => {} + Err(error) => { + return Err(GitSnafu { + what: "open existing workspace repository", + } + .into_error(error)); + } + } + + info!("cloning `{url}` into `{}`", destination.to_string_lossy()); + git2::Repository::clone(url, destination).context(GitSnafu { + what: "clone workspace repository", + })?; + Ok(()) +} + +fn is_generated_path(path: &Path) -> bool { + path.components() + .next() + .map(|component| component.as_os_str() == OsStr::new(BUILD_DIR)) + .unwrap_or(false) +} + +fn dirty_statuses(repo: &git2::Repository) -> Result, Error> { + let mut options = StatusOptions::default(); + options + .include_untracked(true) + .recurse_untracked_dirs(true) + .renames_head_to_index(true) + .renames_index_to_workdir(true); + repo.statuses(Some(&mut options)).context(GitSnafu { + what: "get root repository status", + }) +} + +fn format_dirty_rejection(tracked_paths: &BTreeSet, untracked_count: usize) -> String { + let mut lines = vec![String::from( + "working tree or index has uncommitted modifications; the selected clean source path requires a clean working tree:", + )]; + + for path in tracked_paths.iter().take(DIRTY_PATH_DISPLAY_LIMIT) { + lines.push(format!("- {}", path.to_string_lossy())); + } + + if tracked_paths.len() > DIRTY_PATH_DISPLAY_LIMIT { + lines.push(format!( + "- ... and {} more tracked path(s)", + tracked_paths.len() - DIRTY_PATH_DISPLAY_LIMIT + )); + } + + if untracked_count > 0 { + lines.push(format!( + "- ... plus {} untracked file(s) not listed", + untracked_count + )); + } + + lines.push(String::new()); + + if untracked_count > 0 { + lines.push(String::from( + "For local build/serve/check commands, run without `--clean` to include tracked local changes. For remote/parity/clean runs, commit or stash tracked changes first. Commit/stash/remove untracked files before retrying.", + )); + } else { + lines.push(String::from( + "For local build/serve/check commands, run without `--clean` to include tracked local changes. For remote/parity/clean runs, commit or stash tracked changes first.", + )); + } + + lines.join("\n") +} + pub fn check_dirty(root_path: &Path) -> Result<(), Error> { + let (tracked_paths, untracked_count) = + collect_dirty_paths(root_path, |path| !is_generated_path(path))?; + + if tracked_paths.is_empty() && untracked_count == 0 { + Ok(()) + } else { + DirtySnafu { + message: format_dirty_rejection(&tracked_paths, untracked_count), + } + .fail() + } +} + +fn entry_path(entry: &git2::StatusEntry<'_>) -> Option { + entry + .head_to_index() + .and_then(|delta| delta.new_file().path().or_else(|| delta.old_file().path())) + .or_else(|| { + entry + .index_to_workdir() + .and_then(|delta| delta.new_file().path().or_else(|| delta.old_file().path())) + }) + .or_else(|| entry.path().map(Path::new)) + .map(Path::to_path_buf) +} + +fn collect_dirty_paths( + root_path: &Path, + include_path: impl Fn(&Path) -> bool, +) -> Result<(BTreeSet, usize), Error> { let repo = git2::Repository::open(root_path).context(GitSnafu { what: "open root repository", })?; - let mut options = StatusOptions::default(); - options.include_untracked(true); - let statuses = repo.statuses(Some(&mut options)).context(GitSnafu { - what: "get root repository status", + let statuses = dirty_statuses(&repo)?; + let mut paths = BTreeSet::new(); + let mut untracked_count = 0; + + for entry in statuses.iter() { + let status = entry.status(); + let path = entry_path(&entry).unwrap_or_else(|| PathBuf::from("")); + + if status.contains(Status::CONFLICTED) { + return DirtyConflictSnafu { path }.fail(); + } + + if status == Status::CURRENT || status == Status::IGNORED { + continue; + } + + if status == Status::WT_NEW { + if include_path(&path) { + untracked_count += 1; + } + continue; + } + + if let Some(delta) = entry.head_to_index() { + if let Some(old_path) = delta.old_file().path().filter(|path| include_path(path)) { + paths.insert(old_path.to_path_buf()); + } + if let Some(new_path) = delta.new_file().path().filter(|path| include_path(path)) { + paths.insert(new_path.to_path_buf()); + } + } + + if let Some(delta) = entry.index_to_workdir() { + if let Some(old_path) = delta.old_file().path().filter(|path| include_path(path)) { + paths.insert(old_path.to_path_buf()); + } + if let Some(new_path) = delta.new_file().path().filter(|path| include_path(path)) { + paths.insert(new_path.to_path_buf()); + } + } + + if include_path(&path) { + paths.insert(path); + } + } + + Ok((paths, untracked_count)) +} + +pub fn working_tree_paths(root_path: &Path) -> Result, Error> { + let (paths, _) = collect_dirty_paths(root_path, |path| !is_generated_path(path))?; + Ok(paths.into_iter().collect()) +} + +pub fn tracked_working_tree_paths(root_path: &Path) -> Result, Error> { + let (paths, _) = collect_dirty_paths(root_path, |_| true)?; + Ok(paths.into_iter().collect()) +} + +pub fn materialize_working_tree(source_root: &Path, destination_root: &Path) -> Result<(), Error> { + remove_existing_path(destination_root).with_context(|_| IoSnafu { + path: destination_root.to_path_buf(), + })?; + std::fs::create_dir_all(destination_root).with_context(|_| IoSnafu { + path: destination_root.to_path_buf(), + })?; + + let mut paths = tracked_paths(source_root, |_| true)?; + paths.extend(tracked_working_tree_paths(source_root)?); + sync_working_tree_paths(source_root, destination_root, &paths) +} + +pub fn sync_working_tree_paths( + source_root: &Path, + destination_root: &Path, + relative_paths: &BTreeSet, +) -> Result<(), Error> { + for path in relative_paths { + sync_working_tree_path(source_root, destination_root, path)?; + } + + Ok(()) +} + +pub fn index_path(root_path: &Path) -> Result { + let repo = git2::Repository::open(root_path).context(GitSnafu { + what: "open root repository", + })?; + let index = repo.index().context(GitSnafu { + what: "open root repository index", })?; - let mut statuses = statuses.iter().filter(|x| { - x.path() - .map(|x| !x.trim_end_matches('/').ends_with(super::BUILD_DIR)) - .unwrap_or(false) + index + .path() + .map(Path::to_path_buf) + .with_context(|| UpdateTreeSnafu:: { + msg: "repository index is in-memory".into(), + }) +} + +pub fn sync_materialized_paths( + source_root: &Path, + build_repo_path: &Path, + relative_paths: &BTreeSet, +) -> Result<(), Error> { + if relative_paths.is_empty() { + return Ok(()); + } + + let working_repo = git2::Repository::open(build_repo_path).context(GitSnafu { + what: "open build repository", + })?; + let working_root = working_repo + .workdir() + .with_context(|| UpdateTreeSnafu:: { + msg: "build repository workdir is unavailable".into(), + })?; + let mut index = working_repo.index().context(GitSnafu { + what: "open build repository index", + })?; + + for path in relative_paths { + sync_dirty_path(source_root, working_root, &mut index, path)?; + } + + index.write().context(GitSnafu { + what: "write build repository index", + })?; + + Ok(()) +} + +fn remove_existing_path(path: &Path) -> Result<(), std::io::Error> { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_dir() && !metadata.file_type().is_symlink() => { + std::fs::remove_dir_all(path) + } + Ok(_) => std::fs::remove_file(path), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} + +fn tracked_paths( + root_path: &Path, + include_path: impl Fn(&Path) -> bool, +) -> Result, Error> { + let repo = git2::Repository::open(root_path).context(GitSnafu { + what: "open root repository", + })?; + let head = repo.head().context(GitSnafu { what: "head" })?; + let commit = head.peel_to_commit().context(GitSnafu { + what: "peel head to commit", + })?; + let tree = commit.tree().context(GitSnafu { what: "head tree" })?; + let mut paths = BTreeSet::new(); + let mut walk_error = None; + + let walk_result = tree.walk(git2::TreeWalkMode::PreOrder, |prefix, entry| { + let Some(name) = entry.name() else { + walk_error = Some( + UpdateTreeSnafu { + msg: format!("tree entry without name in `{prefix}`"), + } + .build(), + ); + return TreeWalkResult::Abort; + }; + + match entry.kind() { + Some(ObjectType::Blob) => (), + Some(ObjectType::Tree) => return TreeWalkResult::Ok, + kind => { + walk_error = Some( + UpdateTreeSnafu { + msg: format!("unknown blob type `{kind:?}` for `{}{name}`", prefix), + } + .build(), + ); + return TreeWalkResult::Abort; + } + } + + let path = PathBuf::from(format!("{prefix}{name}")); + if include_path(&path) { + paths.insert(path); + } + + TreeWalkResult::Ok }); - if statuses.next().is_some() { - DirtySnafu.fail() + + if let Some(error) = walk_error { + return Err(error); + } + + walk_result.context(GitSnafu { + what: "traverse tree", + })?; + + Ok(paths) +} + +fn remove_index_path(index: &mut git2::Index, path: &Path) -> Result<(), Error> { + match index.remove_path(path) { + Ok(()) => Ok(()), + Err(error) if error.code() == git2::ErrorCode::NotFound => match index.remove_dir(path, -1) + { + Ok(()) => Ok(()), + Err(error) if error.code() == git2::ErrorCode::NotFound => Ok(()), + Err(error) => Err(GitSnafu { + what: "remove dirty path from index", + } + .into_error(error)), + }, + Err(error) => Err(GitSnafu { + what: "remove dirty path from index", + } + .into_error(error)), + } +} + +#[cfg(target_family = "unix")] +fn copy_symlink(source: &Path, destination: &Path) -> Result<(), std::io::Error> { + let target = std::fs::read_link(source)?; + std::os::unix::fs::symlink(target, destination) +} + +#[cfg(target_family = "windows")] +fn copy_symlink(source: &Path, destination: &Path) -> Result<(), std::io::Error> { + let target = std::fs::read_link(source)?; + let resolved_target = source + .parent() + .map(|parent| parent.join(&target)) + .unwrap_or_else(|| target.clone()); + + if std::fs::metadata(&resolved_target) + .map(|metadata| metadata.is_dir()) + .unwrap_or(false) + { + std::os::windows::fs::symlink_dir(target, destination) } else { - Ok(()) + std::os::windows::fs::symlink_file(target, destination) } } +#[cfg(not(any(target_family = "unix", target_family = "windows")))] +fn copy_symlink(_source: &Path, _destination: &Path) -> Result<(), std::io::Error> { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "no symlink implementation available", + )) +} + +fn sync_dirty_path( + source_root: &Path, + working_root: &Path, + index: &mut git2::Index, + relative_path: &Path, +) -> Result<(), Error> { + let source_path = source_root.join(relative_path); + let working_path = working_root.join(relative_path); + + match std::fs::symlink_metadata(&source_path) { + Ok(metadata) if metadata.file_type().is_dir() && !metadata.file_type().is_symlink() => { + remove_existing_path(&working_path).with_context(|_| IoSnafu { + path: working_path.clone(), + })?; + remove_index_path(index, relative_path)?; + Ok(()) + } + Ok(metadata) if metadata.file_type().is_file() || metadata.file_type().is_symlink() => { + if let Some(parent) = working_path.parent() { + std::fs::create_dir_all(parent).with_context(|_| IoSnafu { + path: parent.to_path_buf(), + })?; + } + + remove_existing_path(&working_path).with_context(|_| IoSnafu { + path: working_path.clone(), + })?; + + if metadata.file_type().is_symlink() { + copy_symlink(&source_path, &working_path).with_context(|_| IoSnafu { + path: working_path.clone(), + })?; + } else { + std::fs::copy(&source_path, &working_path).with_context(|_| IoSnafu { + path: source_path.clone(), + })?; + } + + index.add_path(relative_path).context(GitSnafu { + what: "add dirty path to index", + })?; + Ok(()) + } + Ok(_) => DirtyUnsupportedPathSnafu { + path: relative_path.to_path_buf(), + } + .fail(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + remove_existing_path(&working_path).with_context(|_| IoSnafu { + path: working_path.clone(), + })?; + remove_index_path(index, relative_path)?; + Ok(()) + } + Err(error) => Err(IoSnafu { path: source_path }.into_error(error)), + } +} + +fn sync_working_tree_path( + source_root: &Path, + destination_root: &Path, + relative_path: &Path, +) -> Result<(), Error> { + let source_path = source_root.join(relative_path); + let destination_path = destination_root.join(relative_path); + + match std::fs::symlink_metadata(&source_path) { + Ok(metadata) if metadata.file_type().is_dir() && !metadata.file_type().is_symlink() => { + remove_existing_path(&destination_path).with_context(|_| IoSnafu { + path: destination_path.clone(), + }) + } + Ok(metadata) if metadata.file_type().is_file() || metadata.file_type().is_symlink() => { + if let Some(parent) = destination_path.parent() { + std::fs::create_dir_all(parent).with_context(|_| IoSnafu { + path: parent.to_path_buf(), + })?; + } + + remove_existing_path(&destination_path).with_context(|_| IoSnafu { + path: destination_path.clone(), + })?; + + if metadata.file_type().is_symlink() { + copy_symlink(&source_path, &destination_path).with_context(|_| IoSnafu { + path: destination_path.clone(), + })?; + } else { + std::fs::copy(&source_path, &destination_path).with_context(|_| IoSnafu { + path: source_path.clone(), + })?; + } + + Ok(()) + } + Ok(_) => DirtyUnsupportedPathSnafu { + path: relative_path.to_path_buf(), + } + .fail(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + remove_existing_path(&destination_path).with_context(|_| IoSnafu { + path: destination_path, + }) + } + Err(error) => Err(IoSnafu { path: source_path }.into_error(error)), + } +} + +fn materialize_dirty_tree( + source_root: &Path, + working_repo: &git2::Repository, + local_head: Oid, +) -> Result { + let (dirty_paths, untracked_count) = + collect_dirty_paths(source_root, |path| !is_generated_path(path))?; + if untracked_count > 0 { + info!("dirty mode ignores untracked files in the active content repo"); + } + + if dirty_paths.is_empty() { + return Ok(local_head); + } + + let working_root = working_repo + .workdir() + .with_context(|| UpdateTreeSnafu:: { + msg: "build repository workdir is unavailable".into(), + })?; + let mut index = working_repo.index().context(GitSnafu { + what: "open build repository index", + })?; + + for path in dirty_paths { + sync_dirty_path(source_root, working_root, &mut index, &path)?; + } + + index.write().context(GitSnafu { + what: "write build repository index", + })?; + let tree_id = index.write_tree().context(GitSnafu { + what: "write dirty materialization tree", + })?; + let tree = working_repo.find_tree(tree_id).context(GitSnafu { + what: "find dirty materialization tree", + })?; + + let sig = Signature::now("eips-build", "eips-build@eips-build.invalid").context(GitSnafu { + what: "dirty commit signature", + })?; + let parent = working_repo.find_commit(local_head).context(GitSnafu { + what: "find clean local head commit", + })?; + let dirty_head = working_repo + .commit( + Some("HEAD"), + &sig, + &sig, + "Dirty working tree materialization", + &tree, + &[&parent], + ) + .context(GitSnafu { + what: "commit dirty working tree materialization", + })?; + + info!( + "materialized tracked dirty changes from the active content repo into `{}`", + working_root.to_string_lossy() + ); + + Ok(dirty_head) +} + fn check_conflict(master_tree: &Tree, path: &Path, entry: &TreeEntry) -> Result<(), Error> { let original = match master_tree.get_path(path) { Err(_) => return Ok(()), @@ -170,34 +720,51 @@ fn check_conflict(master_tree: &Tree, path: &Path, entry: &TreeEntry) -> Result< pub struct Fresh { src_repo_use: RepositoryUse, + src_repo_path: PathBuf, src_repo_url: Url, + source_materialization: SourceMaterialization, working_repo: git2::Repository, } impl Fresh { - pub fn new(root_path: &Path, build_path: &Path, locations: &Locations) -> Result { - let root_path = absolute(root_path).context(IoSnafu { path: root_path })?; - check_dirty(&root_path)?; - let src_repo_use = locations.identify_repository(&root_path)?; - let src_repo_url = Url::from_directory_path(&root_path) - .ok() - .context(PathUrlSnafu { path: root_path })?; + pub fn new( + root_path: &Path, + repo_path: &Path, + src_repo_use: RepositoryUse, + source_materialization: SourceMaterialization, + ) -> Result { + let root_path = absolute(root_path).with_context(|_| IoSnafu { path: root_path })?; + if source_materialization == SourceMaterialization::Clean { + check_dirty(&root_path)?; + } + let src_repo_url = + Url::from_directory_path(&root_path) + .ok() + .with_context(|| PathUrlSnafu { + path: root_path.clone(), + })?; debug!("source repository at `{src_repo_url}`"); - let working_repo = open_or_init(build_path)?; + let working_repo = open_or_init(repo_path)?; Ok(Self { working_repo, + src_repo_path: root_path, src_repo_url, src_repo_use, + source_materialization, }) } pub fn clone_src(self) -> Result { info!("cloning local repository"); - let master = fetch(&self.working_repo, self.src_repo_url.as_str(), "HEAD")?; + let master = fetch( + &self.working_repo, + self.src_repo_url.as_str(), + "HEAD:refs/build-eips/source-head", + )?; self.working_repo .set_head_detached(master.id()) .context(GitSnafu { what: "detach" })?; @@ -226,9 +793,15 @@ impl Fresh { panic!("submodules not supported yet"); } - let local_head = master.id(); + let mut local_head = master.id(); drop(master); drop(branch); + + if self.source_materialization == SourceMaterialization::Dirty { + local_head = + materialize_dirty_tree(&self.src_repo_path, &self.working_repo, local_head)?; + } + Ok(SourceOnly { local_head, src_repo_use: self.src_repo_use, @@ -250,7 +823,7 @@ impl SourceOnly { let latest_master = fetch( &self.working_repo, self.src_repo_use.location.repository.as_str(), - "master", + "master:refs/build-eips/upstream-head", )?; let upstream_head = latest_master.id(); drop(latest_master); @@ -374,26 +947,40 @@ impl SourceWithUpstream { pub fn merge(&self) -> Result<(), Error> { let repo_use = &self.src_repo_use; - let master_tree = self.local_head_tree()?; let mut local_head = self.local_head; - for (other_kind, other_repo) in repo_use.other_repos.iter().progress_ext("Merge Repos") { + for (index, (other_kind, other_repo)) in repo_use + .other_repos + .iter() + .progress_ext("Merge Repos") + .enumerate() + { + let local_commit = self + .working_repo + .find_commit(local_head) + .context(GitSnafu { + what: "find local head commit", + })?; + let local_tree = local_commit.tree().context(GitSnafu { + what: "getting local head tree", + })?; info!("fetching {other_kind} repository"); - let master_other = fetch( - &self.working_repo, - other_repo.as_str(), - "master:master-other", - )?; + // Local sibling overrides should follow the checked-out repo HEAD instead of assuming `master`. + let other_ref = format!("refs/build-eips/other-head-{index}"); + let other_refspec = if other_repo.scheme() == "file" { + format!("+HEAD:{other_ref}") + } else { + format!("+master:{other_ref}") + }; + let master_other = fetch(&self.working_repo, other_repo.as_str(), &other_refspec)?; let other_tree = master_other.tree().context(GitSnafu { what: "getting other tree", })?; let mut tree_builder = TreeUpdateBuilder::new(); - let prefix = format!("{}/", super::CONTENT_DIR); + let prefix = format!("{}/", CONTENT_DIR); let mut walk_error: Option = None; let walk_result = other_tree.walk(git2::TreeWalkMode::PreOrder, |a, b| { - if !a.starts_with(&prefix) - && (!a.is_empty() || b.name() != Some(super::CONTENT_DIR)) - { + if !a.starts_with(&prefix) && (!a.is_empty() || b.name() != Some(CONTENT_DIR)) { return TreeWalkResult::Skip; } @@ -425,7 +1012,7 @@ impl SourceWithUpstream { } } - if let Err(e) = check_conflict(&master_tree, Path::new(&path), b) { + if let Err(e) = check_conflict(&local_tree, Path::new(&path), b) { walk_error = Some(e); return TreeWalkResult::Abort; } @@ -444,7 +1031,7 @@ impl SourceWithUpstream { })?; let merged_tree_oid = tree_builder - .create_updated(&self.working_repo, &master_tree) + .create_updated(&self.working_repo, &local_tree) .context(GitSnafu { what: "build tree" })?; let merged_tree = self.working_repo.find_tree(merged_tree_oid).unwrap(); @@ -456,12 +1043,6 @@ impl SourceWithUpstream { }, )?; let msg = format!("Merge {other_repo}"); - let master = self - .working_repo - .find_commit(local_head) - .context(GitSnafu { - what: "find local head commit", - })?; local_head = self .working_repo .commit( @@ -470,7 +1051,7 @@ impl SourceWithUpstream { &sig, &msg, &merged_tree, - &[&master, &master_other], + &[&local_commit, &master_other], ) .context(GitSnafu { what: "committing" })?; @@ -480,15 +1061,21 @@ impl SourceWithUpstream { what: "checkout merged", })?; - self.working_repo - .find_branch("master-other", BranchType::Local) - .context(GitSnafu { - what: "find master-other", - })? - .delete() - .context(GitSnafu { - what: "delete master-other", - })?; + drop(merged_tree); + drop(other_tree); + drop(master_other); + drop(local_tree); + drop(local_commit); + match self.working_repo.find_reference(&other_ref) { + Ok(mut reference) => { + if let Err(error) = reference.delete() { + debug!("unable to delete temporary sibling ref `{other_ref}`: {error}"); + } + } + Err(error) => { + debug!("temporary sibling ref `{other_ref}` was not deleted: {error}"); + } + } } Ok(()) @@ -501,7 +1088,19 @@ fn fetch<'a>( refspec: &'_ str, ) -> Result, Error> { debug!("fetching repository at `{url}`"); - let mut remote = repo.remote_anonymous(url).context(GitSnafu { + let remote_name = "__build_eips_fetch"; + match repo.remote_delete(remote_name) { + Ok(()) => (), + Err(error) if error.code() == git2::ErrorCode::NotFound => (), + Err(error) => { + return Err(GitSnafu { + what: "deleting temporary remote", + } + .into_error(error)) + } + } + + let mut remote = repo.remote(remote_name, url).context(GitSnafu { what: "creating remote", })?; { @@ -514,14 +1113,24 @@ fn fetch<'a>( what: "fetching repo", })?; } + drop(remote); + repo.remote_delete(remote_name).context(GitSnafu { + what: "deleting temporary remote", + })?; + + let fetched_ref = refspec + .split_once(':') + .map(|(_, destination)| destination) + .filter(|destination| !destination.is_empty()) + .unwrap_or("FETCH_HEAD"); let commit = repo - .revparse_single("FETCH_HEAD") + .revparse_single(fetched_ref) .context(GitSnafu { - what: "revparse FETCH_HEAD", + what: "revparse fetched ref", })? .peel_to_commit() .context(GitSnafu { - what: "peel FETCH_HEAD", + what: "peel fetched ref", })?; Ok(commit) } @@ -571,3 +1180,194 @@ impl Cache { Ok(dir) } } + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use git2::{IndexAddOption, Repository, Signature}; + use tempfile::TempDir; + + use super::{materialize_working_tree, sync_working_tree_paths, tracked_working_tree_paths}; + + fn write_file(root: &Path, relative: impl AsRef, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn commit_all(repo: &Repository, message: &str) { + let mut index = repo.index().unwrap(); + index + .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + let parents = repo + .head() + .ok() + .and_then(|head| head.target()) + .map(|oid| repo.find_commit(oid).unwrap()) + .into_iter() + .collect::>(); + let parent_refs = parents.iter().collect::>(); + + repo.commit( + Some("HEAD"), + &signature, + &signature, + message, + &tree, + &parent_refs, + ) + .unwrap(); + } + + fn init_repo(path: &Path, files: &[(&str, &str)]) -> Repository { + std::fs::create_dir_all(path).unwrap(); + let repo = Repository::init(path).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + for (relative, contents) in files { + write_file(path, relative, contents); + } + commit_all(&repo, "initial"); + repo + } + + fn stage_path(repo: &Repository, relative: &str) { + let mut index = repo.index().unwrap(); + index.add_path(Path::new(relative)).unwrap(); + index.write().unwrap(); + } + + #[test] + fn materialize_working_tree_uses_tracked_theme_scope() { + let temp = TempDir::new().unwrap(); + let theme = temp.path().join("theme"); + let mounted = temp.path().join("repo/themes/eips-theme"); + let repo = init_repo( + &theme, + &[ + ("config/zola.toml", "title = 'theme'\n"), + ("build/generated.txt", "tracked build path\n"), + ("delete.txt", "delete me\n"), + ("staged.txt", "old staged\n"), + ("tracked.txt", "old tracked\n"), + ], + ); + + write_file(&theme, "tracked.txt", "unstaged tracked edit\n"); + write_file(&theme, "staged.txt", "staged tracked edit\n"); + stage_path(&repo, "staged.txt"); + write_file(&theme, "new-staged.txt", "new staged file\n"); + stage_path(&repo, "new-staged.txt"); + std::fs::remove_file(theme.join("delete.txt")).unwrap(); + let mut index = repo.index().unwrap(); + index.remove_path(Path::new("delete.txt")).unwrap(); + index.write().unwrap(); + write_file( + &theme, + "untracked.txt", + "ignored by theme materialization\n", + ); + + materialize_working_tree(&theme, &mounted).unwrap(); + + assert_eq!( + std::fs::read_to_string(mounted.join("config/zola.toml")).unwrap(), + "title = 'theme'\n" + ); + assert_eq!( + std::fs::read_to_string(mounted.join("build/generated.txt")).unwrap(), + "tracked build path\n" + ); + assert_eq!( + std::fs::read_to_string(mounted.join("tracked.txt")).unwrap(), + "unstaged tracked edit\n" + ); + assert_eq!( + std::fs::read_to_string(mounted.join("staged.txt")).unwrap(), + "staged tracked edit\n" + ); + assert_eq!( + std::fs::read_to_string(mounted.join("new-staged.txt")).unwrap(), + "new staged file\n" + ); + assert!(!mounted.join("delete.txt").exists()); + assert!(!mounted.join("untracked.txt").exists()); + } + + #[test] + fn newly_staged_theme_file_syncs_after_git_index_rescan() { + let temp = TempDir::new().unwrap(); + let theme = temp.path().join("theme"); + let mounted = temp.path().join("repo/themes/eips-theme"); + let repo = init_repo(&theme, &[("config/zola.toml", "title = 'theme'\n")]); + materialize_working_tree(&theme, &mounted).unwrap(); + let mut previous_dirty_paths = tracked_working_tree_paths(&theme) + .unwrap() + .into_iter() + .collect::>(); + + write_file(&theme, "templates/new.html", "new staged template\n"); + stage_path(&repo, "templates/new.html"); + let current_dirty_paths = tracked_working_tree_paths(&theme) + .unwrap() + .into_iter() + .collect::>(); + let affected_paths = previous_dirty_paths + .union(¤t_dirty_paths) + .cloned() + .collect(); + + sync_working_tree_paths(&theme, &mounted, &affected_paths).unwrap(); + previous_dirty_paths = current_dirty_paths; + + assert_eq!( + std::fs::read_to_string(mounted.join("templates/new.html")).unwrap(), + "new staged template\n" + ); + assert!(previous_dirty_paths.contains(Path::new("templates/new.html"))); + } + + #[cfg(target_family = "unix")] + #[test] + fn tracked_theme_symlinks_are_materialized_as_symlinks() { + let temp = TempDir::new().unwrap(); + let theme = temp.path().join("theme"); + let mounted = temp.path().join("repo/themes/eips-theme"); + std::fs::create_dir_all(&theme).unwrap(); + let repo = Repository::init(&theme).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + write_file(&theme, "target.txt", "target\n"); + std::os::unix::fs::symlink("target.txt", theme.join("linked.txt")).unwrap(); + commit_all(&repo, "initial"); + + materialize_working_tree(&theme, &mounted).unwrap(); + + assert!(std::fs::symlink_metadata(mounted.join("linked.txt")) + .unwrap() + .file_type() + .is_symlink()); + assert_eq!( + std::fs::read_link(mounted.join("linked.txt")).unwrap(), + PathBuf::from("target.txt") + ); + } + + #[test] + fn materialize_working_tree_requires_git_repository() { + let temp = TempDir::new().unwrap(); + let theme = temp.path().join("theme"); + let mounted = temp.path().join("repo/themes/eips-theme"); + std::fs::create_dir_all(&theme).unwrap(); + + let error = materialize_working_tree(&theme, &mounted).unwrap_err(); + + assert!(error.to_string().contains("unable to open root repository")); + } +} diff --git a/src/identity.rs b/src/identity.rs index cbb73c9..bb120f0 100644 --- a/src/identity.rs +++ b/src/identity.rs @@ -31,11 +31,9 @@ impl ActiveRepoIdentity { match Config::production() .locations - .identify_repository(root_path) + .identify_repository_title(root_path) { - Ok(repository_use) => Ok(Self::Legacy { - repo_id: repository_use.title, - }), + Ok(repo_id) => Ok(Self::Legacy { repo_id }), Err(git::Error::NoIdentify { .. }) => { snafu::whatever!( "active repository `{}` does not carry `{}` and does not match the legacy EIPs/ERCs identity fallback", diff --git a/src/main.rs b/src/main.rs index bc0449b..81f1a87 100644 --- a/src/main.rs +++ b/src/main.rs @@ -33,6 +33,17 @@ use crate::{ layout::{BUILD_DIR, CONTENT_DIR, OUTPUT_DIR, REPO_DIR}, }; +fn repository_use(config: &Config, root_path: &Path) -> Result { + let repo_id = config + .locations + .identify_repository_title(root_path) + .whatever_context("cannot identify repository use")?; + let Some(repository_use) = config.locations.repository_use_for_title(&repo_id) else { + snafu::whatever!("repository metadata for `{repo_id}` is unavailable"); + }; + Ok(repository_use) +} + fn lock(build_path: &Path) -> Result { let lock_path = build_path.join(".lock"); let mut lock_file = @@ -82,12 +93,18 @@ impl Prepared { let content_path = repo_path.join(CONTENT_DIR); let output_path = build_path.join(OUTPUT_DIR); - let both = git::Fresh::new(&root_path, &repo_path, &config.locations) - .whatever_context("initializing build repo")? - .clone_src() - .whatever_context("cloning source repo")? - .fetch_upstream() - .whatever_context("fetching upstream repo")?; + let repository_use = repository_use(&config, &root_path)?; + let both = git::Fresh::new( + &root_path, + &repo_path, + repository_use, + git::SourceMaterialization::Clean, + ) + .whatever_context("initializing build repo")? + .clone_src() + .whatever_context("cloning source repo")? + .fetch_upstream() + .whatever_context("fetching upstream repo")?; let changed_files: Vec<_> = both .changed_files() @@ -125,11 +142,7 @@ impl Prepared { } fn build(self) -> Result<(), Whatever> { - let repository_use = self - .config - .locations - .identify_repository(&self.root_path) - .whatever_context("cannot identify repository use")?; + let repository_use = repository_use(&self.config, &self.root_path)?; zola::build( self.config.theme.repository.as_str(), &self.config.theme.commit, From a71fe3a5202f38387f2b5fd8f78573d28ef1b449 Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Sun, 3 May 2026 20:43:18 -0400 Subject: [PATCH 04/20] Add workspace init baseline Add build-eips init to create a workspace root, clone declared sibling repos and the shared theme, optionally clone template, and create the local build root. Write starter .build-eips.toml only when missing and regenerate a base WORKSPACE.md guide for the initialized workspace. Use active repository identity and staging repo metadata for workspace bootstrap while leaving doctor, platform-dev repos, and runtime behavior to later PRs. --- src/cli.rs | 10 ++ src/context.rs | 50 +++++- src/identity.rs | 25 +++ src/main.rs | 8 + src/workspace.rs | 408 +++++++++++++++++++++++++++++++++++++++++++ src/workspace_doc.md | 42 +++++ 6 files changed, 540 insertions(+), 3 deletions(-) create mode 100644 src/workspace.rs create mode 100644 src/workspace_doc.md diff --git a/src/cli.rs b/src/cli.rs index f7a2480..618fba5 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -65,6 +65,16 @@ pub(crate) enum Operation { #[clap(long, value_enum, default_value_t)] format: ChangedFormat, }, + + /// Create workspace config, docs, build root, and missing local repos + Init { + /// Workspace root directory + path: PathBuf, + + /// Also clone template for proposal-family scaffold work + #[arg(long)] + template: bool, + }, } #[derive(Debug, clap::ValueEnum, Clone, Default)] diff --git a/src/context.rs b/src/context.rs index 0fffb43..1beb395 100644 --- a/src/context.rs +++ b/src/context.rs @@ -4,17 +4,61 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -use std::path::PathBuf; +//! Command context and path resolution helpers. + +use std::path::{Path, PathBuf}; use snafu::{ResultExt, Whatever}; -use crate::{cli::Args, find_root}; +use crate::{cli::Args, config, find_root}; + +#[derive(Debug, Clone)] +pub(crate) struct WorkspaceCommandContext { + pub(crate) search_from: PathBuf, + pub(crate) config_path: Option, +} + +pub(crate) fn resolve_input_path(path: &Path) -> Result { + if path.is_absolute() { + Ok(path.to_path_buf()) + } else { + let cwd = std::env::current_dir().whatever_context("unable to get current directory")?; + Ok(cwd.join(path)) + } +} pub(crate) fn root(args: &Args) -> Result { let dir = match &args.root { None => find_root::find_root().whatever_context("cannot find repository root")?, - Some(p) => p.to_path_buf(), + Some(path) => { + find_root::is_root(path).whatever_context("invalid root directory")?; + path.canonicalize() + .whatever_context("unable to canonicalize root directory")? + } }; find_root::is_root(&dir).whatever_context("invalid root directory")?; Ok(dir) } + +fn workspace_search_start(args: &Args) -> Result { + match &args.root { + Some(path) => { + let path = resolve_input_path(path)?; + path.canonicalize() + .whatever_context("unable to canonicalize workspace search path") + } + None => std::env::current_dir().whatever_context("unable to get current directory"), + } +} + +pub(crate) fn load_workspace_command_context( + args: &Args, +) -> Result { + let search_from = workspace_search_start(args)?; + let config_path = config::discover_path(&search_from); + + Ok(WorkspaceCommandContext { + search_from, + config_path, + }) +} diff --git a/src/identity.rs b/src/identity.rs index bb120f0..d8b6209 100644 --- a/src/identity.rs +++ b/src/identity.rs @@ -65,4 +65,29 @@ impl ActiveRepoIdentity { Self::Legacy { .. } => None, } } + + pub(crate) fn repository_use(&self, staging: bool) -> Result { + match self { + Self::Manifest(manifest) => { + let manifest = manifest.manifest(); + Ok(git::RepositoryUse { + title: manifest.repo_id.clone(), + location: manifest.active_endpoint(staging), + other_repos: manifest.sibling_repositories(staging), + }) + } + Self::Legacy { repo_id } => { + let baseline = if staging { + Config::staging() + } else { + Config::production() + }; + let Some(repository_use) = baseline.locations.repository_use_for_title(repo_id) + else { + snafu::whatever!("legacy repository metadata for `{repo_id}` is unavailable"); + }; + Ok(repository_use) + } + } + } } diff --git a/src/main.rs b/src/main.rs index 81f1a87..d2202fc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,6 +18,7 @@ mod lint; mod markdown; mod print; mod progress; +mod workspace; mod zola; use std::path::{Path, PathBuf}; @@ -31,6 +32,7 @@ use crate::{ cli::{Args, Operation}, config::Config, layout::{BUILD_DIR, CONTENT_DIR, OUTPUT_DIR, REPO_DIR}, + workspace::init_workspace, }; fn repository_use(config: &Config, root_path: &Path) -> Result { @@ -186,6 +188,11 @@ fn run() -> Result<(), Whatever> { return Ok(()); } + if let Operation::Init { path, template } = &args.operation { + init_workspace(&args, path.clone(), *template)?; + return Ok(()); + } + let config = if args.staging { Config::staging() } else { @@ -199,6 +206,7 @@ fn run() -> Result<(), Whatever> { match args.operation { Operation::Print { .. } => unreachable!(), + Operation::Init { .. } => unreachable!(), Operation::Clean => { // TODO: There's a race condition here. Maybe we move the lockfile to the repository // root? diff --git a/src/workspace.rs b/src/workspace.rs new file mode 100644 index 0000000..a6439b8 --- /dev/null +++ b/src/workspace.rs @@ -0,0 +1,408 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Local workspace setup. + +use std::{ + fs::OpenOptions, + io::{ErrorKind, Write}, + path::{Path, PathBuf}, +}; + +use log::info; +use snafu::{ResultExt, Whatever}; +use url::Url; + +use crate::{ + cli::Args, + config, + context::{resolve_input_path, root}, + git, + identity::ActiveRepoIdentity, +}; + +const WORKSPACE_THEME_URL: &str = "https://github.com/eips-wg/theme.git"; +const PROPOSAL_TEMPLATE_URL: &str = "https://github.com/eips-wg/template.git"; +const WORKSPACE_DOC_FILE: &str = "WORKSPACE.md"; + +struct WorkspaceInitRepositories<'a> { + theme: &'a Url, + template: &'a Url, +} + +pub(crate) fn init_workspace( + args: &Args, + workspace_root: PathBuf, + include_template: bool, +) -> Result<(), Whatever> { + let theme_repository = Url::parse(WORKSPACE_THEME_URL) + .whatever_context("invalid workspace theme repository URL")?; + let template_repository = Url::parse(PROPOSAL_TEMPLATE_URL) + .whatever_context("invalid proposal template repository URL")?; + let repositories = WorkspaceInitRepositories { + theme: &theme_repository, + template: &template_repository, + }; + + init_workspace_with_repositories(args, workspace_root, include_template, &repositories) +} + +fn init_workspace_with_repositories( + args: &Args, + workspace_root: PathBuf, + include_template: bool, + repositories: &WorkspaceInitRepositories<'_>, +) -> Result<(), Whatever> { + let root_path = root(args)?; + let active_repo = ActiveRepoIdentity::load(&root_path)?; + let workspace_root = resolve_input_path(&workspace_root)?; + std::fs::create_dir_all(&workspace_root) + .whatever_context("unable to create workspace root directory")?; + let workspace_root = workspace_root + .canonicalize() + .whatever_context("unable to canonicalize workspace root directory")?; + + // Workspace init is a local-dev bootstrap path, so it intentionally uses staging URLs. + let repository_use = active_repo.repository_use(true)?; + + let expected_root = workspace_root.join(&repository_use.title); + if root_path != expected_root { + snafu::whatever!( + "init expects the active repository at `{}`, found `{}`", + expected_root.to_string_lossy(), + root_path.to_string_lossy(), + ); + } + + for (sibling_id, sibling_url) in repository_use.other_repos { + git::clone_missing_repo(sibling_url.as_str(), &workspace_root.join(&sibling_id)) + .with_whatever_context(|_| { + format!("unable to clone workspace sibling repo `{sibling_id}`") + })?; + } + + git::clone_missing_repo( + repositories.theme.as_str(), + &workspace_root.join(config::DEFAULT_THEME_DIR), + ) + .whatever_context("unable to clone workspace theme repo")?; + + if include_template { + git::clone_missing_repo( + repositories.template.as_str(), + &workspace_root.join("template"), + ) + .whatever_context("unable to clone workspace template repo")?; + } + + std::fs::create_dir_all(workspace_root.join(config::DEFAULT_BUILD_ROOT_BASE)) + .whatever_context("unable to create local build root")?; + + let config_path = workspace_root.join(config::LOCAL_CONFIG_FILE); + match OpenOptions::new() + .write(true) + .create_new(true) + .open(&config_path) + { + Ok(mut config_file) => { + config_file + .write_all(config::default_workspace_config_text().as_bytes()) + .whatever_context("unable to write workspace config")?; + } + Err(error) if error.kind() == ErrorKind::AlreadyExists => { + info!( + "leaving existing workspace config `{}` in place", + config_path.to_string_lossy() + ); + } + Err(error) => { + return Err(error).whatever_context("unable to write workspace config"); + } + } + + write_workspace_doc(&workspace_root)?; + + Ok(()) +} + +fn workspace_doc_text() -> &'static str { + include_str!("workspace_doc.md") +} + +fn write_workspace_doc(workspace_root: &Path) -> Result<(), Whatever> { + let doc_path = workspace_root.join(WORKSPACE_DOC_FILE); + std::fs::write(&doc_path, workspace_doc_text()).with_whatever_context(|_| { + format!( + "unable to write workspace document `{}`", + doc_path.to_string_lossy() + ) + })?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use clap::Parser; + use git2::{IndexAddOption, Repository, Signature}; + use tempfile::TempDir; + use url::Url; + + use crate::{ + cli::{Args, Operation}, + config::{self, LoadedWorkspaceConfig}, + }; + + use super::{ + init_workspace_with_repositories, workspace_doc_text, WorkspaceInitRepositories, + WORKSPACE_DOC_FILE, WORKSPACE_THEME_URL, + }; + + fn parse_args(arguments: &[&str]) -> Args { + Args::try_parse_from(arguments).unwrap() + } + + fn file_url(path: &Path) -> Url { + Url::from_directory_path(path).unwrap() + } + + fn write_file(root: &Path, relative: impl AsRef, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn commit_all(repo: &Repository, message: &str) { + let mut index = repo.index().unwrap(); + index + .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + let parents = repo + .head() + .ok() + .and_then(|head| head.target()) + .map(|oid| repo.find_commit(oid).unwrap()) + .into_iter() + .collect::>(); + let parent_refs = parents.iter().collect::>(); + + repo.commit( + Some("HEAD"), + &signature, + &signature, + message, + &tree, + &parent_refs, + ) + .unwrap(); + } + + fn init_repo(path: &Path, files: &[(&str, &str)]) -> Repository { + std::fs::create_dir_all(path).unwrap(); + let repo = Repository::init(path).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + for (relative, contents) in files { + write_file(path, relative, contents); + } + commit_all(&repo, "initial"); + repo + } + + fn repo_manifest_text(repo_id: &str, repository: &Url, siblings: &[(&str, Url)]) -> String { + let mut manifest = format!( + r#" +repo_id = "{repo_id}" + +[production] +repository = "{repository}" +base_url = "https://example.test/{repo_id}/" + +[staging] +repository = "{repository}" +base_url = "https://staging.example.test/{repo_id}/" +"# + ); + + for (sibling_id, sibling_repository) in siblings { + manifest.push_str(&format!( + r#" +[siblings.{sibling_id}.production] +repository = "{sibling_repository}" +base_url = "https://example.test/{sibling_id}/" + +[siblings.{sibling_id}.staging] +repository = "{sibling_repository}" +base_url = "https://staging.example.test/{sibling_id}/" +"# + )); + } + + manifest + } + + fn write_manifest_repo( + path: &Path, + repo_id: &str, + upstream: &Url, + siblings: &[(&str, Url)], + ) -> Repository { + let repo = init_repo(path, &[("content/0001.md", "# Proposal\n")]); + write_file( + path, + config::REPO_MANIFEST_FILE, + &repo_manifest_text(repo_id, upstream, siblings), + ); + commit_all(&repo, "add repo manifest"); + repo + } + + fn init_workspace_source_repo(remotes_root: &Path, name: &str) -> Url { + let path = remotes_root.join(name); + init_repo(&path, &[("README.md", "init test repo\n")]); + file_url(&path) + } + + fn workspace_init_test_repository_urls(remotes_root: &Path) -> (Url, Url) { + ( + init_workspace_source_repo(remotes_root, "theme"), + init_workspace_source_repo(remotes_root, "template"), + ) + } + + #[test] + fn init_command_parses_with_optional_template_flag() { + let plain = parse_args(&["build-eips", "init", "/tmp/workspace"]); + let template = parse_args(&["build-eips", "init", "/tmp/workspace", "--template"]); + + assert!(matches!( + plain.operation, + Operation::Init { + template: false, + .. + } + )); + assert!(matches!( + template.operation, + Operation::Init { template: true, .. } + )); + } + + #[test] + fn workspace_theme_url_is_bootstrap_metadata() { + assert_eq!( + Url::parse(WORKSPACE_THEME_URL).unwrap().as_str(), + "https://github.com/eips-wg/theme.git" + ); + } + + #[test] + fn workspace_doc_text_mentions_base_workspace_content() { + let text = workspace_doc_text(); + + for expected in [ + ".build-eips.toml", + ".local-build", + "build-eips init", + "build-eips build", + "build-eips check", + "build-eips serve", + ] { + assert!( + text.contains(expected), + "workspace document text should contain `{expected}`" + ); + } + + assert!(text.ends_with('\n')); + } + + #[test] + fn workspace_init_clones_required_repos_and_writes_config_and_doc() { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let remotes_root = temp.path().join("remotes"); + let (theme_url, template_url) = workspace_init_test_repository_urls(&remotes_root); + let repositories = WorkspaceInitRepositories { + theme: &theme_url, + template: &template_url, + }; + + let sibling_path = remotes_root.join("ERCs"); + let sibling_url = file_url(&sibling_path); + write_manifest_repo(&sibling_path, "ERCs", &sibling_url, &[]); + + let active_path = workspace_root.join("EIPs"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "EIPs", &active_url, &[("ERCs", sibling_url)]); + + let init_args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "init", + workspace_root.to_str().unwrap(), + "--template", + ]); + + init_workspace_with_repositories(&init_args, workspace_root.clone(), true, &repositories) + .unwrap(); + + assert!(Repository::open(workspace_root.join(config::DEFAULT_THEME_DIR)).is_ok()); + assert!(Repository::open(workspace_root.join("ERCs")).is_ok()); + assert!(Repository::open(workspace_root.join("template")).is_ok()); + assert!(workspace_root + .join(config::DEFAULT_BUILD_ROOT_BASE) + .is_dir()); + assert!( + LoadedWorkspaceConfig::from_path(&workspace_root.join(config::LOCAL_CONFIG_FILE)) + .is_ok() + ); + assert_eq!( + std::fs::read_to_string(workspace_root.join(WORKSPACE_DOC_FILE)).unwrap(), + workspace_doc_text() + ); + } + + #[test] + fn workspace_init_leaves_existing_config_unchanged() { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let remotes_root = temp.path().join("remotes"); + let (theme_url, template_url) = workspace_init_test_repository_urls(&remotes_root); + let repositories = WorkspaceInitRepositories { + theme: &theme_url, + template: &template_url, + }; + let active_path = workspace_root.join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + let existing_config = "[server]\nhost = \"127.0.0.1\"\nport = 1111\n"; + write_file(&workspace_root, config::LOCAL_CONFIG_FILE, existing_config); + let init_args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "init", + workspace_root.to_str().unwrap(), + ]); + + init_workspace_with_repositories(&init_args, workspace_root.clone(), false, &repositories) + .unwrap(); + + assert_eq!( + std::fs::read_to_string(workspace_root.join(config::LOCAL_CONFIG_FILE)).unwrap(), + existing_config + ); + } +} diff --git a/src/workspace_doc.md b/src/workspace_doc.md new file mode 100644 index 0000000..c6191dd --- /dev/null +++ b/src/workspace_doc.md @@ -0,0 +1,42 @@ +# build-eips Workspace + +This directory is a local multi-repo workspace for building and checking EIPs/ERCs with the shared theme and proposal sibling repos. + +## Workspace Layout + +After initialization, the minimal workspace should look like this: + +```text +EIPs-project/ +├── .build-eips.toml +├── WORKSPACE.md +├── .local-build/ +├── EIPs/ +├── ERCs/ +└── theme/ +``` + +Use `build-eips init ..` from an active proposal repo such as `EIPs/` or `ERCs/` to create missing sibling repos, clone `theme/`, create `.local-build/`, write `.build-eips.toml`, and generate this guide. + +Pass `--template` when proposal template work also needs the optional `template/` repo. + +## Local Commands + +Use the active proposal repo for local build commands: + +```sh +build-eips check +build-eips build +build-eips serve +``` + +The workspace config starts with local server and site defaults: + +```toml +[server] +host = "127.0.0.1" +port = 1111 + +[site] +base_url = "http://127.0.0.1:1111" +``` From 3e4c5074d8c6b79362bc232ab98a72ddfcef99ed Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Sun, 3 May 2026 20:48:46 -0400 Subject: [PATCH 05/20] Add workspace doctor Add build-eips doctor for checking workspace config discovery, active repo identity, repository layout, sibling manifests, theme checkout, and required local tools. Report ok/warn/fail diagnostics and fail the command when any check records a failure. Keep this focused on workspace diagnostics; execution policy, runtime commands, and platform-dev setup land in later PRs. --- src/cli.rs | 3 + src/identity.rs | 14 ++ src/main.rs | 8 +- src/workspace.rs | 514 ++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 532 insertions(+), 7 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 618fba5..dc57267 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -75,6 +75,9 @@ pub(crate) enum Operation { #[arg(long)] template: bool, }, + + /// Check workspace layout, local repos, and required tools + Doctor, } #[derive(Debug, clap::ValueEnum, Clone, Default)] diff --git a/src/identity.rs b/src/identity.rs index d8b6209..dc470ab 100644 --- a/src/identity.rs +++ b/src/identity.rs @@ -66,6 +66,20 @@ impl ActiveRepoIdentity { } } + pub(crate) fn sibling_ids(&self) -> Vec { + match self { + Self::Manifest(manifest) => manifest.manifest().siblings.keys().cloned().collect(), + Self::Legacy { repo_id } => Config::production() + .locations + .repository_use_for_title(repo_id) + .expect("legacy repository id should have metadata") + .other_repos + .keys() + .cloned() + .collect(), + } + } + pub(crate) fn repository_use(&self, staging: bool) -> Result { match self { Self::Manifest(manifest) => { diff --git a/src/main.rs b/src/main.rs index d2202fc..8143637 100644 --- a/src/main.rs +++ b/src/main.rs @@ -32,7 +32,7 @@ use crate::{ cli::{Args, Operation}, config::Config, layout::{BUILD_DIR, CONTENT_DIR, OUTPUT_DIR, REPO_DIR}, - workspace::init_workspace, + workspace::{doctor_workspace, init_workspace}, }; fn repository_use(config: &Config, root_path: &Path) -> Result { @@ -193,6 +193,11 @@ fn run() -> Result<(), Whatever> { return Ok(()); } + if let Operation::Doctor = &args.operation { + doctor_workspace(&args)?; + return Ok(()); + } + let config = if args.staging { Config::staging() } else { @@ -207,6 +212,7 @@ fn run() -> Result<(), Whatever> { match args.operation { Operation::Print { .. } => unreachable!(), Operation::Init { .. } => unreachable!(), + Operation::Doctor => unreachable!(), Operation::Clean => { // TODO: There's a race condition here. Maybe we move the lockfile to the repository // root? diff --git a/src/workspace.rs b/src/workspace.rs index a6439b8..637d461 100644 --- a/src/workspace.rs +++ b/src/workspace.rs @@ -4,22 +4,23 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -//! Local workspace setup. +//! Local workspace setup and diagnostics. use std::{ + fmt, fs::OpenOptions, io::{ErrorKind, Write}, path::{Path, PathBuf}, }; use log::info; -use snafu::{ResultExt, Whatever}; +use snafu::{Report, ResultExt, Whatever}; use url::Url; use crate::{ cli::Args, - config, - context::{resolve_input_path, root}, + config::{self, LoadedRepoManifest, LoadedWorkspaceConfig}, + context::{load_workspace_command_context, resolve_input_path, root}, git, identity::ActiveRepoIdentity, }; @@ -33,6 +34,384 @@ struct WorkspaceInitRepositories<'a> { template: &'a Url, } +#[derive(Debug, Clone, Copy)] +enum DoctorStatus { + Ok, + Warn, + Fail, +} + +#[derive(Debug, Default)] +struct DoctorReport { + warnings: usize, + failures: usize, +} + +impl fmt::Display for DoctorStatus { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let label = match self { + Self::Ok => "ok", + Self::Warn => "warn", + Self::Fail => "fail", + }; + + f.write_str(label) + } +} + +impl DoctorReport { + fn record(&mut self, status: DoctorStatus, message: impl AsRef) { + match status { + DoctorStatus::Ok => (), + DoctorStatus::Warn => self.warnings += 1, + DoctorStatus::Fail => self.failures += 1, + } + + println!("[{status}] {}", message.as_ref()); + } +} + +fn command_path(command: &str) -> Option { + let path = std::env::var_os("PATH")?; + + #[cfg(not(windows))] + let candidates = [command.to_owned()]; + + #[cfg(windows)] + { + use std::ffi::OsString; + + let mut candidates = vec![command.to_owned()]; + let command = OsString::from(command); + let path_exts = std::env::var_os("PATHEXT") + .unwrap_or_default() + .to_string_lossy() + .split(';') + .filter(|ext| !ext.is_empty()) + .map(|ext| format!("{}{}", command.to_string_lossy(), ext)) + .collect::>(); + candidates.extend(path_exts); + std::env::split_paths(&path).find_map(|entry| { + candidates + .iter() + .map(|candidate| entry.join(candidate)) + .find(|candidate| candidate.is_file()) + }) + } + + #[cfg(not(windows))] + { + std::env::split_paths(&path).find_map(|entry| { + candidates + .iter() + .map(|candidate| entry.join(candidate)) + .find(|candidate| candidate.is_file()) + }) + } +} + +fn check_workspace_repo( + report: &mut DoctorReport, + workspace_root: &Path, + name: &str, +) -> Option { + let path = workspace_root.join(name); + match git2::Repository::open(&path) { + Ok(_) => { + report.record( + DoctorStatus::Ok, + format!( + "found workspace repo `{}` at `{}`", + name, + path.to_string_lossy() + ), + ); + Some(path) + } + Err(_) if !path.exists() => { + report.record( + DoctorStatus::Fail, + format!( + "expected workspace repo `{}` at `{}`", + name, + path.to_string_lossy() + ), + ); + None + } + Err(_) => { + report.record( + DoctorStatus::Fail, + format!( + "expected `{}` to be a git repository at `{}`", + name, + path.to_string_lossy() + ), + ); + None + } + } +} + +fn check_sibling_manifest_id( + report: &mut DoctorReport, + sibling_path: &Path, + expected_repo_id: &str, +) { + match LoadedRepoManifest::load(sibling_path) { + Ok(Some(manifest)) if manifest.manifest().repo_id == expected_repo_id => report.record( + DoctorStatus::Ok, + format!("sibling `{expected_repo_id}` manifest repo_id matches workspace key"), + ), + Ok(Some(manifest)) => report.record( + DoctorStatus::Fail, + format!( + "sibling `{expected_repo_id}` manifest declares repo_id `{}`", + manifest.manifest().repo_id + ), + ), + Ok(None) => (), + Err(error) => report.record( + DoctorStatus::Fail, + format!( + "sibling `{expected_repo_id}` repo manifest could not be loaded: {}", + Report::from_error(error) + ), + ), + } +} + +fn check_tool(report: &mut DoctorReport, command: &str, why: &str) -> bool { + match command_path(command) { + Some(path) => { + report.record( + DoctorStatus::Ok, + format!( + "found required tool `{}` at `{}`", + command, + path.to_string_lossy() + ), + ); + true + } + None => { + report.record( + DoctorStatus::Fail, + format!("missing required tool `{}`: {}", command, why), + ); + false + } + } +} + +#[cfg(windows)] +fn check_default_windows_build_eips_path(report: &mut DoctorReport) { + let Some(local_app_data) = std::env::var_os("LOCALAPPDATA") else { + return; + }; + + let install_dir = PathBuf::from(local_app_data).join("build-eips").join("bin"); + let build_eips_path = install_dir.join("build-eips.exe"); + + if build_eips_path.is_file() { + report.record( + DoctorStatus::Warn, + format!( + "found build-eips at the default user-local install path `{}`, but `{}` is not on PATH", + build_eips_path.to_string_lossy(), + install_dir.to_string_lossy() + ), + ); + } +} + +#[cfg(not(windows))] +fn check_default_windows_build_eips_path(_report: &mut DoctorReport) {} + +fn collect_doctor_report(args: &Args, check_tools: bool) -> Result { + let context = load_workspace_command_context(args)?; + let mut report = DoctorReport::default(); + let (root_path, active_repo) = match root(args) { + Ok(root_path) => match ActiveRepoIdentity::load(&root_path) { + Ok(active_repo) => { + report.record( + DoctorStatus::Ok, + format!( + "identified active repo `{}` from {}", + active_repo.repo_id(), + active_repo.source_description() + ), + ); + if let Some(manifest) = active_repo.manifest() { + report.record( + DoctorStatus::Ok, + format!( + "repo manifest parses at `{}`", + manifest.manifest_path().to_string_lossy() + ), + ); + } + (Some(root_path), Some(active_repo)) + } + Err(error) => { + report.record( + DoctorStatus::Fail, + format!("active repo identity could not be resolved: {error}"), + ); + (Some(root_path), None) + } + }, + Err(error) => { + report.record( + DoctorStatus::Fail, + format!( + "active repo root could not be resolved: {}", + Report::from_error(error) + ), + ); + (None, None) + } + }; + + match context.config_path.as_ref() { + Some(path) => report.record( + DoctorStatus::Ok, + format!( + "found workspace config candidate `{}`", + path.to_string_lossy() + ), + ), + None => report.record( + DoctorStatus::Fail, + format!( + "could not find `{}` while searching upward from `{}`", + config::LOCAL_CONFIG_FILE, + context.search_from.to_string_lossy() + ), + ), + } + + let parsed_config = context + .config_path + .as_deref() + .map(LoadedWorkspaceConfig::from_path) + .transpose(); + + match parsed_config { + Ok(Some(config)) => { + report.record( + DoctorStatus::Ok, + format!( + "workspace config parses at `{}`", + config.config_path().to_string_lossy() + ), + ); + + let workspace_root = config.workspace_root(); + if workspace_root.is_dir() { + report.record( + DoctorStatus::Ok, + format!( + "workspace root exists at `{}`", + workspace_root.to_string_lossy() + ), + ); + } else { + report.record( + DoctorStatus::Fail, + format!( + "workspace root is missing at `{}`", + workspace_root.to_string_lossy() + ), + ); + } + + if let (Some(root_path), Some(active_repo)) = (root_path.as_ref(), active_repo.as_ref()) + { + let expected_root = workspace_root.join(active_repo.repo_id()); + if root_path == &expected_root { + report.record( + DoctorStatus::Ok, + format!( + "active repo `{}` is checked out at `{}`", + active_repo.repo_id(), + expected_root.to_string_lossy() + ), + ); + } else { + report.record( + DoctorStatus::Fail, + format!( + "active repo `{}` should be checked out at `{}`, found `{}`", + active_repo.repo_id(), + expected_root.to_string_lossy(), + root_path.to_string_lossy() + ), + ); + } + + check_workspace_repo(&mut report, workspace_root, active_repo.repo_id()); + for sibling_id in active_repo.sibling_ids() { + if let Some(sibling_path) = + check_workspace_repo(&mut report, workspace_root, &sibling_id) + { + check_sibling_manifest_id(&mut report, &sibling_path, &sibling_id); + } + } + } else { + report.record( + DoctorStatus::Warn, + "workspace repo layout checks were skipped because active repo identity was unavailable", + ); + } + + check_workspace_repo(&mut report, workspace_root, config::DEFAULT_THEME_DIR); + } + Err(error) => { + report.record( + DoctorStatus::Fail, + format!( + "workspace config could not be parsed: {}", + Report::from_error(error) + ), + ); + } + Ok(None) => (), + } + + if check_tools { + if !check_tool( + &mut report, + "build-eips", + "workspace bootstrap and build-eips commands expect `build-eips` on PATH", + ) { + check_default_windows_build_eips_path(&mut report); + } + check_tool( + &mut report, + "git", + "workspace bootstrap and build-eips commands expect git to be available", + ); + check_tool( + &mut report, + "zola", + "build, check, and serve commands need a working zola binary", + ); + } + + Ok(report) +} + +pub(crate) fn doctor_workspace(args: &Args) -> Result<(), Whatever> { + let report = collect_doctor_report(args, true)?; + + if report.failures > 0 { + snafu::whatever!("doctor found {} failing check(s)", report.failures); + } + + Ok(()) +} + pub(crate) fn init_workspace( args: &Args, workspace_root: PathBuf, @@ -159,8 +538,8 @@ mod tests { }; use super::{ - init_workspace_with_repositories, workspace_doc_text, WorkspaceInitRepositories, - WORKSPACE_DOC_FILE, WORKSPACE_THEME_URL, + collect_doctor_report, init_workspace_with_repositories, workspace_doc_text, + WorkspaceInitRepositories, WORKSPACE_DOC_FILE, WORKSPACE_THEME_URL, }; fn parse_args(arguments: &[&str]) -> Args { @@ -280,6 +659,58 @@ base_url = "https://staging.example.test/{sibling_id}/" ) } + fn assert_workspace_init_and_doctor_for_siblings(sibling_ids: &[&str]) { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let remotes_root = temp.path().join("remotes"); + let (theme_url, template_url) = workspace_init_test_repository_urls(&remotes_root); + let repositories = WorkspaceInitRepositories { + theme: &theme_url, + template: &template_url, + }; + + let sibling_repositories = sibling_ids + .iter() + .map(|sibling_id| { + let sibling_path = remotes_root.join(sibling_id); + let sibling_url = file_url(&sibling_path); + write_manifest_repo(&sibling_path, sibling_id, &sibling_url, &[]); + ((*sibling_id).to_owned(), sibling_url) + }) + .collect::>(); + let sibling_manifest_entries = sibling_repositories + .iter() + .map(|(repo_id, url)| (repo_id.as_str(), url.clone())) + .collect::>(); + + let active_path = workspace_root.join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &sibling_manifest_entries); + let init_args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "init", + workspace_root.to_str().unwrap(), + ]); + + init_workspace_with_repositories(&init_args, workspace_root.clone(), false, &repositories) + .unwrap(); + + assert!(workspace_root.join(config::LOCAL_CONFIG_FILE).is_file()); + assert!(Repository::open(workspace_root.join(config::DEFAULT_THEME_DIR)).is_ok()); + for sibling_id in sibling_ids { + assert!(Repository::open(workspace_root.join(sibling_id)).is_ok()); + } + + let doctor_args = + parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "doctor"]); + let report = collect_doctor_report(&doctor_args, false).unwrap(); + + assert_eq!(report.failures, 0); + assert_eq!(report.warnings, 0); + } + #[test] fn init_command_parses_with_optional_template_flag() { let plain = parse_args(&["build-eips", "init", "/tmp/workspace"]); @@ -298,6 +729,13 @@ base_url = "https://staging.example.test/{sibling_id}/" )); } + #[test] + fn doctor_command_parses() { + let args = parse_args(&["build-eips", "doctor"]); + + assert!(matches!(args.operation, Operation::Doctor)); + } + #[test] fn workspace_theme_url_is_bootstrap_metadata() { assert_eq!( @@ -374,6 +812,13 @@ base_url = "https://staging.example.test/{sibling_id}/" ); } + #[test] + fn workspace_init_and_doctor_cover_zero_one_and_many_siblings() { + assert_workspace_init_and_doctor_for_siblings(&[]); + assert_workspace_init_and_doctor_for_siblings(&["ERCs"]); + assert_workspace_init_and_doctor_for_siblings(&["EIPs", "ERCs"]); + } + #[test] fn workspace_init_leaves_existing_config_unchanged() { let temp = TempDir::new().unwrap(); @@ -405,4 +850,61 @@ base_url = "https://staging.example.test/{sibling_id}/" existing_config ); } + + #[test] + fn workspace_doctor_missing_config_reports_one_failure_without_skip_warning() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + let args = parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "doctor"]); + + let report = collect_doctor_report(&args, false).unwrap(); + + assert_eq!(report.failures, 1); + assert_eq!(report.warnings, 0); + } + + #[test] + fn workspace_doctor_parse_failed_config_reports_one_failure_without_skip_warning() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + std::fs::write(workspace.path().join(config::LOCAL_CONFIG_FILE), "[").unwrap(); + let args = parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "doctor"]); + + let report = collect_doctor_report(&args, false).unwrap(); + + assert_eq!(report.failures, 1); + assert_eq!(report.warnings, 0); + } + + #[test] + fn workspace_doctor_removed_config_fields_report_parse_failure_check() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + let config_path = workspace.path().join(config::LOCAL_CONFIG_FILE); + std::fs::write( + &config_path, + r#" +build_root_base = ".local-build" +default_profile = "local" + +[profiles.local] +staging = true +"#, + ) + .unwrap(); + let error = LoadedWorkspaceConfig::from_path(&config_path).unwrap_err(); + assert!(matches!(error, config::WorkspaceError::Parse { .. })); + let args = parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "doctor"]); + + let report = collect_doctor_report(&args, false).unwrap(); + + assert_eq!(report.failures, 1); + assert_eq!(report.warnings, 0); + } } From c63d888b00187ee5899c7e2539806e2919894e4f Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Sun, 3 May 2026 20:59:00 -0400 Subject: [PATCH 06/20] Resolve workspace execution policy Add ResolvedExecution for command source policy, build roots, base URL overrides, staging/production selection, and clean versus dirty source materialization. Add CLI execution controls for production, remote siblings, build roots, build/serve base URL resolution, plain build/serve/check clean mode, and parity commands, with tests for the command matrix. Route build, check, serve, clean, and changed-file listing through the resolved policy while leaving targeted --only behavior to later PRs. --- src/changed.rs | 19 +- src/cli.rs | 376 ++++++++++++++++++++++++- src/execution.rs | 715 +++++++++++++++++++++++++++++++++++++++++++++++ src/lint.rs | 2 +- src/main.rs | 112 ++++---- 5 files changed, 1154 insertions(+), 70 deletions(-) create mode 100644 src/execution.rs diff --git a/src/changed.rs b/src/changed.rs index 62b6a9b..cf53c46 100644 --- a/src/changed.rs +++ b/src/changed.rs @@ -13,7 +13,7 @@ use std::{ use snafu::{ResultExt, Whatever}; -use crate::{cli::ChangedFormat, config::Config, git, layout::REPO_DIR}; +use crate::{cli::ChangedFormat, execution::ResolvedExecution, git, layout::REPO_DIR}; pub(crate) fn is_proposal_path(mut p: PathBuf) -> bool { // Only lint `content/00001.md` and `content/00001/index.md` files. @@ -53,27 +53,18 @@ pub(crate) fn is_proposal_path(mut p: PathBuf) -> bool { } pub(crate) fn run( - root_path: &Path, + resolved: &ResolvedExecution, build_path: &Path, - config: &Config, all: bool, format: &ChangedFormat, ) -> Result<(), Whatever> { let repo_path = build_path.join(REPO_DIR); - let repo_id = config - .locations - .identify_repository_title(root_path) - .whatever_context("cannot identify repository use")?; - let Some(repository_use) = config.locations.repository_use_for_title(&repo_id) else { - snafu::whatever!("repository metadata for `{repo_id}` is unavailable"); - }; - let both = git::Fresh::new( - root_path, + &resolved.root_path, &repo_path, - repository_use, - git::SourceMaterialization::Clean, + resolved.repository_use.clone(), + resolved.source_materialization, ) .whatever_context("initializing build repo")? .clone_src() diff --git a/src/cli.rs b/src/cli.rs index dc57267..bf16e89 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -9,6 +9,7 @@ use std::path::{Path, PathBuf}; use clap::{Parser, Subcommand}; +use url::Url; use crate::{lint, print}; @@ -20,17 +21,43 @@ pub(crate) struct Args { #[clap(short = 'C')] pub(crate) root: Option, - /// Use the staging repositories (for testing) - #[clap(long = "staging")] + /// Force the staging repositories and base URLs + #[clap(long)] pub(crate) staging: bool, + /// Force the production repositories and base URLs + #[clap(long)] + pub(crate) production: bool, + + /// Use the configured remote sibling content repositories + #[clap(long)] + pub(crate) remote_siblings: bool, + + /// Write build artifacts under BUILD_ROOT instead of the default location + #[clap(long)] + pub(crate) build_root: Option, + #[clap(subcommand)] pub(crate) operation: Operation, } -#[derive(Debug, Subcommand)] +#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)] +pub(crate) struct BaseUrlCliArgs { + /// Override the rendered-site base URL for this command + #[arg(long, value_parser = clap::value_parser!(Url))] + pub(crate) base_url: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)] +pub(crate) struct CleanCliArgs { + /// Ignore tracked working-tree changes in the active repo + #[arg(long)] + pub(crate) clean: bool, +} + +#[derive(Debug, Clone, Subcommand)] pub(crate) enum Operation { - /// Print various useful things, like available lints + /// Print linter schema metadata and lint configuration Print { #[command(flatten)] print: print::CmdArgs, @@ -40,21 +67,36 @@ pub(crate) enum Operation { Build { #[command(flatten)] eipw: lint::CmdArgs, + + #[command(flatten)] + base_url: BaseUrlCliArgs, + + #[command(flatten)] + clean: CleanCliArgs, }, /// Build the project and launch a web server to preview it Serve { #[command(flatten)] eipw: lint::CmdArgs, + + #[command(flatten)] + base_url: BaseUrlCliArgs, + + #[command(flatten)] + clean: CleanCliArgs, }, - /// Remove temporary and output files + /// Remove the selected build directory and generated output Clean, - /// Analyze the repository and report errors, but don't build HTML files + /// Validate that the site builds cleanly without writing HTML output Check { #[command(flatten)] eipw: lint::CmdArgs, + + #[command(flatten)] + clean: CleanCliArgs, }, /// List files changed since the last commit common to both the local and upstream repositories @@ -78,6 +120,39 @@ pub(crate) enum Operation { /// Check workspace layout, local repos, and required tools Doctor, + + /// Run build, serve, or check with staging remote proposal sources + Parity { + #[command(subcommand)] + command: ProfiledOperation, + }, +} + +#[derive(Debug, Clone, Subcommand)] +pub(crate) enum ProfiledOperation { + /// Build the project and output HTML + Build { + #[command(flatten)] + eipw: lint::CmdArgs, + + #[command(flatten)] + base_url: BaseUrlCliArgs, + }, + + /// Build the project and launch a web server to preview it + Serve { + #[command(flatten)] + eipw: lint::CmdArgs, + + #[command(flatten)] + base_url: BaseUrlCliArgs, + }, + + /// Validate that the site builds cleanly without writing HTML output + Check { + #[command(flatten)] + eipw: lint::CmdArgs, + }, } #[derive(Debug, clap::ValueEnum, Clone, Default)] @@ -88,6 +163,91 @@ pub(crate) enum ChangedFormat { Json, } +#[derive(Debug, Clone)] +pub(crate) enum RuntimeOperation { + Build { eipw: lint::CmdArgs }, + Serve { eipw: lint::CmdArgs }, + Clean, + Check { eipw: lint::CmdArgs }, + Changed { all: bool, format: ChangedFormat }, +} + +impl Operation { + pub(crate) fn base_url_cli_args(&self) -> BaseUrlCliArgs { + match self { + Self::Build { base_url, .. } | Self::Serve { base_url, .. } => base_url.clone(), + Self::Parity { command } => command.base_url_cli_args(), + Self::Print { .. } + | Self::Clean + | Self::Check { .. } + | Self::Changed { .. } + | Self::Init { .. } + | Self::Doctor => BaseUrlCliArgs::default(), + } + } + + pub(crate) fn clean_cli_args(&self) -> CleanCliArgs { + match self { + Self::Build { clean, .. } | Self::Serve { clean, .. } | Self::Check { clean, .. } => { + clean.clone() + } + Self::Print { .. } + | Self::Clean + | Self::Changed { .. } + | Self::Init { .. } + | Self::Doctor + | Self::Parity { .. } => CleanCliArgs::default(), + } + } + + pub(crate) fn is_plain_site_command(&self) -> bool { + matches!( + self, + Self::Build { .. } | Self::Serve { .. } | Self::Check { .. } + ) + } + + pub(crate) fn runtime_operation(&self) -> Option { + match self { + Self::Print { .. } | Self::Init { .. } | Self::Doctor => None, + Self::Build { eipw, .. } => Some(RuntimeOperation::Build { eipw: eipw.clone() }), + Self::Serve { eipw, .. } => Some(RuntimeOperation::Serve { eipw: eipw.clone() }), + Self::Clean => Some(RuntimeOperation::Clean), + Self::Check { eipw, .. } => Some(RuntimeOperation::Check { eipw: eipw.clone() }), + Self::Changed { all, format } => Some(RuntimeOperation::Changed { + all: *all, + format: format.clone(), + }), + Self::Parity { command } => Some(command.runtime_operation()), + } + } + + pub(crate) fn is_workspace_lifecycle_command(&self) -> bool { + matches!(self, Self::Init { .. } | Self::Doctor) + } + + pub(crate) fn is_print_command(&self) -> bool { + matches!(self, Self::Print { .. }) + } +} + +impl ProfiledOperation { + fn base_url_cli_args(&self) -> BaseUrlCliArgs { + match self { + Self::Build { base_url, .. } | Self::Serve { base_url, .. } => base_url.clone(), + Self::Check { .. } => BaseUrlCliArgs::default(), + } + } + + fn runtime_operation(&self) -> RuntimeOperation { + match self { + Self::Build { eipw, .. } => RuntimeOperation::Build { eipw: eipw.clone() }, + Self::Serve { eipw, .. } => RuntimeOperation::Serve { eipw: eipw.clone() }, + Self::Check { eipw } => RuntimeOperation::Check { eipw: eipw.clone() }, + } + } +} + impl ChangedFormat { fn print_sep(files: &[&Path], sep: &str) { let files: Vec<_> = files @@ -121,3 +281,207 @@ impl ChangedFormat { } } } + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::{Args, Operation, ProfiledOperation, RuntimeOperation}; + + fn parse_args(arguments: &[&str]) -> Args { + Args::try_parse_from(arguments).unwrap() + } + + #[test] + fn parity_command_parses_as_command_prefix() { + let args = parse_args(&["build-eips", "parity", "build"]); + + assert!(matches!( + args.operation, + Operation::Parity { + command: ProfiledOperation::Build { .. } + } + )); + } + + #[test] + fn profile_flag_is_rejected() { + let error = + Args::try_parse_from(["build-eips", "--profile", "local", "build"]).unwrap_err(); + + assert!(error + .to_string() + .contains("unexpected argument '--profile'")); + } + + #[test] + fn removed_theme_flag_is_rejected() { + let removed_flag = concat!("--remote", "-theme"); + let error = Args::try_parse_from(["build-eips", removed_flag, "build"]).unwrap_err(); + + assert!(error + .to_string() + .contains(&format!("unexpected argument '{removed_flag}'"))); + } + + #[test] + fn base_url_flags_parse_on_build_and_serve_forms() { + let cases: &[(&[&str], &str)] = &[ + ( + &["build-eips", "build", "--base-url", "http://localhost:4000"], + "build", + ), + ( + &["build-eips", "serve", "--base-url", "http://localhost:4000"], + "serve", + ), + ( + &[ + "build-eips", + "parity", + "build", + "--base-url", + "http://localhost:4000", + ], + "build", + ), + ( + &[ + "build-eips", + "parity", + "serve", + "--base-url", + "http://localhost:4000", + ], + "serve", + ), + ]; + + for (arguments, expected_runtime_operation) in cases { + let args = parse_args(arguments); + + assert!(matches!( + ( + args.operation.runtime_operation().unwrap(), + *expected_runtime_operation + ), + (RuntimeOperation::Build { .. }, "build") + | (RuntimeOperation::Serve { .. }, "serve") + )); + assert_eq!( + args.operation + .base_url_cli_args() + .base_url + .as_ref() + .unwrap() + .as_str(), + "http://localhost:4000/" + ); + } + } + + #[test] + fn clean_flags_parse_only_on_plain_site_commands() { + for arguments in [ + &["build-eips", "build", "--clean"][..], + &["build-eips", "serve", "--clean"][..], + &["build-eips", "check", "--clean"][..], + ] { + let args = parse_args(arguments); + assert!(args.operation.clean_cli_args().clean); + } + + for arguments in [ + &["build-eips", "parity", "build", "--clean"][..], + &["build-eips", "parity", "serve", "--clean"][..], + &["build-eips", "parity", "check", "--clean"][..], + &["build-eips", "changed", "--clean"][..], + &["build-eips", "clean", "--clean"][..], + ] { + assert!(Args::try_parse_from(arguments).is_err()); + } + } + + #[test] + fn removed_command_surface_is_rejected() { + for arguments in [ + &["build-eips", "dirty", "build"][..], + &["build-eips", "--allow-dirty", "build"][..], + &["build-eips", "--no-allow-dirty", "build"][..], + &["build-eips", "--no-staging", "build"][..], + &["build-eips", "--remote-sibling-repo", "build"][..], + &["build-eips", "workspace", "init", "/tmp/workspace"][..], + &["build-eips", "workspace", "doctor"][..], + &["build-eips", "parity", "clean"][..], + &["build-eips", "parity", "changed"][..], + ] { + assert!(Args::try_parse_from(arguments).is_err()); + } + } + + #[test] + fn base_url_flag_is_rejected_on_non_rendering_forms() { + let cases: &[&[&str]] = &[ + &["build-eips", "check", "--base-url", "http://localhost:4000"], + &[ + "build-eips", + "changed", + "--base-url", + "http://localhost:4000", + ], + &[ + "build-eips", + "doctor", + "--base-url", + "http://localhost:4000", + ], + &[ + "build-eips", + "init", + "/tmp/workspace", + "--base-url", + "http://localhost:4000", + ], + &["build-eips", "print", "--base-url", "http://localhost:4000"], + ]; + + for arguments in cases { + assert!(Args::try_parse_from(*arguments).is_err()); + } + } + + #[test] + fn workspace_lifecycle_commands_parse() { + let plain = parse_args(&["build-eips", "init", "/tmp/workspace"]); + let template = parse_args(&["build-eips", "init", "/tmp/workspace", "--template"]); + let doctor = parse_args(&["build-eips", "doctor"]); + + assert!(matches!( + plain.operation, + Operation::Init { + template: false, + .. + } + )); + assert!(matches!( + template.operation, + Operation::Init { template: true, .. } + )); + assert!(matches!(doctor.operation, Operation::Doctor)); + } + + #[test] + fn remote_siblings_flag_parses() { + let args = parse_args(&["build-eips", "--remote-siblings", "build"]); + + assert!(args.remote_siblings); + } + + #[test] + fn explicit_workspace_config_path_is_not_accepted() { + let error = Args::try_parse_from(["build-eips", "--config", "/tmp/config.toml", "build"]) + .unwrap_err(); + + assert!(error.to_string().contains("unexpected argument '--config'")); + } +} diff --git a/src/execution.rs b/src/execution.rs new file mode 100644 index 0000000..8809e9a --- /dev/null +++ b/src/execution.rs @@ -0,0 +1,715 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Execution source and path resolution. + +use std::path::{Path, PathBuf}; + +use log::{debug, info}; +use snafu::{OptionExt, ResultExt, Whatever}; +use url::Url; + +use crate::{ + cli::{Args, Operation}, + config::{self, LoadedWorkspaceConfig}, + context::{resolve_input_path, root}, + git, + identity::ActiveRepoIdentity, + layout::BUILD_DIR, +}; + +#[derive(Debug, Clone)] +pub(crate) struct ResolvedExecution { + pub(crate) root_path: PathBuf, + pub(crate) build_path: PathBuf, + pub(crate) repository_use: git::RepositoryUse, + pub(crate) source_materialization: git::SourceMaterialization, + pub(crate) base_url_override: Option, + pub(crate) staging: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum SelectedSource { + WorkspaceLocal, + Remote, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ExecutionSettings { + pub(crate) build_root: Option, + pub(crate) staging: bool, + pub(crate) allow_dirty: bool, + pub(crate) sibling: SelectedSource, +} + +fn has_execution_override_flags(args: &Args) -> bool { + args.staging || args.production || args.remote_siblings || args.build_root.is_some() +} + +pub(crate) fn validate_non_execution_command_flags(args: &Args) -> Result<(), Whatever> { + if args.operation.is_workspace_lifecycle_command() && has_execution_override_flags(args) { + snafu::whatever!("execution override flags cannot be used with `init` or `doctor`"); + } + + if args.operation.is_print_command() && has_execution_override_flags(args) { + snafu::whatever!("execution override flags cannot be used with `print`"); + } + + Ok(()) +} + +fn resolve_bool_override( + enabled: bool, + disabled: bool, + enabled_flag: &str, + disabled_flag: &str, +) -> Result, Whatever> { + match (enabled, disabled) { + (true, true) => { + snafu::whatever!("cannot pass both `{enabled_flag}` and `{disabled_flag}`") + } + (true, false) => Ok(Some(true)), + (false, true) => Ok(Some(false)), + (false, false) => Ok(None), + } +} + +fn remote_source_override(force_remote: bool) -> Option { + force_remote.then_some(SelectedSource::Remote) +} + +fn format_sibling_ids(sibling_ids: &[String]) -> String { + sibling_ids.join(", ") +} + +fn resolve_environment_override(args: &Args) -> Result, Whatever> { + resolve_bool_override(args.staging, args.production, "--staging", "--production") +} + +fn explicit_environment_or_parity(args: &Args) -> Result, Whatever> { + if let Some(staging) = resolve_environment_override(args)? { + return Ok(Some(staging)); + } + + if matches!(args.operation, Operation::Parity { .. }) { + return Ok(Some(true)); + } + + Ok(None) +} + +pub(crate) fn resolve_execution_settings( + args: &Args, + sibling_ids: &[String], + workspace_config: Option<&LoadedWorkspaceConfig>, +) -> Result { + let build_root = args + .build_root + .as_deref() + .map(resolve_input_path) + .transpose()?; + let explicit_environment = explicit_environment_or_parity(args)?; + let sibling_override = remote_source_override(args.remote_siblings); + let clean = args.operation.clean_cli_args().clean; + + let (staging, allow_dirty, default_sibling) = if let Some(staging) = explicit_environment { + (staging, false, SelectedSource::Remote) + } else if args.operation.is_plain_site_command() { + (true, !clean, SelectedSource::WorkspaceLocal) + } else { + (false, false, SelectedSource::Remote) + }; + + if sibling_override.is_none() + && default_sibling == SelectedSource::WorkspaceLocal + && !sibling_ids.is_empty() + && workspace_config.is_none() + { + snafu::whatever!( + "the selected command requires workspace-local sibling sources, but no `{}` was found to provide them.\nResolve this by doing one of the following:\n1. run `build-eips init ` so the workspace config supplies the local sources\n2. pass `--remote-siblings` for remote sibling source overrides\n3. use `parity `, `--staging `, or `--production ` for remote clean environment behavior", + config::LOCAL_CONFIG_FILE + ); + } + + let sibling = sibling_override.unwrap_or(default_sibling); + + Ok(ExecutionSettings { + build_root, + staging, + allow_dirty, + sibling, + }) +} + +fn local_repo_url(path: &Path) -> Result { + Url::from_directory_path(path) + .ok() + .whatever_context("unable to convert local sibling repository path into a file URL") +} + +fn apply_sibling_sources( + repository_use: &mut git::RepositoryUse, + sibling_ids: &[String], + workspace_config: Option<&LoadedWorkspaceConfig>, + sibling: &SelectedSource, +) -> Result<(), Whatever> { + match sibling { + SelectedSource::Remote => Ok(()), + SelectedSource::WorkspaceLocal => { + if sibling_ids.is_empty() { + return Ok(()); + } + + let workspace_config = workspace_config.whatever_context( + "workspace-local sibling selection requires a workspace config", + )?; + let mut missing = Vec::new(); + let mut local_repositories = Vec::new(); + + for repo_id in sibling_ids { + let path = workspace_config.local_repo_path(repo_id); + if git::repository_available(&path) { + local_repositories.push((repo_id.clone(), local_repo_url(&path)?)); + } else { + missing.push(repo_id.clone()); + } + } + + if !missing.is_empty() { + snafu::whatever!( + "workspace-local sibling selection requires all declared sibling repos; missing or invalid sibling repo(s): {}", + format_sibling_ids(&missing) + ); + } + + for (repo_id, url) in local_repositories { + repository_use.other_repos.insert(repo_id, url); + } + + Ok(()) + } + } +} + +fn build_path( + root_path: &Path, + repository_use: &git::RepositoryUse, + workspace_config: Option<&LoadedWorkspaceConfig>, + build_root: Option<&Path>, +) -> PathBuf { + build_root + .map(Path::to_path_buf) + .or_else(|| { + workspace_config.map(|workspace_config| { + workspace_config.workspace_build_root(&repository_use.title) + }) + }) + .unwrap_or_else(|| root_path.join(BUILD_DIR)) +} + +fn resolve_base_url_override( + args: &Args, + workspace_config: Option<&LoadedWorkspaceConfig>, +) -> Result, Whatever> { + if let Some(base_url) = args.operation.base_url_cli_args().base_url { + return Ok(Some(base_url)); + } + + if explicit_environment_or_parity(args)?.is_some() { + return Ok(None); + } + + Ok(workspace_config.and_then(|config| config.site_settings().base_url.clone())) +} + +pub(crate) fn resolve_execution(args: &Args) -> Result { + let root_path = root(args)?; + let active_repo = ActiveRepoIdentity::load(&root_path)?; + let sibling_ids = active_repo.sibling_ids(); + let workspace_config = LoadedWorkspaceConfig::discover(&root_path) + .whatever_context("unable to load workspace config")?; + + if let Some(workspace_config) = workspace_config.as_ref() { + debug!( + "using workspace config `{}`", + workspace_config.config_path().to_string_lossy() + ); + } + + let settings = resolve_execution_settings(args, &sibling_ids, workspace_config.as_ref())?; + let mut repository_use = active_repo.repository_use(settings.staging)?; + apply_sibling_sources( + &mut repository_use, + &sibling_ids, + workspace_config.as_ref(), + &settings.sibling, + )?; + + let build_path = build_path( + &root_path, + &repository_use, + workspace_config.as_ref(), + settings.build_root.as_deref(), + ); + let source_materialization = if settings.allow_dirty { + info!( + "dirty mode is enabled; tracked working-tree changes from the active content repo will be materialized into the build input" + ); + git::SourceMaterialization::Dirty + } else { + git::SourceMaterialization::Clean + }; + let base_url_override = resolve_base_url_override(args, workspace_config.as_ref())?; + + Ok(ResolvedExecution { + root_path, + build_path, + repository_use, + source_materialization, + base_url_override, + staging: settings.staging, + }) +} + +#[cfg(test)] +mod tests { + use clap::Parser; + use tempfile::TempDir; + + use crate::{ + cli::Args, + config::{self, LoadedWorkspaceConfig}, + }; + + use super::{ + explicit_environment_or_parity, resolve_base_url_override, resolve_execution_settings, + validate_non_execution_command_flags, ExecutionSettings, SelectedSource, + }; + + fn parse_args(arguments: &[&str]) -> Args { + Args::try_parse_from(arguments).unwrap() + } + + fn load_workspace_config(contents: &str) -> LoadedWorkspaceConfig { + let workspace = TempDir::new().unwrap(); + let config_path = workspace.path().join(config::LOCAL_CONFIG_FILE); + std::fs::write(&config_path, contents).unwrap(); + LoadedWorkspaceConfig::from_path(&config_path).unwrap() + } + + fn settings_for( + arguments: &[&str], + sibling_ids: &[&str], + workspace_config: Option<&LoadedWorkspaceConfig>, + ) -> ExecutionSettings { + let args = parse_args(arguments); + let sibling_ids = sibling_ids + .iter() + .map(|sibling_id| (*sibling_id).to_owned()) + .collect::>(); + + resolve_execution_settings(&args, &sibling_ids, workspace_config).unwrap() + } + + fn assert_settings( + arguments: &[&str], + sibling_ids: &[&str], + workspace_config: Option<&LoadedWorkspaceConfig>, + expected: ExecutionSettings, + ) { + assert_eq!( + settings_for(arguments, sibling_ids, workspace_config), + expected + ); + } + + #[test] + fn explicit_env_or_parity_provenance_is_classified_separately_from_local_defaults() { + let cases: &[(&[&str], Option)] = &[ + (&["build-eips", "--staging", "build"], Some(true)), + (&["build-eips", "--production", "build"], Some(false)), + (&["build-eips", "parity", "build"], Some(true)), + (&["build-eips", "build"], None), + (&["build-eips", "serve"], None), + (&["build-eips", "check"], None), + ]; + + for (arguments, expected) in cases { + let args = parse_args(arguments); + assert_eq!(explicit_environment_or_parity(&args).unwrap(), *expected); + } + } + + #[test] + fn base_url_override_resolution_uses_cli_config_then_provenance() { + let workspace_config = load_workspace_config( + r#" +[site] +base_url = "http://localhost:4000" +"#, + ); + let none = parse_args(&["build-eips", "build"]); + assert!(resolve_base_url_override(&none, None).unwrap().is_none()); + + for arguments in [&["build-eips", "build"][..], &["build-eips", "serve"][..]] { + let args = parse_args(arguments); + assert_eq!( + resolve_base_url_override(&args, Some(&workspace_config)) + .unwrap() + .unwrap() + .as_str(), + "http://localhost:4000/" + ); + } + + let cli = parse_args(&["build-eips", "build", "--base-url", "http://localhost:5000"]); + assert_eq!( + resolve_base_url_override(&cli, Some(&workspace_config)) + .unwrap() + .unwrap() + .as_str(), + "http://localhost:5000/" + ); + + for arguments in [ + &["build-eips", "--staging", "build"][..], + &["build-eips", "--production", "build"][..], + &["build-eips", "parity", "build"][..], + &["build-eips", "parity", "serve"][..], + ] { + let args = parse_args(arguments); + assert!(resolve_base_url_override(&args, Some(&workspace_config)) + .unwrap() + .is_none()); + } + } + + #[test] + fn local_site_base_url_override_does_not_change_execution_settings() { + let workspace_config = load_workspace_config( + r#" +[site] +base_url = "http://localhost:4000" +"#, + ); + let args = parse_args(&["build-eips", "build"]); + let settings = resolve_execution_settings(&args, &[], Some(&workspace_config)).unwrap(); + + assert_eq!( + resolve_base_url_override(&args, Some(&workspace_config)) + .unwrap() + .unwrap() + .as_str(), + "http://localhost:4000/" + ); + assert_eq!( + settings, + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: true, + sibling: SelectedSource::WorkspaceLocal, + } + ); + } + + #[test] + fn plain_site_commands_are_local_first_dirty_staging() { + let workspace_config = load_workspace_config(""); + let expected = ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: true, + sibling: SelectedSource::WorkspaceLocal, + }; + + for arguments in [ + &["build-eips", "build"][..], + &["build-eips", "serve"][..], + &["build-eips", "check"][..], + ] { + assert_settings( + arguments, + &["ERCs"], + Some(&workspace_config), + expected.clone(), + ); + } + } + + #[test] + fn clean_plain_site_commands_keep_local_sources_but_disable_dirty_materialization() { + let workspace_config = load_workspace_config(""); + let expected = ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: false, + sibling: SelectedSource::WorkspaceLocal, + }; + + for arguments in [ + &["build-eips", "build", "--clean"][..], + &["build-eips", "serve", "--clean"][..], + &["build-eips", "check", "--clean"][..], + ] { + assert_settings( + arguments, + &["ERCs"], + Some(&workspace_config), + expected.clone(), + ); + } + } + + #[test] + fn explicit_environment_site_commands_are_remote_clean_for_proposals() { + for (arguments, expected_staging) in [ + (&["build-eips", "--staging", "build"][..], true), + (&["build-eips", "--staging", "serve"][..], true), + (&["build-eips", "--staging", "check"][..], true), + (&["build-eips", "--production", "build"][..], false), + (&["build-eips", "--production", "serve"][..], false), + (&["build-eips", "--production", "check"][..], false), + ] { + assert_settings( + arguments, + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: expected_staging, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + } + } + + #[test] + fn clean_environment_commands_are_accepted_as_redundant_remote_clean() { + for (arguments, expected_staging) in [ + (&["build-eips", "--staging", "build", "--clean"][..], true), + (&["build-eips", "--staging", "serve", "--clean"][..], true), + (&["build-eips", "--staging", "check", "--clean"][..], true), + ( + &["build-eips", "--production", "build", "--clean"][..], + false, + ), + ( + &["build-eips", "--production", "serve", "--clean"][..], + false, + ), + ( + &["build-eips", "--production", "check", "--clean"][..], + false, + ), + ] { + assert_settings( + arguments, + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: expected_staging, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + } + } + + #[test] + fn remote_source_overrides_compose_with_local_dirty_and_clean_modes() { + let workspace_config = load_workspace_config(""); + let cases = [ + ( + &["build-eips", "--remote-siblings", "build"][..], + true, + SelectedSource::Remote, + ), + ( + &["build-eips", "--remote-siblings", "build", "--clean"][..], + false, + SelectedSource::Remote, + ), + ]; + + for (arguments, allow_dirty, sibling) in cases { + assert_settings( + arguments, + &["ERCs"], + Some(&workspace_config), + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty, + sibling, + }, + ); + } + } + + #[test] + fn non_site_commands_do_not_require_workspace_local_sources() { + for arguments in [&["build-eips", "changed"][..], &["build-eips", "clean"][..]] { + assert_settings( + arguments, + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: false, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + } + } + + #[test] + fn changed_environment_flags_use_remote_clean_metadata_without_workspace_config() { + for (arguments, expected_staging) in [ + (&["build-eips", "--staging", "changed"][..], true), + (&["build-eips", "--production", "changed"][..], false), + ] { + assert_settings( + arguments, + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: expected_staging, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + } + } + + #[test] + fn parity_site_commands_remain_remote_clean_staging() { + for arguments in [ + &["build-eips", "parity", "build"][..], + &["build-eips", "parity", "serve"][..], + &["build-eips", "parity", "check"][..], + ] { + assert_settings( + arguments, + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + } + } + + #[test] + fn boolean_override_conflicts_are_hard_errors() { + let args = parse_args(&["build-eips", "--staging", "--production", "build"]); + let error = resolve_execution_settings(&args, &[], None).unwrap_err(); + + assert!(error + .to_string() + .contains("cannot pass both `--staging` and `--production`")); + } + + #[test] + fn non_execution_commands_reject_execution_override_flags() { + for arguments in [ + &["build-eips", "--staging", "init", "/tmp/workspace"][..], + &["build-eips", "--production", "init", "/tmp/workspace"][..], + &["build-eips", "--remote-siblings", "init", "/tmp/workspace"][..], + &[ + "build-eips", + "--build-root", + "/tmp/build", + "init", + "/tmp/workspace", + ][..], + &["build-eips", "--staging", "doctor"][..], + &["build-eips", "--production", "doctor"][..], + &["build-eips", "--remote-siblings", "doctor"][..], + &["build-eips", "--build-root", "/tmp/build", "doctor"][..], + &["build-eips", "--remote-siblings", "print", "schema-version"][..], + ] { + let args = parse_args(arguments); + let error = validate_non_execution_command_flags(&args).unwrap_err(); + let message = error.to_string(); + + assert!( + message.contains("execution override flags cannot be used"), + "unexpected error for {arguments:?}: {message}" + ); + } + } + + #[test] + fn local_first_commands_without_workspace_config_report_sibling_setup_error() { + for arguments in [ + &["build-eips", "build"][..], + &["build-eips", "serve"][..], + &["build-eips", "check"][..], + ] { + let args = parse_args(arguments); + let sibling_ids = vec!["ERCs".to_owned()]; + let error = resolve_execution_settings(&args, &sibling_ids, None).unwrap_err(); + let message = error.to_string(); + + assert!(message.contains("workspace-local sibling sources")); + assert!(message.contains("build-eips init ")); + assert!(message.contains("--remote-siblings")); + } + } + + #[test] + fn local_first_with_remote_sibling_override_is_not_parity() { + let local_args = parse_args(&["build-eips", "--remote-siblings", "build"]); + let sibling_ids = vec!["ERCs".to_owned()]; + let local_settings = resolve_execution_settings(&local_args, &sibling_ids, None).unwrap(); + + let parity_args = parse_args(&["build-eips", "parity", "build"]); + let parity_settings = resolve_execution_settings(&parity_args, &sibling_ids, None).unwrap(); + + assert_eq!( + local_settings, + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: true, + sibling: SelectedSource::Remote, + } + ); + assert_eq!( + parity_settings, + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: false, + sibling: SelectedSource::Remote, + } + ); + } + + #[test] + fn zero_sibling_remote_override_is_noop() { + let remote_args = parse_args(&["build-eips", "--remote-siblings", "parity", "build"]); + let remote_settings = resolve_execution_settings(&remote_args, &[], None).unwrap(); + + assert_eq!(remote_settings.sibling, SelectedSource::Remote); + } + + #[test] + fn zero_sibling_local_first_without_workspace_config_can_resolve_sibling_policy() { + let args = parse_args(&["build-eips", "build"]); + let settings = resolve_execution_settings(&args, &[], None).unwrap(); + + assert_eq!(settings.sibling, SelectedSource::WorkspaceLocal); + } +} diff --git a/src/lint.rs b/src/lint.rs index a433b37..6c45c15 100644 --- a/src/lint.rs +++ b/src/lint.rs @@ -92,7 +92,7 @@ struct Config { eipw: eipw_lint::config::DefaultOptions, } -#[derive(Debug, clap::Args, Serialize, Deserialize)] +#[derive(Debug, Clone, clap::Args, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct CmdArgs { /// Disable linting entirely diff --git a/src/main.rs b/src/main.rs index 8143637..f9740ff 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,6 +9,7 @@ mod changed; mod cli; mod config; mod context; +mod execution; mod find_root; mod git; mod github; @@ -29,23 +30,13 @@ use log::{debug, info}; use snafu::{Report, ResultExt, Whatever}; use crate::{ - cli::{Args, Operation}, + cli::{Args, Operation, RuntimeOperation}, config::Config, - layout::{BUILD_DIR, CONTENT_DIR, OUTPUT_DIR, REPO_DIR}, + execution::{resolve_execution, validate_non_execution_command_flags, ResolvedExecution}, + layout::{CONTENT_DIR, OUTPUT_DIR, REPO_DIR}, workspace::{doctor_workspace, init_workspace}, }; -fn repository_use(config: &Config, root_path: &Path) -> Result { - let repo_id = config - .locations - .identify_repository_title(root_path) - .whatever_context("cannot identify repository use")?; - let Some(repository_use) = config.locations.repository_use_for_title(&repo_id) else { - snafu::whatever!("repository metadata for `{repo_id}` is unavailable"); - }; - Ok(repository_use) -} - fn lock(build_path: &Path) -> Result { let lock_path = build_path.join(".lock"); let mut lock_file = @@ -62,23 +53,23 @@ fn lock(build_path: &Path) -> Result { Ok(lock_file) } -fn make_build_dir(root: &Path) -> Result { - let build_path = root.join(BUILD_DIR); - if let Err(e) = std::fs::create_dir_all(&build_path) { +fn make_build_dir(build_path: &Path) -> Result { + if let Err(e) = std::fs::create_dir_all(build_path) { debug!( "got while creating build directory: {}", Report::from_error(e) ); } - Ok(build_path) + Ok(build_path.to_path_buf()) } #[derive(Debug)] struct Prepared { cache: cache::Cache, - root_path: PathBuf, repo_path: PathBuf, output_path: PathBuf, + repository_use: git::RepositoryUse, + base_url_override: Option, config: Config, } @@ -86,21 +77,28 @@ impl Prepared { fn prepare( eipw: lint::CmdArgs, config: Config, - root_path: PathBuf, - build_path: PathBuf, + resolved: ResolvedExecution, ) -> Result { zola::find_zola().whatever_context("unable to find suitable zola binary")?; + let ResolvedExecution { + root_path, + build_path, + repository_use, + source_materialization, + base_url_override, + staging: _, + } = resolved; + let repo_path = build_path.join(REPO_DIR); let content_path = repo_path.join(CONTENT_DIR); let output_path = build_path.join(OUTPUT_DIR); - let repository_use = repository_use(&config, &root_path)?; let both = git::Fresh::new( &root_path, &repo_path, - repository_use, - git::SourceMaterialization::Clean, + repository_use.clone(), + source_materialization, ) .whatever_context("initializing build repo")? .clone_src() @@ -136,22 +134,26 @@ impl Prepared { Ok(Prepared { config, - root_path, cache, repo_path, output_path, + repository_use, + base_url_override, }) } fn build(self) -> Result<(), Whatever> { - let repository_use = repository_use(&self.config, &self.root_path)?; + let base_url = self + .base_url_override + .as_ref() + .unwrap_or(&self.repository_use.location.base_url); zola::build( self.config.theme.repository.as_str(), &self.config.theme.commit, &self.cache, &self.repo_path, &self.output_path, - repository_use.location.base_url.as_str(), + base_url.as_str(), ) .whatever_context("zola build failed")?; Ok(()) @@ -183,8 +185,10 @@ impl Prepared { fn run() -> Result<(), Whatever> { let args = Args::parse(); - if let Operation::Print { print } = args.operation { - print::print(print); + validate_non_execution_command_flags(&args)?; + + if let Operation::Print { print } = &args.operation { + print::print(print.clone()); return Ok(()); } @@ -198,22 +202,17 @@ fn run() -> Result<(), Whatever> { return Ok(()); } - let config = if args.staging { - Config::staging() - } else { - Config::production() - }; - - let root_path = context::root(&args)?; - let build_path = make_build_dir(&root_path)?; + let runtime_operation = args + .operation + .runtime_operation() + .expect("non-execution commands should have returned earlier"); + let resolved = resolve_execution(&args)?; + let build_path = make_build_dir(&resolved.build_path)?; let mut lock_file = lock(&build_path)?; - match args.operation { - Operation::Print { .. } => unreachable!(), - Operation::Init { .. } => unreachable!(), - Operation::Doctor => unreachable!(), - Operation::Clean => { + match runtime_operation { + RuntimeOperation::Clean => { // TODO: There's a race condition here. Maybe we move the lockfile to the repository // root? lock_file @@ -223,17 +222,32 @@ fn run() -> Result<(), Whatever> { .whatever_context("unable to remove build directory")?; return Ok(()); } - Operation::Check { eipw } => { - Prepared::prepare(eipw, config, root_path, build_path)?.check()?; + RuntimeOperation::Check { eipw } => { + let config = if resolved.staging { + Config::staging() + } else { + Config::production() + }; + Prepared::prepare(eipw, config, resolved)?.check()?; } - Operation::Build { eipw } => { - Prepared::prepare(eipw, config, root_path, build_path)?.build()?; + RuntimeOperation::Build { eipw } => { + let config = if resolved.staging { + Config::staging() + } else { + Config::production() + }; + Prepared::prepare(eipw, config, resolved)?.build()?; } - Operation::Serve { eipw } => { - Prepared::prepare(eipw, config, root_path, build_path)?.serve()?; + RuntimeOperation::Serve { eipw } => { + let config = if resolved.staging { + Config::staging() + } else { + Config::production() + }; + Prepared::prepare(eipw, config, resolved)?.serve()?; } - Operation::Changed { all, format } => { - changed::run(&root_path, &build_path, &config, all, &format)?; + RuntimeOperation::Changed { all, format } => { + changed::run(&resolved, &build_path, all, &format)?; } } From f915e5ee26a843687606007ee3d8f48be8b18f59 Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Sun, 3 May 2026 21:06:52 -0400 Subject: [PATCH 07/20] Run Zola with workspace theme Resolve a workspace-local theme for Zola runtime commands and remove the remote theme cache path. Mount the selected theme under repo/themes/eips-theme for Zola, load Zola config from the mounted theme, and load eipw config from the local theme checkout. Require workspace theme setup for build, check, serve, and parity commands while leaving the prepared runtime pipeline to the next PR. --- Cargo.lock | 38 +-------- Cargo.toml | 2 - src/cache.rs | 84 ------------------ src/execution.rs | 218 +++++++++++++++++++++++++++++++++++++++++++---- src/git.rs | 40 --------- src/layout.rs | 18 +++- src/lint.rs | 15 +--- src/main.rs | 78 ++++------------- src/zola.rs | 201 +++++++++++++++++++++++++++++++++---------- 9 files changed, 392 insertions(+), 302 deletions(-) delete mode 100644 src/cache.rs diff --git a/Cargo.lock b/Cargo.lock index aa9d0aa..715f3ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -237,7 +237,6 @@ dependencies = [ "chrono", "citationberg", "clap", - "directories", "duct", "eipw-lint", "eipw-preamble", @@ -258,7 +257,6 @@ dependencies = [ "semver", "serde", "serde_json", - "sha3", "snafu", "tempfile", "tokio", @@ -533,22 +531,13 @@ dependencies = [ "crypto-common", ] -[[package]] -name = "directories" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" -dependencies = [ - "dirs-sys 0.5.0", -] - [[package]] name = "dirs" version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" dependencies = [ - "dirs-sys 0.4.1", + "dirs-sys", ] [[package]] @@ -559,22 +548,10 @@ checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" dependencies = [ "libc", "option-ext", - "redox_users 0.4.6", + "redox_users", "windows-sys 0.48.0", ] -[[package]] -name = "dirs-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" -dependencies = [ - "libc", - "option-ext", - "redox_users 0.5.2", - "windows-sys 0.61.2", -] - [[package]] name = "displaydoc" version = "0.2.5" @@ -1934,17 +1911,6 @@ dependencies = [ "thiserror 1.0.69", ] -[[package]] -name = "redox_users" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" -dependencies = [ - "getrandom 0.2.17", - "libredox", - "thiserror 2.0.18", -] - [[package]] name = "ref-cast" version = "1.0.25" diff --git a/Cargo.toml b/Cargo.toml index 4529c21..2a48446 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,6 @@ depends = "$auto, libc6, libssl3, zlib1g, libgcc-s1, libgit2-1.5, git" [dependencies] chrono = "0.4.42" clap = { version = "4.5.53", features = ["cargo", "derive"] } -directories = "6.0.0" duct = "1.1.1" eipw-lint = { version = "0.10.0", features = [ "tokio", "schema-version" ] } eipw-snippets = "0.2.0" @@ -40,7 +39,6 @@ regex = "1.12.2" semver = {version = "1.0.27", features = ["serde"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.148" -sha3 = "0.10.8" snafu = { version = "0.8.9", features = ["rust_1_81"] } tokio = { version = "1.48.0", features = ["fs", "rt", "macros"] } toml = "0.9.10" diff --git a/src/cache.rs b/src/cache.rs deleted file mode 100644 index 9c7eb2d..0000000 --- a/src/cache.rs +++ /dev/null @@ -1,84 +0,0 @@ -/* - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. - */ - -use std::{io::ErrorKind, path::PathBuf, sync::Arc}; - -use directories::ProjectDirs; -use fslock::LockFile; -use log::{debug, info}; -use sha3::{Digest, Sha3_256}; -use snafu::{Backtrace, IntoError, OptionExt, Report, ResultExt, Snafu}; - -#[derive(Debug, Snafu)] -pub enum Error { - #[snafu(display("unable to discover application directories (unset $HOME?)"))] - Directories { backtrace: Backtrace }, - #[snafu(display("unable to access the path `{}`", path.to_string_lossy()))] - Fs { - path: PathBuf, - backtrace: Backtrace, - source: std::io::Error, - }, -} - -#[derive(Debug)] -struct Inner { - _lock: LockFile, - dir: PathBuf, -} - -#[derive(Debug, Clone)] -pub struct Cache(Arc); - -impl Cache { - pub fn open() -> Result { - debug!("opening local file cache"); - - let dirs = - ProjectDirs::from("org.ethereum", "eips", "eips-build").context(DirectoriesSnafu)?; - let cache_path = dirs.cache_dir(); - if let Err(e) = std::fs::create_dir_all(cache_path) { - debug!( - "got while creating cache directory: {}", - Report::from_error(e) - ); - } - - let lock_path = cache_path.join(".lock"); - let mut lock = LockFile::open(&lock_path).context(FsSnafu { path: &lock_path })?; - - let locked = lock - .try_lock_with_pid() - .context(FsSnafu { path: &lock_path })?; - - if !locked { - info!("waiting on cache directory..."); - lock.lock_with_pid().context(FsSnafu { path: &lock_path })?; - } - - Ok(Self(Arc::new(Inner { - _lock: lock, - dir: cache_path.into(), - }))) - } - - pub fn dir(&self, key: &str) -> Result { - let mut hasher = Sha3_256::new(); - hasher.update(key.as_bytes()); - let hash = hasher.finalize(); - let hash_text = format!("{:x}", hash); - let path = self.0.dir.join(hash_text); - - debug!("creating cache directory `{}`", path.to_string_lossy()); - match std::fs::create_dir(&path) { - Ok(()) => (), - Err(e) if e.kind() == ErrorKind::AlreadyExists => (), - Err(e) => return Err(FsSnafu { path }.into_error(e)), - } - - Ok(path) - } -} diff --git a/src/execution.rs b/src/execution.rs index 8809e9a..ea558ab 100644 --- a/src/execution.rs +++ b/src/execution.rs @@ -6,7 +6,10 @@ //! Execution source and path resolution. -use std::path::{Path, PathBuf}; +use std::{ + io::ErrorKind, + path::{Path, PathBuf}, +}; use log::{debug, info}; use snafu::{OptionExt, ResultExt, Whatever}; @@ -26,9 +29,17 @@ pub(crate) struct ResolvedExecution { pub(crate) root_path: PathBuf, pub(crate) build_path: PathBuf, pub(crate) repository_use: git::RepositoryUse, + pub(crate) theme_path: Option, pub(crate) source_materialization: git::SourceMaterialization, pub(crate) base_url_override: Option, - pub(crate) staging: bool, +} + +impl ResolvedExecution { + pub(crate) fn theme_path(&self) -> Result<&Path, Whatever> { + self.theme_path + .as_deref() + .whatever_context("the selected command requires a resolved workspace-local theme") + } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -123,15 +134,26 @@ pub(crate) fn resolve_execution_settings( (false, false, SelectedSource::Remote) }; - if sibling_override.is_none() + let missing_theme = operation_requires_theme(&args.operation) && workspace_config.is_none(); + let missing_sibling = sibling_override.is_none() && default_sibling == SelectedSource::WorkspaceLocal && !sibling_ids.is_empty() - && workspace_config.is_none() - { - snafu::whatever!( - "the selected command requires workspace-local sibling sources, but no `{}` was found to provide them.\nResolve this by doing one of the following:\n1. run `build-eips init ` so the workspace config supplies the local sources\n2. pass `--remote-siblings` for remote sibling source overrides\n3. use `parity `, `--staging `, or `--production ` for remote clean environment behavior", - config::LOCAL_CONFIG_FILE - ); + && workspace_config.is_none(); + + match (missing_theme, missing_sibling) { + (true, true) => { + snafu::whatever!( + "the selected command requires a workspace config with local theme and sibling sources, but no `{}` was found.\n\nRun:\n build-eips init \n\nThen retry from that workspace, or pass `--remote-siblings` if you intentionally want remote sibling proposal sources.", + config::LOCAL_CONFIG_FILE + ); + } + (false, true) => { + snafu::whatever!( + "the selected command requires workspace-local sibling sources, but no `{}` was found to provide them.\nResolve this by doing one of the following:\n1. run `build-eips init ` so the workspace config supplies the local sources\n2. pass `--remote-siblings` for remote sibling source overrides\n3. use `parity `, `--staging `, or `--production ` for remote clean environment behavior", + config::LOCAL_CONFIG_FILE + ); + } + _ => {} } let sibling = sibling_override.unwrap_or(default_sibling); @@ -210,6 +232,49 @@ fn build_path( .unwrap_or_else(|| root_path.join(BUILD_DIR)) } +fn operation_requires_theme(operation: &Operation) -> bool { + matches!( + operation, + Operation::Build { .. } + | Operation::Serve { .. } + | Operation::Check { .. } + | Operation::Parity { .. } + ) +} + +fn resolve_theme_path( + workspace_config: Option<&LoadedWorkspaceConfig>, + operation: &Operation, +) -> Result, Whatever> { + if !operation_requires_theme(operation) { + return Ok(None); + } + + let workspace_config = workspace_config.with_whatever_context(|| { + format!( + "the selected command requires a workspace config with a local theme, but no `{}` was found.\n\nRun:\n build-eips init \n\nThen retry from that workspace.", + config::LOCAL_CONFIG_FILE + ) + })?; + let theme_path = workspace_config.local_theme_path(); + + match std::fs::metadata(&theme_path) { + Ok(_) => Ok(Some(theme_path)), + Err(error) if matches!(error.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => { + snafu::whatever!( + "workspace-local theme path `{}` does not exist.\n\nRun `build-eips init ` to bootstrap the workspace, or\nclone/update the theme repository at the configured path.", + theme_path.to_string_lossy() + ); + } + Err(error) => { + snafu::whatever!( + "unable to access workspace-local theme path `{}`: {error}", + theme_path.to_string_lossy() + ); + } + } +} + fn resolve_base_url_override( args: &Args, workspace_config: Option<&LoadedWorkspaceConfig>, @@ -240,6 +305,7 @@ pub(crate) fn resolve_execution(args: &Args) -> Result Result")); + assert!(!message.contains("theme and sibling")); + assert!(!message.contains(concat!("--remote", "-theme"))); + } + + fn assert_combined_missing_workspace_error(arguments: &[&str]) { + let args = parse_args(arguments); + let sibling_ids = vec!["ERCs".to_owned()]; + let error = resolve_execution_settings(&args, &sibling_ids, None).unwrap_err(); + let message = error.to_string(); + + assert!(message + .contains("the selected command requires a workspace config with local theme and sibling sources")); + assert!(message.contains("no `.build-eips.toml` was found")); + assert!(message.contains("build-eips init ")); + assert!(message.contains( + "pass `--remote-siblings` if you intentionally want remote sibling proposal sources" + )); + assert!(!message.contains(concat!("--remote", "-theme"))); + assert!(!message.contains("--profile")); + assert!(!message.contains("--allow-dirty")); + assert!(!message.contains("--theme ")); + assert!(!message.contains("--sibling-repo ")); + } + #[test] fn explicit_env_or_parity_provenance_is_classified_separately_from_local_defaults() { let cases: &[(&[&str], Option)] = &[ @@ -651,20 +750,54 @@ base_url = "http://localhost:4000" } #[test] - fn local_first_commands_without_workspace_config_report_sibling_setup_error() { + fn zola_runtime_commands_require_workspace_local_theme() { + let workspace = TempDir::new().unwrap(); + let config_path = workspace.path().join(config::LOCAL_CONFIG_FILE); + std::fs::write(&config_path, "").unwrap(); + std::fs::create_dir(workspace.path().join(config::DEFAULT_THEME_DIR)).unwrap(); + let workspace_config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + for arguments in [ &["build-eips", "build"][..], - &["build-eips", "serve"][..], &["build-eips", "check"][..], + &["build-eips", "serve"][..], + &["build-eips", "--staging", "build"][..], + &["build-eips", "--production", "check"][..], + &["build-eips", "parity", "build"][..], ] { let args = parse_args(arguments); - let sibling_ids = vec!["ERCs".to_owned()]; - let error = resolve_execution_settings(&args, &sibling_ids, None).unwrap_err(); - let message = error.to_string(); + let theme_path = super::resolve_theme_path(Some(&workspace_config), &args.operation) + .unwrap() + .unwrap(); + + assert_eq!(theme_path, workspace.path().join(config::DEFAULT_THEME_DIR)); + } + } + + #[test] + fn non_theme_commands_do_not_require_workspace_local_theme() { + for arguments in [ + &["build-eips", "changed"][..], + &["build-eips", "clean"][..], + &["build-eips", "doctor"][..], + &["build-eips", "print", "schema-version"][..], + ] { + let args = parse_args(arguments); + + assert!(super::resolve_theme_path(None, &args.operation) + .unwrap() + .is_none()); + } + } - assert!(message.contains("workspace-local sibling sources")); - assert!(message.contains("build-eips init ")); - assert!(message.contains("--remote-siblings")); + #[test] + fn local_first_theme_commands_without_workspace_config_report_combined_setup_error() { + for arguments in [ + &["build-eips", "build"][..], + &["build-eips", "serve"][..], + &["build-eips", "check"][..], + ] { + assert_combined_missing_workspace_error(arguments); } } @@ -711,5 +844,54 @@ base_url = "http://localhost:4000" let settings = resolve_execution_settings(&args, &[], None).unwrap(); assert_eq!(settings.sibling, SelectedSource::WorkspaceLocal); + assert_theme_only_missing_workspace_error(&["build-eips", "build"]); + } + + #[test] + fn remote_sibling_override_without_workspace_config_only_requires_theme_resolution() { + let args = parse_args(&["build-eips", "--remote-siblings", "build"]); + let sibling_ids = vec!["ERCs".to_owned()]; + let settings = resolve_execution_settings(&args, &sibling_ids, None).unwrap(); + + assert_eq!(settings.sibling, SelectedSource::Remote); + assert_theme_only_missing_workspace_error(&["build-eips", "--remote-siblings", "build"]); + } + + #[test] + fn environment_and_parity_zola_commands_without_workspace_config_only_require_theme() { + for arguments in [ + &["build-eips", "--staging", "build"][..], + &["build-eips", "--production", "serve"][..], + &["build-eips", "parity", "check"][..], + ] { + let args = parse_args(arguments); + let sibling_ids = vec!["ERCs".to_owned()]; + let settings = resolve_execution_settings(&args, &sibling_ids, None).unwrap(); + + assert_eq!(settings.sibling, SelectedSource::Remote); + assert_theme_only_missing_workspace_error(arguments); + } + } + + #[test] + fn missing_workspace_theme_path_reports_clear_error() { + let workspace = TempDir::new().unwrap(); + let config_path = workspace.path().join(config::LOCAL_CONFIG_FILE); + std::fs::write(&config_path, "").unwrap(); + let workspace_config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + let args = parse_args(&["build-eips", "build"]); + + let error = + super::resolve_theme_path(Some(&workspace_config), &args.operation).unwrap_err(); + let message = error.to_string(); + + assert!(message.contains(&format!( + "workspace-local theme path `{}` does not exist", + workspace + .path() + .join(config::DEFAULT_THEME_DIR) + .to_string_lossy() + ))); + assert!(message.contains("build-eips init ")); } } diff --git a/src/git.rs b/src/git.rs index 1d35379..c77c3a5 100644 --- a/src/git.rs +++ b/src/git.rs @@ -11,7 +11,6 @@ use std::{ }; use crate::{ - cache::Cache, config::{LegacyLocations, RepositoryEndpoint}, layout::{BUILD_DIR, CONTENT_DIR}, progress::{Git, ProgressIteratorExt}, @@ -67,11 +66,6 @@ pub enum Error { DirtyUnsupportedPath { path: PathBuf, backtrace: Backtrace }, #[snafu(display("unable to update tree ({msg})"))] UpdateTree { msg: String, backtrace: Backtrace }, - #[snafu(context(false))] - Cache { - #[snafu(backtrace)] - source: crate::cache::Error, - }, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1147,40 +1141,6 @@ fn open_or_init(dir: &Path) -> Result { Ok(repo) } -impl Cache { - pub fn repo(&self, url: &str, commit: &str) -> Result { - let key = format!("git\0{url}"); - let dir = self.dir(&key)?; - - let repo = open_or_init(&dir)?; - let object = match repo.revparse_single(commit) { - Ok(c) => c, - Err(e) if e.code() == git2::ErrorCode::NotFound => { - fetch(&repo, url, "master")?; - repo.revparse_single(commit).context(GitSnafu { - what: "revparse cached commit", - })? - } - Err(e) => { - return Err(GitSnafu { - what: "revparse cached commit", - } - .into_error(e)) - } - }; - - repo.checkout_tree(&object, Some(CheckoutBuilder::new().force())) - .context(GitSnafu { - what: "checkout cached commit", - })?; - repo.set_head_detached(object.id()).context(GitSnafu { - what: "set detached head", - })?; - - Ok(dir) - } -} - #[cfg(test)] mod tests { use std::path::{Path, PathBuf}; diff --git a/src/layout.rs b/src/layout.rs index cc2faa6..3725a1f 100644 --- a/src/layout.rs +++ b/src/layout.rs @@ -4,7 +4,23 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +use std::path::{Path, PathBuf}; + pub(crate) const CONTENT_DIR: &str = "content"; pub(crate) const BUILD_DIR: &str = "build"; pub(crate) const REPO_DIR: &str = "repo"; -pub(crate) const OUTPUT_DIR: &str = "output"; +const OUTPUT_DIR: &str = "output"; + +pub(crate) fn output_path(build_path: &Path) -> PathBuf { + build_path.join(OUTPUT_DIR) +} + +pub(crate) fn mounted_theme_path(project_path: &Path) -> PathBuf { + project_path.join("themes").join("eips-theme") +} + +pub(crate) fn theme_config_path(theme_path: &Path) -> PathBuf { + [theme_path, Path::new("config"), Path::new("zola.toml")] + .iter() + .collect() +} diff --git a/src/lint.rs b/src/lint.rs index 6c45c15..9fa9e25 100644 --- a/src/lint.rs +++ b/src/lint.rs @@ -11,7 +11,6 @@ use clap::ValueEnum; use log::debug; use semver::{Comparator, Op, VersionReq}; -use crate::cache::Cache; use crate::progress::ProgressIteratorExt; use eipw_lint::reporters::{AdditionalHelp, Count, Json, Reporter, Text}; @@ -51,11 +50,6 @@ pub enum Error { source: std::io::Error, }, #[snafu(transparent)] - Git { - #[snafu(backtrace)] - source: crate::git::Error, - }, - #[snafu(transparent)] SchemaVersion { #[snafu(backtrace)] source: SchemaVersionError, @@ -257,9 +251,7 @@ fn version_cmp( #[tokio::main(flavor = "current_thread")] pub async fn eipw( - theme_repo: &str, - theme_rev: &str, - cache: &Cache, + theme_path: &Path, root_dir: &Path, repo_dir: &Path, changed_paths: Vec, @@ -271,10 +263,7 @@ pub async fn eipw( let mut stdout = std::io::stdout(); - let mut config_path = cache.repo(theme_repo, theme_rev)?; - - config_path.push("config"); - config_path.push("eipw.toml"); + let config_path = theme_path.join("config").join("eipw.toml"); let toml_file = Toml::file_exact(&config_path); diff --git a/src/main.rs b/src/main.rs index f9740ff..3a2e3b8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,6 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -mod cache; mod changed; mod cli; mod config; @@ -31,9 +30,8 @@ use snafu::{Report, ResultExt, Whatever}; use crate::{ cli::{Args, Operation, RuntimeOperation}, - config::Config, execution::{resolve_execution, validate_non_execution_command_flags, ResolvedExecution}, - layout::{CONTENT_DIR, OUTPUT_DIR, REPO_DIR}, + layout::{output_path, CONTENT_DIR, REPO_DIR}, workspace::{doctor_workspace, init_workspace}, }; @@ -65,34 +63,30 @@ fn make_build_dir(build_path: &Path) -> Result { #[derive(Debug)] struct Prepared { - cache: cache::Cache, repo_path: PathBuf, output_path: PathBuf, repository_use: git::RepositoryUse, + theme_path: PathBuf, base_url_override: Option, - config: Config, } impl Prepared { - fn prepare( - eipw: lint::CmdArgs, - config: Config, - resolved: ResolvedExecution, - ) -> Result { + fn prepare(eipw: lint::CmdArgs, resolved: ResolvedExecution) -> Result { zola::find_zola().whatever_context("unable to find suitable zola binary")?; + let theme_path = resolved.theme_path()?.to_path_buf(); let ResolvedExecution { root_path, build_path, repository_use, + theme_path: _, source_materialization, base_url_override, - staging: _, } = resolved; let repo_path = build_path.join(REPO_DIR); let content_path = repo_path.join(CONTENT_DIR); - let output_path = build_path.join(OUTPUT_DIR); + let output_path = output_path(&build_path); let both = git::Fresh::new( &root_path, @@ -117,27 +111,16 @@ impl Prepared { both.merge() .whatever_context("unable to merge ERC/EIP repositories")?; - let cache = cache::Cache::open().whatever_context("unable to open cache")?; - - lint::eipw( - config.theme.repository.as_str(), - &config.theme.commit, - &cache, - &root_path, - &repo_path, - changed_files, - eipw, - ) - .whatever_context("linting failed")?; + lint::eipw(&theme_path, &root_path, &repo_path, changed_files, eipw) + .whatever_context("linting failed")?; markdown::preprocess(&content_path).whatever_context("unable to preprocess markdown")?; Ok(Prepared { - config, - cache, repo_path, output_path, repository_use, + theme_path, base_url_override, }) } @@ -148,9 +131,7 @@ impl Prepared { .as_ref() .unwrap_or(&self.repository_use.location.base_url); zola::build( - self.config.theme.repository.as_str(), - &self.config.theme.commit, - &self.cache, + &self.theme_path, &self.repo_path, &self.output_path, base_url.as_str(), @@ -160,25 +141,13 @@ impl Prepared { } fn serve(self) -> Result<(), Whatever> { - zola::serve( - self.config.theme.repository.as_str(), - &self.config.theme.commit, - &self.cache, - &self.repo_path, - &self.output_path, - ) - .whatever_context("zola serve failed")?; + zola::serve(&self.theme_path, &self.repo_path, &self.output_path) + .whatever_context("zola serve failed")?; Ok(()) } fn check(self) -> Result<(), Whatever> { - zola::check( - self.config.theme.repository.as_str(), - &self.config.theme.commit, - &self.cache, - &self.repo_path, - ) - .whatever_context("zola check failed")?; + zola::check(&self.theme_path, &self.repo_path).whatever_context("zola check failed")?; Ok(()) } } @@ -223,28 +192,13 @@ fn run() -> Result<(), Whatever> { return Ok(()); } RuntimeOperation::Check { eipw } => { - let config = if resolved.staging { - Config::staging() - } else { - Config::production() - }; - Prepared::prepare(eipw, config, resolved)?.check()?; + Prepared::prepare(eipw, resolved)?.check()?; } RuntimeOperation::Build { eipw } => { - let config = if resolved.staging { - Config::staging() - } else { - Config::production() - }; - Prepared::prepare(eipw, config, resolved)?.build()?; + Prepared::prepare(eipw, resolved)?.build()?; } RuntimeOperation::Serve { eipw } => { - let config = if resolved.staging { - Config::staging() - } else { - Config::production() - }; - Prepared::prepare(eipw, config, resolved)?.serve()?; + Prepared::prepare(eipw, resolved)?.serve()?; } RuntimeOperation::Changed { all, format } => { changed::run(&resolved, &build_path, all, &format)?; diff --git a/src/zola.rs b/src/zola.rs index fc65638..82c3503 100644 --- a/src/zola.rs +++ b/src/zola.rs @@ -15,7 +15,7 @@ use semver::Version; use snafu::{ensure, Backtrace, IntoError, Report, ResultExt, Snafu}; use url::Url; -use crate::{cache::Cache, git}; +use crate::layout::{mounted_theme_path, theme_config_path}; const MINIMUM_VERSION: Version = Version::new(0, 22, 1); @@ -38,8 +38,11 @@ fn symlink_dir(original: &Path, link: &Path) -> Result<(), std::io::Error> { } fn force_symlink_dir(original: &Path, link: &Path) -> Result<(), std::io::Error> { - match std::fs::remove_file(link) { - Ok(()) => (), + match std::fs::symlink_metadata(link) { + Ok(metadata) if metadata.file_type().is_dir() && !metadata.file_type().is_symlink() => { + std::fs::remove_dir_all(link)?; + } + Ok(_) => std::fs::remove_file(link)?, Err(e) if e.kind() == ErrorKind::NotFound => (), Err(e) => return Err(e), } @@ -47,6 +50,19 @@ fn force_symlink_dir(original: &Path, link: &Path) -> Result<(), std::io::Error> symlink_dir(original, link) } +fn mount_theme(theme_dir: &Path, project_path: &Path) -> Result { + let mounted_theme_path = mounted_theme_path(project_path); + if theme_dir == mounted_theme_path { + return Ok(mounted_theme_path); + } + + if let Some(parent) = mounted_theme_path.parent() { + std::fs::create_dir_all(parent)?; + } + force_symlink_dir(theme_dir, &mounted_theme_path)?; + Ok(mounted_theme_path) +} + #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("could not find zola binary (requires at least version {MINIMUM_VERSION})"))] @@ -71,11 +87,6 @@ pub enum Error { backtrace: Backtrace, source: std::io::Error, }, - #[snafu(context(false))] - Git { - #[snafu(backtrace)] - source: git::Error, - }, } pub fn find_zola() -> Result<(), Error> { @@ -96,21 +107,14 @@ pub fn find_zola() -> Result<(), Error> { Ok(()) } -pub fn check( - theme_repo: &str, - theme_rev: &str, - cache: &Cache, - project_path: &Path, -) -> Result<(), Error> { +pub fn check(theme_dir: &Path, project_path: &Path) -> Result<(), Error> { let args = ["check", "--drafts", "--skip-external-links"]; - spawn_log(theme_repo, theme_rev, cache, project_path, args)?; + spawn_log(theme_dir, project_path, args)?; Ok(()) } pub fn build( - theme_repo: &str, - theme_rev: &str, - cache: &Cache, + theme_dir: &Path, project_path: &Path, output_path: &Path, base_url: &str, @@ -120,20 +124,14 @@ pub fn build( .map(OsString::from) .into_iter() .chain(std::iter::once(output_path.into())); - spawn_log(theme_repo, theme_rev, cache, project_path, args)?; + spawn_log(theme_dir, project_path, args)?; if let Ok(url) = Url::from_file_path(output_path) { info!("HTML output to: {}", url); } Ok(()) } -pub fn serve( - theme_repo: &str, - theme_rev: &str, - cache: &Cache, - project_path: &Path, - output_path: &Path, -) -> Result<(), Error> { +pub fn serve(theme_dir: &Path, project_path: &Path, output_path: &Path) -> Result<(), Error> { // TODO: Properly kill the child process when we receive ctrl-c. warn!("live reloading is not implemented"); remove_output(output_path); @@ -141,7 +139,7 @@ pub fn serve( .map(OsString::from) .into_iter() .chain(std::iter::once(output_path.into())); - spawn_log(theme_repo, theme_rev, cache, project_path, args)?; + spawn_log(theme_dir, project_path, args)?; Ok(()) } @@ -154,13 +152,7 @@ fn remove_output(output_path: &Path) { } } -fn spawn_log( - theme_repo: &str, - theme_rev: &str, - cache: &Cache, - project_path: &Path, - args: U, -) -> Result<(), Error> +fn spawn_log(theme_dir: &Path, project_path: &Path, args: U) -> Result<(), Error> where U: IntoIterator, I: Into, @@ -173,18 +165,9 @@ where find_zola()?; - let theme_dir = cache.repo(theme_repo, theme_rev)?; - - let mut themes_dir = project_path.join("themes"); - if let Err(e) = std::fs::create_dir(&themes_dir) { - debug!("got while creating themes dir: {}", Report::from_error(e)); - } - themes_dir.push("eips-theme"); - force_symlink_dir(&theme_dir, &themes_dir).context(FsSnafu { path: &themes_dir })?; - - let config_path: PathBuf = [&theme_dir, Path::new("config"), Path::new("zola.toml")] - .iter() - .collect(); + let mounted_theme_path = + mount_theme(theme_dir, project_path).context(FsSnafu { path: theme_dir })?; + let config_path = theme_config_path(&mounted_theme_path); let prefix = [OsString::from("-c"), config_path.into()].into_iter(); let args = prefix.chain(args.into_iter().map(Into::into)); @@ -221,3 +204,129 @@ where Ok(()) } + +#[cfg(test)] +mod tests { + use std::{ + fs, + path::{Path, PathBuf}, + process::{Command, ExitStatus}, + }; + + use tempfile::TempDir; + + use crate::layout::{mounted_theme_path, theme_config_path}; + + use super::{find_zola, mount_theme}; + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).unwrap(); + } + fs::write(path, contents).unwrap(); + } + + fn zola_build_status(project_root: &Path) -> ExitStatus { + Command::new("zola") + .arg("build") + .arg("--drafts") + .current_dir(project_root) + .status() + .unwrap() + } + + #[test] + fn zola_rejects_missing_internal_links_but_accepts_external_links() { + if find_zola().is_err() { + eprintln!("skipping zola link behavior fixture because zola is not installed"); + return; + } + + let temp = TempDir::new().unwrap(); + let internal = temp.path().join("internal"); + write_file( + &internal, + "config.toml", + "base_url = \"https://example.test\"\n", + ); + write_file( + &internal, + "content/_index.md", + "+++\ntitle = \"Internal\"\n+++\n[Missing](@/missing.md)\n", + ); + let external = temp.path().join("external"); + write_file( + &external, + "config.toml", + "base_url = \"https://example.test\"\n", + ); + write_file( + &external, + "content/_index.md", + "+++\ntitle = \"External\"\n+++\n[External](https://eips.ethereum.org/EIPS/eip-1)\n", + ); + + assert!(!zola_build_status(&internal).success()); + assert!(zola_build_status(&external).success()); + } + + #[test] + fn mounted_theme_paths_are_under_project_themes_directory() { + let project_path = PathBuf::from("/tmp/project"); + let mounted_theme = mounted_theme_path(&project_path); + + assert_eq!( + mounted_theme, + PathBuf::from("/tmp/project/themes/eips-theme") + ); + assert_eq!( + theme_config_path(&mounted_theme), + PathBuf::from("/tmp/project/themes/eips-theme/config/zola.toml") + ); + } + + #[test] + fn mount_theme_does_not_symlink_mounted_local_theme_onto_itself() { + let temp = TempDir::new().unwrap(); + let project_path = temp.path().join("repo"); + let mounted_theme = mounted_theme_path(&project_path); + fs::create_dir_all(mounted_theme.join("config")).unwrap(); + fs::write(mounted_theme.join("config/zola.toml"), "title = 'local'\n").unwrap(); + + let result = mount_theme(&mounted_theme, &project_path).unwrap(); + + assert_eq!(result, mounted_theme); + assert!(mounted_theme.join("config/zola.toml").is_file()); + assert!(!fs::symlink_metadata(&mounted_theme) + .unwrap() + .file_type() + .is_symlink()); + } + + #[cfg(target_family = "unix")] + #[test] + fn theme_mount_replaces_prior_real_mounted_theme_directory() { + let temp = TempDir::new().unwrap(); + let project_path = temp.path().join("repo"); + let source_theme = temp.path().join("source-theme"); + fs::create_dir_all(source_theme.join("config")).unwrap(); + fs::write(source_theme.join("config/zola.toml"), "title = 'source'\n").unwrap(); + + let mounted_theme = mounted_theme_path(&project_path); + fs::create_dir_all(&mounted_theme).unwrap(); + fs::write(mounted_theme.join("stale.txt"), "stale").unwrap(); + + let result = mount_theme(&source_theme, &project_path).unwrap(); + + assert_eq!(result, mounted_theme); + assert!(fs::symlink_metadata(&mounted_theme) + .unwrap() + .file_type() + .is_symlink()); + assert_eq!( + fs::read_to_string(theme_config_path(&mounted_theme)).unwrap(), + "title = 'source'\n" + ); + } +} From 8c31bb84a883b8f316727d50b27bde9c16d0372b Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Sun, 3 May 2026 21:11:13 -0400 Subject: [PATCH 08/20] Add prepared runtime build pipeline Move the Prepared runtime pipeline out of main.rs into pipeline.rs and keep main.rs focused on dispatching resolved runtime operations. Prepare runtime inputs from ResolvedExecution by cloning and fetching sources, force-refreshing prepared git scratch refs, merging sibling proposal content while keeping the active homepage, running eipw lint, preprocessing markdown, and materializing the local theme for Zola. Keep the existing minimal Prepared::serve method with the type, while leaving serve watcher and sync behavior to the serve runtime PR. --- src/execution.rs | 5 +- src/find_root.rs | 2 +- src/git.rs | 75 ++++++++++++++++- src/main.rs | 96 +-------------------- src/pipeline.rs | 211 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 290 insertions(+), 99 deletions(-) create mode 100644 src/pipeline.rs diff --git a/src/execution.rs b/src/execution.rs index ea558ab..c634e3c 100644 --- a/src/execution.rs +++ b/src/execution.rs @@ -411,8 +411,9 @@ mod tests { let error = resolve_execution_settings(&args, &sibling_ids, None).unwrap_err(); let message = error.to_string(); - assert!(message - .contains("the selected command requires a workspace config with local theme and sibling sources")); + assert!(message.contains( + "the selected command requires a workspace config with local theme and sibling sources" + )); assert!(message.contains("no `.build-eips.toml` was found")); assert!(message.contains("build-eips init ")); assert!(message.contains( diff --git a/src/find_root.rs b/src/find_root.rs index c2db1e0..77bb66d 100644 --- a/src/find_root.rs +++ b/src/find_root.rs @@ -4,7 +4,7 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -use super::CONTENT_DIR; +use crate::layout::CONTENT_DIR; use snafu::{ResultExt, Snafu}; use std::{ backtrace::Backtrace, diff --git a/src/git.rs b/src/git.rs index c77c3a5..8ef5346 100644 --- a/src/git.rs +++ b/src/git.rs @@ -25,6 +25,7 @@ use snafu::{ensure, Backtrace, IntoError, OptionExt, ResultExt, Snafu}; use url::Url; const DIRTY_PATH_DISPLAY_LIMIT: usize = 10; +const CONTENT_INDEX_PATH: &str = "content/_index.md"; #[derive(Debug, Snafu)] pub enum Error { @@ -757,7 +758,7 @@ impl Fresh { let master = fetch( &self.working_repo, self.src_repo_url.as_str(), - "HEAD:refs/build-eips/source-head", + "+HEAD:refs/build-eips/source-head", )?; self.working_repo .set_head_detached(master.id()) @@ -817,7 +818,7 @@ impl SourceOnly { let latest_master = fetch( &self.working_repo, self.src_repo_use.location.repository.as_str(), - "master:refs/build-eips/upstream-head", + "+master:refs/build-eips/upstream-head", )?; let upstream_head = latest_master.id(); drop(latest_master); @@ -1006,6 +1007,11 @@ impl SourceWithUpstream { } } + if path == CONTENT_INDEX_PATH { + debug!("skip sibling homepage `{path}`"); + return TreeWalkResult::Ok; + } + if let Err(e) = check_conflict(&local_tree, Path::new(&path), b) { walk_error = Some(e); return TreeWalkResult::Abort; @@ -1148,7 +1154,15 @@ mod tests { use git2::{IndexAddOption, Repository, Signature}; use tempfile::TempDir; - use super::{materialize_working_tree, sync_working_tree_paths, tracked_working_tree_paths}; + use super::{ + materialize_working_tree, sync_working_tree_paths, tracked_working_tree_paths, Fresh, + RepositoryUse, SourceMaterialization, + }; + use crate::config::RepositoryEndpoint; + + fn file_url(path: &Path) -> url::Url { + url::Url::from_directory_path(path).unwrap() + } fn write_file(root: &Path, relative: impl AsRef, contents: &str) { let path = root.join(relative); @@ -1204,6 +1218,61 @@ mod tests { index.write().unwrap(); } + #[test] + fn merge_skips_sibling_homepage_and_keeps_sibling_proposals() { + let temp = TempDir::new().unwrap(); + let active = temp.path().join("active"); + let sibling = temp.path().join("sibling"); + let prepared = temp.path().join("prepared"); + + init_repo( + &active, + &[ + ("content/_index.md", "active homepage\n"), + ("content/00555.md", "# Active proposal\n"), + ], + ); + init_repo( + &sibling, + &[ + ("content/_index.md", "sibling homepage\n"), + ("content/00678.md", "# Sibling proposal\n"), + ], + ); + + let mut other_repos = std::collections::BTreeMap::new(); + other_repos.insert("ERCs".to_owned(), file_url(&sibling)); + let active_url = file_url(&active); + let repository_use = RepositoryUse { + title: "EIPs".to_owned(), + location: RepositoryEndpoint { + repository: active_url, + base_url: "https://eips.example.test/".parse().unwrap(), + }, + other_repos, + }; + + Fresh::new( + &active, + &prepared, + repository_use, + SourceMaterialization::Clean, + ) + .unwrap() + .clone_src() + .unwrap() + .fetch_upstream() + .unwrap() + .merge() + .unwrap(); + + assert_eq!( + std::fs::read_to_string(prepared.join("content/_index.md")).unwrap(), + "active homepage\n" + ); + assert!(prepared.join("content/00678.md").is_file()); + } + #[test] fn materialize_working_tree_uses_tracked_theme_scope() { let temp = TempDir::new().unwrap(); diff --git a/src/main.rs b/src/main.rs index 3a2e3b8..5880c8c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,6 +16,7 @@ mod identity; mod layout; mod lint; mod markdown; +mod pipeline; mod print; mod progress; mod workspace; @@ -30,8 +31,8 @@ use snafu::{Report, ResultExt, Whatever}; use crate::{ cli::{Args, Operation, RuntimeOperation}, - execution::{resolve_execution, validate_non_execution_command_flags, ResolvedExecution}, - layout::{output_path, CONTENT_DIR, REPO_DIR}, + execution::{resolve_execution, validate_non_execution_command_flags}, + pipeline::Prepared, workspace::{doctor_workspace, init_workspace}, }; @@ -61,97 +62,6 @@ fn make_build_dir(build_path: &Path) -> Result { Ok(build_path.to_path_buf()) } -#[derive(Debug)] -struct Prepared { - repo_path: PathBuf, - output_path: PathBuf, - repository_use: git::RepositoryUse, - theme_path: PathBuf, - base_url_override: Option, -} - -impl Prepared { - fn prepare(eipw: lint::CmdArgs, resolved: ResolvedExecution) -> Result { - zola::find_zola().whatever_context("unable to find suitable zola binary")?; - let theme_path = resolved.theme_path()?.to_path_buf(); - - let ResolvedExecution { - root_path, - build_path, - repository_use, - theme_path: _, - source_materialization, - base_url_override, - } = resolved; - - let repo_path = build_path.join(REPO_DIR); - let content_path = repo_path.join(CONTENT_DIR); - let output_path = output_path(&build_path); - - let both = git::Fresh::new( - &root_path, - &repo_path, - repository_use.clone(), - source_materialization, - ) - .whatever_context("initializing build repo")? - .clone_src() - .whatever_context("cloning source repo")? - .fetch_upstream() - .whatever_context("fetching upstream repo")?; - - let changed_files: Vec<_> = both - .changed_files() - .whatever_context("unable to list changed files")? - .into_iter() - .filter(|p| changed::is_proposal_path(p.into())) - .map(|p| repo_path.join(p)) - .collect(); - - both.merge() - .whatever_context("unable to merge ERC/EIP repositories")?; - - lint::eipw(&theme_path, &root_path, &repo_path, changed_files, eipw) - .whatever_context("linting failed")?; - - markdown::preprocess(&content_path).whatever_context("unable to preprocess markdown")?; - - Ok(Prepared { - repo_path, - output_path, - repository_use, - theme_path, - base_url_override, - }) - } - - fn build(self) -> Result<(), Whatever> { - let base_url = self - .base_url_override - .as_ref() - .unwrap_or(&self.repository_use.location.base_url); - zola::build( - &self.theme_path, - &self.repo_path, - &self.output_path, - base_url.as_str(), - ) - .whatever_context("zola build failed")?; - Ok(()) - } - - fn serve(self) -> Result<(), Whatever> { - zola::serve(&self.theme_path, &self.repo_path, &self.output_path) - .whatever_context("zola serve failed")?; - Ok(()) - } - - fn check(self) -> Result<(), Whatever> { - zola::check(&self.theme_path, &self.repo_path).whatever_context("zola check failed")?; - Ok(()) - } -} - fn run() -> Result<(), Whatever> { let args = Args::parse(); validate_non_execution_command_flags(&args)?; diff --git a/src/pipeline.rs b/src/pipeline.rs new file mode 100644 index 0000000..172d87c --- /dev/null +++ b/src/pipeline.rs @@ -0,0 +1,211 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Prepared Zola runtime pipeline. + +use std::path::{Path, PathBuf}; + +use snafu::{OptionExt, ResultExt, Whatever}; +use url::Url; + +use crate::{ + changed, + execution::ResolvedExecution, + git, + layout::{mounted_theme_path, output_path, CONTENT_DIR, REPO_DIR}, + lint, markdown, zola, +}; + +fn prepare_theme_for_zola(theme_path: PathBuf, repo_path: &Path) -> Result { + let mounted_theme_dir = mounted_theme_path(repo_path); + git::materialize_working_tree(&theme_path, &mounted_theme_dir) + .whatever_context("unable to materialize workspace-local theme")?; + + Ok(mounted_theme_dir) +} + +#[derive(Debug)] +pub(crate) struct Prepared { + repo_path: PathBuf, + output_path: PathBuf, + repository_use: git::RepositoryUse, + theme_path: PathBuf, + base_url_override: Option, +} + +impl Prepared { + pub(crate) fn prepare( + eipw: lint::CmdArgs, + resolved: ResolvedExecution, + ) -> Result { + zola::find_zola().whatever_context("unable to find suitable zola binary")?; + + let ResolvedExecution { + root_path, + build_path, + repository_use, + theme_path, + source_materialization, + base_url_override, + } = resolved; + let theme_path = + theme_path.whatever_context("Zola runtime requires a workspace-local theme path")?; + + let repo_path = build_path.join(REPO_DIR); + let content_path = repo_path.join(CONTENT_DIR); + let output_path = output_path(&build_path); + + let both = git::Fresh::new( + &root_path, + &repo_path, + repository_use.clone(), + source_materialization, + ) + .whatever_context("initializing build repo")? + .clone_src() + .whatever_context("cloning source repo")? + .fetch_upstream() + .whatever_context("fetching upstream repo")?; + + let changed_files: Vec<_> = both + .changed_files() + .whatever_context("unable to list changed files")? + .into_iter() + .filter(|p| changed::is_proposal_path(p.into())) + .map(|p| repo_path.join(p)) + .collect(); + + both.merge() + .whatever_context("unable to merge ERC/EIP repositories")?; + + lint::eipw(&theme_path, &root_path, &repo_path, changed_files, eipw) + .whatever_context("linting failed")?; + + markdown::preprocess(&content_path).whatever_context("unable to preprocess markdown")?; + let theme_path = prepare_theme_for_zola(theme_path, &repo_path)?; + + Ok(Prepared { + repository_use, + theme_path, + repo_path, + output_path, + base_url_override, + }) + } + + pub(crate) fn build(self) -> Result<(), Whatever> { + let base_url = self + .base_url_override + .as_ref() + .unwrap_or(&self.repository_use.location.base_url); + zola::build( + &self.theme_path, + &self.repo_path, + &self.output_path, + base_url.as_str(), + ) + .whatever_context("zola build failed")?; + Ok(()) + } + + pub(crate) fn serve(self) -> Result<(), Whatever> { + zola::serve(&self.theme_path, &self.repo_path, &self.output_path) + .whatever_context("zola serve failed")?; + Ok(()) + } + + pub(crate) fn check(self) -> Result<(), Whatever> { + zola::check(&self.theme_path, &self.repo_path).whatever_context("zola check failed")?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use git2::{IndexAddOption, Repository, Signature}; + use tempfile::TempDir; + + use crate::layout::{mounted_theme_path, theme_config_path}; + + use super::prepare_theme_for_zola; + + fn write_file(root: &Path, relative: impl AsRef, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn commit_all(repo: &Repository, message: &str) { + let mut index = repo.index().unwrap(); + index + .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + let parents = repo + .head() + .ok() + .and_then(|head| head.target()) + .map(|oid| repo.find_commit(oid).unwrap()) + .into_iter() + .collect::>(); + let parent_refs = parents.iter().collect::>(); + + repo.commit( + Some("HEAD"), + &signature, + &signature, + message, + &tree, + &parent_refs, + ) + .unwrap(); + } + + fn init_repo(path: &Path, files: &[(&str, &str)]) -> Repository { + std::fs::create_dir_all(path).unwrap(); + let repo = Repository::init(path).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + for (relative, contents) in files { + write_file(path, relative, contents); + } + commit_all(&repo, "initial"); + repo + } + + #[test] + fn workspace_local_theme_is_materialized_as_mounted_theme_for_zola() { + let temp = TempDir::new().unwrap(); + let theme_root = temp.path().join("workspace/theme"); + init_repo( + &theme_root, + &[ + ("config/zola.toml", "title = 'theme'\n"), + ("templates/index.html", "local theme\n"), + ], + ); + let repo_path = temp.path().join("workspace/.local-build/Core/repo"); + + let theme_path = prepare_theme_for_zola(theme_root, &repo_path).unwrap(); + + let mounted_theme_dir = mounted_theme_path(&repo_path); + assert_eq!(theme_path, mounted_theme_dir); + assert_eq!( + theme_config_path(&mounted_theme_dir), + repo_path.join("themes/eips-theme/config/zola.toml") + ); + assert_eq!( + std::fs::read_to_string(mounted_theme_dir.join("templates/index.html")).unwrap(), + "local theme\n" + ); + } +} From fcfad996d19ab8d8110ea6b1da9d7168571d69cb Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Sun, 3 May 2026 21:18:29 -0400 Subject: [PATCH 09/20] Add serve runtime Add server binding resolution and serve-only host/port flags for local Zola serve commands. Run Zola serve with the resolved server binding, optional base URL override, fast/force serve flags, and generated output directory. Add dirty serve watching for dirty active-repo paths and local theme changes. Clean mode disables active-repo sync but keeps theme sync. --- Cargo.lock | 141 +++++++++++++- Cargo.toml | 1 + src/cli.rs | 66 +++++++ src/execution.rs | 88 ++++++++- src/main.rs | 1 + src/pipeline.rs | 69 ++++++- src/serve.rs | 494 +++++++++++++++++++++++++++++++++++++++++++++++ src/zola.rs | 114 ++++++++++- 8 files changed, 943 insertions(+), 31 deletions(-) create mode 100644 src/serve.rs diff --git a/Cargo.lock b/Cargo.lock index 715f3ee..a7da737 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -187,9 +187,15 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.10.0" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "block-buffer" @@ -251,6 +257,7 @@ dependencies = [ "iref", "lazy_static", "log", + "notify", "pulldown-cmark", "pulldown-cmark-to-cmark", "regex", @@ -465,6 +472,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crunchy" version = "0.2.4" @@ -782,6 +804,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -830,6 +863,15 @@ dependencies = [ "num", ] +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "fslock" version = "0.2.1" @@ -913,7 +955,7 @@ version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" dependencies = [ - "bitflags", + "bitflags 2.11.1", "libc", "libgit2-sys", "log", @@ -1198,6 +1240,26 @@ dependencies = [ "rustversion", ] +[[package]] +name = "inotify" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + [[package]] name = "iref" version = "3.2.2" @@ -1317,6 +1379,26 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "kqueue" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7b65860415f949f23fa882e669f2dbd4a0f0eeb1acdd56790b30494afd7da2f" +dependencies = [ + "bitflags 2.11.1", + "libc", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1349,8 +1431,9 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ - "bitflags", + "bitflags 2.11.1", "libc", + "redox_syscall 0.7.4", ] [[package]] @@ -1478,6 +1561,18 @@ dependencies = [ "adler2", ] +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.48.0", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -1494,6 +1589,25 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "notify" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" +dependencies = [ + "bitflags 2.11.1", + "crossbeam-channel", + "filetime", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "walkdir", + "windows-sys 0.48.0", +] + [[package]] name = "num" version = "0.4.3" @@ -1663,7 +1777,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link", ] @@ -1823,7 +1937,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" dependencies = [ - "bitflags", + "bitflags 2.11.1", "getopts", "memchr", "pulldown-cmark-escape", @@ -1897,7 +2011,16 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.11.1", +] + +[[package]] +name = "redox_syscall" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" +dependencies = [ + "bitflags 2.11.1", ] [[package]] @@ -1992,7 +2115,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys", @@ -2070,7 +2193,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8" dependencies = [ - "bitflags", + "bitflags 2.11.1", "cssparser", "derive_more", "fxhash", diff --git a/Cargo.toml b/Cargo.toml index 2a48446..2f03e2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ indicatif = "0.18.3" indicatif-log-bridge = "0.2.3" lazy_static = "1.5.0" log = { version = "0.4.29", features = ["std"] } +notify = "6.1.1" pulldown-cmark = "0.13.0" pulldown-cmark-to-cmark = "22.0.0" regex = "1.12.2" diff --git a/src/cli.rs b/src/cli.rs index bf16e89..d70e3ab 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -41,6 +41,17 @@ pub(crate) struct Args { pub(crate) operation: Operation, } +#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)] +pub(crate) struct ServerCliArgs { + /// Host/interface for the local server to bind + #[arg(long)] + pub(crate) host: Option, + + /// Port for the local server to bind + #[arg(long)] + pub(crate) port: Option, +} + #[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)] pub(crate) struct BaseUrlCliArgs { /// Override the rendered-site base URL for this command @@ -80,6 +91,9 @@ pub(crate) enum Operation { #[command(flatten)] eipw: lint::CmdArgs, + #[command(flatten)] + server: ServerCliArgs, + #[command(flatten)] base_url: BaseUrlCliArgs, @@ -144,6 +158,9 @@ pub(crate) enum ProfiledOperation { #[command(flatten)] eipw: lint::CmdArgs, + #[command(flatten)] + server: ServerCliArgs, + #[command(flatten)] base_url: BaseUrlCliArgs, }, @@ -173,6 +190,20 @@ pub(crate) enum RuntimeOperation { } impl Operation { + pub(crate) fn server_cli_args(&self) -> ServerCliArgs { + match self { + Self::Serve { server, .. } => server.clone(), + Self::Parity { command } => command.server_cli_args(), + Self::Print { .. } + | Self::Build { .. } + | Self::Clean + | Self::Check { .. } + | Self::Changed { .. } + | Self::Init { .. } + | Self::Doctor => ServerCliArgs::default(), + } + } + pub(crate) fn base_url_cli_args(&self) -> BaseUrlCliArgs { match self { Self::Build { base_url, .. } | Self::Serve { base_url, .. } => base_url.clone(), @@ -232,6 +263,13 @@ impl Operation { } impl ProfiledOperation { + fn server_cli_args(&self) -> ServerCliArgs { + match self { + Self::Serve { server, .. } => server.clone(), + Self::Build { .. } | Self::Check { .. } => ServerCliArgs::default(), + } + } + fn base_url_cli_args(&self) -> BaseUrlCliArgs { match self { Self::Build { base_url, .. } | Self::Serve { base_url, .. } => base_url.clone(), @@ -402,6 +440,34 @@ mod tests { } } + #[test] + fn server_flags_parse_on_serve_forms() { + let cases: &[&[&str]] = &[ + &["build-eips", "serve", "--host", "0.0.0.0", "--port", "8080"], + &[ + "build-eips", + "parity", + "serve", + "--host", + "0.0.0.0", + "--port", + "8080", + ], + ]; + + for arguments in cases { + let args = parse_args(arguments); + assert!(matches!( + args.operation.runtime_operation(), + Some(RuntimeOperation::Serve { .. }) + )); + let server = args.operation.server_cli_args(); + + assert_eq!(server.host.as_deref(), Some("0.0.0.0")); + assert_eq!(server.port, Some(8080)); + } + } + #[test] fn removed_command_surface_is_rejected() { for arguments in [ diff --git a/src/execution.rs b/src/execution.rs index c634e3c..46d7c22 100644 --- a/src/execution.rs +++ b/src/execution.rs @@ -16,8 +16,8 @@ use snafu::{OptionExt, ResultExt, Whatever}; use url::Url; use crate::{ - cli::{Args, Operation}, - config::{self, LoadedWorkspaceConfig}, + cli::{Args, Operation, ServerCliArgs}, + config::{self, LoadedWorkspaceConfig, ServerBinding}, context::{resolve_input_path, root}, git, identity::ActiveRepoIdentity, @@ -31,6 +31,7 @@ pub(crate) struct ResolvedExecution { pub(crate) repository_use: git::RepositoryUse, pub(crate) theme_path: Option, pub(crate) source_materialization: git::SourceMaterialization, + pub(crate) server_binding: ServerBinding, pub(crate) base_url_override: Option, } @@ -275,6 +276,25 @@ fn resolve_theme_path( } } +fn resolve_server_binding( + workspace_config: Option<&LoadedWorkspaceConfig>, + server_cli: &ServerCliArgs, +) -> ServerBinding { + let mut binding = workspace_config + .map(|workspace_config| ServerBinding::from(workspace_config.server_settings())) + .unwrap_or_default(); + + if let Some(host) = &server_cli.host { + binding.host = host.clone(); + } + + if let Some(port) = server_cli.port { + binding.port = port; + } + + binding +} + fn resolve_base_url_override( args: &Args, workspace_config: Option<&LoadedWorkspaceConfig>, @@ -336,6 +356,10 @@ pub(crate) fn resolve_execution(args: &Args) -> Result Args { @@ -443,6 +468,59 @@ mod tests { } } + #[test] + fn server_binding_resolution_uses_cli_config_then_defaults() { + assert_eq!( + resolve_server_binding(None, &Default::default()), + ServerBinding { + host: "127.0.0.1".to_owned(), + port: 1111, + } + ); + + let workspace_config = load_workspace_config( + r#" +[server] +host = "0.0.0.0" +port = 8080 +"#, + ); + + assert_eq!( + resolve_server_binding(Some(&workspace_config), &Default::default()), + ServerBinding { + host: "0.0.0.0".to_owned(), + port: 8080, + } + ); + assert_eq!( + resolve_server_binding( + Some(&workspace_config), + &ServerCliArgs { + host: Some("127.0.0.1".to_owned()), + port: Some(4000), + }, + ), + ServerBinding { + host: "127.0.0.1".to_owned(), + port: 4000, + } + ); + assert_eq!( + resolve_server_binding( + Some(&workspace_config), + &ServerCliArgs { + host: None, + port: Some(4000), + }, + ), + ServerBinding { + host: "0.0.0.0".to_owned(), + port: 4000, + } + ); + } + #[test] fn base_url_override_resolution_uses_cli_config_then_provenance() { let workspace_config = load_workspace_config( diff --git a/src/main.rs b/src/main.rs index 5880c8c..6aee051 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,6 +19,7 @@ mod markdown; mod pipeline; mod print; mod progress; +mod serve; mod workspace; mod zola; diff --git a/src/pipeline.rs b/src/pipeline.rs index 172d87c..413d0dc 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -13,18 +13,33 @@ use url::Url; use crate::{ changed, + config::ServerBinding, execution::ResolvedExecution, git, layout::{mounted_theme_path, output_path, CONTENT_DIR, REPO_DIR}, - lint, markdown, zola, + lint, markdown, + serve::{serve_sync_config, DirtyServeWatcher, LocalThemeServeSync}, + zola, }; -fn prepare_theme_for_zola(theme_path: PathBuf, repo_path: &Path) -> Result { +fn prepare_theme_for_zola( + theme_path: PathBuf, + repo_path: &Path, +) -> Result<(PathBuf, LocalThemeServeSync), Whatever> { let mounted_theme_dir = mounted_theme_path(repo_path); git::materialize_working_tree(&theme_path, &mounted_theme_dir) .whatever_context("unable to materialize workspace-local theme")?; + let theme_index_path = git::index_path(&theme_path) + .whatever_context("unable to resolve workspace-local theme Git index path")?; - Ok(mounted_theme_dir) + Ok(( + mounted_theme_dir.clone(), + LocalThemeServeSync { + theme_source_root: theme_path, + mounted_theme_dir, + theme_index_path, + }, + )) } #[derive(Debug)] @@ -33,6 +48,10 @@ pub(crate) struct Prepared { output_path: PathBuf, repository_use: git::RepositoryUse, theme_path: PathBuf, + local_theme_sync: Option, + source_root: PathBuf, + source_materialization: git::SourceMaterialization, + server_binding: ServerBinding, base_url_override: Option, } @@ -49,6 +68,7 @@ impl Prepared { repository_use, theme_path, source_materialization, + server_binding, base_url_override, } = resolved; let theme_path = @@ -85,13 +105,17 @@ impl Prepared { .whatever_context("linting failed")?; markdown::preprocess(&content_path).whatever_context("unable to preprocess markdown")?; - let theme_path = prepare_theme_for_zola(theme_path, &repo_path)?; + let (theme_path, local_theme_sync) = prepare_theme_for_zola(theme_path, &repo_path)?; Ok(Prepared { repository_use, theme_path, + local_theme_sync: Some(local_theme_sync), repo_path, output_path, + source_root: root_path, + source_materialization, + server_binding, base_url_override, }) } @@ -112,9 +136,35 @@ impl Prepared { } pub(crate) fn serve(self) -> Result<(), Whatever> { - zola::serve(&self.theme_path, &self.repo_path, &self.output_path) - .whatever_context("zola serve failed")?; - Ok(()) + let sync_config = serve_sync_config( + self.source_materialization, + &self.source_root, + &self.repo_path, + self.local_theme_sync.clone(), + ); + let dirty_watcher = if sync_config.has_targets() { + Some( + DirtyServeWatcher::start(sync_config) + .whatever_context("unable to start dirty serve watcher")?, + ) + } else { + None + }; + + let result = zola::serve( + &self.theme_path, + &self.repo_path, + &self.output_path, + &self.server_binding, + self.base_url_override.as_ref(), + ) + .whatever_context("zola serve failed"); + + if let Some(dirty_watcher) = dirty_watcher { + dirty_watcher.stop(); + } + + result } pub(crate) fn check(self) -> Result<(), Whatever> { @@ -195,7 +245,7 @@ mod tests { ); let repo_path = temp.path().join("workspace/.local-build/Core/repo"); - let theme_path = prepare_theme_for_zola(theme_root, &repo_path).unwrap(); + let (theme_path, sync) = prepare_theme_for_zola(theme_root.clone(), &repo_path).unwrap(); let mounted_theme_dir = mounted_theme_path(&repo_path); assert_eq!(theme_path, mounted_theme_dir); @@ -207,5 +257,8 @@ mod tests { std::fs::read_to_string(mounted_theme_dir.join("templates/index.html")).unwrap(), "local theme\n" ); + assert_eq!(sync.theme_source_root, theme_root); + assert_eq!(sync.mounted_theme_dir, mounted_theme_dir); + assert!(sync.theme_index_path.ends_with(".git/index")); } } diff --git a/src/serve.rs b/src/serve.rs new file mode 100644 index 0000000..cf347cb --- /dev/null +++ b/src/serve.rs @@ -0,0 +1,494 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Dirty active-repo and local-theme serve synchronization. + +use std::{ + collections::BTreeSet, + ffi::OsStr, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc::{self, RecvTimeoutError}, + Arc, + }, + thread::{self, JoinHandle}, + time::Duration, +}; + +use log::{debug, info, warn}; +use notify::{Event, RecursiveMode, Watcher}; +use snafu::{Report, ResultExt, Whatever}; + +use crate::{git, layout::CONTENT_DIR, markdown}; + +#[derive(Debug)] +pub(crate) struct DirtyServeWatcher { + stop: Arc, + thread: JoinHandle<()>, +} + +#[derive(Debug, Clone)] +struct ActiveRepoServeSync { + source_root: PathBuf, + build_repo_path: PathBuf, +} + +#[derive(Debug, Clone)] +pub(crate) struct LocalThemeServeSync { + pub(crate) theme_source_root: PathBuf, + pub(crate) mounted_theme_dir: PathBuf, + pub(crate) theme_index_path: PathBuf, +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct ServeSyncConfig { + active_repo: Option, + local_theme: Option, +} + +impl ServeSyncConfig { + pub(crate) fn has_targets(&self) -> bool { + self.active_repo.is_some() || self.local_theme.is_some() + } +} + +impl DirtyServeWatcher { + pub(crate) fn start(sync_config: ServeSyncConfig) -> Result { + let stop = Arc::new(AtomicBool::new(false)); + let stop_thread = stop.clone(); + let (ready_tx, ready_rx) = mpsc::channel(); + let thread = + thread::spawn(move || dirty_serve_sync_loop(sync_config, stop_thread, ready_tx)); + + match ready_rx + .recv() + .whatever_context("dirty serve watcher exited before initialization")? + { + Ok(()) => Ok(Self { stop, thread }), + Err(message) => { + stop.store(true, Ordering::Relaxed); + let _ = thread.join(); + snafu::whatever!("{message}"); + } + } + } + + pub(crate) fn stop(self) { + self.stop.store(true, Ordering::Relaxed); + let _ = self.thread.join(); + } +} + +fn path_is_watched_source_path(root_path: &Path, path: &Path) -> bool { + let Ok(relative_path) = path.strip_prefix(root_path) else { + return false; + }; + + relative_path + .components() + .next() + .map(|component| component.as_os_str() != OsStr::new(".git")) + .unwrap_or(false) +} + +fn event_has_watched_source_path(root_path: &Path, event: &Event) -> bool { + event + .paths + .iter() + .any(|path| path_is_watched_source_path(root_path, path)) +} + +fn index_lock_path(index_path: &Path) -> Option { + let file_name = index_path.file_name()?.to_string_lossy(); + Some(index_path.with_file_name(format!("{file_name}.lock"))) +} + +fn event_has_theme_index_path(index_path: &Path, event: &Event) -> bool { + let lock_path = index_lock_path(index_path); + event.paths.iter().any(|path| { + path == index_path + || lock_path + .as_ref() + .map(|lock_path| path == lock_path) + .unwrap_or(false) + }) +} + +fn sync_dirty_serve_state( + source_root: &Path, + build_repo_path: &Path, + previous_dirty_paths: &mut BTreeSet, +) -> Result<(), Whatever> { + let current_dirty_paths: BTreeSet<_> = git::working_tree_paths(source_root) + .whatever_context("unable to list tracked dirty paths for dirty serve")? + .into_iter() + .collect(); + + let affected_paths: BTreeSet<_> = previous_dirty_paths + .union(¤t_dirty_paths) + .cloned() + .collect(); + + if affected_paths.is_empty() { + *previous_dirty_paths = current_dirty_paths; + return Ok(()); + } + + git::sync_materialized_paths(source_root, build_repo_path, &affected_paths) + .whatever_context("unable to synchronize tracked paths into the materialized repo")?; + markdown::preprocess(&build_repo_path.join(CONTENT_DIR)) + .whatever_context("unable to preprocess synchronized markdown during dirty serve")?; + + info!( + "synchronized {} tracked path(s) into the materialized repo for dirty serve", + affected_paths.len() + ); + + *previous_dirty_paths = current_dirty_paths; + Ok(()) +} + +fn capture_active_dirty_paths(source_root: &Path) -> Result, Whatever> { + Ok(git::working_tree_paths(source_root) + .whatever_context("unable to list tracked dirty paths for dirty serve")? + .into_iter() + .collect()) +} + +fn sync_theme_serve_state( + theme_source_root: &Path, + mounted_theme_dir: &Path, + previous_dirty_paths: &mut BTreeSet, +) -> Result<(), Whatever> { + let current_dirty_paths: BTreeSet<_> = git::tracked_working_tree_paths(theme_source_root) + .whatever_context("unable to list tracked dirty paths for local theme serve")? + .into_iter() + .collect(); + + let affected_paths: BTreeSet<_> = previous_dirty_paths + .union(¤t_dirty_paths) + .cloned() + .collect(); + + if affected_paths.is_empty() { + *previous_dirty_paths = current_dirty_paths; + return Ok(()); + } + + git::sync_working_tree_paths(theme_source_root, mounted_theme_dir, &affected_paths) + .whatever_context("unable to synchronize tracked local theme paths")?; + + info!( + "synchronized {} tracked path(s) into the mounted local theme for serve", + affected_paths.len() + ); + + *previous_dirty_paths = current_dirty_paths; + Ok(()) +} + +fn watch_theme_index( + watcher: &mut notify::RecommendedWatcher, + theme_index_path: &Path, +) -> Result<(), String> { + let file_result = watcher.watch(theme_index_path, RecursiveMode::NonRecursive); + let Some(parent) = theme_index_path.parent() else { + return file_result.map_err(|file_error| { + format!( + "unable to watch local theme Git index `{}`: {file_error}", + theme_index_path.to_string_lossy() + ) + }); + }; + let parent_result = watcher.watch(parent, RecursiveMode::NonRecursive); + + match (file_result, parent_result) { + (Ok(()), Ok(())) => Ok(()), + (Ok(()), Err(parent_error)) => { + debug!( + "unable to watch local theme Git index parent `{}`: {parent_error}", + parent.to_string_lossy() + ); + Ok(()) + } + (Err(file_error), Ok(())) => { + debug!( + "using local theme Git index parent watch for `{}` after file watch failed: {file_error}", + theme_index_path.to_string_lossy() + ); + Ok(()) + } + (Err(file_error), Err(parent_error)) => Err(format!( + "unable to watch local theme Git index `{}`: {file_error}; fallback watch on `{}` also failed: {parent_error}", + theme_index_path.to_string_lossy(), + parent.to_string_lossy() + )), + } +} + +fn dirty_serve_sync_loop( + sync_config: ServeSyncConfig, + stop: Arc, + ready_tx: mpsc::Sender>, +) { + let (event_tx, event_rx) = mpsc::channel(); + let mut watcher = match notify::recommended_watcher(move |result| { + let _ = event_tx.send(result); + }) { + Ok(watcher) => watcher, + Err(error) => { + let _ = ready_tx.send(Err(format!("unable to start dirty serve watcher: {error}"))); + return; + } + }; + + if let Some(active_repo) = &sync_config.active_repo { + if let Err(error) = watcher.watch(&active_repo.source_root, RecursiveMode::Recursive) { + let _ = ready_tx.send(Err(format!( + "unable to watch `{}` for dirty serve changes: {error}", + active_repo.source_root.to_string_lossy() + ))); + return; + } + } + + if let Some(local_theme) = &sync_config.local_theme { + if let Err(error) = watcher.watch(&local_theme.theme_source_root, RecursiveMode::Recursive) + { + let _ = ready_tx.send(Err(format!( + "unable to watch local theme `{}` for serve changes: {error}", + local_theme.theme_source_root.to_string_lossy() + ))); + return; + } + + if let Err(message) = watch_theme_index(&mut watcher, &local_theme.theme_index_path) { + let _ = ready_tx.send(Err(message)); + return; + } + } + + let mut previous_active_dirty_paths: BTreeSet<_> = match &sync_config.active_repo { + Some(active_repo) => match capture_active_dirty_paths(&active_repo.source_root) { + Ok(paths) => paths, + Err(error) => { + let _ = ready_tx.send(Err(format!( + "unable to capture initial dirty serve state: {}", + Report::from_error(error) + ))); + return; + } + }, + None => BTreeSet::new(), + }; + + let mut previous_theme_dirty_paths: BTreeSet<_> = match &sync_config.local_theme { + Some(local_theme) => { + if let Err(error) = git::materialize_working_tree( + &local_theme.theme_source_root, + &local_theme.mounted_theme_dir, + ) { + let _ = ready_tx.send(Err(format!( + "unable to synchronize initial local theme state after watcher setup: {}", + Report::from_error(error) + ))); + return; + } + + match git::tracked_working_tree_paths(&local_theme.theme_source_root) { + Ok(paths) => paths.into_iter().collect(), + Err(error) => { + let _ = ready_tx.send(Err(format!( + "unable to capture initial local theme dirty state: {}", + Report::from_error(error) + ))); + return; + } + } + } + None => BTreeSet::new(), + }; + + if let Some(active_repo) = &sync_config.active_repo { + info!( + "watching `{}` for dirty serve changes", + active_repo.source_root.to_string_lossy() + ); + } + if let Some(local_theme) = &sync_config.local_theme { + info!( + "watching `{}` for local theme serve changes", + local_theme.theme_source_root.to_string_lossy() + ); + } + let _ = ready_tx.send(Ok(())); + + while !stop.load(Ordering::Relaxed) { + let first_event = match event_rx.recv_timeout(Duration::from_millis(250)) { + Ok(event) => Some(event), + Err(RecvTimeoutError::Timeout) => None, + Err(RecvTimeoutError::Disconnected) => break, + }; + + let Some(first_event) = first_event else { + continue; + }; + + let mut saw_active_event = false; + let mut saw_theme_event = false; + + match first_event { + Ok(event) => { + if let Some(active_repo) = &sync_config.active_repo { + saw_active_event |= + event_has_watched_source_path(&active_repo.source_root, &event); + } + if let Some(local_theme) = &sync_config.local_theme { + saw_theme_event |= + event_has_watched_source_path(&local_theme.theme_source_root, &event) + || event_has_theme_index_path(&local_theme.theme_index_path, &event); + } + } + Err(error) => { + warn!("filesystem watcher error: {error}"); + } + } + + loop { + match event_rx.recv_timeout(Duration::from_millis(75)) { + Ok(Ok(event)) => { + if let Some(active_repo) = &sync_config.active_repo { + saw_active_event |= + event_has_watched_source_path(&active_repo.source_root, &event); + } + if let Some(local_theme) = &sync_config.local_theme { + saw_theme_event |= + event_has_watched_source_path(&local_theme.theme_source_root, &event) + || event_has_theme_index_path( + &local_theme.theme_index_path, + &event, + ); + } + } + Ok(Err(error)) => warn!("filesystem watcher error: {error}"), + Err(RecvTimeoutError::Timeout) => break, + Err(RecvTimeoutError::Disconnected) => return, + } + } + + if saw_active_event { + if let Some(active_repo) = &sync_config.active_repo { + if let Err(error) = sync_dirty_serve_state( + &active_repo.source_root, + &active_repo.build_repo_path, + &mut previous_active_dirty_paths, + ) { + warn!( + "unable to synchronize dirty serve changes: {}", + Report::from_error(error) + ); + } + } + } + + if saw_theme_event { + if let Some(local_theme) = &sync_config.local_theme { + if let Err(error) = sync_theme_serve_state( + &local_theme.theme_source_root, + &local_theme.mounted_theme_dir, + &mut previous_theme_dirty_paths, + ) { + warn!( + "unable to synchronize local theme serve changes: {}", + Report::from_error(error) + ); + } + } + } + } +} + +pub(crate) fn serve_sync_config( + source_materialization: git::SourceMaterialization, + source_root: &Path, + repo_path: &Path, + local_theme_sync: Option, +) -> ServeSyncConfig { + ServeSyncConfig { + active_repo: (source_materialization == git::SourceMaterialization::Dirty).then(|| { + ActiveRepoServeSync { + source_root: source_root.to_path_buf(), + build_repo_path: repo_path.to_path_buf(), + } + }), + local_theme: local_theme_sync, + } +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use notify::{Event, EventKind}; + use tempfile::TempDir; + + use crate::git::SourceMaterialization; + + use super::{event_has_theme_index_path, serve_sync_config, LocalThemeServeSync}; + + fn fake_theme_sync(root: &Path) -> LocalThemeServeSync { + LocalThemeServeSync { + theme_source_root: root.join("theme"), + mounted_theme_dir: root.join("repo/themes/eips-theme"), + theme_index_path: root.join("theme/.git/index"), + } + } + + #[test] + fn local_theme_index_events_trigger_rescan() { + let index_path = PathBuf::from("/workspace/theme/.git/index"); + let index_event = Event::new(EventKind::Any).add_path(index_path.clone()); + let lock_event = + Event::new(EventKind::Any).add_path(PathBuf::from("/workspace/theme/.git/index.lock")); + let unrelated_event = + Event::new(EventKind::Any).add_path(PathBuf::from("/workspace/theme/.git/config")); + + assert!(event_has_theme_index_path(&index_path, &index_event)); + assert!(event_has_theme_index_path(&index_path, &lock_event)); + assert!(!event_has_theme_index_path(&index_path, &unrelated_event)); + } + + #[test] + fn local_serve_syncs_theme_and_dirty_active_repo() { + let temp = TempDir::new().unwrap(); + + let sync_config = serve_sync_config( + SourceMaterialization::Dirty, + &temp.path().join("Core"), + &temp.path().join(".local-build/Core/repo"), + Some(fake_theme_sync(temp.path())), + ); + + assert!(sync_config.active_repo.is_some()); + assert!(sync_config.local_theme.is_some()); + } + + #[test] + fn clean_local_serve_keeps_theme_sync_but_disables_active_repo_dirty_sync() { + let temp = TempDir::new().unwrap(); + + let sync_config = serve_sync_config( + SourceMaterialization::Clean, + &temp.path().join("Core"), + &temp.path().join(".local-build/Core/repo"), + Some(fake_theme_sync(temp.path())), + ); + + assert!(sync_config.active_repo.is_none()); + assert!(sync_config.local_theme.is_some()); + } +} diff --git a/src/zola.rs b/src/zola.rs index 82c3503..125a166 100644 --- a/src/zola.rs +++ b/src/zola.rs @@ -15,7 +15,10 @@ use semver::Version; use snafu::{ensure, Backtrace, IntoError, Report, ResultExt, Snafu}; use url::Url; -use crate::layout::{mounted_theme_path, theme_config_path}; +use crate::{ + config::ServerBinding, + layout::{mounted_theme_path, theme_config_path}, +}; const MINIMUM_VERSION: Version = Version::new(0, 22, 1); @@ -131,18 +134,51 @@ pub fn build( Ok(()) } -pub fn serve(theme_dir: &Path, project_path: &Path, output_path: &Path) -> Result<(), Error> { +pub fn serve( + theme_dir: &Path, + project_path: &Path, + output_path: &Path, + server_binding: &ServerBinding, + base_url_override: Option<&Url>, +) -> Result<(), Error> { // TODO: Properly kill the child process when we receive ctrl-c. - warn!("live reloading is not implemented"); remove_output(output_path); - let args = ["serve", "--drafts", "-o"] - .map(OsString::from) - .into_iter() - .chain(std::iter::once(output_path.into())); + let args = serve_args(server_binding, output_path, base_url_override); spawn_log(theme_dir, project_path, args)?; Ok(()) } +fn serve_args( + server_binding: &ServerBinding, + output_path: &Path, + base_url_override: Option<&Url>, +) -> Vec { + let mut args = [ + "serve", + "--drafts", + "--fast", + "--force", + "--interface", + server_binding.host.as_str(), + "--port", + ] + .map(OsString::from) + .to_vec(); + + args.push(OsString::from(server_binding.port.to_string())); + + if let Some(base_url) = base_url_override { + args.extend([ + OsString::from("-u"), + OsString::from(base_url.as_str()), + OsString::from("--no-port-append"), + ]); + } + + args.extend([OsString::from("-o"), output_path.as_os_str().to_os_string()]); + args +} + fn remove_output(output_path: &Path) { if let Err(e) = std::fs::remove_dir_all(output_path) { debug!( @@ -208,6 +244,7 @@ where #[cfg(test)] mod tests { use std::{ + ffi::OsString, fs, path::{Path, PathBuf}, process::{Command, ExitStatus}, @@ -215,9 +252,12 @@ mod tests { use tempfile::TempDir; - use crate::layout::{mounted_theme_path, theme_config_path}; + use crate::{ + config::ServerBinding, + layout::{mounted_theme_path, theme_config_path}, + }; - use super::{find_zola, mount_theme}; + use super::{find_zola, mount_theme, serve_args}; fn write_file(root: &Path, relative: &str, contents: &str) { let path = root.join(relative); @@ -286,6 +326,62 @@ mod tests { ); } + #[test] + fn serve_args_include_configured_interface_and_port() { + let server_binding = ServerBinding { + host: "0.0.0.0".to_owned(), + port: 8080, + }; + + assert_eq!( + serve_args(&server_binding, Path::new("/tmp/build-output"), None), + vec![ + OsString::from("serve"), + OsString::from("--drafts"), + OsString::from("--fast"), + OsString::from("--force"), + OsString::from("--interface"), + OsString::from("0.0.0.0"), + OsString::from("--port"), + OsString::from("8080"), + OsString::from("-o"), + OsString::from("/tmp/build-output"), + ] + ); + } + + #[test] + fn serve_args_include_base_url_override_when_present() { + let server_binding = ServerBinding { + host: "127.0.0.1".to_owned(), + port: 1111, + }; + let base_url = "http://127.0.0.1:1111".parse().unwrap(); + + assert_eq!( + serve_args( + &server_binding, + Path::new("/tmp/build-output"), + Some(&base_url) + ), + vec![ + OsString::from("serve"), + OsString::from("--drafts"), + OsString::from("--fast"), + OsString::from("--force"), + OsString::from("--interface"), + OsString::from("127.0.0.1"), + OsString::from("--port"), + OsString::from("1111"), + OsString::from("-u"), + OsString::from("http://127.0.0.1:1111/"), + OsString::from("--no-port-append"), + OsString::from("-o"), + OsString::from("/tmp/build-output"), + ] + ); + } + #[test] fn mount_theme_does_not_symlink_mounted_local_theme_onto_itself() { let temp = TempDir::new().unwrap(); From e96e6e729dbde8a2e4ff991a574eb825c60bb46c Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Sun, 3 May 2026 21:23:44 -0400 Subject: [PATCH 10/20] Add static preview runtime Add build-eips preview for serving the existing resolved output directory without rebuilding or starting dirty sync. Reuse server binding resolution and preview-only host/port flags, and report missing output before binding the local server. Add a tiny_http static file server with safe path resolution, index-file fallback, basic content types, and preview path tests. --- Cargo.lock | 31 ++++++ Cargo.toml | 1 + src/cli.rs | 82 +++++++++++---- src/execution.rs | 7 +- src/main.rs | 10 ++ src/preview.rs | 263 +++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 374 insertions(+), 20 deletions(-) create mode 100644 src/preview.rs diff --git a/Cargo.lock b/Cargo.lock index a7da737..2c52e21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,6 +134,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + [[package]] name = "atomic" version = "0.6.1" @@ -266,6 +272,7 @@ dependencies = [ "serde_json", "snafu", "tempfile", + "tiny_http", "tokio", "toml 0.9.11+spec-1.1.0", "toml_datetime 0.7.5+spec-1.1.0", @@ -346,6 +353,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "chunked_transfer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" + [[package]] name = "ciborium" version = "0.2.2" @@ -1070,6 +1083,12 @@ dependencies = [ "match_token", ] +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -2603,6 +2622,18 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii", + "chunked_transfer", + "httpdate", + "log", +] + [[package]] name = "tinystr" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index 2f03e2f..4bbfb89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ semver = {version = "1.0.27", features = ["serde"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.148" snafu = { version = "0.8.9", features = ["rust_1_81"] } +tiny_http = "0.12.0" tokio = { version = "1.48.0", features = ["fs", "rt", "macros"] } toml = "0.9.10" toml_datetime = { version = "0.7.5", features = ["serde"] } diff --git a/src/cli.rs b/src/cli.rs index d70e3ab..2b55f6d 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -86,7 +86,13 @@ pub(crate) enum Operation { clean: CleanCliArgs, }, - /// Build the project and launch a web server to preview it + /// Serve the existing built output without rebuilding it + Preview { + #[command(flatten)] + server: ServerCliArgs, + }, + + /// Build a fresh temporary site, serve it locally, and watch tracked edits Serve { #[command(flatten)] eipw: lint::CmdArgs, @@ -184,6 +190,7 @@ pub(crate) enum ChangedFormat { pub(crate) enum RuntimeOperation { Build { eipw: lint::CmdArgs }, Serve { eipw: lint::CmdArgs }, + Preview, Clean, Check { eipw: lint::CmdArgs }, Changed { all: bool, format: ChangedFormat }, @@ -192,7 +199,7 @@ pub(crate) enum RuntimeOperation { impl Operation { pub(crate) fn server_cli_args(&self) -> ServerCliArgs { match self { - Self::Serve { server, .. } => server.clone(), + Self::Serve { server, .. } | Self::Preview { server } => server.clone(), Self::Parity { command } => command.server_cli_args(), Self::Print { .. } | Self::Build { .. } @@ -209,6 +216,7 @@ impl Operation { Self::Build { base_url, .. } | Self::Serve { base_url, .. } => base_url.clone(), Self::Parity { command } => command.base_url_cli_args(), Self::Print { .. } + | Self::Preview { .. } | Self::Clean | Self::Check { .. } | Self::Changed { .. } @@ -223,6 +231,7 @@ impl Operation { clean.clone() } Self::Print { .. } + | Self::Preview { .. } | Self::Clean | Self::Changed { .. } | Self::Init { .. } @@ -243,6 +252,7 @@ impl Operation { Self::Print { .. } | Self::Init { .. } | Self::Doctor => None, Self::Build { eipw, .. } => Some(RuntimeOperation::Build { eipw: eipw.clone() }), Self::Serve { eipw, .. } => Some(RuntimeOperation::Serve { eipw: eipw.clone() }), + Self::Preview { .. } => Some(RuntimeOperation::Preview), Self::Clean => Some(RuntimeOperation::Clean), Self::Check { eipw, .. } => Some(RuntimeOperation::Check { eipw: eipw.clone() }), Self::Changed { all, format } => Some(RuntimeOperation::Changed { @@ -433,6 +443,7 @@ mod tests { &["build-eips", "parity", "build", "--clean"][..], &["build-eips", "parity", "serve", "--clean"][..], &["build-eips", "parity", "check", "--clean"][..], + &["build-eips", "preview", "--clean"][..], &["build-eips", "changed", "--clean"][..], &["build-eips", "clean", "--clean"][..], ] { @@ -441,26 +452,45 @@ mod tests { } #[test] - fn server_flags_parse_on_serve_forms() { - let cases: &[&[&str]] = &[ - &["build-eips", "serve", "--host", "0.0.0.0", "--port", "8080"], - &[ - "build-eips", - "parity", - "serve", - "--host", - "0.0.0.0", - "--port", - "8080", - ], + fn server_flags_parse_on_serve_and_preview_forms() { + let cases: &[(&[&str], bool)] = &[ + ( + &["build-eips", "serve", "--host", "0.0.0.0", "--port", "8080"], + true, + ), + ( + &[ + "build-eips", + "preview", + "--host", + "0.0.0.0", + "--port", + "8080", + ], + false, + ), + ( + &[ + "build-eips", + "parity", + "serve", + "--host", + "0.0.0.0", + "--port", + "8080", + ], + true, + ), ]; - for arguments in cases { + for (arguments, expect_serve) in cases { let args = parse_args(arguments); - assert!(matches!( - args.operation.runtime_operation(), - Some(RuntimeOperation::Serve { .. }) - )); + let runtime_operation = args.operation.runtime_operation().unwrap(); + match runtime_operation { + RuntimeOperation::Serve { .. } if *expect_serve => {} + RuntimeOperation::Preview if !*expect_serve => {} + other => panic!("unexpected runtime operation: {other:?}"), + } let server = args.operation.server_cli_args(); assert_eq!(server.host.as_deref(), Some("0.0.0.0")); @@ -478,6 +508,7 @@ mod tests { &["build-eips", "--remote-sibling-repo", "build"][..], &["build-eips", "workspace", "init", "/tmp/workspace"][..], &["build-eips", "workspace", "doctor"][..], + &["build-eips", "parity", "preview"][..], &["build-eips", "parity", "clean"][..], &["build-eips", "parity", "changed"][..], ] { @@ -488,6 +519,19 @@ mod tests { #[test] fn base_url_flag_is_rejected_on_non_rendering_forms() { let cases: &[&[&str]] = &[ + &[ + "build-eips", + "preview", + "--base-url", + "http://localhost:4000", + ], + &[ + "build-eips", + "parity", + "preview", + "--base-url", + "http://localhost:4000", + ], &["build-eips", "check", "--base-url", "http://localhost:4000"], &[ "build-eips", diff --git a/src/execution.rs b/src/execution.rs index 46d7c22..bffa6ac 100644 --- a/src/execution.rs +++ b/src/execution.rs @@ -732,7 +732,11 @@ base_url = "http://localhost:4000" #[test] fn non_site_commands_do_not_require_workspace_local_sources() { - for arguments in [&["build-eips", "changed"][..], &["build-eips", "clean"][..]] { + for arguments in [ + &["build-eips", "changed"][..], + &["build-eips", "clean"][..], + &["build-eips", "preview"][..], + ] { assert_settings( arguments, &["ERCs"], @@ -858,6 +862,7 @@ base_url = "http://localhost:4000" for arguments in [ &["build-eips", "changed"][..], &["build-eips", "clean"][..], + &["build-eips", "preview"][..], &["build-eips", "doctor"][..], &["build-eips", "print", "schema-version"][..], ] { diff --git a/src/main.rs b/src/main.rs index 6aee051..8e3eb51 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,6 +17,7 @@ mod layout; mod lint; mod markdown; mod pipeline; +mod preview; mod print; mod progress; mod serve; @@ -33,6 +34,7 @@ use snafu::{Report, ResultExt, Whatever}; use crate::{ cli::{Args, Operation, RuntimeOperation}, execution::{resolve_execution, validate_non_execution_command_flags}, + layout::output_path, pipeline::Prepared, workspace::{doctor_workspace, init_workspace}, }; @@ -87,6 +89,13 @@ fn run() -> Result<(), Whatever> { .runtime_operation() .expect("non-execution commands should have returned earlier"); let resolved = resolve_execution(&args)?; + + if matches!(runtime_operation, RuntimeOperation::Preview) { + preview::serve(&output_path(&resolved.build_path), &resolved.server_binding) + .whatever_context("preview server failed")?; + return Ok(()); + } + let build_path = make_build_dir(&resolved.build_path)?; let mut lock_file = lock(&build_path)?; @@ -111,6 +120,7 @@ fn run() -> Result<(), Whatever> { RuntimeOperation::Serve { eipw } => { Prepared::prepare(eipw, resolved)?.serve()?; } + RuntimeOperation::Preview => unreachable!(), RuntimeOperation::Changed { all, format } => { changed::run(&resolved, &build_path, all, &format)?; } diff --git a/src/preview.rs b/src/preview.rs new file mode 100644 index 0000000..4f91335 --- /dev/null +++ b/src/preview.rs @@ -0,0 +1,263 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +use std::{ + fs::File, + io::ErrorKind, + path::{Component, Path, PathBuf}, +}; + +use log::info; +use snafu::{ResultExt, Whatever}; +use tiny_http::{Header, Method, Request, Response, Server, StatusCode}; + +use crate::config::ServerBinding; + +const INDEX_HTML: &str = "index.html"; + +pub fn serve(output_path: &Path, server_binding: &ServerBinding) -> Result<(), Whatever> { + if !output_path.is_dir() { + snafu::whatever!( + "preview output directory `{}` is missing; run `build-eips build` for this profile first", + output_path.to_string_lossy() + ); + } + + let server = match Server::http((server_binding.host.as_str(), server_binding.port)) { + Ok(server) => server, + Err(error) => { + snafu::whatever!("unable to bind preview server on {server_binding}: {error}") + } + }; + + info!( + "serving static preview from `{}` at http://{server_binding}/", + output_path.to_string_lossy() + ); + + for request in server.incoming_requests() { + handle_request(output_path, request)?; + } + + Ok(()) +} + +fn handle_request(output_path: &Path, request: Request) -> Result<(), Whatever> { + match *request.method() { + Method::Get | Method::Head => {} + _ => { + request + .respond(Response::empty(StatusCode(405))) + .whatever_context("unable to send preview method error response")?; + return Ok(()); + } + } + + let Some(paths) = resolve_request_paths(output_path, request.url()) else { + request + .respond(Response::empty(StatusCode(400))) + .whatever_context("unable to send preview bad request response")?; + return Ok(()); + }; + + let Some((path, file)) = open_preview_asset(paths)? else { + request + .respond(Response::empty(StatusCode(404))) + .whatever_context("unable to send preview not found response")?; + return Ok(()); + }; + + let response = if let Some(value) = content_type(&path) { + Response::from_file(file).with_header(content_type_header(value)) + } else { + Response::from_file(file) + }; + + request + .respond(response) + .with_whatever_context(|e| format!("unable to send preview response: {e}"))?; + + Ok(()) +} + +fn open_preview_asset(paths: Vec) -> Result, Whatever> { + for path in paths { + let file = match File::open(&path) { + Ok(file) => file, + Err(error) + if matches!(error.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => + { + continue; + } + Err(error) => { + snafu::whatever!( + "unable to open preview asset `{}`: {error}", + path.to_string_lossy() + ); + } + }; + + if !file + .metadata() + .with_whatever_context(|e| { + format!( + "unable to inspect preview asset `{}`: {e}", + path.to_string_lossy() + ) + })? + .is_file() + { + continue; + } + + return Ok(Some((path, file))); + } + + Ok(None) +} + +fn resolve_request_paths(output_path: &Path, url: &str) -> Option> { + let raw_path = url.split('?').next().unwrap_or("/"); + let mut resolved = output_path.to_path_buf(); + let mut saw_normal_component = false; + + for component in Path::new(raw_path.trim_start_matches('/')).components() { + match component { + Component::CurDir | Component::RootDir => {} + Component::Normal(component) => { + saw_normal_component = true; + resolved.push(component); + } + Component::ParentDir | Component::Prefix(_) => return None, + } + } + + if raw_path.ends_with('/') || !saw_normal_component { + return Some(vec![resolved.join(INDEX_HTML)]); + } + + if resolved.extension().is_none() { + return Some(vec![resolved.join(INDEX_HTML), resolved]); + } + + Some(vec![resolved]) +} + +fn content_type(path: &Path) -> Option<&'static str> { + match path.extension().and_then(|extension| extension.to_str()) { + Some("css") => Some("text/css; charset=utf-8"), + Some("gif") => Some("image/gif"), + Some("htm" | "html") => Some("text/html; charset=utf-8"), + Some("ico") => Some("image/x-icon"), + Some("jpeg" | "jpg") => Some("image/jpeg"), + Some("js") => Some("application/javascript; charset=utf-8"), + Some("json") => Some("application/json; charset=utf-8"), + Some("mjs") => Some("application/javascript; charset=utf-8"), + Some("png") => Some("image/png"), + Some("svg") => Some("image/svg+xml"), + Some("txt") => Some("text/plain; charset=utf-8"), + Some("webp") => Some("image/webp"), + Some("xml") => Some("application/xml; charset=utf-8"), + _ => None, + } +} + +fn content_type_header(value: &str) -> Header { + Header::from_bytes(b"Content-Type", value.as_bytes()) + .expect("hard-coded content-type headers must be valid") +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use tempfile::TempDir; + + use super::{open_preview_asset, resolve_request_paths, INDEX_HTML}; + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn candidates(root: &Path, url: &str) -> Vec { + resolve_request_paths(root, url).unwrap() + } + + fn selected_path(root: &Path, url: &str) -> Option { + open_preview_asset(candidates(root, url)) + .unwrap() + .map(|(path, _file)| path) + } + + #[test] + fn slash_path_resolves_to_index_candidate() { + let temp = TempDir::new().unwrap(); + let root = temp.path(); + + assert_eq!( + candidates(root, "/foo/"), + vec![root.join("foo").join(INDEX_HTML)] + ); + } + + #[test] + fn extensionless_path_prefers_index_then_file_candidate() { + let temp = TempDir::new().unwrap(); + let root = temp.path(); + + assert_eq!( + candidates(root, "/foo"), + vec![root.join("foo").join(INDEX_HTML), root.join("foo")] + ); + } + + #[test] + fn extension_path_resolves_to_file_candidate() { + let temp = TempDir::new().unwrap(); + let root = temp.path(); + + assert_eq!(candidates(root, "/foo.css"), vec![root.join("foo.css")]); + } + + #[test] + fn parent_traversal_is_rejected() { + let temp = TempDir::new().unwrap(); + + assert!(resolve_request_paths(temp.path(), "/../secret.txt").is_none()); + } + + #[test] + fn extensionless_path_serves_index_when_present() { + let temp = TempDir::new().unwrap(); + let root = temp.path(); + write_file(root, "foo/index.html", "index"); + + assert_eq!( + selected_path(root, "/foo"), + Some(root.join("foo").join(INDEX_HTML)) + ); + } + + #[test] + fn extensionless_path_serves_file_without_index() { + let temp = TempDir::new().unwrap(); + let root = temp.path(); + write_file(root, "foo", "file"); + + assert_eq!(selected_path(root, "/foo"), Some(root.join("foo"))); + } + + #[test] + fn missing_path_returns_no_asset() { + let temp = TempDir::new().unwrap(); + + assert!(selected_path(temp.path(), "/missing").is_none()); + } +} From 4b766c7ed1b957f319873a80ac0d88ebe0034354 Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Sun, 3 May 2026 21:28:53 -0400 Subject: [PATCH 11/20] Add proposal selection foundation Add proposal number parsing, editorial selector classification, and content-path helpers for flat and directory proposal layouts. Add OnlyRenderPlan to index proposal content for selected rendering, derive EIP/ERC public URLs, choose internal versus external required references, filter/prune content, and gate dirty path sync. Keep this as internal foundation for editorial integration and targeted build/serve rendering; no user-facing --only config lands here. --- src/main.rs | 1 + src/proposal.rs | 1024 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1025 insertions(+) create mode 100644 src/proposal.rs diff --git a/src/main.rs b/src/main.rs index 8e3eb51..0b6e954 100644 --- a/src/main.rs +++ b/src/main.rs @@ -20,6 +20,7 @@ mod pipeline; mod preview; mod print; mod progress; +mod proposal; mod serve; mod workspace; mod zola; diff --git a/src/proposal.rs b/src/proposal.rs new file mode 100644 index 0000000..d30fb80 --- /dev/null +++ b/src/proposal.rs @@ -0,0 +1,1024 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Proposal path classification and targeted render policy. + +use std::{ + collections::{BTreeMap, BTreeSet}, + ffi::OsStr, + fmt, + num::NonZeroU32, + path::{Path, PathBuf}, +}; + +use eipw_preamble::Preamble; +use serde::{ + de::{self, Unexpected, Visitor}, + Deserialize, Deserializer, Serialize, Serializer, +}; +use snafu::{OptionExt, ResultExt, Whatever}; + +use crate::layout::CONTENT_DIR; + +/// Positive proposal number used by CLI selectors and `[render].only` config. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ProposalNumber(NonZeroU32); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ProposalNumberParseFailure { + Empty, + NonDigit, + Zero, + Overflow, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum EditorialNumberSelector { + Number(ProposalNumber), + InvalidNumberLike(ProposalNumberParseFailure), + PathLike, +} + +impl ProposalNumber { + fn parse_selector(selector: &str) -> Result { + if selector.is_empty() { + return Err(ProposalNumberParseFailure::Empty); + } + + if !selector.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(ProposalNumberParseFailure::NonDigit); + } + + let number = selector + .parse::() + .map_err(|_| ProposalNumberParseFailure::Overflow)?; + + Self::from_u32(number).map_err(|_| ProposalNumberParseFailure::Zero) + } + + pub(crate) fn parse_cli_selector(selector: &str) -> Result { + Self::parse_selector(selector).map_err(|_| { + format!( + "`{selector}` is not a valid --only selector; expected a positive proposal number" + ) + }) + } + + pub(crate) fn from_u32(number: u32) -> Result { + NonZeroU32::new(number).map(Self).ok_or(()) + } + + pub(crate) fn get(self) -> u32 { + self.0.get() + } +} + +pub(crate) fn classify_editorial_number_selector(selector: &str) -> EditorialNumberSelector { + match ProposalNumber::parse_selector(selector) { + Ok(number) => EditorialNumberSelector::Number(number), + Err(failure) if is_number_like_selector(selector) => { + EditorialNumberSelector::InvalidNumberLike(failure) + } + Err(_) => EditorialNumberSelector::PathLike, + } +} + +fn is_number_like_selector(selector: &str) -> bool { + !selector.is_empty() + && selector + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'+' | b'-' | b',')) +} + +impl fmt::Display for ProposalNumber { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}", self.get()) + } +} + +impl Serialize for ProposalNumber { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_u32(self.get()) + } +} + +impl<'de> Deserialize<'de> for ProposalNumber { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct ProposalNumberVisitor; + + impl Visitor<'_> for ProposalNumberVisitor { + type Value = ProposalNumber; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a positive proposal number") + } + + fn visit_i64(self, value: i64) -> Result + where + E: de::Error, + { + let value = u32::try_from(value).map_err(|_| { + E::invalid_value(Unexpected::Signed(value), &"a positive u32 proposal number") + })?; + ProposalNumber::from_u32(value).map_err(|_| { + E::invalid_value( + Unexpected::Unsigned(u64::from(value)), + &"a non-zero proposal number", + ) + }) + } + + fn visit_u64(self, value: u64) -> Result + where + E: de::Error, + { + let value = u32::try_from(value).map_err(|_| { + E::invalid_value( + Unexpected::Unsigned(value), + &"a positive u32 proposal number", + ) + })?; + ProposalNumber::from_u32(value).map_err(|_| { + E::invalid_value( + Unexpected::Unsigned(u64::from(value)), + &"a non-zero proposal number", + ) + }) + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + Err(E::invalid_type(Unexpected::Str(value), &self)) + } + } + + deserializer.deserialize_any(ProposalNumberVisitor) + } +} + +#[derive(Debug)] +pub(crate) enum ProposalReference<'a> { + Internal(String), + External(&'a str), +} + +#[derive(Debug, Clone)] +pub(crate) struct OnlyRenderPlan { + selected_numbers: BTreeSet, + canonical_proposal_numbers: BTreeMap, + markdown_paths_by_number: BTreeMap>, + public_urls_by_number: BTreeMap, +} + +impl OnlyRenderPlan { + pub(crate) fn build( + content_root: &Path, + selected_numbers: BTreeSet, + ) -> Result { + let mut plan = Self { + selected_numbers, + canonical_proposal_numbers: BTreeMap::new(), + markdown_paths_by_number: BTreeMap::new(), + public_urls_by_number: BTreeMap::new(), + }; + + let entries = std::fs::read_dir(content_root).with_whatever_context(|_| { + format!( + "unable to read materialized content directory `{}`", + content_root.to_string_lossy() + ) + })?; + + for entry in entries { + let entry = entry.with_whatever_context(|_| { + format!( + "unable to read materialized content directory entry in `{}`", + content_root.to_string_lossy() + ) + })?; + let entry_path = entry.path(); + let file_type = entry.file_type().with_whatever_context(|_| { + format!( + "unable to inspect materialized content path `{}`", + entry_path.to_string_lossy() + ) + })?; + + if file_type.is_file() { + let Some(number) = flat_proposal_number(&entry_path) else { + continue; + }; + plan.record_markdown_path(content_root, number, &entry_path)?; + } else if file_type.is_dir() { + let Some(number) = path_component_proposal_number(entry_path.file_name()) else { + continue; + }; + let index_path = entry_path.join("index.md"); + match std::fs::read_to_string(&index_path) { + Ok(contents) => { + plan.record_markdown_contents(content_root, number, &index_path, &contents)? + } + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => {} + Err(error) => { + snafu::whatever!( + "unable to read proposal markdown `{}`: {error}", + index_path.to_string_lossy() + ); + } + } + } + } + + for selected_number in &plan.selected_numbers { + if !plan.markdown_paths_by_number.contains_key(selected_number) { + snafu::whatever!("selected proposal `{selected_number}` was not found"); + } + } + + Ok(plan) + } + + fn record_markdown_path( + &mut self, + content_root: &Path, + proposal_number: ProposalNumber, + markdown_path: &Path, + ) -> Result<(), Whatever> { + let contents = std::fs::read_to_string(markdown_path).with_whatever_context(|_| { + format!( + "unable to read proposal markdown `{}`", + markdown_path.to_string_lossy() + ) + })?; + self.record_markdown_contents(content_root, proposal_number, markdown_path, &contents) + } + + fn record_markdown_contents( + &mut self, + content_root: &Path, + proposal_number: ProposalNumber, + markdown_path: &Path, + contents: &str, + ) -> Result<(), Whatever> { + let relative_path = markdown_path + .strip_prefix(content_root) + .with_whatever_context(|_| { + format!( + "proposal markdown `{}` is outside content root `{}`", + markdown_path.to_string_lossy(), + content_root.to_string_lossy() + ) + })? + .to_path_buf(); + let canonical_path = std::fs::canonicalize(markdown_path).with_whatever_context(|_| { + format!( + "unable to canonicalize proposal markdown `{}`", + markdown_path.to_string_lossy() + ) + })?; + let public_url = public_url_for_markdown(markdown_path, proposal_number, contents)?; + + match self.public_urls_by_number.get(&proposal_number) { + Some(existing_url) if existing_url != &public_url => { + snafu::whatever!( + "proposal `{proposal_number}` has conflicting public URLs `{existing_url}` and `{public_url}`" + ); + } + Some(_) => {} + None => { + self.public_urls_by_number + .insert(proposal_number, public_url); + } + } + + self.canonical_proposal_numbers + .insert(canonical_path, proposal_number); + self.markdown_paths_by_number + .entry(proposal_number) + .or_default() + .insert(relative_path); + + Ok(()) + } + + pub(crate) fn external_url_for_canonical_target( + &self, + canonical_target: &Path, + ) -> Option<&str> { + let proposal_number = self.canonical_proposal_numbers.get(canonical_target)?; + if self.selected_numbers.contains(proposal_number) { + return None; + } + + self.public_urls_by_number + .get(proposal_number) + .map(String::as_str) + } + + pub(crate) fn external_url_for_content_target( + &self, + content_relative_path: &Path, + ) -> Option<&str> { + let proposal_number = proposal_number_from_content_markdown_path(content_relative_path)?; + if self.selected_numbers.contains(&proposal_number) { + return None; + } + + self.public_urls_by_number + .get(&proposal_number) + .map(String::as_str) + } + + pub(crate) fn reference_for_required_number( + &self, + proposal_number: ProposalNumber, + ) -> Result, Whatever> { + if self.selected_numbers.contains(&proposal_number) { + let markdown_path = self + .markdown_paths_by_number + .get(&proposal_number) + .and_then(|paths| paths.iter().next()) + .with_whatever_context(|| { + format!("required selected proposal `{proposal_number}` was not found") + })?; + return Ok(ProposalReference::Internal(format!( + "@/{}", + markdown_path.to_string_lossy() + ))); + } + + let public_url = self + .public_urls_by_number + .get(&proposal_number) + .with_whatever_context(|| { + format!("required proposal `{proposal_number}` was not found") + })?; + Ok(ProposalReference::External(public_url)) + } + + pub(crate) fn should_preprocess_markdown(&self, content_relative_path: &Path) -> bool { + match proposal_number_from_content_markdown_path(content_relative_path) { + Some(proposal_number) => { + self.selected_numbers.contains(&proposal_number) + && self + .markdown_paths_by_number + .get(&proposal_number) + .map(|paths| paths.contains(content_relative_path)) + .unwrap_or(false) + } + None => true, + } + } + + pub(crate) fn should_process_proposal_dir(&self, content_relative_path: &Path) -> bool { + path_component_proposal_number(content_relative_path.file_name()) + .map(|proposal_number| self.selected_numbers.contains(&proposal_number)) + .unwrap_or(true) + } + + pub(crate) fn should_sync_dirty_path(&self, repo_relative_path: &Path) -> bool { + let Ok(content_relative_path) = repo_relative_path.strip_prefix(CONTENT_DIR) else { + return true; + }; + + self.should_sync_content_dirty_path(content_relative_path) + } + + pub(crate) fn is_selected_proposal_markdown_path(&self, repo_relative_path: &Path) -> bool { + let Ok(content_relative_path) = repo_relative_path.strip_prefix(CONTENT_DIR) else { + return false; + }; + + self.is_selected_content_proposal_markdown_path(content_relative_path) + } + + fn is_selected_content_proposal_markdown_path(&self, content_relative_path: &Path) -> bool { + let Some(proposal_number) = + proposal_number_from_content_markdown_path(content_relative_path) + else { + return false; + }; + + self.selected_numbers.contains(&proposal_number) + && self + .markdown_paths_by_number + .get(&proposal_number) + .map(|paths| paths.contains(content_relative_path)) + .unwrap_or(false) + } + + fn should_sync_content_dirty_path(&self, content_relative_path: &Path) -> bool { + if proposal_number_from_content_markdown_path(content_relative_path).is_some() { + return self.is_selected_content_proposal_markdown_path(content_relative_path); + } + + let mut components = content_relative_path.components(); + let Some(first) = components.next() else { + return true; + }; + + path_component_proposal_number(Some(first.as_os_str())) + .map(|proposal_number| self.selected_numbers.contains(&proposal_number)) + .unwrap_or(true) + } + + pub(crate) fn prune_content(&self, content_root: &Path) -> Result<(), Whatever> { + let entries = std::fs::read_dir(content_root).with_whatever_context(|_| { + format!( + "unable to read materialized content directory `{}` for pruning", + content_root.to_string_lossy() + ) + })?; + + for entry in entries { + let entry = entry.with_whatever_context(|_| { + format!( + "unable to read materialized content directory entry in `{}` for pruning", + content_root.to_string_lossy() + ) + })?; + let entry_path = entry.path(); + let file_type = entry.file_type().with_whatever_context(|_| { + format!( + "unable to inspect materialized content path `{}` for pruning", + entry_path.to_string_lossy() + ) + })?; + + if file_type.is_file() { + let Some(number) = flat_proposal_number(&entry_path) else { + continue; + }; + if !self.selected_numbers.contains(&number) { + remove_file_if_present(&entry_path)?; + } + } else if file_type.is_dir() { + let Some(number) = path_component_proposal_number(entry_path.file_name()) else { + continue; + }; + if !self.selected_numbers.contains(&number) { + remove_dir_if_present(&entry_path)?; + } + } + } + + Ok(()) + } +} + +fn remove_file_if_present(path: &Path) -> Result<(), Whatever> { + match std::fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => + { + Ok(()) + } + Err(error) => { + snafu::whatever!( + "unable to prune unselected proposal file `{}`: {error}", + path.to_string_lossy() + ); + } + } +} + +fn remove_dir_if_present(path: &Path) -> Result<(), Whatever> { + match std::fs::remove_dir_all(path) { + Ok(()) => Ok(()), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => + { + Ok(()) + } + Err(error) => { + snafu::whatever!( + "unable to prune unselected proposal directory `{}`: {error}", + path.to_string_lossy() + ); + } + } +} + +fn public_url_for_markdown( + markdown_path: &Path, + proposal_number: ProposalNumber, + contents: &str, +) -> Result { + let path_lossy = markdown_path.to_string_lossy(); + let (preamble, _) = Preamble::split(contents) + .with_whatever_context(|_| format!("couldn't split preamble for `{path_lossy}`"))?; + let preamble = Preamble::parse(Some(&path_lossy), preamble) + .ok() + .with_whatever_context(|| format!("couldn't parse preamble in `{path_lossy}`"))?; + let is_erc = preamble + .fields() + .any(|field| field.name() == "category" && field.value().trim() == "ERC"); + + if is_erc { + Ok(format!( + "https://ercs.ethereum.org/ERCS/erc-{}", + proposal_number.get() + )) + } else { + Ok(format!( + "https://eips.ethereum.org/EIPS/eip-{}", + proposal_number.get() + )) + } +} + +fn flat_proposal_number(path: &Path) -> Option { + if path.extension().and_then(OsStr::to_str) != Some("md") { + return None; + } + + path_component_proposal_number(path.file_stem()) +} + +fn path_component_proposal_number(component: Option<&OsStr>) -> Option { + let name = component?.to_str()?; + if name.is_empty() || !name.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + + name.parse::() + .ok() + .and_then(|number| NonZeroU32::new(number).map(ProposalNumber)) +} + +pub(crate) fn proposal_number_from_content_markdown_path( + content_relative_path: &Path, +) -> Option { + let mut components = content_relative_path.components(); + let first = components.next()?; + let first_path = Path::new(first.as_os_str()); + + match components.next() { + None => flat_proposal_number(first_path), + Some(component) + if component.as_os_str() == OsStr::new("index.md") && components.next().is_none() => + { + path_component_proposal_number(Some(first.as_os_str())) + } + Some(_) => None, + } +} + +pub(crate) fn is_proposal_path(path: &Path) -> bool { + let Ok(content_relative_path) = path.strip_prefix(CONTENT_DIR) else { + return false; + }; + + proposal_number_from_content_markdown_path(content_relative_path).is_some() +} + +pub(crate) fn resolve_proposal_number_markdown_path( + active_repo_root: &Path, + proposal_number: ProposalNumber, +) -> Result { + let content_root = active_repo_root.join(CONTENT_DIR); + let mut matches = BTreeSet::new(); + let entries = std::fs::read_dir(&content_root).with_whatever_context(|_| { + format!( + "unable to read active repository content directory `{}`", + content_root.to_string_lossy() + ) + })?; + + for entry in entries { + let entry = entry.with_whatever_context(|_| { + format!( + "unable to read active repository content directory entry in `{}`", + content_root.to_string_lossy() + ) + })?; + let entry_path = entry.path(); + let file_type = entry.file_type().with_whatever_context(|_| { + format!( + "unable to inspect active repository content path `{}`", + entry_path.to_string_lossy() + ) + })?; + + if file_type.is_file() { + if flat_proposal_number(&entry_path) == Some(proposal_number) { + matches.insert(PathBuf::from(CONTENT_DIR).join(entry.file_name())); + } + } else if file_type.is_dir() + && path_component_proposal_number(Some(entry.file_name().as_os_str())) + == Some(proposal_number) + { + let index_path = entry_path.join("index.md"); + match std::fs::metadata(&index_path) { + Ok(metadata) if metadata.is_file() => { + matches.insert( + PathBuf::from(CONTENT_DIR) + .join(entry.file_name()) + .join("index.md"), + ); + } + Ok(_) => {} + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => {} + Err(error) => { + snafu::whatever!( + "unable to inspect proposal markdown `{}`: {error}", + index_path.to_string_lossy() + ); + } + } + } + } + + match matches.len() { + 0 => snafu::whatever!( + "proposal `{proposal_number}` was not found in active repository content" + ), + 1 => Ok(matches.into_iter().next().expect("one proposal path")), + _ => { + let paths = matches + .iter() + .map(|path| format!("`{}`", path.to_string_lossy())) + .collect::>() + .join(", "); + snafu::whatever!( + "proposal `{proposal_number}` has more than one markdown path in active repository content: {paths}" + ); + } + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use tempfile::TempDir; + + use super::{ + classify_editorial_number_selector, is_proposal_path, + proposal_number_from_content_markdown_path, resolve_proposal_number_markdown_path, + EditorialNumberSelector, OnlyRenderPlan, ProposalNumber, ProposalNumberParseFailure, + }; + + fn number(value: u32) -> ProposalNumber { + ProposalNumber::from_u32(value).unwrap() + } + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn proposal_markdown(number: u32, category: Option<&str>) -> String { + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!("---\neip: {number}\ntitle: Test\n{category}---\nBody\n") + } + + #[test] + fn proposal_numbers_parse_cli_selectors_strictly() { + assert_eq!( + ProposalNumber::parse_cli_selector("555").unwrap(), + number(555) + ); + assert_eq!( + ProposalNumber::parse_cli_selector("00555").unwrap(), + number(555) + ); + + for selector in ["+555", "0", "-555", "abc", "555,678", "content/00555.md"] { + let error = ProposalNumber::parse_cli_selector(selector).unwrap_err(); + assert_eq!( + error, + format!( + "`{selector}` is not a valid --only selector; expected a positive proposal number" + ) + ); + } + + let error = ProposalNumber::parse_cli_selector("4294967296").unwrap_err(); + assert_eq!( + error, + "`4294967296` is not a valid --only selector; expected a positive proposal number" + ); + } + + #[test] + fn editorial_number_selector_classifier_splits_numbers_invalid_numbers_and_paths() { + assert_eq!( + classify_editorial_number_selector("000555"), + EditorialNumberSelector::Number(number(555)) + ); + + for (selector, expected_failure) in [ + ("0", ProposalNumberParseFailure::Zero), + ("+555", ProposalNumberParseFailure::NonDigit), + ("-555", ProposalNumberParseFailure::NonDigit), + ("555,678", ProposalNumberParseFailure::NonDigit), + ("4294967296", ProposalNumberParseFailure::Overflow), + ] { + assert_eq!( + classify_editorial_number_selector(selector), + EditorialNumberSelector::InvalidNumberLike(expected_failure) + ); + } + + for selector in ["foo", "draft.md", "4a", "draft-4.md", "content/00555.md"] { + assert_eq!( + classify_editorial_number_selector(selector), + EditorialNumberSelector::PathLike + ); + } + } + + #[test] + fn proposal_path_matching_normalizes_numeric_paths() { + assert_eq!( + proposal_number_from_content_markdown_path(Path::new("555.md")), + Some(number(555)) + ); + assert_eq!( + proposal_number_from_content_markdown_path(Path::new("00555.md")), + Some(number(555)) + ); + assert_eq!( + proposal_number_from_content_markdown_path(Path::new("000555/index.md")), + Some(number(555)) + ); + assert!(is_proposal_path(Path::new("content/000555/index.md"))); + assert!(!is_proposal_path(Path::new( + "content/000555/assets/readme.md" + ))); + } + + #[test] + fn proposal_number_resolver_returns_exact_flat_markdown_path() { + for (selector, existing_path) in [ + ("4", "content/4.md"), + ("004", "content/0004.md"), + ("0004", "content/004.md"), + ] { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), existing_path, ""); + + assert_eq!( + resolve_proposal_number_markdown_path( + temp.path(), + ProposalNumber::parse_cli_selector(selector).unwrap(), + ) + .unwrap(), + Path::new(existing_path) + ); + } + } + + #[test] + fn proposal_number_resolver_returns_exact_directory_index_path() { + for (selector, existing_path) in [ + ("4", "content/4/index.md"), + ("004", "content/0004/index.md"), + ("0004", "content/004/index.md"), + ] { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), existing_path, ""); + + assert_eq!( + resolve_proposal_number_markdown_path( + temp.path(), + ProposalNumber::parse_cli_selector(selector).unwrap(), + ) + .unwrap(), + Path::new(existing_path) + ); + } + } + + #[test] + fn proposal_number_resolver_reports_missing_and_ignores_assets_only_dirs() { + let missing = TempDir::new().unwrap(); + write_file(missing.path(), "content/0005.md", ""); + let error = resolve_proposal_number_markdown_path(missing.path(), number(4)) + .unwrap_err() + .to_string(); + assert!(error.contains("proposal `4` was not found in active repository content")); + + let assets_only = TempDir::new().unwrap(); + write_file(assets_only.path(), "content/0004/assets/foo.png", ""); + let error = resolve_proposal_number_markdown_path(assets_only.path(), number(4)) + .unwrap_err() + .to_string(); + assert!(error.contains("proposal `4` was not found in active repository content")); + } + + #[test] + fn proposal_number_resolver_reports_ambiguous_markdown_paths() { + for paths in [ + &["content/4.md", "content/0004/index.md"][..], + &["content/4.md", "content/0004.md"][..], + &["content/4/index.md", "content/0004/index.md"][..], + ] { + let temp = TempDir::new().unwrap(); + for path in paths { + write_file(temp.path(), path, ""); + } + + let error = resolve_proposal_number_markdown_path(temp.path(), number(4)) + .unwrap_err() + .to_string(); + + assert!(error.contains( + "proposal `4` has more than one markdown path in active repository content" + )); + } + } + + #[test] + fn proposal_number_resolver_searches_only_active_repo_content() { + let temp = TempDir::new().unwrap(); + let active_repo = temp.path().join("active"); + let sibling_repo = temp.path().join("sibling"); + write_file(&active_repo, "content/0005.md", ""); + write_file(&sibling_repo, "content/0004.md", ""); + + let error = resolve_proposal_number_markdown_path(&active_repo, number(4)) + .unwrap_err() + .to_string(); + + assert!(error.contains("proposal `4` was not found in active repository content")); + } + + #[test] + fn only_render_plan_requires_selected_markdown_not_assets_only() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555/assets/foo.png", ""); + + let error = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()) + .unwrap_err() + .to_string(); + + assert!(error.contains("selected proposal `555` was not found")); + } + + #[test] + fn only_render_plan_reports_missing_selected_proposal() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + + let error = OnlyRenderPlan::build(content, [number(678)].into_iter().collect()) + .unwrap_err() + .to_string(); + + assert!(error.contains("selected proposal `678` was not found")); + } + + #[test] + fn only_render_plan_records_exact_public_urls_by_category() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", &proposal_markdown(678, Some("ERC"))); + write_file( + content, + "00777.md", + &proposal_markdown(777, Some("Standards Track")), + ); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert_eq!( + plan.public_urls_by_number.get(&number(555)).unwrap(), + "https://eips.ethereum.org/EIPS/eip-555" + ); + assert_eq!( + plan.public_urls_by_number.get(&number(678)).unwrap(), + "https://ercs.ethereum.org/ERCS/erc-678" + ); + assert_eq!( + plan.public_urls_by_number.get(&number(777)).unwrap(), + "https://eips.ethereum.org/EIPS/eip-777" + ); + } + + #[test] + fn only_render_plan_does_not_mask_missing_required_targets() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + let error = plan + .reference_for_required_number(number(678)) + .unwrap_err() + .to_string(); + + assert!(error.contains("required proposal `678` was not found")); + } + + #[test] + fn only_render_plan_does_not_mask_malformed_target_front_matter() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", "not front matter"); + + let error = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()) + .unwrap_err() + .to_string(); + + assert!(error.contains("couldn't split preamble")); + } + + #[test] + fn only_render_plan_detects_conflicting_public_urls() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file( + content, + "00555/index.md", + &proposal_markdown(555, Some("ERC")), + ); + + let error = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()) + .unwrap_err() + .to_string(); + + assert!(error.contains("conflicting public URLs")); + } + + #[test] + fn only_render_plan_prunes_unselected_proposal_content_only() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00555/assets/foo.png", ""); + write_file(content, "00678.md", &proposal_markdown(678, None)); + write_file(content, "00777/index.md", &proposal_markdown(777, None)); + write_file(content, "00777/assets/foo.png", ""); + write_file(content, "_index.md", "+++\ntitle = \"Home\"\n+++\n"); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + plan.prune_content(content).unwrap(); + + assert!(content.join("00555.md").is_file()); + assert!(content.join("00555/assets/foo.png").is_file()); + assert!(!content.join("00678.md").exists()); + assert!(!content.join("00777").exists()); + assert!(content.join("_index.md").is_file()); + } + + #[test] + fn only_render_plan_filters_dirty_paths_without_filesystem_state() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", &proposal_markdown(678, None)); + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert!(plan.should_sync_dirty_path(Path::new("content/00555.md"))); + assert!(plan.should_sync_dirty_path(Path::new("content/00555/assets/diagram.png"))); + assert!(plan.should_sync_dirty_path(Path::new("content/_index.md"))); + assert!(plan.should_sync_dirty_path(Path::new(".build-eips.repo.toml"))); + assert!(!plan.should_sync_dirty_path(Path::new("content/00678.md"))); + assert!(!plan.should_sync_dirty_path(Path::new("content/00678/assets/diagram.png"))); + assert!(!plan.should_sync_dirty_path(Path::new("content/00999.md"))); + + assert!(plan.is_selected_proposal_markdown_path(Path::new("content/00555.md"))); + assert!( + !plan.is_selected_proposal_markdown_path(Path::new("content/00555/assets/diagram.png")) + ); + assert!(!plan.is_selected_proposal_markdown_path(Path::new("content/_index.md"))); + assert!(!plan.is_selected_proposal_markdown_path(Path::new("content/00678.md"))); + } +} From a4577b6e1fed4d5b5cbbe1126dd7e897618530a4 Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Sun, 3 May 2026 21:35:02 -0400 Subject: [PATCH 12/20] Add editorial command integration Add `build-eips editorial lint` and `build-eips editorial check` as the first user-facing proposal-selection commands. Keep eipw options scoped to editorial lint/check. Normal build, check, and serve prepare runtime sources, preprocess markdown, and run Zola without carrying eipw source-selection flags. Run editorial-selected eipw lint against the prepared merged source tree so cross-repo EIP/ERC references resolve through the same content layout used by local builds. Prepare runtime sources from the local active checkout, merge sibling repositories, and keep active-upstream fetches in changed-file comparison and editorial `--against-upstream` target selection. --- src/cli.rs | 180 +++++++-- src/editorial.rs | 926 +++++++++++++++++++++++++++++++++++++++++++++++ src/execution.rs | 130 ++++++- src/find_root.rs | 2 +- src/git.rs | 313 ++++++++-------- src/lint.rs | 51 +-- src/main.rs | 25 +- src/pipeline.rs | 274 ++++++++++++-- 8 files changed, 1632 insertions(+), 269 deletions(-) create mode 100644 src/editorial.rs diff --git a/src/cli.rs b/src/cli.rs index 2b55f6d..c4af61f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -76,9 +76,6 @@ pub(crate) enum Operation { /// Build the project and output HTML Build { - #[command(flatten)] - eipw: lint::CmdArgs, - #[command(flatten)] base_url: BaseUrlCliArgs, @@ -94,9 +91,6 @@ pub(crate) enum Operation { /// Build a fresh temporary site, serve it locally, and watch tracked edits Serve { - #[command(flatten)] - eipw: lint::CmdArgs, - #[command(flatten)] server: ServerCliArgs, @@ -112,9 +106,6 @@ pub(crate) enum Operation { /// Validate that the site builds cleanly without writing HTML output Check { - #[command(flatten)] - eipw: lint::CmdArgs, - #[command(flatten)] clean: CleanCliArgs, }, @@ -128,6 +119,12 @@ pub(crate) enum Operation { format: ChangedFormat, }, + /// Run targeted editorial lint or check workflows + Editorial { + #[command(subcommand)] + command: EditorialCommand, + }, + /// Create workspace config, docs, build root, and missing local repos Init { /// Workspace root directory @@ -152,18 +149,12 @@ pub(crate) enum Operation { pub(crate) enum ProfiledOperation { /// Build the project and output HTML Build { - #[command(flatten)] - eipw: lint::CmdArgs, - #[command(flatten)] base_url: BaseUrlCliArgs, }, /// Build the project and launch a web server to preview it Serve { - #[command(flatten)] - eipw: lint::CmdArgs, - #[command(flatten)] server: ServerCliArgs, @@ -172,12 +163,49 @@ pub(crate) enum ProfiledOperation { }, /// Validate that the site builds cleanly without writing HTML output + Check, +} + +#[derive(Debug, Subcommand, Clone)] +pub(crate) enum EditorialCommand { + /// Run eipw lint checks on selected proposal files + Lint { + #[command(flatten)] + selectors: EditorialSelectorArgs, + + #[command(flatten)] + eipw: lint::CmdArgs, + }, + + /// Run eipw lint checks, then validate the site build Check { + #[command(flatten)] + selectors: EditorialSelectorArgs, + #[command(flatten)] eipw: lint::CmdArgs, }, } +#[derive(Debug, clap::Args, Clone)] +pub(crate) struct EditorialSelectorArgs { + /// Proposal number(s) or repo-relative proposal path(s), such as `4` or `content/07949.md` + #[arg(value_name = "TARGET")] + pub(crate) paths: Vec, + + /// Read proposal numbers or repo-relative proposal paths from BATCH, one per line + #[arg(long)] + pub(crate) batch: Option, + + /// Select tracked dirty proposal files from the active content repo + #[arg(long)] + pub(crate) working_tree: bool, + + /// Select proposal files changed versus the upstream merge-base + #[arg(long)] + pub(crate) against_upstream: bool, +} + #[derive(Debug, clap::ValueEnum, Clone, Default)] pub(crate) enum ChangedFormat { #[default] @@ -188,12 +216,13 @@ pub(crate) enum ChangedFormat { #[derive(Debug, Clone)] pub(crate) enum RuntimeOperation { - Build { eipw: lint::CmdArgs }, - Serve { eipw: lint::CmdArgs }, + Build, + Serve, Preview, Clean, - Check { eipw: lint::CmdArgs }, + Check, Changed { all: bool, format: ChangedFormat }, + Editorial { command: EditorialCommand }, } impl Operation { @@ -206,6 +235,7 @@ impl Operation { | Self::Clean | Self::Check { .. } | Self::Changed { .. } + | Self::Editorial { .. } | Self::Init { .. } | Self::Doctor => ServerCliArgs::default(), } @@ -220,6 +250,7 @@ impl Operation { | Self::Clean | Self::Check { .. } | Self::Changed { .. } + | Self::Editorial { .. } | Self::Init { .. } | Self::Doctor => BaseUrlCliArgs::default(), } @@ -234,6 +265,7 @@ impl Operation { | Self::Preview { .. } | Self::Clean | Self::Changed { .. } + | Self::Editorial { .. } | Self::Init { .. } | Self::Doctor | Self::Parity { .. } => CleanCliArgs::default(), @@ -247,18 +279,25 @@ impl Operation { ) } + pub(crate) fn is_editorial_command(&self) -> bool { + matches!(self, Self::Editorial { .. }) + } + pub(crate) fn runtime_operation(&self) -> Option { match self { Self::Print { .. } | Self::Init { .. } | Self::Doctor => None, - Self::Build { eipw, .. } => Some(RuntimeOperation::Build { eipw: eipw.clone() }), - Self::Serve { eipw, .. } => Some(RuntimeOperation::Serve { eipw: eipw.clone() }), + Self::Build { .. } => Some(RuntimeOperation::Build), + Self::Serve { .. } => Some(RuntimeOperation::Serve), Self::Preview { .. } => Some(RuntimeOperation::Preview), Self::Clean => Some(RuntimeOperation::Clean), - Self::Check { eipw, .. } => Some(RuntimeOperation::Check { eipw: eipw.clone() }), + Self::Check { .. } => Some(RuntimeOperation::Check), Self::Changed { all, format } => Some(RuntimeOperation::Changed { all: *all, format: format.clone(), }), + Self::Editorial { command } => Some(RuntimeOperation::Editorial { + command: command.clone(), + }), Self::Parity { command } => Some(command.runtime_operation()), } } @@ -289,9 +328,9 @@ impl ProfiledOperation { fn runtime_operation(&self) -> RuntimeOperation { match self { - Self::Build { eipw, .. } => RuntimeOperation::Build { eipw: eipw.clone() }, - Self::Serve { eipw, .. } => RuntimeOperation::Serve { eipw: eipw.clone() }, - Self::Check { eipw } => RuntimeOperation::Check { eipw: eipw.clone() }, + Self::Build { .. } => RuntimeOperation::Build, + Self::Serve { .. } => RuntimeOperation::Serve, + Self::Check => RuntimeOperation::Check, } } } @@ -330,11 +369,20 @@ impl ChangedFormat { } } +impl EditorialSelectorArgs { + pub(crate) fn selector_count(&self) -> usize { + usize::from(!self.paths.is_empty()) + + usize::from(self.batch.is_some()) + + usize::from(self.working_tree) + + usize::from(self.against_upstream) + } +} + #[cfg(test)] mod tests { use clap::Parser; - use super::{Args, Operation, ProfiledOperation, RuntimeOperation}; + use super::{Args, EditorialCommand, Operation, ProfiledOperation, RuntimeOperation}; fn parse_args(arguments: &[&str]) -> Args { Args::try_parse_from(arguments).unwrap() @@ -413,8 +461,7 @@ mod tests { args.operation.runtime_operation().unwrap(), *expected_runtime_operation ), - (RuntimeOperation::Build { .. }, "build") - | (RuntimeOperation::Serve { .. }, "serve") + (RuntimeOperation::Build, "build") | (RuntimeOperation::Serve, "serve") )); assert_eq!( args.operation @@ -487,7 +534,7 @@ mod tests { let args = parse_args(arguments); let runtime_operation = args.operation.runtime_operation().unwrap(); match runtime_operation { - RuntimeOperation::Serve { .. } if *expect_serve => {} + RuntimeOperation::Serve if *expect_serve => {} RuntimeOperation::Preview if !*expect_serve => {} other => panic!("unexpected runtime operation: {other:?}"), } @@ -508,6 +555,7 @@ mod tests { &["build-eips", "--remote-sibling-repo", "build"][..], &["build-eips", "workspace", "init", "/tmp/workspace"][..], &["build-eips", "workspace", "doctor"][..], + &["build-eips", "editorial", "build", "1"][..], &["build-eips", "parity", "preview"][..], &["build-eips", "parity", "clean"][..], &["build-eips", "parity", "changed"][..], @@ -516,6 +564,62 @@ mod tests { } } + #[test] + fn eipw_flags_parse_only_on_editorial_commands() { + let command_prefixes: &[&[&str]] = &[ + &["build-eips", "build"], + &["build-eips", "check"], + &["build-eips", "serve"], + &["build-eips", "parity", "build"], + &["build-eips", "parity", "check"], + &["build-eips", "parity", "serve"], + ]; + let eipw_flags: &[&[&str]] = &[ + &["--no-default-lints"], + &["-D", "markdown-refs"], + &["--deny", "markdown-refs"], + &["-W", "markdown-link-status"], + &["--warn", "markdown-link-status"], + &["-A", "preamble-required"], + &["--allow", "preamble-required"], + ]; + + for command_prefix in command_prefixes { + for eipw_flag in eipw_flags { + let arguments = command_prefix + .iter() + .chain(eipw_flag.iter()) + .copied() + .collect::>(); + assert!( + Args::try_parse_from(arguments.clone()).is_err(), + "expected {arguments:?} to reject eipw flags" + ); + } + } + + for command in ["lint", "check"] { + let args = parse_args(&[ + "build-eips", + "editorial", + command, + "content/00001.md", + "--no-default-lints", + "-D", + "markdown-refs", + "--warn", + "markdown-link-status", + "--allow", + "preamble-required", + ]); + + assert!(matches!( + args.operation.runtime_operation(), + Some(RuntimeOperation::Editorial { .. }) + )); + } + } + #[test] fn base_url_flag_is_rejected_on_non_rendering_forms() { let cases: &[&[&str]] = &[ @@ -552,6 +656,14 @@ mod tests { "--base-url", "http://localhost:4000", ], + &[ + "build-eips", + "editorial", + "lint", + "--working-tree", + "--base-url", + "http://localhost:4000", + ], &["build-eips", "print", "--base-url", "http://localhost:4000"], ]; @@ -580,6 +692,18 @@ mod tests { assert!(matches!(doctor.operation, Operation::Doctor)); } + #[test] + fn editorial_check_parses_as_runtime_editorial_command() { + let args = parse_args(&["build-eips", "editorial", "check", "--working-tree"]); + + assert!(matches!( + args.operation.runtime_operation(), + Some(RuntimeOperation::Editorial { + command: EditorialCommand::Check { .. } + }) + )); + } + #[test] fn remote_siblings_flag_parses() { let args = parse_args(&["build-eips", "--remote-siblings", "build"]); diff --git a/src/editorial.rs b/src/editorial.rs new file mode 100644 index 0000000..33f8f10 --- /dev/null +++ b/src/editorial.rs @@ -0,0 +1,926 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Editorial target selection and runtime helpers. + +use std::{ + collections::BTreeSet, + fs::File, + io::ErrorKind, + path::{Path, PathBuf}, +}; + +use log::info; +use snafu::{OptionExt, ResultExt, Whatever}; + +use crate::{ + cli::EditorialSelectorArgs, + context::resolve_input_path, + execution::ResolvedExecution, + git, + layout::REPO_DIR, + lint, + proposal::{ + classify_editorial_number_selector, is_proposal_path, + resolve_proposal_number_markdown_path, EditorialNumberSelector, + }, +}; + +fn repo_relative_canonical_path( + root_path: &Path, + path: &Path, + canonical_path: &Path, +) -> Result { + if path.is_absolute() { + snafu::whatever!( + "editorial selectors require repo-relative proposal paths, got `{}`", + path.to_string_lossy() + ); + } + + let relative = canonical_path + .strip_prefix(root_path) + .with_whatever_context(|_| { + format!( + "editorial target `{}` escapes the active repository root", + path.to_string_lossy() + ) + })? + .to_path_buf(); + + Ok(relative) +} + +fn validate_editorial_targets( + root_path: &Path, + paths: Vec, + strict: bool, +) -> Result, Whatever> { + let mut unique = BTreeSet::new(); + let mut targets = Vec::new(); + + for path in paths { + if path.is_absolute() { + snafu::whatever!( + "editorial selectors require repo-relative proposal paths, got `{}`", + path.to_string_lossy() + ); + } + + let full_path = root_path.join(&path); + let canonical_path = match full_path.canonicalize() { + Ok(canonical_path) => canonical_path, + Err(error) + if !strict + && matches!(error.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => + { + continue; + } + Err(error) => { + return Err(error).with_whatever_context(|_| { + format!( + "unable to resolve editorial target `{}`", + full_path.to_string_lossy() + ) + }); + } + }; + + let relative = repo_relative_canonical_path(root_path, &path, &canonical_path)?; + + if !is_proposal_path(&relative) { + if strict { + snafu::whatever!( + "editorial target `{}` is not a supported proposal path", + relative.to_string_lossy() + ); + } + continue; + } + + if unique.insert(relative.clone()) { + targets.push(relative); + } + } + + if strict && targets.is_empty() { + snafu::whatever!("editorial selector resolved no proposal files"); + } + + Ok(targets) +} + +fn read_editorial_batch(path: &Path) -> Result, Whatever> { + let contents = + std::fs::read_to_string(path).whatever_context("unable to read editorial batch file")?; + let mut paths = Vec::new(); + + for line in contents.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + paths.push(PathBuf::from(line)); + } + + Ok(paths) +} + +fn normalize_editorial_selector(root_path: &Path, path: PathBuf) -> Result { + let Some(selector) = path.as_os_str().to_str() else { + return Ok(path); + }; + + match classify_editorial_number_selector(selector) { + EditorialNumberSelector::Number(proposal_number) => { + resolve_proposal_number_markdown_path(root_path, proposal_number) + } + EditorialNumberSelector::InvalidNumberLike(_failure) => { + snafu::whatever!( + "editorial number selector `{selector}` is invalid; expected a positive proposal number that fits in u32, without signs or commas" + ); + } + EditorialNumberSelector::PathLike => Ok(path), + } +} + +fn normalize_editorial_selectors( + root_path: &Path, + paths: Vec, +) -> Result, Whatever> { + paths + .into_iter() + .map(|path| normalize_editorial_selector(root_path, path)) + .collect::>() +} + +fn prepare_editorial_lint_source( + resolved: &ResolvedExecution, +) -> Result<(PathBuf, git::SourceOnly), Whatever> { + let repo_path = resolved.build_path.join(REPO_DIR); + let source = git::Fresh::new( + &resolved.root_path, + &repo_path, + resolved.repository_use.clone(), + resolved.source_materialization, + ) + .whatever_context("initializing build repo for editorial source preparation")? + .clone_src() + .whatever_context("cloning source repo for editorial source preparation")?; + + Ok((repo_path, source)) +} + +fn prepare_editorial_lint_source_with_upstream( + resolved: &ResolvedExecution, +) -> Result<(PathBuf, git::SourceWithUpstream), Whatever> { + let (repo_path, source) = prepare_editorial_lint_source(resolved)?; + let source = source + .fetch_upstream() + .whatever_context("fetching upstream repo for editorial source preparation")?; + + Ok((repo_path, source)) +} + +fn raw_editorial_targets( + selectors: &EditorialSelectorArgs, + resolved: &ResolvedExecution, + upstream_source: Option<&git::SourceWithUpstream>, +) -> Result, Whatever> { + if selectors.selector_count() != 1 { + snafu::whatever!( + "choose exactly one editorial selector: explicit proposal targets, `--batch`, `--working-tree`, or `--against-upstream`" + ); + } + + let raw_targets = if !selectors.paths.is_empty() { + selectors.paths.clone() + } else if let Some(batch) = selectors.batch.as_deref() { + let batch = resolve_input_path(batch)?; + read_editorial_batch(&batch)? + } else if selectors.working_tree { + git::working_tree_paths(&resolved.root_path) + .whatever_context("unable to resolve working-tree editorial targets")? + } else { + upstream_source + .whatever_context( + "against-upstream editorial target selection requires upstream source", + )? + .changed_files() + .whatever_context("unable to list editorial targets against upstream")? + }; + + Ok(raw_targets) +} + +fn validate_raw_editorial_targets( + selectors: &EditorialSelectorArgs, + resolved: &ResolvedExecution, + raw_targets: Vec, +) -> Result, Whatever> { + let strict = !selectors.paths.is_empty() || selectors.batch.is_some(); + let targets = if strict { + normalize_editorial_selectors(&resolved.root_path, raw_targets)? + } else { + raw_targets + }; + validate_editorial_targets(&resolved.root_path, targets, strict) +} + +fn editorial_targets_from_source( + selectors: &EditorialSelectorArgs, + resolved: &ResolvedExecution, + upstream_source: Option<&git::SourceWithUpstream>, +) -> Result, Whatever> { + let raw_targets = raw_editorial_targets(selectors, resolved, upstream_source)?; + + validate_raw_editorial_targets(selectors, resolved, raw_targets) +} + +pub(crate) fn editorial_targets( + selectors: &EditorialSelectorArgs, + resolved: &ResolvedExecution, +) -> Result, Whatever> { + editorial_targets_from_source(selectors, resolved, None) +} + +fn validate_prepared_editorial_targets( + prepared_repo_path: &Path, + targets: &[PathBuf], +) -> Result<(), Whatever> { + for target in targets { + if target.is_absolute() { + snafu::whatever!( + "editorial selectors require repo-relative proposal paths, got `{}`", + target.to_string_lossy() + ); + } + + let prepared_target = prepared_repo_path.join(target); + let file = match File::open(&prepared_target) { + Ok(file) => file, + Err(error) + if matches!(error.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => + { + snafu::whatever!( + "editorial target `{}` exists in the active repo but was not materialized into the prepared source tree; untracked files are not supported", + target.to_string_lossy() + ); + } + Err(error) => { + return Err(error).with_whatever_context(|_| { + format!( + "unable to open prepared editorial target `{}`", + prepared_target.to_string_lossy() + ) + }); + } + }; + let metadata = file.metadata().with_whatever_context(|_| { + format!( + "unable to inspect prepared editorial target `{}`", + prepared_target.to_string_lossy() + ) + })?; + + if !metadata.is_file() { + snafu::whatever!( + "prepared editorial target `{}` is not a file", + target.to_string_lossy() + ); + } + } + + Ok(()) +} + +pub(crate) fn run_editorial_lint( + resolved: &ResolvedExecution, + selectors: &EditorialSelectorArgs, + eipw: lint::CmdArgs, +) -> Result { + if selectors.against_upstream { + let (repo_path, source) = prepare_editorial_lint_source_with_upstream(resolved)?; + let targets = editorial_targets_from_source(selectors, resolved, Some(&source))?; + if targets.is_empty() { + info!("editorial selector resolved no proposal files; skipping editorial lint"); + return Ok(false); + } + source + .merge() + .whatever_context("unable to merge ERC/EIP repositories for editorial lint")?; + validate_prepared_editorial_targets(&repo_path, &targets)?; + + lint::eipw(resolved.theme_path()?, &repo_path, targets, eipw) + .whatever_context("editorial lint failed")?; + + return Ok(true); + } + + let targets = editorial_targets(selectors, resolved)?; + if targets.is_empty() { + info!("editorial selector resolved no proposal files; skipping editorial lint"); + return Ok(false); + } + + let (repo_path, source) = prepare_editorial_lint_source(resolved)?; + source + .merge() + .whatever_context("unable to merge ERC/EIP repositories for editorial lint")?; + validate_prepared_editorial_targets(&repo_path, &targets)?; + + lint::eipw(resolved.theme_path()?, &repo_path, targets, eipw) + .whatever_context("editorial lint failed")?; + + Ok(true) +} + +pub(crate) fn editorial_runtime_execution( + mut resolved: ResolvedExecution, + selectors: &EditorialSelectorArgs, +) -> ResolvedExecution { + if selectors.working_tree { + resolved.source_materialization = git::SourceMaterialization::Dirty; + } + resolved +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use clap::Parser; + use eipw_lint::config::DefaultOptions; + use git2::{IndexAddOption, Repository, Signature}; + use tempfile::TempDir; + use url::Url; + + use crate::{ + cli::{Args, EditorialCommand, EditorialSelectorArgs, RuntimeOperation}, + config::{self, ServerBinding}, + execution::{resolve_execution, ResolvedExecution}, + }; + + use super::{ + editorial_runtime_execution, editorial_targets, run_editorial_lint, + validate_editorial_targets, + }; + + struct EditorialWorkspace { + _temp: TempDir, + active_path: PathBuf, + } + + fn write_file(root: &Path, relative: impl AsRef, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn commit_all(repo: &Repository, message: &str) { + let mut index = repo.index().unwrap(); + index + .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + let parents = repo + .head() + .ok() + .and_then(|head| head.target()) + .map(|oid| repo.find_commit(oid).unwrap()) + .into_iter() + .collect::>(); + let parent_refs = parents.iter().collect::>(); + + repo.commit( + Some("HEAD"), + &signature, + &signature, + message, + &tree, + &parent_refs, + ) + .unwrap(); + } + + fn init_repo(path: &Path, files: &[(&str, &str)]) -> Repository { + std::fs::create_dir_all(path).unwrap(); + let repo = Repository::init(path).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + for (relative, contents) in files { + write_file(path, relative, contents); + } + commit_all(&repo, "initial"); + repo + } + + fn file_url(path: &Path) -> Url { + Url::from_directory_path(path).unwrap() + } + + fn repo_manifest_text(repo_id: &str, repository: &Url, siblings: &[(&str, Url)]) -> String { + let mut manifest = format!( + r#" +repo_id = "{repo_id}" + +[production] +repository = "{repository}" +base_url = "https://example.test/{repo_id}/" + +[staging] +repository = "{repository}" +base_url = "https://staging.example.test/{repo_id}/" +"# + ); + + for (sibling_id, sibling_repository) in siblings { + manifest.push_str(&format!( + r#" +[siblings.{sibling_id}.production] +repository = "{sibling_repository}" +base_url = "https://example.test/{sibling_id}/" + +[siblings.{sibling_id}.staging] +repository = "{sibling_repository}" +base_url = "https://staging.example.test/{sibling_id}/" +"# + )); + } + + manifest + } + + fn proposal_markdown(number: u32, category: Option<&str>, body: &str) -> String { + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!( + "---\neip: {number}\ntitle: Proposal {number}\ndescription: Proposal {number}\nauthor: Test Author \ndiscussions-to: https://ethereum-magicians.org/t/test/{number}\nstatus: Draft\ntype: Standards Track\n{category}created: 2025-01-01\n---\n\n{body}\n" + ) + } + + fn write_eipw_config(workspace_root: &Path) { + let schema_version = DefaultOptions::::schema_version(); + write_file( + workspace_root, + "theme/config/eipw.toml", + &format!( + "schema-version = \"{schema_version}\"\n\n[fetch]\nproposal-format = \"{{:05}}\"\n" + ), + ); + } + + fn missing_file_url() -> Url { + let temp = TempDir::new().unwrap(); + file_url(&temp.path().join("missing-upstream")) + } + + fn editorial_workspace_with_upstream( + active_body: &str, + sibling_body: &str, + upstream_url: Option, + ) -> EditorialWorkspace { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("EIPs"); + let sibling_path = workspace_root.join("ERCs"); + let active_url = upstream_url.unwrap_or_else(|| file_url(&active_path)); + let sibling_url = file_url(&sibling_path); + let manifest = repo_manifest_text("EIPs", &active_url, &[("ERCs", sibling_url)]); + let active_markdown = proposal_markdown(1, None, active_body); + let sibling_markdown = proposal_markdown(2, Some("ERC"), sibling_body); + + write_file(&workspace_root, config::LOCAL_CONFIG_FILE, ""); + write_eipw_config(&workspace_root); + let _active_repo = init_repo( + &active_path, + &[ + (config::REPO_MANIFEST_FILE, manifest.as_str()), + ("content/00001.md", active_markdown.as_str()), + ], + ); + let _sibling_repo = init_repo( + &sibling_path, + &[("content/00002.md", sibling_markdown.as_str())], + ); + + EditorialWorkspace { + _temp: temp, + active_path, + } + } + + fn editorial_workspace(active_body: &str, sibling_body: &str) -> EditorialWorkspace { + editorial_workspace_with_upstream(active_body, sibling_body, None) + } + + fn editorial_workspace_with_missing_upstream( + active_body: &str, + sibling_body: &str, + ) -> EditorialWorkspace { + editorial_workspace_with_upstream(active_body, sibling_body, Some(missing_file_url())) + } + + fn parsed_editorial_lint( + active_path: &Path, + lint_args: &[&str], + ) -> ( + ResolvedExecution, + EditorialSelectorArgs, + crate::lint::CmdArgs, + ) { + let active_path = active_path.to_str().unwrap(); + let mut arguments = vec!["build-eips", "-C", active_path, "editorial", "lint"]; + arguments.extend_from_slice(lint_args); + let args = Args::try_parse_from(arguments).unwrap(); + let resolved = resolve_execution(&args).unwrap(); + + match args.operation.runtime_operation().unwrap() { + RuntimeOperation::Editorial { + command: EditorialCommand::Lint { selectors, eipw }, + } => (resolved, selectors, eipw), + _ => panic!("expected editorial lint command"), + } + } + + fn run_lint( + workspace: &EditorialWorkspace, + lint_args: &[&str], + ) -> Result { + let (resolved, selectors, eipw) = parsed_editorial_lint(&workspace.active_path, lint_args); + + run_editorial_lint(&resolved, &selectors, eipw) + } + + fn resolved_execution(root_path: PathBuf) -> ResolvedExecution { + ResolvedExecution { + root_path, + build_path: PathBuf::from("/workspace/build/Core"), + repository_use: crate::git::RepositoryUse { + title: "Core".to_owned(), + location: config::RepositoryEndpoint { + repository: "https://example.test/Core.git".parse().unwrap(), + base_url: "https://example.test/Core/".parse().unwrap(), + }, + other_repos: Default::default(), + }, + theme_path: Some(PathBuf::from("/workspace/theme")), + source_materialization: crate::git::SourceMaterialization::Clean, + server_binding: ServerBinding::default(), + base_url_override: None, + } + } + + fn explicit_selectors(paths: &[&str]) -> EditorialSelectorArgs { + EditorialSelectorArgs { + paths: paths.iter().map(|path| PathBuf::from(*path)).collect(), + batch: None, + working_tree: false, + against_upstream: false, + } + } + + #[test] + fn editorial_lint_resolves_sibling_proposals_from_prepared_sources() { + let workspace = editorial_workspace_with_missing_upstream( + "Reference [ERC-2](./00002.md).", + "Sibling proposal.", + ); + + assert!(run_lint( + &workspace, + &[ + "content/00001.md", + "--no-default-lints", + "-D", + "markdown-refs" + ] + ) + .unwrap()); + } + + #[test] + fn editorial_batch_lint_resolves_siblings_without_fetching_active_upstream() { + let workspace = editorial_workspace_with_missing_upstream( + "Reference [ERC-2](./00002.md).", + "Sibling proposal.", + ); + let batch_path = workspace.active_path.join("targets.txt"); + write_file(&workspace.active_path, "targets.txt", "content/00001.md\n"); + let batch_path = batch_path.to_str().unwrap(); + + assert!(run_lint( + &workspace, + &[ + "--batch", + batch_path, + "--no-default-lints", + "-D", + "markdown-refs" + ] + ) + .unwrap()); + } + + #[test] + fn editorial_working_tree_lint_uses_dirty_content_without_fetching_active_upstream() { + let workspace = editorial_workspace_with_missing_upstream( + "Reference [ERC-9999](./09999.md).", + "Sibling proposal.", + ); + write_file( + &workspace.active_path, + "content/00001.md", + &proposal_markdown(1, None, "Reference [ERC-2](./00002.md)."), + ); + + assert!(run_lint( + &workspace, + &[ + "--working-tree", + "--no-default-lints", + "-D", + "markdown-refs" + ] + ) + .unwrap()); + } + + #[test] + fn editorial_against_upstream_lint_still_requires_active_upstream() { + let workspace = + editorial_workspace_with_missing_upstream("Active proposal.", "Sibling proposal."); + + let error = run_lint( + &workspace, + &[ + "--against-upstream", + "--no-default-lints", + "-D", + "markdown-refs", + ], + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("fetching upstream repo for editorial source preparation")); + } + + #[test] + fn editorial_lint_rejects_sibling_only_target_as_non_active_target() { + let workspace = editorial_workspace("Active proposal.", "Sibling proposal."); + + let error = run_lint( + &workspace, + &[ + "content/00002.md", + "--no-default-lints", + "-D", + "markdown-refs", + ], + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("unable to resolve editorial target")); + } + + #[test] + fn editorial_lint_reports_untracked_targets_missing_from_prepared_sources() { + let workspace = editorial_workspace("Active proposal.", "Sibling proposal."); + write_file( + &workspace.active_path, + "content/00003.md", + &proposal_markdown(3, None, "Untracked proposal."), + ); + + let error = run_lint( + &workspace, + &[ + "content/00003.md", + "--no-default-lints", + "-D", + "markdown-refs", + ], + ) + .unwrap_err() + .to_string(); + + assert!(error.contains( + "editorial target `content/00003.md` exists in the active repo but was not materialized into the prepared source tree" + )); + assert!(error.contains("untracked files are not supported")); + } + + #[test] + fn editorial_lint_materializes_tracked_dirty_working_tree_targets() { + let workspace = + editorial_workspace("Reference [ERC-9999](./09999.md).", "Sibling proposal."); + write_file( + &workspace.active_path, + "content/00001.md", + &proposal_markdown(1, None, "Reference [ERC-2](./00002.md)."), + ); + + assert!(run_lint( + &workspace, + &[ + "--working-tree", + "--no-default-lints", + "-D", + "markdown-refs" + ] + ) + .unwrap()); + } + + #[test] + fn editorial_working_tree_check_still_forces_dirty_runtime_materialization() { + let resolved = ResolvedExecution { + root_path: PathBuf::from("/workspace/Core"), + build_path: PathBuf::from("/workspace/build/Core"), + repository_use: crate::git::RepositoryUse { + title: "Core".to_owned(), + location: config::RepositoryEndpoint { + repository: "https://example.test/Core.git".parse().unwrap(), + base_url: "https://example.test/Core/".parse().unwrap(), + }, + other_repos: Default::default(), + }, + theme_path: Some(PathBuf::from("/workspace/theme")), + source_materialization: crate::git::SourceMaterialization::Clean, + server_binding: ServerBinding::default(), + base_url_override: None, + }; + let selectors = EditorialSelectorArgs { + paths: Vec::new(), + batch: None, + working_tree: true, + against_upstream: false, + }; + + assert_eq!( + editorial_runtime_execution(resolved, &selectors).source_materialization, + crate::git::SourceMaterialization::Dirty + ); + } + + #[test] + fn editorial_explicit_numeric_selectors_resolve_to_markdown_paths() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/0004.md", ""); + let resolved = resolved_execution(temp.path().to_path_buf()); + + for selector in ["4", "004", "0004"] { + assert_eq!( + editorial_targets(&explicit_selectors(&[selector]), &resolved).unwrap(), + vec![PathBuf::from("content/0004.md")] + ); + } + } + + #[test] + fn editorial_explicit_numeric_selectors_support_multiple_and_dedupe() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/0004.md", ""); + write_file(temp.path(), "content/0005/index.md", ""); + let resolved = resolved_execution(temp.path().to_path_buf()); + + assert_eq!( + editorial_targets(&explicit_selectors(&["4", "0004", "005"]), &resolved).unwrap(), + vec![ + PathBuf::from("content/0004.md"), + PathBuf::from("content/0005/index.md"), + ] + ); + } + + #[test] + fn editorial_batch_accepts_numbers_paths_comments_and_empty_lines() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/0004.md", ""); + write_file(temp.path(), "content/0005/index.md", ""); + let batch_path = temp.path().join("targets.txt"); + write_file( + temp.path(), + "targets.txt", + "\n# comment\n \n4\ncontent/0005/index.md\n", + ); + let resolved = resolved_execution(temp.path().to_path_buf()); + let selectors = EditorialSelectorArgs { + paths: Vec::new(), + batch: Some(batch_path), + working_tree: false, + against_upstream: false, + }; + + assert_eq!( + editorial_targets(&selectors, &resolved).unwrap(), + vec![ + PathBuf::from("content/0004.md"), + PathBuf::from("content/0005/index.md"), + ] + ); + } + + #[test] + fn editorial_explicit_repo_relative_path_selectors_still_work() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/0004.md", ""); + let resolved = resolved_execution(temp.path().to_path_buf()); + + assert_eq!( + editorial_targets(&explicit_selectors(&["content/0004.md"]), &resolved).unwrap(), + vec![PathBuf::from("content/0004.md")] + ); + } + + #[test] + fn editorial_invalid_number_like_selectors_fail_with_editorial_error() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/0004.md", ""); + let resolved = resolved_execution(temp.path().to_path_buf()); + + for selector in [ + "0", + "+4", + "-4", + "4,5", + "4,,5", + ",4", + "4,", + "+", + "-", + "4294967296", + ] { + let error = editorial_targets(&explicit_selectors(&[selector]), &resolved) + .unwrap_err() + .to_string(); + assert!(error.contains(&format!( + "editorial number selector `{selector}` is invalid" + ))); + assert!(error.contains( + "expected a positive proposal number that fits in u32, without signs or commas" + )); + } + } + + #[test] + fn editorial_path_like_selectors_continue_through_path_validation() { + let temp = TempDir::new().unwrap(); + let resolved = resolved_execution(temp.path().to_path_buf()); + + for selector in ["foo", "draft.md", "4a", "draft-4.md"] { + write_file(temp.path(), selector, ""); + + let error = editorial_targets(&explicit_selectors(&[selector]), &resolved) + .unwrap_err() + .to_string(); + + assert!(error.contains("is not a supported proposal path")); + assert!(!error.contains("editorial number selector")); + } + } + + #[cfg(unix)] + #[test] + fn editorial_non_utf8_selector_continues_through_path_validation() { + use std::{ffi::OsStr, os::unix::ffi::OsStrExt}; + + let temp = TempDir::new().unwrap(); + let resolved = resolved_execution(temp.path().to_path_buf()); + let selectors = EditorialSelectorArgs { + paths: vec![PathBuf::from(OsStr::from_bytes(b"\xff"))], + batch: None, + working_tree: false, + against_upstream: false, + }; + + let error = editorial_targets(&selectors, &resolved) + .unwrap_err() + .to_string(); + + assert!(error.contains("unable to resolve editorial target")); + assert!(!error.contains("editorial number selector")); + } + + #[test] + fn non_strict_editorial_target_validation_does_not_normalize_numeric_paths() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "4", ""); + write_file(temp.path(), "content/0004.md", ""); + + assert_eq!( + validate_editorial_targets(temp.path(), vec![PathBuf::from("4")], false).unwrap(), + Vec::::new() + ); + } +} diff --git a/src/execution.rs b/src/execution.rs index bffa6ac..d0eb81e 100644 --- a/src/execution.rs +++ b/src/execution.rs @@ -129,7 +129,7 @@ pub(crate) fn resolve_execution_settings( let (staging, allow_dirty, default_sibling) = if let Some(staging) = explicit_environment { (staging, false, SelectedSource::Remote) - } else if args.operation.is_plain_site_command() { + } else if args.operation.is_plain_site_command() || args.operation.is_editorial_command() { (true, !clean, SelectedSource::WorkspaceLocal) } else { (false, false, SelectedSource::Remote) @@ -239,6 +239,7 @@ fn operation_requires_theme(operation: &Operation) -> bool { Operation::Build { .. } | Operation::Serve { .. } | Operation::Check { .. } + | Operation::Editorial { .. } | Operation::Parity { .. } ) } @@ -713,6 +714,17 @@ base_url = "http://localhost:4000" false, SelectedSource::Remote, ), + ( + &[ + "build-eips", + "--remote-siblings", + "editorial", + "lint", + "content/0001.md", + ][..], + true, + SelectedSource::Remote, + ), ]; for (arguments, allow_dirty, sibling) in cases { @@ -847,6 +859,7 @@ base_url = "http://localhost:4000" &["build-eips", "--staging", "build"][..], &["build-eips", "--production", "check"][..], &["build-eips", "parity", "build"][..], + &["build-eips", "editorial", "check", "--against-upstream"][..], ] { let args = parse_args(arguments); let theme_path = super::resolve_theme_path(Some(&workspace_config), &args.operation) @@ -874,12 +887,127 @@ base_url = "http://localhost:4000" } } + #[test] + fn editorial_dispatch_uses_local_first_for_all_editorial_commands() { + let workspace_config = load_workspace_config(""); + + assert_settings( + &["build-eips", "editorial", "lint", "content/0001.md"], + &["ERCs"], + Some(&workspace_config), + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: true, + sibling: SelectedSource::WorkspaceLocal, + }, + ); + assert_settings( + &["build-eips", "editorial", "check", "content/0001.md"], + &["ERCs"], + Some(&workspace_config), + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: true, + sibling: SelectedSource::WorkspaceLocal, + }, + ); + assert_settings( + &[ + "build-eips", + "--remote-siblings", + "editorial", + "lint", + "content/0001.md", + ], + &["ERCs"], + Some(&workspace_config), + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: true, + sibling: SelectedSource::Remote, + }, + ); + assert_settings( + &[ + "build-eips", + "--staging", + "editorial", + "lint", + "content/0001.md", + ], + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + assert_settings( + &[ + "build-eips", + "--production", + "editorial", + "lint", + "content/0001.md", + ], + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: false, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + assert_settings( + &[ + "build-eips", + "--staging", + "editorial", + "check", + "--against-upstream", + ], + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + assert_settings( + &[ + "build-eips", + "--production", + "editorial", + "check", + "--against-upstream", + ], + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: false, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + } + #[test] fn local_first_theme_commands_without_workspace_config_report_combined_setup_error() { for arguments in [ &["build-eips", "build"][..], &["build-eips", "serve"][..], &["build-eips", "check"][..], + &["build-eips", "editorial", "lint", "content/0001.md"][..], + &["build-eips", "editorial", "check", "content/0001.md"][..], ] { assert_combined_missing_workspace_error(arguments); } diff --git a/src/find_root.rs b/src/find_root.rs index 77bb66d..40cde8c 100644 --- a/src/find_root.rs +++ b/src/find_root.rs @@ -34,7 +34,7 @@ pub enum Error { pub fn is_root(path: &Path) -> Result<(), Error> { let git = path.join(".git"); let contents = path.join(CONTENT_DIR); - if git.is_dir() && contents.is_dir() { + if (git.is_dir() || git.is_file()) && contents.is_dir() { Ok(()) } else { NoRootSnafu.fail() diff --git a/src/git.rs b/src/git.rs index 8ef5346..04d1956 100644 --- a/src/git.rs +++ b/src/git.rs @@ -813,6 +813,10 @@ pub struct SourceOnly { } impl SourceOnly { + pub fn merge(&self) -> Result<(), Error> { + merge_sibling_repositories(&self.working_repo, &self.src_repo_use, self.local_head) + } + pub fn fetch_upstream(self) -> Result { info!("fetching latest {} repository", self.src_repo_use.title); let latest_master = fetch( @@ -892,41 +896,138 @@ impl SourceWithUpstream { Ok(changed_files) } - fn check_ignored(&self, tree: &Tree) -> Result<(), Error> { - let mut walk_error = None; - let walk_result = tree.walk(git2::TreeWalkMode::PreOrder, |a, b| { - if b.kind() != Some(ObjectType::Blob) { - return TreeWalkResult::Ok; + pub fn merge(&self) -> Result<(), Error> { + merge_sibling_repositories(&self.working_repo, &self.src_repo_use, self.local_head) + } +} + +fn check_ignored(working_repo: &git2::Repository, tree: &Tree) -> Result<(), Error> { + let mut walk_error = None; + let walk_result = tree.walk(git2::TreeWalkMode::PreOrder, |a, b| { + if b.kind() != Some(ObjectType::Blob) { + return TreeWalkResult::Ok; + } + + let path = match b.name() { + None => a.to_owned(), + Some(p) => format!("{a}{p}"), + }; + + debug!("checking if `{path}` is ignored"); + + match working_repo.is_path_ignored(&path) { + Ok(false) => TreeWalkResult::Ok, + Ok(true) => { + walk_error = Some( + UpdateTreeSnafu { + msg: format!("contains ignored path `{path}`"), + } + .build(), + ); + TreeWalkResult::Abort + } + Err(e) => { + walk_error = Some( + GitSnafu { + what: "check ignored", + } + .into_error(e), + ); + TreeWalkResult::Abort } + } + }); - let path = match b.name() { - None => a.to_owned(), - Some(p) => format!("{a}{p}"), - }; + if let Some(error) = walk_error { + return Err(error); + } + + walk_result.context(GitSnafu { + what: "traverse tree", + })?; - debug!("checking if `{path}` is ignored"); + Ok(()) +} - match self.working_repo.is_path_ignored(&path) { - Ok(false) => TreeWalkResult::Ok, - Ok(true) => { +fn merge_sibling_repositories( + working_repo: &git2::Repository, + repo_use: &RepositoryUse, + mut local_head: Oid, +) -> Result<(), Error> { + for (index, (other_kind, other_repo)) in repo_use + .other_repos + .iter() + .progress_ext("Merge Repos") + .enumerate() + { + let local_commit = working_repo.find_commit(local_head).context(GitSnafu { + what: "find local head commit", + })?; + let local_tree = local_commit.tree().context(GitSnafu { + what: "getting local head tree", + })?; + info!("fetching {other_kind} repository"); + // Local sibling overrides should follow the checked-out repo HEAD instead of assuming `master`. + let other_ref = format!("refs/build-eips/other-head-{index}"); + let other_refspec = if other_repo.scheme() == "file" { + format!("+HEAD:{other_ref}") + } else { + format!("+master:{other_ref}") + }; + let master_other = fetch(working_repo, other_repo.as_str(), &other_refspec)?; + let other_tree = master_other.tree().context(GitSnafu { + what: "getting other tree", + })?; + + let mut tree_builder = TreeUpdateBuilder::new(); + let prefix = format!("{}/", CONTENT_DIR); + let mut walk_error: Option = None; + let walk_result = other_tree.walk(git2::TreeWalkMode::PreOrder, |a, b| { + if !a.starts_with(&prefix) && (!a.is_empty() || b.name() != Some(CONTENT_DIR)) { + return TreeWalkResult::Skip; + } + + let name = match b.name() { + Some(n) => n, + None => { walk_error = Some( UpdateTreeSnafu { - msg: format!("contains ignored path `{path}`"), + msg: format!("tree entry without name in `{a}`"), } .build(), ); - TreeWalkResult::Abort + return TreeWalkResult::Abort; } - Err(e) => { + }; + + let path = format!("{}{}", a, name); + match b.kind() { + Some(ObjectType::Blob) => (), + Some(ObjectType::Tree) => return TreeWalkResult::Ok, + kind => { walk_error = Some( - GitSnafu { - what: "check ignored", + UpdateTreeSnafu { + msg: format!("unknown blob type `{kind:?}` for `{path}`"), } - .into_error(e), + .build(), ); - TreeWalkResult::Abort + return TreeWalkResult::Abort; } } + + if path == CONTENT_INDEX_PATH { + debug!("skip sibling homepage `{path}`"); + return TreeWalkResult::Ok; + } + + if let Err(e) = check_conflict(&local_tree, Path::new(&path), b) { + walk_error = Some(e); + return TreeWalkResult::Abort; + } + + debug!("upsert `{path}`"); + tree_builder.upsert(path, b.id(), FileMode::Blob); + TreeWalkResult::Ok }); if let Some(error) = walk_error { @@ -937,149 +1038,53 @@ impl SourceWithUpstream { what: "traverse tree", })?; - Ok(()) - } + let merged_tree_oid = tree_builder + .create_updated(working_repo, &local_tree) + .context(GitSnafu { what: "build tree" })?; + let merged_tree = working_repo.find_tree(merged_tree_oid).unwrap(); - pub fn merge(&self) -> Result<(), Error> { - let repo_use = &self.src_repo_use; - let mut local_head = self.local_head; - for (index, (other_kind, other_repo)) in repo_use - .other_repos - .iter() - .progress_ext("Merge Repos") - .enumerate() - { - let local_commit = self - .working_repo - .find_commit(local_head) - .context(GitSnafu { - what: "find local head commit", - })?; - let local_tree = local_commit.tree().context(GitSnafu { - what: "getting local head tree", + check_ignored(working_repo, &merged_tree)?; + + let sig = + Signature::now("eips-build", "eips-build@eips-build.invalid").context(GitSnafu { + what: "commit signature", })?; - info!("fetching {other_kind} repository"); - // Local sibling overrides should follow the checked-out repo HEAD instead of assuming `master`. - let other_ref = format!("refs/build-eips/other-head-{index}"); - let other_refspec = if other_repo.scheme() == "file" { - format!("+HEAD:{other_ref}") - } else { - format!("+master:{other_ref}") - }; - let master_other = fetch(&self.working_repo, other_repo.as_str(), &other_refspec)?; - let other_tree = master_other.tree().context(GitSnafu { - what: "getting other tree", + let msg = format!("Merge {other_repo}"); + local_head = working_repo + .commit( + Some("HEAD"), + &sig, + &sig, + &msg, + &merged_tree, + &[&local_commit, &master_other], + ) + .context(GitSnafu { what: "committing" })?; + + working_repo + .checkout_head(Some(CheckoutBuilder::default().force())) + .context(GitSnafu { + what: "checkout merged", })?; - let mut tree_builder = TreeUpdateBuilder::new(); - let prefix = format!("{}/", CONTENT_DIR); - let mut walk_error: Option = None; - let walk_result = other_tree.walk(git2::TreeWalkMode::PreOrder, |a, b| { - if !a.starts_with(&prefix) && (!a.is_empty() || b.name() != Some(CONTENT_DIR)) { - return TreeWalkResult::Skip; - } - - let name = match b.name() { - Some(n) => n, - None => { - walk_error = Some( - UpdateTreeSnafu { - msg: format!("tree entry without name in `{a}`"), - } - .build(), - ); - return TreeWalkResult::Abort; - } - }; - - let path = format!("{}{}", a, name); - match b.kind() { - Some(ObjectType::Blob) => (), - Some(ObjectType::Tree) => return TreeWalkResult::Ok, - kind => { - walk_error = Some( - UpdateTreeSnafu { - msg: format!("unknown blob type `{kind:?}` for `{path}`"), - } - .build(), - ); - return TreeWalkResult::Abort; - } - } - - if path == CONTENT_INDEX_PATH { - debug!("skip sibling homepage `{path}`"); - return TreeWalkResult::Ok; - } - - if let Err(e) = check_conflict(&local_tree, Path::new(&path), b) { - walk_error = Some(e); - return TreeWalkResult::Abort; + drop(merged_tree); + drop(other_tree); + drop(master_other); + drop(local_tree); + drop(local_commit); + match working_repo.find_reference(&other_ref) { + Ok(mut reference) => { + if let Err(error) = reference.delete() { + debug!("unable to delete temporary sibling ref `{other_ref}`: {error}"); } - - debug!("upsert `{path}`"); - tree_builder.upsert(path, b.id(), FileMode::Blob); - TreeWalkResult::Ok - }); - - if let Some(error) = walk_error { - return Err(error); } - - walk_result.context(GitSnafu { - what: "traverse tree", - })?; - - let merged_tree_oid = tree_builder - .create_updated(&self.working_repo, &local_tree) - .context(GitSnafu { what: "build tree" })?; - let merged_tree = self.working_repo.find_tree(merged_tree_oid).unwrap(); - - self.check_ignored(&merged_tree)?; - - let sig = Signature::now("eips-build", "eips-build@eips-build.invalid").context( - GitSnafu { - what: "commit signature", - }, - )?; - let msg = format!("Merge {other_repo}"); - local_head = self - .working_repo - .commit( - Some("HEAD"), - &sig, - &sig, - &msg, - &merged_tree, - &[&local_commit, &master_other], - ) - .context(GitSnafu { what: "committing" })?; - - self.working_repo - .checkout_head(Some(CheckoutBuilder::default().force())) - .context(GitSnafu { - what: "checkout merged", - })?; - - drop(merged_tree); - drop(other_tree); - drop(master_other); - drop(local_tree); - drop(local_commit); - match self.working_repo.find_reference(&other_ref) { - Ok(mut reference) => { - if let Err(error) = reference.delete() { - debug!("unable to delete temporary sibling ref `{other_ref}`: {error}"); - } - } - Err(error) => { - debug!("temporary sibling ref `{other_ref}` was not deleted: {error}"); - } + Err(error) => { + debug!("temporary sibling ref `{other_ref}` was not deleted: {error}"); } } - - Ok(()) } + + Ok(()) } fn fetch<'a>( diff --git a/src/lint.rs b/src/lint.rs index 9fa9e25..f845458 100644 --- a/src/lint.rs +++ b/src/lint.rs @@ -89,14 +89,6 @@ struct Config { #[derive(Debug, Clone, clap::Args, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct CmdArgs { - /// Disable linting entirely - #[arg(long, exclusive(true))] - no_lint: bool, - - /// Restrict linting to specific files and/or directories (relative to project root) - #[clap(required(false))] - sources: Vec, - /// Lint output format #[clap(long, value_enum, default_value_t)] format: Format, @@ -252,18 +244,15 @@ fn version_cmp( #[tokio::main(flavor = "current_thread")] pub async fn eipw( theme_path: &Path, - root_dir: &Path, repo_dir: &Path, - changed_paths: Vec, + sources: Vec, opts: CmdArgs, ) -> Result<(), Error> { - if opts.no_lint { - return Ok(()); - } - let mut stdout = std::io::stdout(); - let config_path = theme_path.join("config").join("eipw.toml"); + let mut config_path = theme_path.to_path_buf(); + config_path.push("config"); + config_path.push("eipw.toml"); let toml_file = Toml::file_exact(&config_path); @@ -290,36 +279,8 @@ pub async fn eipw( .await .context(FsSnafu { path: repo_dir })?; - let paths = if opts.sources.is_empty() { - changed_paths - } else { - let root_dir = tokio::fs::canonicalize(root_dir) - .await - .context(FsSnafu { path: root_dir })?; - let mut repo_relative_sources = Vec::with_capacity(opts.sources.len()); - for source in &opts.sources { - let root_relative_source = root_dir.join(source); - let full_source = tokio::fs::canonicalize(&root_relative_source) - .await - .context(FsSnafu { - path: root_relative_source, - })?; - - let relative_source = match full_source.strip_prefix(&root_dir) { - Ok(r) => r, - Err(e) => { - let err = std::io::Error::new(std::io::ErrorKind::NotFound, e); - return Err(FsSnafu { path: full_source }.into_error(err)); - } - }; - - repo_relative_sources.push(repo_dir.join(relative_source)); - } - - repo_relative_sources - }; - - let sources = collect_sources(paths).await?; + let sources: Vec<_> = sources.iter().map(|source| repo_dir.join(source)).collect(); + let sources = collect_sources(sources).await?; let reporter = match opts.format { Format::Json => EitherReporter::Json(Json::default()), diff --git a/src/main.rs b/src/main.rs index 0b6e954..7d092a3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ mod changed; mod cli; mod config; mod context; +mod editorial; mod execution; mod find_root; mod git; @@ -33,7 +34,8 @@ use log::{debug, info}; use snafu::{Report, ResultExt, Whatever}; use crate::{ - cli::{Args, Operation, RuntimeOperation}, + cli::{Args, EditorialCommand, Operation, RuntimeOperation}, + editorial::{editorial_runtime_execution, run_editorial_lint}, execution::{resolve_execution, validate_non_execution_command_flags}, layout::output_path, pipeline::Prepared, @@ -112,19 +114,28 @@ fn run() -> Result<(), Whatever> { .whatever_context("unable to remove build directory")?; return Ok(()); } - RuntimeOperation::Check { eipw } => { - Prepared::prepare(eipw, resolved)?.check()?; + RuntimeOperation::Check => { + Prepared::prepare(resolved)?.check()?; } - RuntimeOperation::Build { eipw } => { - Prepared::prepare(eipw, resolved)?.build()?; + RuntimeOperation::Build => { + Prepared::prepare(resolved)?.build()?; } - RuntimeOperation::Serve { eipw } => { - Prepared::prepare(eipw, resolved)?.serve()?; + RuntimeOperation::Serve => { + Prepared::prepare(resolved)?.serve()?; } RuntimeOperation::Preview => unreachable!(), RuntimeOperation::Changed { all, format } => { changed::run(&resolved, &build_path, all, &format)?; } + RuntimeOperation::Editorial { command } => match command { + EditorialCommand::Lint { selectors, eipw } => { + run_editorial_lint(&resolved, &selectors, eipw)?; + } + EditorialCommand::Check { selectors, eipw } => { + run_editorial_lint(&resolved, &selectors, eipw.clone())?; + Prepared::prepare(editorial_runtime_execution(resolved, &selectors))?.check()?; + } + }, } lock_file diff --git a/src/pipeline.rs b/src/pipeline.rs index 413d0dc..4858f68 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -12,12 +12,11 @@ use snafu::{OptionExt, ResultExt, Whatever}; use url::Url; use crate::{ - changed, config::ServerBinding, execution::ResolvedExecution, git, layout::{mounted_theme_path, output_path, CONTENT_DIR, REPO_DIR}, - lint, markdown, + markdown, serve::{serve_sync_config, DirtyServeWatcher, LocalThemeServeSync}, zola, }; @@ -42,6 +41,29 @@ fn prepare_theme_for_zola( )) } +fn prepare_runtime_source( + root_path: &Path, + repo_path: &Path, + repository_use: &git::RepositoryUse, + source_materialization: git::SourceMaterialization, +) -> Result<(), Whatever> { + let source = git::Fresh::new( + root_path, + repo_path, + repository_use.clone(), + source_materialization, + ) + .whatever_context("initializing build repo")? + .clone_src() + .whatever_context("cloning source repo")?; + + source + .merge() + .whatever_context("unable to merge ERC/EIP repositories")?; + + Ok(()) +} + #[derive(Debug)] pub(crate) struct Prepared { repo_path: PathBuf, @@ -56,10 +78,7 @@ pub(crate) struct Prepared { } impl Prepared { - pub(crate) fn prepare( - eipw: lint::CmdArgs, - resolved: ResolvedExecution, - ) -> Result { + pub(crate) fn prepare(resolved: ResolvedExecution) -> Result { zola::find_zola().whatever_context("unable to find suitable zola binary")?; let ResolvedExecution { @@ -78,31 +97,12 @@ impl Prepared { let content_path = repo_path.join(CONTENT_DIR); let output_path = output_path(&build_path); - let both = git::Fresh::new( + prepare_runtime_source( &root_path, &repo_path, - repository_use.clone(), + &repository_use, source_materialization, - ) - .whatever_context("initializing build repo")? - .clone_src() - .whatever_context("cloning source repo")? - .fetch_upstream() - .whatever_context("fetching upstream repo")?; - - let changed_files: Vec<_> = both - .changed_files() - .whatever_context("unable to list changed files")? - .into_iter() - .filter(|p| changed::is_proposal_path(p.into())) - .map(|p| repo_path.join(p)) - .collect(); - - both.merge() - .whatever_context("unable to merge ERC/EIP repositories")?; - - lint::eipw(&theme_path, &root_path, &repo_path, changed_files, eipw) - .whatever_context("linting failed")?; + )?; markdown::preprocess(&content_path).whatever_context("unable to preprocess markdown")?; let (theme_path, local_theme_sync) = prepare_theme_for_zola(theme_path, &repo_path)?; @@ -175,14 +175,29 @@ impl Prepared { #[cfg(test)] mod tests { - use std::path::Path; + use std::path::{Path, PathBuf}; + use clap::Parser; use git2::{IndexAddOption, Repository, Signature}; use tempfile::TempDir; - - use crate::layout::{mounted_theme_path, theme_config_path}; - - use super::prepare_theme_for_zola; + use url::Url; + + use crate::{ + changed, + cli::{Args, ChangedFormat, EditorialCommand, RuntimeOperation}, + config, + editorial::editorial_runtime_execution, + execution::{resolve_execution, ResolvedExecution}, + git::SourceMaterialization, + layout::{mounted_theme_path, theme_config_path, REPO_DIR}, + }; + + use super::{prepare_runtime_source, prepare_theme_for_zola}; + + struct RuntimeWorkspace { + _temp: TempDir, + active_path: PathBuf, + } fn write_file(root: &Path, relative: impl AsRef, contents: &str) { let path = root.join(relative); @@ -232,6 +247,96 @@ mod tests { repo } + fn file_url(path: &Path) -> Url { + Url::from_directory_path(path).unwrap() + } + + fn repo_manifest_text(repo_id: &str, repository: &Url, siblings: &[(&str, Url)]) -> String { + let mut manifest = format!( + r#" +repo_id = "{repo_id}" + +[production] +repository = "{repository}" +base_url = "https://example.test/{repo_id}/" + +[staging] +repository = "{repository}" +base_url = "https://staging.example.test/{repo_id}/" +"# + ); + + for (sibling_id, sibling_repository) in siblings { + manifest.push_str(&format!( + r#" +[siblings.{sibling_id}.production] +repository = "{sibling_repository}" +base_url = "https://example.test/{sibling_id}/" + +[siblings.{sibling_id}.staging] +repository = "{sibling_repository}" +base_url = "https://staging.example.test/{sibling_id}/" +"# + )); + } + + manifest + } + + fn runtime_workspace(with_sibling: bool) -> RuntimeWorkspace { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("EIPs"); + let sibling_path = workspace_root.join("ERCs"); + let missing_upstream = file_url(&temp.path().join("missing-upstream")); + let siblings = with_sibling.then(|| ("ERCs", file_url(&sibling_path))); + let siblings = siblings.into_iter().collect::>(); + let manifest = repo_manifest_text("EIPs", &missing_upstream, &siblings); + + write_file(&workspace_root, config::LOCAL_CONFIG_FILE, ""); + std::fs::create_dir_all(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + let _active_repo = init_repo( + &active_path, + &[ + (config::REPO_MANIFEST_FILE, manifest.as_str()), + ("content/00001.md", "active proposal\n"), + ], + ); + + if with_sibling { + let _sibling_repo = init_repo(&sibling_path, &[("content/00002.md", "sibling\n")]); + } + + RuntimeWorkspace { + _temp: temp, + active_path, + } + } + + fn resolved_runtime(workspace: &RuntimeWorkspace, command: &[&str]) -> ResolvedExecution { + let active_path = workspace.active_path.to_str().unwrap(); + let mut arguments = vec!["build-eips", "-C", active_path]; + arguments.extend_from_slice(command); + let args = Args::try_parse_from(arguments).unwrap(); + + resolve_execution(&args).unwrap() + } + + fn prepare_resolved_source(resolved: &ResolvedExecution) -> Result<(), snafu::Whatever> { + std::fs::create_dir_all(&resolved.build_path).unwrap(); + let repo_path = resolved.build_path.join(REPO_DIR); + prepare_runtime_source( + &resolved.root_path, + &repo_path, + &resolved.repository_use, + resolved.source_materialization, + ) + } + + fn prepared_path(resolved: &ResolvedExecution, relative: impl AsRef) -> PathBuf { + resolved.build_path.join(REPO_DIR).join(relative) + } + #[test] fn workspace_local_theme_is_materialized_as_mounted_theme_for_zola() { let temp = TempDir::new().unwrap(); @@ -261,4 +366,107 @@ mod tests { assert_eq!(sync.mounted_theme_dir, mounted_theme_dir); assert!(sync.theme_index_path.ends_with(".git/index")); } + + #[test] + fn prepared_runtime_source_succeeds_with_unreachable_active_upstream() { + for command in [&["build"][..], &["check"][..], &["serve"][..]] { + let workspace = runtime_workspace(false); + let resolved = resolved_runtime(&workspace, command); + + prepare_resolved_source(&resolved).unwrap(); + + assert_eq!( + std::fs::read_to_string(prepared_path(&resolved, "content/00001.md")).unwrap(), + "active proposal\n" + ); + } + } + + #[test] + fn prepared_runtime_source_uses_remote_siblings_without_active_upstream_fetch() { + let workspace = runtime_workspace(true); + let resolved = resolved_runtime(&workspace, &["--remote-siblings", "build"]); + + prepare_resolved_source(&resolved).unwrap(); + + assert_eq!( + std::fs::read_to_string(prepared_path(&resolved, "content/00001.md")).unwrap(), + "active proposal\n" + ); + assert_eq!( + std::fs::read_to_string(prepared_path(&resolved, "content/00002.md")).unwrap(), + "sibling\n" + ); + } + + #[test] + fn remote_environment_runtime_source_prep_keeps_local_active_checkout() { + for command in [ + &["--staging", "build"][..], + &["--production", "check"][..], + &["parity", "serve"][..], + ] { + let workspace = runtime_workspace(false); + let resolved = resolved_runtime(&workspace, command); + + assert_eq!( + resolved.source_materialization, + SourceMaterialization::Clean + ); + prepare_resolved_source(&resolved).unwrap(); + + assert_eq!( + std::fs::read_to_string(prepared_path(&resolved, "content/00001.md")).unwrap(), + "active proposal\n" + ); + } + } + + #[test] + fn changed_still_requires_active_upstream() { + let workspace = runtime_workspace(false); + let resolved = resolved_runtime(&workspace, &["changed"]); + std::fs::create_dir_all(&resolved.build_path).unwrap(); + + let error = changed::run( + &resolved, + &resolved.build_path, + false, + &ChangedFormat::Newline, + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("fetching upstream repo")); + } + + #[test] + fn editorial_check_site_phase_source_prep_does_not_fetch_active_upstream() { + let workspace = runtime_workspace(false); + let active_path = workspace.active_path.to_str().unwrap(); + let args = Args::try_parse_from([ + "build-eips", + "-C", + active_path, + "editorial", + "check", + "--against-upstream", + ]) + .unwrap(); + let resolved = resolve_execution(&args).unwrap(); + let RuntimeOperation::Editorial { + command: EditorialCommand::Check { selectors, .. }, + } = args.operation.runtime_operation().unwrap() + else { + panic!("expected editorial check runtime operation"); + }; + let resolved = editorial_runtime_execution(resolved, &selectors); + + prepare_resolved_source(&resolved).unwrap(); + + assert_eq!( + std::fs::read_to_string(prepared_path(&resolved, "content/00001.md")).unwrap(), + "active proposal\n" + ); + } } From fce78c00a1ccda89edb1540417dbc7bc3d786bd0 Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Sun, 3 May 2026 21:49:21 -0400 Subject: [PATCH 13/20] Add targeted build rendering Add build-only render selection from --only and workspace [render].only, deduping proposal numbers into ResolvedExecution. Build OnlyRenderPlan during prepared runtime setup, rewrite omitted proposal links and requires entries to public URLs, and prune unselected proposal content before Zola runs. Restrict targeted rendering to local dirty build mode for now, leaving targeted serve sync to the next PR. Remove the eipw lint step from the prepared runtime pipeline so linting is reached only through editorial commands. --- src/cli.rs | 84 +++++++++++- src/config.rs | 90 +++++++++++- src/editorial.rs | 2 + src/execution.rs | 151 +++++++++++++++++++- src/markdown.rs | 350 ++++++++++++++++++++++++++++++++++++++++++++--- src/pipeline.rs | 16 ++- src/serve.rs | 2 +- 7 files changed, 668 insertions(+), 27 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index c4af61f..7b0bdc8 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -11,7 +11,7 @@ use std::path::{Path, PathBuf}; use clap::{Parser, Subcommand}; use url::Url; -use crate::{lint, print}; +use crate::{lint, print, proposal::ProposalNumber}; /// Build script for Ethereum EIPs and ERCs. #[derive(Parser, Debug)] @@ -66,6 +66,13 @@ pub(crate) struct CleanCliArgs { pub(crate) clean: bool, } +#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)] +pub(crate) struct OnlyCliArgs { + /// Render only the selected proposal number(s) + #[arg(long, value_name = "NUMBER", value_parser = ProposalNumber::parse_cli_selector, num_args = 1..)] + pub(crate) only: Vec, +} + #[derive(Debug, Clone, Subcommand)] pub(crate) enum Operation { /// Print linter schema metadata and lint configuration @@ -81,6 +88,9 @@ pub(crate) enum Operation { #[command(flatten)] clean: CleanCliArgs, + + #[command(flatten)] + only: OnlyCliArgs, }, /// Serve the existing built output without rebuilding it @@ -153,7 +163,7 @@ pub(crate) enum ProfiledOperation { base_url: BaseUrlCliArgs, }, - /// Build the project and launch a web server to preview it + /// Build a fresh temporary site, serve it locally, and watch tracked edits Serve { #[command(flatten)] server: ServerCliArgs, @@ -272,6 +282,22 @@ impl Operation { } } + pub(crate) fn only_cli_args(&self) -> Option<&OnlyCliArgs> { + match self { + Self::Build { only, .. } => Some(only), + Self::Print { .. } + | Self::Preview { .. } + | Self::Serve { .. } + | Self::Clean + | Self::Check { .. } + | Self::Changed { .. } + | Self::Editorial { .. } + | Self::Init { .. } + | Self::Doctor + | Self::Parity { .. } => None, + } + } + pub(crate) fn is_plain_site_command(&self) -> bool { matches!( self, @@ -315,14 +341,14 @@ impl ProfiledOperation { fn server_cli_args(&self) -> ServerCliArgs { match self { Self::Serve { server, .. } => server.clone(), - Self::Build { .. } | Self::Check { .. } => ServerCliArgs::default(), + Self::Build { .. } | Self::Check => ServerCliArgs::default(), } } fn base_url_cli_args(&self) -> BaseUrlCliArgs { match self { Self::Build { base_url, .. } | Self::Serve { base_url, .. } => base_url.clone(), - Self::Check { .. } => BaseUrlCliArgs::default(), + Self::Check => BaseUrlCliArgs::default(), } } @@ -382,6 +408,8 @@ impl EditorialSelectorArgs { mod tests { use clap::Parser; + use crate::proposal::ProposalNumber; + use super::{Args, EditorialCommand, Operation, ProfiledOperation, RuntimeOperation}; fn parse_args(arguments: &[&str]) -> Args { @@ -420,6 +448,54 @@ mod tests { .contains(&format!("unexpected argument '{removed_flag}'"))); } + #[test] + fn only_flag_parses_one_or_more_proposal_numbers_on_build() { + let one = parse_args(&["build-eips", "build", "--only", "00555"]); + let many = parse_args(&["build-eips", "build", "--only", "555", "678", "897"]); + + match one.operation { + Operation::Build { only, .. } => { + assert_eq!(only.only, vec![ProposalNumber::from_u32(555).unwrap()]); + } + other => panic!("unexpected operation: {other:?}"), + } + match many.operation { + Operation::Build { only, .. } => { + assert_eq!( + only.only, + vec![ + ProposalNumber::from_u32(555).unwrap(), + ProposalNumber::from_u32(678).unwrap(), + ProposalNumber::from_u32(897).unwrap(), + ] + ); + } + other => panic!("unexpected operation: {other:?}"), + } + } + + #[test] + fn only_flag_rejects_invalid_selectors_and_non_build_commands() { + for selector in [ + "+555", + "0", + "-555", + "abc", + "555,678", + "content/00555.md", + "4294967296", + ] { + assert!( + Args::try_parse_from(["build-eips", "build", "--only", selector]).is_err(), + "expected `{selector}` to be rejected" + ); + } + + assert!(Args::try_parse_from(["build-eips", "serve", "--only", "555"]).is_err()); + assert!(Args::try_parse_from(["build-eips", "check", "--only", "555"]).is_err()); + assert!(Args::try_parse_from(["build-eips", "parity", "build", "--only", "555"]).is_err()); + } + #[test] fn base_url_flags_parse_on_build_and_serve_forms() { let cases: &[(&[&str], &str)] = &[ diff --git a/src/config.rs b/src/config.rs index c9aa8bc..9730aab 100644 --- a/src/config.rs +++ b/src/config.rs @@ -14,6 +14,8 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; use snafu::{Backtrace, IntoError, OptionExt, ResultExt, Snafu}; use url::{Position, Url}; +use crate::proposal::ProposalNumber; + pub const LOCAL_CONFIG_FILE: &str = ".build-eips.toml"; pub const REPO_MANIFEST_FILE: &str = ".build-eips.repo.toml"; pub const DEFAULT_BUILD_ROOT_BASE: &str = ".local-build"; @@ -468,6 +470,10 @@ pub struct WorkspaceConfig { /// Local rendered-site URL defaults for build and serve commands. #[serde(default)] pub site: SiteSettings, + + /// Local render filtering defaults. + #[serde(default)] + pub render: RenderSettings, } impl WorkspaceConfig { @@ -475,10 +481,20 @@ impl WorkspaceConfig { Self { server: ServerSettings::default(), site: SiteSettings::starter(), + render: RenderSettings::default(), } } } +/// Local render filtering settings. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct RenderSettings { + /// Proposal numbers to render for applicable local build commands. + #[serde(default)] + pub only: Vec, +} + /// Workspace-local bind address defaults for local server commands. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] @@ -642,6 +658,10 @@ impl LoadedWorkspaceConfig { &self.config.site } + pub fn render_settings(&self) -> &RenderSettings { + &self.config.render + } + pub fn local_theme_path(&self) -> PathBuf { self.workspace_root.join(DEFAULT_THEME_DIR) } @@ -689,6 +709,7 @@ mod tests { RepoManifestError, ServerBinding, ServerSettings, WorkspaceError, DEFAULT_SERVER_HOST, DEFAULT_SERVER_PORT, DEFAULT_SITE_BASE_URL, LOCAL_CONFIG_FILE, REPO_MANIFEST_FILE, }; + use crate::proposal::ProposalNumber; struct TestRepo { tempdir: TempDir, @@ -941,9 +962,10 @@ base_url = "https://staging.example.test/ERCs/" assert!(original.contains("port = 1111")); assert!(original.contains("[site]")); assert!(original.contains(&format!("base_url = \"{DEFAULT_SITE_BASE_URL}\""))); + assert!(original.contains("[render]")); + assert!(original.contains("only = []")); assert!(!original.contains("default_profile")); assert!(!original.contains("[profiles")); - assert!(!original.contains("[render]")); } #[test] @@ -1074,6 +1096,72 @@ base_url = "http://127.0.0.1:1111" assert_eq!(config.server_settings(), &ServerSettings::default()); assert!(config.site_settings().base_url.is_none()); + assert!(config.render_settings().only.is_empty()); + } + + #[test] + fn parses_workspace_config_render_only_settings() { + let repo = TestRepo::new(); + let config_path = repo.write_file( + LOCAL_CONFIG_FILE, + r#" +[render] +only = [555, 678, 555] +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!( + config.render_settings().only, + vec![ + ProposalNumber::from_u32(555).unwrap(), + ProposalNumber::from_u32(678).unwrap(), + ProposalNumber::from_u32(555).unwrap(), + ] + ); + } + + #[test] + fn missing_render_missing_only_and_empty_only_disable_filtering() { + let cases = [ + ("missing render", ""), + ("missing only", "[render]\n"), + ("empty only", "[render]\nonly = []\n"), + ]; + + for (name, contents) in cases { + let repo = TestRepo::new(); + let config_path = repo.write_file(LOCAL_CONFIG_FILE, contents); + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert!( + config.render_settings().only.is_empty(), + "expected `{name}` to disable render filtering" + ); + } + } + + #[test] + fn workspace_config_render_only_rejects_non_positive_and_non_integer_values() { + let cases = [ + ("zero", "only = [0]"), + ("negative", "only = [-555]"), + ("quoted", "only = [\"555\"]"), + ("overflow", "only = [4294967296]"), + ]; + + for (name, contents) in cases { + let repo = TestRepo::new(); + let config_path = + repo.write_file(LOCAL_CONFIG_FILE, &format!("[render]\n{contents}\n")); + let error = LoadedWorkspaceConfig::from_path(&config_path).unwrap_err(); + + assert!( + matches!(error, WorkspaceError::Parse { .. }), + "expected `{name}` render only config to fail, got {error:?}" + ); + } } #[test] diff --git a/src/editorial.rs b/src/editorial.rs index 33f8f10..e516004 100644 --- a/src/editorial.rs +++ b/src/editorial.rs @@ -573,6 +573,7 @@ base_url = "https://staging.example.test/{sibling_id}/" other_repos: Default::default(), }, theme_path: Some(PathBuf::from("/workspace/theme")), + only: None, source_materialization: crate::git::SourceMaterialization::Clean, server_binding: ServerBinding::default(), base_url_override: None, @@ -756,6 +757,7 @@ base_url = "https://staging.example.test/{sibling_id}/" other_repos: Default::default(), }, theme_path: Some(PathBuf::from("/workspace/theme")), + only: None, source_materialization: crate::git::SourceMaterialization::Clean, server_binding: ServerBinding::default(), base_url_override: None, diff --git a/src/execution.rs b/src/execution.rs index d0eb81e..bd75fa3 100644 --- a/src/execution.rs +++ b/src/execution.rs @@ -7,6 +7,7 @@ //! Execution source and path resolution. use std::{ + collections::BTreeSet, io::ErrorKind, path::{Path, PathBuf}, }; @@ -22,6 +23,7 @@ use crate::{ git, identity::ActiveRepoIdentity, layout::BUILD_DIR, + proposal::ProposalNumber, }; #[derive(Debug, Clone)] @@ -30,6 +32,7 @@ pub(crate) struct ResolvedExecution { pub(crate) build_path: PathBuf, pub(crate) repository_use: git::RepositoryUse, pub(crate) theme_path: Option, + pub(crate) only: Option>, pub(crate) source_materialization: git::SourceMaterialization, pub(crate) server_binding: ServerBinding, pub(crate) base_url_override: Option, @@ -113,6 +116,20 @@ fn explicit_environment_or_parity(args: &Args) -> Result, Whatever> Ok(None) } +fn cli_only_requested(args: &Args) -> bool { + args.operation + .only_cli_args() + .map(|only| !only.only.is_empty()) + .unwrap_or(false) +} + +fn only_cli_is_applicable(args: &Args, explicit_environment: Option) -> bool { + matches!(args.operation, Operation::Build { .. }) + && explicit_environment.is_none() + && !args.operation.clean_cli_args().clean + && !args.remote_siblings +} + pub(crate) fn resolve_execution_settings( args: &Args, sibling_ids: &[String], @@ -127,6 +144,10 @@ pub(crate) fn resolve_execution_settings( let sibling_override = remote_source_override(args.remote_siblings); let clean = args.operation.clean_cli_args().clean; + if cli_only_requested(args) && !only_cli_is_applicable(args, explicit_environment) { + snafu::whatever!("--only is supported only for local dirty build commands"); + } + let (staging, allow_dirty, default_sibling) = if let Some(staging) = explicit_environment { (staging, false, SelectedSource::Remote) } else if args.operation.is_plain_site_command() || args.operation.is_editorial_command() { @@ -167,6 +188,39 @@ pub(crate) fn resolve_execution_settings( }) } +fn dedupe_only_numbers(numbers: &[ProposalNumber]) -> Option> { + let numbers = numbers.iter().copied().collect::>(); + (!numbers.is_empty()).then_some(numbers) +} + +fn resolve_only_selection( + args: &Args, + settings: &ExecutionSettings, + workspace_config: Option<&LoadedWorkspaceConfig>, +) -> Result>, Whatever> { + let explicit_environment = explicit_environment_or_parity(args)?; + let applicable = matches!(args.operation, Operation::Build { .. }) + && explicit_environment.is_none() + && settings.allow_dirty + && settings.sibling == SelectedSource::WorkspaceLocal; + + if let Some(only) = args.operation.only_cli_args() { + if let Some(numbers) = dedupe_only_numbers(&only.only) { + if !applicable { + snafu::whatever!("--only is supported only for local dirty build commands"); + } + return Ok(Some(numbers)); + } + } + + if !applicable { + return Ok(None); + } + + Ok(workspace_config + .and_then(|workspace_config| dedupe_only_numbers(&workspace_config.render_settings().only))) +} + fn local_repo_url(path: &Path) -> Result { Url::from_directory_path(path) .ok() @@ -326,6 +380,7 @@ pub(crate) fn resolve_execution(args: &Args) -> Result Result Args { @@ -452,6 +508,17 @@ mod tests { assert!(!message.contains("--sibling-repo ")); } + fn only_selection_for( + arguments: &[&str], + workspace_config: Option<&LoadedWorkspaceConfig>, + ) -> Option> { + let args = parse_args(arguments); + let settings = resolve_execution_settings(&args, &[], workspace_config).unwrap(); + resolve_only_selection(&args, &settings, workspace_config) + .unwrap() + .map(|numbers| numbers.into_iter().map(|number| number.get()).collect()) + } + #[test] fn explicit_env_or_parity_provenance_is_classified_separately_from_local_defaults() { let cases: &[(&[&str], Option)] = &[ @@ -595,6 +662,86 @@ base_url = "http://localhost:4000" ); } + #[test] + fn build_only_cli_selection_overrides_config_and_dedupes() { + let workspace_config = load_workspace_config( + r#" +[render] +only = [555] +"#, + ); + + assert_eq!( + only_selection_for( + &["build-eips", "build", "--only", "678", "555", "678"], + Some(&workspace_config), + ), + Some(vec![555, 678]) + ); + } + + #[test] + fn build_only_config_selection_applies_to_local_dirty_build_only() { + let workspace_config = load_workspace_config( + r#" +[render] +only = [555, 678, 555] +"#, + ); + + assert_eq!( + only_selection_for(&["build-eips", "build"], Some(&workspace_config)), + Some(vec![555, 678]) + ); + + for arguments in [ + &["build-eips", "build", "--clean"][..], + &["build-eips", "--remote-siblings", "build"][..], + &["build-eips", "--staging", "build"][..], + &["build-eips", "--production", "build"][..], + &["build-eips", "serve"][..], + &["build-eips", "check"][..], + &["build-eips", "parity", "build"][..], + ] { + assert!(only_selection_for(arguments, Some(&workspace_config)).is_none()); + } + } + + #[test] + fn build_only_cli_selection_rejects_non_local_dirty_modes() { + let workspace_config = load_workspace_config(""); + + for arguments in [ + &["build-eips", "build", "--only", "555", "--clean"][..], + &["build-eips", "--remote-siblings", "build", "--only", "555"][..], + &["build-eips", "--staging", "build", "--only", "555"][..], + &["build-eips", "--production", "build", "--only", "555"][..], + ] { + let args = parse_args(arguments); + let error = resolve_execution_settings(&args, &[], Some(&workspace_config)) + .unwrap_err() + .to_string(); + + assert!(error.contains("--only is supported only for local dirty build commands")); + } + } + + #[test] + fn missing_render_config_and_empty_only_disable_filtering() { + let missing_render = load_workspace_config(""); + let missing_only = load_workspace_config("[render]\n"); + let empty_only = load_workspace_config( + r#" +[render] +only = [] +"#, + ); + + assert!(only_selection_for(&["build-eips", "build"], Some(&missing_render)).is_none()); + assert!(only_selection_for(&["build-eips", "build"], Some(&missing_only)).is_none()); + assert!(only_selection_for(&["build-eips", "build"], Some(&empty_only)).is_none()); + } + #[test] fn plain_site_commands_are_local_first_dirty_staging() { let workspace_config = load_workspace_config(""); diff --git a/src/markdown.rs b/src/markdown.rs index 06198e6..1418b1d 100644 --- a/src/markdown.rs +++ b/src/markdown.rs @@ -38,7 +38,10 @@ use walkdir::WalkDir; use iref::IriRefBuf; -use crate::progress::ProgressIteratorExt; +use crate::{ + progress::ProgressIteratorExt, + proposal::{OnlyRenderPlan, ProposalNumber, ProposalReference}, +}; #[derive(Debug, Serialize, Deserialize)] struct Author { @@ -232,7 +235,7 @@ fn extract_authors(value: &str) -> Result, Whatever> { Ok(authors) } -pub fn preprocess(root_path: &Path) -> Result<(), Whatever> { +pub fn preprocess(root_path: &Path, only_plan: Option<&OnlyRenderPlan>) -> Result<(), Whatever> { let dir = std::fs::read_dir(root_path).with_whatever_context(|_| { format!("could not read directory `{}`", root_path.to_string_lossy()) })?; @@ -268,10 +271,46 @@ pub fn preprocess(root_path: &Path) -> Result<(), Whatever> { } if file_type.is_dir() { - process_eip(root_path, &entry_path.join("index.md"))?; - process_assets(root_path, &entry_path)?; + let relative_path = entry_path + .strip_prefix(root_path) + .with_whatever_context(|_| { + format!( + "content directory entry `{}` is outside `{}`", + entry_path.to_string_lossy(), + root_path.to_string_lossy() + ) + })?; + if only_plan + .map(|plan| plan.should_process_proposal_dir(relative_path)) + .unwrap_or(true) + { + let index_path = entry_path.join("index.md"); + if let Some(plan) = only_plan { + let index_relative_path = relative_path.join("index.md"); + if plan.should_preprocess_markdown(&index_relative_path) { + process_eip(root_path, &index_path, only_plan)?; + } + } else { + process_eip(root_path, &index_path, only_plan)?; + } + process_assets(root_path, &entry_path, only_plan)?; + } } else if entry_path.extension().and_then(OsStr::to_str) == Some("md") { - process_eip(root_path, &entry_path)?; + let relative_path = entry_path + .strip_prefix(root_path) + .with_whatever_context(|_| { + format!( + "content file `{}` is outside `{}`", + entry_path.to_string_lossy(), + root_path.to_string_lossy() + ) + })?; + if only_plan + .map(|plan| plan.should_preprocess_markdown(relative_path)) + .unwrap_or(true) + { + process_eip(root_path, &entry_path, only_plan)?; + } } } @@ -336,6 +375,7 @@ fn canonicalize_md(path: &Path) -> Result { fn fix_links<'a, 'b>( root: &'a Path, parent: &'a Path, + only_plan: Option<&'a OnlyRenderPlan>, mut e: Event<'b>, ) -> Result, Whatever> { match &mut e { @@ -354,6 +394,31 @@ fn fix_links<'a, 'b>( return Ok(e); } + let iri_path: &str = iri_ref.path().as_ref(); + let child = if iri_path.starts_with("/") { + let mut path = Path::new(iri_path); + path = path.strip_prefix("/").unwrap(); + root.join(path) + } else { + parent.join(Path::new(iri_path)) + }; + let canonicalized = canonicalize_md(&child)?; + if let Some(public_url) = + only_plan.and_then(|plan| plan.external_url_for_canonical_target(&canonicalized)) + { + let mut external_url = public_url.to_owned(); + if let Some(query) = iri_ref.query() { + external_url.push('?'); + external_url.push_str(query.as_str()); + } + if let Some(fragment) = iri_ref.fragment() { + external_url.push('#'); + external_url.push_str(fragment.as_str()); + } + *dest_url = CowStr::from(external_url); + return Ok(e); + } + let canonicalized = path_to_at(root, parent, iri_ref.path())?; let path = iref::iri::Path::new(&canonicalized).expect("path is valid IRI"); iri_ref.set_path(path); @@ -429,7 +494,12 @@ impl RenderCsl { } } -fn transform_markdown(root: &Path, path: &Path, body: &str) -> Result { +fn transform_markdown( + root: &Path, + path: &Path, + body: &str, + only_plan: Option<&OnlyRenderPlan>, +) -> Result { let mut opts = Options::empty(); opts.insert(Options::ENABLE_TABLES); opts.insert(Options::ENABLE_FOOTNOTES); @@ -441,7 +511,7 @@ fn transform_markdown(root: &Path, path: &Path, body: &str) -> Result csl.render_csl(e).transpose(), err => Some(err), @@ -456,7 +526,11 @@ fn transform_markdown(root: &Path, path: &Path, body: &str) -> Result Result<(), Whatever> { +fn process_assets( + root: &Path, + path: &Path, + only_plan: Option<&OnlyRenderPlan>, +) -> Result<(), Whatever> { let canon_root = std::fs::canonicalize(root).whatever_context("could not canonicalize root")?; let number_txt = path .file_name() @@ -516,12 +590,13 @@ fn process_assets(root: &Path, path: &Path) -> Result<(), Whatever> { format!("could not read file `{}`", path.to_string_lossy()) })?; - let contents = transform_markdown(root, path, &contents).with_whatever_context(|_| { - format!( - "unable to transform markdown for `{}`", - path.to_string_lossy() - ) - })?; + let contents = + transform_markdown(root, path, &contents, only_plan).with_whatever_context(|_| { + format!( + "unable to transform markdown for `{}`", + path.to_string_lossy() + ) + })?; let relative_path = path.strip_prefix(&assets_dir).unwrap(); let relative_path = relative_path.with_file_name(relative_path.file_stem().unwrap()); @@ -556,7 +631,11 @@ fn process_assets(root: &Path, path: &Path) -> Result<(), Whatever> { Ok(()) } -fn process_eip(root: &Path, path: &Path) -> Result<(), Whatever> { +fn process_eip( + root: &Path, + path: &Path, + only_plan: Option<&OnlyRenderPlan>, +) -> Result<(), Whatever> { let path_lossy = path.to_string_lossy(); let contents = read_to_string(path) .with_whatever_context(|_| format!("could not read file `{}`", path_lossy))?; @@ -564,7 +643,7 @@ fn process_eip(root: &Path, path: &Path) -> Result<(), Whatever> { let (preamble, body) = Preamble::split(&contents) .with_whatever_context(|_| format!("couldn't split preamble for `{}`", path_lossy))?; - let body = transform_markdown(root, path, body) + let body = transform_markdown(root, path, body, only_plan) .with_whatever_context(|_| format!("unable to transform markdown for `{path_lossy}`"))?; let preamble = Preamble::parse(Some(&path_lossy), preamble) @@ -654,8 +733,24 @@ fn process_eip(root: &Path, path: &Path) -> Result<(), Whatever> { .whatever_context("could not parse requires")? .into_iter() .map(|eip| { - let path = format!("/{eip:0>5}.md"); - path_to_at(root, root, &path) + let proposal_number = match ProposalNumber::from_u32(eip) { + Ok(proposal_number) => proposal_number, + Err(()) => snafu::whatever!("could not parse requires"), + }; + match only_plan { + Some(plan) => { + match plan.reference_for_required_number(proposal_number)? { + ProposalReference::Internal(path) => Ok(path), + ProposalReference::External(public_url) => { + Ok(public_url.to_owned()) + } + } + } + None => { + let path = format!("/{eip:0>5}.md"); + path_to_at(root, root, &path) + } + } }) .collect::>()?; front_matter @@ -673,3 +768,222 @@ fn process_eip(root: &Path, path: &Path) -> Result<(), Whatever> { Ok(()) } + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + use std::path::{Path, PathBuf}; + + use git2::{IndexAddOption, Repository, Signature}; + use snafu::Report; + use tempfile::TempDir; + use toml::Value as TomlValue; + + use super::preprocess; + use crate::proposal::{OnlyRenderPlan, ProposalNumber}; + + fn number(value: u32) -> ProposalNumber { + ProposalNumber::from_u32(value).unwrap() + } + + fn write_file(root: &Path, relative: &str, contents: impl AsRef) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents.as_ref()).unwrap(); + } + + fn commit_all(repo: &Repository) { + let mut index = repo.index().unwrap(); + index + .add_all(["content"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + + repo.commit(Some("HEAD"), &signature, &signature, "initial", &tree, &[]) + .unwrap(); + } + + fn content_repo(files: &[(&str, String)]) -> (TempDir, PathBuf) { + let temp = TempDir::new().unwrap(); + let repo_root = temp.path().join("repo"); + let content_root = repo_root.join("content"); + std::fs::create_dir_all(&content_root).unwrap(); + let repo = Repository::init(&repo_root).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + + for (relative, contents) in files { + write_file(&content_root, relative, contents); + } + + commit_all(&repo); + (temp, content_root) + } + + fn proposal_markdown( + proposal_number: u32, + category: Option<&str>, + extra_preamble: &str, + body: &str, + ) -> String { + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!( + "---\neip: {proposal_number}\ntitle: Proposal {proposal_number}\n{category}{extra_preamble}---\n{body}\n" + ) + } + + fn only_plan(content_root: &Path, selected: &[u32]) -> OnlyRenderPlan { + let selected = selected + .iter() + .copied() + .map(number) + .collect::>(); + OnlyRenderPlan::build(content_root, selected).unwrap() + } + + fn rendered_body(path: &Path) -> String { + let contents = std::fs::read_to_string(path).unwrap(); + contents.split_once("\n+++\n").unwrap().1.to_owned() + } + + fn rendered_front_matter(path: &Path) -> TomlValue { + let contents = std::fs::read_to_string(path).unwrap(); + let front_matter = contents + .strip_prefix("+++\n") + .unwrap() + .split_once("\n+++\n") + .unwrap() + .0; + toml::from_str(front_matter).unwrap() + } + + #[test] + fn targeted_preprocess_rewrites_selected_body_links_to_unselected_public_urls() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [ERC-678](/00678.md)."), + ), + ( + "00678.md", + proposal_markdown(678, Some("ERC"), "", "Target."), + ), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("https://ercs.ethereum.org/ERCS/erc-678")); + assert!(!body.contains("@/00678.md")); + } + + #[test] + fn targeted_preprocess_rewrites_retained_non_proposal_links_to_public_urls() { + let (_temp, content) = content_repo(&[ + ( + "_index.md", + "---\ntitle: Home\n---\nSee [EIP-678](/00678.md).\n".to_owned(), + ), + ("00555.md", proposal_markdown(555, None, "", "Selected.")), + ("00678.md", proposal_markdown(678, None, "", "Unselected.")), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("_index.md")); + assert!(body.contains("https://eips.ethereum.org/EIPS/eip-678")); + assert!(!body.contains("@/00678.md")); + } + + #[test] + fn targeted_preprocess_preserves_query_and_fragment_on_external_links() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "", + "See [Fragment](./00155.md#list-of-chain-id-s).\nSee [Query](./00155.md?foo=bar#list-of-chain-id-s).", + ), + ), + ("00155.md", proposal_markdown(155, None, "", "Unselected.")), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("https://eips.ethereum.org/EIPS/eip-155#list-of-chain-id-s")); + assert!(body.contains("https://eips.ethereum.org/EIPS/eip-155?foo=bar#list-of-chain-id-s")); + } + + #[test] + fn targeted_preprocess_rewrites_requires_to_unselected_public_urls() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "requires: 678\n", "Selected."), + ), + ( + "00678.md", + proposal_markdown(678, Some("ERC"), "", "Target."), + ), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let front_matter = rendered_front_matter(&content.join("00555.md")); + let requires = front_matter["extra"]["requires"].as_array().unwrap(); + assert_eq!( + requires[0].as_str().unwrap(), + "https://ercs.ethereum.org/ERCS/erc-678" + ); + } + + #[test] + fn targeted_preprocess_keeps_internal_references_between_selected_proposals() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "requires: 678\n", "See [EIP-678](/00678.md)."), + ), + ( + "00678.md", + proposal_markdown(678, Some("ERC"), "", "Target."), + ), + ]); + let plan = only_plan(&content, &[555, 678]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + let front_matter = rendered_front_matter(&content.join("00555.md")); + let requires = front_matter["extra"]["requires"].as_array().unwrap(); + assert!(body.contains("@/00678.md")); + assert_eq!(requires[0].as_str().unwrap(), "@/00678.md"); + } + + #[test] + fn targeted_preprocess_does_not_mask_missing_body_link_targets() { + let (_temp, content) = content_repo(&[( + "00555.md", + proposal_markdown(555, None, "", "See [Missing](/00678.md)."), + )]); + let plan = only_plan(&content, &[555]); + + let error = Report::from_error(preprocess(&content, Some(&plan)).unwrap_err()).to_string(); + + assert!(error.contains("could not canonicalize")); + assert!(error.contains("00678.md")); + } +} diff --git a/src/pipeline.rs b/src/pipeline.rs index 4858f68..a83fd89 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -17,6 +17,7 @@ use crate::{ git, layout::{mounted_theme_path, output_path, CONTENT_DIR, REPO_DIR}, markdown, + proposal::OnlyRenderPlan, serve::{serve_sync_config, DirtyServeWatcher, LocalThemeServeSync}, zola, }; @@ -71,6 +72,7 @@ pub(crate) struct Prepared { repository_use: git::RepositoryUse, theme_path: PathBuf, local_theme_sync: Option, + only_plan: Option, source_root: PathBuf, source_materialization: git::SourceMaterialization, server_binding: ServerBinding, @@ -86,6 +88,7 @@ impl Prepared { build_path, repository_use, theme_path, + only, source_materialization, server_binding, base_url_override, @@ -104,7 +107,17 @@ impl Prepared { source_materialization, )?; - markdown::preprocess(&content_path).whatever_context("unable to preprocess markdown")?; + let only_plan = only + .map(|selected_numbers| OnlyRenderPlan::build(&content_path, selected_numbers)) + .transpose() + .whatever_context("unable to build targeted render plan")?; + markdown::preprocess(&content_path, only_plan.as_ref()) + .whatever_context("unable to preprocess markdown")?; + if let Some(only_plan) = &only_plan { + only_plan + .prune_content(&content_path) + .whatever_context("unable to prune unselected proposals")?; + } let (theme_path, local_theme_sync) = prepare_theme_for_zola(theme_path, &repo_path)?; Ok(Prepared { @@ -113,6 +126,7 @@ impl Prepared { local_theme_sync: Some(local_theme_sync), repo_path, output_path, + only_plan, source_root: root_path, source_materialization, server_binding, diff --git a/src/serve.rs b/src/serve.rs index cf347cb..a1153b7 100644 --- a/src/serve.rs +++ b/src/serve.rs @@ -140,7 +140,7 @@ fn sync_dirty_serve_state( git::sync_materialized_paths(source_root, build_repo_path, &affected_paths) .whatever_context("unable to synchronize tracked paths into the materialized repo")?; - markdown::preprocess(&build_repo_path.join(CONTENT_DIR)) + markdown::preprocess(&build_repo_path.join(CONTENT_DIR), None) .whatever_context("unable to preprocess synchronized markdown during dirty serve")?; info!( From 9e42fa85eff612e3a04cb144646ffef0b659ad6c Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Wed, 6 May 2026 00:53:08 -0400 Subject: [PATCH 14/20] Fix cross-proposal asset links Resolve cross-proposal asset links before Zola sees prepared markdown. Add proposal asset path resolution, rendered URL builders, and an OnlyRenderPlan asset inventory so links can be validated before targeted pruning removes omitted proposal content. Rewrite static asset links to rendered relative URLs when targets are available locally, and to public EIP/ERC asset URLs when targeted rendering omits the target proposal. Keep selected asset markdown links on the existing Zola @/... path, while omitted asset markdown links use public page URLs. Preserve query strings and fragments, leave fragment-only and raw HTML links untouched, and skip already-generated Zola markdown so repeated preprocessing remains idempotent. --- src/markdown.rs | 1278 ++++++++++++++++++++++++++++++++++++++++++++++- src/proposal.rs | 388 +++++++++++++- 2 files changed, 1634 insertions(+), 32 deletions(-) diff --git a/src/markdown.rs b/src/markdown.rs index 1418b1d..e8432d0 100644 --- a/src/markdown.rs +++ b/src/markdown.rs @@ -26,7 +26,7 @@ use std::collections::HashMap; use std::ffi::OsStr; use std::fs::read_to_string; use std::io::Write; -use std::path::{Path, PathBuf}; +use std::path::{Component, Path, PathBuf}; use snafu::{whatever, OptionExt, ResultExt, Whatever}; @@ -40,7 +40,10 @@ use iref::IriRefBuf; use crate::{ progress::ProgressIteratorExt, - proposal::{OnlyRenderPlan, ProposalNumber, ProposalReference}, + proposal::{ + path_component_proposal_number, proposal_number_from_content_markdown_path, OnlyRenderPlan, + ProposalAssetKind, ProposalNumber, ProposalReference, + }, }; #[derive(Debug, Serialize, Deserialize)] @@ -235,6 +238,452 @@ fn extract_authors(value: &str) -> Result, Whatever> { Ok(authors) } +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ProposalAssetPathResolution { + NotAProposalAsset, + ProposalAsset(ProposalAssetPath), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ProposalAssetPath { + pub(crate) target_proposal_number: ProposalNumber, + pub(crate) content_relative_asset_path: PathBuf, + pub(crate) asset_relative_path: PathBuf, + pub(crate) kind: ProposalAssetKind, + pub(crate) rendered_target_path: String, +} + +#[derive(Debug)] +struct DecodedPathSegment { + value: String, +} + +pub(crate) fn resolve_proposal_asset_path( + content_root: &Path, + source_md_path: &Path, + iri_path: &str, +) -> Result { + let source_parent = source_md_path.parent().with_whatever_context(|| { + format!( + "source markdown path `{}` has no parent", + source_md_path.to_string_lossy() + ) + })?; + let normalized_root = normalize_path_lexically(content_root); + if !raw_iri_resolves_to_proposal_asset_candidate( + &normalized_root, + content_root, + source_parent, + iri_path, + ) { + return Ok(ProposalAssetPathResolution::NotAProposalAsset); + } + + let decoded_segments = decode_iri_path_segments(iri_path)?; + reject_unsafe_asset_segments(&decoded_segments)?; + + let target_path = + resolve_url_path_lexically(content_root, source_parent, iri_path, &decoded_segments); + let normalized_target = normalize_path_lexically(&target_path); + let Ok(content_relative_path) = normalized_target.strip_prefix(&normalized_root) else { + return Ok(ProposalAssetPathResolution::NotAProposalAsset); + }; + + let Some((target_proposal_number, asset_relative_path)) = + proposal_asset_parts(content_relative_path) + else { + return Ok(ProposalAssetPathResolution::NotAProposalAsset); + }; + + let kind = if iri_path.ends_with(".md") { + ProposalAssetKind::Markdown + } else { + ProposalAssetKind::Static + }; + let rendered_target_path = + rendered_asset_path(target_proposal_number, &asset_relative_path, kind)?; + + Ok(ProposalAssetPathResolution::ProposalAsset( + ProposalAssetPath { + target_proposal_number, + content_relative_asset_path: content_relative_path.to_path_buf(), + asset_relative_path, + kind, + rendered_target_path, + }, + )) +} + +pub(crate) fn absolute_rendered_path_for_content_path( + content_relative_path: &Path, +) -> Result { + if content_relative_path == Path::new("_index.md") { + return Ok("/".to_owned()); + } + + if let Some(proposal_number) = proposal_number_from_content_markdown_path(content_relative_path) + { + return Ok(format!("/{proposal_number}/")); + } + + if let Some((proposal_number, asset_relative_path)) = + proposal_asset_parts(content_relative_path) + { + return rendered_asset_path( + proposal_number, + &asset_relative_path, + ProposalAssetKind::from_path(&asset_relative_path), + ); + } + + snafu::whatever!( + "content path `{}` is not a proposal page or proposal asset", + content_relative_path.to_string_lossy() + ); +} + +pub(crate) fn relative_url_from_rendered_paths( + source_rendered_path: &str, + target_rendered_path: &str, +) -> Result { + let source_segments = rendered_directory_segments(source_rendered_path)?; + let (target_segments, target_is_directory) = rendered_path_segments(target_rendered_path)?; + let common_len = source_segments + .iter() + .zip(target_segments.iter()) + .take_while(|(source, target)| source == target) + .count(); + + let mut relative_segments = Vec::new(); + relative_segments.extend(std::iter::repeat_n( + "..", + source_segments.len() - common_len, + )); + relative_segments.extend(target_segments[common_len..].iter().copied()); + + let mut relative_url = if relative_segments.is_empty() { + ".".to_owned() + } else { + relative_segments.join("/") + }; + if target_is_directory && !relative_url.ends_with('/') { + relative_url.push('/'); + } + + Ok(relative_url) +} + +pub(crate) fn proposal_asset_exists_in_content_tree( + content_root: &Path, + content_relative_asset_path: &Path, +) -> bool { + if !content_relative_asset_path + .components() + .all(|component| matches!(component, Component::Normal(_))) + { + return false; + } + + let Ok(canonical_content_root) = std::fs::canonicalize(content_root) else { + return false; + }; + let Ok(canonical_target) = + std::fs::canonicalize(content_root.join(content_relative_asset_path)) + else { + return false; + }; + + if !canonical_target.starts_with(canonical_content_root) { + return false; + } + + std::fs::metadata(canonical_target) + .map(|metadata| metadata.is_file()) + .unwrap_or(false) +} + +fn decode_iri_path_segments(iri_path: &str) -> Result, Whatever> { + iri_path + .split('/') + .map(|segment| { + Ok(DecodedPathSegment { + value: percent_decode_url_segment(segment)?, + }) + }) + .collect::, _>>() +} + +fn percent_decode_url_segment(segment: &str) -> Result { + let bytes = segment.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut index = 0; + + while index < bytes.len() { + if bytes[index] != b'%' { + decoded.push(bytes[index]); + index += 1; + continue; + } + + if index + 2 >= bytes.len() { + snafu::whatever!("invalid percent encoding in URL path segment `{segment}`"); + } + + let high = hex_value(bytes[index + 1]).with_whatever_context(|| { + format!("invalid percent encoding in URL path segment `{segment}`") + })?; + let low = hex_value(bytes[index + 2]).with_whatever_context(|| { + format!("invalid percent encoding in URL path segment `{segment}`") + })?; + let value = (high << 4) | low; + if matches!(value, b'/' | b'\\' | b'\0') { + snafu::whatever!("unsafe percent encoding in URL path segment `{segment}`"); + } + decoded.push(value); + index += 3; + } + + String::from_utf8(decoded) + .with_whatever_context(|_| format!("URL path segment `{segment}` is not UTF-8")) +} + +fn hex_value(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + b'A'..=b'F' => Some(byte - b'A' + 10), + _ => None, + } +} + +fn reject_unsafe_asset_segments(segments: &[DecodedPathSegment]) -> Result<(), Whatever> { + let Some(assets_index) = segments.windows(2).position(|window| { + path_component_proposal_number(Some(OsStr::new(window[0].value.as_str()))).is_some() + && window[1].value == "assets" + }) else { + return Ok(()); + }; + + for segment in &segments[assets_index + 2..] { + if segment.value.is_empty() { + continue; + } + if segment.value == "." || segment.value == ".." { + snafu::whatever!("unsafe proposal asset path segment `{}`", segment.value); + } + if segment.value.contains(['/', '\\', '\0']) { + snafu::whatever!("unsafe proposal asset path segment `{}`", segment.value); + } + } + + Ok(()) +} + +fn resolve_url_path_lexically( + content_root: &Path, + source_parent: &Path, + iri_path: &str, + decoded_segments: &[DecodedPathSegment], +) -> PathBuf { + let mut path = if iri_path.starts_with('/') { + content_root.to_path_buf() + } else { + source_parent.to_path_buf() + }; + + for segment in decoded_segments { + match segment.value.as_str() { + "" | "." => {} + ".." => { + path.pop(); + } + _ => path.push(&segment.value), + } + } + + path +} + +fn raw_iri_resolves_to_proposal_asset_candidate( + normalized_root: &Path, + content_root: &Path, + source_parent: &Path, + iri_path: &str, +) -> bool { + let mut path = if iri_path.starts_with('/') { + content_root.to_path_buf() + } else { + source_parent.to_path_buf() + }; + let raw_segments = iri_path.split('/').collect::>(); + + for (index, segment) in raw_segments.iter().enumerate() { + match *segment { + "" | "." => {} + ".." => { + path.pop(); + } + _ => path.push(segment), + } + + let normalized_path = normalize_path_lexically(&path); + let Ok(content_relative_path) = normalized_path.strip_prefix(normalized_root) else { + continue; + }; + + if proposal_asset_parts(content_relative_path).is_some() { + return true; + } + + if proposal_asset_dir_prefix(content_relative_path).is_some() + && raw_segments[index + 1..] + .iter() + .any(|remaining_segment| !remaining_segment.is_empty()) + { + return true; + } + } + + false +} + +fn normalize_path_lexically(path: &Path) -> PathBuf { + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), + Component::RootDir => normalized.push(component.as_os_str()), + Component::CurDir => {} + Component::ParentDir => { + normalized.pop(); + } + Component::Normal(part) => normalized.push(part), + } + } + normalized +} + +fn proposal_asset_dir_prefix(content_relative_path: &Path) -> Option { + let mut components = content_relative_path.components(); + let proposal_component = components.next()?; + let assets_component = components.next()?; + if components.next().is_some() || assets_component.as_os_str() != OsStr::new("assets") { + return None; + } + + path_component_proposal_number(Some(proposal_component.as_os_str())) +} + +fn proposal_asset_parts(content_relative_path: &Path) -> Option<(ProposalNumber, PathBuf)> { + let mut components = content_relative_path.components(); + let proposal_component = components.next()?; + let assets_component = components.next()?; + if assets_component.as_os_str() != OsStr::new("assets") { + return None; + } + + let proposal_number = path_component_proposal_number(Some(proposal_component.as_os_str()))?; + let asset_relative_path = components.as_path(); + if asset_relative_path.as_os_str().is_empty() { + return None; + } + if !asset_relative_path + .components() + .all(|component| matches!(component, Component::Normal(_))) + { + return None; + } + + Some((proposal_number, asset_relative_path.to_path_buf())) +} + +fn rendered_asset_path( + proposal_number: ProposalNumber, + asset_relative_path: &Path, + kind: ProposalAssetKind, +) -> Result { + let mut segments = vec![proposal_number.to_string(), "assets".to_owned()]; + let asset_segments = asset_relative_path + .components() + .map(|component| match component { + Component::Normal(part) => { + part.to_str().map(str::to_owned).with_whatever_context(|| { + format!( + "non-UTF-8 proposal asset path `{}`", + asset_relative_path.to_string_lossy() + ) + }) + } + _ => snafu::whatever!( + "unsupported proposal asset path component in `{}`", + asset_relative_path.to_string_lossy() + ), + }) + .collect::, _>>()?; + + let last_index = asset_segments.len().saturating_sub(1); + for (index, mut segment) in asset_segments.into_iter().enumerate() { + if kind == ProposalAssetKind::Markdown && index == last_index { + segment = segment + .strip_suffix(".md") + .with_whatever_context(|| { + format!( + "proposal asset markdown path `{}` does not end in `.md`", + asset_relative_path.to_string_lossy() + ) + })? + .to_owned(); + } + segments.push(percent_encode_url_segment(&segment)); + } + + let mut rendered_path = format!("/{}", segments.join("/")); + if kind == ProposalAssetKind::Markdown { + rendered_path.push('/'); + } + + Ok(rendered_path) +} + +fn percent_encode_url_segment(segment: &str) -> String { + let mut encoded = String::with_capacity(segment.len()); + for byte in segment.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +fn rendered_directory_segments(rendered_path: &str) -> Result, Whatever> { + let (mut segments, is_directory) = rendered_path_segments(rendered_path)?; + if !is_directory { + segments.pop(); + } + Ok(segments) +} + +fn rendered_path_segments(rendered_path: &str) -> Result<(Vec<&str>, bool), Whatever> { + if !rendered_path.starts_with('/') { + snafu::whatever!("rendered path `{rendered_path}` is not absolute"); + } + + let is_directory = rendered_path.ends_with('/'); + let segments = rendered_path + .trim_matches('/') + .split('/') + .filter(|segment| !segment.is_empty()) + .collect(); + + Ok((segments, is_directory)) +} + +fn is_generated_zola_markdown(contents: &str) -> bool { + contents.starts_with("+++\n") +} + pub fn preprocess(root_path: &Path, only_plan: Option<&OnlyRenderPlan>) -> Result<(), Whatever> { let dir = std::fs::read_dir(root_path).with_whatever_context(|_| { format!("could not read directory `{}`", root_path.to_string_lossy()) @@ -372,9 +821,107 @@ fn canonicalize_md(path: &Path) -> Result { }) } +enum AssetLinkRewrite { + Rewrite(String), + FallThrough, +} + +fn resolve_asset_link_rewrite( + root: &Path, + source_md_path: &Path, + only_plan: Option<&OnlyRenderPlan>, + iri_path: &str, +) -> Result, Whatever> { + if iri_path.is_empty() { + return Ok(None); + } + + let ProposalAssetPathResolution::ProposalAsset(asset_path) = + resolve_proposal_asset_path(root, source_md_path, iri_path)? + else { + return Ok(None); + }; + + if let Some(plan) = only_plan { + if let Some(public_url) = + plan.public_url_for_omitted_proposal_asset(&asset_path.content_relative_asset_path) + { + return Ok(Some(AssetLinkRewrite::Rewrite(public_url))); + } + + if plan.has_proposal_asset(&asset_path.content_relative_asset_path) { + validate_local_proposal_asset(root, source_md_path, iri_path, &asset_path)?; + return local_asset_link_rewrite(root, source_md_path, asset_path).map(Some); + } + + snafu::whatever!( + "proposal asset link `{iri_path}` in `{}` resolved to `{}` but was not found in targeted render inventory", + source_md_path.to_string_lossy(), + asset_path.content_relative_asset_path.to_string_lossy() + ); + } + + validate_local_proposal_asset(root, source_md_path, iri_path, &asset_path)?; + local_asset_link_rewrite(root, source_md_path, asset_path).map(Some) +} + +fn validate_local_proposal_asset( + root: &Path, + source_md_path: &Path, + iri_path: &str, + asset_path: &ProposalAssetPath, +) -> Result<(), Whatever> { + if proposal_asset_exists_in_content_tree(root, &asset_path.content_relative_asset_path) { + return Ok(()); + } + + snafu::whatever!( + "proposal asset link `{iri_path}` in `{}` resolved to missing asset `{}`", + source_md_path.to_string_lossy(), + asset_path.content_relative_asset_path.to_string_lossy() + ); +} + +fn local_asset_link_rewrite( + root: &Path, + source_md_path: &Path, + asset_path: ProposalAssetPath, +) -> Result { + if asset_path.kind == ProposalAssetKind::Markdown { + return Ok(AssetLinkRewrite::FallThrough); + } + + let source_relative_path = source_md_path + .strip_prefix(root) + .with_whatever_context(|_| { + format!( + "source markdown `{}` is outside content root `{}`", + source_md_path.to_string_lossy(), + root.to_string_lossy() + ) + })?; + let source_rendered_path = absolute_rendered_path_for_content_path(source_relative_path)?; + let relative_url = + relative_url_from_rendered_paths(&source_rendered_path, &asset_path.rendered_target_path)?; + + Ok(AssetLinkRewrite::Rewrite(relative_url)) +} + +fn append_query_and_fragment(mut url: String, iri_ref: &IriRefBuf) -> String { + if let Some(query) = iri_ref.query() { + url.push('?'); + url.push_str(query.as_str()); + } + if let Some(fragment) = iri_ref.fragment() { + url.push('#'); + url.push_str(fragment.as_str()); + } + url +} + fn fix_links<'a, 'b>( root: &'a Path, - parent: &'a Path, + source_md_path: &'a Path, only_plan: Option<&'a OnlyRenderPlan>, mut e: Event<'b>, ) -> Result, Whatever> { @@ -384,17 +931,31 @@ fn fix_links<'a, 'b>( .map_err(|e| e.to_string()) .whatever_context("invalid URL in image/link")?; - if iri_ref.authority().is_some() { + if iri_ref.scheme().is_some() || iri_ref.authority().is_some() { // Is a protocol-relative or absolute URL. return Ok(e); } + let iri_path: &str = iri_ref.path().as_ref(); + match resolve_asset_link_rewrite(root, source_md_path, only_plan, iri_path)? { + Some(AssetLinkRewrite::Rewrite(url)) => { + *dest_url = CowStr::from(append_query_and_fragment(url, &iri_ref)); + return Ok(e); + } + Some(AssetLinkRewrite::FallThrough) | None => {} + } + if !iri_ref.path().ends_with(".md") { // Only markdown files need the `@` syntax. return Ok(e); } - let iri_path: &str = iri_ref.path().as_ref(); + let parent = source_md_path.parent().with_whatever_context(|| { + format!( + "source markdown path `{}` has no parent", + source_md_path.to_string_lossy() + ) + })?; let child = if iri_path.starts_with("/") { let mut path = Path::new(iri_path); path = path.strip_prefix("/").unwrap(); @@ -406,16 +967,8 @@ fn fix_links<'a, 'b>( if let Some(public_url) = only_plan.and_then(|plan| plan.external_url_for_canonical_target(&canonicalized)) { - let mut external_url = public_url.to_owned(); - if let Some(query) = iri_ref.query() { - external_url.push('?'); - external_url.push_str(query.as_str()); - } - if let Some(fragment) = iri_ref.fragment() { - external_url.push('#'); - external_url.push_str(fragment.as_str()); - } - *dest_url = CowStr::from(external_url); + *dest_url = + CowStr::from(append_query_and_fragment(public_url.to_owned(), &iri_ref)); return Ok(e); } @@ -507,11 +1060,10 @@ fn transform_markdown( opts.insert(Options::ENABLE_TASKLISTS); opts.insert(Options::ENABLE_HEADING_ATTRIBUTES); - let parent = path.parent().unwrap(); let mut csl = RenderCsl { contents: None }; let events = Parser::new_ext(body, opts) - .map(|e| fix_links(root, parent, only_plan, e)) + .map(|e| fix_links(root, path, only_plan, e)) .filter_map(|r| match r { Ok(e) => csl.render_csl(e).transpose(), err => Some(err), @@ -589,6 +1141,9 @@ fn process_assets( let contents = read_to_string(path).with_whatever_context(|_| { format!("could not read file `{}`", path.to_string_lossy()) })?; + if is_generated_zola_markdown(&contents) { + continue; + } let contents = transform_markdown(root, path, &contents, only_plan).with_whatever_context(|_| { @@ -639,6 +1194,9 @@ fn process_eip( let path_lossy = path.to_string_lossy(); let contents = read_to_string(path) .with_whatever_context(|_| format!("could not read file `{}`", path_lossy))?; + if is_generated_zola_markdown(&contents) { + return Ok(()); + } let (preamble, body) = Preamble::split(&contents) .with_whatever_context(|_| format!("couldn't split preamble for `{}`", path_lossy))?; @@ -779,7 +1337,12 @@ mod tests { use tempfile::TempDir; use toml::Value as TomlValue; - use super::preprocess; + use super::{ + absolute_rendered_path_for_content_path, preprocess, proposal_asset_exists_in_content_tree, + relative_url_from_rendered_paths, resolve_proposal_asset_path, ProposalAssetPath, + ProposalAssetPathResolution, + }; + use crate::proposal::ProposalAssetKind; use crate::proposal::{OnlyRenderPlan, ProposalNumber}; fn number(value: u32) -> ProposalNumber { @@ -863,6 +1426,685 @@ mod tests { toml::from_str(front_matter).unwrap() } + fn resolved_asset( + content_root: &Path, + source_md_path: &Path, + iri_path: &str, + ) -> ProposalAssetPath { + match resolve_proposal_asset_path(content_root, source_md_path, iri_path).unwrap() { + ProposalAssetPathResolution::ProposalAsset(asset_path) => asset_path, + ProposalAssetPathResolution::NotAProposalAsset => { + panic!("expected `{iri_path}` to resolve as proposal asset") + } + } + } + + #[test] + fn resolver_detects_flat_source_cross_proposal_static_asset() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let asset_path = resolved_asset(&content, &source, "./00678/assets/foo.pdf"); + + assert_eq!(asset_path.target_proposal_number, number(678)); + assert_eq!( + asset_path.content_relative_asset_path, + Path::new("00678/assets/foo.pdf") + ); + assert_eq!(asset_path.asset_relative_path, Path::new("foo.pdf")); + assert_eq!(asset_path.kind, ProposalAssetKind::Static); + assert_eq!(asset_path.rendered_target_path, "/678/assets/foo.pdf"); + } + + #[test] + fn resolver_detects_directory_source_cross_proposal_static_asset() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555/index.md"); + + let asset_path = resolved_asset(&content, &source, "../00678/assets/foo.pdf"); + + assert_eq!( + asset_path.content_relative_asset_path, + Path::new("00678/assets/foo.pdf") + ); + assert_eq!(asset_path.rendered_target_path, "/678/assets/foo.pdf"); + } + + #[test] + fn resolver_detects_asset_markdown_lexically_without_filesystem() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555/assets/guide.md"); + + let asset_path = resolved_asset(&content, &source, "../../00678/assets/guide.md"); + + assert_eq!(asset_path.kind, ProposalAssetKind::Markdown); + assert_eq!( + asset_path.content_relative_asset_path, + Path::new("00678/assets/guide.md") + ); + assert_eq!(asset_path.rendered_target_path, "/678/assets/guide/"); + } + + #[test] + fn resolver_decodes_safe_percent_paths_and_renders_encoded_urls() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let asset_path = resolved_asset( + &content, + &source, + "./00678/assets/Contract%20Interactions%20diagram.svg", + ); + + assert_eq!( + asset_path.content_relative_asset_path, + Path::new("00678/assets/Contract Interactions diagram.svg") + ); + assert_eq!( + asset_path.rendered_target_path, + "/678/assets/Contract%20Interactions%20diagram.svg" + ); + } + + #[test] + fn resolver_rejects_unsafe_percent_and_asset_segments() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + for iri_path in [ + "./00678/assets/foo%2Fbar.pdf", + "./00678/assets/foo%5Cbar.pdf", + "./00678/assets/foo%00bar.pdf", + "./00678/assets/.", + "./00678/assets/..", + "./00678/assets/%2E", + "./00678/assets/%2E%2E", + ] { + let error = resolve_proposal_asset_path(&content, &source, iri_path) + .unwrap_err() + .to_string(); + assert!( + error.contains("unsafe"), + "expected unsafe path error for `{iri_path}`, got `{error}`" + ); + } + } + + #[test] + fn resolver_allows_unsafe_percent_encodings_for_non_proposal_paths() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let resolution = + resolve_proposal_asset_path(&content, &source, "./images/foo%2Fbar.pdf").unwrap(); + + assert_eq!(resolution, ProposalAssetPathResolution::NotAProposalAsset); + } + + #[test] + fn resolver_still_rejects_unsafe_percent_encodings_for_proposal_assets() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let error = resolve_proposal_asset_path(&content, &source, "./00678/assets/foo%2Fbar.pdf") + .unwrap_err() + .to_string(); + + assert!(error.contains("unsafe")); + } + + #[test] + fn resolver_returns_passthrough_for_outside_root_without_error() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let resolution = + resolve_proposal_asset_path(&content, &source, "../elsewhere/foo.pdf").unwrap(); + + assert_eq!(resolution, ProposalAssetPathResolution::NotAProposalAsset); + } + + #[test] + fn resolver_returns_passthrough_for_non_proposal_asset_paths() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let resolution = + resolve_proposal_asset_path(&content, &source, "./images/foo.pdf").unwrap(); + + assert_eq!(resolution, ProposalAssetPathResolution::NotAProposalAsset); + } + + #[test] + fn rendered_path_helper_maps_proposal_and_asset_content_paths() { + for (content_relative_path, expected_rendered_path) in [ + ("_index.md", "/"), + ("00555.md", "/555/"), + ("00555/index.md", "/555/"), + ("00555/assets/guide.md", "/555/assets/guide/"), + ("00555/assets/README.md", "/555/assets/README/"), + ("00555/assets/index.md", "/555/assets/index/"), + ("00678/assets/foo.pdf", "/678/assets/foo.pdf"), + ( + "00678/assets/Contract Interactions diagram.svg", + "/678/assets/Contract%20Interactions%20diagram.svg", + ), + ] { + assert_eq!( + absolute_rendered_path_for_content_path(Path::new(content_relative_path)).unwrap(), + expected_rendered_path + ); + } + } + + #[test] + fn relative_url_helper_uses_rendered_paths() { + assert_eq!( + relative_url_from_rendered_paths("/555/", "/678/assets/foo.pdf").unwrap(), + "../678/assets/foo.pdf" + ); + assert_eq!( + relative_url_from_rendered_paths("/555/assets/guide/", "/678/assets/foo.pdf").unwrap(), + "../../../678/assets/foo.pdf" + ); + assert_eq!( + relative_url_from_rendered_paths("/555/", "/678/assets/guide/").unwrap(), + "../678/assets/guide/" + ); + } + + #[test] + fn filesystem_validator_checks_content_relative_assets() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + write_file(&content, "00678/assets/foo.pdf", ""); + + assert!(proposal_asset_exists_in_content_tree( + &content, + Path::new("00678/assets/foo.pdf") + )); + assert!(!proposal_asset_exists_in_content_tree( + &content, + Path::new("00678/assets/missing.pdf") + )); + assert!(!proposal_asset_exists_in_content_tree( + &content, + Path::new("../00678/assets/foo.pdf") + )); + } + + #[cfg(unix)] + #[test] + fn filesystem_validator_rejects_symlink_targets_outside_content_root() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + write_file(&content, "00678/assets/placeholder", ""); + let outside = temp.path().join("outside.pdf"); + std::fs::write(&outside, "").unwrap(); + std::os::unix::fs::symlink(&outside, content.join("00678/assets/outside.pdf")).unwrap(); + + assert!(!proposal_asset_exists_in_content_tree( + &content, + Path::new("00678/assets/outside.pdf") + )); + } + + #[test] + fn preprocess_rewrites_flat_source_cross_proposal_static_asset() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](./00678/assets/foo.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(../678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_rewrites_directory_source_cross_proposal_static_asset() { + let (_temp, content) = content_repo(&[ + ( + "00555/index.md", + proposal_markdown(555, None, "", "See [asset](../00678/assets/foo.pdf)."), + ), + ("00555/assets/.keep", "".to_owned()), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555/index.md")); + assert!(body.contains("(../678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_rewrites_root_index_source_cross_proposal_static_asset() { + let (_temp, content) = content_repo(&[ + ( + "_index.md", + "---\ntitle: Home\n---\nSee [asset](/00678/assets/foo.pdf).\n".to_owned(), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("_index.md")); + assert!(body.contains("(678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_rewrites_asset_markdown_source_using_rendered_source_path() { + let (_temp, content) = content_repo(&[ + ( + "00555/index.md", + proposal_markdown(555, None, "", "Source."), + ), + ( + "00555/assets/guide.md", + "See [asset](../../00678/assets/foo.pdf).".to_owned(), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555/assets/guide.md")); + assert!(body.contains("(../../../678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_rewrites_source_root_absolute_asset_path() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](/00678/assets/foo.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(../678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_rewrites_cross_proposal_image_links() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "![diagram](./00678/assets/diagram.png)"), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/diagram.png", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("![diagram](../678/assets/diagram.png)")); + } + + #[test] + fn preprocess_preserves_query_and_fragment_on_asset_links() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "", + "See [asset](./00678/assets/foo.pdf?download=1#page=2).", + ), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(../678/assets/foo.pdf?download=1#page=2)")); + } + + #[test] + fn preprocess_decodes_asset_paths_and_keeps_generated_urls_encoded() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "", + "See [asset](./00678/assets/Contract%20Interactions%20diagram.svg).", + ), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ( + "00678/assets/Contract Interactions diagram.svg", + "".to_owned(), + ), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("../678/assets/Contract%20Interactions%20diagram.svg")); + } + + #[test] + fn preprocess_keeps_selected_or_full_asset_markdown_links_on_existing_md_path() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [guide](./00678/assets/guide.md)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/guide.md", "Guide.".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(@/00678/assets/guide.md)")); + } + + #[test] + fn preprocess_keeps_asset_markdown_fragment_links_unchanged() { + let (_temp, content) = content_repo(&[ + ( + "00555/index.md", + proposal_markdown(555, None, "", "Source."), + ), + ( + "00555/assets/guide.md", + "See [heading](#heading).\n\n## Heading\n".to_owned(), + ), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555/assets/guide.md")); + assert!(body.contains("[heading](#heading)")); + } + + #[test] + fn preprocess_keeps_ordinary_proposal_markdown_links_on_existing_path() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [proposal](./00678.md)."), + ), + ("00678.md", proposal_markdown(678, None, "", "Target.")), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(@/00678.md)")); + } + + #[test] + fn targeted_preprocess_rewrites_omitted_static_asset_to_public_url() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](./00678/assets/foo.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(https://eips.ethereum.org/678/assets/foo.pdf)")); + } + + #[test] + fn targeted_preprocess_rewrites_omitted_asset_markdown_to_public_url() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [guide](./00678/assets/guide.md)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/guide.md", "Guide.".to_owned()), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(https://eips.ethereum.org/678/assets/guide/)")); + } + + #[test] + fn targeted_preprocess_rewrites_omitted_readme_and_index_asset_markdown_public_urls() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "", + "See [readme](./00678/assets/README.md) and [index](./00678/assets/index.md).", + ), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/README.md", "Readme.".to_owned()), + ("00678/assets/index.md", "Index.".to_owned()), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(https://eips.ethereum.org/678/assets/README/)")); + assert!(body.contains("(https://eips.ethereum.org/678/assets/index/)")); + } + + #[test] + fn targeted_preprocess_keeps_selected_static_asset_local() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](./00678/assets/foo.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + let plan = only_plan(&content, &[555, 678]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(../678/assets/foo.pdf)")); + assert!(!body.contains("https://eips.ethereum.org/678/assets/foo.pdf")); + } + + #[test] + fn targeted_dirty_preprocess_uses_inventory_after_omitted_target_is_pruned() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](./00678/assets/foo.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + let plan = only_plan(&content, &[555]); + plan.prune_content(&content).unwrap(); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(https://eips.ethereum.org/678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_errors_clearly_for_missing_selected_or_full_asset_target() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](./00678/assets/missing.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/.keep", "".to_owned()), + ]); + + let error = Report::from_error(preprocess(&content, None).unwrap_err()).to_string(); + + assert!(error.contains("proposal asset link")); + assert!(error.contains("00555.md")); + assert!(error.contains("./00678/assets/missing.pdf")); + assert!(error.contains("00678/assets/missing.pdf")); + } + + #[test] + fn preprocess_skips_generated_zola_markdown_files() { + let original = + "+++\ntitle = \"Generated\"\n+++\nSee [asset](./00678/assets/missing.pdf).\n"; + let (_temp, content) = content_repo(&[("00555.md", original.to_owned())]); + + preprocess(&content, None).unwrap(); + + assert_eq!( + std::fs::read_to_string(content.join("00555.md")).unwrap(), + original + ); + } + + #[test] + fn process_assets_skips_only_generated_asset_markdown_file() { + let generated = + "+++\ntitle = \"Generated\"\n+++\nSee [missing](../../00678/assets/missing.pdf).\n"; + let (_temp, content) = content_repo(&[ + ( + "00555/index.md", + proposal_markdown(555, None, "", "Source."), + ), + ("00555/assets/generated.md", generated.to_owned()), + ("00555/assets/fresh.md", "Fresh asset markdown.".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + assert_eq!( + std::fs::read_to_string(content.join("00555/assets/generated.md")).unwrap(), + generated + ); + assert!( + std::fs::read_to_string(content.join("00555/assets/fresh.md")) + .unwrap() + .starts_with("+++\n") + ); + } + + #[test] + fn preprocess_leaves_non_proposal_relative_asset_links_unchanged() { + let (_temp, content) = content_repo(&[( + "00555.md", + proposal_markdown(555, None, "", "See [local](./images/foo.pdf)."), + )]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(./images/foo.pdf)")); + } + + #[test] + fn preprocess_leaves_raw_html_asset_references_unchanged() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "", + r#"asset"#, + ), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains(r#"asset"#)); + } + #[test] fn targeted_preprocess_rewrites_selected_body_links_to_unselected_public_urls() { let (_temp, content) = content_repo(&[ diff --git a/src/proposal.rs b/src/proposal.rs index d30fb80..787f3ba 100644 --- a/src/proposal.rs +++ b/src/proposal.rs @@ -15,11 +15,13 @@ use std::{ }; use eipw_preamble::Preamble; +use log::warn; use serde::{ de::{self, Unexpected, Visitor}, Deserialize, Deserializer, Serialize, Serializer, }; use snafu::{OptionExt, ResultExt, Whatever}; +use walkdir::WalkDir; use crate::layout::CONTENT_DIR; @@ -173,11 +175,66 @@ pub(crate) enum ProposalReference<'a> { External(&'a str), } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ProposalAssetKind { + Static, + Markdown, +} + +impl ProposalAssetKind { + pub(crate) fn from_path(path: &Path) -> Self { + if path.extension().and_then(OsStr::to_str) == Some("md") { + Self::Markdown + } else { + Self::Static + } + } +} + +#[derive(Debug, Clone)] +#[allow(dead_code)] +struct ProposalAssetInventoryEntry { + proposal_number: ProposalNumber, + site: ProposalPublicSite, + asset_relative_path: PathBuf, + kind: ProposalAssetKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ProposalPublicSite { + Eips, + Ercs, +} + +impl ProposalPublicSite { + fn proposal_url(self, proposal_number: ProposalNumber) -> String { + match self { + Self::Eips => format!( + "https://eips.ethereum.org/EIPS/eip-{}", + proposal_number.get() + ), + Self::Ercs => format!( + "https://ercs.ethereum.org/ERCS/erc-{}", + proposal_number.get() + ), + } + } + + fn asset_base_url(self) -> &'static str { + match self { + Self::Eips => "https://eips.ethereum.org", + Self::Ercs => "https://ercs.ethereum.org", + } + } +} + #[derive(Debug, Clone)] pub(crate) struct OnlyRenderPlan { selected_numbers: BTreeSet, + asset_inventory: BTreeMap, canonical_proposal_numbers: BTreeMap, markdown_paths_by_number: BTreeMap>, + public_sites_by_number: BTreeMap, public_urls_by_number: BTreeMap, } @@ -188,8 +245,10 @@ impl OnlyRenderPlan { ) -> Result { let mut plan = Self { selected_numbers, + asset_inventory: BTreeMap::new(), canonical_proposal_numbers: BTreeMap::new(), markdown_paths_by_number: BTreeMap::new(), + public_sites_by_number: BTreeMap::new(), public_urls_by_number: BTreeMap::new(), }; @@ -244,6 +303,8 @@ impl OnlyRenderPlan { } } + plan.inventory_assets(content_root)?; + for selected_number in &plan.selected_numbers { if !plan.markdown_paths_by_number.contains_key(selected_number) { snafu::whatever!("selected proposal `{selected_number}` was not found"); @@ -291,7 +352,8 @@ impl OnlyRenderPlan { markdown_path.to_string_lossy() ) })?; - let public_url = public_url_for_markdown(markdown_path, proposal_number, contents)?; + let site = public_site_for_markdown(markdown_path, contents)?; + let public_url = site.proposal_url(proposal_number); match self.public_urls_by_number.get(&proposal_number) { Some(existing_url) if existing_url != &public_url => { @@ -301,6 +363,7 @@ impl OnlyRenderPlan { } Some(_) => {} None => { + self.public_sites_by_number.insert(proposal_number, site); self.public_urls_by_number .insert(proposal_number, public_url); } @@ -316,6 +379,127 @@ impl OnlyRenderPlan { Ok(()) } + fn inventory_assets(&mut self, content_root: &Path) -> Result<(), Whatever> { + let canon_root = std::fs::canonicalize(content_root).with_whatever_context(|_| { + format!( + "unable to canonicalize content root `{}` for proposal asset inventory", + content_root.to_string_lossy() + ) + })?; + + let proposal_assets = self + .markdown_paths_by_number + .iter() + .flat_map(|(proposal_number, markdown_paths)| { + markdown_paths.iter().map(move |markdown_path| { + ( + *proposal_number, + markdown_path.clone(), + asset_dir_for_markdown_path(markdown_path), + ) + }) + }) + .collect::>(); + + for (proposal_number, markdown_path, asset_dir) in proposal_assets { + let Some(site) = self.public_sites_by_number.get(&proposal_number).copied() else { + continue; + }; + + let absolute_asset_dir = content_root.join(&asset_dir); + for entry in WalkDir::new(&absolute_asset_dir) + .follow_links(true) + .into_iter() + { + let entry = match entry { + Ok(entry) => entry, + Err(error) if missing_asset_dir(&error) => continue, + Err(error) => { + return Err(error).with_whatever_context(|_| { + format!( + "couldn't read proposal asset inventory entry in `{}`", + absolute_asset_dir.to_string_lossy() + ) + }); + } + }; + + if !entry.file_type().is_file() { + continue; + } + + let candidate = match std::fs::canonicalize(entry.path()) { + Ok(candidate) => candidate, + Err(error) => { + warn!( + "unable to canonicalize `{}`: {error}", + entry.path().to_string_lossy() + ); + continue; + } + }; + + if !candidate.starts_with(&canon_root) { + warn!( + "asset `{}` not in root, skipping", + entry.path().to_string_lossy() + ); + continue; + } + + let content_relative_path = entry + .path() + .strip_prefix(content_root) + .with_whatever_context(|_| { + format!( + "proposal asset `{}` for `{}` is outside content root `{}`", + entry.path().to_string_lossy(), + markdown_path.to_string_lossy(), + content_root.to_string_lossy() + ) + })?; + let asset_relative_path = entry + .path() + .strip_prefix(&absolute_asset_dir) + .with_whatever_context(|_| { + format!( + "proposal asset `{}` is outside asset directory `{}`", + entry.path().to_string_lossy(), + absolute_asset_dir.to_string_lossy() + ) + })?; + let entry = ProposalAssetInventoryEntry { + proposal_number, + site, + asset_relative_path: asset_relative_path.to_path_buf(), + kind: ProposalAssetKind::from_path(asset_relative_path), + }; + + self.asset_inventory + .insert(content_relative_path.to_path_buf(), entry); + } + } + + Ok(()) + } + + pub(crate) fn has_proposal_asset(&self, content_relative_asset_path: &Path) -> bool { + self.asset_inventory + .contains_key(content_relative_asset_path) + } + + pub(crate) fn public_url_for_omitted_proposal_asset( + &self, + content_relative_asset_path: &Path, + ) -> Option { + let entry = self.asset_inventory.get(content_relative_asset_path)?; + if self.selected_numbers.contains(&entry.proposal_number) { + return None; + } + + Some(public_asset_url(entry)) + } + pub(crate) fn external_url_for_canonical_target( &self, canonical_target: &Path, @@ -481,6 +665,27 @@ impl OnlyRenderPlan { } } +fn missing_asset_dir(error: &walkdir::Error) -> bool { + error.depth() == 0 + && error.io_error().is_some_and(|io_error| { + matches!( + io_error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) + }) +} + +fn asset_dir_for_markdown_path(markdown_path: &Path) -> PathBuf { + if markdown_path.file_name() == Some(OsStr::new("index.md")) { + markdown_path + .parent() + .map(|proposal_dir| proposal_dir.join("assets")) + .expect("index path has proposal parent") + } else { + markdown_path.with_extension("").join("assets") + } +} + fn remove_file_if_present(path: &Path) -> Result<(), Whatever> { match std::fs::remove_file(path) { Ok(()) => Ok(()), @@ -521,11 +726,10 @@ fn remove_dir_if_present(path: &Path) -> Result<(), Whatever> { } } -fn public_url_for_markdown( +fn public_site_for_markdown( markdown_path: &Path, - proposal_number: ProposalNumber, contents: &str, -) -> Result { +) -> Result { let path_lossy = markdown_path.to_string_lossy(); let (preamble, _) = Preamble::split(contents) .with_whatever_context(|_| format!("couldn't split preamble for `{path_lossy}`"))?; @@ -537,19 +741,49 @@ fn public_url_for_markdown( .any(|field| field.name() == "category" && field.value().trim() == "ERC"); if is_erc { - Ok(format!( - "https://ercs.ethereum.org/ERCS/erc-{}", - proposal_number.get() - )) + Ok(ProposalPublicSite::Ercs) } else { - Ok(format!( - "https://eips.ethereum.org/EIPS/eip-{}", - proposal_number.get() - )) + Ok(ProposalPublicSite::Eips) } } -fn flat_proposal_number(path: &Path) -> Option { +#[allow(dead_code)] +fn public_asset_url(entry: &ProposalAssetInventoryEntry) -> String { + let mut url = format!( + "{}/{}/assets", + entry.site.asset_base_url(), + entry.proposal_number.get() + ); + for component in entry.asset_relative_path.components() { + let std::path::Component::Normal(component) = component else { + continue; + }; + url.push('/'); + url.push_str(&percent_encode_url_segment(&component.to_string_lossy())); + } + + if entry.kind == ProposalAssetKind::Markdown { + url.truncate(url.len() - ".md".len()); + url.push('/'); + } + + url +} + +#[allow(dead_code)] +fn percent_encode_url_segment(segment: &str) -> String { + let mut encoded = String::with_capacity(segment.len()); + for byte in segment.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +pub(crate) fn flat_proposal_number(path: &Path) -> Option { if path.extension().and_then(OsStr::to_str) != Some("md") { return None; } @@ -557,7 +791,7 @@ fn flat_proposal_number(path: &Path) -> Option { path_component_proposal_number(path.file_stem()) } -fn path_component_proposal_number(component: Option<&OsStr>) -> Option { +pub(crate) fn path_component_proposal_number(component: Option<&OsStr>) -> Option { let name = component?.to_str()?; if name.is_empty() || !name.bytes().all(|byte| byte.is_ascii_digit()) { return None; @@ -930,6 +1164,132 @@ mod tests { ); } + #[test] + fn only_render_plan_inventories_directory_proposal_assets() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file( + content, + "00678/index.md", + &proposal_markdown(678, Some("ERC")), + ); + write_file(content, "00678/assets/foo.pdf", ""); + write_file(content, "00678/assets/guide.md", ""); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert!(plan.has_proposal_asset(Path::new("00678/assets/foo.pdf"))); + assert!(plan.has_proposal_asset(Path::new("00678/assets/guide.md"))); + assert_eq!( + plan.public_url_for_omitted_proposal_asset(Path::new("00678/assets/foo.pdf")) + .unwrap(), + "https://ercs.ethereum.org/678/assets/foo.pdf" + ); + assert_eq!( + plan.public_url_for_omitted_proposal_asset(Path::new("00678/assets/guide.md")) + .unwrap(), + "https://ercs.ethereum.org/678/assets/guide/" + ); + } + + #[test] + fn only_render_plan_inventories_flat_proposal_assets_when_directory_exists() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", &proposal_markdown(678, None)); + write_file(content, "00678/assets/README.md", ""); + write_file(content, "00678/assets/index.md", ""); + write_file( + content, + "00678/assets/Contract Interactions diagram.svg", + "", + ); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert_eq!( + plan.public_url_for_omitted_proposal_asset(Path::new("00678/assets/README.md")) + .unwrap(), + "https://eips.ethereum.org/678/assets/README/" + ); + assert_eq!( + plan.public_url_for_omitted_proposal_asset(Path::new("00678/assets/index.md")) + .unwrap(), + "https://eips.ethereum.org/678/assets/index/" + ); + assert_eq!( + plan.public_url_for_omitted_proposal_asset(Path::new( + "00678/assets/Contract Interactions diagram.svg" + )) + .unwrap(), + "https://eips.ethereum.org/678/assets/Contract%20Interactions%20diagram.svg" + ); + } + + #[test] + fn only_render_plan_records_flat_proposals_without_assets_as_empty() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", &proposal_markdown(678, None)); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert!(!plan.has_proposal_asset(Path::new("00678/assets/foo.pdf"))); + assert!(plan + .public_url_for_omitted_proposal_asset(Path::new("00678/assets/foo.pdf")) + .is_none()); + } + + #[test] + fn only_render_plan_does_not_inventory_assets_only_numeric_dirs() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678/assets/foo.pdf", ""); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert!(!plan.has_proposal_asset(Path::new("00678/assets/foo.pdf"))); + assert!(plan + .public_url_for_omitted_proposal_asset(Path::new("00678/assets/foo.pdf")) + .is_none()); + } + + #[test] + fn only_render_plan_does_not_construct_public_asset_urls_for_selected_targets() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", &proposal_markdown(678, None)); + write_file(content, "00678/assets/foo.pdf", ""); + + let plan = OnlyRenderPlan::build(content, [number(555), number(678)].into_iter().collect()) + .unwrap(); + + assert!(plan.has_proposal_asset(Path::new("00678/assets/foo.pdf"))); + assert!(plan + .public_url_for_omitted_proposal_asset(Path::new("00678/assets/foo.pdf")) + .is_none()); + } + + #[test] + fn only_render_plan_errors_on_malformed_proposal_before_asset_inventory_policy() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", "not front matter"); + write_file(content, "00678/assets/foo.pdf", ""); + + let error = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()) + .unwrap_err() + .to_string(); + + assert!(error.contains("couldn't split preamble")); + } + #[test] fn only_render_plan_does_not_mask_missing_required_targets() { let temp = TempDir::new().unwrap(); From 478ff8403cbcace881de14ea1945cbdcfad650a1 Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Sun, 3 May 2026 21:53:32 -0400 Subject: [PATCH 15/20] Add targeted serve rendering Extend targeted rendering to local dirty serve by accepting --only on serve and applying workspace [render].only to local serve runs. Pass OnlyRenderPlan into dirty serve sync and filter active-repo dirty paths to selected proposal content. Avoid reintroducing omitted proposal markdown or assets into the materialized repo. Add incremental targeted markdown preprocessing for dirty serve updates, including selected asset markdown, retained non-proposal pages, selected deletions, and filesystem timestamp fallback for new dirty files. --- src/cli.rs | 25 ++- src/execution.rs | 52 ++++-- src/markdown.rs | 395 ++++++++++++++++++++++++++++++++++++------- src/pipeline.rs | 1 + src/serve.rs | 361 ++++++++++++++++++++++++++++++++++++--- src/workspace_doc.md | 27 +++ 6 files changed, 765 insertions(+), 96 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 7b0bdc8..2ea0951 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -109,6 +109,9 @@ pub(crate) enum Operation { #[command(flatten)] clean: CleanCliArgs, + + #[command(flatten)] + only: OnlyCliArgs, }, /// Remove the selected build directory and generated output @@ -284,10 +287,9 @@ impl Operation { pub(crate) fn only_cli_args(&self) -> Option<&OnlyCliArgs> { match self { - Self::Build { only, .. } => Some(only), + Self::Build { only, .. } | Self::Serve { only, .. } => Some(only), Self::Print { .. } | Self::Preview { .. } - | Self::Serve { .. } | Self::Clean | Self::Check { .. } | Self::Changed { .. } @@ -449,9 +451,10 @@ mod tests { } #[test] - fn only_flag_parses_one_or_more_proposal_numbers_on_build() { + fn only_flag_parses_one_or_more_proposal_numbers_on_build_and_serve() { let one = parse_args(&["build-eips", "build", "--only", "00555"]); let many = parse_args(&["build-eips", "build", "--only", "555", "678", "897"]); + let serve = parse_args(&["build-eips", "serve", "--only", "555", "678"]); match one.operation { Operation::Build { only, .. } => { @@ -472,10 +475,22 @@ mod tests { } other => panic!("unexpected operation: {other:?}"), } + match serve.operation { + Operation::Serve { only, .. } => { + assert_eq!( + only.only, + vec![ + ProposalNumber::from_u32(555).unwrap(), + ProposalNumber::from_u32(678).unwrap(), + ] + ); + } + other => panic!("unexpected operation: {other:?}"), + } } #[test] - fn only_flag_rejects_invalid_selectors_and_non_build_commands() { + fn only_flag_rejects_invalid_selectors_and_non_targeted_commands() { for selector in [ "+555", "0", @@ -491,9 +506,9 @@ mod tests { ); } - assert!(Args::try_parse_from(["build-eips", "serve", "--only", "555"]).is_err()); assert!(Args::try_parse_from(["build-eips", "check", "--only", "555"]).is_err()); assert!(Args::try_parse_from(["build-eips", "parity", "build", "--only", "555"]).is_err()); + assert!(Args::try_parse_from(["build-eips", "parity", "serve", "--only", "555"]).is_err()); } #[test] diff --git a/src/execution.rs b/src/execution.rs index bd75fa3..364811e 100644 --- a/src/execution.rs +++ b/src/execution.rs @@ -124,8 +124,10 @@ fn cli_only_requested(args: &Args) -> bool { } fn only_cli_is_applicable(args: &Args, explicit_environment: Option) -> bool { - matches!(args.operation, Operation::Build { .. }) - && explicit_environment.is_none() + matches!( + args.operation, + Operation::Build { .. } | Operation::Serve { .. } + ) && explicit_environment.is_none() && !args.operation.clean_cli_args().clean && !args.remote_siblings } @@ -145,7 +147,7 @@ pub(crate) fn resolve_execution_settings( let clean = args.operation.clean_cli_args().clean; if cli_only_requested(args) && !only_cli_is_applicable(args, explicit_environment) { - snafu::whatever!("--only is supported only for local dirty build commands"); + snafu::whatever!("--only is supported only for local dirty build and serve commands"); } let (staging, allow_dirty, default_sibling) = if let Some(staging) = explicit_environment { @@ -199,15 +201,19 @@ fn resolve_only_selection( workspace_config: Option<&LoadedWorkspaceConfig>, ) -> Result>, Whatever> { let explicit_environment = explicit_environment_or_parity(args)?; - let applicable = matches!(args.operation, Operation::Build { .. }) - && explicit_environment.is_none() + let applicable = matches!( + args.operation, + Operation::Build { .. } | Operation::Serve { .. } + ) && explicit_environment.is_none() && settings.allow_dirty && settings.sibling == SelectedSource::WorkspaceLocal; if let Some(only) = args.operation.only_cli_args() { if let Some(numbers) = dedupe_only_numbers(&only.only) { if !applicable { - snafu::whatever!("--only is supported only for local dirty build commands"); + snafu::whatever!( + "--only is supported only for local dirty build and serve commands" + ); } return Ok(Some(numbers)); } @@ -663,7 +669,7 @@ base_url = "http://localhost:4000" } #[test] - fn build_only_cli_selection_overrides_config_and_dedupes() { + fn only_cli_selection_overrides_config_and_dedupes_for_build_and_serve() { let workspace_config = load_workspace_config( r#" [render] @@ -678,10 +684,17 @@ only = [555] ), Some(vec![555, 678]) ); + assert_eq!( + only_selection_for( + &["build-eips", "serve", "--only", "678", "555", "678"], + Some(&workspace_config), + ), + Some(vec![555, 678]) + ); } #[test] - fn build_only_config_selection_applies_to_local_dirty_build_only() { + fn render_only_config_selection_applies_to_local_dirty_build_and_serve_only() { let workspace_config = load_workspace_config( r#" [render] @@ -693,22 +706,30 @@ only = [555, 678, 555] only_selection_for(&["build-eips", "build"], Some(&workspace_config)), Some(vec![555, 678]) ); + assert_eq!( + only_selection_for(&["build-eips", "serve"], Some(&workspace_config)), + Some(vec![555, 678]) + ); for arguments in [ &["build-eips", "build", "--clean"][..], &["build-eips", "--remote-siblings", "build"][..], &["build-eips", "--staging", "build"][..], &["build-eips", "--production", "build"][..], - &["build-eips", "serve"][..], + &["build-eips", "serve", "--clean"][..], + &["build-eips", "--remote-siblings", "serve"][..], + &["build-eips", "--staging", "serve"][..], + &["build-eips", "--production", "serve"][..], &["build-eips", "check"][..], &["build-eips", "parity", "build"][..], + &["build-eips", "parity", "serve"][..], ] { assert!(only_selection_for(arguments, Some(&workspace_config)).is_none()); } } #[test] - fn build_only_cli_selection_rejects_non_local_dirty_modes() { + fn only_cli_selection_rejects_non_local_dirty_build_and_serve_modes() { let workspace_config = load_workspace_config(""); for arguments in [ @@ -716,13 +737,19 @@ only = [555, 678, 555] &["build-eips", "--remote-siblings", "build", "--only", "555"][..], &["build-eips", "--staging", "build", "--only", "555"][..], &["build-eips", "--production", "build", "--only", "555"][..], + &["build-eips", "serve", "--only", "555", "--clean"][..], + &["build-eips", "--remote-siblings", "serve", "--only", "555"][..], + &["build-eips", "--staging", "serve", "--only", "555"][..], + &["build-eips", "--production", "serve", "--only", "555"][..], ] { let args = parse_args(arguments); let error = resolve_execution_settings(&args, &[], Some(&workspace_config)) .unwrap_err() .to_string(); - assert!(error.contains("--only is supported only for local dirty build commands")); + assert!( + error.contains("--only is supported only for local dirty build and serve commands") + ); } } @@ -740,6 +767,9 @@ only = [] assert!(only_selection_for(&["build-eips", "build"], Some(&missing_render)).is_none()); assert!(only_selection_for(&["build-eips", "build"], Some(&missing_only)).is_none()); assert!(only_selection_for(&["build-eips", "build"], Some(&empty_only)).is_none()); + assert!(only_selection_for(&["build-eips", "serve"], Some(&missing_render)).is_none()); + assert!(only_selection_for(&["build-eips", "serve"], Some(&missing_only)).is_none()); + assert!(only_selection_for(&["build-eips", "serve"], Some(&empty_only)).is_none()); } #[test] diff --git a/src/markdown.rs b/src/markdown.rs index e8432d0..6d02a6f 100644 --- a/src/markdown.rs +++ b/src/markdown.rs @@ -22,10 +22,10 @@ use regex::Regex; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::ffi::OsStr; use std::fs::read_to_string; -use std::io::Write; +use std::io::{ErrorKind, Write}; use std::path::{Component, Path, PathBuf}; use snafu::{whatever, OptionExt, ResultExt, Whatever}; @@ -46,6 +46,26 @@ use crate::{ }, }; +#[derive(Clone, Copy)] +enum MissingPathMode { + Error, + Ignore, +} + +impl MissingPathMode { + fn should_ignore_io_error(self, error: &std::io::Error) -> bool { + matches!(self, Self::Ignore) + && matches!(error.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) + } + + fn should_ignore_walkdir_error(self, error: &walkdir::Error) -> bool { + error + .io_error() + .map(|error| self.should_ignore_io_error(error)) + .unwrap_or(false) + } +} + #[derive(Debug, Serialize, Deserialize)] struct Author { name: String, @@ -143,6 +163,19 @@ impl Default for FrontMatter { } } +fn filesystem_modified(p: &Path) -> Result { + let metadata = std::fs::metadata(p) + .with_whatever_context(|e| format!("unable to read metadata for `{}`: {e}", p.display()))?; + let modified = metadata.modified().with_whatever_context(|e| { + format!( + "unable to read filesystem modified time for `{}`: {e}", + p.display() + ) + })?; + let date_time: DateTime = modified.into(); + Ok(date_time.to_rfc3339().parse().unwrap()) +} + fn last_modified(p: &Path) -> Result { // TODO: Replace this with `git2` let mut command = std::process::Command::new("git"); @@ -164,7 +197,16 @@ fn last_modified(p: &Path) -> Result { } let date_str = std::str::from_utf8(&output.stdout) - .with_whatever_context(|e| format!("command {:?} output not UTF-8: {e}", command))?; + .with_whatever_context(|e| format!("command {:?} output not UTF-8: {e}", command))? + .trim(); + + if date_str.is_empty() { + debug!( + "falling back to filesystem modified time for `{}` because git has no timestamp for the current path", + p.to_string_lossy() + ); + return filesystem_modified(p); + } let unix: i64 = date_str.parse().with_whatever_context(|e| { let err_str = std::str::from_utf8(&output.stderr).unwrap_or(""); @@ -737,12 +779,12 @@ pub fn preprocess(root_path: &Path, only_plan: Option<&OnlyRenderPlan>) -> Resul if let Some(plan) = only_plan { let index_relative_path = relative_path.join("index.md"); if plan.should_preprocess_markdown(&index_relative_path) { - process_eip(root_path, &index_path, only_plan)?; + process_eip(root_path, &index_path, only_plan, MissingPathMode::Error)?; } } else { - process_eip(root_path, &index_path, only_plan)?; + process_eip(root_path, &index_path, only_plan, MissingPathMode::Error)?; } - process_assets(root_path, &entry_path, only_plan)?; + process_assets(root_path, &entry_path, only_plan, MissingPathMode::Error)?; } } else if entry_path.extension().and_then(OsStr::to_str) == Some("md") { let relative_path = entry_path @@ -758,7 +800,7 @@ pub fn preprocess(root_path: &Path, only_plan: Option<&OnlyRenderPlan>) -> Resul .map(|plan| plan.should_preprocess_markdown(relative_path)) .unwrap_or(true) { - process_eip(root_path, &entry_path, only_plan)?; + process_eip(root_path, &entry_path, only_plan, MissingPathMode::Error)?; } } } @@ -766,6 +808,63 @@ pub fn preprocess(root_path: &Path, only_plan: Option<&OnlyRenderPlan>) -> Resul Ok(()) } +pub fn preprocess_paths( + root_path: &Path, + relative_paths: &BTreeSet, + only_plan: Option<&OnlyRenderPlan>, +) -> Result<(), Whatever> { + let mut eips = BTreeSet::new(); + let mut asset_dirs = BTreeSet::new(); + + for relative_path in relative_paths { + let Ok(content_relative_path) = relative_path.strip_prefix("content") else { + continue; + }; + + if content_relative_path.as_os_str().is_empty() { + continue; + } + + if content_relative_path.extension().and_then(OsStr::to_str) != Some("md") { + continue; + } + + if only_plan + .map(|plan| !plan.should_sync_dirty_path(relative_path)) + .unwrap_or(false) + { + continue; + } + + let mut components = content_relative_path.components(); + let Some(first_component) = components.next() else { + continue; + }; + + if matches!( + components.next(), + Some(component) if component.as_os_str() == OsStr::new("assets") + ) { + let proposal_dir = root_path.join(first_component.as_os_str()); + asset_dirs.insert(proposal_dir); + continue; + } + + let path = root_path.join(content_relative_path); + eips.insert(path); + } + + for path in eips { + process_eip(root_path, &path, only_plan, MissingPathMode::Ignore)?; + } + + for path in asset_dirs { + process_assets(root_path, &path, only_plan, MissingPathMode::Ignore)?; + } + + Ok(()) +} + fn path_to_at(root: &Path, parent: &Path, input: &str) -> Result { let croot = std::fs::canonicalize(root).with_whatever_context(|_| { format!("could not canonicalize `{}`", root.to_string_lossy()) @@ -963,6 +1062,26 @@ fn fix_links<'a, 'b>( } else { parent.join(Path::new(iri_path)) }; + let normalized_root = normalize_path_lexically(root); + let normalized_child = normalize_path_lexically(&child); + if let Some(public_url) = only_plan.and_then(|plan| { + normalized_child + .strip_prefix(&normalized_root) + .ok() + .and_then(|relative_path| plan.external_url_for_content_target(relative_path)) + }) { + let mut external_url = public_url.to_owned(); + if let Some(query) = iri_ref.query() { + external_url.push('?'); + external_url.push_str(query.as_str()); + } + if let Some(fragment) = iri_ref.fragment() { + external_url.push('#'); + external_url.push_str(fragment.as_str()); + } + *dest_url = CowStr::from(external_url); + return Ok(e); + } let canonicalized = canonicalize_md(&child)?; if let Some(public_url) = only_plan.and_then(|plan| plan.external_url_for_canonical_target(&canonicalized)) @@ -1082,65 +1201,84 @@ fn process_assets( root: &Path, path: &Path, only_plan: Option<&OnlyRenderPlan>, + missing_path_mode: MissingPathMode, ) -> Result<(), Whatever> { let canon_root = std::fs::canonicalize(root).whatever_context("could not canonicalize root")?; - let number_txt = path - .file_name() - .with_whatever_context(|| format!("no file name for `{}`", path.to_string_lossy()))? - .to_str() - .with_whatever_context(|| format!("non-UTF-8 in `{}`", path.to_string_lossy()))?; + let assets_dir = path.join("assets"); - let number: u32 = number_txt.parse().with_whatever_context(|_| { - format!("can't parse number for `{}`", path.to_string_lossy()) - })?; + let mut entries = Vec::new(); + let mut ignored_missing_path = false; - let assets_dir = path.join("assets"); + for entry in WalkDir::new(&assets_dir).follow_links(true).into_iter() { + let entry = match entry { + Ok(entry) => entry, + Err(error) if missing_path_mode.should_ignore_walkdir_error(&error) => { + ignored_missing_path = true; + continue; + } + Err(error) => { + return Err(error).with_whatever_context(|_| { + format!("couldn't read entry in `{}`", assets_dir.to_string_lossy()) + }); + } + }; - let dir = WalkDir::new(&assets_dir) - .follow_links(true) - .into_iter() - .filter(|e| match e { - Ok(f) if !f.file_type().is_file() => false, - Ok(f) => f.path().extension().and_then(OsStr::to_str) == Some("md"), - Err(_) => true, - }) - .filter(|e| { - let f = match e { - Ok(f) => f, - _ => return true, - }; + if !entry.file_type().is_file() { + continue; + } - let candidate = match std::fs::canonicalize(f.path()) { - Ok(c) => c, - Err(e) => { - warn!( - "unable to canonicalize `{}`: {e}", - f.path().to_string_lossy() - ); - return false; - } - }; + if entry.path().extension().and_then(OsStr::to_str) != Some("md") { + continue; + } - let in_root = candidate.starts_with(&canon_root); - if !in_root { + let candidate = match std::fs::canonicalize(entry.path()) { + Ok(c) => c, + Err(e) => { warn!( - "asset `{}` not in root, skipping", - f.path().to_string_lossy() + "unable to canonicalize `{}`: {e}", + entry.path().to_string_lossy() ); + continue; } - in_root - }); - let dirs: Vec<_> = dir.collect(); + }; - for entry in dirs.into_iter().progress_ext("Assets") { - let entry = entry.with_whatever_context(|_| { - format!("couldn't read entry in `{}`", assets_dir.to_string_lossy()) - })?; + let in_root = candidate.starts_with(&canon_root); + if !in_root { + warn!( + "asset `{}` not in root, skipping", + entry.path().to_string_lossy() + ); + continue; + } + + entries.push(entry); + } + + if entries.is_empty() && ignored_missing_path { + return Ok(()); + } + + let number_txt = path + .file_name() + .with_whatever_context(|| format!("no file name for `{}`", path.to_string_lossy()))? + .to_str() + .with_whatever_context(|| format!("non-UTF-8 in `{}`", path.to_string_lossy()))?; + + let number: u32 = number_txt.parse().with_whatever_context(|_| { + format!("can't parse number for `{}`", path.to_string_lossy()) + })?; + for entry in entries.into_iter().progress_ext("Assets") { let path = entry.path(); - let contents = read_to_string(path).with_whatever_context(|_| { - format!("could not read file `{}`", path.to_string_lossy()) - })?; + let contents = match read_to_string(path) { + Ok(contents) => contents, + Err(error) if missing_path_mode.should_ignore_io_error(&error) => continue, + Err(error) => { + return Err(error).with_whatever_context(|_| { + format!("could not read file `{}`", path.to_string_lossy()) + }); + } + }; if is_generated_zola_markdown(&contents) { continue; } @@ -1180,7 +1318,11 @@ fn process_assets( ..Default::default() }; - write_file(path, front_matter, &contents).whatever_context("couldn't write file")?; + match write_file(path, front_matter, &contents) { + Ok(()) => {} + Err(error) if missing_path_mode.should_ignore_io_error(&error) => continue, + Err(error) => return Err(error).whatever_context("couldn't write file"), + } } Ok(()) @@ -1190,10 +1332,17 @@ fn process_eip( root: &Path, path: &Path, only_plan: Option<&OnlyRenderPlan>, + missing_path_mode: MissingPathMode, ) -> Result<(), Whatever> { let path_lossy = path.to_string_lossy(); - let contents = read_to_string(path) - .with_whatever_context(|_| format!("could not read file `{}`", path_lossy))?; + let contents = match read_to_string(path) { + Ok(contents) => contents, + Err(error) if missing_path_mode.should_ignore_io_error(&error) => return Ok(()), + Err(error) => { + return Err(error) + .with_whatever_context(|_| format!("could not read file `{}`", path_lossy)); + } + }; if is_generated_zola_markdown(&contents) { return Ok(()); } @@ -1322,7 +1471,11 @@ fn process_eip( } } - write_file(Path::new(&path), front_matter, &body).whatever_context("couldn't write file")?; + match write_file(path, front_matter, &body) { + Ok(()) => {} + Err(error) if missing_path_mode.should_ignore_io_error(&error) => return Ok(()), + Err(error) => return Err(error).whatever_context("couldn't write file"), + } Ok(()) } @@ -1338,9 +1491,9 @@ mod tests { use toml::Value as TomlValue; use super::{ - absolute_rendered_path_for_content_path, preprocess, proposal_asset_exists_in_content_tree, - relative_url_from_rendered_paths, resolve_proposal_asset_path, ProposalAssetPath, - ProposalAssetPathResolution, + absolute_rendered_path_for_content_path, preprocess, preprocess_paths, + proposal_asset_exists_in_content_tree, relative_url_from_rendered_paths, + resolve_proposal_asset_path, ProposalAssetPath, ProposalAssetPathResolution, }; use crate::proposal::ProposalAssetKind; use crate::proposal::{OnlyRenderPlan, ProposalNumber}; @@ -1410,6 +1563,10 @@ mod tests { OnlyRenderPlan::build(content_root, selected).unwrap() } + fn repo_paths(paths: &[&str]) -> BTreeSet { + paths.iter().map(PathBuf::from).collect() + } + fn rendered_body(path: &Path) -> String { let contents = std::fs::read_to_string(path).unwrap(); contents.split_once("\n+++\n").unwrap().1.to_owned() @@ -2126,6 +2283,46 @@ mod tests { assert!(!body.contains("@/00678.md")); } + #[test] + fn targeted_preprocess_paths_rewrites_selected_dirty_markdown_with_plan() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "requires: 155\n", + "See [EIP-155](./00155.md#list-of-chain-id-s).", + ), + ), + ("00155.md", proposal_markdown(155, None, "", "Unselected.")), + ]); + let plan = only_plan(&content, &[555]); + + preprocess_paths(&content, &repo_paths(&["content/00555.md"]), Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + let front_matter = rendered_front_matter(&content.join("00555.md")); + let requires = front_matter["extra"]["requires"].as_array().unwrap(); + assert!(body.contains("https://eips.ethereum.org/EIPS/eip-155#list-of-chain-id-s")); + assert_eq!( + requires[0].as_str().unwrap(), + "https://eips.ethereum.org/EIPS/eip-155" + ); + } + + #[test] + fn targeted_preprocess_paths_ignores_deleted_dirty_markdown() { + let (_temp, content) = + content_repo(&[("00555.md", proposal_markdown(555, None, "", "Selected."))]); + let plan = only_plan(&content, &[555]); + std::fs::remove_file(content.join("00555.md")).unwrap(); + + preprocess_paths(&content, &repo_paths(&["content/00555.md"]), Some(&plan)).unwrap(); + + assert!(!content.join("00555.md").exists()); + } + #[test] fn targeted_preprocess_rewrites_retained_non_proposal_links_to_public_urls() { let (_temp, content) = content_repo(&[ @@ -2145,6 +2342,79 @@ mod tests { assert!(!body.contains("@/00678.md")); } + #[test] + fn targeted_preprocess_paths_rewrites_retained_non_proposal_markdown_with_plan() { + let (_temp, content) = content_repo(&[ + ( + "_index.md", + "---\ntitle: Home\n---\nSee [EIP-678](/00678.md).\n".to_owned(), + ), + ("00555.md", proposal_markdown(555, None, "", "Selected.")), + ("00678.md", proposal_markdown(678, None, "", "Unselected.")), + ]); + let plan = only_plan(&content, &[555]); + + preprocess_paths(&content, &repo_paths(&["content/_index.md"]), Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("_index.md")); + assert!(body.contains("https://eips.ethereum.org/EIPS/eip-678")); + assert!(!body.contains("@/00678.md")); + } + + #[test] + fn targeted_preprocess_paths_rewrites_selected_asset_markdown_with_plan() { + let (_temp, content) = content_repo(&[ + ("00555.md", proposal_markdown(555, None, "", "Selected.")), + ( + "00555/assets/guide.md", + "See [EIP-678](/00678.md).\n".to_owned(), + ), + ("00555/assets/diagram.png", "image\n".to_owned()), + ( + "00678.md", + proposal_markdown(678, Some("ERC"), "", "Unselected."), + ), + ]); + let plan = only_plan(&content, &[555]); + + preprocess_paths( + &content, + &repo_paths(&["content/00555/assets/guide.md"]), + Some(&plan), + ) + .unwrap(); + + let body = rendered_body(&content.join("00555/assets/guide.md")); + assert!(body.contains("https://ercs.ethereum.org/ERCS/erc-678")); + assert_eq!( + std::fs::read_to_string(content.join("00555/assets/diagram.png")).unwrap(), + "image\n" + ); + } + + #[test] + fn targeted_preprocess_paths_ignores_deleted_dirty_asset_dir() { + let (_temp, content) = content_repo(&[ + ("00555.md", proposal_markdown(555, None, "", "Selected.")), + ( + "00555/assets/guide.md", + "See [EIP-678](/00678.md).\n".to_owned(), + ), + ("00678.md", proposal_markdown(678, None, "", "Unselected.")), + ]); + let plan = only_plan(&content, &[555]); + std::fs::remove_dir_all(content.join("00555/assets")).unwrap(); + + preprocess_paths( + &content, + &repo_paths(&["content/00555/assets/guide.md"]), + Some(&plan), + ) + .unwrap(); + + assert!(!content.join("00555/assets").exists()); + } + #[test] fn targeted_preprocess_preserves_query_and_fragment_on_external_links() { let (_temp, content) = content_repo(&[ @@ -2157,7 +2427,10 @@ mod tests { "See [Fragment](./00155.md#list-of-chain-id-s).\nSee [Query](./00155.md?foo=bar#list-of-chain-id-s).", ), ), - ("00155.md", proposal_markdown(155, None, "", "Unselected.")), + ( + "00155.md", + proposal_markdown(155, None, "", "Unselected."), + ), ]); let plan = only_plan(&content, &[555]); diff --git a/src/pipeline.rs b/src/pipeline.rs index a83fd89..6638746 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -154,6 +154,7 @@ impl Prepared { self.source_materialization, &self.source_root, &self.repo_path, + self.only_plan.clone(), self.local_theme_sync.clone(), ); let dirty_watcher = if sync_config.has_targets() { diff --git a/src/serve.rs b/src/serve.rs index a1153b7..326cb31 100644 --- a/src/serve.rs +++ b/src/serve.rs @@ -23,7 +23,7 @@ use log::{debug, info, warn}; use notify::{Event, RecursiveMode, Watcher}; use snafu::{Report, ResultExt, Whatever}; -use crate::{git, layout::CONTENT_DIR, markdown}; +use crate::{git, layout::CONTENT_DIR, markdown, proposal::OnlyRenderPlan}; #[derive(Debug)] pub(crate) struct DirtyServeWatcher { @@ -35,6 +35,7 @@ pub(crate) struct DirtyServeWatcher { struct ActiveRepoServeSync { source_root: PathBuf, build_repo_path: PathBuf, + only_plan: Option, } #[derive(Debug, Clone)] @@ -121,27 +122,37 @@ fn event_has_theme_index_path(index_path: &Path, event: &Event) -> bool { fn sync_dirty_serve_state( source_root: &Path, build_repo_path: &Path, + only_plan: Option<&OnlyRenderPlan>, previous_dirty_paths: &mut BTreeSet, ) -> Result<(), Whatever> { - let current_dirty_paths: BTreeSet<_> = git::working_tree_paths(source_root) - .whatever_context("unable to list tracked dirty paths for dirty serve")? - .into_iter() - .collect(); + let current_dirty_paths = filter_dirty_paths( + git::working_tree_paths(source_root) + .whatever_context("unable to list tracked dirty paths for dirty serve")?, + only_plan, + ); - let affected_paths: BTreeSet<_> = previous_dirty_paths - .union(¤t_dirty_paths) - .cloned() - .collect(); + let affected_paths = affected_dirty_paths(previous_dirty_paths, ¤t_dirty_paths); if affected_paths.is_empty() { *previous_dirty_paths = current_dirty_paths; return Ok(()); } + for path in selected_deleted_proposal_markdown_paths(source_root, &affected_paths, only_plan) { + warn!( + "selected proposal path `{}` was removed from the source repo; removing it from the targeted serve build input", + path.to_string_lossy() + ); + } + git::sync_materialized_paths(source_root, build_repo_path, &affected_paths) .whatever_context("unable to synchronize tracked paths into the materialized repo")?; - markdown::preprocess(&build_repo_path.join(CONTENT_DIR), None) - .whatever_context("unable to preprocess synchronized markdown during dirty serve")?; + markdown::preprocess_paths( + &build_repo_path.join(CONTENT_DIR), + &affected_paths, + only_plan, + ) + .whatever_context("unable to preprocess synchronized markdown during dirty serve")?; info!( "synchronized {} tracked path(s) into the materialized repo for dirty serve", @@ -152,11 +163,69 @@ fn sync_dirty_serve_state( Ok(()) } -fn capture_active_dirty_paths(source_root: &Path) -> Result, Whatever> { - Ok(git::working_tree_paths(source_root) - .whatever_context("unable to list tracked dirty paths for dirty serve")? +fn filter_dirty_paths( + dirty_paths: impl IntoIterator, + only_plan: Option<&OnlyRenderPlan>, +) -> BTreeSet { + dirty_paths .into_iter() - .collect()) + .filter(|path| { + only_plan + .map(|plan| plan.should_sync_dirty_path(path)) + .unwrap_or(true) + }) + .collect() +} + +fn affected_dirty_paths( + previous_dirty_paths: &BTreeSet, + current_dirty_paths: &BTreeSet, +) -> BTreeSet { + previous_dirty_paths + .union(current_dirty_paths) + .cloned() + .collect() +} + +fn selected_deleted_proposal_markdown_paths( + source_root: &Path, + affected_paths: &BTreeSet, + only_plan: Option<&OnlyRenderPlan>, +) -> Vec { + let Some(only_plan) = only_plan else { + return Vec::new(); + }; + + affected_paths + .iter() + .filter(|path| only_plan.is_selected_proposal_markdown_path(path)) + .filter( + |path| match std::fs::symlink_metadata(source_root.join(path)) { + Ok(_) => false, + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => + { + true + } + Err(_) => false, + }, + ) + .cloned() + .collect() +} + +fn capture_active_dirty_paths( + source_root: &Path, + only_plan: Option<&OnlyRenderPlan>, +) -> Result, Whatever> { + Ok(filter_dirty_paths( + git::working_tree_paths(source_root) + .whatever_context("unable to list tracked dirty paths for dirty serve")?, + only_plan, + )) } fn sync_theme_serve_state( @@ -273,7 +342,10 @@ fn dirty_serve_sync_loop( } let mut previous_active_dirty_paths: BTreeSet<_> = match &sync_config.active_repo { - Some(active_repo) => match capture_active_dirty_paths(&active_repo.source_root) { + Some(active_repo) => match capture_active_dirty_paths( + &active_repo.source_root, + active_repo.only_plan.as_ref(), + ) { Ok(paths) => paths, Err(error) => { let _ = ready_tx.send(Err(format!( @@ -385,6 +457,7 @@ fn dirty_serve_sync_loop( if let Err(error) = sync_dirty_serve_state( &active_repo.source_root, &active_repo.build_repo_path, + active_repo.only_plan.as_ref(), &mut previous_active_dirty_paths, ) { warn!( @@ -416,6 +489,7 @@ pub(crate) fn serve_sync_config( source_materialization: git::SourceMaterialization, source_root: &Path, repo_path: &Path, + only_plan: Option, local_theme_sync: Option, ) -> ServeSyncConfig { ServeSyncConfig { @@ -423,6 +497,7 @@ pub(crate) fn serve_sync_config( ActiveRepoServeSync { source_root: source_root.to_path_buf(), build_repo_path: repo_path.to_path_buf(), + only_plan, } }), local_theme: local_theme_sync, @@ -431,14 +506,25 @@ pub(crate) fn serve_sync_config( #[cfg(test)] mod tests { - use std::path::{Path, PathBuf}; + use std::{ + collections::BTreeSet, + path::{Path, PathBuf}, + }; + use git2::{IndexAddOption, Repository, Signature}; use notify::{Event, EventKind}; use tempfile::TempDir; - use crate::git::SourceMaterialization; + use crate::{ + git::SourceMaterialization, + proposal::{OnlyRenderPlan, ProposalNumber}, + }; - use super::{event_has_theme_index_path, serve_sync_config, LocalThemeServeSync}; + use super::{ + affected_dirty_paths, event_has_theme_index_path, filter_dirty_paths, + selected_deleted_proposal_markdown_paths, serve_sync_config, sync_dirty_serve_state, + LocalThemeServeSync, + }; fn fake_theme_sync(root: &Path) -> LocalThemeServeSync { LocalThemeServeSync { @@ -448,6 +534,114 @@ mod tests { } } + fn number(value: u32) -> ProposalNumber { + ProposalNumber::from_u32(value).unwrap() + } + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn commit_all(repo: &Repository, message: &str) { + let mut index = repo.index().unwrap(); + index + .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + let parents = repo + .head() + .ok() + .and_then(|head| head.target()) + .map(|oid| repo.find_commit(oid).unwrap()) + .into_iter() + .collect::>(); + let parent_refs = parents.iter().collect::>(); + + repo.commit( + Some("HEAD"), + &signature, + &signature, + message, + &tree, + &parent_refs, + ) + .unwrap(); + } + + fn init_repo(root: &Path, files: &[(&str, &str)]) -> Repository { + std::fs::create_dir_all(root).unwrap(); + let repo = Repository::init(root).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + for (relative, contents) in files { + write_file(root, relative, contents); + } + commit_all(&repo, "initial"); + repo + } + + fn proposal_markdown(value: u32, extra_preamble: &str, body: &str) -> String { + format!("---\neip: {value}\ntitle: Proposal {value}\n{extra_preamble}---\n{body}\n") + } + + fn only_plan(root: &Path) -> OnlyRenderPlan { + let content = root.join("content"); + write_file(&content, "00555.md", &proposal_markdown(555, "", "Body")); + write_file(&content, "00678.md", &proposal_markdown(678, "", "Body")); + OnlyRenderPlan::build(&content, [number(555)].into_iter().collect()).unwrap() + } + + fn paths(paths: &[&str]) -> BTreeSet { + paths.iter().map(PathBuf::from).collect() + } + + fn rendered_body(path: &Path) -> String { + let contents = std::fs::read_to_string(path).unwrap(); + contents.split_once("\n+++\n").unwrap().1.to_owned() + } + + fn dirty_sync_fixture() -> (TempDir, PathBuf, PathBuf, OnlyRenderPlan) { + let temp = TempDir::new().unwrap(); + let source = temp.path().join("source"); + let build = temp.path().join("build/repo"); + let selected = proposal_markdown(555, "requires: 678\n", "See [EIP-678](/00678.md)."); + let unselected = proposal_markdown(678, "", "Unselected."); + init_repo( + &source, + &[ + ("content/00555.md", selected.as_str()), + ("content/00555/assets/diagram.png", "selected image\n"), + ("content/00678.md", unselected.as_str()), + ("content/00678/assets/diagram.png", "unselected image\n"), + ( + "content/_index.md", + "---\ntitle: Home\n---\nSee [EIP-678](/00678.md).\n", + ), + ], + ); + let plan = + OnlyRenderPlan::build(&source.join("content"), [number(555)].into_iter().collect()) + .unwrap(); + init_repo( + &build, + &[ + ("content/00555.md", selected.as_str()), + ("content/00555/assets/diagram.png", "selected image\n"), + ( + "content/_index.md", + "---\ntitle: Home\n---\nSee [EIP-678](/00678.md).\n", + ), + ], + ); + (temp, source, build, plan) + } + #[test] fn local_theme_index_events_trigger_rescan() { let index_path = PathBuf::from("/workspace/theme/.git/index"); @@ -465,15 +659,23 @@ mod tests { #[test] fn local_serve_syncs_theme_and_dirty_active_repo() { let temp = TempDir::new().unwrap(); + let plan = only_plan(temp.path()); let sync_config = serve_sync_config( SourceMaterialization::Dirty, &temp.path().join("Core"), &temp.path().join(".local-build/Core/repo"), + Some(plan), Some(fake_theme_sync(temp.path())), ); assert!(sync_config.active_repo.is_some()); + assert!(sync_config + .active_repo + .as_ref() + .unwrap() + .only_plan + .is_some()); assert!(sync_config.local_theme.is_some()); } @@ -485,10 +687,131 @@ mod tests { SourceMaterialization::Clean, &temp.path().join("Core"), &temp.path().join(".local-build/Core/repo"), + None, Some(fake_theme_sync(temp.path())), ); assert!(sync_config.active_repo.is_none()); assert!(sync_config.local_theme.is_some()); } + + #[test] + fn only_dirty_path_filter_runs_before_union_and_keeps_selected_deletions() { + let temp = TempDir::new().unwrap(); + let plan = only_plan(temp.path()); + let previous_raw = paths(&[ + "content/00555.md", + "content/00678.md", + "content/00678/assets/diagram.png", + ]); + let current_raw = paths(&["content/00555/assets/diagram.png", "content/00999.md"]); + + let previous_filtered = filter_dirty_paths(previous_raw, Some(&plan)); + let current_filtered = filter_dirty_paths(current_raw, Some(&plan)); + let affected = affected_dirty_paths(&previous_filtered, ¤t_filtered); + + assert_eq!( + affected, + paths(&["content/00555.md", "content/00555/assets/diagram.png"]) + ); + } + + #[test] + fn selected_deleted_proposal_markdown_paths_reports_only_selected_markdown_deletions() { + let (_temp, source, _build, plan) = dirty_sync_fixture(); + std::fs::remove_file(source.join("content/00555.md")).unwrap(); + std::fs::remove_file(source.join("content/00555/assets/diagram.png")).unwrap(); + std::fs::remove_file(source.join("content/_index.md")).unwrap(); + + let affected_paths = paths(&[ + "content/00555.md", + "content/00555/assets/diagram.png", + "content/_index.md", + "content/00678.md", + ]); + + assert_eq!( + selected_deleted_proposal_markdown_paths(&source, &affected_paths, Some(&plan)), + vec![PathBuf::from("content/00555.md")] + ); + assert!( + selected_deleted_proposal_markdown_paths(&source, &affected_paths, None).is_empty() + ); + } + + #[test] + fn only_dirty_sync_does_not_reintroduce_unselected_markdown_or_assets() { + let (_temp, source, build, plan) = dirty_sync_fixture(); + write_file( + &source, + "content/00678.md", + &proposal_markdown(678, "", "Dirty unselected."), + ); + write_file( + &source, + "content/00678/assets/diagram.png", + "dirty unselected image\n", + ); + let mut previous_dirty_paths = BTreeSet::new(); + + sync_dirty_serve_state(&source, &build, Some(&plan), &mut previous_dirty_paths).unwrap(); + + assert!(!build.join("content/00678.md").exists()); + assert!(!build.join("content/00678/assets/diagram.png").exists()); + assert!(previous_dirty_paths.is_empty()); + } + + #[test] + fn only_dirty_sync_copies_selected_assets_without_markdown_preprocessing() { + let (_temp, source, build, plan) = dirty_sync_fixture(); + write_file( + &source, + "content/00555/assets/diagram.png", + "dirty selected image\n", + ); + let mut previous_dirty_paths = BTreeSet::new(); + + sync_dirty_serve_state(&source, &build, Some(&plan), &mut previous_dirty_paths).unwrap(); + + assert_eq!( + std::fs::read_to_string(build.join("content/00555/assets/diagram.png")).unwrap(), + "dirty selected image\n" + ); + assert!(previous_dirty_paths.contains(Path::new("content/00555/assets/diagram.png"))); + } + + #[test] + fn only_dirty_sync_preprocesses_selected_and_retained_markdown_with_plan() { + let (_temp, source, build, plan) = dirty_sync_fixture(); + write_file( + &source, + "content/00555.md", + &proposal_markdown(555, "requires: 678\n", "Dirty [EIP-678](/00678.md)."), + ); + write_file( + &source, + "content/_index.md", + "---\ntitle: Home\n---\nDirty [EIP-678](/00678.md).\n", + ); + let mut previous_dirty_paths = BTreeSet::new(); + + sync_dirty_serve_state(&source, &build, Some(&plan), &mut previous_dirty_paths).unwrap(); + + let selected = std::fs::read_to_string(build.join("content/00555.md")).unwrap(); + let index_body = rendered_body(&build.join("content/_index.md")); + assert!(selected.contains("https://eips.ethereum.org/EIPS/eip-678")); + assert!(index_body.contains("https://eips.ethereum.org/EIPS/eip-678")); + } + + #[test] + fn only_dirty_sync_propagates_selected_proposal_deletion() { + let (_temp, source, build, plan) = dirty_sync_fixture(); + std::fs::remove_file(source.join("content/00555.md")).unwrap(); + let mut previous_dirty_paths = BTreeSet::new(); + + sync_dirty_serve_state(&source, &build, Some(&plan), &mut previous_dirty_paths).unwrap(); + + assert!(!build.join("content/00555.md").exists()); + assert!(previous_dirty_paths.contains(Path::new("content/00555.md"))); + } } diff --git a/src/workspace_doc.md b/src/workspace_doc.md index c6191dd..eddce25 100644 --- a/src/workspace_doc.md +++ b/src/workspace_doc.md @@ -40,3 +40,30 @@ port = 1111 [site] base_url = "http://127.0.0.1:1111" ``` + +## Render Specific Proposals Only + +Full local `build` and `serve` runs can take time because they process every +proposal file. When you want to quickly test a single proposal or a specific +batch, add a list of desired proposal numbers to the workspace +`.build-eips.toml`: + +```toml +[render] +only = [555, 678] +``` + +Whenever `[render].only` is populated, regular local dirty `build` and `serve` +commands render only those proposal pages. Links and references to excluded +proposals are rewritten to the canonical public site. + +Use CLI `--only` when you want a one-run target list; it overrides any +proposals in `[render].only` for that run: + +```sh +build-eips serve --only 555 +build-eips build --only 555 +build-eips build --only 555 678 +``` + +Multiple proposal numbers in the CLI are space-separated; no commas. From c48d2993558d709944d49d85509819b573bb26a0 Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Mon, 4 May 2026 03:45:27 -0400 Subject: [PATCH 16/20] Add platform development setup Add --platform-dev workspace initialization for cloning optional preprocessor and eipw repos alongside proposal repos and theme. Add POSIX and PowerShell dev-setup scripts that build the local build-eips binary, ensure a supported Zola is available, and install 0.22.1 when needed while reusing existing 0.22.1-or-newer installs. Add setup documentation, release archive checksum sidecars, doctor helper checks, and focused setup tests for contributor workspace setup. --- .github/workflows/release.yml | 11 + README.md | 103 +++++-- scripts/dev-setup | 558 ++++++++++++++++++++++++++++++++++ scripts/dev-setup.ps1 | 518 +++++++++++++++++++++++++++++++ src/cli.rs | 26 +- src/config.rs | 38 ++- src/identity.rs | 2 +- src/main.rs | 9 +- src/workspace.rs | 451 ++++++++++++++++++++------- src/workspace_doc.md | 82 ++++- 10 files changed, 1638 insertions(+), 160 deletions(-) create mode 100755 scripts/dev-setup create mode 100644 scripts/dev-setup.ps1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 39b359c..9501934 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,6 +40,11 @@ jobs: cd target/release $targetPath = Join-Path -Path (Resolve-Path ..\..\dist) -ChildPath $Env:EIPS_BUILD_ARCHIVE Compress-Archive -Path build-eips.exe -DestinationPath $targetPath + $archivePath = $targetPath + $archiveName = Split-Path -Leaf $archivePath + $sidecarPath = "$archivePath.sha256" + $hash = (Get-FileHash -Algorithm SHA256 -Path $archivePath).Hash.ToLower() + [System.IO.File]::WriteAllText($sidecarPath, "$hash $archiveName`n") - name: Compress (Unix) if: matrix.os != 'windows' env: @@ -47,6 +52,12 @@ jobs: run: | mkdir dist tar cavf "$EIPS_BUILD_ARCHIVE" -C target/release build-eips + archive_name=$(basename "$EIPS_BUILD_ARCHIVE") + if [ "${{ matrix.os }}" = "macos" ]; then + (cd dist && shasum -a 256 "$archive_name" > "$archive_name.sha256") + else + (cd dist && sha256sum "$archive_name" > "$archive_name.sha256") + fi - name: Release uses: softprops/action-gh-release@c95fe1489396fe8a9eb87c0abf8aa5b2ef267fda with: diff --git a/README.md b/README.md index 8e6293e..fe5a569 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,111 @@ build-eips ========== -Build system for linting and rendering Ethereum Improvement Proposals ([EIPs] / -[ERCs]). +`build-eips` is the local and CI build tool for Ethereum Improvement Proposal repositories. It prepares a multi-repo workspace with [EIPs], [ERCs] and the site theme, runs proposal validation, and coordinates the full site build pipeline. ## Prerequisites -`build-eips` requires a few runtime dependencies, available from wherever you -get your software: +`build-eips` requires a few runtime dependencies, available from wherever you get your software: - git - libgit2 - openssl -- [zola](https://github.com/getzola/zola/tree/next)[^1] +- [Zola](https://github.com/getzola/zola) 0.22.1[^1] -[^1]: Requires at least commit [`ead17d0a3`] for full functionality. - -[`ead17d0a3`]: https://github.com/getzola/zola/commit/ead17d0a3a20bfb67043a076c061b35ae6b6ddea +[^1]: The setup scripts can install or reuse the pinned Zola version for local workspace testing. ## Installation ### Pre-compiled Binaries -Pre-compiled binaries for Ubuntu, Windows, and macOS are available from -[GitHub Releases]. +Pre-compiled binaries for Ubuntu, Windows, and macOS are available from [GitHub Releases]. [GitHub Releases]: https://github.com/ethereum/build-eips/releases ### From Source -If you're feeling particularly adventurous, you can install the latest version -of `build-eips` like so: +If you're feeling particularly adventurous, you can install the latest version of `build-eips` like so: -```bash +```sh cargo install --git https://github.com/ethereum/build-eips.git ``` [EIPs]: https://github.com/ethereum/EIPs/ [ERCs]: https://github.com/ethereum/ERCs/ +### Local Testing + +For direct manual testing, build and run the local binary explicitly: + +```bash +cargo build +./target/debug/build-eips --help +./target/debug/build-eips -C ../EIPs check +``` + +```powershell +cargo build +.\target\debug\build-eips.exe --help +.\target\debug\build-eips.exe -C ..\EIPs check +``` + +For repeated manual testing, you can put the local debug binary first on `PATH` for the current shell: + +```bash +export PATH="$PWD/target/debug:$PATH" +build-eips --help +build-eips -C ../EIPs check +``` + +```powershell +$env:Path = "$PWD\target\debug;$env:Path" +build-eips --help +build-eips -C ..\EIPs check +``` + +## Workspace Bootstrap + +Use the contributor setup script when you are changing `build-eips` itself and need the full local multi-repo workspace that runs this checkout. + +That workspace lets you test the proposal validation and site build pipeline against local EIPs/ERCs/theme checkouts without installing or reusing a released `build-eips`. + +Linux and macOS: + +```sh +./scripts/dev-setup +``` + +Windows PowerShell: + +```powershell +.\scripts\dev-setup.ps1 +``` + +The setup script builds the local debug binary, ensures the pinned Zola version is available, bootstraps the workspace with that binary, and runs `doctor`. It does not install or reuse a released `build-eips`. + +The script anchors setup through `../EIPs` by default and clones `https://github.com/ethereum/EIPs.git` there when that checkout is missing. Set `ACTIVE_REPO_ROOT` to use another proposal repo checkout. + +After setup, the workspace has this layout: + +```text +EIPs-project/ +├── .build-eips.toml +├── WORKSPACE.md +├── .local-build/ +├── EIPs/ +├── ERCs/ +├── theme/ +├── preprocessor/ +└── eipw/ +``` + +## Workspace Reference + +After bootstrapping the workspace, use the generated workspace guide at `../WORKSPACE.md` for the full local command reference, workspace layout, configuration, source modes, build outputs, and troubleshooting notes. -## Usage +The generated guide is built from [`src/workspace_doc.md`](src/workspace_doc.md). Update that source file when changing workspace documentation. -1. Clone either [`ethereum/EIPs`] or [`ethereum/ERCs`], and change directory - into it. -1. Modify whatever proposal you'd like. -1. Commit your changes. -1. Build the project. You can use: - - `build-eips check` to quickly check for problems like missing sections, - broken internal links, etc. - - `build-eips build` to create an on-disk bundle of HTML, ready to be - deployed. - - `build-eips serve` to launch a web server to preview changes locally. - **NB: live reload is not yet implemented.** +Use `build-eips doctor` when the workspace does not behave as expected. It checks the active repo, sibling repos, theme checkout, local config, required tools, and generated workspace docs. [`ethereum/EIPs`]: https://github.com/ethereum/EIPs/ [`ethereum/ERCs`]: https://github.com/ethereum/ERCs/ diff --git a/scripts/dev-setup b/scripts/dev-setup new file mode 100755 index 0000000..46fbbc3 --- /dev/null +++ b/scripts/dev-setup @@ -0,0 +1,558 @@ +#!/bin/sh + +set -eu + +DEFAULT_ACTIVE_REPO_URL=https://github.com/ethereum/EIPs.git + +say() { + printf '%s\n' "$*" +} + +die() { + printf 'error: %s\n' "$*" >&2 + exit 1 +} + +shell_quote() { + printf "'%s'" "$(printf '%s' "$1" | sed "s/'/'\\\\''/g")" +} + +absolute_path() { + case "$1" in + /*) + printf '%s\n' "$1" + ;; + *) + printf '%s/%s\n' "$INVOCATION_DIR" "$1" + ;; + esac +} + +ensure_active_repo_root() { + if [ "$ACTIVE_REPO_EXPLICIT" = true ]; then + if [ ! -d "$ACTIVE_REPO_ROOT" ]; then + die "configured ACTIVE_REPO_ROOT does not exist: $ACTIVE_REPO_ROOT. Fix ACTIVE_REPO_ROOT or unset it to let this script clone EIPs." + fi + if [ ! -e "$ACTIVE_REPO_ROOT/.git" ]; then + die "configured ACTIVE_REPO_ROOT is not a git checkout: $ACTIVE_REPO_ROOT. Fix ACTIVE_REPO_ROOT or unset it to let this script clone EIPs." + fi + else + if [ ! -e "$ACTIVE_REPO_ROOT" ]; then + command -v git >/dev/null 2>&1 || die "need git to clone default proposal repo from $DEFAULT_ACTIVE_REPO_URL" + + say "No active proposal repo found at $ACTIVE_REPO_ROOT." + say "Cloning default proposal repo from $DEFAULT_ACTIVE_REPO_URL" + say "This may take a few minutes..." + if ! git clone "$DEFAULT_ACTIVE_REPO_URL" "$ACTIVE_REPO_ROOT"; then + die "failed to clone default proposal repo from $DEFAULT_ACTIVE_REPO_URL to $ACTIVE_REPO_ROOT" + fi + say "Cloned default proposal repo to $ACTIVE_REPO_ROOT" + fi + + if [ ! -d "$ACTIVE_REPO_ROOT" ]; then + die "default active proposal repo path exists but is not a directory: $ACTIVE_REPO_ROOT. Remove it or set ACTIVE_REPO_ROOT." + fi + if [ ! -e "$ACTIVE_REPO_ROOT/.git" ]; then + die "default active proposal repo path exists but is not a git checkout: $ACTIVE_REPO_ROOT. Remove it or set ACTIVE_REPO_ROOT." + fi + fi + + ACTIVE_REPO_ROOT=$(CDPATH= cd -- "$ACTIVE_REPO_ROOT" && pwd) +} + +normalize_dir_for_path_compare() { + dir=$1 + [ -n "$dir" ] || return 1 + + while [ "$dir" != "/" ] && [ "${dir%/}" != "$dir" ]; do + dir=${dir%/} + done + + case "$dir" in + /*) + absolute_dir=$dir + ;; + *) + absolute_dir=$(pwd -P)/$dir + ;; + esac + + if [ -d "$absolute_dir" ]; then + (CDPATH= cd -P -- "$absolute_dir" 2>/dev/null && pwd -P) + else + printf '%s\n' "$absolute_dir" + fi +} + +move_dir_to_front_of_script_path() { + dir=$1 + target=$(normalize_dir_for_path_compare "$dir") || return 1 + new_path= + old_ifs=$IFS + IFS=: + for path_entry in ${PATH:-}; do + [ -n "$path_entry" ] || continue + candidate=$(normalize_dir_for_path_compare "$path_entry") || continue + if [ "$candidate" = "$target" ]; then + continue + fi + + if [ -n "$new_path" ]; then + new_path=$new_path:$path_entry + else + new_path=$path_entry + fi + done + IFS=$old_ifs + + if [ -n "$new_path" ]; then + updated_path=$dir:$new_path + else + updated_path=$dir + fi + + if [ "${PATH:-}" != "$updated_path" ]; then + PATH=$updated_path + add_path_note "$dir" + fi +} + +add_path_note() { + dir=$1 + target=$(normalize_dir_for_path_compare "$dir") || return 1 + old_ifs=$IFS + IFS=' +' + for path_note in $PATH_NOTES; do + candidate=$(normalize_dir_for_path_compare "$path_note") || continue + if [ "$candidate" = "$target" ]; then + IFS=$old_ifs + return 0 + fi + done + IFS=$old_ifs + + case " +$PATH_NOTES +" in + *" +$dir +"*) + return 0 + ;; + esac + + if [ -n "$PATH_NOTES" ]; then + PATH_NOTES=$PATH_NOTES' +'$dir + else + PATH_NOTES=$dir + fi +} + +pick_home_install_dir() { + [ -n "${HOME:-}" ] || return 1 + dir=$HOME/.local/bin + mkdir -p "$dir" 2>/dev/null || return 1 + [ -d "$dir" ] || return 1 + [ -w "$dir" ] || return 1 + printf '%s\n' "$dir" +} + +pick_writable_path_dir() { + old_ifs=$IFS + IFS=: + for dir in ${PATH:-}; do + [ -n "$dir" ] || continue + [ -d "$dir" ] || continue + [ -w "$dir" ] || continue + printf '%s\n' "$dir" + IFS=$old_ifs + return 0 + done + IFS=$old_ifs + return 1 +} + +pick_install_dir() { + if dir=$(pick_home_install_dir); then + printf '%s\n' "$dir" + return 0 + fi + + pick_writable_path_dir +} + +ensure_install_dir() { + [ -n "${INSTALL_DIR:-}" ] && return 0 + + INSTALL_DIR=$(pick_install_dir) || die "could not find a writable install directory" + move_dir_to_front_of_script_path "$INSTALL_DIR" +} + +download() { + url=$1 + destination=$2 + + if command -v curl >/dev/null 2>&1; then + curl -fsSL "$url" -o "$destination" + return 0 + fi + + if command -v wget >/dev/null 2>&1; then + wget -qO "$destination" "$url" + return 0 + fi + + die "need curl or wget to download release binaries" +} + +file_sha256() { + path=$1 + + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$path" | awk '{ print $1 }' | tr 'A-F' 'a-f' + return 0 + fi + + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$path" | awk '{ print $1 }' | tr 'A-F' 'a-f' + return 0 + fi + + if command -v openssl >/dev/null 2>&1; then + openssl dgst -sha256 -r "$path" | awk '{ print $1 }' | tr 'A-F' 'a-f' + return 0 + fi + + die "need sha256sum, shasum, or openssl to verify release checksums" +} + +verify_file_hash() { + archive_path=$1 + expected_hash=$2 + archive_name=$3 + + actual_hash=$(file_sha256 "$archive_path") + [ "$actual_hash" = "$expected_hash" ] || die "checksum mismatch for $archive_name" +} + +cleanup_tmpdir() { + if [ -n "${INSTALL_TMP_TO_CLEAN:-}" ]; then + rm -f "$INSTALL_TMP_TO_CLEAN" + INSTALL_TMP_TO_CLEAN= + fi + if [ -n "${TMPDIR_TO_CLEAN:-}" ]; then + rm -rf "$TMPDIR_TO_CLEAN" + TMPDIR_TO_CLEAN= + fi +} + +cleanup_for_signal() { + cleanup_tmpdir + trap - EXIT INT TERM HUP QUIT + exit 1 +} + +set_cleanup_traps() { + trap cleanup_tmpdir EXIT + trap cleanup_for_signal INT TERM HUP QUIT +} + +clear_cleanup_traps() { + trap - EXIT INT TERM HUP QUIT +} + +unsupported_zola_platform() { + os=$1 + arch=$2 + die "Unsupported platform $os/$arch for automatic Zola install. Install Zola 0.22.1 manually from https://github.com/getzola/zola/releases and ensure it is on PATH." +} + +is_linux_musl() { + if command -v ldd >/dev/null 2>&1 && ldd --version 2>&1 | grep -qi musl; then + return 0 + fi + + set -- /lib/ld-musl-*.so.1 + [ -e "$1" ] +} + +select_zola_release() { + os=$(uname -s) + arch=$(uname -m) + + case "$os" in + Linux) + if is_linux_musl; then + case "$arch" in + x86_64) + ZOLA_ARCHIVE_NAME=zola-v0.22.1-x86_64-unknown-linux-musl.tar.gz + ZOLA_ARCHIVE_HASH=227df99b664421240a8ba77747067c51259b08159125d5603763b3b173b9a881 + ;; + *) + unsupported_zola_platform "$os" "$arch" + ;; + esac + else + case "$arch" in + x86_64) + ZOLA_ARCHIVE_NAME=zola-v0.22.1-x86_64-unknown-linux-gnu.tar.gz + ZOLA_ARCHIVE_HASH=0ca09aa40376aaa9ddfb512ff9ad963262ef95edb0d0f2d5ec6961b6f5cf22ef + ;; + aarch64|arm64) + ZOLA_ARCHIVE_NAME=zola-v0.22.1-aarch64-unknown-linux-gnu.tar.gz + ZOLA_ARCHIVE_HASH=8af437ec6352f33ccd24d7a1cfcb54a3db95d3ce376dc69525b4ef3fb6b8c1d1 + ;; + *) + unsupported_zola_platform "$os" "$arch" + ;; + esac + fi + ;; + Darwin) + case "$arch" in + x86_64) + ZOLA_ARCHIVE_NAME=zola-v0.22.1-x86_64-apple-darwin.tar.gz + ZOLA_ARCHIVE_HASH=3898709e154ae0593933264a540c869348bdb10d7f1b03a42dfb78d63703b3b5 + ;; + arm64|aarch64) + ZOLA_ARCHIVE_NAME=zola-v0.22.1-aarch64-apple-darwin.tar.gz + ZOLA_ARCHIVE_HASH=46ac45a9e7628dba8593b124ee8794f4f9aa1c6b569918ecd4bbc5d0be190515 + ;; + *) + unsupported_zola_platform "$os" "$arch" + ;; + esac + ;; + *) + unsupported_zola_platform "$os" "$arch" + ;; + esac +} + +parse_zola_version() { + output=$1 + + set -f + set -- $output + set +f + [ "$#" -ge 2 ] || return 1 + + ZOLA_FOUND_VERSION=$2 + version_core=$(printf '%s\n' "$ZOLA_FOUND_VERSION" | sed -n 's/^\([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\).*/\1 \2 \3/p') + [ -n "$version_core" ] || return 1 + + set -- $version_core + ZOLA_VERSION_MAJOR=$1 + ZOLA_VERSION_MINOR=$2 + ZOLA_VERSION_PATCH=$3 + ZOLA_VERSION_SUFFIX=$(printf '%s\n' "$ZOLA_FOUND_VERSION" | sed -n 's/^[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*//p') + + return 0 +} + +zola_version_relation() { + if [ "$ZOLA_VERSION_MAJOR" -lt 0 ]; then + printf '%s\n' below + elif [ "$ZOLA_VERSION_MAJOR" -gt 0 ]; then + printf '%s\n' newer + elif [ "$ZOLA_VERSION_MINOR" -lt 22 ]; then + printf '%s\n' below + elif [ "$ZOLA_VERSION_MINOR" -gt 22 ]; then + printf '%s\n' newer + elif [ "$ZOLA_VERSION_PATCH" -lt 1 ]; then + printf '%s\n' below + elif [ "$ZOLA_VERSION_PATCH" -gt 1 ]; then + printf '%s\n' newer + elif [ -n "$ZOLA_VERSION_SUFFIX" ]; then + printf '%s\n' below + else + printf '%s\n' equal + fi +} + +install_zola() { + command -v tar >/dev/null 2>&1 || die "need tar to unpack the zola release archive" + select_zola_release + ensure_install_dir + + tmpdir=$(mktemp -d) + TMPDIR_TO_CLEAN=$tmpdir + set_cleanup_traps + + archive_path=$tmpdir/$ZOLA_ARCHIVE_NAME + release_base_url=https://github.com/getzola/zola/releases/download/v0.22.1 + + say "Installing zola 0.22.1 from $release_base_url/$ZOLA_ARCHIVE_NAME" + download "$release_base_url/$ZOLA_ARCHIVE_NAME" "$archive_path" + verify_file_hash "$archive_path" "$ZOLA_ARCHIVE_HASH" "$ZOLA_ARCHIVE_NAME" + + tar -xzf "$archive_path" -C "$tmpdir" + [ -f "$tmpdir/zola" ] || die "zola release archive did not contain zola" + + install_tmp=$(mktemp "$INSTALL_DIR/.zola.XXXXXX") || die "could not create temporary zola install file in $INSTALL_DIR" + INSTALL_TMP_TO_CLEAN=$install_tmp + cp "$tmpdir/zola" "$install_tmp" + chmod +x "$install_tmp" + mv "$install_tmp" "$INSTALL_DIR/zola" + INSTALL_TMP_TO_CLEAN= + + ZOLA=$INSTALL_DIR/zola + cleanup_tmpdir + clear_cleanup_traps +} + +ensure_zola() { + if command -v zola >/dev/null 2>&1; then + ZOLA=$(command -v zola) + elif [ -n "${HOME:-}" ] && [ -x "$HOME/.local/bin/zola" ]; then + INSTALL_DIR=$HOME/.local/bin + ZOLA=$INSTALL_DIR/zola + move_dir_to_front_of_script_path "$INSTALL_DIR" + else + install_zola + return 0 + fi + + if [ -n "${ZOLA:-}" ]; then + zola_output=$("$ZOLA" --version 2>/dev/null || true) + if parse_zola_version "$zola_output"; then + relation=$(zola_version_relation) + case "$relation" in + below) + say "Found zola $ZOLA_FOUND_VERSION below supported 0.22.1. Installing zola 0.22.1." + install_zola + ;; + equal) + say "Using existing zola $ZOLA_FOUND_VERSION at $ZOLA" + ;; + newer) + say "Found zola $ZOLA_FOUND_VERSION. build-eips is tested with zola 0.22.1 or newer. Continuing with the installed version." + ;; + esac + else + say "Found zola with unparseable version output. Installing zola 0.22.1." + install_zola + fi + fi +} + +INCLUDE_TEMPLATE=false + +while [ "$#" -gt 0 ]; do + case "$1" in + --template) + INCLUDE_TEMPLATE=true + ;; + -h|--help) + say "usage: $0 [--template]" + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac + shift +done + +INVOCATION_DIR=$(pwd) +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +PREPROCESSOR_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd) +INSTALL_DIR= +PATH_NOTES= +TMPDIR_TO_CLEAN= +INSTALL_TMP_TO_CLEAN= + +if [ -n "${WORKSPACE_ROOT:-}" ]; then + WORKSPACE_ROOT=$(absolute_path "$WORKSPACE_ROOT") +else + WORKSPACE_ROOT=$(CDPATH= cd -- "$PREPROCESSOR_ROOT/.." && pwd) +fi +if [ -d "$WORKSPACE_ROOT" ]; then + WORKSPACE_ROOT=$(CDPATH= cd -- "$WORKSPACE_ROOT" && pwd) +fi + +if [ -n "${ACTIVE_REPO_ROOT:-}" ]; then + ACTIVE_REPO_EXPLICIT=true + ACTIVE_REPO_ROOT=$(absolute_path "$ACTIVE_REPO_ROOT") +else + ACTIVE_REPO_EXPLICIT=false + ACTIVE_REPO_ROOT=$WORKSPACE_ROOT/EIPs +fi + +ensure_active_repo_root +BUILD_EIPS=$PREPROCESSOR_ROOT/target/debug/build-eips + +say "Workspace root: $WORKSPACE_ROOT" +say "Active proposal repo: $ACTIVE_REPO_ROOT" +say "Local build-eips: $BUILD_EIPS" + +command -v cargo >/dev/null 2>&1 || die "need cargo to build local build-eips. Install Rust from https://rustup.rs/ and re-run this script." + +say "Building local build-eips" +if ! (CDPATH= cd -- "$PREPROCESSOR_ROOT" && cargo build); then + die "cargo build failed" +fi + +[ -x "$BUILD_EIPS" ] || die "local build-eips was not built at $BUILD_EIPS" + +ensure_zola + +say "Bootstrapping local contributor workspace" +if [ "$INCLUDE_TEMPLATE" = true ]; then + if ! "$BUILD_EIPS" -C "$ACTIVE_REPO_ROOT" init "$WORKSPACE_ROOT" --platform-dev --template; then + die "build-eips init failed" + fi +else + if ! "$BUILD_EIPS" -C "$ACTIVE_REPO_ROOT" init "$WORKSPACE_ROOT" --platform-dev; then + die "build-eips init failed" + fi +fi + +say "Running build-eips doctor" +if ! "$BUILD_EIPS" -C "$ACTIVE_REPO_ROOT" doctor; then + say "Warning: build-eips doctor reported issues above. Fix them before relying on local build-eips commands." +fi + +WORKSPACE_DOC=$WORKSPACE_ROOT/WORKSPACE.md +say "" +if [ -f "$WORKSPACE_DOC" ]; then + say "Workspace docs: $WORKSPACE_DOC" +else + say "Warning: workspace docs were not found at $WORKSPACE_DOC after build-eips init" +fi + +if [ -n "$PATH_NOTES" ]; then + say "" + say "Updated PATH for this script process only:" + old_ifs=$IFS + IFS=' +' + for path_note in $PATH_NOTES; do + say " $path_note" + done + say "To make this permanent in your shell, add:" + for path_note in $PATH_NOTES; do + say " export PATH=$(shell_quote "$path_note"):\$PATH" + done + IFS=$old_ifs +fi + +TARGET_DEBUG_DIR=$PREPROCESSOR_ROOT/target/debug + +say "" +say "Next commands:" +say " cd $(shell_quote "$PREPROCESSOR_ROOT")" +say " cargo test" +say " $(shell_quote "$BUILD_EIPS") -C $(shell_quote "$ACTIVE_REPO_ROOT") check" +say " $(shell_quote "$BUILD_EIPS") -C $(shell_quote "$ACTIVE_REPO_ROOT") build" +say " $(shell_quote "$BUILD_EIPS") -C $(shell_quote "$ACTIVE_REPO_ROOT") serve" +say "" +say "For shorter one-off local commands in this shell, run:" +say " export PATH=$(shell_quote "$TARGET_DEBUG_DIR"):\$PATH" +if [ "$INCLUDE_TEMPLATE" = true ]; then + say "" + say "Template check:" + say " $(shell_quote "$BUILD_EIPS") -C $(shell_quote "$WORKSPACE_ROOT/template") check" +fi diff --git a/scripts/dev-setup.ps1 b/scripts/dev-setup.ps1 new file mode 100644 index 0000000..4406601 --- /dev/null +++ b/scripts/dev-setup.ps1 @@ -0,0 +1,518 @@ +param( + [switch]$Template +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$DefaultActiveRepoUrl = "https://github.com/ethereum/EIPs.git" + +function Say { + param([string]$Message) + + Write-Host $Message +} + +function Die { + param([string]$Message) + + throw "error: $Message" +} + +function ConvertTo-NormalizedDirectoryPath { + param([string]$Directory) + + $trimmed = $Directory.Trim('"') + $trimChars = [char[]]@([System.IO.Path]::DirectorySeparatorChar, [System.IO.Path]::AltDirectorySeparatorChar) + + try { + return ([System.IO.Path]::GetFullPath($trimmed)).TrimEnd($trimChars) + } catch { + return $trimmed.TrimEnd($trimChars) + } +} + +function Add-PathNote { + param([string]$InstallDir) + + $target = ConvertTo-NormalizedDirectoryPath -Directory $InstallDir + foreach ($pathNote in $script:PathNotes) { + $candidate = ConvertTo-NormalizedDirectoryPath -Directory $pathNote + if ([string]::Equals($candidate, $target, [System.StringComparison]::OrdinalIgnoreCase)) { + return + } + } + + $script:PathNotes += $InstallDir +} + +function Move-DirectoryToFrontOfSessionPath { + param([string]$InstallDir) + + $target = ConvertTo-NormalizedDirectoryPath -Directory $InstallDir + $remainingEntries = @() + + if (-not [string]::IsNullOrWhiteSpace($env:Path)) { + foreach ($entry in ($env:Path -split ";")) { + if ([string]::IsNullOrWhiteSpace($entry)) { + continue + } + + $candidate = ConvertTo-NormalizedDirectoryPath -Directory $entry + if ([string]::Equals($candidate, $target, [System.StringComparison]::OrdinalIgnoreCase)) { + continue + } + + $remainingEntries += $entry + } + } + + $newEntries = @($InstallDir) + if ($remainingEntries.Count -gt 0) { + $newEntries += $remainingEntries + } + + $updatedPath = [string]::Join(";", $newEntries) + if (-not [string]::Equals($env:Path, $updatedPath, [System.StringComparison]::Ordinal)) { + $env:Path = $updatedPath + + Add-PathNote -InstallDir $InstallDir + } +} + +function Find-ZolaOnPath { + foreach ($commandName in @("zola", "zola.exe")) { + $commands = @(Get-Command -Name $commandName -CommandType Application -ErrorAction SilentlyContinue) + if ($commands.Count -gt 0) { + return $commands[0].Source + } + } + + return $null +} + +function Assert-InstallDirWritable { + param([string]$InstallDir) + + $probePath = $null + + try { + New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null + $probeName = ".build-eips-write-test-{0}.tmp" -f ([System.Guid]::NewGuid().ToString("N")) + $probePath = Join-Path -Path $InstallDir -ChildPath $probeName + [System.IO.File]::WriteAllText($probePath, "") + Remove-Item -LiteralPath $probePath -Force + $probePath = $null + } catch { + Die ("install directory cannot be created or written ({0}): {1}" -f $InstallDir, $_.Exception.Message) + } finally { + if (($null -ne $probePath) -and (Test-Path -LiteralPath $probePath)) { + Remove-Item -LiteralPath $probePath -Force -ErrorAction SilentlyContinue + } + } +} + +function Invoke-ReleaseDownload { + param( + [string]$Url, + [string]$Destination + ) + + $previousProgressPreference = $ProgressPreference + $ProgressPreference = "SilentlyContinue" + try { + Invoke-WebRequest -Uri $Url -OutFile $Destination -UseBasicParsing + } catch { + Die ("failed to download {0}: {1}" -f $Url, $_.Exception.Message) + } finally { + $ProgressPreference = $previousProgressPreference + } +} + +function Assert-FileSha256 { + param( + [string]$ArchivePath, + [string]$ExpectedHash, + [string]$ArchiveName + ) + + $actualHash = (Get-FileHash -Algorithm SHA256 -Path $ArchivePath).Hash.ToLowerInvariant() + if ($actualHash -ne $ExpectedHash.ToLowerInvariant()) { + Die "checksum mismatch for $ArchiveName" + } +} + +function Get-ZolaReleaseAsset { + $architecture = $env:PROCESSOR_ARCHITECTURE + if ([string]::IsNullOrWhiteSpace($architecture)) { + $architecture = "unknown" + } + if (($architecture -eq "x86") -and (-not [string]::IsNullOrWhiteSpace($env:PROCESSOR_ARCHITEW6432))) { + $architecture = $env:PROCESSOR_ARCHITEW6432 + } + + switch ($architecture.ToUpperInvariant()) { + "AMD64" { + return @{ + ArchiveName = "zola-v0.22.1-x86_64-pc-windows-msvc.zip" + Hash = "2c8b368f5abdf2b2478748f9549a761fd6599238e18948eccb76a7cae51f5dc1" + } + } + default { + Die "Unsupported platform Windows/$architecture for automatic Zola install. Install Zola 0.22.1 manually from https://github.com/getzola/zola/releases and ensure it is on PATH." + } + } +} + +function Install-Zola { + param( + [string]$InstallDir, + [string]$ZolaPath + ) + + $asset = Get-ZolaReleaseAsset + $archiveName = $asset.ArchiveName + $archiveHash = $asset.Hash + $releaseBaseUrl = "https://github.com/getzola/zola/releases/download/v0.22.1" + $tmpRoot = Join-Path -Path ([System.IO.Path]::GetTempPath()) -ChildPath ("zola-" + [System.Guid]::NewGuid().ToString("N")) + $archivePath = Join-Path -Path $tmpRoot -ChildPath $archiveName + $extractDir = Join-Path -Path $tmpRoot -ChildPath "extract" + + try { + Assert-InstallDirWritable -InstallDir $InstallDir + + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + + Say "Installing zola 0.22.1 from $releaseBaseUrl/$archiveName" + Invoke-ReleaseDownload -Url "$releaseBaseUrl/$archiveName" -Destination $archivePath + Assert-FileSha256 -ArchivePath $archivePath -ExpectedHash $archiveHash -ArchiveName $archiveName + + Expand-Archive -LiteralPath $archivePath -DestinationPath $extractDir -Force + + $extractedZola = Join-Path -Path $extractDir -ChildPath "zola.exe" + if (-not (Test-Path -LiteralPath $extractedZola -PathType Leaf)) { + Die "zola release archive did not contain expected zola.exe" + } + + try { + Move-Item -LiteralPath $extractedZola -Destination $ZolaPath -Force + } catch { + Die ("zola.exe is in use. Close any running zola process and re-run this script. Details: {0}" -f $_.Exception.Message) + } + + return $ZolaPath + } catch { + Die ("failed to install zola: {0}" -f $_.Exception.Message) + } finally { + if (Test-Path -LiteralPath $tmpRoot) { + Remove-Item -LiteralPath $tmpRoot -Recurse -Force -ErrorAction SilentlyContinue + } + } +} + +function Get-ZolaVersionInfo { + param([string]$ZolaPath) + + try { + $output = & $ZolaPath --version 2>$null + if ($LASTEXITCODE -ne 0) { + return $null + } + } catch { + return $null + } + + $fields = @(($output -join " ") -split "\s+" | Where-Object { $_.Length -gt 0 }) + if ($fields.Count -lt 2) { + return $null + } + + $versionToken = $fields[1] + if ($versionToken -notmatch "^([0-9]+)\.([0-9]+)\.([0-9]+)(.*)$") { + return $null + } + + return @{ + VersionToken = $versionToken + Version = [version]("{0}.{1}.{2}" -f $Matches[1], $Matches[2], $Matches[3]) + Suffix = $Matches[4] + } +} + +function Get-ZolaVersionRelation { + param([hashtable]$VersionInfo) + + $minimumVersion = [version]"0.22.1" + if ($VersionInfo.Version -lt $minimumVersion) { + return "below" + } + if ($VersionInfo.Version -gt $minimumVersion) { + return "newer" + } + if (-not [string]::IsNullOrEmpty($VersionInfo.Suffix)) { + return "below" + } + + return "equal" +} + +function Get-DefaultInstallPaths { + if ([string]::IsNullOrWhiteSpace($env:LOCALAPPDATA)) { + Die "LOCALAPPDATA is not set; cannot determine the user-local Zola install directory" + } + + $installDir = Join-Path -Path (Join-Path -Path $env:LOCALAPPDATA -ChildPath "build-eips") -ChildPath "bin" + $zolaPath = Join-Path -Path $installDir -ChildPath "zola.exe" + + return @{ + InstallDir = $installDir + ZolaPath = $zolaPath + } +} + +function Install-PinnedZola { + $defaultPaths = Get-DefaultInstallPaths + $installedZola = Install-Zola -InstallDir $defaultPaths.InstallDir -ZolaPath $defaultPaths.ZolaPath + Move-DirectoryToFrontOfSessionPath -InstallDir $defaultPaths.InstallDir + + return $installedZola +} + +function Initialize-Zola { + $zolaPath = Find-ZolaOnPath + if ($null -eq $zolaPath) { + $defaultPaths = Get-DefaultInstallPaths + if (Test-Path -LiteralPath $defaultPaths.ZolaPath -PathType Leaf) { + $zolaPath = $defaultPaths.ZolaPath + Move-DirectoryToFrontOfSessionPath -InstallDir $defaultPaths.InstallDir + } + } + + if ($null -eq $zolaPath) { + return (Install-PinnedZola) + } + + $versionInfo = Get-ZolaVersionInfo -ZolaPath $zolaPath + if ($null -eq $versionInfo) { + Say "Found zola with unparseable version output. Installing zola 0.22.1." + return (Install-PinnedZola) + } + + $relation = Get-ZolaVersionRelation -VersionInfo $versionInfo + switch ($relation) { + "below" { + Say ("Found zola {0} below supported 0.22.1. Installing zola 0.22.1." -f $versionInfo.VersionToken) + return (Install-PinnedZola) + } + "equal" { + Say ("Using existing zola {0} at {1}" -f $versionInfo.VersionToken, $zolaPath) + return $zolaPath + } + "newer" { + Say ("Found zola {0}. build-eips is tested with zola 0.22.1 or newer. Continuing with the installed version." -f $versionInfo.VersionToken) + return $zolaPath + } + } +} + +function ConvertTo-PowerShellQuotedPath { + param([string]$Path) + + return "'{0}'" -f ($Path -replace "'", "''") +} + +function Resolve-ConfiguredPath { + param( + [string]$PathValue, + [string]$BaseDir + ) + + if ([System.IO.Path]::IsPathRooted($PathValue)) { + $candidate = $PathValue + } else { + $candidate = Join-Path -Path $BaseDir -ChildPath $PathValue + } + + try { + return (Resolve-Path -LiteralPath $candidate).ProviderPath + } catch { + return [System.IO.Path]::GetFullPath($candidate) + } +} + +function Resolve-ActiveRepoRoot { + param( + [string]$InvocationDir, + [string]$WorkspaceRoot + ) + + $activeRepoExplicit = $false + if (-not [string]::IsNullOrWhiteSpace($env:ACTIVE_REPO_ROOT)) { + $activeRepoExplicit = $true + if ([System.IO.Path]::IsPathRooted($env:ACTIVE_REPO_ROOT)) { + $activeRepoCandidate = $env:ACTIVE_REPO_ROOT + } else { + $activeRepoCandidate = Join-Path -Path $InvocationDir -ChildPath $env:ACTIVE_REPO_ROOT + } + } else { + $activeRepoCandidate = Join-Path -Path $WorkspaceRoot -ChildPath "EIPs" + } + + if ($activeRepoExplicit) { + try { + $resolved = (Resolve-Path -LiteralPath $activeRepoCandidate).ProviderPath + } catch { + Die "configured ACTIVE_REPO_ROOT does not exist: $activeRepoCandidate. Fix ACTIVE_REPO_ROOT or unset it to let this script clone EIPs." + } + + if (-not (Test-Path -LiteralPath $resolved -PathType Container)) { + Die "configured ACTIVE_REPO_ROOT is not a directory: $resolved. Fix ACTIVE_REPO_ROOT or unset it to let this script clone EIPs." + } + if (-not (Test-Path -LiteralPath (Join-Path -Path $resolved -ChildPath ".git"))) { + Die "configured ACTIVE_REPO_ROOT is not a git checkout: $resolved. Fix ACTIVE_REPO_ROOT or unset it to let this script clone EIPs." + } + + return @{ + Path = $resolved + Explicit = $true + } + } + + if (-not (Test-Path -LiteralPath $activeRepoCandidate)) { + $gitCommand = Get-Command git -CommandType Application -ErrorAction SilentlyContinue + if ($null -eq $gitCommand) { + Die "need git to clone default proposal repo from $DefaultActiveRepoUrl" + } + + Say "No active proposal repo found at $activeRepoCandidate." + Say "Cloning default proposal repo from $DefaultActiveRepoUrl" + Say "This may take a few minutes..." + & git clone $DefaultActiveRepoUrl $activeRepoCandidate + $GitCloneExitCode = $LASTEXITCODE + if ($GitCloneExitCode -ne 0) { + Die "failed to clone default proposal repo from $DefaultActiveRepoUrl to $activeRepoCandidate" + } + Say "Cloned default proposal repo to $activeRepoCandidate" + } + + try { + $resolved = (Resolve-Path -LiteralPath $activeRepoCandidate).ProviderPath + } catch { + Die "default active proposal repo path does not exist after clone: $activeRepoCandidate" + } + + if (-not (Test-Path -LiteralPath $resolved -PathType Container)) { + Die "default active proposal repo path exists but is not a directory: $resolved. Remove it or set ACTIVE_REPO_ROOT." + } + if (-not (Test-Path -LiteralPath (Join-Path -Path $resolved -ChildPath ".git"))) { + Die "default active proposal repo path exists but is not a git checkout: $resolved. Remove it or set ACTIVE_REPO_ROOT." + } + + return @{ + Path = $resolved + Explicit = $false + } +} + +$PathNotes = @() + +$InvocationDir = (Get-Location).ProviderPath +$ScriptDir = (Resolve-Path -LiteralPath $PSScriptRoot).ProviderPath +$PreprocessorRoot = (Resolve-Path -LiteralPath (Split-Path -Path $ScriptDir -Parent)).ProviderPath + +if (-not [string]::IsNullOrWhiteSpace($env:WORKSPACE_ROOT)) { + $WorkspaceRoot = Resolve-ConfiguredPath -PathValue $env:WORKSPACE_ROOT -BaseDir $InvocationDir +} else { + $WorkspaceRoot = (Resolve-Path -LiteralPath (Split-Path -Path $PreprocessorRoot -Parent)).ProviderPath +} + +$ActiveRepoInfo = Resolve-ActiveRepoRoot -InvocationDir $InvocationDir -WorkspaceRoot $WorkspaceRoot +$ActiveRepoRoot = $ActiveRepoInfo["Path"] +$ActiveRepoExplicit = $ActiveRepoInfo["Explicit"] +$BuildEipsPath = Join-Path -Path $PreprocessorRoot -ChildPath "target\debug\build-eips.exe" + +Say "Workspace root: $WorkspaceRoot" +Say "Active proposal repo: $ActiveRepoRoot" +Say "Local build-eips: $BuildEipsPath" +Say "If PowerShell blocks this script, run:" +Say " powershell -ExecutionPolicy Bypass -File .\scripts\dev-setup.ps1" + +$cargoCommand = Get-Command cargo -CommandType Application -ErrorAction SilentlyContinue +if ($null -eq $cargoCommand) { + Die "need cargo to build local build-eips. Install Rust from https://rustup.rs/ and re-run this script." +} + +Say "Building local build-eips" +$CargoExitCode = 0 +Push-Location -LiteralPath $PreprocessorRoot +try { + & cargo build + $CargoExitCode = $LASTEXITCODE +} finally { + Pop-Location +} +if ($CargoExitCode -ne 0) { + Die "cargo build failed with exit code $CargoExitCode" +} + +if (-not (Test-Path -LiteralPath $BuildEipsPath -PathType Leaf)) { + Die "local build-eips was not built at $BuildEipsPath" +} + +$ZolaPath = Initialize-Zola + +Say "Bootstrapping local contributor workspace" +if ($Template) { + & $BuildEipsPath -C $ActiveRepoRoot init $WorkspaceRoot --platform-dev --template + $WorkspaceInitExitCode = $LASTEXITCODE +} else { + & $BuildEipsPath -C $ActiveRepoRoot init $WorkspaceRoot --platform-dev + $WorkspaceInitExitCode = $LASTEXITCODE +} +if ($WorkspaceInitExitCode -ne 0) { + Die "build-eips init failed with exit code $WorkspaceInitExitCode" +} + +Say "Running build-eips doctor" +& $BuildEipsPath -C $ActiveRepoRoot doctor +$WorkspaceDoctorExitCode = $LASTEXITCODE +if ($WorkspaceDoctorExitCode -ne 0) { + Say "Warning: build-eips doctor reported issues above. Fix them before relying on local build-eips commands." +} + +$WorkspaceDocPath = Join-Path -Path $WorkspaceRoot -ChildPath "WORKSPACE.md" +Say "" +if (Test-Path -LiteralPath $WorkspaceDocPath -PathType Leaf) { + Say "Workspace docs: $WorkspaceDocPath" +} else { + Say "Warning: workspace docs were not found at $WorkspaceDocPath after build-eips init" +} + +if ($PathNotes.Count -gt 0) { + Say "" + Say 'Updated PATH for this PowerShell session only:' + foreach ($pathNote in $PathNotes) { + Say " $pathNote" + } + Say "To make this permanent, add the listed directory or directories to your user Path in Windows Environment Variables." +} + +$TargetDebugDir = Split-Path -Path $BuildEipsPath -Parent + +Say "" +Say "Next commands:" +Say (" cd {0}" -f (ConvertTo-PowerShellQuotedPath -Path $PreprocessorRoot)) +Say " cargo test" +Say (" {0} -C {1} check" -f (ConvertTo-PowerShellQuotedPath -Path $BuildEipsPath), (ConvertTo-PowerShellQuotedPath -Path $ActiveRepoRoot)) +Say (" {0} -C {1} build" -f (ConvertTo-PowerShellQuotedPath -Path $BuildEipsPath), (ConvertTo-PowerShellQuotedPath -Path $ActiveRepoRoot)) +Say (" {0} -C {1} serve" -f (ConvertTo-PowerShellQuotedPath -Path $BuildEipsPath), (ConvertTo-PowerShellQuotedPath -Path $ActiveRepoRoot)) +Say "" +Say "For shorter one-off local commands in this PowerShell session, run:" +Say (" `$env:Path = {0} + ';' + `$env:Path" -f (ConvertTo-PowerShellQuotedPath -Path $TargetDebugDir)) +if ($Template) { + $TemplateRoot = Join-Path -Path $WorkspaceRoot -ChildPath "template" + Say "" + Say "Template check:" + Say (" {0} -C {1} check" -f (ConvertTo-PowerShellQuotedPath -Path $BuildEipsPath), (ConvertTo-PowerShellQuotedPath -Path $TemplateRoot)) +} diff --git a/src/cli.rs b/src/cli.rs index 2ea0951..487ef12 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -146,6 +146,10 @@ pub(crate) enum Operation { /// Also clone template for proposal-family scaffold work #[arg(long)] template: bool, + + /// Also clone preprocessor and eipw for platform development + #[arg(long)] + platform_dev: bool, }, /// Check workspace layout, local repos, and required tools @@ -767,18 +771,38 @@ mod tests { fn workspace_lifecycle_commands_parse() { let plain = parse_args(&["build-eips", "init", "/tmp/workspace"]); let template = parse_args(&["build-eips", "init", "/tmp/workspace", "--template"]); + let combined = parse_args(&[ + "build-eips", + "init", + "/tmp/workspace", + "--template", + "--platform-dev", + ]); let doctor = parse_args(&["build-eips", "doctor"]); assert!(matches!( plain.operation, Operation::Init { template: false, + platform_dev: false, .. } )); assert!(matches!( template.operation, - Operation::Init { template: true, .. } + Operation::Init { + template: true, + platform_dev: false, + .. + } + )); + assert!(matches!( + combined.operation, + Operation::Init { + template: true, + platform_dev: true, + .. + } )); assert!(matches!(doctor.operation, Operation::Doctor)); } diff --git a/src/config.rs b/src/config.rs index 9730aab..60f5ada 100644 --- a/src/config.rs +++ b/src/config.rs @@ -23,21 +23,27 @@ pub const DEFAULT_THEME_DIR: &str = "theme"; pub const DEFAULT_SERVER_HOST: &str = "127.0.0.1"; pub const DEFAULT_SERVER_PORT: u16 = 1111; pub const DEFAULT_SITE_BASE_URL: &str = "http://127.0.0.1:1111"; -const RESERVED_REPO_IDS: &[&str] = &["theme", "preprocessor", "eipw"]; +const RESERVED_WORKSPACE_NAMES: &[&str] = &[DEFAULT_THEME_DIR, "preprocessor", "eipw"]; #[derive(Debug, Snafu)] pub enum RepoManifestError { - #[snafu(display("i/o error while accessing `{}`", path.to_string_lossy()))] + #[snafu( + context(name(RepoManifestIoSnafu)), + display("i/o error while accessing `{}`", path.to_string_lossy()) + )] Io { path: PathBuf, source: std::io::Error, backtrace: Backtrace, }, - #[snafu(display( - "unable to parse repo manifest `{}`", - manifest_path.to_string_lossy() - ))] + #[snafu( + context(name(RepoManifestParseSnafu)), + display( + "unable to parse repo manifest `{}`", + manifest_path.to_string_lossy() + ) + )] Parse { manifest_path: PathBuf, #[snafu(source(from(toml::de::Error, Box::new)))] @@ -226,7 +232,7 @@ impl RepoManifest { Some("must not be `.` or `..`") } else if key.contains('/') || key.contains('\\') { Some("must be a single safe path component") - } else if RESERVED_REPO_IDS.contains(&key) { + } else if RESERVED_WORKSPACE_NAMES.contains(&key) { Some("collides with a reserved workspace/platform directory name") } else { None @@ -312,7 +318,7 @@ impl LoadedRepoManifest { { Ok(None) } - Err(error) => Err(IoSnafu { + Err(error) => Err(RepoManifestIoSnafu { path: manifest_path, } .into_error(error)), @@ -321,20 +327,22 @@ impl LoadedRepoManifest { #[cfg(test)] pub fn from_path(path: &Path) -> Result { - let manifest_path = path.canonicalize().with_context(|_| IoSnafu { + let manifest_path = path.canonicalize().with_context(|_| RepoManifestIoSnafu { path: path.to_path_buf(), })?; - let contents = std::fs::read_to_string(&manifest_path).with_context(|_| IoSnafu { - path: manifest_path.clone(), - })?; + let contents = + std::fs::read_to_string(&manifest_path).with_context(|_| RepoManifestIoSnafu { + path: manifest_path.clone(), + })?; Self::from_contents(manifest_path, &contents) } fn from_contents(manifest_path: PathBuf, contents: &str) -> Result { - let manifest = - toml::from_str::(contents).with_context(|_| ParseSnafu { + let manifest = toml::from_str::(contents).with_context(|_| { + RepoManifestParseSnafu { manifest_path: manifest_path.clone(), - })?; + } + })?; let manifest = RepoManifest::from_raw(manifest, &manifest_path)?; Ok(Self { diff --git a/src/identity.rs b/src/identity.rs index dc470ab..6db5d08 100644 --- a/src/identity.rs +++ b/src/identity.rs @@ -86,7 +86,7 @@ impl ActiveRepoIdentity { let manifest = manifest.manifest(); Ok(git::RepositoryUse { title: manifest.repo_id.clone(), - location: manifest.active_endpoint(staging), + location: manifest.active_endpoint(staging).clone(), other_repos: manifest.sibling_repositories(staging), }) } diff --git a/src/main.rs b/src/main.rs index 7d092a3..cb6de2c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -77,8 +77,13 @@ fn run() -> Result<(), Whatever> { return Ok(()); } - if let Operation::Init { path, template } = &args.operation { - init_workspace(&args, path.clone(), *template)?; + if let Operation::Init { + path, + template, + platform_dev, + } = &args.operation + { + init_workspace(&args, path.clone(), *template, *platform_dev)?; return Ok(()); } diff --git a/src/workspace.rs b/src/workspace.rs index 637d461..6892655 100644 --- a/src/workspace.rs +++ b/src/workspace.rs @@ -27,13 +27,10 @@ use crate::{ const WORKSPACE_THEME_URL: &str = "https://github.com/eips-wg/theme.git"; const PROPOSAL_TEMPLATE_URL: &str = "https://github.com/eips-wg/template.git"; +const PLATFORM_PREPROCESSOR_URL: &str = "https://github.com/eips-wg/preprocessor.git"; +const PLATFORM_EIPW_URL: &str = "https://github.com/ethereum/eipw.git"; const WORKSPACE_DOC_FILE: &str = "WORKSPACE.md"; -struct WorkspaceInitRepositories<'a> { - theme: &'a Url, - template: &'a Url, -} - #[derive(Debug, Clone, Copy)] enum DoctorStatus { Ok, @@ -47,6 +44,13 @@ struct DoctorReport { failures: usize, } +struct WorkspaceInitRepositories<'a> { + theme: &'a Url, + template: &'a Url, + preprocessor: &'a Url, + eipw: &'a Url, +} + impl fmt::Display for DoctorStatus { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let label = match self { @@ -228,6 +232,74 @@ fn check_default_windows_build_eips_path(report: &mut DoctorReport) { #[cfg(not(windows))] fn check_default_windows_build_eips_path(_report: &mut DoctorReport) {} +#[cfg(not(windows))] +fn check_optional_download_tool(report: &mut DoctorReport) { + let curl = command_path("curl"); + let wget = command_path("wget"); + + record_optional_download_tool(report, curl.as_deref(), wget.as_deref()); +} + +#[cfg(not(windows))] +fn record_optional_download_tool( + report: &mut DoctorReport, + curl: Option<&Path>, + wget: Option<&Path>, +) { + match (curl, wget) { + (Some(path), _) => report.record( + DoctorStatus::Ok, + format!( + "found front-door download helper `curl` at `{}`", + path.to_string_lossy() + ), + ), + (None, Some(path)) => report.record( + DoctorStatus::Ok, + format!( + "found front-door download helper `wget` at `{}`", + path.to_string_lossy() + ), + ), + (None, None) => report.record( + DoctorStatus::Warn, + "missing both `curl` and `wget`; `scripts/dev-setup` will not be able to download a release binary", + ), + } +} + +#[cfg(not(windows))] +fn check_front_door_archive_tool(report: &mut DoctorReport) { + let tar = command_path("tar"); + record_front_door_archive_tool(report, tar.as_deref()); +} + +#[cfg(not(windows))] +fn record_front_door_archive_tool(report: &mut DoctorReport, tar: Option<&Path>) { + match tar { + Some(path) => report.record( + DoctorStatus::Ok, + format!( + "found front-door archive tool `tar` at `{}`", + path.to_string_lossy() + ), + ), + None => report.record( + DoctorStatus::Warn, + "missing `tar`; `scripts/dev-setup` will not be able to unpack the release binary", + ), + } +} + +#[cfg(not(windows))] +fn check_front_door_setup_tools(report: &mut DoctorReport) { + check_optional_download_tool(report); + check_front_door_archive_tool(report); +} + +#[cfg(windows)] +fn check_front_door_setup_tools(_report: &mut DoctorReport) {} + fn collect_doctor_report(args: &Args, check_tools: bool) -> Result { let context = load_workspace_command_context(args)?; let mut report = DoctorReport::default(); @@ -397,6 +469,7 @@ fn collect_doctor_report(args: &Args, check_tools: bool) -> Result Result<(), Whatever> { let theme_repository = Url::parse(WORKSPACE_THEME_URL) .whatever_context("invalid workspace theme repository URL")?; let template_repository = Url::parse(PROPOSAL_TEMPLATE_URL) .whatever_context("invalid proposal template repository URL")?; + let preprocessor_repository = Url::parse(PLATFORM_PREPROCESSOR_URL) + .whatever_context("invalid platform preprocessor repository URL")?; + let eipw_repository = + Url::parse(PLATFORM_EIPW_URL).whatever_context("invalid platform eipw repository URL")?; let repositories = WorkspaceInitRepositories { theme: &theme_repository, template: &template_repository, + preprocessor: &preprocessor_repository, + eipw: &eipw_repository, }; - init_workspace_with_repositories(args, workspace_root, include_template, &repositories) + init_workspace_with_repositories( + args, + workspace_root, + include_template, + platform_dev, + &repositories, + ) } fn init_workspace_with_repositories( args: &Args, workspace_root: PathBuf, include_template: bool, + platform_dev: bool, repositories: &WorkspaceInitRepositories<'_>, ) -> Result<(), Whatever> { let root_path = root(args)?; @@ -477,6 +564,16 @@ fn init_workspace_with_repositories( .whatever_context("unable to clone workspace template repo")?; } + if platform_dev { + git::clone_missing_repo( + repositories.preprocessor.as_str(), + &workspace_root.join("preprocessor"), + ) + .whatever_context("unable to clone workspace preprocessor repo")?; + git::clone_missing_repo(repositories.eipw.as_str(), &workspace_root.join("eipw")) + .whatever_context("unable to clone workspace eipw repo")?; + } + std::fs::create_dir_all(workspace_root.join(config::DEFAULT_BUILD_ROOT_BASE)) .whatever_context("unable to create local build root")?; @@ -630,18 +727,27 @@ base_url = "https://staging.example.test/{sibling_id}/" manifest } - fn write_manifest_repo( + fn write_repo_manifest_file( path: &Path, repo_id: &str, upstream: &Url, siblings: &[(&str, Url)], - ) -> Repository { - let repo = init_repo(path, &[("content/0001.md", "# Proposal\n")]); + ) { write_file( path, config::REPO_MANIFEST_FILE, &repo_manifest_text(repo_id, upstream, siblings), ); + } + + fn write_manifest_repo( + path: &Path, + repo_id: &str, + upstream: &Url, + siblings: &[(&str, Url)], + ) -> Repository { + let repo = init_repo(path, &[("content/0001.md", "# Proposal\n")]); + write_repo_manifest_file(path, repo_id, upstream, siblings); commit_all(&repo, "add repo manifest"); repo } @@ -652,40 +758,41 @@ base_url = "https://staging.example.test/{sibling_id}/" file_url(&path) } - fn workspace_init_test_repository_urls(remotes_root: &Path) -> (Url, Url) { + fn workspace_init_test_repository_urls(remotes_root: &Path) -> (Url, Url, Url, Url) { ( init_workspace_source_repo(remotes_root, "theme"), init_workspace_source_repo(remotes_root, "template"), + init_workspace_source_repo(remotes_root, "preprocessor"), + init_workspace_source_repo(remotes_root, "eipw"), ) } - fn assert_workspace_init_and_doctor_for_siblings(sibling_ids: &[&str]) { + fn run_workspace_init_for_docs( + existing_doc: Option<&str>, + existing_config: Option<&str>, + ) -> (TempDir, std::path::PathBuf) { let temp = TempDir::new().unwrap(); let workspace_root = temp.path().join("workspace"); let remotes_root = temp.path().join("remotes"); - let (theme_url, template_url) = workspace_init_test_repository_urls(&remotes_root); + let (theme_url, template_url, preprocessor_url, eipw_url) = + workspace_init_test_repository_urls(&remotes_root); let repositories = WorkspaceInitRepositories { theme: &theme_url, template: &template_url, + preprocessor: &preprocessor_url, + eipw: &eipw_url, }; - - let sibling_repositories = sibling_ids - .iter() - .map(|sibling_id| { - let sibling_path = remotes_root.join(sibling_id); - let sibling_url = file_url(&sibling_path); - write_manifest_repo(&sibling_path, sibling_id, &sibling_url, &[]); - ((*sibling_id).to_owned(), sibling_url) - }) - .collect::>(); - let sibling_manifest_entries = sibling_repositories - .iter() - .map(|(repo_id, url)| (repo_id.as_str(), url.clone())) - .collect::>(); - let active_path = workspace_root.join("Core"); let active_url = file_url(&active_path); - write_manifest_repo(&active_path, "Core", &active_url, &sibling_manifest_entries); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + + if let Some(contents) = existing_doc { + write_file(&workspace_root, WORKSPACE_DOC_FILE, contents); + } + if let Some(contents) = existing_config { + write_file(&workspace_root, config::LOCAL_CONFIG_FILE, contents); + } + let init_args = parse_args(&[ "build-eips", "-C", @@ -694,67 +801,79 @@ base_url = "https://staging.example.test/{sibling_id}/" workspace_root.to_str().unwrap(), ]); - init_workspace_with_repositories(&init_args, workspace_root.clone(), false, &repositories) - .unwrap(); - - assert!(workspace_root.join(config::LOCAL_CONFIG_FILE).is_file()); - assert!(Repository::open(workspace_root.join(config::DEFAULT_THEME_DIR)).is_ok()); - for sibling_id in sibling_ids { - assert!(Repository::open(workspace_root.join(sibling_id)).is_ok()); - } + init_workspace_with_repositories( + &init_args, + workspace_root.clone(), + false, + false, + &repositories, + ) + .unwrap(); - let doctor_args = - parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "doctor"]); - let report = collect_doctor_report(&doctor_args, false).unwrap(); + (temp, workspace_root) + } - assert_eq!(report.failures, 0); - assert_eq!(report.warnings, 0); + #[test] + fn workspace_theme_url_is_bootstrap_metadata() { + assert_eq!( + Url::parse(WORKSPACE_THEME_URL).unwrap().as_str(), + "https://github.com/eips-wg/theme.git" + ); } + #[cfg(not(windows))] #[test] - fn init_command_parses_with_optional_template_flag() { - let plain = parse_args(&["build-eips", "init", "/tmp/workspace"]); - let template = parse_args(&["build-eips", "init", "/tmp/workspace", "--template"]); - - assert!(matches!( - plain.operation, - Operation::Init { - template: false, - .. - } - )); - assert!(matches!( - template.operation, - Operation::Init { template: true, .. } - )); + fn front_door_setup_tool_records_posix_helper_warnings() { + let mut report = super::DoctorReport::default(); + + super::record_optional_download_tool(&mut report, None, None); + super::record_front_door_archive_tool(&mut report, None); + + assert_eq!(report.warnings, 2); + assert_eq!(report.failures, 0); } + #[cfg(not(windows))] #[test] - fn doctor_command_parses() { - let args = parse_args(&["build-eips", "doctor"]); + fn front_door_setup_tool_accepts_posix_helpers() { + let mut report = super::DoctorReport::default(); + let tool_path = Path::new("/usr/bin/tool"); - assert!(matches!(args.operation, Operation::Doctor)); + super::record_optional_download_tool(&mut report, Some(tool_path), None); + super::record_front_door_archive_tool(&mut report, Some(tool_path)); + + assert_eq!(report.warnings, 0); + assert_eq!(report.failures, 0); } + #[cfg(windows)] #[test] - fn workspace_theme_url_is_bootstrap_metadata() { - assert_eq!( - Url::parse(WORKSPACE_THEME_URL).unwrap().as_str(), - "https://github.com/eips-wg/theme.git" - ); + fn front_door_setup_tools_skip_posix_helpers_on_windows() { + let mut report = super::DoctorReport::default(); + + super::check_front_door_setup_tools(&mut report); + + assert_eq!(report.warnings, 0); + assert_eq!(report.failures, 0); } #[test] - fn workspace_doc_text_mentions_base_workspace_content() { + fn workspace_doc_text_mentions_setup_reference_content() { let text = workspace_doc_text(); for expected in [ ".build-eips.toml", ".local-build", "build-eips init", + "build-eips doctor", "build-eips build", - "build-eips check", "build-eips serve", + "--platform-dev", + "preprocessor/", + "eipw/", + "[render]", + "only = [", + "--only", ] { assert!( text.contains(expected), @@ -766,88 +885,196 @@ base_url = "https://staging.example.test/{sibling_id}/" } #[test] - fn workspace_init_clones_required_repos_and_writes_config_and_doc() { + fn workspace_init_writes_workspace_doc() { + let (_temp, workspace_root) = run_workspace_init_for_docs(None, None); + + let doc = std::fs::read_to_string(workspace_root.join(WORKSPACE_DOC_FILE)).unwrap(); + assert_eq!(doc, workspace_doc_text()); + } + + #[test] + fn workspace_init_overwrites_existing_workspace_doc() { + let existing_doc = "Old workspace docs\n"; + let (_temp, workspace_root) = run_workspace_init_for_docs(Some(existing_doc), None); + + let doc = std::fs::read_to_string(workspace_root.join(WORKSPACE_DOC_FILE)).unwrap(); + assert_ne!(doc, existing_doc); + assert_eq!(doc, workspace_doc_text()); + } + + #[test] + fn workspace_init_leaves_existing_config_without_render_unchanged() { + let existing_config = "[server]\nhost = \"127.0.0.1\"\nport = 1111\n"; + let (_temp, workspace_root) = run_workspace_init_for_docs(None, Some(existing_config)); + + assert_eq!( + std::fs::read_to_string(workspace_root.join(config::LOCAL_CONFIG_FILE)).unwrap(), + existing_config + ); + } + + fn assert_workspace_init_optional_repos( + workspace_root: &Path, + expect_template: bool, + expect_platform_dev: bool, + ) { + assert!(Repository::open(workspace_root.join(config::DEFAULT_THEME_DIR)).is_ok()); + assert_eq!( + Repository::open(workspace_root.join("template")).is_ok(), + expect_template + ); + assert_eq!( + Repository::open(workspace_root.join("preprocessor")).is_ok(), + expect_platform_dev + ); + assert_eq!( + Repository::open(workspace_root.join("eipw")).is_ok(), + expect_platform_dev + ); + } + + fn assert_workspace_init_and_doctor_for_siblings(sibling_ids: &[&str]) { let temp = TempDir::new().unwrap(); let workspace_root = temp.path().join("workspace"); let remotes_root = temp.path().join("remotes"); - let (theme_url, template_url) = workspace_init_test_repository_urls(&remotes_root); + let (theme_url, template_url, preprocessor_url, eipw_url) = + workspace_init_test_repository_urls(&remotes_root); let repositories = WorkspaceInitRepositories { theme: &theme_url, template: &template_url, + preprocessor: &preprocessor_url, + eipw: &eipw_url, }; - let sibling_path = remotes_root.join("ERCs"); - let sibling_url = file_url(&sibling_path); - write_manifest_repo(&sibling_path, "ERCs", &sibling_url, &[]); - - let active_path = workspace_root.join("EIPs"); + let sibling_repositories = sibling_ids + .iter() + .map(|sibling_id| { + let sibling_id = *sibling_id; + let sibling_path = remotes_root.join(sibling_id); + let sibling_url = file_url(&sibling_path); + write_manifest_repo(&sibling_path, sibling_id, &sibling_url, &[]); + (sibling_id.to_owned(), sibling_url) + }) + .collect::>(); + let sibling_manifest_entries = sibling_repositories + .iter() + .map(|(repo_id, url)| (repo_id.as_str(), url.clone())) + .collect::>(); + let active_path = workspace_root.join("Core"); let active_url = file_url(&active_path); - write_manifest_repo(&active_path, "EIPs", &active_url, &[("ERCs", sibling_url)]); - + write_manifest_repo(&active_path, "Core", &active_url, &sibling_manifest_entries); let init_args = parse_args(&[ "build-eips", "-C", active_path.to_str().unwrap(), "init", workspace_root.to_str().unwrap(), - "--template", ]); - init_workspace_with_repositories(&init_args, workspace_root.clone(), true, &repositories) - .unwrap(); + init_workspace_with_repositories( + &init_args, + workspace_root.clone(), + false, + false, + &repositories, + ) + .unwrap(); - assert!(Repository::open(workspace_root.join(config::DEFAULT_THEME_DIR)).is_ok()); - assert!(Repository::open(workspace_root.join("ERCs")).is_ok()); - assert!(Repository::open(workspace_root.join("template")).is_ok()); - assert!(workspace_root - .join(config::DEFAULT_BUILD_ROOT_BASE) - .is_dir()); - assert!( - LoadedWorkspaceConfig::from_path(&workspace_root.join(config::LOCAL_CONFIG_FILE)) - .is_ok() - ); - assert_eq!( - std::fs::read_to_string(workspace_root.join(WORKSPACE_DOC_FILE)).unwrap(), - workspace_doc_text() - ); - } + assert!(workspace_root.join(config::LOCAL_CONFIG_FILE).is_file()); + assert_workspace_init_optional_repos(&workspace_root, false, false); + for sibling_id in sibling_ids { + assert!(Repository::open(workspace_root.join(sibling_id)).is_ok()); + } - #[test] - fn workspace_init_and_doctor_cover_zero_one_and_many_siblings() { - assert_workspace_init_and_doctor_for_siblings(&[]); - assert_workspace_init_and_doctor_for_siblings(&["ERCs"]); - assert_workspace_init_and_doctor_for_siblings(&["EIPs", "ERCs"]); + let doctor_args = + parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "doctor"]); + let report = collect_doctor_report(&doctor_args, false).unwrap(); + + assert_eq!(report.failures, 0); } - #[test] - fn workspace_init_leaves_existing_config_unchanged() { + fn assert_workspace_init_optional_clone_behavior( + flags: &[&str], + expect_template: bool, + expect_platform_dev: bool, + ) { let temp = TempDir::new().unwrap(); let workspace_root = temp.path().join("workspace"); let remotes_root = temp.path().join("remotes"); - let (theme_url, template_url) = workspace_init_test_repository_urls(&remotes_root); + let (theme_url, template_url, preprocessor_url, eipw_url) = + workspace_init_test_repository_urls(&remotes_root); let repositories = WorkspaceInitRepositories { theme: &theme_url, template: &template_url, + preprocessor: &preprocessor_url, + eipw: &eipw_url, }; let active_path = workspace_root.join("Core"); let active_url = file_url(&active_path); write_manifest_repo(&active_path, "Core", &active_url, &[]); - let existing_config = "[server]\nhost = \"127.0.0.1\"\nport = 1111\n"; - write_file(&workspace_root, config::LOCAL_CONFIG_FILE, existing_config); - let init_args = parse_args(&[ + let active_path = active_path.to_string_lossy(); + let workspace_root_arg = workspace_root.to_string_lossy(); + let mut arguments = vec![ "build-eips", "-C", - active_path.to_str().unwrap(), + active_path.as_ref(), "init", - workspace_root.to_str().unwrap(), - ]); + workspace_root_arg.as_ref(), + ]; + arguments.extend_from_slice(flags); + let init_args = parse_args(&arguments); + let Operation::Init { + path: workspace_root_path, + template, + platform_dev, + } = init_args.operation.clone() + else { + panic!("expected init command"); + }; - init_workspace_with_repositories(&init_args, workspace_root.clone(), false, &repositories) - .unwrap(); + assert_eq!(template, expect_template); + assert_eq!(platform_dev, expect_platform_dev); - assert_eq!( - std::fs::read_to_string(workspace_root.join(config::LOCAL_CONFIG_FILE)).unwrap(), - existing_config + init_workspace_with_repositories( + &init_args, + workspace_root_path, + template, + platform_dev, + &repositories, + ) + .unwrap(); + + assert_workspace_init_optional_repos(&workspace_root, expect_template, expect_platform_dev); + } + + #[test] + fn workspace_init_and_doctor_cover_zero_one_and_many_siblings() { + assert_workspace_init_and_doctor_for_siblings(&[]); + assert_workspace_init_and_doctor_for_siblings(&["ERCs"]); + assert_workspace_init_and_doctor_for_siblings(&["EIPs", "ERCs"]); + } + + #[test] + fn default_workspace_init_clones_required_repos_only() { + assert_workspace_init_optional_clone_behavior(&[], false, false); + } + + #[test] + fn workspace_init_template_clones_template_only_as_optional_repo() { + assert_workspace_init_optional_clone_behavior(&["--template"], true, false); + } + + #[test] + fn workspace_init_platform_dev_clones_platform_repos_only_as_optional_repos() { + assert_workspace_init_optional_clone_behavior(&["--platform-dev"], false, true); + } + + #[test] + fn workspace_init_template_and_platform_dev_clone_all_optional_repos() { + assert_workspace_init_optional_clone_behavior( + &["--template", "--platform-dev"], + true, + true, ); } diff --git a/src/workspace_doc.md b/src/workspace_doc.md index eddce25..01b892f 100644 --- a/src/workspace_doc.md +++ b/src/workspace_doc.md @@ -1,10 +1,12 @@ # build-eips Workspace -This directory is a local multi-repo workspace for building and checking EIPs/ERCs with the shared theme and proposal sibling repos. +This directory is a local multi-repo workspace for building, serving, previewing, and validating EIPs/ERCs with the shared theme and proposal sibling repos. + +The workspace keeps the proposal repos, theme repo, generated build output, and local workspace settings in one predictable layout. Run commands from an active proposal repo such as `EIPs/` or `ERCs/`, or from this workspace root with `-C EIPs` or `-C ERCs`. ## Workspace Layout -After initialization, the minimal workspace should look like this: +After running a setup script, the minimal operational workspace should look like this: ```text EIPs-project/ @@ -16,9 +18,81 @@ EIPs-project/ └── theme/ ``` -Use `build-eips init ..` from an active proposal repo such as `EIPs/` or `ERCs/` to create missing sibling repos, clone `theme/`, create `.local-build/`, write `.build-eips.toml`, and generate this guide. +Optional setup flags can add more repos: + +```text +EIPs-project/ +├── template/ # --template +├── preprocessor/ # --platform-dev +└── eipw/ # --platform-dev +``` + +- `.build-eips.toml`: workspace settings. +- `WORKSPACE.md`: generated workspace guide. +- `.local-build/`: generated build output and materialized repositories. +- `EIPs/` and `ERCs/`: proposal source repositories. +- `theme/`: workspace-local Zola theme required by build, serve, and check commands. +- `template/`: optional proposal template repository. +- `preprocessor/`: optional local `build-eips` development checkout. +- `eipw/`: optional local `eipw` development checkout. + +If the optional repos are missing, rerun build-eips init with the needed flags. + +From an active proposal repo: + +```sh +build-eips init .. --template +build-eips init .. --platform-dev +build-eips init .. --template --platform-dev +``` + +From the workspace root: + +```sh +build-eips -C EIPs init . --template +build-eips -C EIPs init . --platform-dev +build-eips -C EIPs init . --template --platform-dev +``` + +## Requirements And Troubleshooting + +Local workspace commands require these tools on `PATH`: + +- Git +- `build-eips` +- Zola 0.22.1 + +Git must be installed separately. The setup scripts locate or install `build-eips` and Zola, add locally installed tool directories to `PATH` for the current shell session, and print guidance for making those `PATH` changes permanent. + +Run `build-eips doctor` after setup and whenever a command cannot find a repo, config file, theme, or required tool: + +```sh +build-eips doctor +``` + +From the workspace root, anchor the command through an active proposal repo: + +```sh +build-eips -C EIPs doctor +build-eips -C ERCs doctor +``` + +`build-eips doctor` checks: + +- required tools: Git, `build-eips`, and Zola 0.22.1 +- the active proposal repo manifest +- `.build-eips.toml` +- workspace-local sibling proposal repos +- workspace-local `theme/` +- optional setup helper tools used by setup scripts + +If a fresh shell cannot find `build-eips` or Zola, rerun the setup script or apply the permanent `PATH` guidance printed by the setup script. + +If a sibling repo, `theme/`, or optional platform repo is missing, rerun `build-eips init` with the needed flags. + +If `theme/` or `preprocessor/` setup cannot create the default `EIPs/` checkout, check that Git is installed and that the workspace `EIPs/` path does not already exist as a non-git directory. Set `ACTIVE_REPO_ROOT` when you want setup to use an existing ERCs or custom proposal repo checkout. -Pass `--template` when proposal template work also needs the optional `template/` repo. +If `build-eips doctor` reports that Zola is missing or too old, rerun the setup script to install the supported Zola version. ## Local Commands From 7b09725c15736b66ee145a05c389db86ba49214c Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Mon, 4 May 2026 03:45:56 -0400 Subject: [PATCH 17/20] Add final preprocessor integration Finalize the preprocessor integration docs and command help. Describe staging, production, and parity as clean local-active runtime modes that use remote sibling sources and selected environment metadata. Keep CLI help, workspace guide, and architecture source-policy wording aligned around the local active checkout source model. --- ARCHITECTURE.md | 184 ++++++++++ Cargo.lock | 26 +- src/README.md | 60 ++++ src/changed.rs | 52 +-- src/cli.rs | 123 +++---- src/config.rs | 144 ++++---- src/editorial.rs | 2 +- src/execution.rs | 375 +++++++++++--------- src/identity.rs | 13 +- src/layout.rs | 2 + src/main.rs | 8 +- src/print.rs | 2 +- src/tests.rs | 814 +++++++++++++++++++++++++++++++++++++++++++ src/workspace.rs | 9 +- src/workspace_doc.md | 158 ++++++++- src/zola.rs | 33 +- 16 files changed, 1588 insertions(+), 417 deletions(-) create mode 100644 ARCHITECTURE.md create mode 100644 src/README.md create mode 100644 src/tests.rs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..26e8a93 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,184 @@ +# Architecture Overview + +This document describes the `build-eips` system model: its system layers, repository responsibilities, trust boundaries, source and generated state, data flow, and the separation between validation, transformation, materialization, and rendering. + +`build-eips` coordinates independent proposal, theme, and tooling repositories into a prepared render tree. The source repositories remain authoritative for their own data, while generated state is isolated to the resolved build root and consumed by validation and rendering paths. + +Workspace setup, command usage, and Rust module ownership are documented in `README.md`, `src/workspace_doc.md`, and `src/README.md`, respectively. + +## System Layers + +The system is organized into layers with separate responsibilities: + +- source repositories: canonical proposal content, tracked topology metadata, theme source files, and tooling crates +- local workspace overlay: local operator state and local checkout paths that let independent repositories work as one system +- preprocessor orchestration: active repository identity, topology, source policy, execution resolution, and runtime handoff +- disposable prepared state: generated Git worktrees, mounted theme copies, and rendered output under the resolved build root +- render/output surface: the prepared tree and mounted theme consumed by Zola, plus static output served by preview + +Validation, transformation, materialization, and rendering are pipeline responsibilities. They are not separate system layers. + +## Repository Responsibilities + +Each repository owns one part of the system: + +- proposal repositories own canonical content and `.build-eips.repo.toml` repo manifests +- the theme owns render templates, runtime configuration, styles, assets, and syntax definitions +- the preprocessor owns orchestration, source resolution, materialization, preprocessing, and runtime handoff +- eipw owns editorial validation rules consumed through Cargo dependencies such as `eipw-lint`, `eipw-preamble`, and `eipw-snippets` +- the template repository owns proposal scaffolding + +## Source And Generated State + +The architecture separates authoritative state from generated state: + +- proposal `content/`, `.build-eips.repo.toml` repo manifests, and theme files are source +- `.build-eips.repo.toml` provides source-controlled topology; its `repo_id` is the stable key used for workspace directories, default build roots, and sibling references +- `.build-eips.toml` is local workspace state for local server, site, and render defaults +- prepared repositories and rendered output are generated under the resolved build root +- the normal workspace build root is `.local-build/` +- without workspace config, the fallback build root is `/build` +- mounted theme copies are generated runtime state +- canonical proposal content is not rewritten by runtime paths + +## Trust Boundaries + +Inputs enter the system with different trust and ownership properties: + +- tracked `.build-eips.repo.toml` manifests are source-controlled topology +- `.build-eips.toml` is local operator state +- CLI flags are per-run user intent +- remote Git refs are fetched external state used for source materialization and merge-base comparison +- proposal markdown is source content that must be transformed and validated before rendering +- theme files are local source input and may include tracked or staged local theme changes during materialization +- prepared repositories, mounted themes, and rendered output are generated state + +## Identity Resolution + +Identity resolution determines the active proposal repository before topology is derived. + +The inputs are: + +- command anchor: current working directory or `-C` +- active proposal repository root +- tracked `.build-eips.repo.toml` manifest, when present +- built-in EIPs/ERCs identifying-commit lookup, when no manifest is present + +The resolution order is manifest-backed identity first, then built-in EIPs/ERCs compatibility identity for checkouts without tracked manifests. + +## Topology Resolution + +Topology follows identity: + +- sibling repositories come from the manifest or built-in compatibility metadata +- remote endpoints come from the selected repository identity +- local workspace paths come from workspace discovery and local config +- build roots come from local workspace defaults or per-run overrides + +## Source Policy + +Source policy determines which repository state is materialized or consumed after identity and topology are resolved. + +| Policy | Active repository source | Dirty active edits | Sibling source | Environment metadata | +| --- | --- | --- | --- | --- | +| Local dirty site validation/rendering | local active checkout | tracked edits included | workspace-local siblings | local site defaults with selected upstream metadata | +| Clean local site validation/rendering | local active checkout | rejected before materialization | workspace-local siblings | local site defaults with selected upstream metadata | +| Remote-sibling site validation/rendering | local active checkout | follows the selected local dirty/clean policy | selected remote sibling endpoints | selected environment metadata | +| Explicit staging, production, and parity | local active checkout | rejected before materialization | selected remote sibling endpoints | selected environment metadata | +| Changed-file comparison and upstream editorial target selection | local active checkout through prepared Git state | clean comparison | not render topology | selected upstream merge base | +| Editorial lint with explicit, batch, or working-tree selectors | selected files in the active checkout | consumed directly by selector | not render topology | lint configuration from the local theme | + +Staging, production, and parity paths do not replace the active render source with a remote active checkout. They require the local active checkout to be clean, use the selected environment metadata for upstream comparison and rendered base URLs, and resolve sibling proposal repositories from remote endpoints. + +Editorial check composes editorial lint with the site-level check path. `--build-root` changes the resolved generated-state location, and `--only` narrows proposal rendering only for local dirty rendering paths; default render selection can also come from `[render].only` in `.build-eips.toml`. + +## Overlay Model + +The overlay model describes the render-time proposal surface: + +- independent proposal repositories are not merged as source repositories +- sibling proposal content is overlaid into one prepared render tree +- the theme is materialized into the prepared tree at `themes/eips-theme` +- the overlay exists only in generated state +- the prepared render tree is the only proposal surface seen by Zola + +Git mechanics belong to the materialization boundary. The overlay model is the architectural result of that materialization. + +## Data Flow + +The system flow has an execution-resolution step, a validation branch, and a prepared-state branch: + +```text +active repo + sibling repos + theme + workspace config + | + v +execution resolution + |-------------------------------| + v v +editorial/changed-file validation prepared build repo + | + v + markdown/proposal transformation + | + v + theme mount + Zola config + | + v + Zola validation or rendering + | + v + static output / preview surface +``` + +Some validation work happens before or beside materialization, while Zola validation and rendering happen against prepared state. `changed` and `editorial --against-upstream` still use prepared Git state to compute merge-base differences. They bypass markdown transformation and Zola; they do not bypass Git source preparation entirely. + +## Materialization Boundary + +Materialization is the boundary between source repositories and runtime state: + +- runtime paths do not render directly from source checkouts +- the preprocessor creates a disposable prepared repository +- active and sibling proposal content are merged into that prepared tree +- tracked dirty active-repo state is included only when the selected source mode allows it +- dirty active-repo materialization ignores untracked active-repo files +- clean source materialization rejects dirty or untracked active-repo state before proceeding +- theme materialization includes tracked theme files plus tracked or staged theme working-tree changes, independent of active-repo dirty mode +- canonical source content is not rewritten + +## Pipeline Responsibilities + +The pipeline separates kinds of work: + +- identity and topology resolution: decide what the runtime path operates on +- materialization: Git clone/fetch/dirty/sibling/theme staging +- transformation: markdown preprocessing, front matter, links, `requires`, and citations +- validation: editorial lint, changed-file selection, and Zola check behavior +- rendering: Zola build and serve + +Preview is the exception: it serves prior rendered output and bypasses materialization, transformation, and Zola. + +Editorial validation is a peer of rendering. It consumes the same identity and source model, and the editorial check path composes with the runtime check path when site-level validation is required. + +## Invariants + +These rules keep the system coherent: + +- canonical proposal content is not rewritten by runtime paths +- generated output lives under the resolved build root +- Zola sees one prepared repo and one mounted theme +- sibling content enters only through materialization +- workspace-local state is not canonical proposal content + +## Failure Boundaries + +The system halts at boundaries that would otherwise make prepared state ambiguous or invalid: + +- prepared-state mutation paths use `/.lock`; preview bypasses the lock because it serves existing output, and clean unlocks before removing the build root +- missing required local theme halts Zola-backed and editorial lint paths +- unresolved active repository identity halts the run +- sibling content path conflicts halt materialization +- invalid discovered workspace config halts execution-resolved runtime paths + +## Deferred Or Compatibility Paths + +Tracked `.build-eips.repo.toml` manifests are the normal topology description. Built-in EIPs/ERCs identity handles checkouts without tracked manifests. The template manifest exists, but supported raw-template bootstrap/developer workflow is not first-class yet. diff --git a/Cargo.lock b/Cargo.lock index 2c52e21..ee2306e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -199,9 +199,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.1" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] name = "block-buffer" @@ -968,7 +968,7 @@ version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.10.0", "libc", "libgit2-sys", "log", @@ -1410,11 +1410,11 @@ dependencies = [ [[package]] name = "kqueue-sys" -version = "1.1.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7b65860415f949f23fa882e669f2dbd4a0f0eeb1acdd56790b30494afd7da2f" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" dependencies = [ - "bitflags 2.11.1", + "bitflags 1.3.2", "libc", ] @@ -1450,7 +1450,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.10.0", "libc", "redox_syscall 0.7.4", ] @@ -1614,7 +1614,7 @@ version = "6.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.10.0", "crossbeam-channel", "filetime", "fsevent-sys", @@ -1956,7 +1956,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.10.0", "getopts", "memchr", "pulldown-cmark-escape", @@ -2030,7 +2030,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.10.0", ] [[package]] @@ -2039,7 +2039,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.10.0", ] [[package]] @@ -2134,7 +2134,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.10.0", "errno", "libc", "linux-raw-sys", @@ -2212,7 +2212,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.10.0", "cssparser", "derive_more", "fxhash", diff --git a/src/README.md b/src/README.md new file mode 100644 index 0000000..9feb0b5 --- /dev/null +++ b/src/README.md @@ -0,0 +1,60 @@ +# Preprocessor Module And Test Ownership + +This file documents how the preprocessor crate is organized: which module owns each behavior, where tests should live, and how to keep module boundaries reviewable. It is for contributors and maintainers, not end-user documentation. + + +## Module Ownership + +- `main.rs` owns CLI entry, top-level dispatch, build locking, and runtime orchestration. +- `changed.rs` owns changed-file command execution, coordinating upstream diffing, proposal filtering, and changed-output formatting. +- `cli.rs` owns the clap command surface and command helper methods. +- `config.rs` owns built-in repository metadata, workspace config schema, repo manifest schema, parsing, defaults, and config discovery. +- `context.rs` owns command input path resolution and workspace command context. +- `editorial.rs` owns editorial selector resolution, active-repo target validation, prepared-source editorial lint orchestration, and editorial check runtime handoff. +- `execution.rs` owns source mode, environment, base URL, server binding, build path, workspace source, and runtime execution resolution. +- `find_root.rs` owns active proposal-repo root detection. +- `git.rs` owns git repository identification, clone/fetch/merge behavior, source materialization, and tracked path synchronization. +- `github.rs` owns GitHub annotation reporting support for lint output. +- `identity.rs` owns active repository identity selection from repo manifests or legacy metadata. +- `layout.rs` owns shared build layout names and path helpers. +- `lint.rs` owns eipw config loading, reporter setup, and invocation for caller-provided source lists. +- `markdown.rs` owns proposal markdown preprocessing. +- `pipeline.rs` owns prepared Zola runtime setup and build/check/serve steps. +- `preview.rs` owns static preview serving for already-built output. +- `print.rs` owns diagnostic print subcommands. +- `progress.rs` owns progress/log rendering helpers. +- `proposal.rs` owns proposal path classification, proposal-number parsing, and targeted-rendering selection policy. +- `serve.rs` owns dirty active-repo and local-theme serve synchronization. +- `workspace.rs` owns local workspace setup and diagnostics. +- `zola.rs` owns Zola discovery, theme mounting, and Zola command invocation. + +## Test Ownership + +New tests should generally live in the module that owns the behavior. Use `super::` from module-local tests where natural, or sibling module paths from the owning module. Module-local tests cover behavior in modules such as `cli.rs`, `config.rs`, `execution.rs`, `git.rs`, `markdown.rs`, `pipeline.rs`, `proposal.rs`, `serve.rs`, `workspace.rs`, and `zola.rs`; `src/tests.rs` intentionally holds the remaining cross-domain behavior tests. + +Use `src/tests.rs` for cross-domain behavior tests, especially tests covering: + +- command dispatch across modules +- CLI plus execution plus runtime behavior +- downstream CI invariance +- multi-repo or manifest-driven flows +- workspace plus execution plus editorial behavior +- source materialization behavior spanning git, execution, pipeline, or serve +- tests that would require exposing more internals just to move them + +Move tests to module-local `#[cfg(test)]` modules only when the behavior is owned by one module and the test remains clearer there. Examples include pure clap parsing in `cli.rs`, execution policy helpers in `execution.rs`, serve event filtering in `serve.rs`, workspace-local theme materialization in `pipeline.rs`, prepared-source editorial lint behavior in `editorial.rs`, workspace setup and diagnostics behavior in `workspace.rs`, active repo identity behavior in `identity.rs` when it does not require the full execution path, and proposal path or proposal-number behavior in `proposal.rs`. + +## Dependency Direction + +Imports should generally flow from orchestration modules toward domain and shared modules. `main.rs` should call into command/runtime modules, while lower-level modules should avoid depending on high-level orchestration. + +`execution.rs` is the runtime resolution layer between command/config inputs and runtime execution: it combines CLI flags, workspace config, active repo identity, source mode, build path, server binding, and base URL decisions before `main.rs` hands work to `changed.rs`, `pipeline.rs`, `serve.rs`, `preview.rs`, or editorial helpers. + +Examples of high-level orchestration modules include `main.rs`, `workspace.rs`, `pipeline.rs`, and `serve.rs`. Shared or lower-level modules include `cli.rs`, `layout.rs`, `proposal.rs`, and focused domain modules such as `git.rs`, `markdown.rs`, and `zola.rs`. + +Lower-level or shared modules should not import higher-level orchestration modules just to reuse behavior. Move shared behavior into the owning domain module instead. + + +## Visibility + +Do not make private helpers `pub` or `pub(crate)` just to move a test. If a test needs direct access to private module behavior, it probably belongs in that module's `#[cfg(test)]` block. Use `src/tests.rs` for cross-module behavior tests that exercise crate-visible paths. diff --git a/src/changed.rs b/src/changed.rs index cf53c46..07ad880 100644 --- a/src/changed.rs +++ b/src/changed.rs @@ -4,53 +4,16 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -//! Changed-file command execution. +//! Changed-file command helpers. -use std::{ - ffi::OsStr, - path::{Path, PathBuf}, -}; +use std::path::Path; use snafu::{ResultExt, Whatever}; -use crate::{cli::ChangedFormat, execution::ResolvedExecution, git, layout::REPO_DIR}; - -pub(crate) fn is_proposal_path(mut p: PathBuf) -> bool { - // Only lint `content/00001.md` and `content/00001/index.md` files. - - // content/00000.md | content/00000/index.md - // ^^^^^^^^ | ^^^^^^^^ - match p.file_name() { - Some(n) if n == "index.md" => { - p.pop(); - } - Some(_) if p.extension().map(|x| x == "md").unwrap_or(false) => { - p.set_extension(""); - } - None | Some(_) => return false, - } - - // content/00000 - // ^^^^^ - match p.file_name().and_then(OsStr::to_str) { - None => return false, - Some(f) if f.parse::().is_err() => return false, - Some(_) => { - p.pop(); - } - } - - // content - // ^^^^^^^ - match p.file_name() { - Some(f) if f == "content" => { - p.pop(); - } - _ => return false, - } - - p == OsStr::new("") -} +use crate::{ + cli::ChangedFormat, execution::ResolvedExecution, git, layout::REPO_DIR, + proposal::is_proposal_path, +}; pub(crate) fn run( resolved: &ResolvedExecution, @@ -76,10 +39,11 @@ pub(crate) fn run( .changed_files() .whatever_context("unable to list changed files")? .into_iter() - .filter(|p| all || is_proposal_path(p.into())) + .filter(|p| all || is_proposal_path(p)) .map(|p| repo_path.join(p)) .collect(); format.print(&changed_files, &repo_path); + Ok(()) } diff --git a/src/cli.rs b/src/cli.rs index 487ef12..6488667 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -21,11 +21,11 @@ pub(crate) struct Args { #[clap(short = 'C')] pub(crate) root: Option, - /// Force the staging repositories and base URLs + /// Use staging sibling repositories and environment metadata #[clap(long)] pub(crate) staging: bool, - /// Force the production repositories and base URLs + /// Use production sibling repositories and environment metadata #[clap(long)] pub(crate) production: bool, @@ -155,7 +155,7 @@ pub(crate) enum Operation { /// Check workspace layout, local repos, and required tools Doctor, - /// Run build, serve, or check with staging remote proposal sources + /// Run build, serve, or check with clean staging settings and remote siblings Parity { #[command(subcommand)] command: ProfiledOperation, @@ -275,7 +275,7 @@ impl Operation { pub(crate) fn clean_cli_args(&self) -> CleanCliArgs { match self { - Self::Build { clean, .. } | Self::Serve { clean, .. } | Self::Check { clean, .. } => { + Self::Build { clean, .. } | Self::Serve { clean, .. } | Self::Check { clean } => { clean.clone() } Self::Print { .. } @@ -353,7 +353,7 @@ impl ProfiledOperation { fn base_url_cli_args(&self) -> BaseUrlCliArgs { match self { - Self::Build { base_url, .. } | Self::Serve { base_url, .. } => base_url.clone(), + Self::Build { base_url } | Self::Serve { base_url, .. } => base_url.clone(), Self::Check => BaseUrlCliArgs::default(), } } @@ -455,7 +455,7 @@ mod tests { } #[test] - fn only_flag_parses_one_or_more_proposal_numbers_on_build_and_serve() { + fn only_flag_parses_one_or_more_proposal_numbers_on_build() { let one = parse_args(&["build-eips", "build", "--only", "00555"]); let many = parse_args(&["build-eips", "build", "--only", "555", "678", "897"]); let serve = parse_args(&["build-eips", "serve", "--only", "555", "678"]); @@ -494,7 +494,7 @@ mod tests { } #[test] - fn only_flag_rejects_invalid_selectors_and_non_targeted_commands() { + fn only_flag_rejects_invalid_selectors_and_non_build_commands() { for selector in [ "+555", "0", @@ -515,16 +515,63 @@ mod tests { assert!(Args::try_parse_from(["build-eips", "parity", "serve", "--only", "555"]).is_err()); } + #[test] + fn server_flags_parse_on_serve_and_preview_forms() { + let cases: &[(&[&str], bool)] = &[ + ( + &["build-eips", "serve", "--host", "0.0.0.0", "--port", "8080"], + true, + ), + ( + &[ + "build-eips", + "preview", + "--host", + "0.0.0.0", + "--port", + "8080", + ], + false, + ), + ( + &[ + "build-eips", + "parity", + "serve", + "--host", + "0.0.0.0", + "--port", + "8080", + ], + true, + ), + ]; + + for (arguments, expect_serve) in cases { + let args = parse_args(arguments); + let runtime_operation = args.operation.runtime_operation().unwrap(); + match runtime_operation { + RuntimeOperation::Serve if *expect_serve => {} + RuntimeOperation::Preview if !*expect_serve => {} + other => panic!("unexpected runtime operation: {other:?}"), + } + let server = args.operation.server_cli_args(); + + assert_eq!(server.host.as_deref(), Some("0.0.0.0")); + assert_eq!(server.port, Some(8080)); + } + } + #[test] fn base_url_flags_parse_on_build_and_serve_forms() { - let cases: &[(&[&str], &str)] = &[ + let cases: &[(&[&str], RuntimeOperation)] = &[ ( &["build-eips", "build", "--base-url", "http://localhost:4000"], - "build", + RuntimeOperation::Build, ), ( &["build-eips", "serve", "--base-url", "http://localhost:4000"], - "serve", + RuntimeOperation::Serve, ), ( &[ @@ -534,7 +581,7 @@ mod tests { "--base-url", "http://localhost:4000", ], - "build", + RuntimeOperation::Build, ), ( &[ @@ -544,7 +591,7 @@ mod tests { "--base-url", "http://localhost:4000", ], - "serve", + RuntimeOperation::Serve, ), ]; @@ -554,9 +601,10 @@ mod tests { assert!(matches!( ( args.operation.runtime_operation().unwrap(), - *expected_runtime_operation + (*expected_runtime_operation).clone() ), - (RuntimeOperation::Build, "build") | (RuntimeOperation::Serve, "serve") + (RuntimeOperation::Build, RuntimeOperation::Build) + | (RuntimeOperation::Serve, RuntimeOperation::Serve) )); assert_eq!( args.operation @@ -593,53 +641,6 @@ mod tests { } } - #[test] - fn server_flags_parse_on_serve_and_preview_forms() { - let cases: &[(&[&str], bool)] = &[ - ( - &["build-eips", "serve", "--host", "0.0.0.0", "--port", "8080"], - true, - ), - ( - &[ - "build-eips", - "preview", - "--host", - "0.0.0.0", - "--port", - "8080", - ], - false, - ), - ( - &[ - "build-eips", - "parity", - "serve", - "--host", - "0.0.0.0", - "--port", - "8080", - ], - true, - ), - ]; - - for (arguments, expect_serve) in cases { - let args = parse_args(arguments); - let runtime_operation = args.operation.runtime_operation().unwrap(); - match runtime_operation { - RuntimeOperation::Serve if *expect_serve => {} - RuntimeOperation::Preview if !*expect_serve => {} - other => panic!("unexpected runtime operation: {other:?}"), - } - let server = args.operation.server_cli_args(); - - assert_eq!(server.host.as_deref(), Some("0.0.0.0")); - assert_eq!(server.port, Some(8080)); - } - } - #[test] fn removed_command_surface_is_rejected() { for arguments in [ diff --git a/src/config.rs b/src/config.rs index 60f5ada..24cd891 100644 --- a/src/config.rs +++ b/src/config.rs @@ -360,15 +360,6 @@ impl LoadedRepoManifest { } } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Theme { - /// Where to fetch the theme from. - pub repository: Url, - - /// Specific revision to checkout from the theme repository. - pub commit: String, -} - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct LegacyLocation { /// Git repository to fetch proposals from. @@ -399,7 +390,6 @@ pub struct LegacyLocations(pub HashMap); #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Config { - pub theme: Theme, pub locations: LegacyLocations, } @@ -426,12 +416,6 @@ impl Config { ); Self { - theme: Theme { - repository: "https://github.com/ethereum/eips-theme.git" - .try_into() - .unwrap(), - commit: "0ddac35da36d311a8401c6cfb79c9991f78b647d".into(), - }, locations: LegacyLocations(locations), } } @@ -458,10 +442,6 @@ impl Config { ); Self { - theme: Theme { - repository: "https://github.com/eips-wg/theme.git".try_into().unwrap(), - commit: "0ddac35da36d311a8401c6cfb79c9991f78b647d".into(), - }, locations: LegacyLocations(locations), } } @@ -498,7 +478,7 @@ impl WorkspaceConfig { #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct RenderSettings { - /// Proposal numbers to render for applicable local build commands. + /// Proposal numbers to render for applicable local build and serve commands. #[serde(default)] pub only: Vec, } @@ -652,10 +632,10 @@ impl LoadedWorkspaceConfig { &self.workspace_root } - pub fn workspace_build_root(&self, repo_id: &str) -> PathBuf { + pub fn workspace_build_root(&self, repo_name: &str) -> PathBuf { self.workspace_root .join(DEFAULT_BUILD_ROOT_BASE) - .join(repo_id) + .join(repo_name) } pub fn server_settings(&self) -> &ServerSettings { @@ -674,8 +654,8 @@ impl LoadedWorkspaceConfig { self.workspace_root.join(DEFAULT_THEME_DIR) } - pub fn local_repo_path(&self, repo_id: &str) -> PathBuf { - self.workspace_root.join(repo_id) + pub fn local_repo_path(&self, repo_name: &str) -> PathBuf { + self.workspace_root.join(repo_name) } } @@ -719,11 +699,11 @@ mod tests { }; use crate::proposal::ProposalNumber; - struct TestRepo { + struct TestWorkspace { tempdir: TempDir, } - impl TestRepo { + impl TestWorkspace { fn new() -> Self { Self { tempdir: TempDir::new().unwrap(), @@ -746,6 +726,12 @@ mod tests { std::fs::write(&path, contents).unwrap(); path } + + fn create_dir(&self, relative: impl AsRef) -> PathBuf { + let path = self.path(relative); + std::fs::create_dir_all(&path).unwrap(); + path + } } fn manifest_text(repo_id: &str, siblings: &str) -> String { @@ -775,15 +761,17 @@ base_url = "https://staging.example.test/{repo_id}/" #[test] fn missing_repo_manifest_loads_as_none() { - let repo = TestRepo::new(); + let workspace = TestWorkspace::new(); - assert!(LoadedRepoManifest::load(repo.root()).unwrap().is_none()); + assert!(LoadedRepoManifest::load(workspace.root()) + .unwrap() + .is_none()); } #[test] fn malformed_repo_manifest_reports_parse_error() { - let repo = TestRepo::new(); - let manifest_path = repo.write_file(REPO_MANIFEST_FILE, "repo_id = ["); + let workspace = TestWorkspace::new(); + let manifest_path = workspace.write_file(REPO_MANIFEST_FILE, "repo_id = ["); let error = LoadedRepoManifest::from_path(&manifest_path).unwrap_err(); @@ -792,8 +780,8 @@ base_url = "https://staging.example.test/{repo_id}/" #[test] fn parses_repo_manifest_with_directional_siblings() { - let repo = TestRepo::new(); - let manifest_path = repo.write_file( + let workspace = TestWorkspace::new(); + let manifest_path = workspace.write_file( REPO_MANIFEST_FILE, &manifest_text( "Core", @@ -818,8 +806,8 @@ base_url = "https://staging.example.test/EIPs/" #[test] fn repo_manifest_requires_identity_and_environments() { - let repo = TestRepo::new(); - let manifest_path = repo.write_file( + let workspace = TestWorkspace::new(); + let manifest_path = workspace.write_file( REPO_MANIFEST_FILE, r#" [production] @@ -833,7 +821,7 @@ base_url = "https://example.test/Core/" assert!(reason.contains("missing required `repo_id` entry")); - let manifest_path = repo.write_file( + let manifest_path = workspace.write_file( REPO_MANIFEST_FILE, r#" repo_id = "Core" @@ -848,7 +836,7 @@ base_url = "https://example.test/Core/" assert!(reason.contains("missing required `staging` entry")); - let manifest_path = repo.write_file( + let manifest_path = workspace.write_file( REPO_MANIFEST_FILE, r#" repo_id = "Core" @@ -866,8 +854,8 @@ base_url = "https://staging.example.test/Core/" #[test] fn repo_manifest_rejects_unsafe_and_reserved_keys() { - let repo = TestRepo::new(); - let manifest_path = repo.write_file(REPO_MANIFEST_FILE, &manifest_text("theme", "")); + let workspace = TestWorkspace::new(); + let manifest_path = workspace.write_file(REPO_MANIFEST_FILE, &manifest_text("theme", "")); let reason = manifest_invalid_reason(LoadedRepoManifest::from_path(&manifest_path).unwrap_err()); @@ -875,7 +863,8 @@ base_url = "https://staging.example.test/Core/" assert!(reason.contains("repo_id `theme`")); assert!(reason.contains("reserved")); - let manifest_path = repo.write_file(REPO_MANIFEST_FILE, &manifest_text("Core/Meta", "")); + let manifest_path = + workspace.write_file(REPO_MANIFEST_FILE, &manifest_text("Core/Meta", "")); let reason = manifest_invalid_reason(LoadedRepoManifest::from_path(&manifest_path).unwrap_err()); @@ -885,8 +874,8 @@ base_url = "https://staging.example.test/Core/" #[test] fn repo_manifest_rejects_self_sibling() { - let repo = TestRepo::new(); - let manifest_path = repo.write_file( + let workspace = TestWorkspace::new(); + let manifest_path = workspace.write_file( REPO_MANIFEST_FILE, &manifest_text( "Core", @@ -909,9 +898,9 @@ base_url = "https://staging.example.test/Core/" } #[test] - fn repo_manifest_rejects_duplicate_sibling_repositories() { - let repo = TestRepo::new(); - let manifest_path = repo.write_file( + fn repo_manifest_rejects_duplicate_sibling_repositories_per_environment() { + let workspace = TestWorkspace::new(); + let manifest_path = workspace.write_file( REPO_MANIFEST_FILE, &manifest_text( "Core", @@ -938,18 +927,17 @@ base_url = "https://staging.example.test/ERCs/" let reason = manifest_invalid_reason(LoadedRepoManifest::from_path(&manifest_path).unwrap_err()); - assert!(reason.contains("duplicate production sibling repository")); - assert!(reason.contains("https://example.test/shared.git")); + assert!(reason.contains("duplicate production sibling repository declaration")); } #[test] fn parses_default_workspace_config() { - let repo = TestRepo::new(); - let config_path = repo.write_file(LOCAL_CONFIG_FILE, &default_workspace_config_text()); + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file(LOCAL_CONFIG_FILE, &default_workspace_config_text()); let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); - assert_eq!(config.workspace_root(), repo.root()); + assert_eq!(config.workspace_root(), workspace.root()); assert_eq!(config.server_settings(), &ServerSettings::default()); assert_eq!( config.site_settings().base_url.as_ref().unwrap().as_str(), @@ -978,8 +966,8 @@ base_url = "https://staging.example.test/ERCs/" #[test] fn parses_workspace_config_server_settings() { - let repo = TestRepo::new(); - let config_path = repo.write_file( + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( LOCAL_CONFIG_FILE, r#" [server] @@ -1001,8 +989,8 @@ port = 8080 #[test] fn missing_server_settings_use_default_binding() { - let repo = TestRepo::new(); - let config_path = repo.write_file( + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( LOCAL_CONFIG_FILE, r#" [site] @@ -1020,8 +1008,8 @@ base_url = "http://localhost:4000" #[test] fn parses_workspace_config_site_settings() { - let repo = TestRepo::new(); - let config_path = repo.write_file( + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( LOCAL_CONFIG_FILE, r#" [site] @@ -1039,8 +1027,8 @@ base_url = "http://localhost:4000" #[test] fn invalid_workspace_config_site_base_url_errors() { - let repo = TestRepo::new(); - let config_path = repo.write_file( + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( LOCAL_CONFIG_FILE, r#" [site] @@ -1056,8 +1044,8 @@ base_url = "not a url" #[test] fn missing_site_settings_preserve_no_base_url_override() { - let repo = TestRepo::new(); - let config_path = repo.write_file( + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( LOCAL_CONFIG_FILE, r#" [server] @@ -1073,8 +1061,8 @@ port = 1111 #[test] fn minimal_workspace_config_parses() { - let repo = TestRepo::new(); - let config_path = repo.write_file( + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( LOCAL_CONFIG_FILE, r#" [server] @@ -1097,8 +1085,8 @@ base_url = "http://127.0.0.1:1111" #[test] fn empty_workspace_config_uses_defaults() { - let repo = TestRepo::new(); - let config_path = repo.write_file(LOCAL_CONFIG_FILE, " \n"); + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file(LOCAL_CONFIG_FILE, " \n"); let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); @@ -1109,8 +1097,8 @@ base_url = "http://127.0.0.1:1111" #[test] fn parses_workspace_config_render_only_settings() { - let repo = TestRepo::new(); - let config_path = repo.write_file( + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( LOCAL_CONFIG_FILE, r#" [render] @@ -1139,8 +1127,8 @@ only = [555, 678, 555] ]; for (name, contents) in cases { - let repo = TestRepo::new(); - let config_path = repo.write_file(LOCAL_CONFIG_FILE, contents); + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file(LOCAL_CONFIG_FILE, contents); let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); assert!( @@ -1160,9 +1148,9 @@ only = [555, 678, 555] ]; for (name, contents) in cases { - let repo = TestRepo::new(); + let workspace = TestWorkspace::new(); let config_path = - repo.write_file(LOCAL_CONFIG_FILE, &format!("[render]\n{contents}\n")); + workspace.write_file(LOCAL_CONFIG_FILE, &format!("[render]\n{contents}\n")); let error = LoadedWorkspaceConfig::from_path(&config_path).unwrap_err(); assert!( @@ -1212,8 +1200,8 @@ repository = "https://github.com/eips-wg/theme.git" ]; for (field, contents) in cases { - let repo = TestRepo::new(); - let config_path = repo.write_file(LOCAL_CONFIG_FILE, &contents); + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file(LOCAL_CONFIG_FILE, &contents); let error = LoadedWorkspaceConfig::from_path(&config_path).unwrap_err(); assert!( @@ -1225,10 +1213,9 @@ repository = "https://github.com/eips-wg/theme.git" #[test] fn discover_path_walks_upward() { - let repo = TestRepo::new(); - let config_path = repo.write_file(LOCAL_CONFIG_FILE, &default_workspace_config_text()); - let nested = repo.path("EIPs/content"); - std::fs::create_dir_all(&nested).unwrap(); + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file(LOCAL_CONFIG_FILE, &default_workspace_config_text()); + let nested = workspace.create_dir("EIPs/content"); assert_eq!(discover_path(&nested).unwrap(), config_path); assert_eq!( @@ -1242,9 +1229,8 @@ repository = "https://github.com/eips-wg/theme.git" #[test] fn missing_workspace_config_is_not_discovered() { - let repo = TestRepo::new(); - let nested = repo.path("EIPs/content"); - std::fs::create_dir_all(&nested).unwrap(); + let workspace = TestWorkspace::new(); + let nested = workspace.create_dir("EIPs/content"); assert!(discover_path(&nested).is_none()); assert!(LoadedWorkspaceConfig::discover(&nested).unwrap().is_none()); diff --git a/src/editorial.rs b/src/editorial.rs index e516004..2b1ce5a 100644 --- a/src/editorial.rs +++ b/src/editorial.rs @@ -230,7 +230,7 @@ fn validate_raw_editorial_targets( validate_editorial_targets(&resolved.root_path, targets, strict) } -fn editorial_targets_from_source( +pub(crate) fn editorial_targets_from_source( selectors: &EditorialSelectorArgs, resolved: &ResolvedExecution, upstream_source: Option<&git::SourceWithUpstream>, diff --git a/src/execution.rs b/src/execution.rs index 364811e..2b10743 100644 --- a/src/execution.rs +++ b/src/execution.rs @@ -388,6 +388,7 @@ pub(crate) fn resolve_execution(args: &Args) -> Result Args { @@ -482,7 +483,7 @@ mod tests { fn assert_theme_only_missing_workspace_error(arguments: &[&str]) { let args = parse_args(arguments); - let error = super::resolve_theme_path(None, &args.operation).unwrap_err(); + let error = resolve_theme_path(None, &args.operation).unwrap_err(); let message = error.to_string(); assert!(message.contains( @@ -525,27 +526,10 @@ mod tests { .map(|numbers| numbers.into_iter().map(|number| number.get()).collect()) } - #[test] - fn explicit_env_or_parity_provenance_is_classified_separately_from_local_defaults() { - let cases: &[(&[&str], Option)] = &[ - (&["build-eips", "--staging", "build"], Some(true)), - (&["build-eips", "--production", "build"], Some(false)), - (&["build-eips", "parity", "build"], Some(true)), - (&["build-eips", "build"], None), - (&["build-eips", "serve"], None), - (&["build-eips", "check"], None), - ]; - - for (arguments, expected) in cases { - let args = parse_args(arguments); - assert_eq!(explicit_environment_or_parity(&args).unwrap(), *expected); - } - } - #[test] fn server_binding_resolution_uses_cli_config_then_defaults() { assert_eq!( - resolve_server_binding(None, &Default::default()), + resolve_server_binding(None, &ServerCliArgs::default()), ServerBinding { host: "127.0.0.1".to_owned(), port: 1111, @@ -561,7 +545,7 @@ port = 8080 ); assert_eq!( - resolve_server_binding(Some(&workspace_config), &Default::default()), + resolve_server_binding(Some(&workspace_config), &ServerCliArgs::default()), ServerBinding { host: "0.0.0.0".to_owned(), port: 8080, @@ -595,6 +579,23 @@ port = 8080 ); } + #[test] + fn explicit_env_or_parity_provenance_is_classified_separately_from_local_defaults() { + let cases: &[(&[&str], Option)] = &[ + (&["build-eips", "--staging", "build"], Some(true)), + (&["build-eips", "--production", "build"], Some(false)), + (&["build-eips", "parity", "build"], Some(true)), + (&["build-eips", "build"], None), + (&["build-eips", "serve"], None), + (&["build-eips", "check"], None), + ]; + + for (arguments, expected) in cases { + let args = parse_args(arguments); + assert_eq!(explicit_environment_or_parity(&args).unwrap(), *expected); + } + } + #[test] fn base_url_override_resolution_uses_cli_config_then_provenance() { let workspace_config = load_workspace_config( @@ -637,6 +638,46 @@ base_url = "http://localhost:4000" .unwrap() .is_none()); } + + for arguments in [ + &[ + "build-eips", + "--staging", + "build", + "--base-url", + "http://localhost:5000", + ][..], + &[ + "build-eips", + "--production", + "build", + "--base-url", + "http://localhost:5000", + ][..], + &[ + "build-eips", + "parity", + "build", + "--base-url", + "http://localhost:5000", + ][..], + &[ + "build-eips", + "parity", + "serve", + "--base-url", + "http://localhost:5000", + ][..], + ] { + let args = parse_args(arguments); + assert_eq!( + resolve_base_url_override(&args, Some(&workspace_config)) + .unwrap() + .unwrap() + .as_str(), + "http://localhost:5000/" + ); + } } #[test] @@ -668,110 +709,6 @@ base_url = "http://localhost:4000" ); } - #[test] - fn only_cli_selection_overrides_config_and_dedupes_for_build_and_serve() { - let workspace_config = load_workspace_config( - r#" -[render] -only = [555] -"#, - ); - - assert_eq!( - only_selection_for( - &["build-eips", "build", "--only", "678", "555", "678"], - Some(&workspace_config), - ), - Some(vec![555, 678]) - ); - assert_eq!( - only_selection_for( - &["build-eips", "serve", "--only", "678", "555", "678"], - Some(&workspace_config), - ), - Some(vec![555, 678]) - ); - } - - #[test] - fn render_only_config_selection_applies_to_local_dirty_build_and_serve_only() { - let workspace_config = load_workspace_config( - r#" -[render] -only = [555, 678, 555] -"#, - ); - - assert_eq!( - only_selection_for(&["build-eips", "build"], Some(&workspace_config)), - Some(vec![555, 678]) - ); - assert_eq!( - only_selection_for(&["build-eips", "serve"], Some(&workspace_config)), - Some(vec![555, 678]) - ); - - for arguments in [ - &["build-eips", "build", "--clean"][..], - &["build-eips", "--remote-siblings", "build"][..], - &["build-eips", "--staging", "build"][..], - &["build-eips", "--production", "build"][..], - &["build-eips", "serve", "--clean"][..], - &["build-eips", "--remote-siblings", "serve"][..], - &["build-eips", "--staging", "serve"][..], - &["build-eips", "--production", "serve"][..], - &["build-eips", "check"][..], - &["build-eips", "parity", "build"][..], - &["build-eips", "parity", "serve"][..], - ] { - assert!(only_selection_for(arguments, Some(&workspace_config)).is_none()); - } - } - - #[test] - fn only_cli_selection_rejects_non_local_dirty_build_and_serve_modes() { - let workspace_config = load_workspace_config(""); - - for arguments in [ - &["build-eips", "build", "--only", "555", "--clean"][..], - &["build-eips", "--remote-siblings", "build", "--only", "555"][..], - &["build-eips", "--staging", "build", "--only", "555"][..], - &["build-eips", "--production", "build", "--only", "555"][..], - &["build-eips", "serve", "--only", "555", "--clean"][..], - &["build-eips", "--remote-siblings", "serve", "--only", "555"][..], - &["build-eips", "--staging", "serve", "--only", "555"][..], - &["build-eips", "--production", "serve", "--only", "555"][..], - ] { - let args = parse_args(arguments); - let error = resolve_execution_settings(&args, &[], Some(&workspace_config)) - .unwrap_err() - .to_string(); - - assert!( - error.contains("--only is supported only for local dirty build and serve commands") - ); - } - } - - #[test] - fn missing_render_config_and_empty_only_disable_filtering() { - let missing_render = load_workspace_config(""); - let missing_only = load_workspace_config("[render]\n"); - let empty_only = load_workspace_config( - r#" -[render] -only = [] -"#, - ); - - assert!(only_selection_for(&["build-eips", "build"], Some(&missing_render)).is_none()); - assert!(only_selection_for(&["build-eips", "build"], Some(&missing_only)).is_none()); - assert!(only_selection_for(&["build-eips", "build"], Some(&empty_only)).is_none()); - assert!(only_selection_for(&["build-eips", "serve"], Some(&missing_render)).is_none()); - assert!(only_selection_for(&["build-eips", "serve"], Some(&missing_only)).is_none()); - assert!(only_selection_for(&["build-eips", "serve"], Some(&empty_only)).is_none()); - } - #[test] fn plain_site_commands_are_local_first_dirty_staging() { let workspace_config = load_workspace_config(""); @@ -796,6 +733,47 @@ only = [] } } + #[test] + fn zola_runtime_commands_require_workspace_local_theme() { + let workspace = TempDir::new().unwrap(); + let config_path = workspace.path().join(config::LOCAL_CONFIG_FILE); + std::fs::write(&config_path, "").unwrap(); + std::fs::create_dir(workspace.path().join(config::DEFAULT_THEME_DIR)).unwrap(); + let workspace_config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + for arguments in [ + &["build-eips", "build"][..], + &["build-eips", "check"][..], + &["build-eips", "serve"][..], + &["build-eips", "--staging", "build"][..], + &["build-eips", "--production", "check"][..], + &["build-eips", "parity", "build"][..], + &["build-eips", "editorial", "check", "--against-upstream"][..], + ] { + let args = parse_args(arguments); + let theme_path = resolve_theme_path(Some(&workspace_config), &args.operation) + .unwrap() + .unwrap(); + + assert_eq!(theme_path, workspace.path().join(config::DEFAULT_THEME_DIR)); + } + } + + #[test] + fn non_theme_commands_do_not_require_workspace_local_theme() { + for arguments in [ + &["build-eips", "changed"][..], + &["build-eips", "clean"][..], + &["build-eips", "preview"][..], + &["build-eips", "doctor"][..], + &["build-eips", "print", "schema-version"][..], + ] { + let args = parse_args(arguments); + + assert!(resolve_theme_path(None, &args.operation).unwrap().is_none()); + } + } + #[test] fn clean_plain_site_commands_keep_local_sources_but_disable_dirty_materialization() { let workspace_config = load_workspace_config(""); @@ -919,6 +897,103 @@ only = [] } } + #[test] + fn only_selection_dedupes_and_cli_replaces_config() { + let workspace_config = load_workspace_config( + r#" +[render] +only = [678, 555, 678] +"#, + ); + + assert_eq!( + only_selection_for(&["build-eips", "build"], Some(&workspace_config)).unwrap(), + vec![555, 678] + ); + assert_eq!( + only_selection_for( + &["build-eips", "build", "--only", "00555", "555", "897"], + Some(&workspace_config) + ) + .unwrap(), + vec![555, 897] + ); + assert_eq!( + only_selection_for(&["build-eips", "serve"], Some(&workspace_config)).unwrap(), + vec![555, 678] + ); + assert_eq!( + only_selection_for( + &["build-eips", "serve", "--only", "00555", "555", "897"], + Some(&workspace_config) + ) + .unwrap(), + vec![555, 897] + ); + } + + #[test] + fn only_cli_rejects_parsed_non_applicable_build_and_serve_modes() { + for arguments in [ + &["build-eips", "--staging", "build", "--only", "555"][..], + &["build-eips", "--production", "build", "--only", "555"][..], + &["build-eips", "build", "--clean", "--only", "555"][..], + &["build-eips", "--remote-siblings", "build", "--only", "555"][..], + &["build-eips", "--staging", "serve", "--only", "555"][..], + &["build-eips", "--production", "serve", "--only", "555"][..], + &["build-eips", "serve", "--clean", "--only", "555"][..], + &["build-eips", "--remote-siblings", "serve", "--only", "555"][..], + ] { + let args = parse_args(arguments); + let error = resolve_execution_settings(&args, &[], None).unwrap_err(); + + assert!(error + .to_string() + .contains("--only is supported only for local dirty build and serve commands")); + } + } + + #[test] + fn render_only_config_is_ignored_outside_applicable_build_commands() { + let workspace_config = load_workspace_config( + r#" +[render] +only = [999999] +"#, + ); + + for arguments in [ + &["build-eips", "check"][..], + &["build-eips", "build", "--clean"][..], + &["build-eips", "serve", "--clean"][..], + &["build-eips", "--staging", "build"][..], + &["build-eips", "--staging", "serve"][..], + &["build-eips", "parity", "build"][..], + &["build-eips", "parity", "serve"][..], + ] { + assert!(only_selection_for(arguments, Some(&workspace_config)).is_none()); + } + } + + #[test] + fn missing_render_config_and_empty_only_disable_filtering() { + let missing_render = load_workspace_config(""); + let missing_only = load_workspace_config("[render]\n"); + let empty_only = load_workspace_config( + r#" +[render] +only = [] +"#, + ); + + assert!(only_selection_for(&["build-eips", "build"], Some(&missing_render)).is_none()); + assert!(only_selection_for(&["build-eips", "build"], Some(&missing_only)).is_none()); + assert!(only_selection_for(&["build-eips", "build"], Some(&empty_only)).is_none()); + assert!(only_selection_for(&["build-eips", "serve"], Some(&missing_render)).is_none()); + assert!(only_selection_for(&["build-eips", "serve"], Some(&missing_only)).is_none()); + assert!(only_selection_for(&["build-eips", "serve"], Some(&empty_only)).is_none()); + } + #[test] fn non_site_commands_do_not_require_workspace_local_sources() { for arguments in [ @@ -1021,49 +1096,6 @@ only = [] } } - #[test] - fn zola_runtime_commands_require_workspace_local_theme() { - let workspace = TempDir::new().unwrap(); - let config_path = workspace.path().join(config::LOCAL_CONFIG_FILE); - std::fs::write(&config_path, "").unwrap(); - std::fs::create_dir(workspace.path().join(config::DEFAULT_THEME_DIR)).unwrap(); - let workspace_config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); - - for arguments in [ - &["build-eips", "build"][..], - &["build-eips", "check"][..], - &["build-eips", "serve"][..], - &["build-eips", "--staging", "build"][..], - &["build-eips", "--production", "check"][..], - &["build-eips", "parity", "build"][..], - &["build-eips", "editorial", "check", "--against-upstream"][..], - ] { - let args = parse_args(arguments); - let theme_path = super::resolve_theme_path(Some(&workspace_config), &args.operation) - .unwrap() - .unwrap(); - - assert_eq!(theme_path, workspace.path().join(config::DEFAULT_THEME_DIR)); - } - } - - #[test] - fn non_theme_commands_do_not_require_workspace_local_theme() { - for arguments in [ - &["build-eips", "changed"][..], - &["build-eips", "clean"][..], - &["build-eips", "preview"][..], - &["build-eips", "doctor"][..], - &["build-eips", "print", "schema-version"][..], - ] { - let args = parse_args(arguments); - - assert!(super::resolve_theme_path(None, &args.operation) - .unwrap() - .is_none()); - } - } - #[test] fn editorial_dispatch_uses_local_first_for_all_editorial_commands() { let workspace_config = load_workspace_config(""); @@ -1228,7 +1260,7 @@ only = [] } #[test] - fn zero_sibling_local_first_without_workspace_config_can_resolve_sibling_policy() { + fn zero_sibling_local_first_without_workspace_config_only_requires_theme_resolution() { let args = parse_args(&["build-eips", "build"]); let settings = resolve_execution_settings(&args, &[], None).unwrap(); @@ -1270,8 +1302,7 @@ only = [] let workspace_config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); let args = parse_args(&["build-eips", "build"]); - let error = - super::resolve_theme_path(Some(&workspace_config), &args.operation).unwrap_err(); + let error = resolve_theme_path(Some(&workspace_config), &args.operation).unwrap_err(); let message = error.to_string(); assert!(message.contains(&format!( diff --git a/src/identity.rs b/src/identity.rs index 6db5d08..68ec03a 100644 --- a/src/identity.rs +++ b/src/identity.rs @@ -8,7 +8,7 @@ use std::path::Path; -use snafu::{ResultExt, Whatever}; +use snafu::{OptionExt, ResultExt, Whatever}; use crate::{ config::{self, Config, LoadedRepoManifest}, @@ -96,11 +96,12 @@ impl ActiveRepoIdentity { } else { Config::production() }; - let Some(repository_use) = baseline.locations.repository_use_for_title(repo_id) - else { - snafu::whatever!("legacy repository metadata for `{repo_id}` is unavailable"); - }; - Ok(repository_use) + baseline + .locations + .repository_use_for_title(repo_id) + .with_whatever_context(|| { + format!("legacy repository metadata for `{repo_id}` is unavailable") + }) } } } diff --git a/src/layout.rs b/src/layout.rs index 3725a1f..dec1039 100644 --- a/src/layout.rs +++ b/src/layout.rs @@ -4,6 +4,8 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +//! Shared build layout names and path helpers. + use std::path::{Path, PathBuf}; pub(crate) const CONTENT_DIR: &str = "content"; diff --git a/src/main.rs b/src/main.rs index cb6de2c..e78c69e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,6 +26,9 @@ mod serve; mod workspace; mod zola; +#[cfg(test)] +mod tests; + use std::path::{Path, PathBuf}; use clap::Parser; @@ -105,7 +108,6 @@ fn run() -> Result<(), Whatever> { } let build_path = make_build_dir(&resolved.build_path)?; - let mut lock_file = lock(&build_path)?; match runtime_operation { @@ -115,7 +117,7 @@ fn run() -> Result<(), Whatever> { lock_file .unlock() .whatever_context("unable to unlock build directory")?; - std::fs::remove_dir_all(build_path) + std::fs::remove_dir_all(&build_path) .whatever_context("unable to remove build directory")?; return Ok(()); } @@ -137,7 +139,7 @@ fn run() -> Result<(), Whatever> { run_editorial_lint(&resolved, &selectors, eipw)?; } EditorialCommand::Check { selectors, eipw } => { - run_editorial_lint(&resolved, &selectors, eipw.clone())?; + run_editorial_lint(&resolved, &selectors, eipw)?; Prepared::prepare(editorial_runtime_execution(resolved, &selectors))?.check()?; } }, diff --git a/src/print.rs b/src/print.rs index 4220cb2..33b0945 100644 --- a/src/print.rs +++ b/src/print.rs @@ -8,7 +8,7 @@ use eipw_lint::config::DefaultOptions; #[derive(Debug, clap::Args, Clone)] pub struct CmdArgs { - /// Thing to print + /// Linter metadata or configuration output to print what: What, } diff --git a/src/tests.rs b/src/tests.rs new file mode 100644 index 0000000..f45eba9 --- /dev/null +++ b/src/tests.rs @@ -0,0 +1,814 @@ +#![cfg(test)] + +// Cross-domain behavior tests live here; see src/README.md for module test ownership. + +use std::{ + collections::BTreeSet, + path::{Path, PathBuf}, +}; + +use clap::Parser; +use git2::{IndexAddOption, Repository, Signature}; +use snafu::Report; +use tempfile::TempDir; +use url::Url; + +use crate::{ + cli::{Args, EditorialCommand, EditorialSelectorArgs, Operation, RuntimeOperation}, + config::{self, LoadedWorkspaceConfig}, + editorial::editorial_targets_from_source, + execution::{ + resolve_execution, resolve_execution_settings, validate_non_execution_command_flags, + ExecutionSettings, ResolvedExecution, SelectedSource, + }, + layout::{BUILD_DIR, CONTENT_DIR, REPO_DIR}, + markdown, + proposal::{OnlyRenderPlan, ProposalNumber}, +}; + +fn parse_args(arguments: &[&str]) -> Args { + Args::try_parse_from(arguments).unwrap() +} + +fn settings_for( + arguments: &[&str], + sibling_ids: &[&str], + workspace_config: Option<&LoadedWorkspaceConfig>, +) -> ExecutionSettings { + let args = parse_args(arguments); + let sibling_ids = sibling_ids + .iter() + .map(|sibling_id| (*sibling_id).to_owned()) + .collect::>(); + + resolve_execution_settings(&args, &sibling_ids, workspace_config).unwrap() +} + +fn assert_settings( + arguments: &[&str], + sibling_ids: &[&str], + workspace_config: Option<&LoadedWorkspaceConfig>, + expected: ExecutionSettings, +) { + assert_eq!( + settings_for(arguments, sibling_ids, workspace_config), + expected + ); +} + +fn file_url(path: &Path) -> Url { + Url::from_directory_path(path).unwrap() +} + +fn write_file(root: &Path, relative: impl AsRef, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); +} + +fn commit_all(repo: &Repository, message: &str) { + let mut index = repo.index().unwrap(); + index + .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + let parents = repo + .head() + .ok() + .and_then(|head| head.target()) + .map(|oid| repo.find_commit(oid).unwrap()) + .into_iter() + .collect::>(); + let parent_refs = parents.iter().collect::>(); + + repo.commit( + Some("HEAD"), + &signature, + &signature, + message, + &tree, + &parent_refs, + ) + .unwrap(); +} + +fn init_repo(path: &Path, files: &[(&str, &str)]) -> Repository { + std::fs::create_dir_all(path).unwrap(); + let repo = Repository::init(path).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + for (relative, contents) in files { + write_file(path, relative, contents); + } + commit_all(&repo, "initial"); + repo +} + +fn append_and_commit(repo: &Repository, root: &Path, files: &[(&str, &str)], message: &str) { + for (relative, contents) in files { + write_file(root, relative, contents); + } + commit_all(repo, message); +} + +fn proposal_markdown(number: u32, category: Option<&str>, body: &str) -> String { + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!("---\neip: {number}\ntitle: Proposal {number}\n{category}---\n{body}\n") +} + +fn materialize_resolved_repo(resolved: &ResolvedExecution) -> PathBuf { + let repo_path = resolved.build_path.join(REPO_DIR); + crate::git::Fresh::new( + &resolved.root_path, + &repo_path, + resolved.repository_use.clone(), + resolved.source_materialization, + ) + .unwrap() + .clone_src() + .unwrap() + .fetch_upstream() + .unwrap() + .merge() + .unwrap(); + + repo_path +} + +fn preprocess_and_prune_only(repo_path: &Path, selected: BTreeSet) { + let content_path = repo_path.join(CONTENT_DIR); + let plan = OnlyRenderPlan::build(&content_path, selected).unwrap(); + markdown::preprocess(&content_path, Some(&plan)).unwrap(); + plan.prune_content(&content_path).unwrap(); +} + +fn repo_manifest_text(repo_id: &str, repository: &Url, siblings: &[(&str, Url)]) -> String { + let mut manifest = format!( + r#" +repo_id = "{repo_id}" + +[production] +repository = "{repository}" +base_url = "https://example.test/{repo_id}/" + +[staging] +repository = "{repository}" +base_url = "https://staging.example.test/{repo_id}/" +"# + ); + + for (sibling_id, sibling_repository) in siblings { + manifest.push_str(&format!( + r#" +[siblings.{sibling_id}.production] +repository = "{sibling_repository}" +base_url = "https://example.test/{sibling_id}/" + +[siblings.{sibling_id}.staging] +repository = "{sibling_repository}" +base_url = "https://staging.example.test/{sibling_id}/" +"# + )); + } + + manifest +} + +fn write_repo_manifest_file(path: &Path, repo_id: &str, upstream: &Url, siblings: &[(&str, Url)]) { + write_file( + path, + config::REPO_MANIFEST_FILE, + &repo_manifest_text(repo_id, upstream, siblings), + ); +} + +fn write_manifest_repo( + path: &Path, + repo_id: &str, + upstream: &Url, + siblings: &[(&str, Url)], +) -> Repository { + let repo = init_repo(path, &[("content/0001.md", "# Proposal\n")]); + write_repo_manifest_file(path, repo_id, upstream, siblings); + commit_all(&repo, "add repo manifest"); + repo +} + +#[test] +fn command_groups_route_separately_from_parity() { + let init = parse_args(&["build-eips", "init", "/tmp/workspace"]); + let doctor = parse_args(&["build-eips", "doctor"]); + let editorial_lint = parse_args(&["build-eips", "editorial", "lint", "--working-tree"]); + let editorial_check = parse_args(&["build-eips", "editorial", "check", "--working-tree"]); + + assert!(matches!(init.operation, Operation::Init { .. })); + assert!(matches!(doctor.operation, Operation::Doctor)); + assert!(matches!( + editorial_lint.operation.runtime_operation(), + Some(RuntimeOperation::Editorial { + command: EditorialCommand::Lint { .. } + }) + )); + assert!(matches!( + editorial_check.operation.runtime_operation(), + Some(RuntimeOperation::Editorial { + command: EditorialCommand::Check { .. } + }) + )); + assert!(validate_non_execution_command_flags(&editorial_lint).is_ok()); +} + +#[test] +fn downstream_ci_changed_forms_do_not_need_workspace_config() { + for (arguments, expected_staging) in [ + (&["build-eips", "--staging", "changed"][..], true), + (&["build-eips", "--production", "changed"][..], false), + ] { + assert_settings( + arguments, + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: expected_staging, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + } +} + +#[test] +fn downstream_ci_zola_forms_require_workspace_local_theme_config() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + let active_path = active_path.to_string_lossy().to_string(); + + for arguments in [ + &["build-eips", "--staging", "build"][..], + &["build-eips", "--production", "build"][..], + &[ + "build-eips", + "--staging", + "editorial", + "check", + "--against-upstream", + ][..], + &["build-eips", "parity", "check"][..], + ] { + let mut cli_arguments = vec!["build-eips", "-C", active_path.as_str()]; + cli_arguments.extend_from_slice(&arguments[1..]); + let args = parse_args(&cli_arguments); + let message = resolve_execution(&args).unwrap_err().to_string(); + + assert!(message.contains("requires a workspace config with a local theme")); + assert!(message.contains("build-eips init ")); + } +} + +#[test] +fn manifest_identity_drives_runtime_resolution() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + std::fs::create_dir(workspace.path().join(config::DEFAULT_THEME_DIR)).unwrap(); + write_file( + workspace.path(), + config::LOCAL_CONFIG_FILE, + &config::default_workspace_config_text(), + ); + let build_root = workspace.path().join("build-root"); + let args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "--build-root", + build_root.to_str().unwrap(), + "parity", + "build", + ]); + + let resolved = resolve_execution(&args).unwrap(); + + assert_eq!(resolved.repository_use.title, "Core"); + assert_eq!(resolved.repository_use.location.repository, active_url); + assert!(resolved.repository_use.other_repos.is_empty()); + assert_eq!(resolved.build_path, build_root); +} + +#[test] +fn execution_commands_discover_workspace_config_from_active_repo_root() { + let workspace = TempDir::new().unwrap(); + let workspace_root = workspace.path().join("workspace"); + let active_path = workspace_root.join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + std::fs::create_dir(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + write_file( + &workspace_root, + config::LOCAL_CONFIG_FILE, + &config::default_workspace_config_text(), + ); + let args = parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "build"]); + + let resolved = resolve_execution(&args).unwrap(); + + assert_eq!( + resolved.build_path, + workspace_root + .join(config::DEFAULT_BUILD_ROOT_BASE) + .join("Core") + ); + assert_eq!( + resolved.source_materialization, + crate::git::SourceMaterialization::Dirty + ); +} + +#[test] +fn build_root_override_wins_with_workspace_config() { + let workspace = TempDir::new().unwrap(); + let workspace_root = workspace.path().join("workspace"); + let active_path = workspace_root.join("Core"); + let active_url = file_url(&active_path); + let build_root = workspace.path().join("override-build-root"); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + std::fs::create_dir(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + write_file( + &workspace_root, + config::LOCAL_CONFIG_FILE, + &config::default_workspace_config_text(), + ); + let args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "--build-root", + build_root.to_str().unwrap(), + "build", + ]); + + let resolved = resolve_execution(&args).unwrap(); + + assert_eq!(resolved.build_path, build_root); +} + +#[test] +fn non_workspace_runtime_path_falls_back_to_active_repo_build_dir() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + let args = parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "changed"]); + + let resolved = resolve_execution(&args).unwrap(); + + assert_eq!(resolved.build_path, active_path.join(BUILD_DIR)); +} + +#[test] +fn non_theme_runtime_commands_resolve_without_workspace_config() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + let active_path = active_path.to_string_lossy().to_string(); + + for command in ["changed", "clean", "preview"] { + let args = parse_args(&["build-eips", "-C", active_path.as_str(), command]); + let resolved = resolve_execution(&args).unwrap(); + + assert!(resolved.theme_path.is_none()); + } +} + +#[test] +fn unknown_repo_without_manifest_or_legacy_identity_errors() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Unknown"); + init_repo(&active_path, &[("content/0001.md", "# Proposal\n")]); + let args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "parity", + "build", + ]); + + let error = resolve_execution(&args).unwrap_err(); + let message = error.to_string(); + + assert!(message.contains(config::REPO_MANIFEST_FILE)); + assert!(message.contains("legacy EIPs/ERCs identity fallback")); +} + +#[test] +fn malformed_repo_manifest_does_not_fall_back_to_legacy_identity() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Malformed"); + init_repo(&active_path, &[("content/0001.md", "# Proposal\n")]); + write_file(&active_path, config::REPO_MANIFEST_FILE, "repo_id = ["); + let args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "parity", + "build", + ]); + + let message = Report::from_error(resolve_execution(&args).unwrap_err()).to_string(); + + assert!(message.contains("unable to load repo manifest")); + assert!(!message.contains("legacy EIPs/ERCs identity fallback")); +} + +#[test] +fn workspace_local_sibling_mode_is_all_or_nothing() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let eips_path = workspace.path().join("EIPs"); + init_repo(&eips_path, &[("content/0002.md", "# EIP\n")]); + let siblings = vec![ + ("EIPs", file_url(&eips_path)), + ("ERCs", file_url(&workspace.path().join("remotes/ERCs"))), + ]; + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &siblings); + std::fs::create_dir(workspace.path().join(config::DEFAULT_THEME_DIR)).unwrap(); + std::fs::write( + workspace.path().join(config::LOCAL_CONFIG_FILE), + config::default_workspace_config_text(), + ) + .unwrap(); + let args = parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "build"]); + + let error = resolve_execution(&args).unwrap_err(); + let message = error.to_string(); + + assert!(message.contains("requires all declared sibling repos")); + assert!(message.contains("ERCs")); +} + +#[test] +fn workspace_local_sources_resolve_from_standard_layout() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let eips_path = workspace.path().join("EIPs"); + let ercs_path = workspace.path().join("ERCs"); + init_repo(&eips_path, &[("content/0002.md", "# EIP\n")]); + init_repo(&ercs_path, &[("content/0003.md", "# ERC\n")]); + std::fs::create_dir(workspace.path().join(config::DEFAULT_THEME_DIR)).unwrap(); + let siblings = vec![ + ("EIPs", file_url(&eips_path)), + ("ERCs", file_url(&ercs_path)), + ]; + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &siblings); + std::fs::write( + workspace.path().join(config::LOCAL_CONFIG_FILE), + config::default_workspace_config_text(), + ) + .unwrap(); + let args = parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "build"]); + + let resolved = resolve_execution(&args).unwrap(); + + assert_eq!( + resolved.theme_path.as_deref(), + Some(workspace.path().join(config::DEFAULT_THEME_DIR).as_path()) + ); + assert_eq!( + resolved.repository_use.other_repos["EIPs"], + file_url(&eips_path) + ); + assert_eq!( + resolved.repository_use.other_repos["ERCs"], + file_url(&ercs_path) + ); +} + +#[test] +fn manifest_driven_multi_repo_build_and_editorial_flows_resolve_siblings() { + let temp = TempDir::new().unwrap(); + let upstream_path = temp.path().join("upstream/Core"); + init_repo( + &upstream_path, + &[("content/0001.md", "# Original proposal\n")], + ); + let upstream_url = file_url(&upstream_path); + + let active_path = temp.path().join("workspace/Core"); + std::fs::create_dir_all(active_path.parent().unwrap()).unwrap(); + std::fs::create_dir(temp.path().join("workspace/theme")).unwrap(); + write_file( + active_path.parent().unwrap(), + config::LOCAL_CONFIG_FILE, + &config::default_workspace_config_text(), + ); + git2::build::RepoBuilder::new() + .clone(upstream_url.as_str(), &active_path) + .unwrap(); + let active_repo = Repository::open(&active_path).unwrap(); + + let eips_path = temp.path().join("remotes/EIPs"); + init_repo(&eips_path, &[("content/0002.md", "# EIP sibling\n")]); + let ercs_path = temp.path().join("remotes/ERCs"); + init_repo(&ercs_path, &[("content/0003.md", "# ERC sibling\n")]); + let siblings = vec![ + ("EIPs", file_url(&eips_path)), + ("ERCs", file_url(&ercs_path)), + ]; + write_repo_manifest_file(&active_path, "Core", &upstream_url, &siblings); + append_and_commit( + &active_repo, + &active_path, + &[("content/0001.md", "# Updated proposal\n")], + "local proposal update", + ); + let build_root = temp.path().join("build-root"); + let args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "--build-root", + build_root.to_str().unwrap(), + "parity", + "build", + ]); + let resolved = resolve_execution(&args).unwrap(); + + assert_eq!(resolved.repository_use.other_repos.len(), 2); + + let repo_path = resolved.build_path.join(REPO_DIR); + let source = crate::git::Fresh::new( + &resolved.root_path, + &repo_path, + resolved.repository_use.clone(), + resolved.source_materialization, + ) + .unwrap() + .clone_src() + .unwrap() + .fetch_upstream() + .unwrap(); + + let selectors = EditorialSelectorArgs { + paths: Vec::::new(), + batch: None, + working_tree: false, + against_upstream: true, + }; + let targets = editorial_targets_from_source(&selectors, &resolved, Some(&source)).unwrap(); + + source.merge().unwrap(); + + assert!(repo_path.join("content/0002.md").is_file()); + assert!(repo_path.join("content/0003.md").is_file()); + + assert_eq!(targets, vec![PathBuf::from("content/0001.md")]); +} + +#[test] +fn only_build_selection_can_come_from_workspace_local_sibling_after_merge() { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("Core"); + let sibling_path = workspace_root.join("ERCs"); + let active_url = file_url(&active_path); + + let active_555 = proposal_markdown(555, None, "Active proposal."); + let active_repo = init_repo(&active_path, &[("content/00555.md", active_555.as_str())]); + let sibling_678 = proposal_markdown(678, Some("ERC"), "Sibling proposal."); + init_repo(&sibling_path, &[("content/00678.md", sibling_678.as_str())]); + write_repo_manifest_file( + &active_path, + "Core", + &active_url, + &[("ERCs", file_url(&sibling_path))], + ); + commit_all(&active_repo, "add manifest"); + std::fs::create_dir(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + write_file( + &workspace_root, + config::LOCAL_CONFIG_FILE, + &config::default_workspace_config_text(), + ); + + let args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "build", + "--only", + "678", + ]); + let resolved = resolve_execution(&args).unwrap(); + let repo_path = materialize_resolved_repo(&resolved); + let selected = resolved.only.clone().unwrap(); + + preprocess_and_prune_only(&repo_path, selected); + + assert!(repo_path.join("content/00678.md").is_file()); + assert!(!repo_path.join("content/00555.md").exists()); +} + +#[test] +fn normal_build_after_only_restores_full_materialized_content_tree() { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("Core"); + let active_url = file_url(&active_path); + let selected_555 = proposal_markdown(555, None, "Selected proposal."); + let unselected_678 = proposal_markdown(678, Some("ERC"), "Unselected proposal."); + let active_repo = init_repo( + &active_path, + &[ + ("content/00555.md", selected_555.as_str()), + ("content/00678.md", unselected_678.as_str()), + ], + ); + write_repo_manifest_file(&active_path, "Core", &active_url, &[]); + commit_all(&active_repo, "add manifest"); + std::fs::create_dir(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + write_file( + &workspace_root, + config::LOCAL_CONFIG_FILE, + &config::default_workspace_config_text(), + ); + let build_root = temp.path().join("build-root"); + + let only_args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "--build-root", + build_root.to_str().unwrap(), + "build", + "--only", + "555", + ]); + let only_resolved = resolve_execution(&only_args).unwrap(); + let repo_path = materialize_resolved_repo(&only_resolved); + preprocess_and_prune_only(&repo_path, only_resolved.only.clone().unwrap()); + + assert!(repo_path.join("content/00555.md").is_file()); + assert!(!repo_path.join("content/00678.md").exists()); + + let normal_args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "--build-root", + build_root.to_str().unwrap(), + "build", + ]); + let normal_resolved = resolve_execution(&normal_args).unwrap(); + assert!(normal_resolved.only.is_none()); + let restored_repo_path = materialize_resolved_repo(&normal_resolved); + markdown::preprocess(&restored_repo_path.join(CONTENT_DIR), None).unwrap(); + + assert!(restored_repo_path.join("content/00555.md").is_file()); + assert!(restored_repo_path.join("content/00678.md").is_file()); +} + +#[test] +fn serve_applies_render_only_config_in_phase_two() { + let workspace = TempDir::new().unwrap(); + let workspace_root = workspace.path().join("workspace"); + let active_path = workspace_root.join("Core"); + let active_url = file_url(&active_path); + let active_555 = proposal_markdown(555, None, "Active proposal."); + let active_repo = init_repo(&active_path, &[("content/00555.md", active_555.as_str())]); + write_repo_manifest_file(&active_path, "Core", &active_url, &[]); + commit_all(&active_repo, "add manifest"); + std::fs::create_dir(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + write_file( + &workspace_root, + config::LOCAL_CONFIG_FILE, + r#" +[render] +only = [555] +"#, + ); + + let args = parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "serve"]); + let resolved = resolve_execution(&args).unwrap(); + + assert_eq!( + resolved + .only + .unwrap() + .into_iter() + .map(|number| number.get()) + .collect::>(), + vec![555] + ); +} + +#[test] +fn only_serve_startup_uses_build_filtering() { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("Core"); + let active_url = file_url(&active_path); + let selected_555 = proposal_markdown(555, None, "Selected proposal."); + let unselected_678 = proposal_markdown(678, Some("ERC"), "Unselected proposal."); + let active_repo = init_repo( + &active_path, + &[ + ("content/00555.md", selected_555.as_str()), + ("content/00678.md", unselected_678.as_str()), + ], + ); + write_repo_manifest_file(&active_path, "Core", &active_url, &[]); + commit_all(&active_repo, "add manifest"); + std::fs::create_dir(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + write_file( + &workspace_root, + config::LOCAL_CONFIG_FILE, + &config::default_workspace_config_text(), + ); + + let args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "serve", + "--only", + "555", + ]); + let resolved = resolve_execution(&args).unwrap(); + let repo_path = materialize_resolved_repo(&resolved); + preprocess_and_prune_only(&repo_path, resolved.only.clone().unwrap()); + + assert!(repo_path.join("content/00555.md").is_file()); + assert!(!repo_path.join("content/00678.md").exists()); +} + +#[test] +fn normal_serve_after_only_restores_full_materialized_content_tree() { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("Core"); + let active_url = file_url(&active_path); + let selected_555 = proposal_markdown(555, None, "Selected proposal."); + let unselected_678 = proposal_markdown(678, Some("ERC"), "Unselected proposal."); + let active_repo = init_repo( + &active_path, + &[ + ("content/00555.md", selected_555.as_str()), + ("content/00678.md", unselected_678.as_str()), + ], + ); + write_repo_manifest_file(&active_path, "Core", &active_url, &[]); + commit_all(&active_repo, "add manifest"); + std::fs::create_dir(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + write_file( + &workspace_root, + config::LOCAL_CONFIG_FILE, + &config::default_workspace_config_text(), + ); + let build_root = temp.path().join("build-root"); + + let only_args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "--build-root", + build_root.to_str().unwrap(), + "serve", + "--only", + "555", + ]); + let only_resolved = resolve_execution(&only_args).unwrap(); + let repo_path = materialize_resolved_repo(&only_resolved); + preprocess_and_prune_only(&repo_path, only_resolved.only.clone().unwrap()); + + assert!(repo_path.join("content/00555.md").is_file()); + assert!(!repo_path.join("content/00678.md").exists()); + + let normal_args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "--build-root", + build_root.to_str().unwrap(), + "serve", + ]); + let normal_resolved = resolve_execution(&normal_args).unwrap(); + assert!(normal_resolved.only.is_none()); + let restored_repo_path = materialize_resolved_repo(&normal_resolved); + markdown::preprocess(&restored_repo_path.join(CONTENT_DIR), None).unwrap(); + + assert!(restored_repo_path.join("content/00555.md").is_file()); + assert!(restored_repo_path.join("content/00678.md").is_file()); +} diff --git a/src/workspace.rs b/src/workspace.rs index 6892655..bd76562 100644 --- a/src/workspace.rs +++ b/src/workspace.rs @@ -858,7 +858,7 @@ base_url = "https://staging.example.test/{sibling_id}/" } #[test] - fn workspace_doc_text_mentions_setup_reference_content() { + fn workspace_doc_text_mentions_required_workspace_reference_content() { let text = workspace_doc_text(); for expected in [ @@ -868,12 +868,13 @@ base_url = "https://staging.example.test/{sibling_id}/" "build-eips doctor", "build-eips build", "build-eips serve", - "--platform-dev", - "preprocessor/", - "eipw/", + "build-eips preview", + "build-eips editorial check", "[render]", "only = [", "--only", + "--remote-siblings", + "--base-url", ] { assert!( text.contains(expected), diff --git a/src/workspace_doc.md b/src/workspace_doc.md index 01b892f..d2216be 100644 --- a/src/workspace_doc.md +++ b/src/workspace_doc.md @@ -94,17 +94,44 @@ If `theme/` or `preprocessor/` setup cannot create the default `EIPs/` checkout, If `build-eips doctor` reports that Zola is missing or too old, rerun the setup script to install the supported Zola version. -## Local Commands +### Build And Serve Locally -Use the active proposal repo for local build commands: +Build the full static site, then preview that built output: -```sh -build-eips check +```bash build-eips build +build-eips preview +``` + +`preview` serves the last output written by `build`. Run `build` again before `preview` when you want to inspect fresh output. + +Use `serve` when you want a live development server that livereloads changes instead of a reusable build output: + +```bash build-eips serve ``` -The workspace config starts with local server and site defaults: +`serve` runs a fresh temporary site build each time it is invoked (without using `build`), starts a local development server, and watches tracked local edits. Its output cannot be reused by `preview`. + +Use `check` to quickly validate whether the site will build cleanly without producing the full built site: + +```bash +build-eips check +``` + +By default, `check`, `build`, and `serve` use the local workspace in dirty mode, which includes tracked working-tree edits from this repo. `preview` serves the last output written by `build`. Use `--clean` when you want to ignore tracked local proposal edits for one command: + +```bash +build-eips check --clean +build-eips build --clean +build-eips serve --clean +``` + +For staging, production, parity, and remote-sibling modes, see `../WORKSPACE.md`. + +### Local Settings + +Local build settings live in `../.build-eips.toml`, which the setup script generates. Use that workspace file to change the local server address or local site URL: ```toml [server] @@ -115,29 +142,128 @@ port = 1111 base_url = "http://127.0.0.1:1111" ``` -## Render Specific Proposals Only +`serve` and `preview` use `[server]` for the local bind address. `build` and `serve` use `[site].base_url` when generating links. -Full local `build` and `serve` runs can take time because they process every -proposal file. When you want to quickly test a single proposal or a specific -batch, add a list of desired proposal numbers to the workspace -`.build-eips.toml`: +CLI flags such as `--host`, `--port`, and `--base-url` override the workspace config for one run: + +```bash +build-eips serve --host 0.0.0.0 --port 3000 --base-url http://127.0.0.1:3000 +``` + +### Render Specific Proposals Only + +Full local `build` and `serve` runs can take time because they process every proposal file. When you want to quickly test a single proposal or a specific batch, add a list of desired proposal numbers to the workspace `.build-eips.toml`: ```toml [render] only = [555, 678] ``` -Whenever `[render].only` is populated, regular local dirty `build` and `serve` -commands render only those proposal pages. Links and references to excluded -proposals are rewritten to the canonical public site. +Add one or more proposal numbers in `[render].only`, separated by commas. It's empty by default, but whenever it is populated, the regular `build` and `serve` commands render only those proposal pages. Links and references to excluded proposals are rewritten to the canonical public site. -Use CLI `--only` when you want a one-run target list; it overrides any -proposals in `[render].only` for that run: +Use CLI `--only` when you want a one-run target list; it also overrides any proposals in `[render].only` for that run: -```sh +```bash build-eips serve --only 555 build-eips build --only 555 build-eips build --only 555 678 ``` Multiple proposal numbers in the CLI are space-separated; no commas. + +### Editorial Validation + +Use editorial commands to validate proposal files before opening or updating a pull request. + +- `editorial lint` runs targeted `eipw` proposal-rule checks. +- `editorial check` runs `editorial lint`, then checks that the selected proposal changes will not prevent the full site from building cleanly. + +Check one or more specific proposals by number: + +```bash +build-eips editorial check 1 +build-eips editorial check 1 123 +``` + +For the closest match to PR CI, use `editorial check` against the proposal files changed versus upstream: + +```bash +build-eips --staging editorial check --against-upstream --format github +``` + +Both commands accept the same selector modes: + +* proposal numbers or repo-relative proposal paths for explicit targets +* `--working-tree` for tracked dirty proposal files +* `--against-upstream` for proposal files changed versus the upstream merge-base +* `--batch ` for a repeatable target list + +They also accept `eipw` options such as `--format github`. + +Use a batch file when you want to lint or check the same proposal set repeatedly. A batch file is a plain text file with one proposal number per line: + +```txt +1 +7949 +``` + +```bash +build-eips editorial lint --batch ../editor-batch.txt +build-eips editorial check --batch ../editor-batch.txt +``` + +### Source And Output Overrides + +Workspace-local sources come from the standard workspace layout. The local theme is `workspace/theme`, and local sibling repos are `workspace/` from the active repo manifest. + +Use `--remote-siblings` when you need to force remote sibling proposal sources for a single command. + +Use global `--build-root ` when you want a separate prepared repo and output directory, for example to compare two builds side by side. The path replaces the default `.local-build/` location for each command where you pass it, so use the same `--build-root` value when serving or previewing builds. + +Example: + +```bash +build-eips -C /work/EIPs-project/EIPs --build-root /tmp/eips-local build --base-url http://127.0.0.1:1111 +build-eips -C /work/EIPs-project/EIPs --build-root /tmp/eips-staging --staging build --base-url http://127.0.0.1:1112 + +build-eips -C /work/EIPs-project/EIPs --build-root /tmp/eips-local preview --port 1111 +build-eips -C /work/EIPs-project/EIPs --build-root /tmp/eips-staging preview --port 1112 + +# Or using serve +build-eips -C /work/EIPs-project/EIPs --build-root /tmp/eips-local serve --port 1111 +build-eips -C /work/EIPs-project/EIPs --build-root /tmp/eips-staging --staging serve --port 1112 +``` + +### Remote And Parity Modes + +Use remote modes when you want a clean render of the local active checkout with staging or production environment metadata and remote sibling proposal sources. + +`--staging` and `--production` use the local active checkout, reject dirty active-repo edits, and select remote sibling sources plus staging or production environment metadata: + +```sh +build-eips --staging check +build-eips --staging build +build-eips --staging serve + +build-eips --production check +build-eips --production build +build-eips --production serve +``` + +`parity` is the built-in clean staging path for checking whether the local active checkout behaves like the staging environment: + +```sh +build-eips parity check +build-eips parity build +build-eips parity serve +``` + +Use `--remote-siblings` when you want to keep the active proposal repo local, but resolve sibling proposal repos from the configured remote environment: + +```sh +build-eips --remote-siblings check +build-eips --remote-siblings build +build-eips --remote-siblings serve +``` + +Remote environment commands and `parity` use the local active checkout, but do not use local dirty proposal edits. They still use the workspace-local `theme/`, so check out the theme commit or branch you want before running them. diff --git a/src/zola.rs b/src/zola.rs index 125a166..eb185af 100644 --- a/src/zola.rs +++ b/src/zola.rs @@ -250,12 +250,11 @@ mod tests { process::{Command, ExitStatus}, }; - use tempfile::TempDir; - use crate::{ config::ServerBinding, layout::{mounted_theme_path, theme_config_path}, }; + use tempfile::TempDir; use super::{find_zola, mount_theme, serve_args}; @@ -311,21 +310,6 @@ mod tests { assert!(zola_build_status(&external).success()); } - #[test] - fn mounted_theme_paths_are_under_project_themes_directory() { - let project_path = PathBuf::from("/tmp/project"); - let mounted_theme = mounted_theme_path(&project_path); - - assert_eq!( - mounted_theme, - PathBuf::from("/tmp/project/themes/eips-theme") - ); - assert_eq!( - theme_config_path(&mounted_theme), - PathBuf::from("/tmp/project/themes/eips-theme/config/zola.toml") - ); - } - #[test] fn serve_args_include_configured_interface_and_port() { let server_binding = ServerBinding { @@ -382,6 +366,21 @@ mod tests { ); } + #[test] + fn mounted_theme_paths_are_under_project_themes_directory() { + let project_path = PathBuf::from("/tmp/project"); + let mounted_theme = mounted_theme_path(&project_path); + + assert_eq!( + mounted_theme, + PathBuf::from("/tmp/project/themes/eips-theme") + ); + assert_eq!( + theme_config_path(&mounted_theme), + PathBuf::from("/tmp/project/themes/eips-theme/config/zola.toml") + ); + } + #[test] fn mount_theme_does_not_symlink_mounted_local_theme_onto_itself() { let temp = TempDir::new().unwrap(); From f5425412e6c291dbb8c61958dc544375c51d1a12 Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Thu, 7 May 2026 00:21:42 -0400 Subject: [PATCH 18/20] Preserve section front matter during preprocessing Parse section index front matter directly as YAML before writing the generated Zola TOML front matter. This preserves structured section metadata such as extra.homepage_badges instead of flattening nested YAML through the proposal preamble parser. Keep proposal markdown on the existing proposal preamble path, and keep body link rewriting active for section index pages. --- Cargo.lock | 1 + Cargo.toml | 1 + src/markdown.rs | 59 +++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ee2306e..496a691 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -270,6 +270,7 @@ dependencies = [ "semver", "serde", "serde_json", + "serde_yaml", "snafu", "tempfile", "tiny_http", diff --git a/Cargo.toml b/Cargo.toml index 4bbfb89..17652cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ regex = "1.12.2" semver = {version = "1.0.27", features = ["serde"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.148" +serde_yaml = "0.9.34" snafu = { version = "0.8.9", features = ["rust_1_81"] } tiny_http = "0.12.0" tokio = { version = "1.48.0", features = ["fs", "rt", "macros"] } diff --git a/src/markdown.rs b/src/markdown.rs index 6d02a6f..9d111d7 100644 --- a/src/markdown.rs +++ b/src/markdown.rs @@ -233,6 +233,10 @@ fn write_file(path: &Path, front_matter: FrontMatter, body: &str) -> std::io::Re Ok(()) } +fn is_section_index_path(path: &Path) -> bool { + path.file_name() == Some(OsStr::new("_index.md")) +} + lazy_static! { // Matches GitHub usernames. static ref RE_GITHUB: Regex = Regex::new(r"^([^()<>,@]+) \(@([a-zA-Z\d-]+)\)$").unwrap(); @@ -1353,14 +1357,26 @@ fn process_eip( let body = transform_markdown(root, path, body, only_plan) .with_whatever_context(|_| format!("unable to transform markdown for `{path_lossy}`"))?; + if is_section_index_path(path) { + let front_matter = + serde_yaml::from_str::(preamble).with_whatever_context(|_| { + format!("couldn't parse section front matter in `{}`", path_lossy) + })?; + + match write_file(path, front_matter, &body) { + Ok(()) => {} + Err(error) if missing_path_mode.should_ignore_io_error(&error) => return Ok(()), + Err(error) => return Err(error).whatever_context("couldn't write file"), + } + + return Ok(()); + } + let preamble = Preamble::parse(Some(&path_lossy), preamble) .ok() .with_whatever_context(|| format!("couldn't parse preamble in `{}`", path_lossy))?; - let updated = match path.file_name() { - Some(x) if x == "_index.md" => None, - _ => Some(last_modified(path)?), - }; + let updated = Some(last_modified(path)?); let mut front_matter = FrontMatter { updated, @@ -2342,6 +2358,41 @@ mod tests { assert!(!body.contains("@/00678.md")); } + #[test] + fn preprocess_preserves_section_extra_front_matter_and_rewrites_body_links() { + let (_temp, content) = content_repo(&[ + ( + "_index.md", + r#"--- +title: Home +extra: + homepage_badges: + - href: https://discord.gg/9FxN6CfaQR + image: https://dcbadge.limes.pink/api/server/9FxN6CfaQR?style=flat + alt: Badge for ERCRef Discord channel +--- +See [EIP-678](/00678.md). +"# + .to_owned(), + ), + ("00555.md", proposal_markdown(555, None, "", "Selected.")), + ("00678.md", proposal_markdown(678, None, "", "Unselected.")), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let front_matter = rendered_front_matter(&content.join("_index.md")); + let badges = front_matter["extra"]["homepage_badges"].as_array().unwrap(); + assert_eq!( + badges[0]["href"].as_str().unwrap(), + "https://discord.gg/9FxN6CfaQR" + ); + let body = rendered_body(&content.join("_index.md")); + assert!(body.contains("https://eips.ethereum.org/EIPS/eip-678")); + assert!(!body.contains("@/00678.md")); + } + #[test] fn targeted_preprocess_paths_rewrites_retained_non_proposal_markdown_with_plan() { let (_temp, content) = content_repo(&[ From cefbda1e5aa0b2c85782b5fdc766a968131c563f Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Sun, 10 May 2026 15:36:12 -0400 Subject: [PATCH 19/20] Generate proposal metadata JSON Generate static/assets/data/proposals.json from prepared proposal sources during the runtime build pipeline. Collect proposal metadata through a shared catalog so JSON writing and future prepared-runtime data passes use the same path validation, preamble parsing, duplicate detection, targeted URL policy, and stable proposal ordering. Preserve the existing JSON shape, active repository prefix selection, pretty formatting, omitted optional fields, and output-collision protection. --- src/main.rs | 2 + src/pipeline.rs | 59 ++++- src/proposal.rs | 33 ++- src/proposal_catalog.rs | 520 +++++++++++++++++++++++++++++++++++++ src/proposal_metadata.rs | 546 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 1150 insertions(+), 10 deletions(-) create mode 100644 src/proposal_catalog.rs create mode 100644 src/proposal_metadata.rs diff --git a/src/main.rs b/src/main.rs index e78c69e..029e488 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,6 +22,8 @@ mod preview; mod print; mod progress; mod proposal; +mod proposal_catalog; +mod proposal_metadata; mod serve; mod workspace; mod zola; diff --git a/src/pipeline.rs b/src/pipeline.rs index 6638746..d86f9cf 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -18,6 +18,7 @@ use crate::{ layout::{mounted_theme_path, output_path, CONTENT_DIR, REPO_DIR}, markdown, proposal::OnlyRenderPlan, + proposal_metadata, serve::{serve_sync_config, DirtyServeWatcher, LocalThemeServeSync}, zola, }; @@ -111,6 +112,12 @@ impl Prepared { .map(|selected_numbers| OnlyRenderPlan::build(&content_path, selected_numbers)) .transpose() .whatever_context("unable to build targeted render plan")?; + proposal_metadata::write_proposal_metadata_json( + &repo_path, + &repository_use.title, + only_plan.as_ref(), + ) + .whatever_context("unable to write proposal metadata JSON")?; markdown::preprocess(&content_path, only_plan.as_ref()) .whatever_context("unable to preprocess markdown")?; if let Some(only_plan) = &only_plan { @@ -204,7 +211,8 @@ mod tests { editorial::editorial_runtime_execution, execution::{resolve_execution, ResolvedExecution}, git::SourceMaterialization, - layout::{mounted_theme_path, theme_config_path, REPO_DIR}, + layout::{mounted_theme_path, theme_config_path, CONTENT_DIR, REPO_DIR}, + proposal_catalog::collect_proposal_catalog, }; use super::{prepare_runtime_source, prepare_theme_for_zola}; @@ -298,6 +306,15 @@ base_url = "https://staging.example.test/{sibling_id}/" manifest } + fn pipeline_proposal_markdown(number: u32, category: Option<&str>, body: &str) -> String { + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!( + "---\neip: {number}\ntitle: Proposal {number}\nstatus: Draft\ntype: Standards Track\n{category}---\n\n{body}\n" + ) + } + fn runtime_workspace(with_sibling: bool) -> RuntimeWorkspace { let temp = TempDir::new().unwrap(); let workspace_root = temp.path().join("workspace"); @@ -484,4 +501,44 @@ base_url = "https://staging.example.test/{sibling_id}/" "active proposal\n" ); } + + #[test] + fn proposal_catalog_collection_uses_prepared_merged_sources() { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("EIPs"); + let sibling_path = workspace_root.join("ERCs"); + let active_url = file_url(&active_path); + let sibling_url = file_url(&sibling_path); + let manifest = repo_manifest_text("EIPs", &active_url, &[("ERCs", sibling_url)]); + let active_markdown = pipeline_proposal_markdown(1, None, "Active proposal."); + let sibling_markdown = pipeline_proposal_markdown(2, Some("ERC"), "Sibling proposal."); + + write_file(&workspace_root, config::LOCAL_CONFIG_FILE, ""); + std::fs::create_dir_all(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + let _active_repo = init_repo( + &active_path, + &[ + (config::REPO_MANIFEST_FILE, manifest.as_str()), + ("content/00001.md", active_markdown.as_str()), + ], + ); + let _sibling_repo = init_repo( + &sibling_path, + &[("content/00002.md", sibling_markdown.as_str())], + ); + let workspace = RuntimeWorkspace { + _temp: temp, + active_path, + }; + let resolved = resolved_runtime(&workspace, &["build"]); + + prepare_resolved_source(&resolved).unwrap(); + let catalog = + collect_proposal_catalog(&prepared_path(&resolved, CONTENT_DIR), None).unwrap(); + let records = catalog.into_records(); + + assert!(!resolved.root_path.join("content/00002.md").exists()); + assert_eq!(records["erc-2"].title, "Proposal 2"); + } } diff --git a/src/proposal.rs b/src/proposal.rs index 787f3ba..f9392df 100644 --- a/src/proposal.rs +++ b/src/proposal.rs @@ -201,13 +201,13 @@ struct ProposalAssetInventoryEntry { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ProposalPublicSite { +pub(crate) enum ProposalPublicSite { Eips, Ercs, } impl ProposalPublicSite { - fn proposal_url(self, proposal_number: ProposalNumber) -> String { + pub(crate) fn proposal_url(self, proposal_number: ProposalNumber) -> String { match self { Self::Eips => format!( "https://eips.ethereum.org/EIPS/eip-{}", @@ -500,6 +500,10 @@ impl OnlyRenderPlan { Some(public_asset_url(entry)) } + pub(crate) fn is_selected_number(&self, proposal_number: ProposalNumber) -> bool { + self.selected_numbers.contains(&proposal_number) + } + pub(crate) fn external_url_for_canonical_target( &self, canonical_target: &Path, @@ -726,27 +730,38 @@ fn remove_dir_if_present(path: &Path) -> Result<(), Whatever> { } } -fn public_site_for_markdown( +pub(crate) fn parse_proposal_preamble<'a>( markdown_path: &Path, - contents: &str, -) -> Result { + contents: &'a str, +) -> Result, Whatever> { let path_lossy = markdown_path.to_string_lossy(); let (preamble, _) = Preamble::split(contents) .with_whatever_context(|_| format!("couldn't split preamble for `{path_lossy}`"))?; - let preamble = Preamble::parse(Some(&path_lossy), preamble) + Preamble::parse(None, preamble) .ok() - .with_whatever_context(|| format!("couldn't parse preamble in `{path_lossy}`"))?; + .with_whatever_context(|| format!("couldn't parse preamble in `{path_lossy}`")) +} + +fn public_site_for_preamble(preamble: &Preamble<'_>) -> ProposalPublicSite { let is_erc = preamble .fields() .any(|field| field.name() == "category" && field.value().trim() == "ERC"); if is_erc { - Ok(ProposalPublicSite::Ercs) + ProposalPublicSite::Ercs } else { - Ok(ProposalPublicSite::Eips) + ProposalPublicSite::Eips } } +pub(crate) fn public_site_for_markdown( + markdown_path: &Path, + contents: &str, +) -> Result { + let preamble = parse_proposal_preamble(markdown_path, contents)?; + Ok(public_site_for_preamble(&preamble)) +} + #[allow(dead_code)] fn public_asset_url(entry: &ProposalAssetInventoryEntry) -> String { let mut url = format!( diff --git a/src/proposal_catalog.rs b/src/proposal_catalog.rs new file mode 100644 index 0000000..0a68992 --- /dev/null +++ b/src/proposal_catalog.rs @@ -0,0 +1,520 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Shared proposal catalog collection from prepared content sources. + +use std::{ + collections::BTreeMap, + fmt, + path::{Path, PathBuf}, +}; + +use eipw_preamble::Preamble; +use serde::Serialize; +use snafu::{ResultExt, Whatever}; + +use crate::proposal::{ + flat_proposal_number, parse_proposal_preamble, path_component_proposal_number, + public_site_for_markdown, OnlyRenderPlan, ProposalNumber, ProposalPublicSite, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub(crate) enum ProposalCatalogPrefix { + #[serde(rename = "EIP")] + Eip, + #[serde(rename = "ERC")] + Erc, +} + +impl ProposalCatalogPrefix { + pub(crate) fn key(self, proposal_number: ProposalNumber) -> String { + format!("{self}-{proposal_number}").to_ascii_lowercase() + } +} + +impl From for ProposalCatalogPrefix { + fn from(site: ProposalPublicSite) -> Self { + match site { + ProposalPublicSite::Eips => Self::Eip, + ProposalPublicSite::Ercs => Self::Erc, + } + } +} + +impl fmt::Display for ProposalCatalogPrefix { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Eip => formatter.write_str("EIP"), + Self::Erc => formatter.write_str("ERC"), + } + } +} + +#[derive(Debug, Clone, Serialize)] +pub(crate) struct ProposalCatalogRecord { + pub(crate) number: ProposalNumber, + pub(crate) prefix: ProposalCatalogPrefix, + pub(crate) title: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) description: Option, + pub(crate) status: String, + #[serde(rename = "type")] + pub(crate) proposal_type: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) category: Option, + pub(crate) url: String, +} + +#[derive(Debug, Clone)] +pub(crate) struct ProposalCatalog { + records: BTreeMap, +} + +impl ProposalCatalog { + pub(crate) fn into_records(self) -> BTreeMap { + self.records + } +} + +#[derive(Debug)] +struct CollectedProposalCatalogRecord { + key: String, + source_path: PathBuf, + record: ProposalCatalogRecord, +} + +#[derive(Debug)] +struct ProposalCatalogFields { + title: String, + description: Option, + status: String, + proposal_type: String, + category: Option, +} + +impl ProposalCatalogFields { + fn parse( + markdown_path: &Path, + proposal_number: ProposalNumber, + preamble: &Preamble<'_>, + ) -> Result { + validate_preamble_proposal_number(markdown_path, proposal_number, preamble)?; + + Ok(Self { + title: required_catalog_field(markdown_path, preamble, "title")?, + description: optional_catalog_field(preamble, "description"), + status: required_catalog_field(markdown_path, preamble, "status")?, + proposal_type: required_catalog_field(markdown_path, preamble, "type")?, + category: optional_catalog_field(preamble, "category"), + }) + } +} + +struct ContentRootEntry { + entry_path: PathBuf, + file_type: std::fs::FileType, +} + +pub(crate) fn collect_proposal_catalog( + content_root: &Path, + only_plan: Option<&OnlyRenderPlan>, +) -> Result { + let mut records = BTreeMap::::new(); + + for entry in sorted_content_root_entries(content_root)? { + if entry.file_type.is_file() { + let Some(proposal_number) = flat_proposal_number(&entry.entry_path) else { + continue; + }; + insert_collected_proposal_catalog_record( + &mut records, + collect_proposal_catalog_from_path( + content_root, + proposal_number, + &entry.entry_path, + only_plan, + )?, + )?; + } else if entry.file_type.is_dir() { + let Some(proposal_number) = + path_component_proposal_number(entry.entry_path.file_name()) + else { + continue; + }; + let index_path = entry.entry_path.join("index.md"); + match std::fs::read_to_string(&index_path) { + Ok(contents) => { + insert_collected_proposal_catalog_record( + &mut records, + collect_proposal_catalog_from_contents( + content_root, + proposal_number, + &index_path, + &contents, + only_plan, + )?, + )?; + } + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => {} + Err(error) => { + snafu::whatever!( + "unable to read proposal markdown `{}`: {error}", + index_path.to_string_lossy() + ); + } + } + } + } + + Ok(ProposalCatalog { + records: records + .into_iter() + .map(|(key, metadata)| (key, metadata.record)) + .collect(), + }) +} + +fn sorted_content_root_entries(content_root: &Path) -> Result, Whatever> { + let entries = std::fs::read_dir(content_root).with_whatever_context(|_| { + format!( + "unable to read materialized content directory `{}` for proposal metadata", + content_root.to_string_lossy() + ) + })?; + + let mut entries = entries + .map(|entry| -> Result { + let entry = entry.with_whatever_context(|_| { + format!( + "unable to read materialized content directory entry in `{}` for proposal metadata", + content_root.to_string_lossy() + ) + })?; + let entry_path = entry.path(); + let file_type = entry.file_type().with_whatever_context(|_| { + format!( + "unable to inspect materialized content path `{}` for proposal metadata", + entry_path.to_string_lossy() + ) + })?; + + Ok(ContentRootEntry { + entry_path, + file_type, + }) + }) + .collect::, Whatever>>()?; + entries.sort_by(|left, right| left.entry_path.cmp(&right.entry_path)); + Ok(entries) +} + +fn collect_proposal_catalog_from_path( + content_root: &Path, + proposal_number: ProposalNumber, + markdown_path: &Path, + only_plan: Option<&OnlyRenderPlan>, +) -> Result { + let contents = std::fs::read_to_string(markdown_path).with_whatever_context(|_| { + format!( + "unable to read proposal markdown `{}`", + markdown_path.to_string_lossy() + ) + })?; + collect_proposal_catalog_from_contents( + content_root, + proposal_number, + markdown_path, + &contents, + only_plan, + ) +} + +fn collect_proposal_catalog_from_contents( + content_root: &Path, + proposal_number: ProposalNumber, + markdown_path: &Path, + contents: &str, + only_plan: Option<&OnlyRenderPlan>, +) -> Result { + markdown_path + .strip_prefix(content_root) + .with_whatever_context(|_| { + format!( + "proposal markdown `{}` is outside content root `{}`", + markdown_path.to_string_lossy(), + content_root.to_string_lossy() + ) + })?; + + let site = public_site_for_markdown(markdown_path, contents)?; + let preamble = parse_proposal_preamble(markdown_path, contents)?; + let prefix = ProposalCatalogPrefix::from(site); + let fields = ProposalCatalogFields::parse(markdown_path, proposal_number, &preamble)?; + + Ok(CollectedProposalCatalogRecord { + key: prefix.key(proposal_number), + source_path: markdown_path.to_path_buf(), + record: ProposalCatalogRecord { + number: proposal_number, + prefix, + title: fields.title, + description: fields.description, + status: fields.status, + proposal_type: fields.proposal_type, + category: fields.category, + url: catalog_record_url(proposal_number, site, only_plan), + }, + }) +} + +fn insert_collected_proposal_catalog_record( + records: &mut BTreeMap, + metadata: CollectedProposalCatalogRecord, +) -> Result<(), Whatever> { + if let Some(existing) = records.get(&metadata.key) { + snafu::whatever!( + "duplicate proposal metadata key `{}` from `{}` and `{}`", + metadata.key, + existing.source_path.to_string_lossy(), + metadata.source_path.to_string_lossy() + ); + } + + records.insert(metadata.key.clone(), metadata); + Ok(()) +} + +fn catalog_record_url( + proposal_number: ProposalNumber, + site: ProposalPublicSite, + only_plan: Option<&OnlyRenderPlan>, +) -> String { + if only_plan.is_some_and(|plan| !plan.is_selected_number(proposal_number)) { + site.proposal_url(proposal_number) + } else { + format!("/{}/", proposal_number.get()) + } +} + +fn required_catalog_field( + markdown_path: &Path, + preamble: &Preamble<'_>, + field_name: &str, +) -> Result { + let Some(field) = preamble.by_name(field_name) else { + snafu::whatever!( + "missing required proposal metadata field `{field_name}` in `{}`", + markdown_path.to_string_lossy() + ); + }; + let value = field.value().trim(); + if value.is_empty() { + snafu::whatever!( + "missing required proposal metadata field `{field_name}` in `{}`", + markdown_path.to_string_lossy() + ); + } + + Ok(value.to_owned()) +} + +fn optional_catalog_field(preamble: &Preamble<'_>, field_name: &str) -> Option { + preamble + .by_name(field_name) + .map(|field| field.value().trim()) + .filter(|value| !value.is_empty()) + .map(str::to_owned) +} + +fn validate_preamble_proposal_number( + markdown_path: &Path, + path_proposal_number: ProposalNumber, + preamble: &Preamble<'_>, +) -> Result<(), Whatever> { + for field in preamble + .fields() + .filter(|field| matches!(field.name(), "eip" | "number")) + { + let field_name = field.name(); + let field_value = field.value().trim(); + let parsed = field_value.parse::().with_whatever_context(|_| { + format!( + "couldn't parse proposal number field `{field_name}` in `{}`", + markdown_path.to_string_lossy() + ) + })?; + let Ok(preamble_proposal_number) = ProposalNumber::from_u32(parsed) else { + snafu::whatever!( + "proposal number field `{field_name}` in `{}` must be positive", + markdown_path.to_string_lossy() + ); + }; + + if preamble_proposal_number != path_proposal_number { + snafu::whatever!( + "proposal metadata number mismatch in `{}`: path indicates `{path_proposal_number}`, but `{field_name}` contains `{preamble_proposal_number}`", + markdown_path.to_string_lossy() + ); + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use tempfile::TempDir; + + use super::{collect_proposal_catalog, ProposalCatalogPrefix}; + use crate::proposal::{OnlyRenderPlan, ProposalNumber}; + + fn number(value: u32) -> ProposalNumber { + ProposalNumber::from_u32(value).unwrap() + } + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn catalog_proposal_markdown( + number: u32, + title: &str, + category: Option<&str>, + description: Option<&str>, + ) -> String { + let description = description + .map(|description| format!("description: {description}\n")) + .unwrap_or_default(); + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!( + "---\neip: {number}\ntitle: {title}\n{description}status: Final\ntype: Standards Track\n{category}---\nBody\n" + ) + } + + #[test] + fn proposal_catalog_collects_flat_and_directory_proposals() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "020.md", + &catalog_proposal_markdown(20, "Flat Proposal", None, None), + ); + write_file( + temp.path(), + "21/index.md", + &catalog_proposal_markdown(21, "Directory Proposal", Some("ERC"), Some("Desc")), + ); + + let catalog = collect_proposal_catalog(temp.path(), None).unwrap(); + let records = catalog.into_records(); + + assert_eq!(records.len(), 2); + assert_eq!(records["eip-20"].number, number(20)); + assert_eq!(records["eip-20"].prefix, ProposalCatalogPrefix::Eip); + assert_eq!(records["eip-20"].title, "Flat Proposal"); + assert_eq!(records["eip-20"].url, "/20/"); + assert_eq!(records["erc-21"].number, number(21)); + assert_eq!(records["erc-21"].prefix, ProposalCatalogPrefix::Erc); + assert_eq!(records["erc-21"].description.as_deref(), Some("Desc")); + } + + #[test] + fn proposal_catalog_reports_malformed_preambles() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "001.md", "not front matter\n"); + + let error = collect_proposal_catalog(temp.path(), None) + .unwrap_err() + .to_string(); + + assert!(error.contains("couldn't split preamble")); + assert!(error.contains("001.md")); + } + + #[test] + fn proposal_catalog_reports_path_preamble_number_mismatch() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "001.md", + "---\neip: 2\ntitle: Proposal 1\nstatus: Final\ntype: Standards Track\n---\nBody\n", + ); + + let error = collect_proposal_catalog(temp.path(), None) + .unwrap_err() + .to_string(); + + assert!(error.contains("proposal metadata number mismatch")); + assert!(error.contains("001.md")); + assert!(error.contains("path indicates `1`")); + assert!(error.contains("`eip` contains `2`")); + } + + #[test] + fn proposal_catalog_collects_independently_of_json_writing() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "001.md", + &catalog_proposal_markdown(1, "Proposal 1", None, None), + ); + + let catalog = collect_proposal_catalog(temp.path(), None).unwrap(); + let records = catalog.into_records(); + + assert_eq!(records["eip-1"].status, "Final"); + assert!(!temp + .path() + .join("static/assets/data/proposals.json") + .exists()); + } + + #[test] + fn proposal_catalog_applies_targeted_url_policy() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "020.md", + &catalog_proposal_markdown(20, "Proposal 20", None, None), + ); + write_file( + temp.path(), + "021.md", + &catalog_proposal_markdown(21, "Proposal 21", None, None), + ); + write_file( + temp.path(), + "022.md", + &catalog_proposal_markdown(22, "Proposal 22", Some("ERC"), None), + ); + let plan = OnlyRenderPlan::build(temp.path(), [number(20)].into_iter().collect()).unwrap(); + + let catalog = collect_proposal_catalog(temp.path(), Some(&plan)).unwrap(); + let records = catalog.into_records(); + + assert_eq!(records["eip-20"].url, "/20/"); + assert_eq!( + records["eip-21"].url, + "https://eips.ethereum.org/EIPS/eip-21" + ); + assert_eq!( + records["erc-22"].url, + "https://ercs.ethereum.org/ERCS/erc-22" + ); + } +} diff --git a/src/proposal_metadata.rs b/src/proposal_metadata.rs new file mode 100644 index 0000000..60573e7 --- /dev/null +++ b/src/proposal_metadata.rs @@ -0,0 +1,546 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Proposal metadata JSON generation for theme popovers. + +use std::{ + collections::BTreeMap, + io::Write, + path::{Path, PathBuf}, +}; + +use serde::Serialize; +use snafu::{OptionExt, ResultExt, Whatever}; + +use crate::{ + layout::CONTENT_DIR, + proposal::OnlyRenderPlan, + proposal_catalog::{collect_proposal_catalog, ProposalCatalogPrefix, ProposalCatalogRecord}, +}; + +const PROPOSAL_METADATA_SCHEMA_VERSION: u8 = 1; + +#[derive(Debug, Serialize)] +struct ProposalMetadataIndex { + schema_version: u8, + active_prefix: ProposalCatalogPrefix, + proposals: BTreeMap, +} + +pub(crate) fn write_proposal_metadata_json( + repo_path: &Path, + repository_title: &str, + only_plan: Option<&OnlyRenderPlan>, +) -> Result<(), Whatever> { + let json_path = proposal_metadata_json_path(repo_path); + ensure_proposal_metadata_output_available(&json_path)?; + + let metadata = ProposalMetadataIndex { + schema_version: PROPOSAL_METADATA_SCHEMA_VERSION, + active_prefix: active_proposal_metadata_prefix(repository_title)?, + proposals: collect_proposal_catalog(&repo_path.join(CONTENT_DIR), only_plan)? + .into_records(), + }; + + write_proposal_metadata_file(&json_path, &metadata) +} + +fn proposal_metadata_json_path(repo_path: &Path) -> PathBuf { + repo_path + .join("static") + .join("assets") + .join("data") + .join("proposals.json") +} + +fn ensure_proposal_metadata_output_available(json_path: &Path) -> Result<(), Whatever> { + match std::fs::metadata(json_path) { + Ok(_) => { + snafu::whatever!( + "proposal metadata output `{}` already exists; refusing to overwrite it", + json_path.to_string_lossy() + ); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => { + snafu::whatever!( + "unable to inspect proposal metadata output `{}`: {error}", + json_path.to_string_lossy() + ); + } + } +} + +fn active_proposal_metadata_prefix( + repository_title: &str, +) -> Result { + match repository_title { + "EIPs" => Ok(ProposalCatalogPrefix::Eip), + "ERCs" => Ok(ProposalCatalogPrefix::Erc), + _ => { + snafu::whatever!( + "unsupported active repository title `{repository_title}` for proposal metadata; expected `EIPs` or `ERCs`" + ); + } + } +} + +fn write_proposal_metadata_file( + json_path: &Path, + metadata: &ProposalMetadataIndex, +) -> Result<(), Whatever> { + let parent = json_path.parent().with_whatever_context(|| { + format!( + "proposal metadata output path `{}` has no parent directory", + json_path.to_string_lossy() + ) + })?; + std::fs::create_dir_all(parent).with_whatever_context(|_| { + format!( + "unable to create proposal metadata directory `{}`", + parent.to_string_lossy() + ) + })?; + + let mut file = match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(json_path) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + snafu::whatever!( + "proposal metadata output `{}` already exists; refusing to overwrite it", + json_path.to_string_lossy() + ); + } + Err(error) => { + snafu::whatever!( + "unable to create proposal metadata output `{}`: {error}", + json_path.to_string_lossy() + ); + } + }; + + serde_json::to_writer_pretty(&mut file, metadata).with_whatever_context(|_| { + format!( + "unable to write proposal metadata JSON `{}`", + json_path.to_string_lossy() + ) + })?; + file.write_all(b"\n").with_whatever_context(|_| { + format!( + "unable to finish proposal metadata JSON `{}`", + json_path.to_string_lossy() + ) + })?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use serde_json::{json, Value}; + use tempfile::TempDir; + + use super::{proposal_metadata_json_path, write_proposal_metadata_json}; + use crate::proposal::{OnlyRenderPlan, ProposalNumber}; + + fn number(value: u32) -> ProposalNumber { + ProposalNumber::from_u32(value).unwrap() + } + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn metadata_proposal_markdown( + number: u32, + title: &str, + category: Option<&str>, + description: Option<&str>, + ) -> String { + let description = description + .map(|description| format!("description: {description}\n")) + .unwrap_or_default(); + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!( + "---\neip: {number}\ntitle: {title}\n{description}status: Final\ntype: Standards Track\n{category}---\nBody\n" + ) + } + + fn write_metadata_json( + repo_path: &Path, + repository_title: &str, + only_plan: Option<&OnlyRenderPlan>, + ) -> Value { + write_proposal_metadata_json(repo_path, repository_title, only_plan).unwrap(); + serde_json::from_str( + &std::fs::read_to_string(proposal_metadata_json_path(repo_path)).unwrap(), + ) + .unwrap() + } + + #[test] + fn proposal_metadata_full_build_writes_json() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown( + 20, + "Token Standard", + Some("ERC"), + Some("A standard interface for tokens."), + ), + ); + + let metadata = write_metadata_json(temp.path(), "EIPs", None); + let proposal = &metadata["proposals"]["erc-20"]; + + assert_eq!(metadata["schema_version"], json!(1)); + assert_eq!(metadata["active_prefix"], json!("EIP")); + assert_eq!(proposal["number"], json!(20)); + assert_eq!(proposal["prefix"], json!("ERC")); + assert_eq!(proposal["title"], json!("Token Standard")); + assert_eq!( + proposal["description"], + json!("A standard interface for tokens.") + ); + assert_eq!(proposal["status"], json!("Final")); + assert_eq!(proposal["type"], json!("Standards Track")); + assert_eq!(proposal["category"], json!("ERC")); + assert_eq!(proposal["url"], json!("/20/")); + } + + #[test] + fn proposal_metadata_full_build_records_use_local_urls() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown(20, "Proposal 20", None, None), + ); + + let metadata = write_metadata_json(temp.path(), "EIPs", None); + + assert_eq!(metadata["proposals"]["eip-20"]["url"], json!("/20/")); + } + + #[test] + fn proposal_metadata_targeted_build_includes_pre_prune_proposals() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown(20, "Proposal 20", None, None), + ); + write_file( + temp.path(), + "content/021.md", + &metadata_proposal_markdown(21, "Proposal 21", None, None), + ); + write_file( + temp.path(), + "content/022.md", + &metadata_proposal_markdown(22, "Proposal 22", Some("ERC"), None), + ); + let plan = OnlyRenderPlan::build( + &temp.path().join("content"), + [number(20)].into_iter().collect(), + ) + .unwrap(); + + let metadata = write_metadata_json(temp.path(), "EIPs", Some(&plan)); + let proposals = metadata["proposals"].as_object().unwrap(); + + assert_eq!(proposals.len(), 3); + assert!(proposals.contains_key("eip-20")); + assert!(proposals.contains_key("eip-21")); + assert!(proposals.contains_key("erc-22")); + } + + #[test] + fn proposal_metadata_targeted_selected_records_use_local_urls() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown(20, "Proposal 20", None, None), + ); + write_file( + temp.path(), + "content/021.md", + &metadata_proposal_markdown(21, "Proposal 21", None, None), + ); + let plan = OnlyRenderPlan::build( + &temp.path().join("content"), + [number(20)].into_iter().collect(), + ) + .unwrap(); + + let metadata = write_metadata_json(temp.path(), "EIPs", Some(&plan)); + + assert_eq!(metadata["proposals"]["eip-20"]["url"], json!("/20/")); + } + + #[test] + fn proposal_metadata_targeted_omitted_records_use_public_urls() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown(20, "Proposal 20", None, None), + ); + write_file( + temp.path(), + "content/021.md", + &metadata_proposal_markdown(21, "Proposal 21", None, None), + ); + write_file( + temp.path(), + "content/022.md", + &metadata_proposal_markdown(22, "Proposal 22", Some("ERC"), None), + ); + let plan = OnlyRenderPlan::build( + &temp.path().join("content"), + [number(20)].into_iter().collect(), + ) + .unwrap(); + + let metadata = write_metadata_json(temp.path(), "EIPs", Some(&plan)); + + assert_eq!( + metadata["proposals"]["eip-21"]["url"], + json!("https://eips.ethereum.org/EIPS/eip-21") + ); + assert_eq!( + metadata["proposals"]["erc-22"]["url"], + json!("https://ercs.ethereum.org/ERCS/erc-22") + ); + } + + #[test] + fn proposal_metadata_active_prefix_comes_from_repository_title() { + for (repository_title, expected_prefix) in [("EIPs", "EIP"), ("ERCs", "ERC")] { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/001.md", + &metadata_proposal_markdown(1, "Proposal 1", None, None), + ); + + let metadata = write_metadata_json(temp.path(), repository_title, None); + + assert_eq!(metadata["active_prefix"], json!(expected_prefix)); + } + } + + #[test] + fn proposal_metadata_erc_category_produces_erc_prefix_and_key() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown(20, "Proposal 20", Some("ERC"), None), + ); + + let metadata = write_metadata_json(temp.path(), "EIPs", None); + let proposals = metadata["proposals"].as_object().unwrap(); + + assert!(proposals.contains_key("erc-20")); + assert_eq!(metadata["proposals"]["erc-20"]["prefix"], json!("ERC")); + } + + #[test] + fn proposal_metadata_non_erc_category_and_default_produce_eip_prefix_and_key() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown(20, "Proposal 20", Some("Core"), None), + ); + write_file( + temp.path(), + "content/021.md", + &metadata_proposal_markdown(21, "Proposal 21", None, None), + ); + + let metadata = write_metadata_json(temp.path(), "EIPs", None); + let proposals = metadata["proposals"].as_object().unwrap(); + + assert!(proposals.contains_key("eip-20")); + assert!(proposals.contains_key("eip-21")); + assert_eq!(metadata["proposals"]["eip-20"]["prefix"], json!("EIP")); + assert_eq!(metadata["proposals"]["eip-21"]["prefix"], json!("EIP")); + } + + #[test] + fn proposal_metadata_missing_optional_description_is_omitted() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/001.md", + &metadata_proposal_markdown(1, "Proposal 1", None, None), + ); + + let metadata = write_metadata_json(temp.path(), "EIPs", None); + let proposal = metadata["proposals"]["eip-1"].as_object().unwrap(); + + assert!(!proposal.contains_key("description")); + } + + #[test] + fn proposal_metadata_malformed_preamble_fails_clearly() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/001.md", "not front matter\n"); + + let error = write_proposal_metadata_json(temp.path(), "EIPs", None) + .unwrap_err() + .to_string(); + + assert!(error.contains("couldn't split preamble")); + assert!(error.contains("001.md")); + } + + #[test] + fn proposal_metadata_missing_required_field_fails_clearly() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/001.md", + "---\neip: 1\ntitle: Proposal 1\ntype: Standards Track\n---\nBody\n", + ); + + let error = write_proposal_metadata_json(temp.path(), "EIPs", None) + .unwrap_err() + .to_string(); + + assert!(error.contains("missing required proposal metadata field `status`")); + assert!(error.contains("001.md")); + } + + #[test] + fn proposal_metadata_path_and_preamble_number_mismatch_fails_clearly() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/001.md", + "---\neip: 2\ntitle: Proposal 1\nstatus: Final\ntype: Standards Track\n---\nBody\n", + ); + + let error = write_proposal_metadata_json(temp.path(), "EIPs", None) + .unwrap_err() + .to_string(); + + assert!(error.contains("proposal metadata number mismatch")); + assert!(error.contains("001.md")); + assert!(error.contains("path indicates `1`")); + assert!(error.contains("`eip` contains `2`")); + } + + #[test] + fn proposal_metadata_allows_eip_and_erc_keys_for_same_number() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown(20, "ERC 20", Some("ERC"), None), + ); + write_file( + temp.path(), + "content/20/index.md", + &metadata_proposal_markdown(20, "EIP 20", None, None), + ); + + let metadata = write_metadata_json(temp.path(), "EIPs", None); + let proposals = metadata["proposals"].as_object().unwrap(); + + assert_eq!(proposals.len(), 2); + assert!(proposals.contains_key("eip-20")); + assert!(proposals.contains_key("erc-20")); + } + + #[test] + fn proposal_metadata_duplicate_same_key_fails_with_both_paths() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown(20, "ERC 20", Some("ERC"), None), + ); + write_file( + temp.path(), + "content/20/index.md", + &metadata_proposal_markdown(20, "Duplicate ERC 20", Some("ERC"), None), + ); + + let error = write_proposal_metadata_json(temp.path(), "EIPs", None) + .unwrap_err() + .to_string(); + + assert!(error.contains("duplicate proposal metadata key `erc-20`")); + assert!(error.contains("020.md")); + assert!(error.contains("20/index.md")); + } + + #[test] + fn proposal_metadata_existing_output_collision_fails_loudly() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/001.md", + &metadata_proposal_markdown(1, "Proposal 1", None, None), + ); + write_file( + temp.path(), + "static/assets/data/proposals.json", + "{\"existing\":true}\n", + ); + + let error = write_proposal_metadata_json(temp.path(), "EIPs", None) + .unwrap_err() + .to_string(); + + assert!(error.contains("already exists")); + assert_eq!( + std::fs::read_to_string(proposal_metadata_json_path(temp.path())).unwrap(), + "{\"existing\":true}\n" + ); + } + + #[test] + fn proposal_metadata_unsupported_markdown_paths_are_ignored() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/_index.md", "not front matter\n"); + write_file(temp.path(), "content/foo.md", "not front matter\n"); + write_file(temp.path(), "content/001/readme.md", "not front matter\n"); + write_file( + temp.path(), + "content/001/assets/readme.md", + "not front matter\n", + ); + write_file( + temp.path(), + "content/002.md", + &metadata_proposal_markdown(2, "Proposal 2", None, None), + ); + + let metadata = write_metadata_json(temp.path(), "EIPs", None); + let proposals = metadata["proposals"].as_object().unwrap(); + + assert_eq!(proposals.len(), 1); + assert!(proposals.contains_key("eip-2")); + } +} From cd7f53da623b4a84e97b5ba335e1462d7e66c077 Mon Sep 17 00:00:00 2001 From: Rito Rhymes Date: Sun, 17 May 2026 20:20:50 -0400 Subject: [PATCH 20/20] Add Pagefind backend indexing and search route Add build-only Pagefind indexing behind the search module boundary. Use the Rust Pagefind crate from build-eips so indexing stays inside the Rust preprocessor. This avoids requiring a separate non-Rust search toolchain, such as Node.js or Python, and avoids shelling out to a Pagefind binary that contributors would need to install, version, and manage on PATH. Add search config and build CLI controls: a workspace [search] block with pagefind = true by default, and build --no-search on build and parity build only. Indexing runs only on build, after Zola has produced rendered HTML, and writes assets under output/pagefind/. Running Pagefind from serve or check would require separate lifecycle support and is out of scope here. Generate the search route page and search state data used by the theme. The route page is written into the prepared repo, not the source worktree, for build, serve, and check so the theme always has a stable shell; its state is marked disabled on serve, check, and --no-search builds, and only an enabled build also writes the Pagefind bundle. Refuse to write the route if user-authored content already occupies content/search.md or content/search/, so the generated route never silently overwrites user content. Keep Pagefind crate imports isolated to src/search/pagefind.rs and expose the rest through build-eips-owned search types. This keeps the integration compartmentalized enough to review, maintain, or remove without intermingling Pagefind API usage through the preprocessor. Add targeted validation for the route and state contract: route collision detection against user content, route placement in the prepared repo under targeted --only builds, resolved base path behavior including --base-url overrides, the disabled state for non-build modes and --no-search builds, and the policy that disabling search does not silently delete previously written Pagefind assets. --- Cargo.lock | 944 ++++++++++++++++++++++++++++++++++++++--- Cargo.toml | 3 +- src/cli.rs | 52 ++- src/config.rs | 53 +++ src/editorial.rs | 2 + src/execution.rs | 62 ++- src/main.rs | 1 + src/pipeline.rs | 310 +++++++++++++- src/search/mod.rs | 423 ++++++++++++++++++ src/search/pagefind.rs | 261 ++++++++++++ 10 files changed, 2045 insertions(+), 66 deletions(-) create mode 100644 src/search/mod.rs create mode 100644 src/search/pagefind.rs diff --git a/Cargo.lock b/Cargo.lock index 496a691..03d2a6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,12 +134,30 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + [[package]] name = "ascii" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "atomic" version = "0.6.1" @@ -182,7 +200,16 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "bit-vec", + "bit-vec 0.8.0", +] + +[[package]] +name = "bit-set" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2f926cc3060f09db9ebc5b52823d85268d24bb917e472c0c4bea35780a7d" +dependencies = [ + "bit-vec 0.9.1", ] [[package]] @@ -191,6 +218,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -199,9 +235,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.10.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "block-buffer" @@ -264,13 +300,14 @@ dependencies = [ "lazy_static", "log", "notify", + "pagefind", "pulldown-cmark", "pulldown-cmark-to-cmark", "regex", "semver", "serde", "serde_json", - "serde_yaml", + "serde_yaml 0.9.34+deprecated", "snafu", "tempfile", "tiny_http", @@ -305,6 +342,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + [[package]] name = "caseless" version = "0.2.2" @@ -426,7 +469,7 @@ version = "4.5.55" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.114", @@ -444,6 +487,23 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + [[package]] name = "comrak" version = "0.37.0" @@ -458,19 +518,60 @@ dependencies = [ "unicode_categories", ] +[[package]] +name = "config-derive" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c547326a30684f853601fb959cc8ecbd0d72abbdd27ba634850a918fa29afc4" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "console" -version = "0.16.2" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03e45a4a8926227e4197636ba97a9fc9b00477e9f4bd711395687c5f0734bec4" +checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" dependencies = [ "encode_unicode", "libc", - "once_cell", "unicode-width", "windows-sys 0.61.2", ] +[[package]] +name = "const_format" +version = "0.2.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e" +dependencies = [ + "const_format_proc_macros", + "konst", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "convert_case" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -486,6 +587,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -495,6 +605,25 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -526,7 +655,20 @@ dependencies = [ "cssparser-macros", "dtoa-short", "itoa", - "phf", + "phf 0.11.3", + "smallvec", +] + +[[package]] +name = "cssparser" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf 0.13.1", "smallvec", ] @@ -551,6 +693,27 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.114", +] + [[package]] name = "deunicode" version = "1.6.2" @@ -717,12 +880,30 @@ dependencies = [ "serde", ] +[[package]] +name = "emojis" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a4d5d50b0b58df5173d8ff1192b4d1422ceae5d981b30d4b6f8ed1d673a2bc4" +dependencies = [ + "phf 0.13.1", +] + [[package]] name = "encode_unicode" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "entities" version = "1.0.1" @@ -772,6 +953,15 @@ dependencies = [ "log", ] +[[package]] +name = "envy" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f47e0157f2cb54f5ae1bd371b30a2ae4311e1c028f575cd4e81de7353215965" +dependencies = [ + "serde", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -794,7 +984,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" dependencies = [ - "bit-set", + "bit-set 0.8.0", "regex-automata", "regex-syntax", ] @@ -835,6 +1025,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fluent-uri" version = "0.3.2" @@ -852,6 +1052,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -906,6 +1112,94 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + [[package]] name = "fxhash" version = "0.2.1" @@ -969,7 +1263,7 @@ version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.1", "libc", "libgit2-sys", "log", @@ -989,6 +1283,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -1007,7 +1307,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -1015,6 +1315,24 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", + "serde", + "serde_core", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "hayagriva" @@ -1024,11 +1342,11 @@ checksum = "1cb69425736f184173b3ca6e27fcba440a61492a790c786b1c6af7e06a03e575" dependencies = [ "ciborium", "citationberg", - "indexmap", + "indexmap 2.13.0", "paste", "roman-numerals-rs", "serde", - "serde_yaml", + "serde_yaml 0.9.34+deprecated", "thiserror 2.0.18", "unic-langid", "unicode-segmentation", @@ -1036,6 +1354,12 @@ dependencies = [ "url", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -1048,6 +1372,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f" +[[package]] +name = "html-escape" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476" +dependencies = [ + "utf8-width", +] + [[package]] name = "html2text" version = "0.14.4" @@ -1216,6 +1549,35 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.13.0" @@ -1316,6 +1678,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -1399,31 +1770,103 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "konst" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128133ed7824fcd73d6e7b17957c5eb7bacb885649bd8c69708b2331a10bcefb" +dependencies = [ + "konst_macro_rules", +] + +[[package]] +name = "konst_macro_rules" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" + [[package]] name = "kqueue" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" dependencies = [ - "kqueue-sys", - "libc", + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", ] [[package]] -name = "kqueue-sys" -version = "1.0.4" +name = "lexical-util" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" dependencies = [ - "bitflags 1.3.2", - "libc", + "lexical-util", + "lexical-write-integer", ] [[package]] -name = "lazy_static" -version = "1.5.0" +name = "lexical-write-integer" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] [[package]] name = "libc" @@ -1451,7 +1894,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.1", "libc", "redox_syscall 0.7.4", ] @@ -1482,6 +1925,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1509,6 +1958,25 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lol_html" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00aad58f6ec3990e795943872f13651e7a5fa59dca2c8f31a74faf8a0e0fb652" +dependencies = [ + "bitflags 2.11.1", + "cfg-if", + "cssparser 0.36.0", + "encoding_rs", + "foldhash 0.2.0", + "hashbrown 0.17.1", + "memchr", + "mime", + "precomputed-hash", + "selectors 0.37.0", + "thiserror 2.0.18", +] + [[package]] name = "lru" version = "0.13.0" @@ -1531,8 +1999,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7a7213d12e1864c0f002f52c2923d4556935a43dec5e71355c2760e0f6e7a18" dependencies = [ "log", - "phf", - "phf_codegen", + "phf 0.11.3", + "phf_codegen 0.11.3", "string_cache", "string_cache_codegen", "tendril", @@ -1562,9 +2030,44 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minicbor" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70eae6d4f18f7d76877fe7b13f0bc21f7c2b7239d2041c338335f7b388d0dd7" +dependencies = [ + "minicbor-derive", +] + +[[package]] +name = "minicbor-derive" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "294f0a0c161c510e9746adf546b8b044fbb0b00677d7dfc9a2452f9fdf63439b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "minifier" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "14f1541610994bba178cb36757e102d06a52a2d9612aa6d34c64b3b377c5d943" +dependencies = [ + "clap", +] [[package]] name = "minimal-lexical" @@ -1579,6 +2082,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -1615,7 +2119,7 @@ version = "6.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.1", "crossbeam-channel", "filetime", "fsevent-sys", @@ -1779,6 +2283,54 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "pagefind" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa7d3c661e0e18b6c9eec1305908baf2bd9564af247727f9e7f4b4d25cd2da39" +dependencies = [ + "anyhow", + "async-compression", + "base64", + "bit-set 0.10.0", + "clap", + "console", + "convert_case", + "either", + "emojis", + "flate2", + "futures", + "hashbrown 0.16.1", + "html-escape", + "include_dir", + "lazy_static", + "lexical-core", + "lol_html", + "minicbor", + "minifier", + "pagefind_stem", + "path-slash", + "rayon", + "regex", + "rust-patch", + "serde", + "serde_json", + "sha-1", + "tikv-jemallocator", + "tokio", + "twelf", + "typed-builder", + "unicode-normalization", + "unicode-segmentation", + "wax", +] + +[[package]] +name = "pagefind_stem" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dfa810b158f3ac364e5acd43ca4a6020a6e729d40c15ce1bed1d911237a52e5" + [[package]] name = "parking_lot" version = "0.12.5" @@ -1808,6 +2360,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "path-slash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42" + [[package]] name = "pct-str" version = "2.0.0" @@ -1830,8 +2388,19 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ - "phf_macros", - "phf_shared", + "phf_macros 0.11.3", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros 0.13.1", + "phf_shared 0.13.1", + "serde", ] [[package]] @@ -1840,8 +2409,18 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.11.3", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator 0.13.1", + "phf_shared 0.13.1", ] [[package]] @@ -1850,18 +2429,41 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ - "phf_shared", + "phf_shared 0.11.3", "rand", ] +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared 0.13.1", +] + [[package]] name = "phf_macros" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.11.3", + "phf_shared 0.11.3", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator 0.13.1", + "phf_shared 0.13.1", "proc-macro2", "quote", "syn 2.0.114", @@ -1876,6 +2478,15 @@ dependencies = [ "siphasher", ] +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -1888,6 +2499,15 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "pori" +version = "0.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a63d338dec139f56dacc692ca63ad35a6be6a797442479b55acd611d79e906" +dependencies = [ + "nom", +] + [[package]] name = "portable-atomic" version = "1.13.1" @@ -1957,7 +2577,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.1", "getopts", "memchr", "pulldown-cmark-escape", @@ -2025,13 +2645,33 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d20581732dd76fa913c7dff1a2412b714afe3573e94d41c34719de73337cc8ab" +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.1", ] [[package]] @@ -2040,7 +2680,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.1", ] [[package]] @@ -2123,19 +2763,55 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c85cd47a33a4510b1424fe796498e174c6a9cf94e606460ef022a19f3e4ff85e" +[[package]] +name = "rust-patch" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4076837f5df7460d37d1e245c966e64f6aaeeb59a76f186f352ca91d6087fb43" +dependencies = [ + "rust-patch-derive", +] + +[[package]] +name = "rust-patch-derive" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9927610a0a7c3e3dece1e89a114c31e435f27db01b1d630e81eb02ecd820f0b" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "rustc-demangle" version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys", @@ -2199,11 +2875,11 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "527e65d9d888567588db4c12da1087598d0f6f8b346cc2c5abc91f05fc2dffe2" dependencies = [ - "cssparser", + "cssparser 0.34.0", "ego-tree", "html5ever 0.29.1", "precomputed-hash", - "selectors", + "selectors 0.26.0", "tendril", ] @@ -2213,15 +2889,34 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8" dependencies = [ - "bitflags 2.10.0", - "cssparser", - "derive_more", + "bitflags 2.11.1", + "cssparser 0.34.0", + "derive_more 0.99.20", "fxhash", "log", "new_debug_unreachable", - "phf", - "phf_codegen", + "phf 0.11.3", + "phf_codegen 0.11.3", + "precomputed-hash", + "servo_arc", + "smallvec", +] + +[[package]] +name = "selectors" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cfaaa6035167f0e604e42723c7650d59ee269ef220d7bbe0565602c8a0173b9" +dependencies = [ + "bitflags 2.11.1", + "cssparser 0.36.0", + "derive_more 2.1.1", + "log", + "new_debug_unreachable", + "phf 0.13.1", + "phf_codegen 0.13.1", "precomputed-hash", + "rustc-hash", "servo_arc", "smallvec", ] @@ -2308,13 +3003,25 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_yaml" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" +dependencies = [ + "indexmap 1.9.3", + "ryu", + "serde", + "yaml-rust", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap", + "indexmap 2.13.0", "itoa", "ryu", "serde", @@ -2330,6 +3037,17 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "sha-1" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.9" @@ -2405,6 +3123,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "siphasher" version = "1.0.2" @@ -2449,7 +3173,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.114", @@ -2489,7 +3213,7 @@ checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" dependencies = [ "new_debug_unreachable", "parking_lot", - "phf_shared", + "phf_shared 0.11.3", "precomputed-hash", "serde", ] @@ -2500,8 +3224,8 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.11.3", + "phf_shared 0.11.3", "proc-macro2", "quote", ] @@ -2525,6 +3249,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", + "quote", "unicode-ident", ] @@ -2623,6 +3348,26 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "tiny_http" version = "0.12.0" @@ -2667,6 +3412,7 @@ version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" dependencies = [ + "bytes", "pin-project-lite", "tokio-macros", ] @@ -2682,6 +3428,15 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + [[package]] name = "toml" version = "0.8.23" @@ -2700,7 +3455,7 @@ version = "0.9.11+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3afc9a848309fe1aaffaed6e1546a7a14de1f935dc9d89d32afd9a44bab7c46" dependencies = [ - "indexmap", + "indexmap 2.13.0", "serde_core", "serde_spanned 1.0.4", "toml_datetime 0.7.5+spec-1.1.0", @@ -2733,7 +3488,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap", + "indexmap 2.13.0", "serde", "serde_spanned 0.6.9", "toml_datetime 0.6.11", @@ -2762,12 +3517,49 @@ version = "1.0.6+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" +[[package]] +name = "twelf" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16de46d08a9d3a25e0a65bb70090797b970bd1e95d72872567ba8d02c0b03bdf" +dependencies = [ + "clap", + "config-derive", + "envy", + "log", + "serde", + "serde_json", + "serde_yaml 0.8.26", + "thiserror 1.0.69", + "toml 0.5.11", +] + [[package]] name = "typed-arena" version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" +[[package]] +name = "typed-builder" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "typenum" version = "1.19.0" @@ -2835,6 +3627,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "unicode_categories" version = "0.1.1" @@ -2884,6 +3682,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca61eb27fa339aa08826a29f03e87b99b4d8f0fc2255306fd266bb1b6a9de498" +[[package]] +name = "utf8-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -3016,6 +3820,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wax" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f8cbf8125142b9b30321ac8721f54c52fbcd6659f76cf863d5e2e38c07a3d7b" +dependencies = [ + "const_format", + "itertools 0.14.0", + "nom", + "pori", + "regex", + "thiserror 2.0.18", + "walkdir", +] + [[package]] name = "web-time" version = "1.1.0" @@ -3032,8 +3851,8 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57ffde1dc01240bdf9992e3205668b235e59421fd085e8a317ed98da0178d414" dependencies = [ - "phf", - "phf_codegen", + "phf 0.11.3", + "phf_codegen 0.11.3", "string_cache", "string_cache_codegen", ] @@ -3304,6 +4123,15 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "yoke" version = "0.8.1" @@ -3419,7 +4247,7 @@ dependencies = [ "dirs", "hashbrown 0.14.5", "indoc", - "itertools", + "itertools 0.13.0", "lazy_static", "regex", "stringmetrics", diff --git a/Cargo.toml b/Cargo.toml index 17652cd..66f6a88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ indicatif-log-bridge = "0.2.3" lazy_static = "1.5.0" log = { version = "0.4.29", features = ["std"] } notify = "6.1.1" +pagefind = { version = "1.5.2", default-features = false } pulldown-cmark = "0.13.0" pulldown-cmark-to-cmark = "22.0.0" regex = "1.12.2" @@ -43,7 +44,7 @@ serde_json = "1.0.148" serde_yaml = "0.9.34" snafu = { version = "0.8.9", features = ["rust_1_81"] } tiny_http = "0.12.0" -tokio = { version = "1.48.0", features = ["fs", "rt", "macros"] } +tokio = { version = "1.48.0", features = ["fs", "io-util", "rt", "macros"] } toml = "0.9.10" toml_datetime = { version = "0.7.5", features = ["serde"] } url = "2.5.7" diff --git a/src/cli.rs b/src/cli.rs index 6488667..93182e4 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -73,6 +73,13 @@ pub(crate) struct OnlyCliArgs { pub(crate) only: Vec, } +#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)] +pub(crate) struct SearchCliArgs { + /// Skip Pagefind search indexing for this build + #[arg(long)] + pub(crate) no_search: bool, +} + #[derive(Debug, Clone, Subcommand)] pub(crate) enum Operation { /// Print linter schema metadata and lint configuration @@ -91,6 +98,9 @@ pub(crate) enum Operation { #[command(flatten)] only: OnlyCliArgs, + + #[command(flatten)] + search: SearchCliArgs, }, /// Serve the existing built output without rebuilding it @@ -168,6 +178,9 @@ pub(crate) enum ProfiledOperation { Build { #[command(flatten)] base_url: BaseUrlCliArgs, + + #[command(flatten)] + search: SearchCliArgs, }, /// Build a fresh temporary site, serve it locally, and watch tracked edits @@ -304,6 +317,22 @@ impl Operation { } } + pub(crate) fn search_cli_args(&self) -> SearchCliArgs { + match self { + Self::Build { search, .. } => search.clone(), + Self::Parity { command } => command.search_cli_args(), + Self::Print { .. } + | Self::Serve { .. } + | Self::Preview { .. } + | Self::Clean + | Self::Check { .. } + | Self::Changed { .. } + | Self::Editorial { .. } + | Self::Init { .. } + | Self::Doctor => SearchCliArgs::default(), + } + } + pub(crate) fn is_plain_site_command(&self) -> bool { matches!( self, @@ -353,11 +382,18 @@ impl ProfiledOperation { fn base_url_cli_args(&self) -> BaseUrlCliArgs { match self { - Self::Build { base_url } | Self::Serve { base_url, .. } => base_url.clone(), + Self::Build { base_url, .. } | Self::Serve { base_url, .. } => base_url.clone(), Self::Check => BaseUrlCliArgs::default(), } } + fn search_cli_args(&self) -> SearchCliArgs { + match self { + Self::Build { search, .. } => search.clone(), + Self::Serve { .. } | Self::Check => SearchCliArgs::default(), + } + } + fn runtime_operation(&self) -> RuntimeOperation { match self { Self::Build { .. } => RuntimeOperation::Build, @@ -515,6 +551,20 @@ mod tests { assert!(Args::try_parse_from(["build-eips", "parity", "serve", "--only", "555"]).is_err()); } + #[test] + fn no_search_flag_parses_only_on_build_commands() { + let build = parse_args(&["build-eips", "build", "--no-search"]); + let parity = parse_args(&["build-eips", "parity", "build", "--no-search"]); + + assert!(build.operation.search_cli_args().no_search); + assert!(parity.operation.search_cli_args().no_search); + + assert!(Args::try_parse_from(["build-eips", "serve", "--no-search"]).is_err()); + assert!(Args::try_parse_from(["build-eips", "check", "--no-search"]).is_err()); + assert!(Args::try_parse_from(["build-eips", "parity", "serve", "--no-search"]).is_err()); + assert!(Args::try_parse_from(["build-eips", "parity", "check", "--no-search"]).is_err()); + } + #[test] fn server_flags_parse_on_serve_and_preview_forms() { let cases: &[(&[&str], bool)] = &[ diff --git a/src/config.rs b/src/config.rs index 24cd891..596df94 100644 --- a/src/config.rs +++ b/src/config.rs @@ -462,6 +462,10 @@ pub struct WorkspaceConfig { /// Local render filtering defaults. #[serde(default)] pub render: RenderSettings, + + /// Local search indexing defaults for rendered output. + #[serde(default)] + pub search: SearchSettings, } impl WorkspaceConfig { @@ -470,6 +474,7 @@ impl WorkspaceConfig { server: ServerSettings::default(), site: SiteSettings::starter(), render: RenderSettings::default(), + search: SearchSettings::default(), } } } @@ -483,6 +488,20 @@ pub struct RenderSettings { pub only: Vec, } +/// Workspace-local search indexing settings. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct SearchSettings { + /// Whether `build-eips build` writes Pagefind search assets after rendering HTML. + pub pagefind: bool, +} + +impl Default for SearchSettings { + fn default() -> Self { + Self { pagefind: true } + } +} + /// Workspace-local bind address defaults for local server commands. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] @@ -650,6 +669,10 @@ impl LoadedWorkspaceConfig { &self.config.render } + pub fn search_settings(&self) -> &SearchSettings { + &self.config.search + } + pub fn local_theme_path(&self) -> PathBuf { self.workspace_root.join(DEFAULT_THEME_DIR) } @@ -943,6 +966,7 @@ base_url = "https://staging.example.test/ERCs/" config.site_settings().base_url.as_ref().unwrap().as_str(), "http://127.0.0.1:1111/" ); + assert!(config.search_settings().pagefind); } #[test] @@ -960,6 +984,8 @@ base_url = "https://staging.example.test/ERCs/" assert!(original.contains(&format!("base_url = \"{DEFAULT_SITE_BASE_URL}\""))); assert!(original.contains("[render]")); assert!(original.contains("only = []")); + assert!(original.contains("[search]")); + assert!(original.contains("pagefind = true")); assert!(!original.contains("default_profile")); assert!(!original.contains("[profiles")); } @@ -1093,6 +1119,33 @@ base_url = "http://127.0.0.1:1111" assert_eq!(config.server_settings(), &ServerSettings::default()); assert!(config.site_settings().base_url.is_none()); assert!(config.render_settings().only.is_empty()); + assert!(config.search_settings().pagefind); + } + + #[test] + fn search_config_defaults_to_pagefind_enabled_when_missing() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file(LOCAL_CONFIG_FILE, ""); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert!(config.search_settings().pagefind); + } + + #[test] + fn parses_workspace_config_search_settings() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( + LOCAL_CONFIG_FILE, + r#" +[search] +pagefind = false +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert!(!config.search_settings().pagefind); } #[test] diff --git a/src/editorial.rs b/src/editorial.rs index 2b1ce5a..ea405cf 100644 --- a/src/editorial.rs +++ b/src/editorial.rs @@ -577,6 +577,7 @@ base_url = "https://staging.example.test/{sibling_id}/" source_materialization: crate::git::SourceMaterialization::Clean, server_binding: ServerBinding::default(), base_url_override: None, + search: Default::default(), } } @@ -761,6 +762,7 @@ base_url = "https://staging.example.test/{sibling_id}/" source_materialization: crate::git::SourceMaterialization::Clean, server_binding: ServerBinding::default(), base_url_override: None, + search: Default::default(), }; let selectors = EditorialSelectorArgs { paths: Vec::new(), diff --git a/src/execution.rs b/src/execution.rs index 2b10743..b714cdb 100644 --- a/src/execution.rs +++ b/src/execution.rs @@ -24,6 +24,7 @@ use crate::{ identity::ActiveRepoIdentity, layout::BUILD_DIR, proposal::ProposalNumber, + search::SearchConfig, }; #[derive(Debug, Clone)] @@ -36,6 +37,7 @@ pub(crate) struct ResolvedExecution { pub(crate) source_materialization: git::SourceMaterialization, pub(crate) server_binding: ServerBinding, pub(crate) base_url_override: Option, + pub(crate) search: SearchConfig, } impl ResolvedExecution { @@ -371,6 +373,23 @@ fn resolve_base_url_override( Ok(workspace_config.and_then(|config| config.site_settings().base_url.clone())) } +fn resolve_search_config( + args: &Args, + workspace_config: Option<&LoadedWorkspaceConfig>, +) -> SearchConfig { + let mut search = workspace_config + .map(|config| SearchConfig { + pagefind: config.search_settings().pagefind, + }) + .unwrap_or_default(); + + if args.operation.search_cli_args().no_search { + search.pagefind = false; + } + + search +} + pub(crate) fn resolve_execution(args: &Args) -> Result { let root_path = root(args)?; let active_repo = ActiveRepoIdentity::load(&root_path)?; @@ -412,6 +431,7 @@ pub(crate) fn resolve_execution(args: &Args) -> Result Result, + search: SearchConfig, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SearchIndexMode { + Build, + #[cfg(test)] + Check, + #[cfg(test)] + Serve, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SearchRouteMode { + Build, + Check, + Serve, +} + +fn should_index_search(mode: SearchIndexMode, search: SearchConfig) -> bool { + matches!(mode, SearchIndexMode::Build) && search.pagefind +} + +fn search_route_enabled(mode: SearchRouteMode, search: SearchConfig) -> bool { + matches!(mode, SearchRouteMode::Build) && search.pagefind +} + +fn run_search_index( + mode: SearchIndexMode, + output_path: &Path, + search_config: SearchConfig, +) -> Result, Whatever> { + if !should_index_search(mode, search_config) { + return Ok(None); + } + + search::index_site(SearchIndexRequest { + output_path: output_path.to_path_buf(), + }) + .map(Some) + .whatever_context("search indexing failed") } impl Prepared { @@ -93,6 +138,7 @@ impl Prepared { source_materialization, server_binding, base_url_override, + search, } = resolved; let theme_path = theme_path.whatever_context("Zola runtime requires a workspace-local theme path")?; @@ -107,6 +153,8 @@ impl Prepared { &repository_use, source_materialization, )?; + search::ensure_search_route_available(&content_path) + .whatever_context("unable to reserve generated search route")?; let only_plan = only .map(|selected_numbers| OnlyRenderPlan::build(&content_path, selected_numbers)) @@ -138,14 +186,13 @@ impl Prepared { source_materialization, server_binding, base_url_override, + search, }) } pub(crate) fn build(self) -> Result<(), Whatever> { - let base_url = self - .base_url_override - .as_ref() - .unwrap_or(&self.repository_use.location.base_url); + let base_url = self.resolved_base_url().clone(); + let _ = self.write_search_route_for_zola(SearchRouteMode::Build)?; zola::build( &self.theme_path, &self.repo_path, @@ -153,10 +200,12 @@ impl Prepared { base_url.as_str(), ) .whatever_context("zola build failed")?; + let _ = run_search_index(SearchIndexMode::Build, &self.output_path, self.search)?; Ok(()) } pub(crate) fn serve(self) -> Result<(), Whatever> { + let _ = self.write_search_route_for_zola(SearchRouteMode::Serve)?; let sync_config = serve_sync_config( self.source_materialization, &self.source_root, @@ -190,9 +239,31 @@ impl Prepared { } pub(crate) fn check(self) -> Result<(), Whatever> { + let _ = self.write_search_route_for_zola(SearchRouteMode::Check)?; zola::check(&self.theme_path, &self.repo_path).whatever_context("zola check failed")?; Ok(()) } + + fn resolved_base_url(&self) -> &Url { + self.base_url_override + .as_ref() + .unwrap_or(&self.repository_use.location.base_url) + } + + fn search_template_state(&self, mode: SearchRouteMode) -> SearchTemplateState { + SearchTemplateState::from_base_url( + search_route_enabled(mode, self.search), + self.resolved_base_url(), + ) + } + + fn write_search_route_for_zola( + &self, + mode: SearchRouteMode, + ) -> Result { + search::write_search_route(&self.repo_path, self.search_template_state(mode)) + .whatever_context("unable to write generated search route") + } } #[cfg(test)] @@ -201,6 +272,7 @@ mod tests { use clap::Parser; use git2::{IndexAddOption, Repository, Signature}; + use snafu::Report; use tempfile::TempDir; use url::Url; @@ -213,9 +285,13 @@ mod tests { git::SourceMaterialization, layout::{mounted_theme_path, theme_config_path, CONTENT_DIR, REPO_DIR}, proposal_catalog::collect_proposal_catalog, + search::{SearchConfig, SEARCH_DATA_FILE}, }; - use super::{prepare_runtime_source, prepare_theme_for_zola}; + use super::{ + prepare_runtime_source, prepare_theme_for_zola, run_search_index, search_route_enabled, + should_index_search, Prepared, SearchIndexMode, SearchRouteMode, + }; struct RuntimeWorkspace { _temp: TempDir, @@ -345,6 +421,42 @@ base_url = "https://staging.example.test/{sibling_id}/" } } + fn runtime_workspace_with_theme(active_files: &[(&str, &str)]) -> RuntimeWorkspace { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("EIPs"); + let theme_path = workspace_root.join("theme"); + let missing_upstream = file_url(&temp.path().join("missing-upstream")); + let manifest = repo_manifest_text("EIPs", &missing_upstream, &[]); + + write_file(&workspace_root, config::LOCAL_CONFIG_FILE, ""); + init_repo( + &theme_path, + &[ + ( + "config/zola.toml", + "title = 'theme'\nbase_url = 'https://example.test/'\ntheme = 'eips-theme'\n", + ), + ("theme.toml", "name = 'eips-theme'\n"), + ("templates/page.html", "{{ page.content | safe }}\n"), + ], + ); + + std::fs::create_dir_all(&active_path).unwrap(); + let active_repo = Repository::init(&active_path).unwrap(); + active_repo.set_head("refs/heads/master").unwrap(); + write_file(&active_path, config::REPO_MANIFEST_FILE, manifest.as_str()); + for (relative, contents) in active_files { + write_file(&active_path, relative, contents); + } + commit_all(&active_repo, "initial"); + + RuntimeWorkspace { + _temp: temp, + active_path, + } + } + fn resolved_runtime(workspace: &RuntimeWorkspace, command: &[&str]) -> ResolvedExecution { let active_path = workspace.active_path.to_str().unwrap(); let mut arguments = vec!["build-eips", "-C", active_path]; @@ -369,6 +481,17 @@ base_url = "https://staging.example.test/{sibling_id}/" resolved.build_path.join(REPO_DIR).join(relative) } + fn rendered_front_matter(path: &Path) -> toml::Value { + let contents = std::fs::read_to_string(path).unwrap(); + let front_matter = contents + .strip_prefix("+++\n") + .unwrap() + .split_once("\n+++\n") + .unwrap() + .0; + toml::from_str(front_matter).unwrap() + } + #[test] fn workspace_local_theme_is_materialized_as_mounted_theme_for_zola() { let temp = TempDir::new().unwrap(); @@ -541,4 +664,181 @@ base_url = "https://staging.example.test/{sibling_id}/" assert!(!resolved.root_path.join("content/00002.md").exists()); assert_eq!(records["erc-2"].title, "Proposal 2"); } + + #[test] + fn search_index_policy_runs_only_for_enabled_builds() { + let enabled = SearchConfig { pagefind: true }; + let disabled = SearchConfig { pagefind: false }; + + assert!(should_index_search(SearchIndexMode::Build, enabled)); + assert!(!should_index_search(SearchIndexMode::Build, disabled)); + assert!(!should_index_search(SearchIndexMode::Check, enabled)); + assert!(!should_index_search(SearchIndexMode::Serve, enabled)); + } + + #[test] + fn search_route_state_policy_matches_runtime_command() { + let enabled = SearchConfig { pagefind: true }; + let disabled = SearchConfig { pagefind: false }; + + assert!(search_route_enabled(SearchRouteMode::Build, enabled)); + assert!(!search_route_enabled(SearchRouteMode::Build, disabled)); + assert!(!search_route_enabled(SearchRouteMode::Check, enabled)); + assert!(!search_route_enabled(SearchRouteMode::Serve, enabled)); + } + + #[test] + fn search_route_collisions_are_detected_in_materialized_content() { + for (relative, expected) in [ + ("content/search.md", "content/search.md"), + ("content/search/placeholder.txt", "content/search"), + ("content/search/index.md", "content/search/index.md"), + ("content/search/_index.md", "content/search/_index.md"), + ] { + let workspace = runtime_workspace_with_theme(&[(relative, "not a proposal\n")]); + let resolved = resolved_runtime(&workspace, &["build"]); + + let error = Report::from_error(Prepared::prepare(resolved).unwrap_err()).to_string(); + + assert!(error.contains("unable to reserve generated search route")); + assert!(error.contains("collides with the generated `/search/` route")); + assert!(error.contains(expected)); + } + } + + #[test] + fn generated_search_route_is_written_to_prepared_repo_not_source_worktree() { + let proposal = pipeline_proposal_markdown(1, None, "Active proposal."); + let workspace = runtime_workspace_with_theme(&[("content/00001.md", proposal.as_str())]); + let resolved = resolved_runtime(&workspace, &["build"]); + let source_search_route = workspace.active_path.join("content/search.md"); + + let prepared = Prepared::prepare(resolved).unwrap(); + let summary = prepared + .write_search_route_for_zola(SearchRouteMode::Build) + .unwrap(); + + assert_eq!( + summary.route_path, + prepared.repo_path.join("content/search.md") + ); + assert_eq!( + summary.data_path, + prepared.repo_path.join("data").join(SEARCH_DATA_FILE) + ); + assert!(!source_search_route.exists()); + assert!(summary.route_path.is_file()); + assert!(summary.data_path.is_file()); + + let front_matter = rendered_front_matter(&summary.route_path); + assert_eq!(front_matter["template"].as_str(), Some("search.html")); + assert_eq!( + front_matter["extra"]["search"]["enabled"].as_bool(), + Some(true) + ); + } + + #[test] + fn targeted_build_writes_search_route_after_preprocess_and_prune() { + let selected = pipeline_proposal_markdown(555, None, "Selected proposal."); + let unselected = pipeline_proposal_markdown(678, Some("ERC"), "Unselected proposal."); + let workspace = runtime_workspace_with_theme(&[ + ("content/00555.md", selected.as_str()), + ("content/00678.md", unselected.as_str()), + ]); + let resolved = resolved_runtime(&workspace, &["build", "--only", "555"]); + + let prepared = Prepared::prepare(resolved).unwrap(); + let selected_path = prepared.repo_path.join("content/00555.md"); + let unselected_path = prepared.repo_path.join("content/00678.md"); + let search_route_path = prepared.repo_path.join("content/search.md"); + let search_data_path = prepared.repo_path.join("data").join(SEARCH_DATA_FILE); + + assert!(selected_path.is_file()); + assert!(!unselected_path.exists()); + assert_eq!( + rendered_front_matter(&selected_path)["extra"]["number"].as_integer(), + Some(555) + ); + assert!(!search_route_path.exists()); + assert!(!search_data_path.exists()); + + let summary = prepared + .write_search_route_for_zola(SearchRouteMode::Build) + .unwrap(); + + assert_eq!(summary.route_path, search_route_path); + assert_eq!(summary.data_path, search_data_path); + assert!(summary.route_path.is_file()); + assert!(summary.data_path.is_file()); + } + + #[test] + fn search_template_state_uses_resolved_runtime_base_url() { + let proposal = pipeline_proposal_markdown(1, None, "Active proposal."); + let repository_workspace = + runtime_workspace_with_theme(&[("content/00001.md", proposal.as_str())]); + + let prepared = + Prepared::prepare(resolved_runtime(&repository_workspace, &["build"])).unwrap(); + let repository_state = prepared.search_template_state(SearchRouteMode::Build); + assert!(repository_state.enabled); + assert_eq!(repository_state.base_path, "/EIPs/"); + assert_eq!( + repository_state.bundle_path.as_deref(), + Some("/EIPs/pagefind/") + ); + + let override_workspace = + runtime_workspace_with_theme(&[("content/00001.md", proposal.as_str())]); + let prepared = Prepared::prepare(resolved_runtime( + &override_workspace, + &["build", "--base-url", "https://wg-eips.ritovision.com/"], + )) + .unwrap(); + let override_state = prepared.search_template_state(SearchRouteMode::Build); + assert!(override_state.enabled); + assert_eq!(override_state.base_path, "/"); + assert_eq!(override_state.bundle_path.as_deref(), Some("/pagefind/")); + } + + #[test] + fn generated_search_route_uses_disabled_state_for_disabled_builds_and_serve() { + let proposal = pipeline_proposal_markdown(1, None, "Active proposal."); + let workspace = runtime_workspace_with_theme(&[("content/00001.md", proposal.as_str())]); + let mut resolved = resolved_runtime(&workspace, &["build"]); + resolved.search = SearchConfig { pagefind: false }; + let prepared = Prepared::prepare(resolved).unwrap(); + + let build_state = prepared.search_template_state(SearchRouteMode::Build); + let serve_state = prepared.search_template_state(SearchRouteMode::Serve); + let check_state = prepared.search_template_state(SearchRouteMode::Check); + + assert!(!build_state.enabled); + assert!(build_state.bundle_path.is_none()); + assert!(!serve_state.enabled); + assert!(serve_state.bundle_path.is_none()); + assert!(!check_state.enabled); + assert!(check_state.bundle_path.is_none()); + } + + #[test] + fn disabled_build_search_hook_leaves_stale_pagefind_assets_untouched() { + let temp = TempDir::new().unwrap(); + let output_path = temp.path().join("output"); + write_file(&output_path, "pagefind/stale.txt", "stale"); + + let summary = run_search_index( + SearchIndexMode::Build, + &output_path, + SearchConfig { pagefind: false }, + ) + .unwrap(); + + assert!(summary.is_none()); + assert_eq!( + std::fs::read_to_string(output_path.join("pagefind/stale.txt")).unwrap(), + "stale" + ); + } } diff --git a/src/search/mod.rs b/src/search/mod.rs new file mode 100644 index 0000000..bc5374d --- /dev/null +++ b/src/search/mod.rs @@ -0,0 +1,423 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Build-side search indexing and generated search route state. + +use std::{ + io::Write, + path::{Path, PathBuf}, +}; + +use serde::Serialize; +use snafu::{OptionExt, ResultExt, Whatever}; +use url::Url; + +use crate::layout::CONTENT_DIR; + +mod pagefind; + +const SEARCH_ROUTE_FILE: &str = "search.md"; +const SEARCH_ROUTE_DIR: &str = "search"; +pub(crate) const SEARCH_DATA_FILE: &str = "build_eips_search.toml"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct SearchConfig { + pub(crate) pagefind: bool, +} + +impl Default for SearchConfig { + fn default() -> Self { + Self { pagefind: true } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct SearchIndexRequest { + pub(crate) output_path: PathBuf, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct SearchIndexSummary { + pub(crate) pages_indexed: usize, + pub(crate) output_path: PathBuf, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct SearchTemplateState { + pub(crate) enabled: bool, + pub(crate) base_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) bundle_path: Option, +} + +impl SearchTemplateState { + pub(crate) fn from_base_url(enabled: bool, base_url: &Url) -> Self { + let base_path = normalized_base_path(base_url); + let bundle_path = enabled.then(|| format!("{base_path}pagefind/")); + + Self { + enabled, + base_path, + bundle_path, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct SearchRouteSummary { + pub(crate) route_path: PathBuf, + pub(crate) data_path: PathBuf, + pub(crate) state: SearchTemplateState, +} + +#[derive(Debug, Serialize)] +struct SearchPageFrontMatter<'a> { + title: &'static str, + template: &'static str, + extra: SearchPageExtra<'a>, +} + +#[derive(Debug, Serialize)] +struct SearchPageExtra<'a> { + search: &'a SearchTemplateState, +} + +pub(crate) fn index_site(request: SearchIndexRequest) -> Result { + pagefind::index_site(request) +} + +pub(crate) fn ensure_search_route_available(content_path: &Path) -> Result<(), Whatever> { + for relative_path in [ + Path::new(SEARCH_ROUTE_FILE).to_path_buf(), + Path::new(SEARCH_ROUTE_DIR).join("index.md"), + Path::new(SEARCH_ROUTE_DIR).join("_index.md"), + Path::new(SEARCH_ROUTE_DIR).to_path_buf(), + ] { + let candidate = content_path.join(&relative_path); + match std::fs::symlink_metadata(&candidate) { + Ok(_) => { + snafu::whatever!( + "materialized content path `{}` collides with the generated `/search/` route; refusing to overwrite user-authored content", + candidate.to_string_lossy() + ); + } + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => {} + Err(error) => { + snafu::whatever!( + "unable to inspect potential search route collision `{}`: {error}", + candidate.to_string_lossy() + ); + } + } + } + + Ok(()) +} + +pub(crate) fn write_search_route( + repo_path: &Path, + state: SearchTemplateState, +) -> Result { + let route_path = repo_path.join(CONTENT_DIR).join(SEARCH_ROUTE_FILE); + let data_path = repo_path.join("data").join(SEARCH_DATA_FILE); + + write_search_data_file(&data_path, &state)?; + write_search_page_file(&route_path, &state)?; + + Ok(SearchRouteSummary { + route_path, + data_path, + state, + }) +} + +fn normalized_base_path(base_url: &Url) -> String { + let path = base_url.path().trim_matches('/'); + + if path.is_empty() { + "/".to_owned() + } else { + format!("/{path}/") + } +} + +fn write_new_file(path: &Path, contents: &str, label: &str) -> Result<(), Whatever> { + let parent = path.parent().with_whatever_context(|| { + format!( + "{label} output path `{}` has no parent directory", + path.to_string_lossy() + ) + })?; + std::fs::create_dir_all(parent).with_whatever_context(|_| { + format!( + "unable to create {label} output directory `{}`", + parent.to_string_lossy() + ) + })?; + + let mut file = match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + snafu::whatever!( + "{label} output `{}` already exists; refusing to overwrite it", + path.to_string_lossy() + ); + } + Err(error) => { + snafu::whatever!( + "unable to create {label} output `{}`: {error}", + path.to_string_lossy() + ); + } + }; + + file.write_all(contents.as_bytes()) + .with_whatever_context(|_| { + format!( + "unable to write {label} output `{}`", + path.to_string_lossy() + ) + })?; + + Ok(()) +} + +fn write_search_data_file(data_path: &Path, state: &SearchTemplateState) -> Result<(), Whatever> { + let mut contents = + toml::to_string_pretty(state).whatever_context("unable to encode search state TOML")?; + if !contents.ends_with('\n') { + contents.push('\n'); + } + write_new_file(data_path, &contents, "search state") +} + +fn write_search_page_file(route_path: &Path, state: &SearchTemplateState) -> Result<(), Whatever> { + let front_matter = SearchPageFrontMatter { + title: "Search", + template: "search.html", + extra: SearchPageExtra { search: state }, + }; + let mut front_matter = toml::to_string_pretty(&front_matter) + .whatever_context("unable to encode search route front matter")?; + if !front_matter.ends_with('\n') { + front_matter.push('\n'); + } + let contents = format!("+++\n{front_matter}+++\n"); + write_new_file(route_path, &contents, "search route") +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use tempfile::TempDir; + use toml::Value as TomlValue; + use url::Url; + + use super::{ + ensure_search_route_available, write_search_route, SearchTemplateState, SEARCH_DATA_FILE, + }; + + fn contains_forbidden_pagefind_reference(path: &Path, src_path: &Path) -> bool { + let relative = path.strip_prefix(src_path).unwrap(); + let allowed_local_call = ["pagefind", "::index_site"].concat(); + let direct_path = ["pagefind", "::"].concat(); + let absolute_direct_path = ["::", "pagefind", "::"].concat(); + let use_direct = ["use", "pagefind"].concat(); + let use_absolute_direct = ["use", "::", "pagefind"].concat(); + let extern_crate = ["extern", "crate", "pagefind"].concat(); + + std::fs::read_to_string(path) + .map(|contents| { + contents.lines().any(|line| { + let code = line.split("//").next().unwrap_or_default(); + let compact = code + .chars() + .filter(|character| !character.is_whitespace()) + .collect::(); + + if relative == Path::new("search/mod.rs") + && compact.contains(&allowed_local_call) + { + return false; + } + + compact.contains(&absolute_direct_path) + || compact.contains(&direct_path) + || compact.starts_with(&use_direct) + || compact.starts_with(&use_absolute_direct) + || compact.starts_with(&extern_crate) + }) + }) + .unwrap_or(false) + } + + #[test] + fn pagefind_crate_imports_stay_inside_pagefind_module() { + let src_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let allowed_path = src_path.join("search/pagefind.rs"); + let violations = walkdir::WalkDir::new(&src_path) + .into_iter() + .filter_map(Result::ok) + .map(|entry| entry.into_path()) + .filter(|path| path.extension().is_some_and(|extension| extension == "rs")) + .filter(|path| path != &allowed_path) + .filter(|path| contains_forbidden_pagefind_reference(path, &src_path)) + .map(|path| path.strip_prefix(&src_path).unwrap().to_path_buf()) + .collect::>(); + + assert!( + violations.is_empty(), + "Pagefind crate imports must stay in src/search/pagefind.rs, found {violations:?}" + ); + } + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn front_matter(path: &Path) -> TomlValue { + let contents = std::fs::read_to_string(path).unwrap(); + let front_matter = contents + .strip_prefix("+++\n") + .unwrap() + .split_once("\n+++\n") + .unwrap() + .0; + toml::from_str(front_matter).unwrap() + } + + #[test] + fn search_template_state_normalizes_base_path_and_bundle_path() { + let root = Url::parse("https://wg-eips.ritovision.com/").unwrap(); + let subpath = Url::parse("https://eips-wg.github.io/EIPs/").unwrap(); + + assert_eq!( + SearchTemplateState::from_base_url(true, &root), + SearchTemplateState { + enabled: true, + base_path: "/".to_owned(), + bundle_path: Some("/pagefind/".to_owned()), + } + ); + assert_eq!( + SearchTemplateState::from_base_url(true, &subpath), + SearchTemplateState { + enabled: true, + base_path: "/EIPs/".to_owned(), + bundle_path: Some("/EIPs/pagefind/".to_owned()), + } + ); + assert_eq!( + SearchTemplateState::from_base_url(false, &subpath), + SearchTemplateState { + enabled: false, + base_path: "/EIPs/".to_owned(), + bundle_path: None, + } + ); + } + + #[test] + fn search_route_collision_checks_all_reserved_route_paths() { + for relative in [ + "content/search.md", + "content/search/placeholder.txt", + "content/search/index.md", + "content/search/_index.md", + ] { + let temp = TempDir::new().unwrap(); + let content_path = temp.path().join("content"); + write_file(temp.path(), relative, "user content\n"); + + let error = ensure_search_route_available(&content_path) + .unwrap_err() + .to_string(); + + assert!(error.contains("collides with the generated `/search/` route")); + assert!(error.contains("content/search")); + } + } + + #[test] + fn search_route_writes_page_and_shared_template_state() { + let temp = TempDir::new().unwrap(); + let state = SearchTemplateState::from_base_url( + true, + &Url::parse("https://eips-wg.github.io/EIPs/").unwrap(), + ); + + let summary = write_search_route(temp.path(), state.clone()).unwrap(); + + assert_eq!(summary.route_path, temp.path().join("content/search.md")); + assert_eq!( + summary.data_path, + temp.path().join("data").join(SEARCH_DATA_FILE) + ); + assert_eq!(summary.state, state); + + let route_front_matter = front_matter(&summary.route_path); + assert_eq!(route_front_matter["title"].as_str(), Some("Search")); + assert_eq!(route_front_matter["template"].as_str(), Some("search.html")); + assert_eq!( + route_front_matter["extra"]["search"]["enabled"].as_bool(), + Some(true) + ); + assert_eq!( + route_front_matter["extra"]["search"]["base_path"].as_str(), + Some("/EIPs/") + ); + assert_eq!( + route_front_matter["extra"]["search"]["bundle_path"].as_str(), + Some("/EIPs/pagefind/") + ); + + let data = std::fs::read_to_string(&summary.data_path).unwrap(); + let data: TomlValue = toml::from_str(&data).unwrap(); + assert_eq!(data["enabled"].as_bool(), Some(true)); + assert_eq!(data["base_path"].as_str(), Some("/EIPs/")); + assert_eq!(data["bundle_path"].as_str(), Some("/EIPs/pagefind/")); + } + + #[test] + fn disabled_search_state_omits_bundle_path() { + let temp = TempDir::new().unwrap(); + let state = SearchTemplateState::from_base_url( + false, + &Url::parse("https://example.test/").unwrap(), + ); + + let summary = write_search_route(temp.path(), state).unwrap(); + + let route_front_matter = front_matter(&summary.route_path); + assert_eq!( + route_front_matter["extra"]["search"]["enabled"].as_bool(), + Some(false) + ); + assert!(route_front_matter["extra"]["search"] + .as_table() + .unwrap() + .get("bundle_path") + .is_none()); + + let data = std::fs::read_to_string(&summary.data_path).unwrap(); + let data: TomlValue = toml::from_str(&data).unwrap(); + assert_eq!(data["enabled"].as_bool(), Some(false)); + assert!(data.as_table().unwrap().get("bundle_path").is_none()); + } +} diff --git a/src/search/pagefind.rs b/src/search/pagefind.rs new file mode 100644 index 0000000..dcd9555 --- /dev/null +++ b/src/search/pagefind.rs @@ -0,0 +1,261 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +use std::{ + fs, + io::ErrorKind, + path::{Path, PathBuf}, +}; + +use log::info; +use pagefind::api::PagefindIndex; +use snafu::{OptionExt, ResultExt, Whatever}; +use tokio::runtime::Builder; + +use super::{SearchIndexRequest, SearchIndexSummary}; + +const PAGEFIND_DIR: &str = "pagefind"; + +pub(super) fn index_site(request: SearchIndexRequest) -> Result { + let runtime = Builder::new_current_thread() + .enable_all() + .build() + .whatever_context("unable to create Pagefind runtime")?; + + runtime.block_on(index_site_async(request)) +} + +async fn index_site_async(request: SearchIndexRequest) -> Result { + let (output_path, pagefind_path) = prepare_pagefind_output_path(&request.output_path)?; + let site_arg = pagefind_path_arg(&output_path, "rendered output directory")?; + let output_arg = pagefind_path_arg(&pagefind_path, "Pagefind output directory")?; + + info!( + "indexing rendered output for search from `{}`", + output_path.to_string_lossy() + ); + + let mut index = PagefindIndex::new(None).whatever_context("unable to create Pagefind index")?; + let pages_indexed = index + .add_directory(site_arg, None) + .await + .whatever_context("unable to index rendered output with Pagefind")?; + let written_path = index + .write_files(Some(output_arg)) + .await + .whatever_context("unable to write Pagefind assets")?; + + let output_path = PathBuf::from(written_path); + info!( + "Pagefind indexed {pages_indexed} page(s) into `{}`", + output_path.to_string_lossy() + ); + + Ok(SearchIndexSummary { + pages_indexed, + output_path, + }) +} + +fn prepare_pagefind_output_path(output_path: &Path) -> Result<(PathBuf, PathBuf), Whatever> { + let output_path = output_path.canonicalize().with_whatever_context(|_| { + format!( + "unable to resolve rendered output directory `{}`", + output_path.to_string_lossy() + ) + })?; + let pagefind_path = output_path.join(PAGEFIND_DIR); + + cleanup_stale_pagefind_path(&output_path, &pagefind_path)?; + + Ok((output_path, pagefind_path)) +} + +fn cleanup_stale_pagefind_path(output_path: &Path, pagefind_path: &Path) -> Result<(), Whatever> { + let metadata = match fs::symlink_metadata(pagefind_path) { + Ok(metadata) => metadata, + Err(error) if matches!(error.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => { + return Ok(()); + } + Err(error) => { + return Err(error).with_whatever_context(|_| { + format!( + "unable to inspect stale Pagefind output path `{}`", + pagefind_path.to_string_lossy() + ) + }); + } + }; + + let cleanup_target = pagefind_path.canonicalize().with_whatever_context(|_| { + format!( + "unable to resolve stale Pagefind output path `{}`", + pagefind_path.to_string_lossy() + ) + })?; + + if !is_strict_descendant(&cleanup_target, output_path) { + snafu::whatever!( + "refusing to clean Pagefind output path `{}` because it resolves outside rendered output directory `{}`", + cleanup_target.to_string_lossy(), + output_path.to_string_lossy() + ); + } + + let file_type = metadata.file_type(); + if file_type.is_symlink() || metadata.is_file() { + fs::remove_file(pagefind_path).with_whatever_context(|_| { + format!( + "unable to remove stale Pagefind output file `{}`", + pagefind_path.to_string_lossy() + ) + })?; + } else if metadata.is_dir() { + fs::remove_dir_all(pagefind_path).with_whatever_context(|_| { + format!( + "unable to remove stale Pagefind output directory `{}`", + pagefind_path.to_string_lossy() + ) + })?; + } else { + snafu::whatever!( + "refusing to clean unsupported Pagefind output path type `{}`", + pagefind_path.to_string_lossy() + ); + } + + Ok(()) +} + +fn is_strict_descendant(path: &Path, parent: &Path) -> bool { + path.starts_with(parent) && path != parent +} + +fn pagefind_path_arg(path: &Path, role: &str) -> Result { + path.to_str().map(str::to_owned).with_whatever_context(|| { + format!( + "unable to pass non-UTF-8 {role} `{}` to Pagefind", + path.to_string_lossy() + ) + }) +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use tempfile::TempDir; + + use crate::search::SearchIndexRequest; + + use super::{index_site, prepare_pagefind_output_path, PAGEFIND_DIR}; + + fn write_file(root: &Path, relative: impl AsRef, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn rendered_site(root: &Path) -> PathBuf { + let output_path = root.join("output"); + write_file( + &output_path, + "index.html", + "Search Fixture

Search Fixture

Rendered proposal body.

", + ); + output_path + } + + fn pagefind_filenames(pagefind_path: &Path) -> Vec { + walkdir::WalkDir::new(pagefind_path) + .into_iter() + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_file()) + .map(|entry| { + entry + .path() + .strip_prefix(pagefind_path) + .unwrap() + .to_string_lossy() + .into_owned() + }) + .collect() + } + + fn assert_pagefind_artifacts(pagefind_path: &Path) { + let filenames = pagefind_filenames(pagefind_path); + + assert!(pagefind_path.join("pagefind.js").is_file()); + assert!(pagefind_path.join("pagefind-entry.json").is_file()); + assert!( + filenames + .iter() + .any(|filename| filename.ends_with(".pf_index")), + "expected at least one Pagefind index artifact, found {filenames:?}" + ); + } + + #[test] + fn pagefind_writes_under_rendered_output_tree() { + let temp = TempDir::new().unwrap(); + let output_path = rendered_site(temp.path()); + let prepared_source_path = temp.path().join("repo"); + std::fs::create_dir_all(&prepared_source_path).unwrap(); + + let summary = index_site(SearchIndexRequest { + output_path: output_path.clone(), + }) + .unwrap(); + + assert!(summary.pages_indexed >= 1); + assert_eq!(summary.output_path, output_path.join(PAGEFIND_DIR)); + assert_pagefind_artifacts(&output_path.join(PAGEFIND_DIR)); + assert!(!prepared_source_path.join(PAGEFIND_DIR).exists()); + assert!(!temp.path().join(PAGEFIND_DIR).exists()); + } + + #[test] + fn stale_pagefind_output_is_removed_before_indexing() { + let temp = TempDir::new().unwrap(); + let output_path = rendered_site(temp.path()); + let stale_path = output_path.join(PAGEFIND_DIR).join("stale.txt"); + write_file( + &output_path, + PathBuf::from(format!("{PAGEFIND_DIR}/stale.txt")), + "stale", + ); + + index_site(SearchIndexRequest { + output_path: output_path.clone(), + }) + .unwrap(); + + assert!(!stale_path.exists()); + assert_pagefind_artifacts(&output_path.join(PAGEFIND_DIR)); + } + + #[cfg(target_family = "unix")] + #[test] + fn stale_pagefind_cleanup_rejects_targets_outside_output_tree() { + use std::os::unix::fs::symlink; + + let temp = TempDir::new().unwrap(); + let output_path = rendered_site(temp.path()); + let outside_path = temp.path().join("outside-pagefind"); + std::fs::create_dir_all(&outside_path).unwrap(); + write_file(&outside_path, "external.txt", "outside"); + symlink(&outside_path, output_path.join(PAGEFIND_DIR)).unwrap(); + + let error = prepare_pagefind_output_path(&output_path) + .unwrap_err() + .to_string(); + + assert!(error.contains("refusing to clean Pagefind output path")); + assert!(outside_path.join("external.txt").is_file()); + } +}