diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 39b359c..9501934 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,6 +40,11 @@ jobs: cd target/release $targetPath = Join-Path -Path (Resolve-Path ..\..\dist) -ChildPath $Env:EIPS_BUILD_ARCHIVE Compress-Archive -Path build-eips.exe -DestinationPath $targetPath + $archivePath = $targetPath + $archiveName = Split-Path -Leaf $archivePath + $sidecarPath = "$archivePath.sha256" + $hash = (Get-FileHash -Algorithm SHA256 -Path $archivePath).Hash.ToLower() + [System.IO.File]::WriteAllText($sidecarPath, "$hash $archiveName`n") - name: Compress (Unix) if: matrix.os != 'windows' env: @@ -47,6 +52,12 @@ jobs: run: | mkdir dist tar cavf "$EIPS_BUILD_ARCHIVE" -C target/release build-eips + archive_name=$(basename "$EIPS_BUILD_ARCHIVE") + if [ "${{ matrix.os }}" = "macos" ]; then + (cd dist && shasum -a 256 "$archive_name" > "$archive_name.sha256") + else + (cd dist && sha256sum "$archive_name" > "$archive_name.sha256") + fi - name: Release uses: softprops/action-gh-release@c95fe1489396fe8a9eb87c0abf8aa5b2ef267fda with: diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..26e8a93 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,184 @@ +# Architecture Overview + +This document describes the `build-eips` system model: its system layers, repository responsibilities, trust boundaries, source and generated state, data flow, and the separation between validation, transformation, materialization, and rendering. + +`build-eips` coordinates independent proposal, theme, and tooling repositories into a prepared render tree. The source repositories remain authoritative for their own data, while generated state is isolated to the resolved build root and consumed by validation and rendering paths. + +Workspace setup, command usage, and Rust module ownership are documented in `README.md`, `src/workspace_doc.md`, and `src/README.md`, respectively. + +## System Layers + +The system is organized into layers with separate responsibilities: + +- source repositories: canonical proposal content, tracked topology metadata, theme source files, and tooling crates +- local workspace overlay: local operator state and local checkout paths that let independent repositories work as one system +- preprocessor orchestration: active repository identity, topology, source policy, execution resolution, and runtime handoff +- disposable prepared state: generated Git worktrees, mounted theme copies, and rendered output under the resolved build root +- render/output surface: the prepared tree and mounted theme consumed by Zola, plus static output served by preview + +Validation, transformation, materialization, and rendering are pipeline responsibilities. They are not separate system layers. + +## Repository Responsibilities + +Each repository owns one part of the system: + +- proposal repositories own canonical content and `.build-eips.repo.toml` repo manifests +- the theme owns render templates, runtime configuration, styles, assets, and syntax definitions +- the preprocessor owns orchestration, source resolution, materialization, preprocessing, and runtime handoff +- eipw owns editorial validation rules consumed through Cargo dependencies such as `eipw-lint`, `eipw-preamble`, and `eipw-snippets` +- the template repository owns proposal scaffolding + +## Source And Generated State + +The architecture separates authoritative state from generated state: + +- proposal `content/`, `.build-eips.repo.toml` repo manifests, and theme files are source +- `.build-eips.repo.toml` provides source-controlled topology; its `repo_id` is the stable key used for workspace directories, default build roots, and sibling references +- `.build-eips.toml` is local workspace state for local server, site, and render defaults +- prepared repositories and rendered output are generated under the resolved build root +- the normal workspace build root is `.local-build/` +- without workspace config, the fallback build root is `/build` +- mounted theme copies are generated runtime state +- canonical proposal content is not rewritten by runtime paths + +## Trust Boundaries + +Inputs enter the system with different trust and ownership properties: + +- tracked `.build-eips.repo.toml` manifests are source-controlled topology +- `.build-eips.toml` is local operator state +- CLI flags are per-run user intent +- remote Git refs are fetched external state used for source materialization and merge-base comparison +- proposal markdown is source content that must be transformed and validated before rendering +- theme files are local source input and may include tracked or staged local theme changes during materialization +- prepared repositories, mounted themes, and rendered output are generated state + +## Identity Resolution + +Identity resolution determines the active proposal repository before topology is derived. + +The inputs are: + +- command anchor: current working directory or `-C` +- active proposal repository root +- tracked `.build-eips.repo.toml` manifest, when present +- built-in EIPs/ERCs identifying-commit lookup, when no manifest is present + +The resolution order is manifest-backed identity first, then built-in EIPs/ERCs compatibility identity for checkouts without tracked manifests. + +## Topology Resolution + +Topology follows identity: + +- sibling repositories come from the manifest or built-in compatibility metadata +- remote endpoints come from the selected repository identity +- local workspace paths come from workspace discovery and local config +- build roots come from local workspace defaults or per-run overrides + +## Source Policy + +Source policy determines which repository state is materialized or consumed after identity and topology are resolved. + +| Policy | Active repository source | Dirty active edits | Sibling source | Environment metadata | +| --- | --- | --- | --- | --- | +| Local dirty site validation/rendering | local active checkout | tracked edits included | workspace-local siblings | local site defaults with selected upstream metadata | +| Clean local site validation/rendering | local active checkout | rejected before materialization | workspace-local siblings | local site defaults with selected upstream metadata | +| Remote-sibling site validation/rendering | local active checkout | follows the selected local dirty/clean policy | selected remote sibling endpoints | selected environment metadata | +| Explicit staging, production, and parity | local active checkout | rejected before materialization | selected remote sibling endpoints | selected environment metadata | +| Changed-file comparison and upstream editorial target selection | local active checkout through prepared Git state | clean comparison | not render topology | selected upstream merge base | +| Editorial lint with explicit, batch, or working-tree selectors | selected files in the active checkout | consumed directly by selector | not render topology | lint configuration from the local theme | + +Staging, production, and parity paths do not replace the active render source with a remote active checkout. They require the local active checkout to be clean, use the selected environment metadata for upstream comparison and rendered base URLs, and resolve sibling proposal repositories from remote endpoints. + +Editorial check composes editorial lint with the site-level check path. `--build-root` changes the resolved generated-state location, and `--only` narrows proposal rendering only for local dirty rendering paths; default render selection can also come from `[render].only` in `.build-eips.toml`. + +## Overlay Model + +The overlay model describes the render-time proposal surface: + +- independent proposal repositories are not merged as source repositories +- sibling proposal content is overlaid into one prepared render tree +- the theme is materialized into the prepared tree at `themes/eips-theme` +- the overlay exists only in generated state +- the prepared render tree is the only proposal surface seen by Zola + +Git mechanics belong to the materialization boundary. The overlay model is the architectural result of that materialization. + +## Data Flow + +The system flow has an execution-resolution step, a validation branch, and a prepared-state branch: + +```text +active repo + sibling repos + theme + workspace config + | + v +execution resolution + |-------------------------------| + v v +editorial/changed-file validation prepared build repo + | + v + markdown/proposal transformation + | + v + theme mount + Zola config + | + v + Zola validation or rendering + | + v + static output / preview surface +``` + +Some validation work happens before or beside materialization, while Zola validation and rendering happen against prepared state. `changed` and `editorial --against-upstream` still use prepared Git state to compute merge-base differences. They bypass markdown transformation and Zola; they do not bypass Git source preparation entirely. + +## Materialization Boundary + +Materialization is the boundary between source repositories and runtime state: + +- runtime paths do not render directly from source checkouts +- the preprocessor creates a disposable prepared repository +- active and sibling proposal content are merged into that prepared tree +- tracked dirty active-repo state is included only when the selected source mode allows it +- dirty active-repo materialization ignores untracked active-repo files +- clean source materialization rejects dirty or untracked active-repo state before proceeding +- theme materialization includes tracked theme files plus tracked or staged theme working-tree changes, independent of active-repo dirty mode +- canonical source content is not rewritten + +## Pipeline Responsibilities + +The pipeline separates kinds of work: + +- identity and topology resolution: decide what the runtime path operates on +- materialization: Git clone/fetch/dirty/sibling/theme staging +- transformation: markdown preprocessing, front matter, links, `requires`, and citations +- validation: editorial lint, changed-file selection, and Zola check behavior +- rendering: Zola build and serve + +Preview is the exception: it serves prior rendered output and bypasses materialization, transformation, and Zola. + +Editorial validation is a peer of rendering. It consumes the same identity and source model, and the editorial check path composes with the runtime check path when site-level validation is required. + +## Invariants + +These rules keep the system coherent: + +- canonical proposal content is not rewritten by runtime paths +- generated output lives under the resolved build root +- Zola sees one prepared repo and one mounted theme +- sibling content enters only through materialization +- workspace-local state is not canonical proposal content + +## Failure Boundaries + +The system halts at boundaries that would otherwise make prepared state ambiguous or invalid: + +- prepared-state mutation paths use `/.lock`; preview bypasses the lock because it serves existing output, and clean unlocks before removing the build root +- missing required local theme halts Zola-backed and editorial lint paths +- unresolved active repository identity halts the run +- sibling content path conflicts halt materialization +- invalid discovered workspace config halts execution-resolved runtime paths + +## Deferred Or Compatibility Paths + +Tracked `.build-eips.repo.toml` manifests are the normal topology description. Built-in EIPs/ERCs identity handles checkouts without tracked manifests. The template manifest exists, but supported raw-template bootstrap/developer workflow is not first-class yet. diff --git a/Cargo.lock b/Cargo.lock index dbca5a3..496a691 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,6 +134,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + [[package]] name = "atomic" version = "0.6.1" @@ -185,6 +191,12 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.10.0" @@ -237,7 +249,6 @@ dependencies = [ "chrono", "citationberg", "clap", - "directories", "duct", "eipw-lint", "eipw-preamble", @@ -252,14 +263,17 @@ dependencies = [ "iref", "lazy_static", "log", + "notify", "pulldown-cmark", "pulldown-cmark-to-cmark", "regex", "semver", "serde", "serde_json", - "sha3", + "serde_yaml", "snafu", + "tempfile", + "tiny_http", "tokio", "toml 0.9.11+spec-1.1.0", "toml_datetime 0.7.5+spec-1.1.0", @@ -340,6 +354,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "chunked_transfer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" + [[package]] name = "ciborium" version = "0.2.2" @@ -466,6 +486,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crunchy" version = "0.2.4" @@ -532,22 +567,13 @@ dependencies = [ "crypto-common", ] -[[package]] -name = "directories" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" -dependencies = [ - "dirs-sys 0.5.0", -] - [[package]] name = "dirs" version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" dependencies = [ - "dirs-sys 0.4.1", + "dirs-sys", ] [[package]] @@ -558,22 +584,10 @@ checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" dependencies = [ "libc", "option-ext", - "redox_users 0.4.6", + "redox_users", "windows-sys 0.48.0", ] -[[package]] -name = "dirs-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" -dependencies = [ - "libc", - "option-ext", - "redox_users 0.5.2", - "windows-sys 0.61.2", -] - [[package]] name = "displaydoc" version = "0.2.5" @@ -785,6 +799,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + [[package]] name = "figment" version = "0.10.19" @@ -798,6 +818,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -846,6 +877,15 @@ dependencies = [ "num", ] +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "fslock" version = "0.2.1" @@ -929,7 +969,7 @@ version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" dependencies = [ - "bitflags", + "bitflags 2.10.0", "libc", "libgit2-sys", "log", @@ -1044,6 +1084,12 @@ dependencies = [ "match_token", ] +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -1214,6 +1260,26 @@ dependencies = [ "rustversion", ] +[[package]] +name = "inotify" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + [[package]] name = "iref" version = "3.2.2" @@ -1333,6 +1399,26 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "kqueue" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1341,9 +1427,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.180" +version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" [[package]] name = "libgit2-sys" @@ -1365,8 +1451,9 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ - "bitflags", + "bitflags 2.10.0", "libc", + "redox_syscall 0.7.4", ] [[package]] @@ -1395,6 +1482,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.1" @@ -1488,6 +1581,18 @@ dependencies = [ "adler2", ] +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.48.0", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -1504,6 +1609,25 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "notify" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" +dependencies = [ + "bitflags 2.10.0", + "crossbeam-channel", + "filetime", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "walkdir", + "windows-sys 0.48.0", +] + [[package]] name = "num" version = "0.4.3" @@ -1673,7 +1797,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link", ] @@ -1833,7 +1957,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" dependencies = [ - "bitflags", + "bitflags 2.10.0", "getopts", "memchr", "pulldown-cmark-escape", @@ -1907,29 +2031,27 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.10.0", ] [[package]] -name = "redox_users" -version = "0.4.6" +name = "redox_syscall" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" dependencies = [ - "getrandom 0.2.17", - "libredox", - "thiserror 1.0.69", + "bitflags 2.10.0", ] [[package]] name = "redox_users" -version = "0.5.2" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.18", + "thiserror 1.0.69", ] [[package]] @@ -2007,6 +2129,19 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.10.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -2078,7 +2213,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8" dependencies = [ - "bitflags", + "bitflags 2.10.0", "cssparser", "derive_more", "fxhash", @@ -2424,6 +2559,19 @@ dependencies = [ "libc", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "tendril" version = "0.4.3" @@ -2475,6 +2623,18 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii", + "chunked_transfer", + "httpdate", + "log", +] + [[package]] name = "tinystr" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index d87e3e7..17652cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,6 @@ depends = "$auto, libc6, libssl3, zlib1g, libgcc-s1, libgit2-1.5, git" [dependencies] chrono = "0.4.42" clap = { version = "4.5.53", features = ["cargo", "derive"] } -directories = "6.0.0" duct = "1.1.1" eipw-lint = { version = "0.10.0", features = [ "tokio", "schema-version" ] } eipw-snippets = "0.2.0" @@ -34,14 +33,16 @@ indicatif = "0.18.3" indicatif-log-bridge = "0.2.3" lazy_static = "1.5.0" log = { version = "0.4.29", features = ["std"] } +notify = "6.1.1" pulldown-cmark = "0.13.0" pulldown-cmark-to-cmark = "22.0.0" regex = "1.12.2" semver = {version = "1.0.27", features = ["serde"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.148" -sha3 = "0.10.8" +serde_yaml = "0.9.34" snafu = { version = "0.8.9", features = ["rust_1_81"] } +tiny_http = "0.12.0" tokio = { version = "1.48.0", features = ["fs", "rt", "macros"] } toml = "0.9.10" toml_datetime = { version = "0.7.5", features = ["serde"] } @@ -53,3 +54,6 @@ iref = "3.2.2" [features] backtrace = [ "snafu/backtrace", "eipw-lint/backtrace" ] + +[dev-dependencies] +tempfile = "3.23.0" diff --git a/README.md b/README.md index 8e6293e..fe5a569 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,111 @@ build-eips ========== -Build system for linting and rendering Ethereum Improvement Proposals ([EIPs] / -[ERCs]). +`build-eips` is the local and CI build tool for Ethereum Improvement Proposal repositories. It prepares a multi-repo workspace with [EIPs], [ERCs] and the site theme, runs proposal validation, and coordinates the full site build pipeline. ## Prerequisites -`build-eips` requires a few runtime dependencies, available from wherever you -get your software: +`build-eips` requires a few runtime dependencies, available from wherever you get your software: - git - libgit2 - openssl -- [zola](https://github.com/getzola/zola/tree/next)[^1] +- [Zola](https://github.com/getzola/zola) 0.22.1[^1] -[^1]: Requires at least commit [`ead17d0a3`] for full functionality. - -[`ead17d0a3`]: https://github.com/getzola/zola/commit/ead17d0a3a20bfb67043a076c061b35ae6b6ddea +[^1]: The setup scripts can install or reuse the pinned Zola version for local workspace testing. ## Installation ### Pre-compiled Binaries -Pre-compiled binaries for Ubuntu, Windows, and macOS are available from -[GitHub Releases]. +Pre-compiled binaries for Ubuntu, Windows, and macOS are available from [GitHub Releases]. [GitHub Releases]: https://github.com/ethereum/build-eips/releases ### From Source -If you're feeling particularly adventurous, you can install the latest version -of `build-eips` like so: +If you're feeling particularly adventurous, you can install the latest version of `build-eips` like so: -```bash +```sh cargo install --git https://github.com/ethereum/build-eips.git ``` [EIPs]: https://github.com/ethereum/EIPs/ [ERCs]: https://github.com/ethereum/ERCs/ +### Local Testing + +For direct manual testing, build and run the local binary explicitly: + +```bash +cargo build +./target/debug/build-eips --help +./target/debug/build-eips -C ../EIPs check +``` + +```powershell +cargo build +.\target\debug\build-eips.exe --help +.\target\debug\build-eips.exe -C ..\EIPs check +``` + +For repeated manual testing, you can put the local debug binary first on `PATH` for the current shell: + +```bash +export PATH="$PWD/target/debug:$PATH" +build-eips --help +build-eips -C ../EIPs check +``` + +```powershell +$env:Path = "$PWD\target\debug;$env:Path" +build-eips --help +build-eips -C ..\EIPs check +``` + +## Workspace Bootstrap + +Use the contributor setup script when you are changing `build-eips` itself and need the full local multi-repo workspace that runs this checkout. + +That workspace lets you test the proposal validation and site build pipeline against local EIPs/ERCs/theme checkouts without installing or reusing a released `build-eips`. + +Linux and macOS: + +```sh +./scripts/dev-setup +``` + +Windows PowerShell: + +```powershell +.\scripts\dev-setup.ps1 +``` + +The setup script builds the local debug binary, ensures the pinned Zola version is available, bootstraps the workspace with that binary, and runs `doctor`. It does not install or reuse a released `build-eips`. + +The script anchors setup through `../EIPs` by default and clones `https://github.com/ethereum/EIPs.git` there when that checkout is missing. Set `ACTIVE_REPO_ROOT` to use another proposal repo checkout. + +After setup, the workspace has this layout: + +```text +EIPs-project/ +├── .build-eips.toml +├── WORKSPACE.md +├── .local-build/ +├── EIPs/ +├── ERCs/ +├── theme/ +├── preprocessor/ +└── eipw/ +``` + +## Workspace Reference + +After bootstrapping the workspace, use the generated workspace guide at `../WORKSPACE.md` for the full local command reference, workspace layout, configuration, source modes, build outputs, and troubleshooting notes. -## Usage +The generated guide is built from [`src/workspace_doc.md`](src/workspace_doc.md). Update that source file when changing workspace documentation. -1. Clone either [`ethereum/EIPs`] or [`ethereum/ERCs`], and change directory - into it. -1. Modify whatever proposal you'd like. -1. Commit your changes. -1. Build the project. You can use: - - `build-eips check` to quickly check for problems like missing sections, - broken internal links, etc. - - `build-eips build` to create an on-disk bundle of HTML, ready to be - deployed. - - `build-eips serve` to launch a web server to preview changes locally. - **NB: live reload is not yet implemented.** +Use `build-eips doctor` when the workspace does not behave as expected. It checks the active repo, sibling repos, theme checkout, local config, required tools, and generated workspace docs. [`ethereum/EIPs`]: https://github.com/ethereum/EIPs/ [`ethereum/ERCs`]: https://github.com/ethereum/ERCs/ diff --git a/scripts/dev-setup b/scripts/dev-setup new file mode 100755 index 0000000..46fbbc3 --- /dev/null +++ b/scripts/dev-setup @@ -0,0 +1,558 @@ +#!/bin/sh + +set -eu + +DEFAULT_ACTIVE_REPO_URL=https://github.com/ethereum/EIPs.git + +say() { + printf '%s\n' "$*" +} + +die() { + printf 'error: %s\n' "$*" >&2 + exit 1 +} + +shell_quote() { + printf "'%s'" "$(printf '%s' "$1" | sed "s/'/'\\\\''/g")" +} + +absolute_path() { + case "$1" in + /*) + printf '%s\n' "$1" + ;; + *) + printf '%s/%s\n' "$INVOCATION_DIR" "$1" + ;; + esac +} + +ensure_active_repo_root() { + if [ "$ACTIVE_REPO_EXPLICIT" = true ]; then + if [ ! -d "$ACTIVE_REPO_ROOT" ]; then + die "configured ACTIVE_REPO_ROOT does not exist: $ACTIVE_REPO_ROOT. Fix ACTIVE_REPO_ROOT or unset it to let this script clone EIPs." + fi + if [ ! -e "$ACTIVE_REPO_ROOT/.git" ]; then + die "configured ACTIVE_REPO_ROOT is not a git checkout: $ACTIVE_REPO_ROOT. Fix ACTIVE_REPO_ROOT or unset it to let this script clone EIPs." + fi + else + if [ ! -e "$ACTIVE_REPO_ROOT" ]; then + command -v git >/dev/null 2>&1 || die "need git to clone default proposal repo from $DEFAULT_ACTIVE_REPO_URL" + + say "No active proposal repo found at $ACTIVE_REPO_ROOT." + say "Cloning default proposal repo from $DEFAULT_ACTIVE_REPO_URL" + say "This may take a few minutes..." + if ! git clone "$DEFAULT_ACTIVE_REPO_URL" "$ACTIVE_REPO_ROOT"; then + die "failed to clone default proposal repo from $DEFAULT_ACTIVE_REPO_URL to $ACTIVE_REPO_ROOT" + fi + say "Cloned default proposal repo to $ACTIVE_REPO_ROOT" + fi + + if [ ! -d "$ACTIVE_REPO_ROOT" ]; then + die "default active proposal repo path exists but is not a directory: $ACTIVE_REPO_ROOT. Remove it or set ACTIVE_REPO_ROOT." + fi + if [ ! -e "$ACTIVE_REPO_ROOT/.git" ]; then + die "default active proposal repo path exists but is not a git checkout: $ACTIVE_REPO_ROOT. Remove it or set ACTIVE_REPO_ROOT." + fi + fi + + ACTIVE_REPO_ROOT=$(CDPATH= cd -- "$ACTIVE_REPO_ROOT" && pwd) +} + +normalize_dir_for_path_compare() { + dir=$1 + [ -n "$dir" ] || return 1 + + while [ "$dir" != "/" ] && [ "${dir%/}" != "$dir" ]; do + dir=${dir%/} + done + + case "$dir" in + /*) + absolute_dir=$dir + ;; + *) + absolute_dir=$(pwd -P)/$dir + ;; + esac + + if [ -d "$absolute_dir" ]; then + (CDPATH= cd -P -- "$absolute_dir" 2>/dev/null && pwd -P) + else + printf '%s\n' "$absolute_dir" + fi +} + +move_dir_to_front_of_script_path() { + dir=$1 + target=$(normalize_dir_for_path_compare "$dir") || return 1 + new_path= + old_ifs=$IFS + IFS=: + for path_entry in ${PATH:-}; do + [ -n "$path_entry" ] || continue + candidate=$(normalize_dir_for_path_compare "$path_entry") || continue + if [ "$candidate" = "$target" ]; then + continue + fi + + if [ -n "$new_path" ]; then + new_path=$new_path:$path_entry + else + new_path=$path_entry + fi + done + IFS=$old_ifs + + if [ -n "$new_path" ]; then + updated_path=$dir:$new_path + else + updated_path=$dir + fi + + if [ "${PATH:-}" != "$updated_path" ]; then + PATH=$updated_path + add_path_note "$dir" + fi +} + +add_path_note() { + dir=$1 + target=$(normalize_dir_for_path_compare "$dir") || return 1 + old_ifs=$IFS + IFS=' +' + for path_note in $PATH_NOTES; do + candidate=$(normalize_dir_for_path_compare "$path_note") || continue + if [ "$candidate" = "$target" ]; then + IFS=$old_ifs + return 0 + fi + done + IFS=$old_ifs + + case " +$PATH_NOTES +" in + *" +$dir +"*) + return 0 + ;; + esac + + if [ -n "$PATH_NOTES" ]; then + PATH_NOTES=$PATH_NOTES' +'$dir + else + PATH_NOTES=$dir + fi +} + +pick_home_install_dir() { + [ -n "${HOME:-}" ] || return 1 + dir=$HOME/.local/bin + mkdir -p "$dir" 2>/dev/null || return 1 + [ -d "$dir" ] || return 1 + [ -w "$dir" ] || return 1 + printf '%s\n' "$dir" +} + +pick_writable_path_dir() { + old_ifs=$IFS + IFS=: + for dir in ${PATH:-}; do + [ -n "$dir" ] || continue + [ -d "$dir" ] || continue + [ -w "$dir" ] || continue + printf '%s\n' "$dir" + IFS=$old_ifs + return 0 + done + IFS=$old_ifs + return 1 +} + +pick_install_dir() { + if dir=$(pick_home_install_dir); then + printf '%s\n' "$dir" + return 0 + fi + + pick_writable_path_dir +} + +ensure_install_dir() { + [ -n "${INSTALL_DIR:-}" ] && return 0 + + INSTALL_DIR=$(pick_install_dir) || die "could not find a writable install directory" + move_dir_to_front_of_script_path "$INSTALL_DIR" +} + +download() { + url=$1 + destination=$2 + + if command -v curl >/dev/null 2>&1; then + curl -fsSL "$url" -o "$destination" + return 0 + fi + + if command -v wget >/dev/null 2>&1; then + wget -qO "$destination" "$url" + return 0 + fi + + die "need curl or wget to download release binaries" +} + +file_sha256() { + path=$1 + + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$path" | awk '{ print $1 }' | tr 'A-F' 'a-f' + return 0 + fi + + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$path" | awk '{ print $1 }' | tr 'A-F' 'a-f' + return 0 + fi + + if command -v openssl >/dev/null 2>&1; then + openssl dgst -sha256 -r "$path" | awk '{ print $1 }' | tr 'A-F' 'a-f' + return 0 + fi + + die "need sha256sum, shasum, or openssl to verify release checksums" +} + +verify_file_hash() { + archive_path=$1 + expected_hash=$2 + archive_name=$3 + + actual_hash=$(file_sha256 "$archive_path") + [ "$actual_hash" = "$expected_hash" ] || die "checksum mismatch for $archive_name" +} + +cleanup_tmpdir() { + if [ -n "${INSTALL_TMP_TO_CLEAN:-}" ]; then + rm -f "$INSTALL_TMP_TO_CLEAN" + INSTALL_TMP_TO_CLEAN= + fi + if [ -n "${TMPDIR_TO_CLEAN:-}" ]; then + rm -rf "$TMPDIR_TO_CLEAN" + TMPDIR_TO_CLEAN= + fi +} + +cleanup_for_signal() { + cleanup_tmpdir + trap - EXIT INT TERM HUP QUIT + exit 1 +} + +set_cleanup_traps() { + trap cleanup_tmpdir EXIT + trap cleanup_for_signal INT TERM HUP QUIT +} + +clear_cleanup_traps() { + trap - EXIT INT TERM HUP QUIT +} + +unsupported_zola_platform() { + os=$1 + arch=$2 + die "Unsupported platform $os/$arch for automatic Zola install. Install Zola 0.22.1 manually from https://github.com/getzola/zola/releases and ensure it is on PATH." +} + +is_linux_musl() { + if command -v ldd >/dev/null 2>&1 && ldd --version 2>&1 | grep -qi musl; then + return 0 + fi + + set -- /lib/ld-musl-*.so.1 + [ -e "$1" ] +} + +select_zola_release() { + os=$(uname -s) + arch=$(uname -m) + + case "$os" in + Linux) + if is_linux_musl; then + case "$arch" in + x86_64) + ZOLA_ARCHIVE_NAME=zola-v0.22.1-x86_64-unknown-linux-musl.tar.gz + ZOLA_ARCHIVE_HASH=227df99b664421240a8ba77747067c51259b08159125d5603763b3b173b9a881 + ;; + *) + unsupported_zola_platform "$os" "$arch" + ;; + esac + else + case "$arch" in + x86_64) + ZOLA_ARCHIVE_NAME=zola-v0.22.1-x86_64-unknown-linux-gnu.tar.gz + ZOLA_ARCHIVE_HASH=0ca09aa40376aaa9ddfb512ff9ad963262ef95edb0d0f2d5ec6961b6f5cf22ef + ;; + aarch64|arm64) + ZOLA_ARCHIVE_NAME=zola-v0.22.1-aarch64-unknown-linux-gnu.tar.gz + ZOLA_ARCHIVE_HASH=8af437ec6352f33ccd24d7a1cfcb54a3db95d3ce376dc69525b4ef3fb6b8c1d1 + ;; + *) + unsupported_zola_platform "$os" "$arch" + ;; + esac + fi + ;; + Darwin) + case "$arch" in + x86_64) + ZOLA_ARCHIVE_NAME=zola-v0.22.1-x86_64-apple-darwin.tar.gz + ZOLA_ARCHIVE_HASH=3898709e154ae0593933264a540c869348bdb10d7f1b03a42dfb78d63703b3b5 + ;; + arm64|aarch64) + ZOLA_ARCHIVE_NAME=zola-v0.22.1-aarch64-apple-darwin.tar.gz + ZOLA_ARCHIVE_HASH=46ac45a9e7628dba8593b124ee8794f4f9aa1c6b569918ecd4bbc5d0be190515 + ;; + *) + unsupported_zola_platform "$os" "$arch" + ;; + esac + ;; + *) + unsupported_zola_platform "$os" "$arch" + ;; + esac +} + +parse_zola_version() { + output=$1 + + set -f + set -- $output + set +f + [ "$#" -ge 2 ] || return 1 + + ZOLA_FOUND_VERSION=$2 + version_core=$(printf '%s\n' "$ZOLA_FOUND_VERSION" | sed -n 's/^\([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\).*/\1 \2 \3/p') + [ -n "$version_core" ] || return 1 + + set -- $version_core + ZOLA_VERSION_MAJOR=$1 + ZOLA_VERSION_MINOR=$2 + ZOLA_VERSION_PATCH=$3 + ZOLA_VERSION_SUFFIX=$(printf '%s\n' "$ZOLA_FOUND_VERSION" | sed -n 's/^[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*//p') + + return 0 +} + +zola_version_relation() { + if [ "$ZOLA_VERSION_MAJOR" -lt 0 ]; then + printf '%s\n' below + elif [ "$ZOLA_VERSION_MAJOR" -gt 0 ]; then + printf '%s\n' newer + elif [ "$ZOLA_VERSION_MINOR" -lt 22 ]; then + printf '%s\n' below + elif [ "$ZOLA_VERSION_MINOR" -gt 22 ]; then + printf '%s\n' newer + elif [ "$ZOLA_VERSION_PATCH" -lt 1 ]; then + printf '%s\n' below + elif [ "$ZOLA_VERSION_PATCH" -gt 1 ]; then + printf '%s\n' newer + elif [ -n "$ZOLA_VERSION_SUFFIX" ]; then + printf '%s\n' below + else + printf '%s\n' equal + fi +} + +install_zola() { + command -v tar >/dev/null 2>&1 || die "need tar to unpack the zola release archive" + select_zola_release + ensure_install_dir + + tmpdir=$(mktemp -d) + TMPDIR_TO_CLEAN=$tmpdir + set_cleanup_traps + + archive_path=$tmpdir/$ZOLA_ARCHIVE_NAME + release_base_url=https://github.com/getzola/zola/releases/download/v0.22.1 + + say "Installing zola 0.22.1 from $release_base_url/$ZOLA_ARCHIVE_NAME" + download "$release_base_url/$ZOLA_ARCHIVE_NAME" "$archive_path" + verify_file_hash "$archive_path" "$ZOLA_ARCHIVE_HASH" "$ZOLA_ARCHIVE_NAME" + + tar -xzf "$archive_path" -C "$tmpdir" + [ -f "$tmpdir/zola" ] || die "zola release archive did not contain zola" + + install_tmp=$(mktemp "$INSTALL_DIR/.zola.XXXXXX") || die "could not create temporary zola install file in $INSTALL_DIR" + INSTALL_TMP_TO_CLEAN=$install_tmp + cp "$tmpdir/zola" "$install_tmp" + chmod +x "$install_tmp" + mv "$install_tmp" "$INSTALL_DIR/zola" + INSTALL_TMP_TO_CLEAN= + + ZOLA=$INSTALL_DIR/zola + cleanup_tmpdir + clear_cleanup_traps +} + +ensure_zola() { + if command -v zola >/dev/null 2>&1; then + ZOLA=$(command -v zola) + elif [ -n "${HOME:-}" ] && [ -x "$HOME/.local/bin/zola" ]; then + INSTALL_DIR=$HOME/.local/bin + ZOLA=$INSTALL_DIR/zola + move_dir_to_front_of_script_path "$INSTALL_DIR" + else + install_zola + return 0 + fi + + if [ -n "${ZOLA:-}" ]; then + zola_output=$("$ZOLA" --version 2>/dev/null || true) + if parse_zola_version "$zola_output"; then + relation=$(zola_version_relation) + case "$relation" in + below) + say "Found zola $ZOLA_FOUND_VERSION below supported 0.22.1. Installing zola 0.22.1." + install_zola + ;; + equal) + say "Using existing zola $ZOLA_FOUND_VERSION at $ZOLA" + ;; + newer) + say "Found zola $ZOLA_FOUND_VERSION. build-eips is tested with zola 0.22.1 or newer. Continuing with the installed version." + ;; + esac + else + say "Found zola with unparseable version output. Installing zola 0.22.1." + install_zola + fi + fi +} + +INCLUDE_TEMPLATE=false + +while [ "$#" -gt 0 ]; do + case "$1" in + --template) + INCLUDE_TEMPLATE=true + ;; + -h|--help) + say "usage: $0 [--template]" + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac + shift +done + +INVOCATION_DIR=$(pwd) +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +PREPROCESSOR_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd) +INSTALL_DIR= +PATH_NOTES= +TMPDIR_TO_CLEAN= +INSTALL_TMP_TO_CLEAN= + +if [ -n "${WORKSPACE_ROOT:-}" ]; then + WORKSPACE_ROOT=$(absolute_path "$WORKSPACE_ROOT") +else + WORKSPACE_ROOT=$(CDPATH= cd -- "$PREPROCESSOR_ROOT/.." && pwd) +fi +if [ -d "$WORKSPACE_ROOT" ]; then + WORKSPACE_ROOT=$(CDPATH= cd -- "$WORKSPACE_ROOT" && pwd) +fi + +if [ -n "${ACTIVE_REPO_ROOT:-}" ]; then + ACTIVE_REPO_EXPLICIT=true + ACTIVE_REPO_ROOT=$(absolute_path "$ACTIVE_REPO_ROOT") +else + ACTIVE_REPO_EXPLICIT=false + ACTIVE_REPO_ROOT=$WORKSPACE_ROOT/EIPs +fi + +ensure_active_repo_root +BUILD_EIPS=$PREPROCESSOR_ROOT/target/debug/build-eips + +say "Workspace root: $WORKSPACE_ROOT" +say "Active proposal repo: $ACTIVE_REPO_ROOT" +say "Local build-eips: $BUILD_EIPS" + +command -v cargo >/dev/null 2>&1 || die "need cargo to build local build-eips. Install Rust from https://rustup.rs/ and re-run this script." + +say "Building local build-eips" +if ! (CDPATH= cd -- "$PREPROCESSOR_ROOT" && cargo build); then + die "cargo build failed" +fi + +[ -x "$BUILD_EIPS" ] || die "local build-eips was not built at $BUILD_EIPS" + +ensure_zola + +say "Bootstrapping local contributor workspace" +if [ "$INCLUDE_TEMPLATE" = true ]; then + if ! "$BUILD_EIPS" -C "$ACTIVE_REPO_ROOT" init "$WORKSPACE_ROOT" --platform-dev --template; then + die "build-eips init failed" + fi +else + if ! "$BUILD_EIPS" -C "$ACTIVE_REPO_ROOT" init "$WORKSPACE_ROOT" --platform-dev; then + die "build-eips init failed" + fi +fi + +say "Running build-eips doctor" +if ! "$BUILD_EIPS" -C "$ACTIVE_REPO_ROOT" doctor; then + say "Warning: build-eips doctor reported issues above. Fix them before relying on local build-eips commands." +fi + +WORKSPACE_DOC=$WORKSPACE_ROOT/WORKSPACE.md +say "" +if [ -f "$WORKSPACE_DOC" ]; then + say "Workspace docs: $WORKSPACE_DOC" +else + say "Warning: workspace docs were not found at $WORKSPACE_DOC after build-eips init" +fi + +if [ -n "$PATH_NOTES" ]; then + say "" + say "Updated PATH for this script process only:" + old_ifs=$IFS + IFS=' +' + for path_note in $PATH_NOTES; do + say " $path_note" + done + say "To make this permanent in your shell, add:" + for path_note in $PATH_NOTES; do + say " export PATH=$(shell_quote "$path_note"):\$PATH" + done + IFS=$old_ifs +fi + +TARGET_DEBUG_DIR=$PREPROCESSOR_ROOT/target/debug + +say "" +say "Next commands:" +say " cd $(shell_quote "$PREPROCESSOR_ROOT")" +say " cargo test" +say " $(shell_quote "$BUILD_EIPS") -C $(shell_quote "$ACTIVE_REPO_ROOT") check" +say " $(shell_quote "$BUILD_EIPS") -C $(shell_quote "$ACTIVE_REPO_ROOT") build" +say " $(shell_quote "$BUILD_EIPS") -C $(shell_quote "$ACTIVE_REPO_ROOT") serve" +say "" +say "For shorter one-off local commands in this shell, run:" +say " export PATH=$(shell_quote "$TARGET_DEBUG_DIR"):\$PATH" +if [ "$INCLUDE_TEMPLATE" = true ]; then + say "" + say "Template check:" + say " $(shell_quote "$BUILD_EIPS") -C $(shell_quote "$WORKSPACE_ROOT/template") check" +fi diff --git a/scripts/dev-setup.ps1 b/scripts/dev-setup.ps1 new file mode 100644 index 0000000..4406601 --- /dev/null +++ b/scripts/dev-setup.ps1 @@ -0,0 +1,518 @@ +param( + [switch]$Template +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$DefaultActiveRepoUrl = "https://github.com/ethereum/EIPs.git" + +function Say { + param([string]$Message) + + Write-Host $Message +} + +function Die { + param([string]$Message) + + throw "error: $Message" +} + +function ConvertTo-NormalizedDirectoryPath { + param([string]$Directory) + + $trimmed = $Directory.Trim('"') + $trimChars = [char[]]@([System.IO.Path]::DirectorySeparatorChar, [System.IO.Path]::AltDirectorySeparatorChar) + + try { + return ([System.IO.Path]::GetFullPath($trimmed)).TrimEnd($trimChars) + } catch { + return $trimmed.TrimEnd($trimChars) + } +} + +function Add-PathNote { + param([string]$InstallDir) + + $target = ConvertTo-NormalizedDirectoryPath -Directory $InstallDir + foreach ($pathNote in $script:PathNotes) { + $candidate = ConvertTo-NormalizedDirectoryPath -Directory $pathNote + if ([string]::Equals($candidate, $target, [System.StringComparison]::OrdinalIgnoreCase)) { + return + } + } + + $script:PathNotes += $InstallDir +} + +function Move-DirectoryToFrontOfSessionPath { + param([string]$InstallDir) + + $target = ConvertTo-NormalizedDirectoryPath -Directory $InstallDir + $remainingEntries = @() + + if (-not [string]::IsNullOrWhiteSpace($env:Path)) { + foreach ($entry in ($env:Path -split ";")) { + if ([string]::IsNullOrWhiteSpace($entry)) { + continue + } + + $candidate = ConvertTo-NormalizedDirectoryPath -Directory $entry + if ([string]::Equals($candidate, $target, [System.StringComparison]::OrdinalIgnoreCase)) { + continue + } + + $remainingEntries += $entry + } + } + + $newEntries = @($InstallDir) + if ($remainingEntries.Count -gt 0) { + $newEntries += $remainingEntries + } + + $updatedPath = [string]::Join(";", $newEntries) + if (-not [string]::Equals($env:Path, $updatedPath, [System.StringComparison]::Ordinal)) { + $env:Path = $updatedPath + + Add-PathNote -InstallDir $InstallDir + } +} + +function Find-ZolaOnPath { + foreach ($commandName in @("zola", "zola.exe")) { + $commands = @(Get-Command -Name $commandName -CommandType Application -ErrorAction SilentlyContinue) + if ($commands.Count -gt 0) { + return $commands[0].Source + } + } + + return $null +} + +function Assert-InstallDirWritable { + param([string]$InstallDir) + + $probePath = $null + + try { + New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null + $probeName = ".build-eips-write-test-{0}.tmp" -f ([System.Guid]::NewGuid().ToString("N")) + $probePath = Join-Path -Path $InstallDir -ChildPath $probeName + [System.IO.File]::WriteAllText($probePath, "") + Remove-Item -LiteralPath $probePath -Force + $probePath = $null + } catch { + Die ("install directory cannot be created or written ({0}): {1}" -f $InstallDir, $_.Exception.Message) + } finally { + if (($null -ne $probePath) -and (Test-Path -LiteralPath $probePath)) { + Remove-Item -LiteralPath $probePath -Force -ErrorAction SilentlyContinue + } + } +} + +function Invoke-ReleaseDownload { + param( + [string]$Url, + [string]$Destination + ) + + $previousProgressPreference = $ProgressPreference + $ProgressPreference = "SilentlyContinue" + try { + Invoke-WebRequest -Uri $Url -OutFile $Destination -UseBasicParsing + } catch { + Die ("failed to download {0}: {1}" -f $Url, $_.Exception.Message) + } finally { + $ProgressPreference = $previousProgressPreference + } +} + +function Assert-FileSha256 { + param( + [string]$ArchivePath, + [string]$ExpectedHash, + [string]$ArchiveName + ) + + $actualHash = (Get-FileHash -Algorithm SHA256 -Path $ArchivePath).Hash.ToLowerInvariant() + if ($actualHash -ne $ExpectedHash.ToLowerInvariant()) { + Die "checksum mismatch for $ArchiveName" + } +} + +function Get-ZolaReleaseAsset { + $architecture = $env:PROCESSOR_ARCHITECTURE + if ([string]::IsNullOrWhiteSpace($architecture)) { + $architecture = "unknown" + } + if (($architecture -eq "x86") -and (-not [string]::IsNullOrWhiteSpace($env:PROCESSOR_ARCHITEW6432))) { + $architecture = $env:PROCESSOR_ARCHITEW6432 + } + + switch ($architecture.ToUpperInvariant()) { + "AMD64" { + return @{ + ArchiveName = "zola-v0.22.1-x86_64-pc-windows-msvc.zip" + Hash = "2c8b368f5abdf2b2478748f9549a761fd6599238e18948eccb76a7cae51f5dc1" + } + } + default { + Die "Unsupported platform Windows/$architecture for automatic Zola install. Install Zola 0.22.1 manually from https://github.com/getzola/zola/releases and ensure it is on PATH." + } + } +} + +function Install-Zola { + param( + [string]$InstallDir, + [string]$ZolaPath + ) + + $asset = Get-ZolaReleaseAsset + $archiveName = $asset.ArchiveName + $archiveHash = $asset.Hash + $releaseBaseUrl = "https://github.com/getzola/zola/releases/download/v0.22.1" + $tmpRoot = Join-Path -Path ([System.IO.Path]::GetTempPath()) -ChildPath ("zola-" + [System.Guid]::NewGuid().ToString("N")) + $archivePath = Join-Path -Path $tmpRoot -ChildPath $archiveName + $extractDir = Join-Path -Path $tmpRoot -ChildPath "extract" + + try { + Assert-InstallDirWritable -InstallDir $InstallDir + + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + + Say "Installing zola 0.22.1 from $releaseBaseUrl/$archiveName" + Invoke-ReleaseDownload -Url "$releaseBaseUrl/$archiveName" -Destination $archivePath + Assert-FileSha256 -ArchivePath $archivePath -ExpectedHash $archiveHash -ArchiveName $archiveName + + Expand-Archive -LiteralPath $archivePath -DestinationPath $extractDir -Force + + $extractedZola = Join-Path -Path $extractDir -ChildPath "zola.exe" + if (-not (Test-Path -LiteralPath $extractedZola -PathType Leaf)) { + Die "zola release archive did not contain expected zola.exe" + } + + try { + Move-Item -LiteralPath $extractedZola -Destination $ZolaPath -Force + } catch { + Die ("zola.exe is in use. Close any running zola process and re-run this script. Details: {0}" -f $_.Exception.Message) + } + + return $ZolaPath + } catch { + Die ("failed to install zola: {0}" -f $_.Exception.Message) + } finally { + if (Test-Path -LiteralPath $tmpRoot) { + Remove-Item -LiteralPath $tmpRoot -Recurse -Force -ErrorAction SilentlyContinue + } + } +} + +function Get-ZolaVersionInfo { + param([string]$ZolaPath) + + try { + $output = & $ZolaPath --version 2>$null + if ($LASTEXITCODE -ne 0) { + return $null + } + } catch { + return $null + } + + $fields = @(($output -join " ") -split "\s+" | Where-Object { $_.Length -gt 0 }) + if ($fields.Count -lt 2) { + return $null + } + + $versionToken = $fields[1] + if ($versionToken -notmatch "^([0-9]+)\.([0-9]+)\.([0-9]+)(.*)$") { + return $null + } + + return @{ + VersionToken = $versionToken + Version = [version]("{0}.{1}.{2}" -f $Matches[1], $Matches[2], $Matches[3]) + Suffix = $Matches[4] + } +} + +function Get-ZolaVersionRelation { + param([hashtable]$VersionInfo) + + $minimumVersion = [version]"0.22.1" + if ($VersionInfo.Version -lt $minimumVersion) { + return "below" + } + if ($VersionInfo.Version -gt $minimumVersion) { + return "newer" + } + if (-not [string]::IsNullOrEmpty($VersionInfo.Suffix)) { + return "below" + } + + return "equal" +} + +function Get-DefaultInstallPaths { + if ([string]::IsNullOrWhiteSpace($env:LOCALAPPDATA)) { + Die "LOCALAPPDATA is not set; cannot determine the user-local Zola install directory" + } + + $installDir = Join-Path -Path (Join-Path -Path $env:LOCALAPPDATA -ChildPath "build-eips") -ChildPath "bin" + $zolaPath = Join-Path -Path $installDir -ChildPath "zola.exe" + + return @{ + InstallDir = $installDir + ZolaPath = $zolaPath + } +} + +function Install-PinnedZola { + $defaultPaths = Get-DefaultInstallPaths + $installedZola = Install-Zola -InstallDir $defaultPaths.InstallDir -ZolaPath $defaultPaths.ZolaPath + Move-DirectoryToFrontOfSessionPath -InstallDir $defaultPaths.InstallDir + + return $installedZola +} + +function Initialize-Zola { + $zolaPath = Find-ZolaOnPath + if ($null -eq $zolaPath) { + $defaultPaths = Get-DefaultInstallPaths + if (Test-Path -LiteralPath $defaultPaths.ZolaPath -PathType Leaf) { + $zolaPath = $defaultPaths.ZolaPath + Move-DirectoryToFrontOfSessionPath -InstallDir $defaultPaths.InstallDir + } + } + + if ($null -eq $zolaPath) { + return (Install-PinnedZola) + } + + $versionInfo = Get-ZolaVersionInfo -ZolaPath $zolaPath + if ($null -eq $versionInfo) { + Say "Found zola with unparseable version output. Installing zola 0.22.1." + return (Install-PinnedZola) + } + + $relation = Get-ZolaVersionRelation -VersionInfo $versionInfo + switch ($relation) { + "below" { + Say ("Found zola {0} below supported 0.22.1. Installing zola 0.22.1." -f $versionInfo.VersionToken) + return (Install-PinnedZola) + } + "equal" { + Say ("Using existing zola {0} at {1}" -f $versionInfo.VersionToken, $zolaPath) + return $zolaPath + } + "newer" { + Say ("Found zola {0}. build-eips is tested with zola 0.22.1 or newer. Continuing with the installed version." -f $versionInfo.VersionToken) + return $zolaPath + } + } +} + +function ConvertTo-PowerShellQuotedPath { + param([string]$Path) + + return "'{0}'" -f ($Path -replace "'", "''") +} + +function Resolve-ConfiguredPath { + param( + [string]$PathValue, + [string]$BaseDir + ) + + if ([System.IO.Path]::IsPathRooted($PathValue)) { + $candidate = $PathValue + } else { + $candidate = Join-Path -Path $BaseDir -ChildPath $PathValue + } + + try { + return (Resolve-Path -LiteralPath $candidate).ProviderPath + } catch { + return [System.IO.Path]::GetFullPath($candidate) + } +} + +function Resolve-ActiveRepoRoot { + param( + [string]$InvocationDir, + [string]$WorkspaceRoot + ) + + $activeRepoExplicit = $false + if (-not [string]::IsNullOrWhiteSpace($env:ACTIVE_REPO_ROOT)) { + $activeRepoExplicit = $true + if ([System.IO.Path]::IsPathRooted($env:ACTIVE_REPO_ROOT)) { + $activeRepoCandidate = $env:ACTIVE_REPO_ROOT + } else { + $activeRepoCandidate = Join-Path -Path $InvocationDir -ChildPath $env:ACTIVE_REPO_ROOT + } + } else { + $activeRepoCandidate = Join-Path -Path $WorkspaceRoot -ChildPath "EIPs" + } + + if ($activeRepoExplicit) { + try { + $resolved = (Resolve-Path -LiteralPath $activeRepoCandidate).ProviderPath + } catch { + Die "configured ACTIVE_REPO_ROOT does not exist: $activeRepoCandidate. Fix ACTIVE_REPO_ROOT or unset it to let this script clone EIPs." + } + + if (-not (Test-Path -LiteralPath $resolved -PathType Container)) { + Die "configured ACTIVE_REPO_ROOT is not a directory: $resolved. Fix ACTIVE_REPO_ROOT or unset it to let this script clone EIPs." + } + if (-not (Test-Path -LiteralPath (Join-Path -Path $resolved -ChildPath ".git"))) { + Die "configured ACTIVE_REPO_ROOT is not a git checkout: $resolved. Fix ACTIVE_REPO_ROOT or unset it to let this script clone EIPs." + } + + return @{ + Path = $resolved + Explicit = $true + } + } + + if (-not (Test-Path -LiteralPath $activeRepoCandidate)) { + $gitCommand = Get-Command git -CommandType Application -ErrorAction SilentlyContinue + if ($null -eq $gitCommand) { + Die "need git to clone default proposal repo from $DefaultActiveRepoUrl" + } + + Say "No active proposal repo found at $activeRepoCandidate." + Say "Cloning default proposal repo from $DefaultActiveRepoUrl" + Say "This may take a few minutes..." + & git clone $DefaultActiveRepoUrl $activeRepoCandidate + $GitCloneExitCode = $LASTEXITCODE + if ($GitCloneExitCode -ne 0) { + Die "failed to clone default proposal repo from $DefaultActiveRepoUrl to $activeRepoCandidate" + } + Say "Cloned default proposal repo to $activeRepoCandidate" + } + + try { + $resolved = (Resolve-Path -LiteralPath $activeRepoCandidate).ProviderPath + } catch { + Die "default active proposal repo path does not exist after clone: $activeRepoCandidate" + } + + if (-not (Test-Path -LiteralPath $resolved -PathType Container)) { + Die "default active proposal repo path exists but is not a directory: $resolved. Remove it or set ACTIVE_REPO_ROOT." + } + if (-not (Test-Path -LiteralPath (Join-Path -Path $resolved -ChildPath ".git"))) { + Die "default active proposal repo path exists but is not a git checkout: $resolved. Remove it or set ACTIVE_REPO_ROOT." + } + + return @{ + Path = $resolved + Explicit = $false + } +} + +$PathNotes = @() + +$InvocationDir = (Get-Location).ProviderPath +$ScriptDir = (Resolve-Path -LiteralPath $PSScriptRoot).ProviderPath +$PreprocessorRoot = (Resolve-Path -LiteralPath (Split-Path -Path $ScriptDir -Parent)).ProviderPath + +if (-not [string]::IsNullOrWhiteSpace($env:WORKSPACE_ROOT)) { + $WorkspaceRoot = Resolve-ConfiguredPath -PathValue $env:WORKSPACE_ROOT -BaseDir $InvocationDir +} else { + $WorkspaceRoot = (Resolve-Path -LiteralPath (Split-Path -Path $PreprocessorRoot -Parent)).ProviderPath +} + +$ActiveRepoInfo = Resolve-ActiveRepoRoot -InvocationDir $InvocationDir -WorkspaceRoot $WorkspaceRoot +$ActiveRepoRoot = $ActiveRepoInfo["Path"] +$ActiveRepoExplicit = $ActiveRepoInfo["Explicit"] +$BuildEipsPath = Join-Path -Path $PreprocessorRoot -ChildPath "target\debug\build-eips.exe" + +Say "Workspace root: $WorkspaceRoot" +Say "Active proposal repo: $ActiveRepoRoot" +Say "Local build-eips: $BuildEipsPath" +Say "If PowerShell blocks this script, run:" +Say " powershell -ExecutionPolicy Bypass -File .\scripts\dev-setup.ps1" + +$cargoCommand = Get-Command cargo -CommandType Application -ErrorAction SilentlyContinue +if ($null -eq $cargoCommand) { + Die "need cargo to build local build-eips. Install Rust from https://rustup.rs/ and re-run this script." +} + +Say "Building local build-eips" +$CargoExitCode = 0 +Push-Location -LiteralPath $PreprocessorRoot +try { + & cargo build + $CargoExitCode = $LASTEXITCODE +} finally { + Pop-Location +} +if ($CargoExitCode -ne 0) { + Die "cargo build failed with exit code $CargoExitCode" +} + +if (-not (Test-Path -LiteralPath $BuildEipsPath -PathType Leaf)) { + Die "local build-eips was not built at $BuildEipsPath" +} + +$ZolaPath = Initialize-Zola + +Say "Bootstrapping local contributor workspace" +if ($Template) { + & $BuildEipsPath -C $ActiveRepoRoot init $WorkspaceRoot --platform-dev --template + $WorkspaceInitExitCode = $LASTEXITCODE +} else { + & $BuildEipsPath -C $ActiveRepoRoot init $WorkspaceRoot --platform-dev + $WorkspaceInitExitCode = $LASTEXITCODE +} +if ($WorkspaceInitExitCode -ne 0) { + Die "build-eips init failed with exit code $WorkspaceInitExitCode" +} + +Say "Running build-eips doctor" +& $BuildEipsPath -C $ActiveRepoRoot doctor +$WorkspaceDoctorExitCode = $LASTEXITCODE +if ($WorkspaceDoctorExitCode -ne 0) { + Say "Warning: build-eips doctor reported issues above. Fix them before relying on local build-eips commands." +} + +$WorkspaceDocPath = Join-Path -Path $WorkspaceRoot -ChildPath "WORKSPACE.md" +Say "" +if (Test-Path -LiteralPath $WorkspaceDocPath -PathType Leaf) { + Say "Workspace docs: $WorkspaceDocPath" +} else { + Say "Warning: workspace docs were not found at $WorkspaceDocPath after build-eips init" +} + +if ($PathNotes.Count -gt 0) { + Say "" + Say 'Updated PATH for this PowerShell session only:' + foreach ($pathNote in $PathNotes) { + Say " $pathNote" + } + Say "To make this permanent, add the listed directory or directories to your user Path in Windows Environment Variables." +} + +$TargetDebugDir = Split-Path -Path $BuildEipsPath -Parent + +Say "" +Say "Next commands:" +Say (" cd {0}" -f (ConvertTo-PowerShellQuotedPath -Path $PreprocessorRoot)) +Say " cargo test" +Say (" {0} -C {1} check" -f (ConvertTo-PowerShellQuotedPath -Path $BuildEipsPath), (ConvertTo-PowerShellQuotedPath -Path $ActiveRepoRoot)) +Say (" {0} -C {1} build" -f (ConvertTo-PowerShellQuotedPath -Path $BuildEipsPath), (ConvertTo-PowerShellQuotedPath -Path $ActiveRepoRoot)) +Say (" {0} -C {1} serve" -f (ConvertTo-PowerShellQuotedPath -Path $BuildEipsPath), (ConvertTo-PowerShellQuotedPath -Path $ActiveRepoRoot)) +Say "" +Say "For shorter one-off local commands in this PowerShell session, run:" +Say (" `$env:Path = {0} + ';' + `$env:Path" -f (ConvertTo-PowerShellQuotedPath -Path $TargetDebugDir)) +if ($Template) { + $TemplateRoot = Join-Path -Path $WorkspaceRoot -ChildPath "template" + Say "" + Say "Template check:" + Say (" {0} -C {1} check" -f (ConvertTo-PowerShellQuotedPath -Path $BuildEipsPath), (ConvertTo-PowerShellQuotedPath -Path $TemplateRoot)) +} diff --git a/src/README.md b/src/README.md new file mode 100644 index 0000000..9feb0b5 --- /dev/null +++ b/src/README.md @@ -0,0 +1,60 @@ +# Preprocessor Module And Test Ownership + +This file documents how the preprocessor crate is organized: which module owns each behavior, where tests should live, and how to keep module boundaries reviewable. It is for contributors and maintainers, not end-user documentation. + + +## Module Ownership + +- `main.rs` owns CLI entry, top-level dispatch, build locking, and runtime orchestration. +- `changed.rs` owns changed-file command execution, coordinating upstream diffing, proposal filtering, and changed-output formatting. +- `cli.rs` owns the clap command surface and command helper methods. +- `config.rs` owns built-in repository metadata, workspace config schema, repo manifest schema, parsing, defaults, and config discovery. +- `context.rs` owns command input path resolution and workspace command context. +- `editorial.rs` owns editorial selector resolution, active-repo target validation, prepared-source editorial lint orchestration, and editorial check runtime handoff. +- `execution.rs` owns source mode, environment, base URL, server binding, build path, workspace source, and runtime execution resolution. +- `find_root.rs` owns active proposal-repo root detection. +- `git.rs` owns git repository identification, clone/fetch/merge behavior, source materialization, and tracked path synchronization. +- `github.rs` owns GitHub annotation reporting support for lint output. +- `identity.rs` owns active repository identity selection from repo manifests or legacy metadata. +- `layout.rs` owns shared build layout names and path helpers. +- `lint.rs` owns eipw config loading, reporter setup, and invocation for caller-provided source lists. +- `markdown.rs` owns proposal markdown preprocessing. +- `pipeline.rs` owns prepared Zola runtime setup and build/check/serve steps. +- `preview.rs` owns static preview serving for already-built output. +- `print.rs` owns diagnostic print subcommands. +- `progress.rs` owns progress/log rendering helpers. +- `proposal.rs` owns proposal path classification, proposal-number parsing, and targeted-rendering selection policy. +- `serve.rs` owns dirty active-repo and local-theme serve synchronization. +- `workspace.rs` owns local workspace setup and diagnostics. +- `zola.rs` owns Zola discovery, theme mounting, and Zola command invocation. + +## Test Ownership + +New tests should generally live in the module that owns the behavior. Use `super::` from module-local tests where natural, or sibling module paths from the owning module. Module-local tests cover behavior in modules such as `cli.rs`, `config.rs`, `execution.rs`, `git.rs`, `markdown.rs`, `pipeline.rs`, `proposal.rs`, `serve.rs`, `workspace.rs`, and `zola.rs`; `src/tests.rs` intentionally holds the remaining cross-domain behavior tests. + +Use `src/tests.rs` for cross-domain behavior tests, especially tests covering: + +- command dispatch across modules +- CLI plus execution plus runtime behavior +- downstream CI invariance +- multi-repo or manifest-driven flows +- workspace plus execution plus editorial behavior +- source materialization behavior spanning git, execution, pipeline, or serve +- tests that would require exposing more internals just to move them + +Move tests to module-local `#[cfg(test)]` modules only when the behavior is owned by one module and the test remains clearer there. Examples include pure clap parsing in `cli.rs`, execution policy helpers in `execution.rs`, serve event filtering in `serve.rs`, workspace-local theme materialization in `pipeline.rs`, prepared-source editorial lint behavior in `editorial.rs`, workspace setup and diagnostics behavior in `workspace.rs`, active repo identity behavior in `identity.rs` when it does not require the full execution path, and proposal path or proposal-number behavior in `proposal.rs`. + +## Dependency Direction + +Imports should generally flow from orchestration modules toward domain and shared modules. `main.rs` should call into command/runtime modules, while lower-level modules should avoid depending on high-level orchestration. + +`execution.rs` is the runtime resolution layer between command/config inputs and runtime execution: it combines CLI flags, workspace config, active repo identity, source mode, build path, server binding, and base URL decisions before `main.rs` hands work to `changed.rs`, `pipeline.rs`, `serve.rs`, `preview.rs`, or editorial helpers. + +Examples of high-level orchestration modules include `main.rs`, `workspace.rs`, `pipeline.rs`, and `serve.rs`. Shared or lower-level modules include `cli.rs`, `layout.rs`, `proposal.rs`, and focused domain modules such as `git.rs`, `markdown.rs`, and `zola.rs`. + +Lower-level or shared modules should not import higher-level orchestration modules just to reuse behavior. Move shared behavior into the owning domain module instead. + + +## Visibility + +Do not make private helpers `pub` or `pub(crate)` just to move a test. If a test needs direct access to private module behavior, it probably belongs in that module's `#[cfg(test)]` block. Use `src/tests.rs` for cross-module behavior tests that exercise crate-visible paths. diff --git a/src/cache.rs b/src/cache.rs deleted file mode 100644 index 9c7eb2d..0000000 --- a/src/cache.rs +++ /dev/null @@ -1,84 +0,0 @@ -/* - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. - */ - -use std::{io::ErrorKind, path::PathBuf, sync::Arc}; - -use directories::ProjectDirs; -use fslock::LockFile; -use log::{debug, info}; -use sha3::{Digest, Sha3_256}; -use snafu::{Backtrace, IntoError, OptionExt, Report, ResultExt, Snafu}; - -#[derive(Debug, Snafu)] -pub enum Error { - #[snafu(display("unable to discover application directories (unset $HOME?)"))] - Directories { backtrace: Backtrace }, - #[snafu(display("unable to access the path `{}`", path.to_string_lossy()))] - Fs { - path: PathBuf, - backtrace: Backtrace, - source: std::io::Error, - }, -} - -#[derive(Debug)] -struct Inner { - _lock: LockFile, - dir: PathBuf, -} - -#[derive(Debug, Clone)] -pub struct Cache(Arc); - -impl Cache { - pub fn open() -> Result { - debug!("opening local file cache"); - - let dirs = - ProjectDirs::from("org.ethereum", "eips", "eips-build").context(DirectoriesSnafu)?; - let cache_path = dirs.cache_dir(); - if let Err(e) = std::fs::create_dir_all(cache_path) { - debug!( - "got while creating cache directory: {}", - Report::from_error(e) - ); - } - - let lock_path = cache_path.join(".lock"); - let mut lock = LockFile::open(&lock_path).context(FsSnafu { path: &lock_path })?; - - let locked = lock - .try_lock_with_pid() - .context(FsSnafu { path: &lock_path })?; - - if !locked { - info!("waiting on cache directory..."); - lock.lock_with_pid().context(FsSnafu { path: &lock_path })?; - } - - Ok(Self(Arc::new(Inner { - _lock: lock, - dir: cache_path.into(), - }))) - } - - pub fn dir(&self, key: &str) -> Result { - let mut hasher = Sha3_256::new(); - hasher.update(key.as_bytes()); - let hash = hasher.finalize(); - let hash_text = format!("{:x}", hash); - let path = self.0.dir.join(hash_text); - - debug!("creating cache directory `{}`", path.to_string_lossy()); - match std::fs::create_dir(&path) { - Ok(()) => (), - Err(e) if e.kind() == ErrorKind::AlreadyExists => (), - Err(e) => return Err(FsSnafu { path }.into_error(e)), - } - - Ok(path) - } -} diff --git a/src/changed.rs b/src/changed.rs index 533ac26..07ad880 100644 --- a/src/changed.rs +++ b/src/changed.rs @@ -4,78 +4,46 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -//! Changed-file command execution. +//! Changed-file command helpers. -use std::{ - ffi::OsStr, - path::{Path, PathBuf}, -}; +use std::path::Path; use snafu::{ResultExt, Whatever}; -use crate::{cli::ChangedFormat, config::Config, git, layout::REPO_DIR}; - -pub(crate) fn is_proposal_path(mut p: PathBuf) -> bool { - // Only lint `content/00001.md` and `content/00001/index.md` files. - - // content/00000.md | content/00000/index.md - // ^^^^^^^^ | ^^^^^^^^ - match p.file_name() { - Some(n) if n == "index.md" => { - p.pop(); - } - Some(_) if p.extension().map(|x| x == "md").unwrap_or(false) => { - p.set_extension(""); - } - None | Some(_) => return false, - } - - // content/00000 - // ^^^^^ - match p.file_name().and_then(OsStr::to_str) { - None => return false, - Some(f) if f.parse::().is_err() => return false, - Some(_) => { - p.pop(); - } - } - - // content - // ^^^^^^^ - match p.file_name() { - Some(f) if f == "content" => { - p.pop(); - } - _ => return false, - } - - p == OsStr::new("") -} +use crate::{ + cli::ChangedFormat, execution::ResolvedExecution, git, layout::REPO_DIR, + proposal::is_proposal_path, +}; pub(crate) fn run( - root_path: &Path, + resolved: &ResolvedExecution, build_path: &Path, - config: &Config, all: bool, format: &ChangedFormat, ) -> Result<(), Whatever> { let repo_path = build_path.join(REPO_DIR); - let both = git::Fresh::new(root_path, &repo_path, &config.locations) - .whatever_context("initializing build repo")? - .clone_src() - .whatever_context("cloning source repo")? - .fetch_upstream() - .whatever_context("fetching upstream repo")?; + let both = git::Fresh::new( + &resolved.root_path, + &repo_path, + resolved.repository_use.clone(), + resolved.source_materialization, + ) + .whatever_context("initializing build repo")? + .clone_src() + .whatever_context("cloning source repo")? + .fetch_upstream() + .whatever_context("fetching upstream repo")?; let changed_files: Vec<_> = both .changed_files() .whatever_context("unable to list changed files")? .into_iter() - .filter(|p| all || is_proposal_path(p.into())) + .filter(|p| all || is_proposal_path(p)) .map(|p| repo_path.join(p)) .collect(); format.print(&changed_files, &repo_path); + Ok(()) } diff --git a/src/cli.rs b/src/cli.rs index f7a2480..6488667 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -9,8 +9,9 @@ use std::path::{Path, PathBuf}; use clap::{Parser, Subcommand}; +use url::Url; -use crate::{lint, print}; +use crate::{lint, print, proposal::ProposalNumber}; /// Build script for Ethereum EIPs and ERCs. #[derive(Parser, Debug)] @@ -20,17 +21,61 @@ pub(crate) struct Args { #[clap(short = 'C')] pub(crate) root: Option, - /// Use the staging repositories (for testing) - #[clap(long = "staging")] + /// Use staging sibling repositories and environment metadata + #[clap(long)] pub(crate) staging: bool, + /// Use production sibling repositories and environment metadata + #[clap(long)] + pub(crate) production: bool, + + /// Use the configured remote sibling content repositories + #[clap(long)] + pub(crate) remote_siblings: bool, + + /// Write build artifacts under BUILD_ROOT instead of the default location + #[clap(long)] + pub(crate) build_root: Option, + #[clap(subcommand)] pub(crate) operation: Operation, } -#[derive(Debug, Subcommand)] +#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)] +pub(crate) struct ServerCliArgs { + /// Host/interface for the local server to bind + #[arg(long)] + pub(crate) host: Option, + + /// Port for the local server to bind + #[arg(long)] + pub(crate) port: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)] +pub(crate) struct BaseUrlCliArgs { + /// Override the rendered-site base URL for this command + #[arg(long, value_parser = clap::value_parser!(Url))] + pub(crate) base_url: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)] +pub(crate) struct CleanCliArgs { + /// Ignore tracked working-tree changes in the active repo + #[arg(long)] + pub(crate) clean: bool, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)] +pub(crate) struct OnlyCliArgs { + /// Render only the selected proposal number(s) + #[arg(long, value_name = "NUMBER", value_parser = ProposalNumber::parse_cli_selector, num_args = 1..)] + pub(crate) only: Vec, +} + +#[derive(Debug, Clone, Subcommand)] pub(crate) enum Operation { - /// Print various useful things, like available lints + /// Print linter schema metadata and lint configuration Print { #[command(flatten)] print: print::CmdArgs, @@ -39,22 +84,43 @@ pub(crate) enum Operation { /// Build the project and output HTML Build { #[command(flatten)] - eipw: lint::CmdArgs, + base_url: BaseUrlCliArgs, + + #[command(flatten)] + clean: CleanCliArgs, + + #[command(flatten)] + only: OnlyCliArgs, }, - /// Build the project and launch a web server to preview it + /// Serve the existing built output without rebuilding it + Preview { + #[command(flatten)] + server: ServerCliArgs, + }, + + /// Build a fresh temporary site, serve it locally, and watch tracked edits Serve { #[command(flatten)] - eipw: lint::CmdArgs, + server: ServerCliArgs, + + #[command(flatten)] + base_url: BaseUrlCliArgs, + + #[command(flatten)] + clean: CleanCliArgs, + + #[command(flatten)] + only: OnlyCliArgs, }, - /// Remove temporary and output files + /// Remove the selected build directory and generated output Clean, - /// Analyze the repository and report errors, but don't build HTML files + /// Validate that the site builds cleanly without writing HTML output Check { #[command(flatten)] - eipw: lint::CmdArgs, + clean: CleanCliArgs, }, /// List files changed since the last commit common to both the local and upstream repositories @@ -65,6 +131,96 @@ pub(crate) enum Operation { #[clap(long, value_enum, default_value_t)] format: ChangedFormat, }, + + /// Run targeted editorial lint or check workflows + Editorial { + #[command(subcommand)] + command: EditorialCommand, + }, + + /// Create workspace config, docs, build root, and missing local repos + Init { + /// Workspace root directory + path: PathBuf, + + /// Also clone template for proposal-family scaffold work + #[arg(long)] + template: bool, + + /// Also clone preprocessor and eipw for platform development + #[arg(long)] + platform_dev: bool, + }, + + /// Check workspace layout, local repos, and required tools + Doctor, + + /// Run build, serve, or check with clean staging settings and remote siblings + Parity { + #[command(subcommand)] + command: ProfiledOperation, + }, +} + +#[derive(Debug, Clone, Subcommand)] +pub(crate) enum ProfiledOperation { + /// Build the project and output HTML + Build { + #[command(flatten)] + base_url: BaseUrlCliArgs, + }, + + /// Build a fresh temporary site, serve it locally, and watch tracked edits + Serve { + #[command(flatten)] + server: ServerCliArgs, + + #[command(flatten)] + base_url: BaseUrlCliArgs, + }, + + /// Validate that the site builds cleanly without writing HTML output + Check, +} + +#[derive(Debug, Subcommand, Clone)] +pub(crate) enum EditorialCommand { + /// Run eipw lint checks on selected proposal files + Lint { + #[command(flatten)] + selectors: EditorialSelectorArgs, + + #[command(flatten)] + eipw: lint::CmdArgs, + }, + + /// Run eipw lint checks, then validate the site build + Check { + #[command(flatten)] + selectors: EditorialSelectorArgs, + + #[command(flatten)] + eipw: lint::CmdArgs, + }, +} + +#[derive(Debug, clap::Args, Clone)] +pub(crate) struct EditorialSelectorArgs { + /// Proposal number(s) or repo-relative proposal path(s), such as `4` or `content/07949.md` + #[arg(value_name = "TARGET")] + pub(crate) paths: Vec, + + /// Read proposal numbers or repo-relative proposal paths from BATCH, one per line + #[arg(long)] + pub(crate) batch: Option, + + /// Select tracked dirty proposal files from the active content repo + #[arg(long)] + pub(crate) working_tree: bool, + + /// Select proposal files changed versus the upstream merge-base + #[arg(long)] + pub(crate) against_upstream: bool, } #[derive(Debug, clap::ValueEnum, Clone, Default)] @@ -75,6 +231,142 @@ pub(crate) enum ChangedFormat { Json, } +#[derive(Debug, Clone)] +pub(crate) enum RuntimeOperation { + Build, + Serve, + Preview, + Clean, + Check, + Changed { all: bool, format: ChangedFormat }, + Editorial { command: EditorialCommand }, +} + +impl Operation { + pub(crate) fn server_cli_args(&self) -> ServerCliArgs { + match self { + Self::Serve { server, .. } | Self::Preview { server } => server.clone(), + Self::Parity { command } => command.server_cli_args(), + Self::Print { .. } + | Self::Build { .. } + | Self::Clean + | Self::Check { .. } + | Self::Changed { .. } + | Self::Editorial { .. } + | Self::Init { .. } + | Self::Doctor => ServerCliArgs::default(), + } + } + + pub(crate) fn base_url_cli_args(&self) -> BaseUrlCliArgs { + match self { + Self::Build { base_url, .. } | Self::Serve { base_url, .. } => base_url.clone(), + Self::Parity { command } => command.base_url_cli_args(), + Self::Print { .. } + | Self::Preview { .. } + | Self::Clean + | Self::Check { .. } + | Self::Changed { .. } + | Self::Editorial { .. } + | Self::Init { .. } + | Self::Doctor => BaseUrlCliArgs::default(), + } + } + + pub(crate) fn clean_cli_args(&self) -> CleanCliArgs { + match self { + Self::Build { clean, .. } | Self::Serve { clean, .. } | Self::Check { clean } => { + clean.clone() + } + Self::Print { .. } + | Self::Preview { .. } + | Self::Clean + | Self::Changed { .. } + | Self::Editorial { .. } + | Self::Init { .. } + | Self::Doctor + | Self::Parity { .. } => CleanCliArgs::default(), + } + } + + pub(crate) fn only_cli_args(&self) -> Option<&OnlyCliArgs> { + match self { + Self::Build { only, .. } | Self::Serve { only, .. } => Some(only), + Self::Print { .. } + | Self::Preview { .. } + | Self::Clean + | Self::Check { .. } + | Self::Changed { .. } + | Self::Editorial { .. } + | Self::Init { .. } + | Self::Doctor + | Self::Parity { .. } => None, + } + } + + pub(crate) fn is_plain_site_command(&self) -> bool { + matches!( + self, + Self::Build { .. } | Self::Serve { .. } | Self::Check { .. } + ) + } + + pub(crate) fn is_editorial_command(&self) -> bool { + matches!(self, Self::Editorial { .. }) + } + + pub(crate) fn runtime_operation(&self) -> Option { + match self { + Self::Print { .. } | Self::Init { .. } | Self::Doctor => None, + Self::Build { .. } => Some(RuntimeOperation::Build), + Self::Serve { .. } => Some(RuntimeOperation::Serve), + Self::Preview { .. } => Some(RuntimeOperation::Preview), + Self::Clean => Some(RuntimeOperation::Clean), + Self::Check { .. } => Some(RuntimeOperation::Check), + Self::Changed { all, format } => Some(RuntimeOperation::Changed { + all: *all, + format: format.clone(), + }), + Self::Editorial { command } => Some(RuntimeOperation::Editorial { + command: command.clone(), + }), + Self::Parity { command } => Some(command.runtime_operation()), + } + } + + pub(crate) fn is_workspace_lifecycle_command(&self) -> bool { + matches!(self, Self::Init { .. } | Self::Doctor) + } + + pub(crate) fn is_print_command(&self) -> bool { + matches!(self, Self::Print { .. }) + } +} + +impl ProfiledOperation { + fn server_cli_args(&self) -> ServerCliArgs { + match self { + Self::Serve { server, .. } => server.clone(), + Self::Build { .. } | Self::Check => ServerCliArgs::default(), + } + } + + fn base_url_cli_args(&self) -> BaseUrlCliArgs { + match self { + Self::Build { base_url } | Self::Serve { base_url, .. } => base_url.clone(), + Self::Check => BaseUrlCliArgs::default(), + } + } + + fn runtime_operation(&self) -> RuntimeOperation { + match self { + Self::Build { .. } => RuntimeOperation::Build, + Self::Serve { .. } => RuntimeOperation::Serve, + Self::Check => RuntimeOperation::Check, + } + } +} + impl ChangedFormat { fn print_sep(files: &[&Path], sep: &str) { let files: Vec<_> = files @@ -108,3 +400,438 @@ impl ChangedFormat { } } } + +impl EditorialSelectorArgs { + pub(crate) fn selector_count(&self) -> usize { + usize::from(!self.paths.is_empty()) + + usize::from(self.batch.is_some()) + + usize::from(self.working_tree) + + usize::from(self.against_upstream) + } +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + use crate::proposal::ProposalNumber; + + use super::{Args, EditorialCommand, Operation, ProfiledOperation, RuntimeOperation}; + + fn parse_args(arguments: &[&str]) -> Args { + Args::try_parse_from(arguments).unwrap() + } + + #[test] + fn parity_command_parses_as_command_prefix() { + let args = parse_args(&["build-eips", "parity", "build"]); + + assert!(matches!( + args.operation, + Operation::Parity { + command: ProfiledOperation::Build { .. } + } + )); + } + + #[test] + fn profile_flag_is_rejected() { + let error = + Args::try_parse_from(["build-eips", "--profile", "local", "build"]).unwrap_err(); + + assert!(error + .to_string() + .contains("unexpected argument '--profile'")); + } + + #[test] + fn removed_theme_flag_is_rejected() { + let removed_flag = concat!("--remote", "-theme"); + let error = Args::try_parse_from(["build-eips", removed_flag, "build"]).unwrap_err(); + + assert!(error + .to_string() + .contains(&format!("unexpected argument '{removed_flag}'"))); + } + + #[test] + fn only_flag_parses_one_or_more_proposal_numbers_on_build() { + let one = parse_args(&["build-eips", "build", "--only", "00555"]); + let many = parse_args(&["build-eips", "build", "--only", "555", "678", "897"]); + let serve = parse_args(&["build-eips", "serve", "--only", "555", "678"]); + + match one.operation { + Operation::Build { only, .. } => { + assert_eq!(only.only, vec![ProposalNumber::from_u32(555).unwrap()]); + } + other => panic!("unexpected operation: {other:?}"), + } + match many.operation { + Operation::Build { only, .. } => { + assert_eq!( + only.only, + vec![ + ProposalNumber::from_u32(555).unwrap(), + ProposalNumber::from_u32(678).unwrap(), + ProposalNumber::from_u32(897).unwrap(), + ] + ); + } + other => panic!("unexpected operation: {other:?}"), + } + match serve.operation { + Operation::Serve { only, .. } => { + assert_eq!( + only.only, + vec![ + ProposalNumber::from_u32(555).unwrap(), + ProposalNumber::from_u32(678).unwrap(), + ] + ); + } + other => panic!("unexpected operation: {other:?}"), + } + } + + #[test] + fn only_flag_rejects_invalid_selectors_and_non_build_commands() { + for selector in [ + "+555", + "0", + "-555", + "abc", + "555,678", + "content/00555.md", + "4294967296", + ] { + assert!( + Args::try_parse_from(["build-eips", "build", "--only", selector]).is_err(), + "expected `{selector}` to be rejected" + ); + } + + assert!(Args::try_parse_from(["build-eips", "check", "--only", "555"]).is_err()); + assert!(Args::try_parse_from(["build-eips", "parity", "build", "--only", "555"]).is_err()); + assert!(Args::try_parse_from(["build-eips", "parity", "serve", "--only", "555"]).is_err()); + } + + #[test] + fn server_flags_parse_on_serve_and_preview_forms() { + let cases: &[(&[&str], bool)] = &[ + ( + &["build-eips", "serve", "--host", "0.0.0.0", "--port", "8080"], + true, + ), + ( + &[ + "build-eips", + "preview", + "--host", + "0.0.0.0", + "--port", + "8080", + ], + false, + ), + ( + &[ + "build-eips", + "parity", + "serve", + "--host", + "0.0.0.0", + "--port", + "8080", + ], + true, + ), + ]; + + for (arguments, expect_serve) in cases { + let args = parse_args(arguments); + let runtime_operation = args.operation.runtime_operation().unwrap(); + match runtime_operation { + RuntimeOperation::Serve if *expect_serve => {} + RuntimeOperation::Preview if !*expect_serve => {} + other => panic!("unexpected runtime operation: {other:?}"), + } + let server = args.operation.server_cli_args(); + + assert_eq!(server.host.as_deref(), Some("0.0.0.0")); + assert_eq!(server.port, Some(8080)); + } + } + + #[test] + fn base_url_flags_parse_on_build_and_serve_forms() { + let cases: &[(&[&str], RuntimeOperation)] = &[ + ( + &["build-eips", "build", "--base-url", "http://localhost:4000"], + RuntimeOperation::Build, + ), + ( + &["build-eips", "serve", "--base-url", "http://localhost:4000"], + RuntimeOperation::Serve, + ), + ( + &[ + "build-eips", + "parity", + "build", + "--base-url", + "http://localhost:4000", + ], + RuntimeOperation::Build, + ), + ( + &[ + "build-eips", + "parity", + "serve", + "--base-url", + "http://localhost:4000", + ], + RuntimeOperation::Serve, + ), + ]; + + for (arguments, expected_runtime_operation) in cases { + let args = parse_args(arguments); + + assert!(matches!( + ( + args.operation.runtime_operation().unwrap(), + (*expected_runtime_operation).clone() + ), + (RuntimeOperation::Build, RuntimeOperation::Build) + | (RuntimeOperation::Serve, RuntimeOperation::Serve) + )); + assert_eq!( + args.operation + .base_url_cli_args() + .base_url + .as_ref() + .unwrap() + .as_str(), + "http://localhost:4000/" + ); + } + } + + #[test] + fn clean_flags_parse_only_on_plain_site_commands() { + for arguments in [ + &["build-eips", "build", "--clean"][..], + &["build-eips", "serve", "--clean"][..], + &["build-eips", "check", "--clean"][..], + ] { + let args = parse_args(arguments); + assert!(args.operation.clean_cli_args().clean); + } + + for arguments in [ + &["build-eips", "parity", "build", "--clean"][..], + &["build-eips", "parity", "serve", "--clean"][..], + &["build-eips", "parity", "check", "--clean"][..], + &["build-eips", "preview", "--clean"][..], + &["build-eips", "changed", "--clean"][..], + &["build-eips", "clean", "--clean"][..], + ] { + assert!(Args::try_parse_from(arguments).is_err()); + } + } + + #[test] + fn removed_command_surface_is_rejected() { + for arguments in [ + &["build-eips", "dirty", "build"][..], + &["build-eips", "--allow-dirty", "build"][..], + &["build-eips", "--no-allow-dirty", "build"][..], + &["build-eips", "--no-staging", "build"][..], + &["build-eips", "--remote-sibling-repo", "build"][..], + &["build-eips", "workspace", "init", "/tmp/workspace"][..], + &["build-eips", "workspace", "doctor"][..], + &["build-eips", "editorial", "build", "1"][..], + &["build-eips", "parity", "preview"][..], + &["build-eips", "parity", "clean"][..], + &["build-eips", "parity", "changed"][..], + ] { + assert!(Args::try_parse_from(arguments).is_err()); + } + } + + #[test] + fn eipw_flags_parse_only_on_editorial_commands() { + let command_prefixes: &[&[&str]] = &[ + &["build-eips", "build"], + &["build-eips", "check"], + &["build-eips", "serve"], + &["build-eips", "parity", "build"], + &["build-eips", "parity", "check"], + &["build-eips", "parity", "serve"], + ]; + let eipw_flags: &[&[&str]] = &[ + &["--no-default-lints"], + &["-D", "markdown-refs"], + &["--deny", "markdown-refs"], + &["-W", "markdown-link-status"], + &["--warn", "markdown-link-status"], + &["-A", "preamble-required"], + &["--allow", "preamble-required"], + ]; + + for command_prefix in command_prefixes { + for eipw_flag in eipw_flags { + let arguments = command_prefix + .iter() + .chain(eipw_flag.iter()) + .copied() + .collect::>(); + assert!( + Args::try_parse_from(arguments.clone()).is_err(), + "expected {arguments:?} to reject eipw flags" + ); + } + } + + for command in ["lint", "check"] { + let args = parse_args(&[ + "build-eips", + "editorial", + command, + "content/00001.md", + "--no-default-lints", + "-D", + "markdown-refs", + "--warn", + "markdown-link-status", + "--allow", + "preamble-required", + ]); + + assert!(matches!( + args.operation.runtime_operation(), + Some(RuntimeOperation::Editorial { .. }) + )); + } + } + + #[test] + fn base_url_flag_is_rejected_on_non_rendering_forms() { + let cases: &[&[&str]] = &[ + &[ + "build-eips", + "preview", + "--base-url", + "http://localhost:4000", + ], + &[ + "build-eips", + "parity", + "preview", + "--base-url", + "http://localhost:4000", + ], + &["build-eips", "check", "--base-url", "http://localhost:4000"], + &[ + "build-eips", + "changed", + "--base-url", + "http://localhost:4000", + ], + &[ + "build-eips", + "doctor", + "--base-url", + "http://localhost:4000", + ], + &[ + "build-eips", + "init", + "/tmp/workspace", + "--base-url", + "http://localhost:4000", + ], + &[ + "build-eips", + "editorial", + "lint", + "--working-tree", + "--base-url", + "http://localhost:4000", + ], + &["build-eips", "print", "--base-url", "http://localhost:4000"], + ]; + + for arguments in cases { + assert!(Args::try_parse_from(*arguments).is_err()); + } + } + + #[test] + fn workspace_lifecycle_commands_parse() { + let plain = parse_args(&["build-eips", "init", "/tmp/workspace"]); + let template = parse_args(&["build-eips", "init", "/tmp/workspace", "--template"]); + let combined = parse_args(&[ + "build-eips", + "init", + "/tmp/workspace", + "--template", + "--platform-dev", + ]); + let doctor = parse_args(&["build-eips", "doctor"]); + + assert!(matches!( + plain.operation, + Operation::Init { + template: false, + platform_dev: false, + .. + } + )); + assert!(matches!( + template.operation, + Operation::Init { + template: true, + platform_dev: false, + .. + } + )); + assert!(matches!( + combined.operation, + Operation::Init { + template: true, + platform_dev: true, + .. + } + )); + assert!(matches!(doctor.operation, Operation::Doctor)); + } + + #[test] + fn editorial_check_parses_as_runtime_editorial_command() { + let args = parse_args(&["build-eips", "editorial", "check", "--working-tree"]); + + assert!(matches!( + args.operation.runtime_operation(), + Some(RuntimeOperation::Editorial { + command: EditorialCommand::Check { .. } + }) + )); + } + + #[test] + fn remote_siblings_flag_parses() { + let args = parse_args(&["build-eips", "--remote-siblings", "build"]); + + assert!(args.remote_siblings); + } + + #[test] + fn explicit_workspace_config_path_is_not_accepted() { + let error = Args::try_parse_from(["build-eips", "--config", "/tmp/config.toml", "build"]) + .unwrap_err(); + + assert!(error.to_string().contains("unexpected argument '--config'")); + } +} diff --git a/src/config.rs b/src/config.rs index 26c4ad4..24cd891 100644 --- a/src/config.rs +++ b/src/config.rs @@ -4,22 +4,364 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -use std::collections::HashMap; +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + fmt, + path::{Path, PathBuf}, +}; -use serde::{Deserialize, Serialize}; -use url::Url; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use snafu::{Backtrace, IntoError, OptionExt, ResultExt, Snafu}; +use url::{Position, Url}; -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Theme { - /// Where to fetch the theme from. +use crate::proposal::ProposalNumber; + +pub const LOCAL_CONFIG_FILE: &str = ".build-eips.toml"; +pub const REPO_MANIFEST_FILE: &str = ".build-eips.repo.toml"; +pub const DEFAULT_BUILD_ROOT_BASE: &str = ".local-build"; +pub const DEFAULT_THEME_DIR: &str = "theme"; +pub const DEFAULT_SERVER_HOST: &str = "127.0.0.1"; +pub const DEFAULT_SERVER_PORT: u16 = 1111; +pub const DEFAULT_SITE_BASE_URL: &str = "http://127.0.0.1:1111"; +const RESERVED_WORKSPACE_NAMES: &[&str] = &[DEFAULT_THEME_DIR, "preprocessor", "eipw"]; + +#[derive(Debug, Snafu)] +pub enum RepoManifestError { + #[snafu( + context(name(RepoManifestIoSnafu)), + display("i/o error while accessing `{}`", path.to_string_lossy()) + )] + Io { + path: PathBuf, + source: std::io::Error, + backtrace: Backtrace, + }, + + #[snafu( + context(name(RepoManifestParseSnafu)), + display( + "unable to parse repo manifest `{}`", + manifest_path.to_string_lossy() + ) + )] + Parse { + manifest_path: PathBuf, + #[snafu(source(from(toml::de::Error, Box::new)))] + source: Box, + backtrace: Backtrace, + }, + + #[snafu(display( + "repo manifest `{}` is invalid: {reason}", + manifest_path.to_string_lossy() + ))] + Invalid { + manifest_path: PathBuf, + reason: String, + backtrace: Backtrace, + }, +} + +#[derive(Debug, Snafu)] +pub enum WorkspaceError { + #[snafu(display("i/o error while accessing `{}`", path.to_string_lossy()))] + Fs { + path: PathBuf, + source: std::io::Error, + backtrace: Backtrace, + }, + + #[snafu(display( + "unable to parse workspace config `{}`", + config_path.to_string_lossy() + ))] + Parse { + config_path: PathBuf, + #[snafu(source(from(toml::de::Error, Box::new)))] + source: Box, + backtrace: Backtrace, + }, +} + +/// Environment-specific repository metadata for an active proposal repo or sibling repo. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RepositoryEndpoint { + /// Git repository to fetch proposal content from. pub repository: Url, - /// Specific revision to checkout from the theme repository. - pub commit: String, + /// Base URL where rendered HTML and assets for this repository are served. + pub base_url: Url, +} + +/// Tracked active-repo manifest loaded from `.build-eips.repo.toml`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RepoManifest { + /// Stable machine key for workspace directory names, build roots, and sibling references. + pub repo_id: String, + + /// Production repository and base URL for this active repo. + pub production: RepositoryEndpoint, + + /// Staging repository and base URL for this active repo. + pub staging: RepositoryEndpoint, + + /// Directional sibling content repos used by this active repo. + #[serde(default)] + pub siblings: BTreeMap, +} + +impl RepoManifest { + fn from_raw(raw: RawRepoManifest, manifest_path: &Path) -> Result { + let repo_id = Self::required_value(manifest_path, "repo_id", raw.repo_id)?; + let production = Self::required_value(manifest_path, "production", raw.production)?; + let staging = Self::required_value(manifest_path, "staging", raw.staging)?; + let siblings = raw + .siblings + .into_iter() + .map(|(repo_id, sibling)| { + let production = Self::required_value( + manifest_path, + &format!("siblings.{repo_id}.production"), + sibling.production, + )?; + let staging = Self::required_value( + manifest_path, + &format!("siblings.{repo_id}.staging"), + sibling.staging, + )?; + + Ok(( + repo_id, + RepoManifestSibling { + production, + staging, + }, + )) + }) + .collect::>()?; + + let manifest = Self { + repo_id, + production, + staging, + siblings, + }; + manifest.validate(manifest_path)?; + Ok(manifest) + } + + fn validate(&self, manifest_path: &Path) -> Result<(), RepoManifestError> { + Self::validate_repo_key(manifest_path, "repo_id", &self.repo_id)?; + + if self.siblings.contains_key(&self.repo_id) { + return InvalidSnafu { + manifest_path: manifest_path.to_path_buf(), + reason: format!( + "repo_id `{}` cannot also be declared as a sibling", + self.repo_id + ), + } + .fail(); + } + + for sibling_id in self.siblings.keys() { + Self::validate_repo_key(manifest_path, "sibling key", sibling_id)?; + } + + Self::validate_unique_sibling_repositories( + manifest_path, + "production", + self.siblings + .iter() + .map(|(id, sibling)| (id.as_str(), sibling.production.repository.as_str())), + )?; + Self::validate_unique_sibling_repositories( + manifest_path, + "staging", + self.siblings + .iter() + .map(|(id, sibling)| (id.as_str(), sibling.staging.repository.as_str())), + )?; + + Ok(()) + } + + pub fn active_endpoint(&self, staging: bool) -> &RepositoryEndpoint { + if staging { + &self.staging + } else { + &self.production + } + } + + pub fn sibling_repositories(&self, staging: bool) -> BTreeMap { + self.siblings + .iter() + .map(|(repo_id, sibling)| { + let endpoint = if staging { + &sibling.staging + } else { + &sibling.production + }; + (repo_id.clone(), endpoint.repository.clone()) + }) + .collect() + } + + fn required_value( + manifest_path: &Path, + field: &str, + value: Option, + ) -> Result { + value.with_context(|| InvalidSnafu { + manifest_path: manifest_path.to_path_buf(), + reason: format!("missing required `{field}` entry"), + }) + } + + fn validate_repo_key( + manifest_path: &Path, + label: &str, + key: &str, + ) -> Result<(), RepoManifestError> { + let invalid_reason = if key.is_empty() { + Some("must not be empty") + } else if matches!(key, "." | "..") { + Some("must not be `.` or `..`") + } else if key.contains('/') || key.contains('\\') { + Some("must be a single safe path component") + } else if RESERVED_WORKSPACE_NAMES.contains(&key) { + Some("collides with a reserved workspace/platform directory name") + } else { + None + }; + + if let Some(reason) = invalid_reason { + return InvalidSnafu { + manifest_path: manifest_path.to_path_buf(), + reason: format!("{label} `{key}` {reason}"), + } + .fail(); + } + + Ok(()) + } + + fn validate_unique_sibling_repositories<'a>( + manifest_path: &Path, + environment: &str, + siblings: impl Iterator, + ) -> Result<(), RepoManifestError> { + let mut seen = HashSet::new(); + for (repo_id, repository) in siblings { + if !seen.insert(repository) { + return InvalidSnafu { + manifest_path: manifest_path.to_path_buf(), + reason: format!( + "duplicate {environment} sibling repository declaration `{repository}` under sibling key `{repo_id}`" + ), + } + .fail(); + } + } + + Ok(()) + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct RawRepoManifest { + repo_id: Option, + production: Option, + staging: Option, + #[serde(default)] + siblings: BTreeMap, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct RawRepoManifestSibling { + production: Option, + staging: Option, +} + +/// Environment-specific metadata for one declared sibling content repo. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RepoManifestSibling { + /// Production repository and base URL for this sibling repo. + pub production: RepositoryEndpoint, + + /// Staging repository and base URL for this sibling repo. + pub staging: RepositoryEndpoint, +} + +#[derive(Debug, Clone)] +pub struct LoadedRepoManifest { + manifest_path: PathBuf, + manifest: RepoManifest, +} + +impl LoadedRepoManifest { + pub fn load(repo_root: &Path) -> Result, RepoManifestError> { + let manifest_path = repo_root.join(REPO_MANIFEST_FILE); + match std::fs::read_to_string(&manifest_path) { + Ok(contents) => Self::from_contents(manifest_path, &contents).map(Some), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => + { + Ok(None) + } + Err(error) => Err(RepoManifestIoSnafu { + path: manifest_path, + } + .into_error(error)), + } + } + + #[cfg(test)] + pub fn from_path(path: &Path) -> Result { + let manifest_path = path.canonicalize().with_context(|_| RepoManifestIoSnafu { + path: path.to_path_buf(), + })?; + let contents = + std::fs::read_to_string(&manifest_path).with_context(|_| RepoManifestIoSnafu { + path: manifest_path.clone(), + })?; + Self::from_contents(manifest_path, &contents) + } + + fn from_contents(manifest_path: PathBuf, contents: &str) -> Result { + let manifest = toml::from_str::(contents).with_context(|_| { + RepoManifestParseSnafu { + manifest_path: manifest_path.clone(), + } + })?; + let manifest = RepoManifest::from_raw(manifest, &manifest_path)?; + + Ok(Self { + manifest_path, + manifest, + }) + } + + pub fn manifest_path(&self) -> &Path { + &self.manifest_path + } + + pub fn manifest(&self) -> &RepoManifest { + &self.manifest + } } #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Location { +pub struct LegacyLocation { /// Git repository to fetch proposals from. pub repository: Url, @@ -33,14 +375,22 @@ pub struct Location { pub identifying_commit: String, } +impl LegacyLocation { + pub fn endpoint(&self) -> RepositoryEndpoint { + RepositoryEndpoint { + repository: self.repository.clone(), + base_url: self.base_url.clone(), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(transparent)] -pub struct Locations(pub HashMap); +pub struct LegacyLocations(pub HashMap); #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Config { - pub theme: Theme, - pub locations: Locations, + pub locations: LegacyLocations, } impl Config { @@ -49,7 +399,7 @@ impl Config { locations.insert( "EIPs".into(), - Location { + LegacyLocation { repository: "https://github.com/ethereum/EIPs.git".try_into().unwrap(), base_url: "https://eips.ethereum.org/".try_into().unwrap(), identifying_commit: "0f44e2b94df4e504bb7b912f56ebd712db2ad396".into(), @@ -58,7 +408,7 @@ impl Config { locations.insert( "ERCs".into(), - Location { + LegacyLocation { repository: "https://github.com/ethereum/ERCs.git".try_into().unwrap(), base_url: "https://ercs.ethereum.org/".try_into().unwrap(), identifying_commit: "8dd085d159cb123f545c272c0d871a5339550e79".into(), @@ -66,13 +416,7 @@ impl Config { ); Self { - theme: Theme { - repository: "https://github.com/ethereum/eips-theme.git" - .try_into() - .unwrap(), - commit: "0ddac35da36d311a8401c6cfb79c9991f78b647d".into(), - }, - locations: Locations(locations), + locations: LegacyLocations(locations), } } @@ -81,7 +425,7 @@ impl Config { locations.insert( "EIPs".into(), - Location { + LegacyLocation { repository: "https://github.com/eips-wg/EIPs.git".try_into().unwrap(), base_url: "https://eips-wg.github.io/EIPs/".try_into().unwrap(), identifying_commit: "0f44e2b94df4e504bb7b912f56ebd712db2ad396".into(), @@ -90,7 +434,7 @@ impl Config { locations.insert( "ERCs".into(), - Location { + LegacyLocation { repository: "https://github.com/eips-wg/ERCs.git".try_into().unwrap(), base_url: "https://eips-wg.github.io/ERCs/".try_into().unwrap(), identifying_commit: "8dd085d159cb123f545c272c0d871a5339550e79".into(), @@ -98,11 +442,797 @@ impl Config { ); Self { - theme: Theme { - repository: "https://github.com/eips-wg/theme.git".try_into().unwrap(), - commit: "0ddac35da36d311a8401c6cfb79c9991f78b647d".into(), - }, - locations: Locations(locations), + locations: LegacyLocations(locations), } } } + +/// Workspace-local configuration loaded from `.build-eips.toml`. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct WorkspaceConfig { + /// Local server defaults for `build-eips serve` and `build-eips preview`. + #[serde(default)] + pub server: ServerSettings, + + /// Local rendered-site URL defaults for build and serve commands. + #[serde(default)] + pub site: SiteSettings, + + /// Local render filtering defaults. + #[serde(default)] + pub render: RenderSettings, +} + +impl WorkspaceConfig { + fn starter() -> Self { + Self { + server: ServerSettings::default(), + site: SiteSettings::starter(), + render: RenderSettings::default(), + } + } +} + +/// Local render filtering settings. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct RenderSettings { + /// Proposal numbers to render for applicable local build and serve commands. + #[serde(default)] + pub only: Vec, +} + +/// Workspace-local bind address defaults for local server commands. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct ServerSettings { + /// Host or interface address used by `serve` and `preview`. + pub host: String, + + /// TCP port used by `serve` and `preview`. + pub port: u16, +} + +impl Default for ServerSettings { + fn default() -> Self { + Self { + host: DEFAULT_SERVER_HOST.to_owned(), + port: DEFAULT_SERVER_PORT, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServerBinding { + pub host: String, + pub port: u16, +} + +impl Default for ServerBinding { + fn default() -> Self { + ServerSettings::default().into() + } +} + +impl From for ServerBinding { + fn from(settings: ServerSettings) -> Self { + Self { + host: settings.host, + port: settings.port, + } + } +} + +impl From<&ServerSettings> for ServerBinding { + fn from(settings: &ServerSettings) -> Self { + settings.clone().into() + } +} + +impl fmt::Display for ServerBinding { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}:{}", self.host, self.port) + } +} + +/// Workspace-local rendered-site URL defaults for build and serve commands. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct SiteSettings { + /// Base URL written into rendered HTML, feeds, canonical links, and sitemaps. + #[serde( + default, + serialize_with = "serialize_optional_base_url", + deserialize_with = "deserialize_optional_base_url" + )] + pub base_url: Option, +} + +impl SiteSettings { + fn starter() -> Self { + Self { + base_url: Some( + DEFAULT_SITE_BASE_URL + .parse() + .expect("default site base URL should parse"), + ), + } + } +} + +fn serialize_optional_base_url(base_url: &Option, serializer: S) -> Result +where + S: Serializer, +{ + match base_url { + Some(base_url) => serializer.serialize_some(&format_base_url(base_url)), + None => serializer.serialize_none(), + } +} + +fn deserialize_optional_base_url<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + Option::::deserialize(deserializer) +} + +fn format_base_url(base_url: &Url) -> String { + if base_url.path() == "/" && base_url.query().is_none() && base_url.fragment().is_none() { + base_url[..Position::BeforePath].to_owned() + } else { + base_url.as_str().to_owned() + } +} + +#[derive(Debug, Clone)] +pub struct LoadedWorkspaceConfig { + config_path: PathBuf, + workspace_root: PathBuf, + config: WorkspaceConfig, +} + +impl LoadedWorkspaceConfig { + pub fn from_path(path: &Path) -> Result { + let config_path = path.canonicalize().with_context(|_| FsSnafu { + path: path.to_path_buf(), + })?; + let contents = std::fs::read_to_string(&config_path).with_context(|_| FsSnafu { + path: config_path.clone(), + })?; + let config = toml::from_str::(&contents).with_context(|_| ParseSnafu { + config_path: config_path.clone(), + })?; + + let workspace_root = config_path + .parent() + .expect("workspace config should always have a parent") + .to_path_buf(); + + Ok(Self { + config_path, + workspace_root, + config, + }) + } + + pub fn discover(start: &Path) -> Result, WorkspaceError> { + match discover_path(start) { + Some(path) => Self::from_path(&path).map(Some), + None => Ok(None), + } + } + + pub fn config_path(&self) -> &Path { + &self.config_path + } + + pub fn workspace_root(&self) -> &Path { + &self.workspace_root + } + + pub fn workspace_build_root(&self, repo_name: &str) -> PathBuf { + self.workspace_root + .join(DEFAULT_BUILD_ROOT_BASE) + .join(repo_name) + } + + pub fn server_settings(&self) -> &ServerSettings { + &self.config.server + } + + pub fn site_settings(&self) -> &SiteSettings { + &self.config.site + } + + pub fn render_settings(&self) -> &RenderSettings { + &self.config.render + } + + pub fn local_theme_path(&self) -> PathBuf { + self.workspace_root.join(DEFAULT_THEME_DIR) + } + + pub fn local_repo_path(&self, repo_name: &str) -> PathBuf { + self.workspace_root.join(repo_name) + } +} + +pub fn discover_path(start: &Path) -> Option { + let mut current = Some(start); + + while let Some(candidate) = current { + let config_path = candidate.join(LOCAL_CONFIG_FILE); + match std::fs::File::open(&config_path) { + Ok(_) => return Some(config_path), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => + { + current = candidate.parent(); + } + Err(_) => return Some(config_path), + } + } + + None +} + +pub fn default_workspace_config_text() -> String { + toml::to_string_pretty(&WorkspaceConfig::starter()) + .expect("workspace starter config should serialize") +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use tempfile::TempDir; + + use super::{ + default_workspace_config_text, discover_path, LoadedRepoManifest, LoadedWorkspaceConfig, + RepoManifestError, ServerBinding, ServerSettings, WorkspaceError, DEFAULT_SERVER_HOST, + DEFAULT_SERVER_PORT, DEFAULT_SITE_BASE_URL, LOCAL_CONFIG_FILE, REPO_MANIFEST_FILE, + }; + use crate::proposal::ProposalNumber; + + struct TestWorkspace { + tempdir: TempDir, + } + + impl TestWorkspace { + fn new() -> Self { + Self { + tempdir: TempDir::new().unwrap(), + } + } + + fn root(&self) -> &Path { + self.tempdir.path() + } + + fn path(&self, relative: impl AsRef) -> PathBuf { + self.root().join(relative) + } + + fn write_file(&self, relative: impl AsRef, contents: &str) -> PathBuf { + let path = self.path(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(&path, contents).unwrap(); + path + } + + fn create_dir(&self, relative: impl AsRef) -> PathBuf { + let path = self.path(relative); + std::fs::create_dir_all(&path).unwrap(); + path + } + } + + fn manifest_text(repo_id: &str, siblings: &str) -> String { + format!( + r#" +repo_id = "{repo_id}" + +[production] +repository = "https://example.test/{repo_id}.git" +base_url = "https://example.test/{repo_id}/" + +[staging] +repository = "https://staging.example.test/{repo_id}.git" +base_url = "https://staging.example.test/{repo_id}/" + +{siblings} +"# + ) + } + + fn manifest_invalid_reason(error: RepoManifestError) -> String { + match error { + RepoManifestError::Invalid { reason, .. } => reason, + other => panic!("expected invalid repo manifest, got {other:?}"), + } + } + + #[test] + fn missing_repo_manifest_loads_as_none() { + let workspace = TestWorkspace::new(); + + assert!(LoadedRepoManifest::load(workspace.root()) + .unwrap() + .is_none()); + } + + #[test] + fn malformed_repo_manifest_reports_parse_error() { + let workspace = TestWorkspace::new(); + let manifest_path = workspace.write_file(REPO_MANIFEST_FILE, "repo_id = ["); + + let error = LoadedRepoManifest::from_path(&manifest_path).unwrap_err(); + + assert!(matches!(error, RepoManifestError::Parse { .. })); + } + + #[test] + fn parses_repo_manifest_with_directional_siblings() { + let workspace = TestWorkspace::new(); + let manifest_path = workspace.write_file( + REPO_MANIFEST_FILE, + &manifest_text( + "Core", + r#" +[siblings.EIPs.production] +repository = "https://example.test/EIPs.git" +base_url = "https://example.test/EIPs/" + +[siblings.EIPs.staging] +repository = "https://staging.example.test/EIPs.git" +base_url = "https://staging.example.test/EIPs/" +"#, + ), + ); + + let manifest = LoadedRepoManifest::from_path(&manifest_path).unwrap(); + + assert_eq!(manifest.manifest().repo_id, "Core"); + assert_eq!(manifest.manifest().siblings.len(), 1); + assert!(manifest.manifest().siblings.contains_key("EIPs")); + } + + #[test] + fn repo_manifest_requires_identity_and_environments() { + let workspace = TestWorkspace::new(); + let manifest_path = workspace.write_file( + REPO_MANIFEST_FILE, + r#" +[production] +repository = "https://example.test/Core.git" +base_url = "https://example.test/Core/" +"#, + ); + + let reason = + manifest_invalid_reason(LoadedRepoManifest::from_path(&manifest_path).unwrap_err()); + + assert!(reason.contains("missing required `repo_id` entry")); + + let manifest_path = workspace.write_file( + REPO_MANIFEST_FILE, + r#" +repo_id = "Core" + +[production] +repository = "https://example.test/Core.git" +base_url = "https://example.test/Core/" +"#, + ); + let reason = + manifest_invalid_reason(LoadedRepoManifest::from_path(&manifest_path).unwrap_err()); + + assert!(reason.contains("missing required `staging` entry")); + + let manifest_path = workspace.write_file( + REPO_MANIFEST_FILE, + r#" +repo_id = "Core" + +[staging] +repository = "https://staging.example.test/Core.git" +base_url = "https://staging.example.test/Core/" +"#, + ); + let reason = + manifest_invalid_reason(LoadedRepoManifest::from_path(&manifest_path).unwrap_err()); + + assert!(reason.contains("missing required `production` entry")); + } + + #[test] + fn repo_manifest_rejects_unsafe_and_reserved_keys() { + let workspace = TestWorkspace::new(); + let manifest_path = workspace.write_file(REPO_MANIFEST_FILE, &manifest_text("theme", "")); + + let reason = + manifest_invalid_reason(LoadedRepoManifest::from_path(&manifest_path).unwrap_err()); + + assert!(reason.contains("repo_id `theme`")); + assert!(reason.contains("reserved")); + + let manifest_path = + workspace.write_file(REPO_MANIFEST_FILE, &manifest_text("Core/Meta", "")); + let reason = + manifest_invalid_reason(LoadedRepoManifest::from_path(&manifest_path).unwrap_err()); + + assert!(reason.contains("repo_id `Core/Meta`")); + assert!(reason.contains("single safe path component")); + } + + #[test] + fn repo_manifest_rejects_self_sibling() { + let workspace = TestWorkspace::new(); + let manifest_path = workspace.write_file( + REPO_MANIFEST_FILE, + &manifest_text( + "Core", + r#" +[siblings.Core.production] +repository = "https://example.test/Core.git" +base_url = "https://example.test/Core/" + +[siblings.Core.staging] +repository = "https://staging.example.test/Core.git" +base_url = "https://staging.example.test/Core/" +"#, + ), + ); + + let reason = + manifest_invalid_reason(LoadedRepoManifest::from_path(&manifest_path).unwrap_err()); + + assert!(reason.contains("cannot also be declared as a sibling")); + } + + #[test] + fn repo_manifest_rejects_duplicate_sibling_repositories_per_environment() { + let workspace = TestWorkspace::new(); + let manifest_path = workspace.write_file( + REPO_MANIFEST_FILE, + &manifest_text( + "Core", + r#" +[siblings.EIPs.production] +repository = "https://example.test/shared.git" +base_url = "https://example.test/EIPs/" + +[siblings.EIPs.staging] +repository = "https://staging.example.test/EIPs.git" +base_url = "https://staging.example.test/EIPs/" + +[siblings.ERCs.production] +repository = "https://example.test/shared.git" +base_url = "https://example.test/ERCs/" + +[siblings.ERCs.staging] +repository = "https://staging.example.test/ERCs.git" +base_url = "https://staging.example.test/ERCs/" +"#, + ), + ); + + let reason = + manifest_invalid_reason(LoadedRepoManifest::from_path(&manifest_path).unwrap_err()); + + assert!(reason.contains("duplicate production sibling repository declaration")); + } + + #[test] + fn parses_default_workspace_config() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file(LOCAL_CONFIG_FILE, &default_workspace_config_text()); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!(config.workspace_root(), workspace.root()); + assert_eq!(config.server_settings(), &ServerSettings::default()); + assert_eq!( + config.site_settings().base_url.as_ref().unwrap().as_str(), + "http://127.0.0.1:1111/" + ); + } + + #[test] + fn starter_workspace_config_roundtrips_stably() { + let original = default_workspace_config_text(); + let parsed = toml::from_str::(&original).unwrap(); + let reparsed = toml::to_string_pretty(&parsed).unwrap(); + + assert_eq!(reparsed, original); + assert!(!original.contains("build_root_base")); + assert!(original.contains("[server]")); + assert!(original.contains("host = \"127.0.0.1\"")); + assert!(original.contains("port = 1111")); + assert!(original.contains("[site]")); + assert!(original.contains(&format!("base_url = \"{DEFAULT_SITE_BASE_URL}\""))); + assert!(original.contains("[render]")); + assert!(original.contains("only = []")); + assert!(!original.contains("default_profile")); + assert!(!original.contains("[profiles")); + } + + #[test] + fn parses_workspace_config_server_settings() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( + LOCAL_CONFIG_FILE, + r#" +[server] +host = "0.0.0.0" +port = 8080 +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!( + config.server_settings(), + &ServerSettings { + host: "0.0.0.0".to_owned(), + port: 8080, + } + ); + } + + #[test] + fn missing_server_settings_use_default_binding() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( + LOCAL_CONFIG_FILE, + r#" +[site] +base_url = "http://localhost:4000" +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + let binding = ServerBinding::from(config.server_settings()); + + assert_eq!(binding.host, DEFAULT_SERVER_HOST); + assert_eq!(binding.port, DEFAULT_SERVER_PORT); + assert_eq!(binding.to_string(), "127.0.0.1:1111"); + } + + #[test] + fn parses_workspace_config_site_settings() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( + LOCAL_CONFIG_FILE, + r#" +[site] +base_url = "http://localhost:4000" +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!( + config.site_settings().base_url.as_ref().unwrap().as_str(), + "http://localhost:4000/" + ); + } + + #[test] + fn invalid_workspace_config_site_base_url_errors() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( + LOCAL_CONFIG_FILE, + r#" +[site] +base_url = "not a url" +"#, + ); + let error = LoadedWorkspaceConfig::from_path(&config_path).unwrap_err(); + + assert!(error + .to_string() + .contains("unable to parse workspace config")); + } + + #[test] + fn missing_site_settings_preserve_no_base_url_override() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( + LOCAL_CONFIG_FILE, + r#" +[server] +host = "127.0.0.1" +port = 1111 +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert!(config.site_settings().base_url.is_none()); + } + + #[test] + fn minimal_workspace_config_parses() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( + LOCAL_CONFIG_FILE, + r#" +[server] +host = "127.0.0.1" +port = 1111 + +[site] +base_url = "http://127.0.0.1:1111" +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!(config.server_settings(), &ServerSettings::default()); + assert_eq!( + config.site_settings().base_url.as_ref().unwrap().as_str(), + "http://127.0.0.1:1111/" + ); + } + + #[test] + fn empty_workspace_config_uses_defaults() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file(LOCAL_CONFIG_FILE, " \n"); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!(config.server_settings(), &ServerSettings::default()); + assert!(config.site_settings().base_url.is_none()); + assert!(config.render_settings().only.is_empty()); + } + + #[test] + fn parses_workspace_config_render_only_settings() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( + LOCAL_CONFIG_FILE, + r#" +[render] +only = [555, 678, 555] +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!( + config.render_settings().only, + vec![ + ProposalNumber::from_u32(555).unwrap(), + ProposalNumber::from_u32(678).unwrap(), + ProposalNumber::from_u32(555).unwrap(), + ] + ); + } + + #[test] + fn missing_render_missing_only_and_empty_only_disable_filtering() { + let cases = [ + ("missing render", ""), + ("missing only", "[render]\n"), + ("empty only", "[render]\nonly = []\n"), + ]; + + for (name, contents) in cases { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file(LOCAL_CONFIG_FILE, contents); + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert!( + config.render_settings().only.is_empty(), + "expected `{name}` to disable render filtering" + ); + } + } + + #[test] + fn workspace_config_render_only_rejects_non_positive_and_non_integer_values() { + let cases = [ + ("zero", "only = [0]"), + ("negative", "only = [-555]"), + ("quoted", "only = [\"555\"]"), + ("overflow", "only = [4294967296]"), + ]; + + for (name, contents) in cases { + let workspace = TestWorkspace::new(); + let config_path = + workspace.write_file(LOCAL_CONFIG_FILE, &format!("[render]\n{contents}\n")); + let error = LoadedWorkspaceConfig::from_path(&config_path).unwrap_err(); + + assert!( + matches!(error, WorkspaceError::Parse { .. }), + "expected `{name}` render only config to fail, got {error:?}" + ); + } + } + + #[test] + fn removed_workspace_config_fields_use_strict_parse_errors() { + let removed_theme_ref_field = concat!("co", "mmit"); + let cases = vec![ + ( + "build_root_base".to_owned(), + r#"build_root_base = ".local-build""#.to_owned(), + ), + ( + "default_profile".to_owned(), + r#"default_profile = "local""#.to_owned(), + ), + ( + "profiles".to_owned(), + r#" +[profiles.local] +staging = true +"# + .to_owned(), + ), + ( + "theme".to_owned(), + r#" +[theme] +repository = "https://github.com/eips-wg/theme.git" +"# + .to_owned(), + ), + ( + format!("theme.{removed_theme_ref_field}"), + format!( + r#" +[theme] +{removed_theme_ref_field} = "3a597d4cd68ec82d36f01c01335492cfa59501ae" +"# + ), + ), + ]; + + for (field, contents) in cases { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file(LOCAL_CONFIG_FILE, &contents); + let error = LoadedWorkspaceConfig::from_path(&config_path).unwrap_err(); + + assert!( + matches!(error, WorkspaceError::Parse { .. }), + "expected strict parse error for removed field `{field}`, got {error:?}" + ); + } + } + + #[test] + fn discover_path_walks_upward() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file(LOCAL_CONFIG_FILE, &default_workspace_config_text()); + let nested = workspace.create_dir("EIPs/content"); + + assert_eq!(discover_path(&nested).unwrap(), config_path); + assert_eq!( + LoadedWorkspaceConfig::discover(&nested) + .unwrap() + .unwrap() + .config_path(), + config_path + ); + } + + #[test] + fn missing_workspace_config_is_not_discovered() { + let workspace = TestWorkspace::new(); + let nested = workspace.create_dir("EIPs/content"); + + assert!(discover_path(&nested).is_none()); + assert!(LoadedWorkspaceConfig::discover(&nested).unwrap().is_none()); + } +} diff --git a/src/context.rs b/src/context.rs index 0fffb43..1beb395 100644 --- a/src/context.rs +++ b/src/context.rs @@ -4,17 +4,61 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -use std::path::PathBuf; +//! Command context and path resolution helpers. + +use std::path::{Path, PathBuf}; use snafu::{ResultExt, Whatever}; -use crate::{cli::Args, find_root}; +use crate::{cli::Args, config, find_root}; + +#[derive(Debug, Clone)] +pub(crate) struct WorkspaceCommandContext { + pub(crate) search_from: PathBuf, + pub(crate) config_path: Option, +} + +pub(crate) fn resolve_input_path(path: &Path) -> Result { + if path.is_absolute() { + Ok(path.to_path_buf()) + } else { + let cwd = std::env::current_dir().whatever_context("unable to get current directory")?; + Ok(cwd.join(path)) + } +} pub(crate) fn root(args: &Args) -> Result { let dir = match &args.root { None => find_root::find_root().whatever_context("cannot find repository root")?, - Some(p) => p.to_path_buf(), + Some(path) => { + find_root::is_root(path).whatever_context("invalid root directory")?; + path.canonicalize() + .whatever_context("unable to canonicalize root directory")? + } }; find_root::is_root(&dir).whatever_context("invalid root directory")?; Ok(dir) } + +fn workspace_search_start(args: &Args) -> Result { + match &args.root { + Some(path) => { + let path = resolve_input_path(path)?; + path.canonicalize() + .whatever_context("unable to canonicalize workspace search path") + } + None => std::env::current_dir().whatever_context("unable to get current directory"), + } +} + +pub(crate) fn load_workspace_command_context( + args: &Args, +) -> Result { + let search_from = workspace_search_start(args)?; + let config_path = config::discover_path(&search_from); + + Ok(WorkspaceCommandContext { + search_from, + config_path, + }) +} diff --git a/src/editorial.rs b/src/editorial.rs new file mode 100644 index 0000000..2b1ce5a --- /dev/null +++ b/src/editorial.rs @@ -0,0 +1,928 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Editorial target selection and runtime helpers. + +use std::{ + collections::BTreeSet, + fs::File, + io::ErrorKind, + path::{Path, PathBuf}, +}; + +use log::info; +use snafu::{OptionExt, ResultExt, Whatever}; + +use crate::{ + cli::EditorialSelectorArgs, + context::resolve_input_path, + execution::ResolvedExecution, + git, + layout::REPO_DIR, + lint, + proposal::{ + classify_editorial_number_selector, is_proposal_path, + resolve_proposal_number_markdown_path, EditorialNumberSelector, + }, +}; + +fn repo_relative_canonical_path( + root_path: &Path, + path: &Path, + canonical_path: &Path, +) -> Result { + if path.is_absolute() { + snafu::whatever!( + "editorial selectors require repo-relative proposal paths, got `{}`", + path.to_string_lossy() + ); + } + + let relative = canonical_path + .strip_prefix(root_path) + .with_whatever_context(|_| { + format!( + "editorial target `{}` escapes the active repository root", + path.to_string_lossy() + ) + })? + .to_path_buf(); + + Ok(relative) +} + +fn validate_editorial_targets( + root_path: &Path, + paths: Vec, + strict: bool, +) -> Result, Whatever> { + let mut unique = BTreeSet::new(); + let mut targets = Vec::new(); + + for path in paths { + if path.is_absolute() { + snafu::whatever!( + "editorial selectors require repo-relative proposal paths, got `{}`", + path.to_string_lossy() + ); + } + + let full_path = root_path.join(&path); + let canonical_path = match full_path.canonicalize() { + Ok(canonical_path) => canonical_path, + Err(error) + if !strict + && matches!(error.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => + { + continue; + } + Err(error) => { + return Err(error).with_whatever_context(|_| { + format!( + "unable to resolve editorial target `{}`", + full_path.to_string_lossy() + ) + }); + } + }; + + let relative = repo_relative_canonical_path(root_path, &path, &canonical_path)?; + + if !is_proposal_path(&relative) { + if strict { + snafu::whatever!( + "editorial target `{}` is not a supported proposal path", + relative.to_string_lossy() + ); + } + continue; + } + + if unique.insert(relative.clone()) { + targets.push(relative); + } + } + + if strict && targets.is_empty() { + snafu::whatever!("editorial selector resolved no proposal files"); + } + + Ok(targets) +} + +fn read_editorial_batch(path: &Path) -> Result, Whatever> { + let contents = + std::fs::read_to_string(path).whatever_context("unable to read editorial batch file")?; + let mut paths = Vec::new(); + + for line in contents.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + paths.push(PathBuf::from(line)); + } + + Ok(paths) +} + +fn normalize_editorial_selector(root_path: &Path, path: PathBuf) -> Result { + let Some(selector) = path.as_os_str().to_str() else { + return Ok(path); + }; + + match classify_editorial_number_selector(selector) { + EditorialNumberSelector::Number(proposal_number) => { + resolve_proposal_number_markdown_path(root_path, proposal_number) + } + EditorialNumberSelector::InvalidNumberLike(_failure) => { + snafu::whatever!( + "editorial number selector `{selector}` is invalid; expected a positive proposal number that fits in u32, without signs or commas" + ); + } + EditorialNumberSelector::PathLike => Ok(path), + } +} + +fn normalize_editorial_selectors( + root_path: &Path, + paths: Vec, +) -> Result, Whatever> { + paths + .into_iter() + .map(|path| normalize_editorial_selector(root_path, path)) + .collect::>() +} + +fn prepare_editorial_lint_source( + resolved: &ResolvedExecution, +) -> Result<(PathBuf, git::SourceOnly), Whatever> { + let repo_path = resolved.build_path.join(REPO_DIR); + let source = git::Fresh::new( + &resolved.root_path, + &repo_path, + resolved.repository_use.clone(), + resolved.source_materialization, + ) + .whatever_context("initializing build repo for editorial source preparation")? + .clone_src() + .whatever_context("cloning source repo for editorial source preparation")?; + + Ok((repo_path, source)) +} + +fn prepare_editorial_lint_source_with_upstream( + resolved: &ResolvedExecution, +) -> Result<(PathBuf, git::SourceWithUpstream), Whatever> { + let (repo_path, source) = prepare_editorial_lint_source(resolved)?; + let source = source + .fetch_upstream() + .whatever_context("fetching upstream repo for editorial source preparation")?; + + Ok((repo_path, source)) +} + +fn raw_editorial_targets( + selectors: &EditorialSelectorArgs, + resolved: &ResolvedExecution, + upstream_source: Option<&git::SourceWithUpstream>, +) -> Result, Whatever> { + if selectors.selector_count() != 1 { + snafu::whatever!( + "choose exactly one editorial selector: explicit proposal targets, `--batch`, `--working-tree`, or `--against-upstream`" + ); + } + + let raw_targets = if !selectors.paths.is_empty() { + selectors.paths.clone() + } else if let Some(batch) = selectors.batch.as_deref() { + let batch = resolve_input_path(batch)?; + read_editorial_batch(&batch)? + } else if selectors.working_tree { + git::working_tree_paths(&resolved.root_path) + .whatever_context("unable to resolve working-tree editorial targets")? + } else { + upstream_source + .whatever_context( + "against-upstream editorial target selection requires upstream source", + )? + .changed_files() + .whatever_context("unable to list editorial targets against upstream")? + }; + + Ok(raw_targets) +} + +fn validate_raw_editorial_targets( + selectors: &EditorialSelectorArgs, + resolved: &ResolvedExecution, + raw_targets: Vec, +) -> Result, Whatever> { + let strict = !selectors.paths.is_empty() || selectors.batch.is_some(); + let targets = if strict { + normalize_editorial_selectors(&resolved.root_path, raw_targets)? + } else { + raw_targets + }; + validate_editorial_targets(&resolved.root_path, targets, strict) +} + +pub(crate) fn editorial_targets_from_source( + selectors: &EditorialSelectorArgs, + resolved: &ResolvedExecution, + upstream_source: Option<&git::SourceWithUpstream>, +) -> Result, Whatever> { + let raw_targets = raw_editorial_targets(selectors, resolved, upstream_source)?; + + validate_raw_editorial_targets(selectors, resolved, raw_targets) +} + +pub(crate) fn editorial_targets( + selectors: &EditorialSelectorArgs, + resolved: &ResolvedExecution, +) -> Result, Whatever> { + editorial_targets_from_source(selectors, resolved, None) +} + +fn validate_prepared_editorial_targets( + prepared_repo_path: &Path, + targets: &[PathBuf], +) -> Result<(), Whatever> { + for target in targets { + if target.is_absolute() { + snafu::whatever!( + "editorial selectors require repo-relative proposal paths, got `{}`", + target.to_string_lossy() + ); + } + + let prepared_target = prepared_repo_path.join(target); + let file = match File::open(&prepared_target) { + Ok(file) => file, + Err(error) + if matches!(error.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => + { + snafu::whatever!( + "editorial target `{}` exists in the active repo but was not materialized into the prepared source tree; untracked files are not supported", + target.to_string_lossy() + ); + } + Err(error) => { + return Err(error).with_whatever_context(|_| { + format!( + "unable to open prepared editorial target `{}`", + prepared_target.to_string_lossy() + ) + }); + } + }; + let metadata = file.metadata().with_whatever_context(|_| { + format!( + "unable to inspect prepared editorial target `{}`", + prepared_target.to_string_lossy() + ) + })?; + + if !metadata.is_file() { + snafu::whatever!( + "prepared editorial target `{}` is not a file", + target.to_string_lossy() + ); + } + } + + Ok(()) +} + +pub(crate) fn run_editorial_lint( + resolved: &ResolvedExecution, + selectors: &EditorialSelectorArgs, + eipw: lint::CmdArgs, +) -> Result { + if selectors.against_upstream { + let (repo_path, source) = prepare_editorial_lint_source_with_upstream(resolved)?; + let targets = editorial_targets_from_source(selectors, resolved, Some(&source))?; + if targets.is_empty() { + info!("editorial selector resolved no proposal files; skipping editorial lint"); + return Ok(false); + } + source + .merge() + .whatever_context("unable to merge ERC/EIP repositories for editorial lint")?; + validate_prepared_editorial_targets(&repo_path, &targets)?; + + lint::eipw(resolved.theme_path()?, &repo_path, targets, eipw) + .whatever_context("editorial lint failed")?; + + return Ok(true); + } + + let targets = editorial_targets(selectors, resolved)?; + if targets.is_empty() { + info!("editorial selector resolved no proposal files; skipping editorial lint"); + return Ok(false); + } + + let (repo_path, source) = prepare_editorial_lint_source(resolved)?; + source + .merge() + .whatever_context("unable to merge ERC/EIP repositories for editorial lint")?; + validate_prepared_editorial_targets(&repo_path, &targets)?; + + lint::eipw(resolved.theme_path()?, &repo_path, targets, eipw) + .whatever_context("editorial lint failed")?; + + Ok(true) +} + +pub(crate) fn editorial_runtime_execution( + mut resolved: ResolvedExecution, + selectors: &EditorialSelectorArgs, +) -> ResolvedExecution { + if selectors.working_tree { + resolved.source_materialization = git::SourceMaterialization::Dirty; + } + resolved +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use clap::Parser; + use eipw_lint::config::DefaultOptions; + use git2::{IndexAddOption, Repository, Signature}; + use tempfile::TempDir; + use url::Url; + + use crate::{ + cli::{Args, EditorialCommand, EditorialSelectorArgs, RuntimeOperation}, + config::{self, ServerBinding}, + execution::{resolve_execution, ResolvedExecution}, + }; + + use super::{ + editorial_runtime_execution, editorial_targets, run_editorial_lint, + validate_editorial_targets, + }; + + struct EditorialWorkspace { + _temp: TempDir, + active_path: PathBuf, + } + + fn write_file(root: &Path, relative: impl AsRef, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn commit_all(repo: &Repository, message: &str) { + let mut index = repo.index().unwrap(); + index + .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + let parents = repo + .head() + .ok() + .and_then(|head| head.target()) + .map(|oid| repo.find_commit(oid).unwrap()) + .into_iter() + .collect::>(); + let parent_refs = parents.iter().collect::>(); + + repo.commit( + Some("HEAD"), + &signature, + &signature, + message, + &tree, + &parent_refs, + ) + .unwrap(); + } + + fn init_repo(path: &Path, files: &[(&str, &str)]) -> Repository { + std::fs::create_dir_all(path).unwrap(); + let repo = Repository::init(path).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + for (relative, contents) in files { + write_file(path, relative, contents); + } + commit_all(&repo, "initial"); + repo + } + + fn file_url(path: &Path) -> Url { + Url::from_directory_path(path).unwrap() + } + + fn repo_manifest_text(repo_id: &str, repository: &Url, siblings: &[(&str, Url)]) -> String { + let mut manifest = format!( + r#" +repo_id = "{repo_id}" + +[production] +repository = "{repository}" +base_url = "https://example.test/{repo_id}/" + +[staging] +repository = "{repository}" +base_url = "https://staging.example.test/{repo_id}/" +"# + ); + + for (sibling_id, sibling_repository) in siblings { + manifest.push_str(&format!( + r#" +[siblings.{sibling_id}.production] +repository = "{sibling_repository}" +base_url = "https://example.test/{sibling_id}/" + +[siblings.{sibling_id}.staging] +repository = "{sibling_repository}" +base_url = "https://staging.example.test/{sibling_id}/" +"# + )); + } + + manifest + } + + fn proposal_markdown(number: u32, category: Option<&str>, body: &str) -> String { + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!( + "---\neip: {number}\ntitle: Proposal {number}\ndescription: Proposal {number}\nauthor: Test Author \ndiscussions-to: https://ethereum-magicians.org/t/test/{number}\nstatus: Draft\ntype: Standards Track\n{category}created: 2025-01-01\n---\n\n{body}\n" + ) + } + + fn write_eipw_config(workspace_root: &Path) { + let schema_version = DefaultOptions::::schema_version(); + write_file( + workspace_root, + "theme/config/eipw.toml", + &format!( + "schema-version = \"{schema_version}\"\n\n[fetch]\nproposal-format = \"{{:05}}\"\n" + ), + ); + } + + fn missing_file_url() -> Url { + let temp = TempDir::new().unwrap(); + file_url(&temp.path().join("missing-upstream")) + } + + fn editorial_workspace_with_upstream( + active_body: &str, + sibling_body: &str, + upstream_url: Option, + ) -> EditorialWorkspace { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("EIPs"); + let sibling_path = workspace_root.join("ERCs"); + let active_url = upstream_url.unwrap_or_else(|| file_url(&active_path)); + let sibling_url = file_url(&sibling_path); + let manifest = repo_manifest_text("EIPs", &active_url, &[("ERCs", sibling_url)]); + let active_markdown = proposal_markdown(1, None, active_body); + let sibling_markdown = proposal_markdown(2, Some("ERC"), sibling_body); + + write_file(&workspace_root, config::LOCAL_CONFIG_FILE, ""); + write_eipw_config(&workspace_root); + let _active_repo = init_repo( + &active_path, + &[ + (config::REPO_MANIFEST_FILE, manifest.as_str()), + ("content/00001.md", active_markdown.as_str()), + ], + ); + let _sibling_repo = init_repo( + &sibling_path, + &[("content/00002.md", sibling_markdown.as_str())], + ); + + EditorialWorkspace { + _temp: temp, + active_path, + } + } + + fn editorial_workspace(active_body: &str, sibling_body: &str) -> EditorialWorkspace { + editorial_workspace_with_upstream(active_body, sibling_body, None) + } + + fn editorial_workspace_with_missing_upstream( + active_body: &str, + sibling_body: &str, + ) -> EditorialWorkspace { + editorial_workspace_with_upstream(active_body, sibling_body, Some(missing_file_url())) + } + + fn parsed_editorial_lint( + active_path: &Path, + lint_args: &[&str], + ) -> ( + ResolvedExecution, + EditorialSelectorArgs, + crate::lint::CmdArgs, + ) { + let active_path = active_path.to_str().unwrap(); + let mut arguments = vec!["build-eips", "-C", active_path, "editorial", "lint"]; + arguments.extend_from_slice(lint_args); + let args = Args::try_parse_from(arguments).unwrap(); + let resolved = resolve_execution(&args).unwrap(); + + match args.operation.runtime_operation().unwrap() { + RuntimeOperation::Editorial { + command: EditorialCommand::Lint { selectors, eipw }, + } => (resolved, selectors, eipw), + _ => panic!("expected editorial lint command"), + } + } + + fn run_lint( + workspace: &EditorialWorkspace, + lint_args: &[&str], + ) -> Result { + let (resolved, selectors, eipw) = parsed_editorial_lint(&workspace.active_path, lint_args); + + run_editorial_lint(&resolved, &selectors, eipw) + } + + fn resolved_execution(root_path: PathBuf) -> ResolvedExecution { + ResolvedExecution { + root_path, + build_path: PathBuf::from("/workspace/build/Core"), + repository_use: crate::git::RepositoryUse { + title: "Core".to_owned(), + location: config::RepositoryEndpoint { + repository: "https://example.test/Core.git".parse().unwrap(), + base_url: "https://example.test/Core/".parse().unwrap(), + }, + other_repos: Default::default(), + }, + theme_path: Some(PathBuf::from("/workspace/theme")), + only: None, + source_materialization: crate::git::SourceMaterialization::Clean, + server_binding: ServerBinding::default(), + base_url_override: None, + } + } + + fn explicit_selectors(paths: &[&str]) -> EditorialSelectorArgs { + EditorialSelectorArgs { + paths: paths.iter().map(|path| PathBuf::from(*path)).collect(), + batch: None, + working_tree: false, + against_upstream: false, + } + } + + #[test] + fn editorial_lint_resolves_sibling_proposals_from_prepared_sources() { + let workspace = editorial_workspace_with_missing_upstream( + "Reference [ERC-2](./00002.md).", + "Sibling proposal.", + ); + + assert!(run_lint( + &workspace, + &[ + "content/00001.md", + "--no-default-lints", + "-D", + "markdown-refs" + ] + ) + .unwrap()); + } + + #[test] + fn editorial_batch_lint_resolves_siblings_without_fetching_active_upstream() { + let workspace = editorial_workspace_with_missing_upstream( + "Reference [ERC-2](./00002.md).", + "Sibling proposal.", + ); + let batch_path = workspace.active_path.join("targets.txt"); + write_file(&workspace.active_path, "targets.txt", "content/00001.md\n"); + let batch_path = batch_path.to_str().unwrap(); + + assert!(run_lint( + &workspace, + &[ + "--batch", + batch_path, + "--no-default-lints", + "-D", + "markdown-refs" + ] + ) + .unwrap()); + } + + #[test] + fn editorial_working_tree_lint_uses_dirty_content_without_fetching_active_upstream() { + let workspace = editorial_workspace_with_missing_upstream( + "Reference [ERC-9999](./09999.md).", + "Sibling proposal.", + ); + write_file( + &workspace.active_path, + "content/00001.md", + &proposal_markdown(1, None, "Reference [ERC-2](./00002.md)."), + ); + + assert!(run_lint( + &workspace, + &[ + "--working-tree", + "--no-default-lints", + "-D", + "markdown-refs" + ] + ) + .unwrap()); + } + + #[test] + fn editorial_against_upstream_lint_still_requires_active_upstream() { + let workspace = + editorial_workspace_with_missing_upstream("Active proposal.", "Sibling proposal."); + + let error = run_lint( + &workspace, + &[ + "--against-upstream", + "--no-default-lints", + "-D", + "markdown-refs", + ], + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("fetching upstream repo for editorial source preparation")); + } + + #[test] + fn editorial_lint_rejects_sibling_only_target_as_non_active_target() { + let workspace = editorial_workspace("Active proposal.", "Sibling proposal."); + + let error = run_lint( + &workspace, + &[ + "content/00002.md", + "--no-default-lints", + "-D", + "markdown-refs", + ], + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("unable to resolve editorial target")); + } + + #[test] + fn editorial_lint_reports_untracked_targets_missing_from_prepared_sources() { + let workspace = editorial_workspace("Active proposal.", "Sibling proposal."); + write_file( + &workspace.active_path, + "content/00003.md", + &proposal_markdown(3, None, "Untracked proposal."), + ); + + let error = run_lint( + &workspace, + &[ + "content/00003.md", + "--no-default-lints", + "-D", + "markdown-refs", + ], + ) + .unwrap_err() + .to_string(); + + assert!(error.contains( + "editorial target `content/00003.md` exists in the active repo but was not materialized into the prepared source tree" + )); + assert!(error.contains("untracked files are not supported")); + } + + #[test] + fn editorial_lint_materializes_tracked_dirty_working_tree_targets() { + let workspace = + editorial_workspace("Reference [ERC-9999](./09999.md).", "Sibling proposal."); + write_file( + &workspace.active_path, + "content/00001.md", + &proposal_markdown(1, None, "Reference [ERC-2](./00002.md)."), + ); + + assert!(run_lint( + &workspace, + &[ + "--working-tree", + "--no-default-lints", + "-D", + "markdown-refs" + ] + ) + .unwrap()); + } + + #[test] + fn editorial_working_tree_check_still_forces_dirty_runtime_materialization() { + let resolved = ResolvedExecution { + root_path: PathBuf::from("/workspace/Core"), + build_path: PathBuf::from("/workspace/build/Core"), + repository_use: crate::git::RepositoryUse { + title: "Core".to_owned(), + location: config::RepositoryEndpoint { + repository: "https://example.test/Core.git".parse().unwrap(), + base_url: "https://example.test/Core/".parse().unwrap(), + }, + other_repos: Default::default(), + }, + theme_path: Some(PathBuf::from("/workspace/theme")), + only: None, + source_materialization: crate::git::SourceMaterialization::Clean, + server_binding: ServerBinding::default(), + base_url_override: None, + }; + let selectors = EditorialSelectorArgs { + paths: Vec::new(), + batch: None, + working_tree: true, + against_upstream: false, + }; + + assert_eq!( + editorial_runtime_execution(resolved, &selectors).source_materialization, + crate::git::SourceMaterialization::Dirty + ); + } + + #[test] + fn editorial_explicit_numeric_selectors_resolve_to_markdown_paths() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/0004.md", ""); + let resolved = resolved_execution(temp.path().to_path_buf()); + + for selector in ["4", "004", "0004"] { + assert_eq!( + editorial_targets(&explicit_selectors(&[selector]), &resolved).unwrap(), + vec![PathBuf::from("content/0004.md")] + ); + } + } + + #[test] + fn editorial_explicit_numeric_selectors_support_multiple_and_dedupe() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/0004.md", ""); + write_file(temp.path(), "content/0005/index.md", ""); + let resolved = resolved_execution(temp.path().to_path_buf()); + + assert_eq!( + editorial_targets(&explicit_selectors(&["4", "0004", "005"]), &resolved).unwrap(), + vec![ + PathBuf::from("content/0004.md"), + PathBuf::from("content/0005/index.md"), + ] + ); + } + + #[test] + fn editorial_batch_accepts_numbers_paths_comments_and_empty_lines() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/0004.md", ""); + write_file(temp.path(), "content/0005/index.md", ""); + let batch_path = temp.path().join("targets.txt"); + write_file( + temp.path(), + "targets.txt", + "\n# comment\n \n4\ncontent/0005/index.md\n", + ); + let resolved = resolved_execution(temp.path().to_path_buf()); + let selectors = EditorialSelectorArgs { + paths: Vec::new(), + batch: Some(batch_path), + working_tree: false, + against_upstream: false, + }; + + assert_eq!( + editorial_targets(&selectors, &resolved).unwrap(), + vec![ + PathBuf::from("content/0004.md"), + PathBuf::from("content/0005/index.md"), + ] + ); + } + + #[test] + fn editorial_explicit_repo_relative_path_selectors_still_work() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/0004.md", ""); + let resolved = resolved_execution(temp.path().to_path_buf()); + + assert_eq!( + editorial_targets(&explicit_selectors(&["content/0004.md"]), &resolved).unwrap(), + vec![PathBuf::from("content/0004.md")] + ); + } + + #[test] + fn editorial_invalid_number_like_selectors_fail_with_editorial_error() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/0004.md", ""); + let resolved = resolved_execution(temp.path().to_path_buf()); + + for selector in [ + "0", + "+4", + "-4", + "4,5", + "4,,5", + ",4", + "4,", + "+", + "-", + "4294967296", + ] { + let error = editorial_targets(&explicit_selectors(&[selector]), &resolved) + .unwrap_err() + .to_string(); + assert!(error.contains(&format!( + "editorial number selector `{selector}` is invalid" + ))); + assert!(error.contains( + "expected a positive proposal number that fits in u32, without signs or commas" + )); + } + } + + #[test] + fn editorial_path_like_selectors_continue_through_path_validation() { + let temp = TempDir::new().unwrap(); + let resolved = resolved_execution(temp.path().to_path_buf()); + + for selector in ["foo", "draft.md", "4a", "draft-4.md"] { + write_file(temp.path(), selector, ""); + + let error = editorial_targets(&explicit_selectors(&[selector]), &resolved) + .unwrap_err() + .to_string(); + + assert!(error.contains("is not a supported proposal path")); + assert!(!error.contains("editorial number selector")); + } + } + + #[cfg(unix)] + #[test] + fn editorial_non_utf8_selector_continues_through_path_validation() { + use std::{ffi::OsStr, os::unix::ffi::OsStrExt}; + + let temp = TempDir::new().unwrap(); + let resolved = resolved_execution(temp.path().to_path_buf()); + let selectors = EditorialSelectorArgs { + paths: vec![PathBuf::from(OsStr::from_bytes(b"\xff"))], + batch: None, + working_tree: false, + against_upstream: false, + }; + + let error = editorial_targets(&selectors, &resolved) + .unwrap_err() + .to_string(); + + assert!(error.contains("unable to resolve editorial target")); + assert!(!error.contains("editorial number selector")); + } + + #[test] + fn non_strict_editorial_target_validation_does_not_normalize_numeric_paths() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "4", ""); + write_file(temp.path(), "content/0004.md", ""); + + assert_eq!( + validate_editorial_targets(temp.path(), vec![PathBuf::from("4")], false).unwrap(), + Vec::::new() + ); + } +} diff --git a/src/execution.rs b/src/execution.rs new file mode 100644 index 0000000..2b10743 --- /dev/null +++ b/src/execution.rs @@ -0,0 +1,1317 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Execution source and path resolution. + +use std::{ + collections::BTreeSet, + io::ErrorKind, + path::{Path, PathBuf}, +}; + +use log::{debug, info}; +use snafu::{OptionExt, ResultExt, Whatever}; +use url::Url; + +use crate::{ + cli::{Args, Operation, ServerCliArgs}, + config::{self, LoadedWorkspaceConfig, ServerBinding}, + context::{resolve_input_path, root}, + git, + identity::ActiveRepoIdentity, + layout::BUILD_DIR, + proposal::ProposalNumber, +}; + +#[derive(Debug, Clone)] +pub(crate) struct ResolvedExecution { + pub(crate) root_path: PathBuf, + pub(crate) build_path: PathBuf, + pub(crate) repository_use: git::RepositoryUse, + pub(crate) theme_path: Option, + pub(crate) only: Option>, + pub(crate) source_materialization: git::SourceMaterialization, + pub(crate) server_binding: ServerBinding, + pub(crate) base_url_override: Option, +} + +impl ResolvedExecution { + pub(crate) fn theme_path(&self) -> Result<&Path, Whatever> { + self.theme_path + .as_deref() + .whatever_context("the selected command requires a resolved workspace-local theme") + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum SelectedSource { + WorkspaceLocal, + Remote, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ExecutionSettings { + pub(crate) build_root: Option, + pub(crate) staging: bool, + pub(crate) allow_dirty: bool, + pub(crate) sibling: SelectedSource, +} + +fn has_execution_override_flags(args: &Args) -> bool { + args.staging || args.production || args.remote_siblings || args.build_root.is_some() +} + +pub(crate) fn validate_non_execution_command_flags(args: &Args) -> Result<(), Whatever> { + if args.operation.is_workspace_lifecycle_command() && has_execution_override_flags(args) { + snafu::whatever!("execution override flags cannot be used with `init` or `doctor`"); + } + + if args.operation.is_print_command() && has_execution_override_flags(args) { + snafu::whatever!("execution override flags cannot be used with `print`"); + } + + Ok(()) +} + +fn resolve_bool_override( + enabled: bool, + disabled: bool, + enabled_flag: &str, + disabled_flag: &str, +) -> Result, Whatever> { + match (enabled, disabled) { + (true, true) => { + snafu::whatever!("cannot pass both `{enabled_flag}` and `{disabled_flag}`") + } + (true, false) => Ok(Some(true)), + (false, true) => Ok(Some(false)), + (false, false) => Ok(None), + } +} + +fn remote_source_override(force_remote: bool) -> Option { + force_remote.then_some(SelectedSource::Remote) +} + +fn format_sibling_ids(sibling_ids: &[String]) -> String { + sibling_ids.join(", ") +} + +fn resolve_environment_override(args: &Args) -> Result, Whatever> { + resolve_bool_override(args.staging, args.production, "--staging", "--production") +} + +fn explicit_environment_or_parity(args: &Args) -> Result, Whatever> { + if let Some(staging) = resolve_environment_override(args)? { + return Ok(Some(staging)); + } + + if matches!(args.operation, Operation::Parity { .. }) { + return Ok(Some(true)); + } + + Ok(None) +} + +fn cli_only_requested(args: &Args) -> bool { + args.operation + .only_cli_args() + .map(|only| !only.only.is_empty()) + .unwrap_or(false) +} + +fn only_cli_is_applicable(args: &Args, explicit_environment: Option) -> bool { + matches!( + args.operation, + Operation::Build { .. } | Operation::Serve { .. } + ) && explicit_environment.is_none() + && !args.operation.clean_cli_args().clean + && !args.remote_siblings +} + +pub(crate) fn resolve_execution_settings( + args: &Args, + sibling_ids: &[String], + workspace_config: Option<&LoadedWorkspaceConfig>, +) -> Result { + let build_root = args + .build_root + .as_deref() + .map(resolve_input_path) + .transpose()?; + let explicit_environment = explicit_environment_or_parity(args)?; + let sibling_override = remote_source_override(args.remote_siblings); + let clean = args.operation.clean_cli_args().clean; + + if cli_only_requested(args) && !only_cli_is_applicable(args, explicit_environment) { + snafu::whatever!("--only is supported only for local dirty build and serve commands"); + } + + let (staging, allow_dirty, default_sibling) = if let Some(staging) = explicit_environment { + (staging, false, SelectedSource::Remote) + } else if args.operation.is_plain_site_command() || args.operation.is_editorial_command() { + (true, !clean, SelectedSource::WorkspaceLocal) + } else { + (false, false, SelectedSource::Remote) + }; + + let missing_theme = operation_requires_theme(&args.operation) && workspace_config.is_none(); + let missing_sibling = sibling_override.is_none() + && default_sibling == SelectedSource::WorkspaceLocal + && !sibling_ids.is_empty() + && workspace_config.is_none(); + + match (missing_theme, missing_sibling) { + (true, true) => { + snafu::whatever!( + "the selected command requires a workspace config with local theme and sibling sources, but no `{}` was found.\n\nRun:\n build-eips init \n\nThen retry from that workspace, or pass `--remote-siblings` if you intentionally want remote sibling proposal sources.", + config::LOCAL_CONFIG_FILE + ); + } + (false, true) => { + snafu::whatever!( + "the selected command requires workspace-local sibling sources, but no `{}` was found to provide them.\nResolve this by doing one of the following:\n1. run `build-eips init ` so the workspace config supplies the local sources\n2. pass `--remote-siblings` for remote sibling source overrides\n3. use `parity `, `--staging `, or `--production ` for remote clean environment behavior", + config::LOCAL_CONFIG_FILE + ); + } + _ => {} + } + + let sibling = sibling_override.unwrap_or(default_sibling); + + Ok(ExecutionSettings { + build_root, + staging, + allow_dirty, + sibling, + }) +} + +fn dedupe_only_numbers(numbers: &[ProposalNumber]) -> Option> { + let numbers = numbers.iter().copied().collect::>(); + (!numbers.is_empty()).then_some(numbers) +} + +fn resolve_only_selection( + args: &Args, + settings: &ExecutionSettings, + workspace_config: Option<&LoadedWorkspaceConfig>, +) -> Result>, Whatever> { + let explicit_environment = explicit_environment_or_parity(args)?; + let applicable = matches!( + args.operation, + Operation::Build { .. } | Operation::Serve { .. } + ) && explicit_environment.is_none() + && settings.allow_dirty + && settings.sibling == SelectedSource::WorkspaceLocal; + + if let Some(only) = args.operation.only_cli_args() { + if let Some(numbers) = dedupe_only_numbers(&only.only) { + if !applicable { + snafu::whatever!( + "--only is supported only for local dirty build and serve commands" + ); + } + return Ok(Some(numbers)); + } + } + + if !applicable { + return Ok(None); + } + + Ok(workspace_config + .and_then(|workspace_config| dedupe_only_numbers(&workspace_config.render_settings().only))) +} + +fn local_repo_url(path: &Path) -> Result { + Url::from_directory_path(path) + .ok() + .whatever_context("unable to convert local sibling repository path into a file URL") +} + +fn apply_sibling_sources( + repository_use: &mut git::RepositoryUse, + sibling_ids: &[String], + workspace_config: Option<&LoadedWorkspaceConfig>, + sibling: &SelectedSource, +) -> Result<(), Whatever> { + match sibling { + SelectedSource::Remote => Ok(()), + SelectedSource::WorkspaceLocal => { + if sibling_ids.is_empty() { + return Ok(()); + } + + let workspace_config = workspace_config.whatever_context( + "workspace-local sibling selection requires a workspace config", + )?; + let mut missing = Vec::new(); + let mut local_repositories = Vec::new(); + + for repo_id in sibling_ids { + let path = workspace_config.local_repo_path(repo_id); + if git::repository_available(&path) { + local_repositories.push((repo_id.clone(), local_repo_url(&path)?)); + } else { + missing.push(repo_id.clone()); + } + } + + if !missing.is_empty() { + snafu::whatever!( + "workspace-local sibling selection requires all declared sibling repos; missing or invalid sibling repo(s): {}", + format_sibling_ids(&missing) + ); + } + + for (repo_id, url) in local_repositories { + repository_use.other_repos.insert(repo_id, url); + } + + Ok(()) + } + } +} + +fn build_path( + root_path: &Path, + repository_use: &git::RepositoryUse, + workspace_config: Option<&LoadedWorkspaceConfig>, + build_root: Option<&Path>, +) -> PathBuf { + build_root + .map(Path::to_path_buf) + .or_else(|| { + workspace_config.map(|workspace_config| { + workspace_config.workspace_build_root(&repository_use.title) + }) + }) + .unwrap_or_else(|| root_path.join(BUILD_DIR)) +} + +fn operation_requires_theme(operation: &Operation) -> bool { + matches!( + operation, + Operation::Build { .. } + | Operation::Serve { .. } + | Operation::Check { .. } + | Operation::Editorial { .. } + | Operation::Parity { .. } + ) +} + +fn resolve_theme_path( + workspace_config: Option<&LoadedWorkspaceConfig>, + operation: &Operation, +) -> Result, Whatever> { + if !operation_requires_theme(operation) { + return Ok(None); + } + + let workspace_config = workspace_config.with_whatever_context(|| { + format!( + "the selected command requires a workspace config with a local theme, but no `{}` was found.\n\nRun:\n build-eips init \n\nThen retry from that workspace.", + config::LOCAL_CONFIG_FILE + ) + })?; + let theme_path = workspace_config.local_theme_path(); + + match std::fs::metadata(&theme_path) { + Ok(_) => Ok(Some(theme_path)), + Err(error) if matches!(error.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => { + snafu::whatever!( + "workspace-local theme path `{}` does not exist.\n\nRun `build-eips init ` to bootstrap the workspace, or\nclone/update the theme repository at the configured path.", + theme_path.to_string_lossy() + ); + } + Err(error) => { + snafu::whatever!( + "unable to access workspace-local theme path `{}`: {error}", + theme_path.to_string_lossy() + ); + } + } +} + +fn resolve_server_binding( + workspace_config: Option<&LoadedWorkspaceConfig>, + server_cli: &ServerCliArgs, +) -> ServerBinding { + let mut binding = workspace_config + .map(|workspace_config| ServerBinding::from(workspace_config.server_settings())) + .unwrap_or_default(); + + if let Some(host) = &server_cli.host { + binding.host = host.clone(); + } + + if let Some(port) = server_cli.port { + binding.port = port; + } + + binding +} + +fn resolve_base_url_override( + args: &Args, + workspace_config: Option<&LoadedWorkspaceConfig>, +) -> Result, Whatever> { + if let Some(base_url) = args.operation.base_url_cli_args().base_url { + return Ok(Some(base_url)); + } + + if explicit_environment_or_parity(args)?.is_some() { + return Ok(None); + } + + Ok(workspace_config.and_then(|config| config.site_settings().base_url.clone())) +} + +pub(crate) fn resolve_execution(args: &Args) -> Result { + let root_path = root(args)?; + let active_repo = ActiveRepoIdentity::load(&root_path)?; + let sibling_ids = active_repo.sibling_ids(); + let workspace_config = LoadedWorkspaceConfig::discover(&root_path) + .whatever_context("unable to load workspace config")?; + + if let Some(workspace_config) = workspace_config.as_ref() { + debug!( + "using workspace config `{}`", + workspace_config.config_path().to_string_lossy() + ); + } + + let settings = resolve_execution_settings(args, &sibling_ids, workspace_config.as_ref())?; + let only = resolve_only_selection(args, &settings, workspace_config.as_ref())?; + let theme_path = resolve_theme_path(workspace_config.as_ref(), &args.operation)?; + + let mut repository_use = active_repo.repository_use(settings.staging)?; + apply_sibling_sources( + &mut repository_use, + &sibling_ids, + workspace_config.as_ref(), + &settings.sibling, + )?; + + let build_path = build_path( + &root_path, + &repository_use, + workspace_config.as_ref(), + settings.build_root.as_deref(), + ); + let source_materialization = if settings.allow_dirty { + info!( + "dirty mode is enabled; tracked working-tree changes from the active content repo will be materialized into the build input" + ); + git::SourceMaterialization::Dirty + } else { + git::SourceMaterialization::Clean + }; + let base_url_override = resolve_base_url_override(args, workspace_config.as_ref())?; + + Ok(ResolvedExecution { + root_path, + build_path, + repository_use, + theme_path, + only, + source_materialization, + server_binding: resolve_server_binding( + workspace_config.as_ref(), + &args.operation.server_cli_args(), + ), + base_url_override, + }) +} + +#[cfg(test)] +mod tests { + use clap::Parser; + use tempfile::TempDir; + + use crate::{ + cli::{Args, ServerCliArgs}, + config::{self, LoadedWorkspaceConfig, ServerBinding}, + }; + + use super::{ + explicit_environment_or_parity, resolve_base_url_override, resolve_execution_settings, + resolve_only_selection, resolve_server_binding, resolve_theme_path, + validate_non_execution_command_flags, ExecutionSettings, SelectedSource, + }; + + fn parse_args(arguments: &[&str]) -> Args { + Args::try_parse_from(arguments).unwrap() + } + + fn load_workspace_config(contents: &str) -> LoadedWorkspaceConfig { + let workspace = TempDir::new().unwrap(); + let config_path = workspace.path().join(config::LOCAL_CONFIG_FILE); + std::fs::write(&config_path, contents).unwrap(); + LoadedWorkspaceConfig::from_path(&config_path).unwrap() + } + + fn settings_for( + arguments: &[&str], + sibling_ids: &[&str], + workspace_config: Option<&LoadedWorkspaceConfig>, + ) -> ExecutionSettings { + let args = parse_args(arguments); + let sibling_ids = sibling_ids + .iter() + .map(|sibling_id| (*sibling_id).to_owned()) + .collect::>(); + + resolve_execution_settings(&args, &sibling_ids, workspace_config).unwrap() + } + + fn assert_settings( + arguments: &[&str], + sibling_ids: &[&str], + workspace_config: Option<&LoadedWorkspaceConfig>, + expected: ExecutionSettings, + ) { + assert_eq!( + settings_for(arguments, sibling_ids, workspace_config), + expected + ); + } + + fn assert_theme_only_missing_workspace_error(arguments: &[&str]) { + let args = parse_args(arguments); + let error = resolve_theme_path(None, &args.operation).unwrap_err(); + let message = error.to_string(); + + assert!(message.contains( + "the selected command requires a workspace config with a local theme, but no `.build-eips.toml` was found" + )); + assert!(message.contains("build-eips init ")); + assert!(!message.contains("theme and sibling")); + assert!(!message.contains(concat!("--remote", "-theme"))); + } + + fn assert_combined_missing_workspace_error(arguments: &[&str]) { + let args = parse_args(arguments); + let sibling_ids = vec!["ERCs".to_owned()]; + let error = resolve_execution_settings(&args, &sibling_ids, None).unwrap_err(); + let message = error.to_string(); + + assert!(message.contains( + "the selected command requires a workspace config with local theme and sibling sources" + )); + assert!(message.contains("no `.build-eips.toml` was found")); + assert!(message.contains("build-eips init ")); + assert!(message.contains( + "pass `--remote-siblings` if you intentionally want remote sibling proposal sources" + )); + assert!(!message.contains(concat!("--remote", "-theme"))); + assert!(!message.contains("--profile")); + assert!(!message.contains("--allow-dirty")); + assert!(!message.contains("--theme ")); + assert!(!message.contains("--sibling-repo ")); + } + + fn only_selection_for( + arguments: &[&str], + workspace_config: Option<&LoadedWorkspaceConfig>, + ) -> Option> { + let args = parse_args(arguments); + let settings = resolve_execution_settings(&args, &[], workspace_config).unwrap(); + resolve_only_selection(&args, &settings, workspace_config) + .unwrap() + .map(|numbers| numbers.into_iter().map(|number| number.get()).collect()) + } + + #[test] + fn server_binding_resolution_uses_cli_config_then_defaults() { + assert_eq!( + resolve_server_binding(None, &ServerCliArgs::default()), + ServerBinding { + host: "127.0.0.1".to_owned(), + port: 1111, + } + ); + + let workspace_config = load_workspace_config( + r#" +[server] +host = "0.0.0.0" +port = 8080 +"#, + ); + + assert_eq!( + resolve_server_binding(Some(&workspace_config), &ServerCliArgs::default()), + ServerBinding { + host: "0.0.0.0".to_owned(), + port: 8080, + } + ); + assert_eq!( + resolve_server_binding( + Some(&workspace_config), + &ServerCliArgs { + host: Some("127.0.0.1".to_owned()), + port: Some(4000), + }, + ), + ServerBinding { + host: "127.0.0.1".to_owned(), + port: 4000, + } + ); + assert_eq!( + resolve_server_binding( + Some(&workspace_config), + &ServerCliArgs { + host: None, + port: Some(4000), + }, + ), + ServerBinding { + host: "0.0.0.0".to_owned(), + port: 4000, + } + ); + } + + #[test] + fn explicit_env_or_parity_provenance_is_classified_separately_from_local_defaults() { + let cases: &[(&[&str], Option)] = &[ + (&["build-eips", "--staging", "build"], Some(true)), + (&["build-eips", "--production", "build"], Some(false)), + (&["build-eips", "parity", "build"], Some(true)), + (&["build-eips", "build"], None), + (&["build-eips", "serve"], None), + (&["build-eips", "check"], None), + ]; + + for (arguments, expected) in cases { + let args = parse_args(arguments); + assert_eq!(explicit_environment_or_parity(&args).unwrap(), *expected); + } + } + + #[test] + fn base_url_override_resolution_uses_cli_config_then_provenance() { + let workspace_config = load_workspace_config( + r#" +[site] +base_url = "http://localhost:4000" +"#, + ); + let none = parse_args(&["build-eips", "build"]); + assert!(resolve_base_url_override(&none, None).unwrap().is_none()); + + for arguments in [&["build-eips", "build"][..], &["build-eips", "serve"][..]] { + let args = parse_args(arguments); + assert_eq!( + resolve_base_url_override(&args, Some(&workspace_config)) + .unwrap() + .unwrap() + .as_str(), + "http://localhost:4000/" + ); + } + + let cli = parse_args(&["build-eips", "build", "--base-url", "http://localhost:5000"]); + assert_eq!( + resolve_base_url_override(&cli, Some(&workspace_config)) + .unwrap() + .unwrap() + .as_str(), + "http://localhost:5000/" + ); + + for arguments in [ + &["build-eips", "--staging", "build"][..], + &["build-eips", "--production", "build"][..], + &["build-eips", "parity", "build"][..], + &["build-eips", "parity", "serve"][..], + ] { + let args = parse_args(arguments); + assert!(resolve_base_url_override(&args, Some(&workspace_config)) + .unwrap() + .is_none()); + } + + for arguments in [ + &[ + "build-eips", + "--staging", + "build", + "--base-url", + "http://localhost:5000", + ][..], + &[ + "build-eips", + "--production", + "build", + "--base-url", + "http://localhost:5000", + ][..], + &[ + "build-eips", + "parity", + "build", + "--base-url", + "http://localhost:5000", + ][..], + &[ + "build-eips", + "parity", + "serve", + "--base-url", + "http://localhost:5000", + ][..], + ] { + let args = parse_args(arguments); + assert_eq!( + resolve_base_url_override(&args, Some(&workspace_config)) + .unwrap() + .unwrap() + .as_str(), + "http://localhost:5000/" + ); + } + } + + #[test] + fn local_site_base_url_override_does_not_change_execution_settings() { + let workspace_config = load_workspace_config( + r#" +[site] +base_url = "http://localhost:4000" +"#, + ); + let args = parse_args(&["build-eips", "build"]); + let settings = resolve_execution_settings(&args, &[], Some(&workspace_config)).unwrap(); + + assert_eq!( + resolve_base_url_override(&args, Some(&workspace_config)) + .unwrap() + .unwrap() + .as_str(), + "http://localhost:4000/" + ); + assert_eq!( + settings, + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: true, + sibling: SelectedSource::WorkspaceLocal, + } + ); + } + + #[test] + fn plain_site_commands_are_local_first_dirty_staging() { + let workspace_config = load_workspace_config(""); + let expected = ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: true, + sibling: SelectedSource::WorkspaceLocal, + }; + + for arguments in [ + &["build-eips", "build"][..], + &["build-eips", "serve"][..], + &["build-eips", "check"][..], + ] { + assert_settings( + arguments, + &["ERCs"], + Some(&workspace_config), + expected.clone(), + ); + } + } + + #[test] + fn zola_runtime_commands_require_workspace_local_theme() { + let workspace = TempDir::new().unwrap(); + let config_path = workspace.path().join(config::LOCAL_CONFIG_FILE); + std::fs::write(&config_path, "").unwrap(); + std::fs::create_dir(workspace.path().join(config::DEFAULT_THEME_DIR)).unwrap(); + let workspace_config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + for arguments in [ + &["build-eips", "build"][..], + &["build-eips", "check"][..], + &["build-eips", "serve"][..], + &["build-eips", "--staging", "build"][..], + &["build-eips", "--production", "check"][..], + &["build-eips", "parity", "build"][..], + &["build-eips", "editorial", "check", "--against-upstream"][..], + ] { + let args = parse_args(arguments); + let theme_path = resolve_theme_path(Some(&workspace_config), &args.operation) + .unwrap() + .unwrap(); + + assert_eq!(theme_path, workspace.path().join(config::DEFAULT_THEME_DIR)); + } + } + + #[test] + fn non_theme_commands_do_not_require_workspace_local_theme() { + for arguments in [ + &["build-eips", "changed"][..], + &["build-eips", "clean"][..], + &["build-eips", "preview"][..], + &["build-eips", "doctor"][..], + &["build-eips", "print", "schema-version"][..], + ] { + let args = parse_args(arguments); + + assert!(resolve_theme_path(None, &args.operation).unwrap().is_none()); + } + } + + #[test] + fn clean_plain_site_commands_keep_local_sources_but_disable_dirty_materialization() { + let workspace_config = load_workspace_config(""); + let expected = ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: false, + sibling: SelectedSource::WorkspaceLocal, + }; + + for arguments in [ + &["build-eips", "build", "--clean"][..], + &["build-eips", "serve", "--clean"][..], + &["build-eips", "check", "--clean"][..], + ] { + assert_settings( + arguments, + &["ERCs"], + Some(&workspace_config), + expected.clone(), + ); + } + } + + #[test] + fn explicit_environment_site_commands_are_remote_clean_for_proposals() { + for (arguments, expected_staging) in [ + (&["build-eips", "--staging", "build"][..], true), + (&["build-eips", "--staging", "serve"][..], true), + (&["build-eips", "--staging", "check"][..], true), + (&["build-eips", "--production", "build"][..], false), + (&["build-eips", "--production", "serve"][..], false), + (&["build-eips", "--production", "check"][..], false), + ] { + assert_settings( + arguments, + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: expected_staging, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + } + } + + #[test] + fn clean_environment_commands_are_accepted_as_redundant_remote_clean() { + for (arguments, expected_staging) in [ + (&["build-eips", "--staging", "build", "--clean"][..], true), + (&["build-eips", "--staging", "serve", "--clean"][..], true), + (&["build-eips", "--staging", "check", "--clean"][..], true), + ( + &["build-eips", "--production", "build", "--clean"][..], + false, + ), + ( + &["build-eips", "--production", "serve", "--clean"][..], + false, + ), + ( + &["build-eips", "--production", "check", "--clean"][..], + false, + ), + ] { + assert_settings( + arguments, + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: expected_staging, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + } + } + + #[test] + fn remote_source_overrides_compose_with_local_dirty_and_clean_modes() { + let workspace_config = load_workspace_config(""); + let cases = [ + ( + &["build-eips", "--remote-siblings", "build"][..], + true, + SelectedSource::Remote, + ), + ( + &["build-eips", "--remote-siblings", "build", "--clean"][..], + false, + SelectedSource::Remote, + ), + ( + &[ + "build-eips", + "--remote-siblings", + "editorial", + "lint", + "content/0001.md", + ][..], + true, + SelectedSource::Remote, + ), + ]; + + for (arguments, allow_dirty, sibling) in cases { + assert_settings( + arguments, + &["ERCs"], + Some(&workspace_config), + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty, + sibling, + }, + ); + } + } + + #[test] + fn only_selection_dedupes_and_cli_replaces_config() { + let workspace_config = load_workspace_config( + r#" +[render] +only = [678, 555, 678] +"#, + ); + + assert_eq!( + only_selection_for(&["build-eips", "build"], Some(&workspace_config)).unwrap(), + vec![555, 678] + ); + assert_eq!( + only_selection_for( + &["build-eips", "build", "--only", "00555", "555", "897"], + Some(&workspace_config) + ) + .unwrap(), + vec![555, 897] + ); + assert_eq!( + only_selection_for(&["build-eips", "serve"], Some(&workspace_config)).unwrap(), + vec![555, 678] + ); + assert_eq!( + only_selection_for( + &["build-eips", "serve", "--only", "00555", "555", "897"], + Some(&workspace_config) + ) + .unwrap(), + vec![555, 897] + ); + } + + #[test] + fn only_cli_rejects_parsed_non_applicable_build_and_serve_modes() { + for arguments in [ + &["build-eips", "--staging", "build", "--only", "555"][..], + &["build-eips", "--production", "build", "--only", "555"][..], + &["build-eips", "build", "--clean", "--only", "555"][..], + &["build-eips", "--remote-siblings", "build", "--only", "555"][..], + &["build-eips", "--staging", "serve", "--only", "555"][..], + &["build-eips", "--production", "serve", "--only", "555"][..], + &["build-eips", "serve", "--clean", "--only", "555"][..], + &["build-eips", "--remote-siblings", "serve", "--only", "555"][..], + ] { + let args = parse_args(arguments); + let error = resolve_execution_settings(&args, &[], None).unwrap_err(); + + assert!(error + .to_string() + .contains("--only is supported only for local dirty build and serve commands")); + } + } + + #[test] + fn render_only_config_is_ignored_outside_applicable_build_commands() { + let workspace_config = load_workspace_config( + r#" +[render] +only = [999999] +"#, + ); + + for arguments in [ + &["build-eips", "check"][..], + &["build-eips", "build", "--clean"][..], + &["build-eips", "serve", "--clean"][..], + &["build-eips", "--staging", "build"][..], + &["build-eips", "--staging", "serve"][..], + &["build-eips", "parity", "build"][..], + &["build-eips", "parity", "serve"][..], + ] { + assert!(only_selection_for(arguments, Some(&workspace_config)).is_none()); + } + } + + #[test] + fn missing_render_config_and_empty_only_disable_filtering() { + let missing_render = load_workspace_config(""); + let missing_only = load_workspace_config("[render]\n"); + let empty_only = load_workspace_config( + r#" +[render] +only = [] +"#, + ); + + assert!(only_selection_for(&["build-eips", "build"], Some(&missing_render)).is_none()); + assert!(only_selection_for(&["build-eips", "build"], Some(&missing_only)).is_none()); + assert!(only_selection_for(&["build-eips", "build"], Some(&empty_only)).is_none()); + assert!(only_selection_for(&["build-eips", "serve"], Some(&missing_render)).is_none()); + assert!(only_selection_for(&["build-eips", "serve"], Some(&missing_only)).is_none()); + assert!(only_selection_for(&["build-eips", "serve"], Some(&empty_only)).is_none()); + } + + #[test] + fn non_site_commands_do_not_require_workspace_local_sources() { + for arguments in [ + &["build-eips", "changed"][..], + &["build-eips", "clean"][..], + &["build-eips", "preview"][..], + ] { + assert_settings( + arguments, + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: false, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + } + } + + #[test] + fn changed_environment_flags_use_remote_clean_metadata_without_workspace_config() { + for (arguments, expected_staging) in [ + (&["build-eips", "--staging", "changed"][..], true), + (&["build-eips", "--production", "changed"][..], false), + ] { + assert_settings( + arguments, + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: expected_staging, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + } + } + + #[test] + fn parity_site_commands_remain_remote_clean_staging() { + for arguments in [ + &["build-eips", "parity", "build"][..], + &["build-eips", "parity", "serve"][..], + &["build-eips", "parity", "check"][..], + ] { + assert_settings( + arguments, + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + } + } + + #[test] + fn boolean_override_conflicts_are_hard_errors() { + let args = parse_args(&["build-eips", "--staging", "--production", "build"]); + let error = resolve_execution_settings(&args, &[], None).unwrap_err(); + + assert!(error + .to_string() + .contains("cannot pass both `--staging` and `--production`")); + } + + #[test] + fn non_execution_commands_reject_execution_override_flags() { + for arguments in [ + &["build-eips", "--staging", "init", "/tmp/workspace"][..], + &["build-eips", "--production", "init", "/tmp/workspace"][..], + &["build-eips", "--remote-siblings", "init", "/tmp/workspace"][..], + &[ + "build-eips", + "--build-root", + "/tmp/build", + "init", + "/tmp/workspace", + ][..], + &["build-eips", "--staging", "doctor"][..], + &["build-eips", "--production", "doctor"][..], + &["build-eips", "--remote-siblings", "doctor"][..], + &["build-eips", "--build-root", "/tmp/build", "doctor"][..], + &["build-eips", "--remote-siblings", "print", "schema-version"][..], + ] { + let args = parse_args(arguments); + let error = validate_non_execution_command_flags(&args).unwrap_err(); + let message = error.to_string(); + + assert!( + message.contains("execution override flags cannot be used"), + "unexpected error for {arguments:?}: {message}" + ); + } + } + + #[test] + fn editorial_dispatch_uses_local_first_for_all_editorial_commands() { + let workspace_config = load_workspace_config(""); + + assert_settings( + &["build-eips", "editorial", "lint", "content/0001.md"], + &["ERCs"], + Some(&workspace_config), + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: true, + sibling: SelectedSource::WorkspaceLocal, + }, + ); + assert_settings( + &["build-eips", "editorial", "check", "content/0001.md"], + &["ERCs"], + Some(&workspace_config), + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: true, + sibling: SelectedSource::WorkspaceLocal, + }, + ); + assert_settings( + &[ + "build-eips", + "--remote-siblings", + "editorial", + "lint", + "content/0001.md", + ], + &["ERCs"], + Some(&workspace_config), + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: true, + sibling: SelectedSource::Remote, + }, + ); + assert_settings( + &[ + "build-eips", + "--staging", + "editorial", + "lint", + "content/0001.md", + ], + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + assert_settings( + &[ + "build-eips", + "--production", + "editorial", + "lint", + "content/0001.md", + ], + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: false, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + assert_settings( + &[ + "build-eips", + "--staging", + "editorial", + "check", + "--against-upstream", + ], + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + assert_settings( + &[ + "build-eips", + "--production", + "editorial", + "check", + "--against-upstream", + ], + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: false, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + } + + #[test] + fn local_first_theme_commands_without_workspace_config_report_combined_setup_error() { + for arguments in [ + &["build-eips", "build"][..], + &["build-eips", "serve"][..], + &["build-eips", "check"][..], + &["build-eips", "editorial", "lint", "content/0001.md"][..], + &["build-eips", "editorial", "check", "content/0001.md"][..], + ] { + assert_combined_missing_workspace_error(arguments); + } + } + + #[test] + fn local_first_with_remote_sibling_override_is_not_parity() { + let local_args = parse_args(&["build-eips", "--remote-siblings", "build"]); + let sibling_ids = vec!["ERCs".to_owned()]; + let local_settings = resolve_execution_settings(&local_args, &sibling_ids, None).unwrap(); + + let parity_args = parse_args(&["build-eips", "parity", "build"]); + let parity_settings = resolve_execution_settings(&parity_args, &sibling_ids, None).unwrap(); + + assert_eq!( + local_settings, + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: true, + sibling: SelectedSource::Remote, + } + ); + assert_eq!( + parity_settings, + ExecutionSettings { + build_root: None, + staging: true, + allow_dirty: false, + sibling: SelectedSource::Remote, + } + ); + } + + #[test] + fn zero_sibling_remote_override_is_noop() { + let remote_args = parse_args(&["build-eips", "--remote-siblings", "parity", "build"]); + let remote_settings = resolve_execution_settings(&remote_args, &[], None).unwrap(); + + assert_eq!(remote_settings.sibling, SelectedSource::Remote); + } + + #[test] + fn zero_sibling_local_first_without_workspace_config_only_requires_theme_resolution() { + let args = parse_args(&["build-eips", "build"]); + let settings = resolve_execution_settings(&args, &[], None).unwrap(); + + assert_eq!(settings.sibling, SelectedSource::WorkspaceLocal); + assert_theme_only_missing_workspace_error(&["build-eips", "build"]); + } + + #[test] + fn remote_sibling_override_without_workspace_config_only_requires_theme_resolution() { + let args = parse_args(&["build-eips", "--remote-siblings", "build"]); + let sibling_ids = vec!["ERCs".to_owned()]; + let settings = resolve_execution_settings(&args, &sibling_ids, None).unwrap(); + + assert_eq!(settings.sibling, SelectedSource::Remote); + assert_theme_only_missing_workspace_error(&["build-eips", "--remote-siblings", "build"]); + } + + #[test] + fn environment_and_parity_zola_commands_without_workspace_config_only_require_theme() { + for arguments in [ + &["build-eips", "--staging", "build"][..], + &["build-eips", "--production", "serve"][..], + &["build-eips", "parity", "check"][..], + ] { + let args = parse_args(arguments); + let sibling_ids = vec!["ERCs".to_owned()]; + let settings = resolve_execution_settings(&args, &sibling_ids, None).unwrap(); + + assert_eq!(settings.sibling, SelectedSource::Remote); + assert_theme_only_missing_workspace_error(arguments); + } + } + + #[test] + fn missing_workspace_theme_path_reports_clear_error() { + let workspace = TempDir::new().unwrap(); + let config_path = workspace.path().join(config::LOCAL_CONFIG_FILE); + std::fs::write(&config_path, "").unwrap(); + let workspace_config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + let args = parse_args(&["build-eips", "build"]); + + let error = resolve_theme_path(Some(&workspace_config), &args.operation).unwrap_err(); + let message = error.to_string(); + + assert!(message.contains(&format!( + "workspace-local theme path `{}` does not exist", + workspace + .path() + .join(config::DEFAULT_THEME_DIR) + .to_string_lossy() + ))); + assert!(message.contains("build-eips init ")); + } +} diff --git a/src/find_root.rs b/src/find_root.rs index c2db1e0..40cde8c 100644 --- a/src/find_root.rs +++ b/src/find_root.rs @@ -4,7 +4,7 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -use super::CONTENT_DIR; +use crate::layout::CONTENT_DIR; use snafu::{ResultExt, Snafu}; use std::{ backtrace::Backtrace, @@ -34,7 +34,7 @@ pub enum Error { pub fn is_root(path: &Path) -> Result<(), Error> { let git = path.join(".git"); let contents = path.join(CONTENT_DIR); - if git.is_dir() && contents.is_dir() { + if (git.is_dir() || git.is_file()) && contents.is_dir() { Ok(()) } else { NoRootSnafu.fail() diff --git a/src/git.rs b/src/git.rs index 357743f..04d1956 100644 --- a/src/git.rs +++ b/src/git.rs @@ -5,25 +5,28 @@ */ use std::{ - collections::HashMap, + collections::{BTreeMap, BTreeSet}, ffi::OsStr, path::{absolute, Path, PathBuf}, }; use crate::{ - cache::Cache, - config::{Location, Locations}, + config::{LegacyLocations, RepositoryEndpoint}, + layout::{BUILD_DIR, CONTENT_DIR}, progress::{Git, ProgressIteratorExt}, }; use git2::{ build::{CheckoutBuilder, TreeUpdateBuilder}, - BranchType, Commit, FetchOptions, FileMode, ObjectType, Oid, RepositoryOpenFlags, Signature, + Commit, FetchOptions, FileMode, ObjectType, Oid, RepositoryOpenFlags, Signature, Status, StatusOptions, Tree, TreeEntry, TreeWalkResult, }; use log::{debug, info}; use snafu::{ensure, Backtrace, IntoError, OptionExt, ResultExt, Snafu}; use url::Url; +const DIRTY_PATH_DISPLAY_LIMIT: usize = 10; +const CONTENT_INDEX_PATH: &str = "content/_index.md"; + #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("cannot convert path into URL (`{}`)", path.to_string_lossy()))] @@ -47,26 +50,44 @@ pub enum Error { }, #[snafu(display("unable to determine which repository is being built (none match)"))] NoIdentify { backtrace: Backtrace }, - #[snafu(display("working tree or index has uncommitted modifications"))] - Dirty { backtrace: Backtrace }, + #[snafu(display("{message}"))] + Dirty { + message: String, + backtrace: Backtrace, + }, + #[snafu(display( + "dirty mode cannot materialize conflicted path `{}`; resolve the conflict and try again", + path.to_string_lossy() + ))] + DirtyConflict { path: PathBuf, backtrace: Backtrace }, + #[snafu(display( + "dirty mode cannot materialize `{}` because it is not a tracked file or symlink in the working tree", + path.to_string_lossy() + ))] + DirtyUnsupportedPath { path: PathBuf, backtrace: Backtrace }, #[snafu(display("unable to update tree ({msg})"))] UpdateTree { msg: String, backtrace: Backtrace }, - #[snafu(context(false))] - Cache { - #[snafu(backtrace)] - source: crate::cache::Error, - }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceMaterialization { + Clean, + Dirty, } #[derive(Debug, Clone)] pub struct RepositoryUse { pub title: String, - pub location: Location, - pub other_repos: HashMap, + pub location: RepositoryEndpoint, + pub other_repos: BTreeMap, } -impl Locations { - pub fn identify_repository(&self, path: &Path) -> Result { +pub fn repository_available(path: &Path) -> bool { + git2::Repository::open(path).is_ok() +} + +impl LegacyLocations { + pub fn identify_repository_title(&self, path: &Path) -> Result { let repo = git2::Repository::open_ext(path, RepositoryOpenFlags::NO_SEARCH, &[] as &[&OsStr]) .context(GitSnafu { @@ -94,9 +115,13 @@ impl Locations { ); ensure!(containing_locations.len() == 1, NoIdentifySnafu); - let (title, location) = containing_locations[0]; + let (title, _) = containing_locations[0]; - // TODO: this is a bit weird, and is a leftover from the previous architecture. + Ok(title.clone()) + } + + pub fn repository_use_for_title(&self, title: &str) -> Option { + let location = self.0.get(title)?; let other_repos = self .0 .iter() @@ -109,35 +134,555 @@ impl Locations { }) .collect(); - Ok(RepositoryUse { - title: title.clone(), - location: location.clone(), + Some(RepositoryUse { + title: title.to_owned(), + location: location.endpoint(), other_repos, }) } } +pub fn clone_missing_repo(url: &str, destination: &Path) -> Result<(), Error> { + match git2::Repository::open(destination) { + Ok(_) => { + info!( + "using existing workspace repo `{}`", + destination.to_string_lossy() + ); + return Ok(()); + } + Err(error) if error.code() == git2::ErrorCode::NotFound => {} + Err(error) => { + return Err(GitSnafu { + what: "open existing workspace repository", + } + .into_error(error)); + } + } + + info!("cloning `{url}` into `{}`", destination.to_string_lossy()); + git2::Repository::clone(url, destination).context(GitSnafu { + what: "clone workspace repository", + })?; + Ok(()) +} + +fn is_generated_path(path: &Path) -> bool { + path.components() + .next() + .map(|component| component.as_os_str() == OsStr::new(BUILD_DIR)) + .unwrap_or(false) +} + +fn dirty_statuses(repo: &git2::Repository) -> Result, Error> { + let mut options = StatusOptions::default(); + options + .include_untracked(true) + .recurse_untracked_dirs(true) + .renames_head_to_index(true) + .renames_index_to_workdir(true); + repo.statuses(Some(&mut options)).context(GitSnafu { + what: "get root repository status", + }) +} + +fn format_dirty_rejection(tracked_paths: &BTreeSet, untracked_count: usize) -> String { + let mut lines = vec![String::from( + "working tree or index has uncommitted modifications; the selected clean source path requires a clean working tree:", + )]; + + for path in tracked_paths.iter().take(DIRTY_PATH_DISPLAY_LIMIT) { + lines.push(format!("- {}", path.to_string_lossy())); + } + + if tracked_paths.len() > DIRTY_PATH_DISPLAY_LIMIT { + lines.push(format!( + "- ... and {} more tracked path(s)", + tracked_paths.len() - DIRTY_PATH_DISPLAY_LIMIT + )); + } + + if untracked_count > 0 { + lines.push(format!( + "- ... plus {} untracked file(s) not listed", + untracked_count + )); + } + + lines.push(String::new()); + + if untracked_count > 0 { + lines.push(String::from( + "For local build/serve/check commands, run without `--clean` to include tracked local changes. For remote/parity/clean runs, commit or stash tracked changes first. Commit/stash/remove untracked files before retrying.", + )); + } else { + lines.push(String::from( + "For local build/serve/check commands, run without `--clean` to include tracked local changes. For remote/parity/clean runs, commit or stash tracked changes first.", + )); + } + + lines.join("\n") +} + pub fn check_dirty(root_path: &Path) -> Result<(), Error> { + let (tracked_paths, untracked_count) = + collect_dirty_paths(root_path, |path| !is_generated_path(path))?; + + if tracked_paths.is_empty() && untracked_count == 0 { + Ok(()) + } else { + DirtySnafu { + message: format_dirty_rejection(&tracked_paths, untracked_count), + } + .fail() + } +} + +fn entry_path(entry: &git2::StatusEntry<'_>) -> Option { + entry + .head_to_index() + .and_then(|delta| delta.new_file().path().or_else(|| delta.old_file().path())) + .or_else(|| { + entry + .index_to_workdir() + .and_then(|delta| delta.new_file().path().or_else(|| delta.old_file().path())) + }) + .or_else(|| entry.path().map(Path::new)) + .map(Path::to_path_buf) +} + +fn collect_dirty_paths( + root_path: &Path, + include_path: impl Fn(&Path) -> bool, +) -> Result<(BTreeSet, usize), Error> { let repo = git2::Repository::open(root_path).context(GitSnafu { what: "open root repository", })?; - let mut options = StatusOptions::default(); - options.include_untracked(true); - let statuses = repo.statuses(Some(&mut options)).context(GitSnafu { - what: "get root repository status", + let statuses = dirty_statuses(&repo)?; + let mut paths = BTreeSet::new(); + let mut untracked_count = 0; + + for entry in statuses.iter() { + let status = entry.status(); + let path = entry_path(&entry).unwrap_or_else(|| PathBuf::from("")); + + if status.contains(Status::CONFLICTED) { + return DirtyConflictSnafu { path }.fail(); + } + + if status == Status::CURRENT || status == Status::IGNORED { + continue; + } + + if status == Status::WT_NEW { + if include_path(&path) { + untracked_count += 1; + } + continue; + } + + if let Some(delta) = entry.head_to_index() { + if let Some(old_path) = delta.old_file().path().filter(|path| include_path(path)) { + paths.insert(old_path.to_path_buf()); + } + if let Some(new_path) = delta.new_file().path().filter(|path| include_path(path)) { + paths.insert(new_path.to_path_buf()); + } + } + + if let Some(delta) = entry.index_to_workdir() { + if let Some(old_path) = delta.old_file().path().filter(|path| include_path(path)) { + paths.insert(old_path.to_path_buf()); + } + if let Some(new_path) = delta.new_file().path().filter(|path| include_path(path)) { + paths.insert(new_path.to_path_buf()); + } + } + + if include_path(&path) { + paths.insert(path); + } + } + + Ok((paths, untracked_count)) +} + +pub fn working_tree_paths(root_path: &Path) -> Result, Error> { + let (paths, _) = collect_dirty_paths(root_path, |path| !is_generated_path(path))?; + Ok(paths.into_iter().collect()) +} + +pub fn tracked_working_tree_paths(root_path: &Path) -> Result, Error> { + let (paths, _) = collect_dirty_paths(root_path, |_| true)?; + Ok(paths.into_iter().collect()) +} + +pub fn materialize_working_tree(source_root: &Path, destination_root: &Path) -> Result<(), Error> { + remove_existing_path(destination_root).with_context(|_| IoSnafu { + path: destination_root.to_path_buf(), + })?; + std::fs::create_dir_all(destination_root).with_context(|_| IoSnafu { + path: destination_root.to_path_buf(), + })?; + + let mut paths = tracked_paths(source_root, |_| true)?; + paths.extend(tracked_working_tree_paths(source_root)?); + sync_working_tree_paths(source_root, destination_root, &paths) +} + +pub fn sync_working_tree_paths( + source_root: &Path, + destination_root: &Path, + relative_paths: &BTreeSet, +) -> Result<(), Error> { + for path in relative_paths { + sync_working_tree_path(source_root, destination_root, path)?; + } + + Ok(()) +} + +pub fn index_path(root_path: &Path) -> Result { + let repo = git2::Repository::open(root_path).context(GitSnafu { + what: "open root repository", + })?; + let index = repo.index().context(GitSnafu { + what: "open root repository index", + })?; + index + .path() + .map(Path::to_path_buf) + .with_context(|| UpdateTreeSnafu:: { + msg: "repository index is in-memory".into(), + }) +} + +pub fn sync_materialized_paths( + source_root: &Path, + build_repo_path: &Path, + relative_paths: &BTreeSet, +) -> Result<(), Error> { + if relative_paths.is_empty() { + return Ok(()); + } + + let working_repo = git2::Repository::open(build_repo_path).context(GitSnafu { + what: "open build repository", + })?; + let working_root = working_repo + .workdir() + .with_context(|| UpdateTreeSnafu:: { + msg: "build repository workdir is unavailable".into(), + })?; + let mut index = working_repo.index().context(GitSnafu { + what: "open build repository index", })?; - let mut statuses = statuses.iter().filter(|x| { - x.path() - .map(|x| !x.trim_end_matches('/').ends_with(super::BUILD_DIR)) - .unwrap_or(false) + + for path in relative_paths { + sync_dirty_path(source_root, working_root, &mut index, path)?; + } + + index.write().context(GitSnafu { + what: "write build repository index", + })?; + + Ok(()) +} + +fn remove_existing_path(path: &Path) -> Result<(), std::io::Error> { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_dir() && !metadata.file_type().is_symlink() => { + std::fs::remove_dir_all(path) + } + Ok(_) => std::fs::remove_file(path), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} + +fn tracked_paths( + root_path: &Path, + include_path: impl Fn(&Path) -> bool, +) -> Result, Error> { + let repo = git2::Repository::open(root_path).context(GitSnafu { + what: "open root repository", + })?; + let head = repo.head().context(GitSnafu { what: "head" })?; + let commit = head.peel_to_commit().context(GitSnafu { + what: "peel head to commit", + })?; + let tree = commit.tree().context(GitSnafu { what: "head tree" })?; + let mut paths = BTreeSet::new(); + let mut walk_error = None; + + let walk_result = tree.walk(git2::TreeWalkMode::PreOrder, |prefix, entry| { + let Some(name) = entry.name() else { + walk_error = Some( + UpdateTreeSnafu { + msg: format!("tree entry without name in `{prefix}`"), + } + .build(), + ); + return TreeWalkResult::Abort; + }; + + match entry.kind() { + Some(ObjectType::Blob) => (), + Some(ObjectType::Tree) => return TreeWalkResult::Ok, + kind => { + walk_error = Some( + UpdateTreeSnafu { + msg: format!("unknown blob type `{kind:?}` for `{}{name}`", prefix), + } + .build(), + ); + return TreeWalkResult::Abort; + } + } + + let path = PathBuf::from(format!("{prefix}{name}")); + if include_path(&path) { + paths.insert(path); + } + + TreeWalkResult::Ok }); - if statuses.next().is_some() { - DirtySnafu.fail() + + if let Some(error) = walk_error { + return Err(error); + } + + walk_result.context(GitSnafu { + what: "traverse tree", + })?; + + Ok(paths) +} + +fn remove_index_path(index: &mut git2::Index, path: &Path) -> Result<(), Error> { + match index.remove_path(path) { + Ok(()) => Ok(()), + Err(error) if error.code() == git2::ErrorCode::NotFound => match index.remove_dir(path, -1) + { + Ok(()) => Ok(()), + Err(error) if error.code() == git2::ErrorCode::NotFound => Ok(()), + Err(error) => Err(GitSnafu { + what: "remove dirty path from index", + } + .into_error(error)), + }, + Err(error) => Err(GitSnafu { + what: "remove dirty path from index", + } + .into_error(error)), + } +} + +#[cfg(target_family = "unix")] +fn copy_symlink(source: &Path, destination: &Path) -> Result<(), std::io::Error> { + let target = std::fs::read_link(source)?; + std::os::unix::fs::symlink(target, destination) +} + +#[cfg(target_family = "windows")] +fn copy_symlink(source: &Path, destination: &Path) -> Result<(), std::io::Error> { + let target = std::fs::read_link(source)?; + let resolved_target = source + .parent() + .map(|parent| parent.join(&target)) + .unwrap_or_else(|| target.clone()); + + if std::fs::metadata(&resolved_target) + .map(|metadata| metadata.is_dir()) + .unwrap_or(false) + { + std::os::windows::fs::symlink_dir(target, destination) } else { - Ok(()) + std::os::windows::fs::symlink_file(target, destination) + } +} + +#[cfg(not(any(target_family = "unix", target_family = "windows")))] +fn copy_symlink(_source: &Path, _destination: &Path) -> Result<(), std::io::Error> { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "no symlink implementation available", + )) +} + +fn sync_dirty_path( + source_root: &Path, + working_root: &Path, + index: &mut git2::Index, + relative_path: &Path, +) -> Result<(), Error> { + let source_path = source_root.join(relative_path); + let working_path = working_root.join(relative_path); + + match std::fs::symlink_metadata(&source_path) { + Ok(metadata) if metadata.file_type().is_dir() && !metadata.file_type().is_symlink() => { + remove_existing_path(&working_path).with_context(|_| IoSnafu { + path: working_path.clone(), + })?; + remove_index_path(index, relative_path)?; + Ok(()) + } + Ok(metadata) if metadata.file_type().is_file() || metadata.file_type().is_symlink() => { + if let Some(parent) = working_path.parent() { + std::fs::create_dir_all(parent).with_context(|_| IoSnafu { + path: parent.to_path_buf(), + })?; + } + + remove_existing_path(&working_path).with_context(|_| IoSnafu { + path: working_path.clone(), + })?; + + if metadata.file_type().is_symlink() { + copy_symlink(&source_path, &working_path).with_context(|_| IoSnafu { + path: working_path.clone(), + })?; + } else { + std::fs::copy(&source_path, &working_path).with_context(|_| IoSnafu { + path: source_path.clone(), + })?; + } + + index.add_path(relative_path).context(GitSnafu { + what: "add dirty path to index", + })?; + Ok(()) + } + Ok(_) => DirtyUnsupportedPathSnafu { + path: relative_path.to_path_buf(), + } + .fail(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + remove_existing_path(&working_path).with_context(|_| IoSnafu { + path: working_path.clone(), + })?; + remove_index_path(index, relative_path)?; + Ok(()) + } + Err(error) => Err(IoSnafu { path: source_path }.into_error(error)), + } +} + +fn sync_working_tree_path( + source_root: &Path, + destination_root: &Path, + relative_path: &Path, +) -> Result<(), Error> { + let source_path = source_root.join(relative_path); + let destination_path = destination_root.join(relative_path); + + match std::fs::symlink_metadata(&source_path) { + Ok(metadata) if metadata.file_type().is_dir() && !metadata.file_type().is_symlink() => { + remove_existing_path(&destination_path).with_context(|_| IoSnafu { + path: destination_path.clone(), + }) + } + Ok(metadata) if metadata.file_type().is_file() || metadata.file_type().is_symlink() => { + if let Some(parent) = destination_path.parent() { + std::fs::create_dir_all(parent).with_context(|_| IoSnafu { + path: parent.to_path_buf(), + })?; + } + + remove_existing_path(&destination_path).with_context(|_| IoSnafu { + path: destination_path.clone(), + })?; + + if metadata.file_type().is_symlink() { + copy_symlink(&source_path, &destination_path).with_context(|_| IoSnafu { + path: destination_path.clone(), + })?; + } else { + std::fs::copy(&source_path, &destination_path).with_context(|_| IoSnafu { + path: source_path.clone(), + })?; + } + + Ok(()) + } + Ok(_) => DirtyUnsupportedPathSnafu { + path: relative_path.to_path_buf(), + } + .fail(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + remove_existing_path(&destination_path).with_context(|_| IoSnafu { + path: destination_path, + }) + } + Err(error) => Err(IoSnafu { path: source_path }.into_error(error)), } } +fn materialize_dirty_tree( + source_root: &Path, + working_repo: &git2::Repository, + local_head: Oid, +) -> Result { + let (dirty_paths, untracked_count) = + collect_dirty_paths(source_root, |path| !is_generated_path(path))?; + if untracked_count > 0 { + info!("dirty mode ignores untracked files in the active content repo"); + } + + if dirty_paths.is_empty() { + return Ok(local_head); + } + + let working_root = working_repo + .workdir() + .with_context(|| UpdateTreeSnafu:: { + msg: "build repository workdir is unavailable".into(), + })?; + let mut index = working_repo.index().context(GitSnafu { + what: "open build repository index", + })?; + + for path in dirty_paths { + sync_dirty_path(source_root, working_root, &mut index, &path)?; + } + + index.write().context(GitSnafu { + what: "write build repository index", + })?; + let tree_id = index.write_tree().context(GitSnafu { + what: "write dirty materialization tree", + })?; + let tree = working_repo.find_tree(tree_id).context(GitSnafu { + what: "find dirty materialization tree", + })?; + + let sig = Signature::now("eips-build", "eips-build@eips-build.invalid").context(GitSnafu { + what: "dirty commit signature", + })?; + let parent = working_repo.find_commit(local_head).context(GitSnafu { + what: "find clean local head commit", + })?; + let dirty_head = working_repo + .commit( + Some("HEAD"), + &sig, + &sig, + "Dirty working tree materialization", + &tree, + &[&parent], + ) + .context(GitSnafu { + what: "commit dirty working tree materialization", + })?; + + info!( + "materialized tracked dirty changes from the active content repo into `{}`", + working_root.to_string_lossy() + ); + + Ok(dirty_head) +} + fn check_conflict(master_tree: &Tree, path: &Path, entry: &TreeEntry) -> Result<(), Error> { let original = match master_tree.get_path(path) { Err(_) => return Ok(()), @@ -170,34 +715,51 @@ fn check_conflict(master_tree: &Tree, path: &Path, entry: &TreeEntry) -> Result< pub struct Fresh { src_repo_use: RepositoryUse, + src_repo_path: PathBuf, src_repo_url: Url, + source_materialization: SourceMaterialization, working_repo: git2::Repository, } impl Fresh { - pub fn new(root_path: &Path, build_path: &Path, locations: &Locations) -> Result { - let root_path = absolute(root_path).context(IoSnafu { path: root_path })?; - check_dirty(&root_path)?; - let src_repo_use = locations.identify_repository(&root_path)?; - let src_repo_url = Url::from_directory_path(&root_path) - .ok() - .context(PathUrlSnafu { path: root_path })?; + pub fn new( + root_path: &Path, + repo_path: &Path, + src_repo_use: RepositoryUse, + source_materialization: SourceMaterialization, + ) -> Result { + let root_path = absolute(root_path).with_context(|_| IoSnafu { path: root_path })?; + if source_materialization == SourceMaterialization::Clean { + check_dirty(&root_path)?; + } + let src_repo_url = + Url::from_directory_path(&root_path) + .ok() + .with_context(|| PathUrlSnafu { + path: root_path.clone(), + })?; debug!("source repository at `{src_repo_url}`"); - let working_repo = open_or_init(build_path)?; + let working_repo = open_or_init(repo_path)?; Ok(Self { working_repo, + src_repo_path: root_path, src_repo_url, src_repo_use, + source_materialization, }) } pub fn clone_src(self) -> Result { info!("cloning local repository"); - let master = fetch(&self.working_repo, self.src_repo_url.as_str(), "HEAD")?; + let master = fetch( + &self.working_repo, + self.src_repo_url.as_str(), + "+HEAD:refs/build-eips/source-head", + )?; self.working_repo .set_head_detached(master.id()) .context(GitSnafu { what: "detach" })?; @@ -226,9 +788,15 @@ impl Fresh { panic!("submodules not supported yet"); } - let local_head = master.id(); + let mut local_head = master.id(); drop(master); drop(branch); + + if self.source_materialization == SourceMaterialization::Dirty { + local_head = + materialize_dirty_tree(&self.src_repo_path, &self.working_repo, local_head)?; + } + Ok(SourceOnly { local_head, src_repo_use: self.src_repo_use, @@ -245,12 +813,16 @@ pub struct SourceOnly { } impl SourceOnly { + pub fn merge(&self) -> Result<(), Error> { + merge_sibling_repositories(&self.working_repo, &self.src_repo_use, self.local_head) + } + pub fn fetch_upstream(self) -> Result { info!("fetching latest {} repository", self.src_repo_use.title); let latest_master = fetch( &self.working_repo, self.src_repo_use.location.repository.as_str(), - "master", + "+master:refs/build-eips/upstream-head", )?; let upstream_head = latest_master.id(); drop(latest_master); @@ -324,41 +896,138 @@ impl SourceWithUpstream { Ok(changed_files) } - fn check_ignored(&self, tree: &Tree) -> Result<(), Error> { - let mut walk_error = None; - let walk_result = tree.walk(git2::TreeWalkMode::PreOrder, |a, b| { - if b.kind() != Some(ObjectType::Blob) { - return TreeWalkResult::Ok; + pub fn merge(&self) -> Result<(), Error> { + merge_sibling_repositories(&self.working_repo, &self.src_repo_use, self.local_head) + } +} + +fn check_ignored(working_repo: &git2::Repository, tree: &Tree) -> Result<(), Error> { + let mut walk_error = None; + let walk_result = tree.walk(git2::TreeWalkMode::PreOrder, |a, b| { + if b.kind() != Some(ObjectType::Blob) { + return TreeWalkResult::Ok; + } + + let path = match b.name() { + None => a.to_owned(), + Some(p) => format!("{a}{p}"), + }; + + debug!("checking if `{path}` is ignored"); + + match working_repo.is_path_ignored(&path) { + Ok(false) => TreeWalkResult::Ok, + Ok(true) => { + walk_error = Some( + UpdateTreeSnafu { + msg: format!("contains ignored path `{path}`"), + } + .build(), + ); + TreeWalkResult::Abort } + Err(e) => { + walk_error = Some( + GitSnafu { + what: "check ignored", + } + .into_error(e), + ); + TreeWalkResult::Abort + } + } + }); - let path = match b.name() { - None => a.to_owned(), - Some(p) => format!("{a}{p}"), - }; + if let Some(error) = walk_error { + return Err(error); + } - debug!("checking if `{path}` is ignored"); + walk_result.context(GitSnafu { + what: "traverse tree", + })?; - match self.working_repo.is_path_ignored(&path) { - Ok(false) => TreeWalkResult::Ok, - Ok(true) => { + Ok(()) +} + +fn merge_sibling_repositories( + working_repo: &git2::Repository, + repo_use: &RepositoryUse, + mut local_head: Oid, +) -> Result<(), Error> { + for (index, (other_kind, other_repo)) in repo_use + .other_repos + .iter() + .progress_ext("Merge Repos") + .enumerate() + { + let local_commit = working_repo.find_commit(local_head).context(GitSnafu { + what: "find local head commit", + })?; + let local_tree = local_commit.tree().context(GitSnafu { + what: "getting local head tree", + })?; + info!("fetching {other_kind} repository"); + // Local sibling overrides should follow the checked-out repo HEAD instead of assuming `master`. + let other_ref = format!("refs/build-eips/other-head-{index}"); + let other_refspec = if other_repo.scheme() == "file" { + format!("+HEAD:{other_ref}") + } else { + format!("+master:{other_ref}") + }; + let master_other = fetch(working_repo, other_repo.as_str(), &other_refspec)?; + let other_tree = master_other.tree().context(GitSnafu { + what: "getting other tree", + })?; + + let mut tree_builder = TreeUpdateBuilder::new(); + let prefix = format!("{}/", CONTENT_DIR); + let mut walk_error: Option = None; + let walk_result = other_tree.walk(git2::TreeWalkMode::PreOrder, |a, b| { + if !a.starts_with(&prefix) && (!a.is_empty() || b.name() != Some(CONTENT_DIR)) { + return TreeWalkResult::Skip; + } + + let name = match b.name() { + Some(n) => n, + None => { walk_error = Some( UpdateTreeSnafu { - msg: format!("contains ignored path `{path}`"), + msg: format!("tree entry without name in `{a}`"), } .build(), ); - TreeWalkResult::Abort + return TreeWalkResult::Abort; } - Err(e) => { + }; + + let path = format!("{}{}", a, name); + match b.kind() { + Some(ObjectType::Blob) => (), + Some(ObjectType::Tree) => return TreeWalkResult::Ok, + kind => { walk_error = Some( - GitSnafu { - what: "check ignored", + UpdateTreeSnafu { + msg: format!("unknown blob type `{kind:?}` for `{path}`"), } - .into_error(e), + .build(), ); - TreeWalkResult::Abort + return TreeWalkResult::Abort; } } + + if path == CONTENT_INDEX_PATH { + debug!("skip sibling homepage `{path}`"); + return TreeWalkResult::Ok; + } + + if let Err(e) = check_conflict(&local_tree, Path::new(&path), b) { + walk_error = Some(e); + return TreeWalkResult::Abort; + } + + debug!("upsert `{path}`"); + tree_builder.upsert(path, b.id(), FileMode::Blob); + TreeWalkResult::Ok }); if let Some(error) = walk_error { @@ -369,130 +1038,53 @@ impl SourceWithUpstream { what: "traverse tree", })?; - Ok(()) - } - - pub fn merge(&self) -> Result<(), Error> { - let repo_use = &self.src_repo_use; - let master_tree = self.local_head_tree()?; - let mut local_head = self.local_head; - for (other_kind, other_repo) in repo_use.other_repos.iter().progress_ext("Merge Repos") { - info!("fetching {other_kind} repository"); - let master_other = fetch( - &self.working_repo, - other_repo.as_str(), - "master:master-other", - )?; - let other_tree = master_other.tree().context(GitSnafu { - what: "getting other tree", - })?; + let merged_tree_oid = tree_builder + .create_updated(working_repo, &local_tree) + .context(GitSnafu { what: "build tree" })?; + let merged_tree = working_repo.find_tree(merged_tree_oid).unwrap(); - let mut tree_builder = TreeUpdateBuilder::new(); - let prefix = format!("{}/", super::CONTENT_DIR); - let mut walk_error: Option = None; - let walk_result = other_tree.walk(git2::TreeWalkMode::PreOrder, |a, b| { - if !a.starts_with(&prefix) - && (!a.is_empty() || b.name() != Some(super::CONTENT_DIR)) - { - return TreeWalkResult::Skip; - } + check_ignored(working_repo, &merged_tree)?; - let name = match b.name() { - Some(n) => n, - None => { - walk_error = Some( - UpdateTreeSnafu { - msg: format!("tree entry without name in `{a}`"), - } - .build(), - ); - return TreeWalkResult::Abort; - } - }; - - let path = format!("{}{}", a, name); - match b.kind() { - Some(ObjectType::Blob) => (), - Some(ObjectType::Tree) => return TreeWalkResult::Ok, - kind => { - walk_error = Some( - UpdateTreeSnafu { - msg: format!("unknown blob type `{kind:?}` for `{path}`"), - } - .build(), - ); - return TreeWalkResult::Abort; - } - } + let sig = + Signature::now("eips-build", "eips-build@eips-build.invalid").context(GitSnafu { + what: "commit signature", + })?; + let msg = format!("Merge {other_repo}"); + local_head = working_repo + .commit( + Some("HEAD"), + &sig, + &sig, + &msg, + &merged_tree, + &[&local_commit, &master_other], + ) + .context(GitSnafu { what: "committing" })?; + + working_repo + .checkout_head(Some(CheckoutBuilder::default().force())) + .context(GitSnafu { + what: "checkout merged", + })?; - if let Err(e) = check_conflict(&master_tree, Path::new(&path), b) { - walk_error = Some(e); - return TreeWalkResult::Abort; + drop(merged_tree); + drop(other_tree); + drop(master_other); + drop(local_tree); + drop(local_commit); + match working_repo.find_reference(&other_ref) { + Ok(mut reference) => { + if let Err(error) = reference.delete() { + debug!("unable to delete temporary sibling ref `{other_ref}`: {error}"); } - - debug!("upsert `{path}`"); - tree_builder.upsert(path, b.id(), FileMode::Blob); - TreeWalkResult::Ok - }); - - if let Some(error) = walk_error { - return Err(error); } - - walk_result.context(GitSnafu { - what: "traverse tree", - })?; - - let merged_tree_oid = tree_builder - .create_updated(&self.working_repo, &master_tree) - .context(GitSnafu { what: "build tree" })?; - let merged_tree = self.working_repo.find_tree(merged_tree_oid).unwrap(); - - self.check_ignored(&merged_tree)?; - - let sig = Signature::now("eips-build", "eips-build@eips-build.invalid").context( - GitSnafu { - what: "commit signature", - }, - )?; - let msg = format!("Merge {other_repo}"); - let master = self - .working_repo - .find_commit(local_head) - .context(GitSnafu { - what: "find local head commit", - })?; - local_head = self - .working_repo - .commit( - Some("HEAD"), - &sig, - &sig, - &msg, - &merged_tree, - &[&master, &master_other], - ) - .context(GitSnafu { what: "committing" })?; - - self.working_repo - .checkout_head(Some(CheckoutBuilder::default().force())) - .context(GitSnafu { - what: "checkout merged", - })?; - - self.working_repo - .find_branch("master-other", BranchType::Local) - .context(GitSnafu { - what: "find master-other", - })? - .delete() - .context(GitSnafu { - what: "delete master-other", - })?; + Err(error) => { + debug!("temporary sibling ref `{other_ref}` was not deleted: {error}"); + } } - - Ok(()) } + + Ok(()) } fn fetch<'a>( @@ -501,7 +1093,19 @@ fn fetch<'a>( refspec: &'_ str, ) -> Result, Error> { debug!("fetching repository at `{url}`"); - let mut remote = repo.remote_anonymous(url).context(GitSnafu { + let remote_name = "__build_eips_fetch"; + match repo.remote_delete(remote_name) { + Ok(()) => (), + Err(error) if error.code() == git2::ErrorCode::NotFound => (), + Err(error) => { + return Err(GitSnafu { + what: "deleting temporary remote", + } + .into_error(error)) + } + } + + let mut remote = repo.remote(remote_name, url).context(GitSnafu { what: "creating remote", })?; { @@ -514,14 +1118,24 @@ fn fetch<'a>( what: "fetching repo", })?; } + drop(remote); + repo.remote_delete(remote_name).context(GitSnafu { + what: "deleting temporary remote", + })?; + + let fetched_ref = refspec + .split_once(':') + .map(|(_, destination)| destination) + .filter(|destination| !destination.is_empty()) + .unwrap_or("FETCH_HEAD"); let commit = repo - .revparse_single("FETCH_HEAD") + .revparse_single(fetched_ref) .context(GitSnafu { - what: "revparse FETCH_HEAD", + what: "revparse fetched ref", })? .peel_to_commit() .context(GitSnafu { - what: "peel FETCH_HEAD", + what: "peel fetched ref", })?; Ok(commit) } @@ -538,36 +1152,256 @@ fn open_or_init(dir: &Path) -> Result { Ok(repo) } -impl Cache { - pub fn repo(&self, url: &str, commit: &str) -> Result { - let key = format!("git\0{url}"); - let dir = self.dir(&key)?; +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; - let repo = open_or_init(&dir)?; - let object = match repo.revparse_single(commit) { - Ok(c) => c, - Err(e) if e.code() == git2::ErrorCode::NotFound => { - fetch(&repo, url, "master")?; - repo.revparse_single(commit).context(GitSnafu { - what: "revparse cached commit", - })? - } - Err(e) => { - return Err(GitSnafu { - what: "revparse cached commit", - } - .into_error(e)) - } + use git2::{IndexAddOption, Repository, Signature}; + use tempfile::TempDir; + + use super::{ + materialize_working_tree, sync_working_tree_paths, tracked_working_tree_paths, Fresh, + RepositoryUse, SourceMaterialization, + }; + use crate::config::RepositoryEndpoint; + + fn file_url(path: &Path) -> url::Url { + url::Url::from_directory_path(path).unwrap() + } + + fn write_file(root: &Path, relative: impl AsRef, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn commit_all(repo: &Repository, message: &str) { + let mut index = repo.index().unwrap(); + index + .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + let parents = repo + .head() + .ok() + .and_then(|head| head.target()) + .map(|oid| repo.find_commit(oid).unwrap()) + .into_iter() + .collect::>(); + let parent_refs = parents.iter().collect::>(); + + repo.commit( + Some("HEAD"), + &signature, + &signature, + message, + &tree, + &parent_refs, + ) + .unwrap(); + } + + fn init_repo(path: &Path, files: &[(&str, &str)]) -> Repository { + std::fs::create_dir_all(path).unwrap(); + let repo = Repository::init(path).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + for (relative, contents) in files { + write_file(path, relative, contents); + } + commit_all(&repo, "initial"); + repo + } + + fn stage_path(repo: &Repository, relative: &str) { + let mut index = repo.index().unwrap(); + index.add_path(Path::new(relative)).unwrap(); + index.write().unwrap(); + } + + #[test] + fn merge_skips_sibling_homepage_and_keeps_sibling_proposals() { + let temp = TempDir::new().unwrap(); + let active = temp.path().join("active"); + let sibling = temp.path().join("sibling"); + let prepared = temp.path().join("prepared"); + + init_repo( + &active, + &[ + ("content/_index.md", "active homepage\n"), + ("content/00555.md", "# Active proposal\n"), + ], + ); + init_repo( + &sibling, + &[ + ("content/_index.md", "sibling homepage\n"), + ("content/00678.md", "# Sibling proposal\n"), + ], + ); + + let mut other_repos = std::collections::BTreeMap::new(); + other_repos.insert("ERCs".to_owned(), file_url(&sibling)); + let active_url = file_url(&active); + let repository_use = RepositoryUse { + title: "EIPs".to_owned(), + location: RepositoryEndpoint { + repository: active_url, + base_url: "https://eips.example.test/".parse().unwrap(), + }, + other_repos, }; - repo.checkout_tree(&object, Some(CheckoutBuilder::new().force())) - .context(GitSnafu { - what: "checkout cached commit", - })?; - repo.set_head_detached(object.id()).context(GitSnafu { - what: "set detached head", - })?; + Fresh::new( + &active, + &prepared, + repository_use, + SourceMaterialization::Clean, + ) + .unwrap() + .clone_src() + .unwrap() + .fetch_upstream() + .unwrap() + .merge() + .unwrap(); + + assert_eq!( + std::fs::read_to_string(prepared.join("content/_index.md")).unwrap(), + "active homepage\n" + ); + assert!(prepared.join("content/00678.md").is_file()); + } + + #[test] + fn materialize_working_tree_uses_tracked_theme_scope() { + let temp = TempDir::new().unwrap(); + let theme = temp.path().join("theme"); + let mounted = temp.path().join("repo/themes/eips-theme"); + let repo = init_repo( + &theme, + &[ + ("config/zola.toml", "title = 'theme'\n"), + ("build/generated.txt", "tracked build path\n"), + ("delete.txt", "delete me\n"), + ("staged.txt", "old staged\n"), + ("tracked.txt", "old tracked\n"), + ], + ); + + write_file(&theme, "tracked.txt", "unstaged tracked edit\n"); + write_file(&theme, "staged.txt", "staged tracked edit\n"); + stage_path(&repo, "staged.txt"); + write_file(&theme, "new-staged.txt", "new staged file\n"); + stage_path(&repo, "new-staged.txt"); + std::fs::remove_file(theme.join("delete.txt")).unwrap(); + let mut index = repo.index().unwrap(); + index.remove_path(Path::new("delete.txt")).unwrap(); + index.write().unwrap(); + write_file( + &theme, + "untracked.txt", + "ignored by theme materialization\n", + ); + + materialize_working_tree(&theme, &mounted).unwrap(); + + assert_eq!( + std::fs::read_to_string(mounted.join("config/zola.toml")).unwrap(), + "title = 'theme'\n" + ); + assert_eq!( + std::fs::read_to_string(mounted.join("build/generated.txt")).unwrap(), + "tracked build path\n" + ); + assert_eq!( + std::fs::read_to_string(mounted.join("tracked.txt")).unwrap(), + "unstaged tracked edit\n" + ); + assert_eq!( + std::fs::read_to_string(mounted.join("staged.txt")).unwrap(), + "staged tracked edit\n" + ); + assert_eq!( + std::fs::read_to_string(mounted.join("new-staged.txt")).unwrap(), + "new staged file\n" + ); + assert!(!mounted.join("delete.txt").exists()); + assert!(!mounted.join("untracked.txt").exists()); + } + + #[test] + fn newly_staged_theme_file_syncs_after_git_index_rescan() { + let temp = TempDir::new().unwrap(); + let theme = temp.path().join("theme"); + let mounted = temp.path().join("repo/themes/eips-theme"); + let repo = init_repo(&theme, &[("config/zola.toml", "title = 'theme'\n")]); + materialize_working_tree(&theme, &mounted).unwrap(); + let mut previous_dirty_paths = tracked_working_tree_paths(&theme) + .unwrap() + .into_iter() + .collect::>(); + + write_file(&theme, "templates/new.html", "new staged template\n"); + stage_path(&repo, "templates/new.html"); + let current_dirty_paths = tracked_working_tree_paths(&theme) + .unwrap() + .into_iter() + .collect::>(); + let affected_paths = previous_dirty_paths + .union(¤t_dirty_paths) + .cloned() + .collect(); + + sync_working_tree_paths(&theme, &mounted, &affected_paths).unwrap(); + previous_dirty_paths = current_dirty_paths; + + assert_eq!( + std::fs::read_to_string(mounted.join("templates/new.html")).unwrap(), + "new staged template\n" + ); + assert!(previous_dirty_paths.contains(Path::new("templates/new.html"))); + } + + #[cfg(target_family = "unix")] + #[test] + fn tracked_theme_symlinks_are_materialized_as_symlinks() { + let temp = TempDir::new().unwrap(); + let theme = temp.path().join("theme"); + let mounted = temp.path().join("repo/themes/eips-theme"); + std::fs::create_dir_all(&theme).unwrap(); + let repo = Repository::init(&theme).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + write_file(&theme, "target.txt", "target\n"); + std::os::unix::fs::symlink("target.txt", theme.join("linked.txt")).unwrap(); + commit_all(&repo, "initial"); + + materialize_working_tree(&theme, &mounted).unwrap(); + + assert!(std::fs::symlink_metadata(mounted.join("linked.txt")) + .unwrap() + .file_type() + .is_symlink()); + assert_eq!( + std::fs::read_link(mounted.join("linked.txt")).unwrap(), + PathBuf::from("target.txt") + ); + } + + #[test] + fn materialize_working_tree_requires_git_repository() { + let temp = TempDir::new().unwrap(); + let theme = temp.path().join("theme"); + let mounted = temp.path().join("repo/themes/eips-theme"); + std::fs::create_dir_all(&theme).unwrap(); + + let error = materialize_working_tree(&theme, &mounted).unwrap_err(); - Ok(dir) + assert!(error.to_string().contains("unable to open root repository")); } } diff --git a/src/identity.rs b/src/identity.rs new file mode 100644 index 0000000..68ec03a --- /dev/null +++ b/src/identity.rs @@ -0,0 +1,108 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Active repository identity selection. + +use std::path::Path; + +use snafu::{OptionExt, ResultExt, Whatever}; + +use crate::{ + config::{self, Config, LoadedRepoManifest}, + git, +}; + +#[derive(Debug, Clone)] +pub(crate) enum ActiveRepoIdentity { + Manifest(Box), + Legacy { repo_id: String }, +} + +impl ActiveRepoIdentity { + pub(crate) fn load(root_path: &Path) -> Result { + if let Some(manifest) = + LoadedRepoManifest::load(root_path).whatever_context("unable to load repo manifest")? + { + return Ok(Self::Manifest(Box::new(manifest))); + } + + match Config::production() + .locations + .identify_repository_title(root_path) + { + Ok(repo_id) => Ok(Self::Legacy { repo_id }), + Err(git::Error::NoIdentify { .. }) => { + snafu::whatever!( + "active repository `{}` does not carry `{}` and does not match the legacy EIPs/ERCs identity fallback", + root_path.to_string_lossy(), + config::REPO_MANIFEST_FILE + ) + } + Err(error) => Err(error).whatever_context("cannot identify legacy repository use"), + } + } + + pub(crate) fn repo_id(&self) -> &str { + match self { + Self::Manifest(manifest) => &manifest.manifest().repo_id, + Self::Legacy { repo_id } => repo_id, + } + } + + pub(crate) fn source_description(&self) -> &'static str { + match self { + Self::Manifest(_) => "repo manifest", + Self::Legacy { .. } => "legacy EIPs/ERCs fallback", + } + } + + pub(crate) fn manifest(&self) -> Option<&LoadedRepoManifest> { + match self { + Self::Manifest(manifest) => Some(manifest.as_ref()), + Self::Legacy { .. } => None, + } + } + + pub(crate) fn sibling_ids(&self) -> Vec { + match self { + Self::Manifest(manifest) => manifest.manifest().siblings.keys().cloned().collect(), + Self::Legacy { repo_id } => Config::production() + .locations + .repository_use_for_title(repo_id) + .expect("legacy repository id should have metadata") + .other_repos + .keys() + .cloned() + .collect(), + } + } + + pub(crate) fn repository_use(&self, staging: bool) -> Result { + match self { + Self::Manifest(manifest) => { + let manifest = manifest.manifest(); + Ok(git::RepositoryUse { + title: manifest.repo_id.clone(), + location: manifest.active_endpoint(staging).clone(), + other_repos: manifest.sibling_repositories(staging), + }) + } + Self::Legacy { repo_id } => { + let baseline = if staging { + Config::staging() + } else { + Config::production() + }; + baseline + .locations + .repository_use_for_title(repo_id) + .with_whatever_context(|| { + format!("legacy repository metadata for `{repo_id}` is unavailable") + }) + } + } + } +} diff --git a/src/layout.rs b/src/layout.rs index cc2faa6..dec1039 100644 --- a/src/layout.rs +++ b/src/layout.rs @@ -4,7 +4,25 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +//! Shared build layout names and path helpers. + +use std::path::{Path, PathBuf}; + pub(crate) const CONTENT_DIR: &str = "content"; pub(crate) const BUILD_DIR: &str = "build"; pub(crate) const REPO_DIR: &str = "repo"; -pub(crate) const OUTPUT_DIR: &str = "output"; +const OUTPUT_DIR: &str = "output"; + +pub(crate) fn output_path(build_path: &Path) -> PathBuf { + build_path.join(OUTPUT_DIR) +} + +pub(crate) fn mounted_theme_path(project_path: &Path) -> PathBuf { + project_path.join("themes").join("eips-theme") +} + +pub(crate) fn theme_config_path(theme_path: &Path) -> PathBuf { + [theme_path, Path::new("config"), Path::new("zola.toml")] + .iter() + .collect() +} diff --git a/src/lint.rs b/src/lint.rs index a433b37..f845458 100644 --- a/src/lint.rs +++ b/src/lint.rs @@ -11,7 +11,6 @@ use clap::ValueEnum; use log::debug; use semver::{Comparator, Op, VersionReq}; -use crate::cache::Cache; use crate::progress::ProgressIteratorExt; use eipw_lint::reporters::{AdditionalHelp, Count, Json, Reporter, Text}; @@ -51,11 +50,6 @@ pub enum Error { source: std::io::Error, }, #[snafu(transparent)] - Git { - #[snafu(backtrace)] - source: crate::git::Error, - }, - #[snafu(transparent)] SchemaVersion { #[snafu(backtrace)] source: SchemaVersionError, @@ -92,17 +86,9 @@ struct Config { eipw: eipw_lint::config::DefaultOptions, } -#[derive(Debug, clap::Args, Serialize, Deserialize)] +#[derive(Debug, Clone, clap::Args, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct CmdArgs { - /// Disable linting entirely - #[arg(long, exclusive(true))] - no_lint: bool, - - /// Restrict linting to specific files and/or directories (relative to project root) - #[clap(required(false))] - sources: Vec, - /// Lint output format #[clap(long, value_enum, default_value_t)] format: Format, @@ -257,22 +243,14 @@ fn version_cmp( #[tokio::main(flavor = "current_thread")] pub async fn eipw( - theme_repo: &str, - theme_rev: &str, - cache: &Cache, - root_dir: &Path, + theme_path: &Path, repo_dir: &Path, - changed_paths: Vec, + sources: Vec, opts: CmdArgs, ) -> Result<(), Error> { - if opts.no_lint { - return Ok(()); - } - let mut stdout = std::io::stdout(); - let mut config_path = cache.repo(theme_repo, theme_rev)?; - + let mut config_path = theme_path.to_path_buf(); config_path.push("config"); config_path.push("eipw.toml"); @@ -301,36 +279,8 @@ pub async fn eipw( .await .context(FsSnafu { path: repo_dir })?; - let paths = if opts.sources.is_empty() { - changed_paths - } else { - let root_dir = tokio::fs::canonicalize(root_dir) - .await - .context(FsSnafu { path: root_dir })?; - let mut repo_relative_sources = Vec::with_capacity(opts.sources.len()); - for source in &opts.sources { - let root_relative_source = root_dir.join(source); - let full_source = tokio::fs::canonicalize(&root_relative_source) - .await - .context(FsSnafu { - path: root_relative_source, - })?; - - let relative_source = match full_source.strip_prefix(&root_dir) { - Ok(r) => r, - Err(e) => { - let err = std::io::Error::new(std::io::ErrorKind::NotFound, e); - return Err(FsSnafu { path: full_source }.into_error(err)); - } - }; - - repo_relative_sources.push(repo_dir.join(relative_source)); - } - - repo_relative_sources - }; - - let sources = collect_sources(paths).await?; + let sources: Vec<_> = sources.iter().map(|source| repo_dir.join(source)).collect(); + let sources = collect_sources(sources).await?; let reporter = match opts.format { Format::Json => EitherReporter::Json(Json::default()), diff --git a/src/main.rs b/src/main.rs index bc01379..bd31014 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,21 +4,34 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -mod cache; mod changed; mod cli; mod config; mod context; +mod editorial; +mod execution; mod find_root; mod git; mod github; +mod identity; mod layout; mod lint; mod markdown; +mod network_upgrades; +mod pipeline; +mod preview; mod print; mod progress; +mod proposal; +mod proposal_catalog; +mod proposal_metadata; +mod serve; +mod workspace; mod zola; +#[cfg(test)] +mod tests; + use std::path::{Path, PathBuf}; use clap::Parser; @@ -27,9 +40,12 @@ use log::{debug, info}; use snafu::{Report, ResultExt, Whatever}; use crate::{ - cli::{Args, Operation}, - config::Config, - layout::{BUILD_DIR, CONTENT_DIR, OUTPUT_DIR, REPO_DIR}, + cli::{Args, EditorialCommand, Operation, RuntimeOperation}, + editorial::{editorial_runtime_execution, run_editorial_lint}, + execution::{resolve_execution, validate_non_execution_command_flags}, + layout::output_path, + pipeline::Prepared, + workspace::{doctor_workspace, init_workspace}, }; fn lock(build_path: &Path) -> Result { @@ -48,165 +64,88 @@ fn lock(build_path: &Path) -> Result { Ok(lock_file) } -fn make_build_dir(root: &Path) -> Result { - let build_path = root.join(BUILD_DIR); - if let Err(e) = std::fs::create_dir_all(&build_path) { +fn make_build_dir(build_path: &Path) -> Result { + if let Err(e) = std::fs::create_dir_all(build_path) { debug!( "got while creating build directory: {}", Report::from_error(e) ); } - Ok(build_path) + Ok(build_path.to_path_buf()) } -#[derive(Debug)] -struct Prepared { - cache: cache::Cache, - root_path: PathBuf, - repo_path: PathBuf, - output_path: PathBuf, - config: Config, -} +fn run() -> Result<(), Whatever> { + let args = Args::parse(); + validate_non_execution_command_flags(&args)?; -impl Prepared { - fn prepare( - eipw: lint::CmdArgs, - config: Config, - root_path: PathBuf, - build_path: PathBuf, - ) -> Result { - zola::find_zola().whatever_context("unable to find suitable zola binary")?; - - let repo_path = build_path.join(REPO_DIR); - let content_path = repo_path.join(CONTENT_DIR); - let output_path = build_path.join(OUTPUT_DIR); - - let both = git::Fresh::new(&root_path, &repo_path, &config.locations) - .whatever_context("initializing build repo")? - .clone_src() - .whatever_context("cloning source repo")? - .fetch_upstream() - .whatever_context("fetching upstream repo")?; - - let changed_files: Vec<_> = both - .changed_files() - .whatever_context("unable to list changed files")? - .into_iter() - .filter(|p| changed::is_proposal_path(p.into())) - .map(|p| repo_path.join(p)) - .collect(); - - both.merge() - .whatever_context("unable to merge ERC/EIP repositories")?; - - let cache = cache::Cache::open().whatever_context("unable to open cache")?; - - lint::eipw( - config.theme.repository.as_str(), - &config.theme.commit, - &cache, - &root_path, - &repo_path, - changed_files, - eipw, - ) - .whatever_context("linting failed")?; - - markdown::preprocess(&content_path).whatever_context("unable to preprocess markdown")?; - - Ok(Prepared { - config, - root_path, - cache, - repo_path, - output_path, - }) + if let Operation::Print { print } = &args.operation { + print::print(print.clone()); + return Ok(()); } - fn build(self) -> Result<(), Whatever> { - let repository_use = self - .config - .locations - .identify_repository(&self.root_path) - .whatever_context("cannot identify repository use")?; - zola::build( - self.config.theme.repository.as_str(), - &self.config.theme.commit, - &self.cache, - &self.repo_path, - &self.output_path, - repository_use.location.base_url.as_str(), - ) - .whatever_context("zola build failed")?; - Ok(()) + if let Operation::Init { + path, + template, + platform_dev, + } = &args.operation + { + init_workspace(&args, path.clone(), *template, *platform_dev)?; + return Ok(()); } - fn serve(self) -> Result<(), Whatever> { - zola::serve( - self.config.theme.repository.as_str(), - &self.config.theme.commit, - &self.cache, - &self.repo_path, - &self.output_path, - ) - .whatever_context("zola serve failed")?; - Ok(()) + if let Operation::Doctor = &args.operation { + doctor_workspace(&args)?; + return Ok(()); } - fn check(self) -> Result<(), Whatever> { - zola::check( - self.config.theme.repository.as_str(), - &self.config.theme.commit, - &self.cache, - &self.repo_path, - ) - .whatever_context("zola check failed")?; - Ok(()) - } -} + let runtime_operation = args + .operation + .runtime_operation() + .expect("non-execution commands should have returned earlier"); + let resolved = resolve_execution(&args)?; -fn run() -> Result<(), Whatever> { - let args = Args::parse(); - if let Operation::Print { print } = args.operation { - print::print(print); + if matches!(runtime_operation, RuntimeOperation::Preview) { + preview::serve(&output_path(&resolved.build_path), &resolved.server_binding) + .whatever_context("preview server failed")?; return Ok(()); } - let config = if args.staging { - Config::staging() - } else { - Config::production() - }; - - let root_path = context::root(&args)?; - let build_path = make_build_dir(&root_path)?; - + let build_path = make_build_dir(&resolved.build_path)?; let mut lock_file = lock(&build_path)?; - match args.operation { - Operation::Print { .. } => unreachable!(), - Operation::Clean => { + match runtime_operation { + RuntimeOperation::Clean => { // TODO: There's a race condition here. Maybe we move the lockfile to the repository // root? lock_file .unlock() .whatever_context("unable to unlock build directory")?; - std::fs::remove_dir_all(build_path) + std::fs::remove_dir_all(&build_path) .whatever_context("unable to remove build directory")?; return Ok(()); } - Operation::Check { eipw } => { - Prepared::prepare(eipw, config, root_path, build_path)?.check()?; + RuntimeOperation::Check => { + Prepared::prepare(resolved)?.check()?; } - Operation::Build { eipw } => { - Prepared::prepare(eipw, config, root_path, build_path)?.build()?; + RuntimeOperation::Build => { + Prepared::prepare(resolved)?.build()?; } - Operation::Serve { eipw } => { - Prepared::prepare(eipw, config, root_path, build_path)?.serve()?; + RuntimeOperation::Serve => { + Prepared::prepare(resolved)?.serve()?; } - Operation::Changed { all, format } => { - changed::run(&root_path, &build_path, &config, all, &format)?; + RuntimeOperation::Preview => unreachable!(), + RuntimeOperation::Changed { all, format } => { + changed::run(&resolved, &build_path, all, &format)?; } + RuntimeOperation::Editorial { command } => match command { + EditorialCommand::Lint { selectors, eipw } => { + run_editorial_lint(&resolved, &selectors, eipw)?; + } + EditorialCommand::Check { selectors, eipw } => { + run_editorial_lint(&resolved, &selectors, eipw)?; + Prepared::prepare(editorial_runtime_execution(resolved, &selectors))?.check()?; + } + }, } lock_file diff --git a/src/markdown.rs b/src/markdown.rs index 06198e6..9d111d7 100644 --- a/src/markdown.rs +++ b/src/markdown.rs @@ -22,11 +22,11 @@ use regex::Regex; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::ffi::OsStr; use std::fs::read_to_string; -use std::io::Write; -use std::path::{Path, PathBuf}; +use std::io::{ErrorKind, Write}; +use std::path::{Component, Path, PathBuf}; use snafu::{whatever, OptionExt, ResultExt, Whatever}; @@ -38,7 +38,33 @@ use walkdir::WalkDir; use iref::IriRefBuf; -use crate::progress::ProgressIteratorExt; +use crate::{ + progress::ProgressIteratorExt, + proposal::{ + path_component_proposal_number, proposal_number_from_content_markdown_path, OnlyRenderPlan, + ProposalAssetKind, ProposalNumber, ProposalReference, + }, +}; + +#[derive(Clone, Copy)] +enum MissingPathMode { + Error, + Ignore, +} + +impl MissingPathMode { + fn should_ignore_io_error(self, error: &std::io::Error) -> bool { + matches!(self, Self::Ignore) + && matches!(error.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) + } + + fn should_ignore_walkdir_error(self, error: &walkdir::Error) -> bool { + error + .io_error() + .map(|error| self.should_ignore_io_error(error)) + .unwrap_or(false) + } +} #[derive(Debug, Serialize, Deserialize)] struct Author { @@ -137,6 +163,19 @@ impl Default for FrontMatter { } } +fn filesystem_modified(p: &Path) -> Result { + let metadata = std::fs::metadata(p) + .with_whatever_context(|e| format!("unable to read metadata for `{}`: {e}", p.display()))?; + let modified = metadata.modified().with_whatever_context(|e| { + format!( + "unable to read filesystem modified time for `{}`: {e}", + p.display() + ) + })?; + let date_time: DateTime = modified.into(); + Ok(date_time.to_rfc3339().parse().unwrap()) +} + fn last_modified(p: &Path) -> Result { // TODO: Replace this with `git2` let mut command = std::process::Command::new("git"); @@ -158,7 +197,16 @@ fn last_modified(p: &Path) -> Result { } let date_str = std::str::from_utf8(&output.stdout) - .with_whatever_context(|e| format!("command {:?} output not UTF-8: {e}", command))?; + .with_whatever_context(|e| format!("command {:?} output not UTF-8: {e}", command))? + .trim(); + + if date_str.is_empty() { + debug!( + "falling back to filesystem modified time for `{}` because git has no timestamp for the current path", + p.to_string_lossy() + ); + return filesystem_modified(p); + } let unix: i64 = date_str.parse().with_whatever_context(|e| { let err_str = std::str::from_utf8(&output.stderr).unwrap_or(""); @@ -185,6 +233,10 @@ fn write_file(path: &Path, front_matter: FrontMatter, body: &str) -> std::io::Re Ok(()) } +fn is_section_index_path(path: &Path) -> bool { + path.file_name() == Some(OsStr::new("_index.md")) +} + lazy_static! { // Matches GitHub usernames. static ref RE_GITHUB: Regex = Regex::new(r"^([^()<>,@]+) \(@([a-zA-Z\d-]+)\)$").unwrap(); @@ -232,7 +284,453 @@ fn extract_authors(value: &str) -> Result, Whatever> { Ok(authors) } -pub fn preprocess(root_path: &Path) -> Result<(), Whatever> { +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ProposalAssetPathResolution { + NotAProposalAsset, + ProposalAsset(ProposalAssetPath), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ProposalAssetPath { + pub(crate) target_proposal_number: ProposalNumber, + pub(crate) content_relative_asset_path: PathBuf, + pub(crate) asset_relative_path: PathBuf, + pub(crate) kind: ProposalAssetKind, + pub(crate) rendered_target_path: String, +} + +#[derive(Debug)] +struct DecodedPathSegment { + value: String, +} + +pub(crate) fn resolve_proposal_asset_path( + content_root: &Path, + source_md_path: &Path, + iri_path: &str, +) -> Result { + let source_parent = source_md_path.parent().with_whatever_context(|| { + format!( + "source markdown path `{}` has no parent", + source_md_path.to_string_lossy() + ) + })?; + let normalized_root = normalize_path_lexically(content_root); + if !raw_iri_resolves_to_proposal_asset_candidate( + &normalized_root, + content_root, + source_parent, + iri_path, + ) { + return Ok(ProposalAssetPathResolution::NotAProposalAsset); + } + + let decoded_segments = decode_iri_path_segments(iri_path)?; + reject_unsafe_asset_segments(&decoded_segments)?; + + let target_path = + resolve_url_path_lexically(content_root, source_parent, iri_path, &decoded_segments); + let normalized_target = normalize_path_lexically(&target_path); + let Ok(content_relative_path) = normalized_target.strip_prefix(&normalized_root) else { + return Ok(ProposalAssetPathResolution::NotAProposalAsset); + }; + + let Some((target_proposal_number, asset_relative_path)) = + proposal_asset_parts(content_relative_path) + else { + return Ok(ProposalAssetPathResolution::NotAProposalAsset); + }; + + let kind = if iri_path.ends_with(".md") { + ProposalAssetKind::Markdown + } else { + ProposalAssetKind::Static + }; + let rendered_target_path = + rendered_asset_path(target_proposal_number, &asset_relative_path, kind)?; + + Ok(ProposalAssetPathResolution::ProposalAsset( + ProposalAssetPath { + target_proposal_number, + content_relative_asset_path: content_relative_path.to_path_buf(), + asset_relative_path, + kind, + rendered_target_path, + }, + )) +} + +pub(crate) fn absolute_rendered_path_for_content_path( + content_relative_path: &Path, +) -> Result { + if content_relative_path == Path::new("_index.md") { + return Ok("/".to_owned()); + } + + if let Some(proposal_number) = proposal_number_from_content_markdown_path(content_relative_path) + { + return Ok(format!("/{proposal_number}/")); + } + + if let Some((proposal_number, asset_relative_path)) = + proposal_asset_parts(content_relative_path) + { + return rendered_asset_path( + proposal_number, + &asset_relative_path, + ProposalAssetKind::from_path(&asset_relative_path), + ); + } + + snafu::whatever!( + "content path `{}` is not a proposal page or proposal asset", + content_relative_path.to_string_lossy() + ); +} + +pub(crate) fn relative_url_from_rendered_paths( + source_rendered_path: &str, + target_rendered_path: &str, +) -> Result { + let source_segments = rendered_directory_segments(source_rendered_path)?; + let (target_segments, target_is_directory) = rendered_path_segments(target_rendered_path)?; + let common_len = source_segments + .iter() + .zip(target_segments.iter()) + .take_while(|(source, target)| source == target) + .count(); + + let mut relative_segments = Vec::new(); + relative_segments.extend(std::iter::repeat_n( + "..", + source_segments.len() - common_len, + )); + relative_segments.extend(target_segments[common_len..].iter().copied()); + + let mut relative_url = if relative_segments.is_empty() { + ".".to_owned() + } else { + relative_segments.join("/") + }; + if target_is_directory && !relative_url.ends_with('/') { + relative_url.push('/'); + } + + Ok(relative_url) +} + +pub(crate) fn proposal_asset_exists_in_content_tree( + content_root: &Path, + content_relative_asset_path: &Path, +) -> bool { + if !content_relative_asset_path + .components() + .all(|component| matches!(component, Component::Normal(_))) + { + return false; + } + + let Ok(canonical_content_root) = std::fs::canonicalize(content_root) else { + return false; + }; + let Ok(canonical_target) = + std::fs::canonicalize(content_root.join(content_relative_asset_path)) + else { + return false; + }; + + if !canonical_target.starts_with(canonical_content_root) { + return false; + } + + std::fs::metadata(canonical_target) + .map(|metadata| metadata.is_file()) + .unwrap_or(false) +} + +fn decode_iri_path_segments(iri_path: &str) -> Result, Whatever> { + iri_path + .split('/') + .map(|segment| { + Ok(DecodedPathSegment { + value: percent_decode_url_segment(segment)?, + }) + }) + .collect::, _>>() +} + +fn percent_decode_url_segment(segment: &str) -> Result { + let bytes = segment.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut index = 0; + + while index < bytes.len() { + if bytes[index] != b'%' { + decoded.push(bytes[index]); + index += 1; + continue; + } + + if index + 2 >= bytes.len() { + snafu::whatever!("invalid percent encoding in URL path segment `{segment}`"); + } + + let high = hex_value(bytes[index + 1]).with_whatever_context(|| { + format!("invalid percent encoding in URL path segment `{segment}`") + })?; + let low = hex_value(bytes[index + 2]).with_whatever_context(|| { + format!("invalid percent encoding in URL path segment `{segment}`") + })?; + let value = (high << 4) | low; + if matches!(value, b'/' | b'\\' | b'\0') { + snafu::whatever!("unsafe percent encoding in URL path segment `{segment}`"); + } + decoded.push(value); + index += 3; + } + + String::from_utf8(decoded) + .with_whatever_context(|_| format!("URL path segment `{segment}` is not UTF-8")) +} + +fn hex_value(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + b'A'..=b'F' => Some(byte - b'A' + 10), + _ => None, + } +} + +fn reject_unsafe_asset_segments(segments: &[DecodedPathSegment]) -> Result<(), Whatever> { + let Some(assets_index) = segments.windows(2).position(|window| { + path_component_proposal_number(Some(OsStr::new(window[0].value.as_str()))).is_some() + && window[1].value == "assets" + }) else { + return Ok(()); + }; + + for segment in &segments[assets_index + 2..] { + if segment.value.is_empty() { + continue; + } + if segment.value == "." || segment.value == ".." { + snafu::whatever!("unsafe proposal asset path segment `{}`", segment.value); + } + if segment.value.contains(['/', '\\', '\0']) { + snafu::whatever!("unsafe proposal asset path segment `{}`", segment.value); + } + } + + Ok(()) +} + +fn resolve_url_path_lexically( + content_root: &Path, + source_parent: &Path, + iri_path: &str, + decoded_segments: &[DecodedPathSegment], +) -> PathBuf { + let mut path = if iri_path.starts_with('/') { + content_root.to_path_buf() + } else { + source_parent.to_path_buf() + }; + + for segment in decoded_segments { + match segment.value.as_str() { + "" | "." => {} + ".." => { + path.pop(); + } + _ => path.push(&segment.value), + } + } + + path +} + +fn raw_iri_resolves_to_proposal_asset_candidate( + normalized_root: &Path, + content_root: &Path, + source_parent: &Path, + iri_path: &str, +) -> bool { + let mut path = if iri_path.starts_with('/') { + content_root.to_path_buf() + } else { + source_parent.to_path_buf() + }; + let raw_segments = iri_path.split('/').collect::>(); + + for (index, segment) in raw_segments.iter().enumerate() { + match *segment { + "" | "." => {} + ".." => { + path.pop(); + } + _ => path.push(segment), + } + + let normalized_path = normalize_path_lexically(&path); + let Ok(content_relative_path) = normalized_path.strip_prefix(normalized_root) else { + continue; + }; + + if proposal_asset_parts(content_relative_path).is_some() { + return true; + } + + if proposal_asset_dir_prefix(content_relative_path).is_some() + && raw_segments[index + 1..] + .iter() + .any(|remaining_segment| !remaining_segment.is_empty()) + { + return true; + } + } + + false +} + +fn normalize_path_lexically(path: &Path) -> PathBuf { + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), + Component::RootDir => normalized.push(component.as_os_str()), + Component::CurDir => {} + Component::ParentDir => { + normalized.pop(); + } + Component::Normal(part) => normalized.push(part), + } + } + normalized +} + +fn proposal_asset_dir_prefix(content_relative_path: &Path) -> Option { + let mut components = content_relative_path.components(); + let proposal_component = components.next()?; + let assets_component = components.next()?; + if components.next().is_some() || assets_component.as_os_str() != OsStr::new("assets") { + return None; + } + + path_component_proposal_number(Some(proposal_component.as_os_str())) +} + +fn proposal_asset_parts(content_relative_path: &Path) -> Option<(ProposalNumber, PathBuf)> { + let mut components = content_relative_path.components(); + let proposal_component = components.next()?; + let assets_component = components.next()?; + if assets_component.as_os_str() != OsStr::new("assets") { + return None; + } + + let proposal_number = path_component_proposal_number(Some(proposal_component.as_os_str()))?; + let asset_relative_path = components.as_path(); + if asset_relative_path.as_os_str().is_empty() { + return None; + } + if !asset_relative_path + .components() + .all(|component| matches!(component, Component::Normal(_))) + { + return None; + } + + Some((proposal_number, asset_relative_path.to_path_buf())) +} + +fn rendered_asset_path( + proposal_number: ProposalNumber, + asset_relative_path: &Path, + kind: ProposalAssetKind, +) -> Result { + let mut segments = vec![proposal_number.to_string(), "assets".to_owned()]; + let asset_segments = asset_relative_path + .components() + .map(|component| match component { + Component::Normal(part) => { + part.to_str().map(str::to_owned).with_whatever_context(|| { + format!( + "non-UTF-8 proposal asset path `{}`", + asset_relative_path.to_string_lossy() + ) + }) + } + _ => snafu::whatever!( + "unsupported proposal asset path component in `{}`", + asset_relative_path.to_string_lossy() + ), + }) + .collect::, _>>()?; + + let last_index = asset_segments.len().saturating_sub(1); + for (index, mut segment) in asset_segments.into_iter().enumerate() { + if kind == ProposalAssetKind::Markdown && index == last_index { + segment = segment + .strip_suffix(".md") + .with_whatever_context(|| { + format!( + "proposal asset markdown path `{}` does not end in `.md`", + asset_relative_path.to_string_lossy() + ) + })? + .to_owned(); + } + segments.push(percent_encode_url_segment(&segment)); + } + + let mut rendered_path = format!("/{}", segments.join("/")); + if kind == ProposalAssetKind::Markdown { + rendered_path.push('/'); + } + + Ok(rendered_path) +} + +fn percent_encode_url_segment(segment: &str) -> String { + let mut encoded = String::with_capacity(segment.len()); + for byte in segment.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +fn rendered_directory_segments(rendered_path: &str) -> Result, Whatever> { + let (mut segments, is_directory) = rendered_path_segments(rendered_path)?; + if !is_directory { + segments.pop(); + } + Ok(segments) +} + +fn rendered_path_segments(rendered_path: &str) -> Result<(Vec<&str>, bool), Whatever> { + if !rendered_path.starts_with('/') { + snafu::whatever!("rendered path `{rendered_path}` is not absolute"); + } + + let is_directory = rendered_path.ends_with('/'); + let segments = rendered_path + .trim_matches('/') + .split('/') + .filter(|segment| !segment.is_empty()) + .collect(); + + Ok((segments, is_directory)) +} + +fn is_generated_zola_markdown(contents: &str) -> bool { + contents.starts_with("+++\n") +} + +pub fn preprocess(root_path: &Path, only_plan: Option<&OnlyRenderPlan>) -> Result<(), Whatever> { let dir = std::fs::read_dir(root_path).with_whatever_context(|_| { format!("could not read directory `{}`", root_path.to_string_lossy()) })?; @@ -268,16 +766,109 @@ pub fn preprocess(root_path: &Path) -> Result<(), Whatever> { } if file_type.is_dir() { - process_eip(root_path, &entry_path.join("index.md"))?; - process_assets(root_path, &entry_path)?; + let relative_path = entry_path + .strip_prefix(root_path) + .with_whatever_context(|_| { + format!( + "content directory entry `{}` is outside `{}`", + entry_path.to_string_lossy(), + root_path.to_string_lossy() + ) + })?; + if only_plan + .map(|plan| plan.should_process_proposal_dir(relative_path)) + .unwrap_or(true) + { + let index_path = entry_path.join("index.md"); + if let Some(plan) = only_plan { + let index_relative_path = relative_path.join("index.md"); + if plan.should_preprocess_markdown(&index_relative_path) { + process_eip(root_path, &index_path, only_plan, MissingPathMode::Error)?; + } + } else { + process_eip(root_path, &index_path, only_plan, MissingPathMode::Error)?; + } + process_assets(root_path, &entry_path, only_plan, MissingPathMode::Error)?; + } } else if entry_path.extension().and_then(OsStr::to_str) == Some("md") { - process_eip(root_path, &entry_path)?; + let relative_path = entry_path + .strip_prefix(root_path) + .with_whatever_context(|_| { + format!( + "content file `{}` is outside `{}`", + entry_path.to_string_lossy(), + root_path.to_string_lossy() + ) + })?; + if only_plan + .map(|plan| plan.should_preprocess_markdown(relative_path)) + .unwrap_or(true) + { + process_eip(root_path, &entry_path, only_plan, MissingPathMode::Error)?; + } } } Ok(()) } +pub fn preprocess_paths( + root_path: &Path, + relative_paths: &BTreeSet, + only_plan: Option<&OnlyRenderPlan>, +) -> Result<(), Whatever> { + let mut eips = BTreeSet::new(); + let mut asset_dirs = BTreeSet::new(); + + for relative_path in relative_paths { + let Ok(content_relative_path) = relative_path.strip_prefix("content") else { + continue; + }; + + if content_relative_path.as_os_str().is_empty() { + continue; + } + + if content_relative_path.extension().and_then(OsStr::to_str) != Some("md") { + continue; + } + + if only_plan + .map(|plan| !plan.should_sync_dirty_path(relative_path)) + .unwrap_or(false) + { + continue; + } + + let mut components = content_relative_path.components(); + let Some(first_component) = components.next() else { + continue; + }; + + if matches!( + components.next(), + Some(component) if component.as_os_str() == OsStr::new("assets") + ) { + let proposal_dir = root_path.join(first_component.as_os_str()); + asset_dirs.insert(proposal_dir); + continue; + } + + let path = root_path.join(content_relative_path); + eips.insert(path); + } + + for path in eips { + process_eip(root_path, &path, only_plan, MissingPathMode::Ignore)?; + } + + for path in asset_dirs { + process_assets(root_path, &path, only_plan, MissingPathMode::Ignore)?; + } + + Ok(()) +} + fn path_to_at(root: &Path, parent: &Path, input: &str) -> Result { let croot = std::fs::canonicalize(root).with_whatever_context(|_| { format!("could not canonicalize `{}`", root.to_string_lossy()) @@ -333,9 +924,108 @@ fn canonicalize_md(path: &Path) -> Result { }) } +enum AssetLinkRewrite { + Rewrite(String), + FallThrough, +} + +fn resolve_asset_link_rewrite( + root: &Path, + source_md_path: &Path, + only_plan: Option<&OnlyRenderPlan>, + iri_path: &str, +) -> Result, Whatever> { + if iri_path.is_empty() { + return Ok(None); + } + + let ProposalAssetPathResolution::ProposalAsset(asset_path) = + resolve_proposal_asset_path(root, source_md_path, iri_path)? + else { + return Ok(None); + }; + + if let Some(plan) = only_plan { + if let Some(public_url) = + plan.public_url_for_omitted_proposal_asset(&asset_path.content_relative_asset_path) + { + return Ok(Some(AssetLinkRewrite::Rewrite(public_url))); + } + + if plan.has_proposal_asset(&asset_path.content_relative_asset_path) { + validate_local_proposal_asset(root, source_md_path, iri_path, &asset_path)?; + return local_asset_link_rewrite(root, source_md_path, asset_path).map(Some); + } + + snafu::whatever!( + "proposal asset link `{iri_path}` in `{}` resolved to `{}` but was not found in targeted render inventory", + source_md_path.to_string_lossy(), + asset_path.content_relative_asset_path.to_string_lossy() + ); + } + + validate_local_proposal_asset(root, source_md_path, iri_path, &asset_path)?; + local_asset_link_rewrite(root, source_md_path, asset_path).map(Some) +} + +fn validate_local_proposal_asset( + root: &Path, + source_md_path: &Path, + iri_path: &str, + asset_path: &ProposalAssetPath, +) -> Result<(), Whatever> { + if proposal_asset_exists_in_content_tree(root, &asset_path.content_relative_asset_path) { + return Ok(()); + } + + snafu::whatever!( + "proposal asset link `{iri_path}` in `{}` resolved to missing asset `{}`", + source_md_path.to_string_lossy(), + asset_path.content_relative_asset_path.to_string_lossy() + ); +} + +fn local_asset_link_rewrite( + root: &Path, + source_md_path: &Path, + asset_path: ProposalAssetPath, +) -> Result { + if asset_path.kind == ProposalAssetKind::Markdown { + return Ok(AssetLinkRewrite::FallThrough); + } + + let source_relative_path = source_md_path + .strip_prefix(root) + .with_whatever_context(|_| { + format!( + "source markdown `{}` is outside content root `{}`", + source_md_path.to_string_lossy(), + root.to_string_lossy() + ) + })?; + let source_rendered_path = absolute_rendered_path_for_content_path(source_relative_path)?; + let relative_url = + relative_url_from_rendered_paths(&source_rendered_path, &asset_path.rendered_target_path)?; + + Ok(AssetLinkRewrite::Rewrite(relative_url)) +} + +fn append_query_and_fragment(mut url: String, iri_ref: &IriRefBuf) -> String { + if let Some(query) = iri_ref.query() { + url.push('?'); + url.push_str(query.as_str()); + } + if let Some(fragment) = iri_ref.fragment() { + url.push('#'); + url.push_str(fragment.as_str()); + } + url +} + fn fix_links<'a, 'b>( root: &'a Path, - parent: &'a Path, + source_md_path: &'a Path, + only_plan: Option<&'a OnlyRenderPlan>, mut e: Event<'b>, ) -> Result, Whatever> { match &mut e { @@ -344,16 +1034,67 @@ fn fix_links<'a, 'b>( .map_err(|e| e.to_string()) .whatever_context("invalid URL in image/link")?; - if iri_ref.authority().is_some() { + if iri_ref.scheme().is_some() || iri_ref.authority().is_some() { // Is a protocol-relative or absolute URL. return Ok(e); } + let iri_path: &str = iri_ref.path().as_ref(); + match resolve_asset_link_rewrite(root, source_md_path, only_plan, iri_path)? { + Some(AssetLinkRewrite::Rewrite(url)) => { + *dest_url = CowStr::from(append_query_and_fragment(url, &iri_ref)); + return Ok(e); + } + Some(AssetLinkRewrite::FallThrough) | None => {} + } + if !iri_ref.path().ends_with(".md") { // Only markdown files need the `@` syntax. return Ok(e); } + let parent = source_md_path.parent().with_whatever_context(|| { + format!( + "source markdown path `{}` has no parent", + source_md_path.to_string_lossy() + ) + })?; + let child = if iri_path.starts_with("/") { + let mut path = Path::new(iri_path); + path = path.strip_prefix("/").unwrap(); + root.join(path) + } else { + parent.join(Path::new(iri_path)) + }; + let normalized_root = normalize_path_lexically(root); + let normalized_child = normalize_path_lexically(&child); + if let Some(public_url) = only_plan.and_then(|plan| { + normalized_child + .strip_prefix(&normalized_root) + .ok() + .and_then(|relative_path| plan.external_url_for_content_target(relative_path)) + }) { + let mut external_url = public_url.to_owned(); + if let Some(query) = iri_ref.query() { + external_url.push('?'); + external_url.push_str(query.as_str()); + } + if let Some(fragment) = iri_ref.fragment() { + external_url.push('#'); + external_url.push_str(fragment.as_str()); + } + *dest_url = CowStr::from(external_url); + return Ok(e); + } + let canonicalized = canonicalize_md(&child)?; + if let Some(public_url) = + only_plan.and_then(|plan| plan.external_url_for_canonical_target(&canonicalized)) + { + *dest_url = + CowStr::from(append_query_and_fragment(public_url.to_owned(), &iri_ref)); + return Ok(e); + } + let canonicalized = path_to_at(root, parent, iri_ref.path())?; let path = iref::iri::Path::new(&canonicalized).expect("path is valid IRI"); iri_ref.set_path(path); @@ -429,7 +1170,12 @@ impl RenderCsl { } } -fn transform_markdown(root: &Path, path: &Path, body: &str) -> Result { +fn transform_markdown( + root: &Path, + path: &Path, + body: &str, + only_plan: Option<&OnlyRenderPlan>, +) -> Result { let mut opts = Options::empty(); opts.insert(Options::ENABLE_TABLES); opts.insert(Options::ENABLE_FOOTNOTES); @@ -437,11 +1183,10 @@ fn transform_markdown(root: &Path, path: &Path, body: &str) -> Result csl.render_csl(e).transpose(), err => Some(err), @@ -456,72 +1201,99 @@ fn transform_markdown(root: &Path, path: &Path, body: &str) -> Result Result<(), Whatever> { +fn process_assets( + root: &Path, + path: &Path, + only_plan: Option<&OnlyRenderPlan>, + missing_path_mode: MissingPathMode, +) -> Result<(), Whatever> { let canon_root = std::fs::canonicalize(root).whatever_context("could not canonicalize root")?; - let number_txt = path - .file_name() - .with_whatever_context(|| format!("no file name for `{}`", path.to_string_lossy()))? - .to_str() - .with_whatever_context(|| format!("non-UTF-8 in `{}`", path.to_string_lossy()))?; + let assets_dir = path.join("assets"); - let number: u32 = number_txt.parse().with_whatever_context(|_| { - format!("can't parse number for `{}`", path.to_string_lossy()) - })?; + let mut entries = Vec::new(); + let mut ignored_missing_path = false; - let assets_dir = path.join("assets"); + for entry in WalkDir::new(&assets_dir).follow_links(true).into_iter() { + let entry = match entry { + Ok(entry) => entry, + Err(error) if missing_path_mode.should_ignore_walkdir_error(&error) => { + ignored_missing_path = true; + continue; + } + Err(error) => { + return Err(error).with_whatever_context(|_| { + format!("couldn't read entry in `{}`", assets_dir.to_string_lossy()) + }); + } + }; - let dir = WalkDir::new(&assets_dir) - .follow_links(true) - .into_iter() - .filter(|e| match e { - Ok(f) if !f.file_type().is_file() => false, - Ok(f) => f.path().extension().and_then(OsStr::to_str) == Some("md"), - Err(_) => true, - }) - .filter(|e| { - let f = match e { - Ok(f) => f, - _ => return true, - }; + if !entry.file_type().is_file() { + continue; + } - let candidate = match std::fs::canonicalize(f.path()) { - Ok(c) => c, - Err(e) => { - warn!( - "unable to canonicalize `{}`: {e}", - f.path().to_string_lossy() - ); - return false; - } - }; + if entry.path().extension().and_then(OsStr::to_str) != Some("md") { + continue; + } - let in_root = candidate.starts_with(&canon_root); - if !in_root { + let candidate = match std::fs::canonicalize(entry.path()) { + Ok(c) => c, + Err(e) => { warn!( - "asset `{}` not in root, skipping", - f.path().to_string_lossy() + "unable to canonicalize `{}`: {e}", + entry.path().to_string_lossy() ); + continue; } - in_root - }); - let dirs: Vec<_> = dir.collect(); + }; - for entry in dirs.into_iter().progress_ext("Assets") { - let entry = entry.with_whatever_context(|_| { - format!("couldn't read entry in `{}`", assets_dir.to_string_lossy()) - })?; + let in_root = candidate.starts_with(&canon_root); + if !in_root { + warn!( + "asset `{}` not in root, skipping", + entry.path().to_string_lossy() + ); + continue; + } + + entries.push(entry); + } + + if entries.is_empty() && ignored_missing_path { + return Ok(()); + } + + let number_txt = path + .file_name() + .with_whatever_context(|| format!("no file name for `{}`", path.to_string_lossy()))? + .to_str() + .with_whatever_context(|| format!("non-UTF-8 in `{}`", path.to_string_lossy()))?; + let number: u32 = number_txt.parse().with_whatever_context(|_| { + format!("can't parse number for `{}`", path.to_string_lossy()) + })?; + + for entry in entries.into_iter().progress_ext("Assets") { let path = entry.path(); - let contents = read_to_string(path).with_whatever_context(|_| { - format!("could not read file `{}`", path.to_string_lossy()) - })?; + let contents = match read_to_string(path) { + Ok(contents) => contents, + Err(error) if missing_path_mode.should_ignore_io_error(&error) => continue, + Err(error) => { + return Err(error).with_whatever_context(|_| { + format!("could not read file `{}`", path.to_string_lossy()) + }); + } + }; + if is_generated_zola_markdown(&contents) { + continue; + } - let contents = transform_markdown(root, path, &contents).with_whatever_context(|_| { - format!( - "unable to transform markdown for `{}`", - path.to_string_lossy() - ) - })?; + let contents = + transform_markdown(root, path, &contents, only_plan).with_whatever_context(|_| { + format!( + "unable to transform markdown for `{}`", + path.to_string_lossy() + ) + })?; let relative_path = path.strip_prefix(&assets_dir).unwrap(); let relative_path = relative_path.with_file_name(relative_path.file_stem().unwrap()); @@ -550,31 +1322,61 @@ fn process_assets(root: &Path, path: &Path) -> Result<(), Whatever> { ..Default::default() }; - write_file(path, front_matter, &contents).whatever_context("couldn't write file")?; + match write_file(path, front_matter, &contents) { + Ok(()) => {} + Err(error) if missing_path_mode.should_ignore_io_error(&error) => continue, + Err(error) => return Err(error).whatever_context("couldn't write file"), + } } Ok(()) } -fn process_eip(root: &Path, path: &Path) -> Result<(), Whatever> { +fn process_eip( + root: &Path, + path: &Path, + only_plan: Option<&OnlyRenderPlan>, + missing_path_mode: MissingPathMode, +) -> Result<(), Whatever> { let path_lossy = path.to_string_lossy(); - let contents = read_to_string(path) - .with_whatever_context(|_| format!("could not read file `{}`", path_lossy))?; + let contents = match read_to_string(path) { + Ok(contents) => contents, + Err(error) if missing_path_mode.should_ignore_io_error(&error) => return Ok(()), + Err(error) => { + return Err(error) + .with_whatever_context(|_| format!("could not read file `{}`", path_lossy)); + } + }; + if is_generated_zola_markdown(&contents) { + return Ok(()); + } let (preamble, body) = Preamble::split(&contents) .with_whatever_context(|_| format!("couldn't split preamble for `{}`", path_lossy))?; - let body = transform_markdown(root, path, body) + let body = transform_markdown(root, path, body, only_plan) .with_whatever_context(|_| format!("unable to transform markdown for `{path_lossy}`"))?; + if is_section_index_path(path) { + let front_matter = + serde_yaml::from_str::(preamble).with_whatever_context(|_| { + format!("couldn't parse section front matter in `{}`", path_lossy) + })?; + + match write_file(path, front_matter, &body) { + Ok(()) => {} + Err(error) if missing_path_mode.should_ignore_io_error(&error) => return Ok(()), + Err(error) => return Err(error).whatever_context("couldn't write file"), + } + + return Ok(()); + } + let preamble = Preamble::parse(Some(&path_lossy), preamble) .ok() .with_whatever_context(|| format!("couldn't parse preamble in `{}`", path_lossy))?; - let updated = match path.file_name() { - Some(x) if x == "_index.md" => None, - _ => Some(last_modified(path)?), - }; + let updated = Some(last_modified(path)?); let mut front_matter = FrontMatter { updated, @@ -654,8 +1456,24 @@ fn process_eip(root: &Path, path: &Path) -> Result<(), Whatever> { .whatever_context("could not parse requires")? .into_iter() .map(|eip| { - let path = format!("/{eip:0>5}.md"); - path_to_at(root, root, &path) + let proposal_number = match ProposalNumber::from_u32(eip) { + Ok(proposal_number) => proposal_number, + Err(()) => snafu::whatever!("could not parse requires"), + }; + match only_plan { + Some(plan) => { + match plan.reference_for_required_number(proposal_number)? { + ProposalReference::Internal(path) => Ok(path), + ProposalReference::External(public_url) => { + Ok(public_url.to_owned()) + } + } + } + None => { + let path = format!("/{eip:0>5}.md"); + path_to_at(root, root, &path) + } + } }) .collect::>()?; front_matter @@ -669,7 +1487,1069 @@ fn process_eip(root: &Path, path: &Path) -> Result<(), Whatever> { } } - write_file(Path::new(&path), front_matter, &body).whatever_context("couldn't write file")?; + match write_file(path, front_matter, &body) { + Ok(()) => {} + Err(error) if missing_path_mode.should_ignore_io_error(&error) => return Ok(()), + Err(error) => return Err(error).whatever_context("couldn't write file"), + } Ok(()) } + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + use std::path::{Path, PathBuf}; + + use git2::{IndexAddOption, Repository, Signature}; + use snafu::Report; + use tempfile::TempDir; + use toml::Value as TomlValue; + + use super::{ + absolute_rendered_path_for_content_path, preprocess, preprocess_paths, + proposal_asset_exists_in_content_tree, relative_url_from_rendered_paths, + resolve_proposal_asset_path, ProposalAssetPath, ProposalAssetPathResolution, + }; + use crate::proposal::ProposalAssetKind; + use crate::proposal::{OnlyRenderPlan, ProposalNumber}; + + fn number(value: u32) -> ProposalNumber { + ProposalNumber::from_u32(value).unwrap() + } + + fn write_file(root: &Path, relative: &str, contents: impl AsRef) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents.as_ref()).unwrap(); + } + + fn commit_all(repo: &Repository) { + let mut index = repo.index().unwrap(); + index + .add_all(["content"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + + repo.commit(Some("HEAD"), &signature, &signature, "initial", &tree, &[]) + .unwrap(); + } + + fn content_repo(files: &[(&str, String)]) -> (TempDir, PathBuf) { + let temp = TempDir::new().unwrap(); + let repo_root = temp.path().join("repo"); + let content_root = repo_root.join("content"); + std::fs::create_dir_all(&content_root).unwrap(); + let repo = Repository::init(&repo_root).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + + for (relative, contents) in files { + write_file(&content_root, relative, contents); + } + + commit_all(&repo); + (temp, content_root) + } + + fn proposal_markdown( + proposal_number: u32, + category: Option<&str>, + extra_preamble: &str, + body: &str, + ) -> String { + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!( + "---\neip: {proposal_number}\ntitle: Proposal {proposal_number}\n{category}{extra_preamble}---\n{body}\n" + ) + } + + fn only_plan(content_root: &Path, selected: &[u32]) -> OnlyRenderPlan { + let selected = selected + .iter() + .copied() + .map(number) + .collect::>(); + OnlyRenderPlan::build(content_root, selected).unwrap() + } + + fn repo_paths(paths: &[&str]) -> BTreeSet { + paths.iter().map(PathBuf::from).collect() + } + + fn rendered_body(path: &Path) -> String { + let contents = std::fs::read_to_string(path).unwrap(); + contents.split_once("\n+++\n").unwrap().1.to_owned() + } + + fn rendered_front_matter(path: &Path) -> TomlValue { + let contents = std::fs::read_to_string(path).unwrap(); + let front_matter = contents + .strip_prefix("+++\n") + .unwrap() + .split_once("\n+++\n") + .unwrap() + .0; + toml::from_str(front_matter).unwrap() + } + + fn resolved_asset( + content_root: &Path, + source_md_path: &Path, + iri_path: &str, + ) -> ProposalAssetPath { + match resolve_proposal_asset_path(content_root, source_md_path, iri_path).unwrap() { + ProposalAssetPathResolution::ProposalAsset(asset_path) => asset_path, + ProposalAssetPathResolution::NotAProposalAsset => { + panic!("expected `{iri_path}` to resolve as proposal asset") + } + } + } + + #[test] + fn resolver_detects_flat_source_cross_proposal_static_asset() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let asset_path = resolved_asset(&content, &source, "./00678/assets/foo.pdf"); + + assert_eq!(asset_path.target_proposal_number, number(678)); + assert_eq!( + asset_path.content_relative_asset_path, + Path::new("00678/assets/foo.pdf") + ); + assert_eq!(asset_path.asset_relative_path, Path::new("foo.pdf")); + assert_eq!(asset_path.kind, ProposalAssetKind::Static); + assert_eq!(asset_path.rendered_target_path, "/678/assets/foo.pdf"); + } + + #[test] + fn resolver_detects_directory_source_cross_proposal_static_asset() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555/index.md"); + + let asset_path = resolved_asset(&content, &source, "../00678/assets/foo.pdf"); + + assert_eq!( + asset_path.content_relative_asset_path, + Path::new("00678/assets/foo.pdf") + ); + assert_eq!(asset_path.rendered_target_path, "/678/assets/foo.pdf"); + } + + #[test] + fn resolver_detects_asset_markdown_lexically_without_filesystem() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555/assets/guide.md"); + + let asset_path = resolved_asset(&content, &source, "../../00678/assets/guide.md"); + + assert_eq!(asset_path.kind, ProposalAssetKind::Markdown); + assert_eq!( + asset_path.content_relative_asset_path, + Path::new("00678/assets/guide.md") + ); + assert_eq!(asset_path.rendered_target_path, "/678/assets/guide/"); + } + + #[test] + fn resolver_decodes_safe_percent_paths_and_renders_encoded_urls() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let asset_path = resolved_asset( + &content, + &source, + "./00678/assets/Contract%20Interactions%20diagram.svg", + ); + + assert_eq!( + asset_path.content_relative_asset_path, + Path::new("00678/assets/Contract Interactions diagram.svg") + ); + assert_eq!( + asset_path.rendered_target_path, + "/678/assets/Contract%20Interactions%20diagram.svg" + ); + } + + #[test] + fn resolver_rejects_unsafe_percent_and_asset_segments() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + for iri_path in [ + "./00678/assets/foo%2Fbar.pdf", + "./00678/assets/foo%5Cbar.pdf", + "./00678/assets/foo%00bar.pdf", + "./00678/assets/.", + "./00678/assets/..", + "./00678/assets/%2E", + "./00678/assets/%2E%2E", + ] { + let error = resolve_proposal_asset_path(&content, &source, iri_path) + .unwrap_err() + .to_string(); + assert!( + error.contains("unsafe"), + "expected unsafe path error for `{iri_path}`, got `{error}`" + ); + } + } + + #[test] + fn resolver_allows_unsafe_percent_encodings_for_non_proposal_paths() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let resolution = + resolve_proposal_asset_path(&content, &source, "./images/foo%2Fbar.pdf").unwrap(); + + assert_eq!(resolution, ProposalAssetPathResolution::NotAProposalAsset); + } + + #[test] + fn resolver_still_rejects_unsafe_percent_encodings_for_proposal_assets() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let error = resolve_proposal_asset_path(&content, &source, "./00678/assets/foo%2Fbar.pdf") + .unwrap_err() + .to_string(); + + assert!(error.contains("unsafe")); + } + + #[test] + fn resolver_returns_passthrough_for_outside_root_without_error() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let resolution = + resolve_proposal_asset_path(&content, &source, "../elsewhere/foo.pdf").unwrap(); + + assert_eq!(resolution, ProposalAssetPathResolution::NotAProposalAsset); + } + + #[test] + fn resolver_returns_passthrough_for_non_proposal_asset_paths() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let resolution = + resolve_proposal_asset_path(&content, &source, "./images/foo.pdf").unwrap(); + + assert_eq!(resolution, ProposalAssetPathResolution::NotAProposalAsset); + } + + #[test] + fn rendered_path_helper_maps_proposal_and_asset_content_paths() { + for (content_relative_path, expected_rendered_path) in [ + ("_index.md", "/"), + ("00555.md", "/555/"), + ("00555/index.md", "/555/"), + ("00555/assets/guide.md", "/555/assets/guide/"), + ("00555/assets/README.md", "/555/assets/README/"), + ("00555/assets/index.md", "/555/assets/index/"), + ("00678/assets/foo.pdf", "/678/assets/foo.pdf"), + ( + "00678/assets/Contract Interactions diagram.svg", + "/678/assets/Contract%20Interactions%20diagram.svg", + ), + ] { + assert_eq!( + absolute_rendered_path_for_content_path(Path::new(content_relative_path)).unwrap(), + expected_rendered_path + ); + } + } + + #[test] + fn relative_url_helper_uses_rendered_paths() { + assert_eq!( + relative_url_from_rendered_paths("/555/", "/678/assets/foo.pdf").unwrap(), + "../678/assets/foo.pdf" + ); + assert_eq!( + relative_url_from_rendered_paths("/555/assets/guide/", "/678/assets/foo.pdf").unwrap(), + "../../../678/assets/foo.pdf" + ); + assert_eq!( + relative_url_from_rendered_paths("/555/", "/678/assets/guide/").unwrap(), + "../678/assets/guide/" + ); + } + + #[test] + fn filesystem_validator_checks_content_relative_assets() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + write_file(&content, "00678/assets/foo.pdf", ""); + + assert!(proposal_asset_exists_in_content_tree( + &content, + Path::new("00678/assets/foo.pdf") + )); + assert!(!proposal_asset_exists_in_content_tree( + &content, + Path::new("00678/assets/missing.pdf") + )); + assert!(!proposal_asset_exists_in_content_tree( + &content, + Path::new("../00678/assets/foo.pdf") + )); + } + + #[cfg(unix)] + #[test] + fn filesystem_validator_rejects_symlink_targets_outside_content_root() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + write_file(&content, "00678/assets/placeholder", ""); + let outside = temp.path().join("outside.pdf"); + std::fs::write(&outside, "").unwrap(); + std::os::unix::fs::symlink(&outside, content.join("00678/assets/outside.pdf")).unwrap(); + + assert!(!proposal_asset_exists_in_content_tree( + &content, + Path::new("00678/assets/outside.pdf") + )); + } + + #[test] + fn preprocess_rewrites_flat_source_cross_proposal_static_asset() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](./00678/assets/foo.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(../678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_rewrites_directory_source_cross_proposal_static_asset() { + let (_temp, content) = content_repo(&[ + ( + "00555/index.md", + proposal_markdown(555, None, "", "See [asset](../00678/assets/foo.pdf)."), + ), + ("00555/assets/.keep", "".to_owned()), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555/index.md")); + assert!(body.contains("(../678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_rewrites_root_index_source_cross_proposal_static_asset() { + let (_temp, content) = content_repo(&[ + ( + "_index.md", + "---\ntitle: Home\n---\nSee [asset](/00678/assets/foo.pdf).\n".to_owned(), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("_index.md")); + assert!(body.contains("(678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_rewrites_asset_markdown_source_using_rendered_source_path() { + let (_temp, content) = content_repo(&[ + ( + "00555/index.md", + proposal_markdown(555, None, "", "Source."), + ), + ( + "00555/assets/guide.md", + "See [asset](../../00678/assets/foo.pdf).".to_owned(), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555/assets/guide.md")); + assert!(body.contains("(../../../678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_rewrites_source_root_absolute_asset_path() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](/00678/assets/foo.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(../678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_rewrites_cross_proposal_image_links() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "![diagram](./00678/assets/diagram.png)"), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/diagram.png", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("![diagram](../678/assets/diagram.png)")); + } + + #[test] + fn preprocess_preserves_query_and_fragment_on_asset_links() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "", + "See [asset](./00678/assets/foo.pdf?download=1#page=2).", + ), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(../678/assets/foo.pdf?download=1#page=2)")); + } + + #[test] + fn preprocess_decodes_asset_paths_and_keeps_generated_urls_encoded() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "", + "See [asset](./00678/assets/Contract%20Interactions%20diagram.svg).", + ), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ( + "00678/assets/Contract Interactions diagram.svg", + "".to_owned(), + ), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("../678/assets/Contract%20Interactions%20diagram.svg")); + } + + #[test] + fn preprocess_keeps_selected_or_full_asset_markdown_links_on_existing_md_path() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [guide](./00678/assets/guide.md)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/guide.md", "Guide.".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(@/00678/assets/guide.md)")); + } + + #[test] + fn preprocess_keeps_asset_markdown_fragment_links_unchanged() { + let (_temp, content) = content_repo(&[ + ( + "00555/index.md", + proposal_markdown(555, None, "", "Source."), + ), + ( + "00555/assets/guide.md", + "See [heading](#heading).\n\n## Heading\n".to_owned(), + ), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555/assets/guide.md")); + assert!(body.contains("[heading](#heading)")); + } + + #[test] + fn preprocess_keeps_ordinary_proposal_markdown_links_on_existing_path() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [proposal](./00678.md)."), + ), + ("00678.md", proposal_markdown(678, None, "", "Target.")), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(@/00678.md)")); + } + + #[test] + fn targeted_preprocess_rewrites_omitted_static_asset_to_public_url() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](./00678/assets/foo.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(https://eips.ethereum.org/678/assets/foo.pdf)")); + } + + #[test] + fn targeted_preprocess_rewrites_omitted_asset_markdown_to_public_url() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [guide](./00678/assets/guide.md)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/guide.md", "Guide.".to_owned()), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(https://eips.ethereum.org/678/assets/guide/)")); + } + + #[test] + fn targeted_preprocess_rewrites_omitted_readme_and_index_asset_markdown_public_urls() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "", + "See [readme](./00678/assets/README.md) and [index](./00678/assets/index.md).", + ), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/README.md", "Readme.".to_owned()), + ("00678/assets/index.md", "Index.".to_owned()), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(https://eips.ethereum.org/678/assets/README/)")); + assert!(body.contains("(https://eips.ethereum.org/678/assets/index/)")); + } + + #[test] + fn targeted_preprocess_keeps_selected_static_asset_local() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](./00678/assets/foo.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + let plan = only_plan(&content, &[555, 678]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(../678/assets/foo.pdf)")); + assert!(!body.contains("https://eips.ethereum.org/678/assets/foo.pdf")); + } + + #[test] + fn targeted_dirty_preprocess_uses_inventory_after_omitted_target_is_pruned() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](./00678/assets/foo.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + let plan = only_plan(&content, &[555]); + plan.prune_content(&content).unwrap(); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(https://eips.ethereum.org/678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_errors_clearly_for_missing_selected_or_full_asset_target() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](./00678/assets/missing.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/.keep", "".to_owned()), + ]); + + let error = Report::from_error(preprocess(&content, None).unwrap_err()).to_string(); + + assert!(error.contains("proposal asset link")); + assert!(error.contains("00555.md")); + assert!(error.contains("./00678/assets/missing.pdf")); + assert!(error.contains("00678/assets/missing.pdf")); + } + + #[test] + fn preprocess_skips_generated_zola_markdown_files() { + let original = + "+++\ntitle = \"Generated\"\n+++\nSee [asset](./00678/assets/missing.pdf).\n"; + let (_temp, content) = content_repo(&[("00555.md", original.to_owned())]); + + preprocess(&content, None).unwrap(); + + assert_eq!( + std::fs::read_to_string(content.join("00555.md")).unwrap(), + original + ); + } + + #[test] + fn process_assets_skips_only_generated_asset_markdown_file() { + let generated = + "+++\ntitle = \"Generated\"\n+++\nSee [missing](../../00678/assets/missing.pdf).\n"; + let (_temp, content) = content_repo(&[ + ( + "00555/index.md", + proposal_markdown(555, None, "", "Source."), + ), + ("00555/assets/generated.md", generated.to_owned()), + ("00555/assets/fresh.md", "Fresh asset markdown.".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + assert_eq!( + std::fs::read_to_string(content.join("00555/assets/generated.md")).unwrap(), + generated + ); + assert!( + std::fs::read_to_string(content.join("00555/assets/fresh.md")) + .unwrap() + .starts_with("+++\n") + ); + } + + #[test] + fn preprocess_leaves_non_proposal_relative_asset_links_unchanged() { + let (_temp, content) = content_repo(&[( + "00555.md", + proposal_markdown(555, None, "", "See [local](./images/foo.pdf)."), + )]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(./images/foo.pdf)")); + } + + #[test] + fn preprocess_leaves_raw_html_asset_references_unchanged() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "", + r#"asset"#, + ), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains(r#"asset"#)); + } + + #[test] + fn targeted_preprocess_rewrites_selected_body_links_to_unselected_public_urls() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [ERC-678](/00678.md)."), + ), + ( + "00678.md", + proposal_markdown(678, Some("ERC"), "", "Target."), + ), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("https://ercs.ethereum.org/ERCS/erc-678")); + assert!(!body.contains("@/00678.md")); + } + + #[test] + fn targeted_preprocess_paths_rewrites_selected_dirty_markdown_with_plan() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "requires: 155\n", + "See [EIP-155](./00155.md#list-of-chain-id-s).", + ), + ), + ("00155.md", proposal_markdown(155, None, "", "Unselected.")), + ]); + let plan = only_plan(&content, &[555]); + + preprocess_paths(&content, &repo_paths(&["content/00555.md"]), Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + let front_matter = rendered_front_matter(&content.join("00555.md")); + let requires = front_matter["extra"]["requires"].as_array().unwrap(); + assert!(body.contains("https://eips.ethereum.org/EIPS/eip-155#list-of-chain-id-s")); + assert_eq!( + requires[0].as_str().unwrap(), + "https://eips.ethereum.org/EIPS/eip-155" + ); + } + + #[test] + fn targeted_preprocess_paths_ignores_deleted_dirty_markdown() { + let (_temp, content) = + content_repo(&[("00555.md", proposal_markdown(555, None, "", "Selected."))]); + let plan = only_plan(&content, &[555]); + std::fs::remove_file(content.join("00555.md")).unwrap(); + + preprocess_paths(&content, &repo_paths(&["content/00555.md"]), Some(&plan)).unwrap(); + + assert!(!content.join("00555.md").exists()); + } + + #[test] + fn targeted_preprocess_rewrites_retained_non_proposal_links_to_public_urls() { + let (_temp, content) = content_repo(&[ + ( + "_index.md", + "---\ntitle: Home\n---\nSee [EIP-678](/00678.md).\n".to_owned(), + ), + ("00555.md", proposal_markdown(555, None, "", "Selected.")), + ("00678.md", proposal_markdown(678, None, "", "Unselected.")), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("_index.md")); + assert!(body.contains("https://eips.ethereum.org/EIPS/eip-678")); + assert!(!body.contains("@/00678.md")); + } + + #[test] + fn preprocess_preserves_section_extra_front_matter_and_rewrites_body_links() { + let (_temp, content) = content_repo(&[ + ( + "_index.md", + r#"--- +title: Home +extra: + homepage_badges: + - href: https://discord.gg/9FxN6CfaQR + image: https://dcbadge.limes.pink/api/server/9FxN6CfaQR?style=flat + alt: Badge for ERCRef Discord channel +--- +See [EIP-678](/00678.md). +"# + .to_owned(), + ), + ("00555.md", proposal_markdown(555, None, "", "Selected.")), + ("00678.md", proposal_markdown(678, None, "", "Unselected.")), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let front_matter = rendered_front_matter(&content.join("_index.md")); + let badges = front_matter["extra"]["homepage_badges"].as_array().unwrap(); + assert_eq!( + badges[0]["href"].as_str().unwrap(), + "https://discord.gg/9FxN6CfaQR" + ); + let body = rendered_body(&content.join("_index.md")); + assert!(body.contains("https://eips.ethereum.org/EIPS/eip-678")); + assert!(!body.contains("@/00678.md")); + } + + #[test] + fn targeted_preprocess_paths_rewrites_retained_non_proposal_markdown_with_plan() { + let (_temp, content) = content_repo(&[ + ( + "_index.md", + "---\ntitle: Home\n---\nSee [EIP-678](/00678.md).\n".to_owned(), + ), + ("00555.md", proposal_markdown(555, None, "", "Selected.")), + ("00678.md", proposal_markdown(678, None, "", "Unselected.")), + ]); + let plan = only_plan(&content, &[555]); + + preprocess_paths(&content, &repo_paths(&["content/_index.md"]), Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("_index.md")); + assert!(body.contains("https://eips.ethereum.org/EIPS/eip-678")); + assert!(!body.contains("@/00678.md")); + } + + #[test] + fn targeted_preprocess_paths_rewrites_selected_asset_markdown_with_plan() { + let (_temp, content) = content_repo(&[ + ("00555.md", proposal_markdown(555, None, "", "Selected.")), + ( + "00555/assets/guide.md", + "See [EIP-678](/00678.md).\n".to_owned(), + ), + ("00555/assets/diagram.png", "image\n".to_owned()), + ( + "00678.md", + proposal_markdown(678, Some("ERC"), "", "Unselected."), + ), + ]); + let plan = only_plan(&content, &[555]); + + preprocess_paths( + &content, + &repo_paths(&["content/00555/assets/guide.md"]), + Some(&plan), + ) + .unwrap(); + + let body = rendered_body(&content.join("00555/assets/guide.md")); + assert!(body.contains("https://ercs.ethereum.org/ERCS/erc-678")); + assert_eq!( + std::fs::read_to_string(content.join("00555/assets/diagram.png")).unwrap(), + "image\n" + ); + } + + #[test] + fn targeted_preprocess_paths_ignores_deleted_dirty_asset_dir() { + let (_temp, content) = content_repo(&[ + ("00555.md", proposal_markdown(555, None, "", "Selected.")), + ( + "00555/assets/guide.md", + "See [EIP-678](/00678.md).\n".to_owned(), + ), + ("00678.md", proposal_markdown(678, None, "", "Unselected.")), + ]); + let plan = only_plan(&content, &[555]); + std::fs::remove_dir_all(content.join("00555/assets")).unwrap(); + + preprocess_paths( + &content, + &repo_paths(&["content/00555/assets/guide.md"]), + Some(&plan), + ) + .unwrap(); + + assert!(!content.join("00555/assets").exists()); + } + + #[test] + fn targeted_preprocess_preserves_query_and_fragment_on_external_links() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "", + "See [Fragment](./00155.md#list-of-chain-id-s).\nSee [Query](./00155.md?foo=bar#list-of-chain-id-s).", + ), + ), + ( + "00155.md", + proposal_markdown(155, None, "", "Unselected."), + ), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("https://eips.ethereum.org/EIPS/eip-155#list-of-chain-id-s")); + assert!(body.contains("https://eips.ethereum.org/EIPS/eip-155?foo=bar#list-of-chain-id-s")); + } + + #[test] + fn targeted_preprocess_rewrites_requires_to_unselected_public_urls() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "requires: 678\n", "Selected."), + ), + ( + "00678.md", + proposal_markdown(678, Some("ERC"), "", "Target."), + ), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let front_matter = rendered_front_matter(&content.join("00555.md")); + let requires = front_matter["extra"]["requires"].as_array().unwrap(); + assert_eq!( + requires[0].as_str().unwrap(), + "https://ercs.ethereum.org/ERCS/erc-678" + ); + } + + #[test] + fn targeted_preprocess_keeps_internal_references_between_selected_proposals() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "requires: 678\n", "See [EIP-678](/00678.md)."), + ), + ( + "00678.md", + proposal_markdown(678, Some("ERC"), "", "Target."), + ), + ]); + let plan = only_plan(&content, &[555, 678]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + let front_matter = rendered_front_matter(&content.join("00555.md")); + let requires = front_matter["extra"]["requires"].as_array().unwrap(); + assert!(body.contains("@/00678.md")); + assert_eq!(requires[0].as_str().unwrap(), "@/00678.md"); + } + + #[test] + fn targeted_preprocess_does_not_mask_missing_body_link_targets() { + let (_temp, content) = content_repo(&[( + "00555.md", + proposal_markdown(555, None, "", "See [Missing](/00678.md)."), + )]); + let plan = only_plan(&content, &[555]); + + let error = Report::from_error(preprocess(&content, Some(&plan)).unwrap_err()).to_string(); + + assert!(error.contains("could not canonicalize")); + assert!(error.contains("00678.md")); + } +} diff --git a/src/network_upgrades.rs b/src/network_upgrades.rs new file mode 100644 index 0000000..8da8990 --- /dev/null +++ b/src/network_upgrades.rs @@ -0,0 +1,1704 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Network upgrade source selection and in-memory membership model. + +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt, + path::Path, +}; + +use chrono::{Datelike, NaiveDate}; +use lazy_static::lazy_static; +use log::warn; +use pulldown_cmark::{CowStr, Event, HeadingLevel, Options, Parser, Tag, TagEnd}; +use regex::Regex; +use snafu::{OptionExt, ResultExt, Whatever}; + +use crate::{ + proposal::{flat_proposal_number, path_component_proposal_number, ProposalNumber}, + proposal_catalog::{ + ProposalCatalog, ProposalCatalogPrefix, ProposalCatalogRecord, + ProposalCatalogSourceDocument, + }, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NetworkUpgradeIndex { + pub(crate) upgrades: Vec, + pub(crate) memberships_by_proposal: BTreeMap>, + pub(crate) selected_sources: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NetworkUpgrade { + pub(crate) display_name: String, + pub(crate) slug: String, + pub(crate) source_meta_eip: ProposalNumber, + pub(crate) source_bucket: NetworkUpgradeSourceBucket, + pub(crate) meta_url: String, + pub(crate) meta_status: String, + pub(crate) stages: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NetworkUpgradeStage { + pub(crate) key: String, + pub(crate) label: String, + pub(crate) rows: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NetworkUpgradeMemberRow { + pub(crate) number: ProposalNumber, + pub(crate) title: String, + pub(crate) status: String, + pub(crate) proposal_type: String, + pub(crate) category: Option, + pub(crate) url: String, + pub(crate) subgroup: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NetworkUpgradeMembership { + pub(crate) display_name: String, + pub(crate) slug: String, + pub(crate) source_meta_eip: ProposalNumber, + pub(crate) meta_url: String, + pub(crate) stage_key: String, + pub(crate) stage_label: String, + pub(crate) subgroup: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum NetworkUpgradeSourceBucket { + Marked, + Transitional, + Permanent, +} + +impl fmt::Display for NetworkUpgradeSourceBucket { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Marked => formatter.write_str("marked modern source"), + Self::Transitional => formatter.write_str("transitional modern registry"), + Self::Permanent => formatter.write_str("permanent registry"), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum NetworkUpgradeParserMode { + ModernStages, + #[allow(dead_code)] + EmptyMembers, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NetworkUpgradeRegistrySource { + pub(crate) bucket: NetworkUpgradeSourceBucket, + pub(crate) source_meta_eip: u32, + pub(crate) parser_mode: NetworkUpgradeParserMode, + pub(crate) upgrades: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NetworkUpgradeRegistryUpgrade { + pub(crate) display_name: &'static str, + pub(crate) slug: Option<&'static str>, + pub(crate) sort_order: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct MarkedNetworkUpgradeSource { + pub(crate) source_meta_eip: ProposalNumber, + pub(crate) display_name: String, + pub(crate) slug: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NetworkUpgradeSelectedSource { + pub(crate) source_meta_eip: ProposalNumber, + pub(crate) source_bucket: NetworkUpgradeSourceBucket, + pub(crate) display_name: String, + pub(crate) slug: String, + pub(crate) parser_mode: NetworkUpgradeParserMode, + pub(crate) sort_order: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct SelectedNetworkUpgradeSource { + details: NetworkUpgradeSelectedSource, + render_sort_key: RenderSortKey, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct RenderSortKey { + value: i64, + source_meta_eip: ProposalNumber, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum NetworkUpgradeStageKey { + Included, + Scheduled, + Considered, + Proposed, + Declined, +} + +impl NetworkUpgradeStageKey { + fn stable_key(self) -> &'static str { + match self { + Self::Included => "included", + Self::Scheduled => "scheduled", + Self::Considered => "considered", + Self::Proposed => "proposed", + Self::Declined => "declined", + } + } + + fn render_rank(self) -> u8 { + match self { + Self::Included => 0, + Self::Scheduled => 10, + Self::Considered => 20, + Self::Proposed => 30, + Self::Declined => 40, + } + } +} + +impl fmt::Display for NetworkUpgradeStageKey { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Included => formatter.write_str("Included"), + Self::Scheduled => formatter.write_str("Scheduled for Inclusion"), + Self::Considered => formatter.write_str("Considered for Inclusion"), + Self::Proposed => formatter.write_str("Proposed for Inclusion"), + Self::Declined => formatter.write_str("Declined for Inclusion"), + } + } +} + +#[derive(Debug, Clone)] +struct ParsedNetworkUpgradeStage { + key: NetworkUpgradeStageKey, + members: Vec, +} + +#[derive(Debug, Clone)] +struct ParsedNetworkUpgradeMember { + number: ProposalNumber, + subgroup: Option, +} + +#[derive(Debug, Clone)] +struct ActiveStage { + key: NetworkUpgradeStageKey, + level: u8, + subgroup: Option, +} + +#[derive(Debug)] +struct HeadingCapture { + level: HeadingLevel, + text: String, +} + +#[derive(Debug)] +struct LinkCapture { + dest_url: String, + text: String, +} + +#[allow(dead_code)] +pub(crate) fn collect_network_upgrades( + catalog: &ProposalCatalog, +) -> Result { + collect_network_upgrades_with_registries( + catalog, + &transitional_modern_registry(), + &permanent_registry(), + ) +} + +pub(crate) fn collect_network_upgrades_with_registries( + catalog: &ProposalCatalog, + transitional_sources: &[NetworkUpgradeRegistrySource], + permanent_sources: &[NetworkUpgradeRegistrySource], +) -> Result { + let selected_sources = + select_network_upgrade_sources(catalog, transitional_sources, permanent_sources)?; + let mut upgrades = Vec::new(); + let mut memberships_by_proposal = + BTreeMap::>::new(); + + for selected_source in &selected_sources { + let source_document = source_document_for_number( + catalog, + selected_source.details.source_meta_eip, + "network upgrade source", + )?; + let meta_record = proposal_record_for_number( + catalog, + selected_source.details.source_meta_eip, + "network upgrade source", + )?; + let parsed_stages = match selected_source.details.parser_mode { + NetworkUpgradeParserMode::ModernStages => parse_modern_stage_members(source_document)?, + NetworkUpgradeParserMode::EmptyMembers => Vec::new(), + }; + let upgrade = build_network_upgrade( + catalog, + &selected_source.details, + meta_record, + parsed_stages, + )?; + + for stage in &upgrade.stages { + for row in &stage.rows { + memberships_by_proposal.entry(row.number).or_default().push( + NetworkUpgradeMembership { + display_name: upgrade.display_name.clone(), + slug: upgrade.slug.clone(), + source_meta_eip: upgrade.source_meta_eip, + meta_url: upgrade.meta_url.clone(), + stage_key: stage.key.clone(), + stage_label: stage.label.clone(), + subgroup: row.subgroup.clone(), + }, + ); + } + } + + upgrades.push(upgrade); + } + + Ok(NetworkUpgradeIndex { + upgrades, + memberships_by_proposal, + selected_sources: selected_sources + .into_iter() + .map(|selected_source| selected_source.details) + .collect(), + }) +} + +pub(crate) fn collect_marked_modern_sources( + catalog: &ProposalCatalog, +) -> Result, Whatever> { + catalog + .source_documents() + .filter_map(|source_document| { + source_document + .network_upgrade() + .map(|display_name| (source_document, display_name)) + }) + .map(|(source_document, display_name)| { + marked_source_from_document(source_document, display_name) + }) + .collect() +} + +pub(crate) fn transitional_modern_registry() -> Vec { + vec![ + NetworkUpgradeRegistrySource { + bucket: NetworkUpgradeSourceBucket::Transitional, + source_meta_eip: 7773, + parser_mode: NetworkUpgradeParserMode::ModernStages, + upgrades: vec![NetworkUpgradeRegistryUpgrade { + display_name: "Glamsterdam", + slug: None, + sort_order: None, + }], + }, + NetworkUpgradeRegistrySource { + bucket: NetworkUpgradeSourceBucket::Transitional, + source_meta_eip: 8081, + parser_mode: NetworkUpgradeParserMode::ModernStages, + upgrades: vec![NetworkUpgradeRegistryUpgrade { + display_name: "Hegotá", + slug: None, + sort_order: None, + }], + }, + ] +} + +pub(crate) fn permanent_registry() -> Vec { + vec![ + NetworkUpgradeRegistrySource { + bucket: NetworkUpgradeSourceBucket::Permanent, + source_meta_eip: 7569, + parser_mode: NetworkUpgradeParserMode::ModernStages, + upgrades: vec![NetworkUpgradeRegistryUpgrade { + display_name: "Dencun", + slug: None, + sort_order: Some(20240313), + }], + }, + NetworkUpgradeRegistrySource { + bucket: NetworkUpgradeSourceBucket::Permanent, + source_meta_eip: 7600, + parser_mode: NetworkUpgradeParserMode::ModernStages, + upgrades: vec![NetworkUpgradeRegistryUpgrade { + display_name: "Pectra", + slug: None, + sort_order: Some(20250507), + }], + }, + NetworkUpgradeRegistrySource { + bucket: NetworkUpgradeSourceBucket::Permanent, + source_meta_eip: 7607, + parser_mode: NetworkUpgradeParserMode::ModernStages, + upgrades: vec![NetworkUpgradeRegistryUpgrade { + display_name: "Fusaka", + slug: None, + sort_order: Some(20251203), + }], + }, + ] +} + +fn marked_source_from_document( + source_document: &ProposalCatalogSourceDocument, + display_name: &str, +) -> Result { + if display_name.trim().is_empty() { + snafu::whatever!( + "network-upgrade marker in `{}` must be non-empty", + source_document.source_path().to_string_lossy() + ); + } + if source_document.proposal_type() != Some("Meta") { + snafu::whatever!( + "network-upgrade marker in `{}` is only allowed on `type: Meta` proposals", + source_document.source_path().to_string_lossy() + ); + } + if source_document.prefix() != ProposalCatalogPrefix::Eip { + snafu::whatever!( + "network-upgrade marker in `{}` is only supported on EIP Meta proposals", + source_document.source_path().to_string_lossy() + ); + } + + let slug = network_upgrade_slug(display_name); + if slug.is_empty() { + snafu::whatever!( + "network-upgrade marker `{display_name}` in `{}` does not derive a stable slug", + source_document.source_path().to_string_lossy() + ); + } + + Ok(MarkedNetworkUpgradeSource { + source_meta_eip: source_document.number(), + display_name: display_name.trim().to_owned(), + slug, + }) +} + +fn select_network_upgrade_sources( + catalog: &ProposalCatalog, + transitional_sources: &[NetworkUpgradeRegistrySource], + permanent_sources: &[NetworkUpgradeRegistrySource], +) -> Result, Whatever> { + let marked_sources = collect_marked_modern_sources(catalog)?; + let marked_source_numbers = marked_sources + .iter() + .map(|source| source.source_meta_eip) + .collect::>(); + let permanent_source_numbers = permanent_sources + .iter() + .map(|source| registry_source_number(source.source_meta_eip)) + .collect::, _>>()?; + + for marked_source in &marked_sources { + if permanent_source_numbers.contains(&marked_source.source_meta_eip) { + snafu::whatever!( + "network-upgrade marker on permanent registry source `{}` is not allowed", + marked_source.source_meta_eip + ); + } + } + + let mut selected_sources = Vec::new(); + for marked_source in marked_sources { + let source_document = source_document_for_number( + catalog, + marked_source.source_meta_eip, + "marked network upgrade source", + )?; + selected_sources.push(SelectedNetworkUpgradeSource { + render_sort_key: created_render_sort_key(source_document)?, + details: NetworkUpgradeSelectedSource { + source_meta_eip: marked_source.source_meta_eip, + source_bucket: NetworkUpgradeSourceBucket::Marked, + display_name: marked_source.display_name, + slug: marked_source.slug, + parser_mode: NetworkUpgradeParserMode::ModernStages, + sort_order: None, + }, + }); + } + + for registry_source in transitional_sources { + let source_meta_eip = registry_source_number(registry_source.source_meta_eip)?; + if marked_source_numbers.contains(&source_meta_eip) { + warn!( + "network-upgrade marker on `{source_meta_eip}` shadows transitional registry entry" + ); + continue; + } + push_registry_selected_sources(catalog, registry_source, &mut selected_sources)?; + } + + for registry_source in permanent_sources { + push_registry_selected_sources(catalog, registry_source, &mut selected_sources)?; + } + + validate_sort_orders(&selected_sources)?; + validate_slugs(&selected_sources)?; + selected_sources.sort_by(|left, right| { + left.render_sort_key + .cmp(&right.render_sort_key) + .then_with(|| left.details.slug.cmp(&right.details.slug)) + }); + + Ok(selected_sources) +} + +fn push_registry_selected_sources( + catalog: &ProposalCatalog, + registry_source: &NetworkUpgradeRegistrySource, + selected_sources: &mut Vec, +) -> Result<(), Whatever> { + let source_meta_eip = registry_source_number(registry_source.source_meta_eip)?; + let source_document = source_document_for_number( + catalog, + source_meta_eip, + "registered network upgrade source", + )?; + let source_created_sort_key = created_render_sort_key(source_document)?; + + for registry_upgrade in ®istry_source.upgrades { + let slug = registry_upgrade + .slug + .map(str::to_owned) + .unwrap_or_else(|| network_upgrade_slug(registry_upgrade.display_name)); + if slug.is_empty() { + snafu::whatever!( + "registered network upgrade `{}` does not derive a stable slug", + registry_upgrade.display_name + ); + } + + selected_sources.push(SelectedNetworkUpgradeSource { + render_sort_key: registry_upgrade + .sort_order + .map(|sort_order| RenderSortKey { + value: i64::from(sort_order), + source_meta_eip, + }) + .unwrap_or(source_created_sort_key), + details: NetworkUpgradeSelectedSource { + source_meta_eip, + source_bucket: registry_source.bucket, + display_name: registry_upgrade.display_name.to_owned(), + slug, + parser_mode: registry_source.parser_mode, + sort_order: registry_upgrade.sort_order, + }, + }); + } + + Ok(()) +} + +fn build_network_upgrade( + catalog: &ProposalCatalog, + selected_source: &NetworkUpgradeSelectedSource, + meta_record: &ProposalCatalogRecord, + parsed_stages: Vec, +) -> Result { + let mut seen_members = BTreeMap::::new(); + let stages = parsed_stages + .into_iter() + .map(|parsed_stage| { + let rows = parsed_stage + .members + .into_iter() + .map(|member| { + if let Some(existing_stage) = + seen_members.insert(member.number, parsed_stage.key) + { + snafu::whatever!( + "network upgrade `{}` lists proposal `{}` more than once, in `{}` and `{}`", + selected_source.display_name, + member.number, + existing_stage, + parsed_stage.key + ); + } + + member_row(catalog, &selected_source.display_name, member) + }) + .collect::, Whatever>>()?; + + Ok(NetworkUpgradeStage { + key: parsed_stage.key.stable_key().to_owned(), + label: parsed_stage.key.to_string(), + rows, + }) + }) + .collect::, Whatever>>()?; + + Ok(NetworkUpgrade { + display_name: selected_source.display_name.clone(), + slug: selected_source.slug.clone(), + source_meta_eip: selected_source.source_meta_eip, + source_bucket: selected_source.source_bucket, + meta_url: meta_record.url.clone(), + meta_status: meta_record.status.clone(), + stages, + }) +} + +fn member_row( + catalog: &ProposalCatalog, + display_name: &str, + member: ParsedNetworkUpgradeMember, +) -> Result { + let record = proposal_record_for_number(catalog, member.number, display_name)?; + Ok(NetworkUpgradeMemberRow { + number: record.number, + title: record.title.clone(), + status: record.status.clone(), + proposal_type: record.proposal_type.clone(), + category: if record.proposal_type == "Standards Track" { + record.category.clone() + } else { + None + }, + url: record.url.clone(), + subgroup: member.subgroup, + }) +} + +fn parse_modern_stage_members( + source_document: &ProposalCatalogSourceDocument, +) -> Result, Whatever> { + let mut options = Options::empty(); + options.insert(Options::ENABLE_TABLES); + options.insert(Options::ENABLE_STRIKETHROUGH); + options.insert(Options::ENABLE_TASKLISTS); + + let mut stages = BTreeMap::::new(); + let mut active_stage = None::; + let mut heading_capture = None::; + let mut link_capture = None::; + + for event in Parser::new_ext(source_document.body(), options) { + match event { + Event::Start(Tag::Heading { level, .. }) => { + heading_capture = Some(HeadingCapture { + level, + text: String::new(), + }); + } + Event::End(TagEnd::Heading(_)) => { + let heading = heading_capture + .take() + .whatever_context("heading end without heading start")?; + handle_heading(source_document, heading, &mut active_stage, &mut stages)?; + } + Event::Text(text) | Event::Code(text) => { + push_cow_text(&mut heading_capture, &mut link_capture, text); + } + Event::SoftBreak | Event::HardBreak => { + if let Some(heading) = &mut heading_capture { + heading.text.push(' '); + } + if let Some(link) = &mut link_capture { + link.text.push(' '); + } + } + Event::Start(Tag::Link { dest_url, .. }) + if active_stage.is_some() && heading_capture.is_none() => + { + link_capture = Some(LinkCapture { + dest_url: dest_url.into_string(), + text: String::new(), + }); + } + Event::End(TagEnd::Link) => { + let Some(link) = link_capture.take() else { + continue; + }; + let Some(active_stage) = &active_stage else { + continue; + }; + let Some(proposal_number) = proposal_number_from_link(&link.dest_url, &link.text) + else { + continue; + }; + let stage = stages + .get_mut(&active_stage.key) + .with_whatever_context(|| { + format!( + "recognized network upgrade stage `{}` was not initialized in `{}`", + active_stage.key, + source_document.source_path().to_string_lossy() + ) + })?; + stage.members.push(ParsedNetworkUpgradeMember { + number: proposal_number, + subgroup: active_stage.subgroup.clone(), + }); + } + _ => {} + } + } + + let mut stages = stages.into_values().collect::>(); + stages.sort_by_key(|stage| stage.key.render_rank()); + Ok(stages) +} + +fn handle_heading( + source_document: &ProposalCatalogSourceDocument, + heading: HeadingCapture, + active_stage: &mut Option, + stages: &mut BTreeMap, +) -> Result<(), Whatever> { + let heading_text = heading.text.trim(); + let heading_level = heading_level_number(heading.level); + if let Some(stage_key) = normalize_stage_heading(heading_text) { + if stages.contains_key(&stage_key) { + snafu::whatever!( + "duplicate network upgrade stage heading `{}` in `{}`", + stage_key, + source_document.source_path().to_string_lossy() + ); + } + stages.insert( + stage_key, + ParsedNetworkUpgradeStage { + key: stage_key, + members: Vec::new(), + }, + ); + *active_stage = Some(ActiveStage { + key: stage_key, + level: heading_level, + subgroup: None, + }); + } else if let Some(stage) = active_stage { + if heading_level <= stage.level { + *active_stage = None; + } else if !heading_text.is_empty() { + stage.subgroup = Some(heading_text.to_owned()); + } + } + + Ok(()) +} + +fn push_cow_text( + heading_capture: &mut Option, + link_capture: &mut Option, + text: CowStr<'_>, +) { + if let Some(heading) = heading_capture { + heading.text.push_str(&text); + } + if let Some(link) = link_capture { + link.text.push_str(&text); + } +} + +fn normalize_stage_heading(heading: &str) -> Option { + let mut normalized = heading.split_whitespace().collect::>().join(" "); + let lower = normalized.to_ascii_lowercase(); + if let Some(stripped) = lower.strip_prefix("eips ") { + normalized = stripped.to_owned(); + } + let lower = normalized.to_ascii_lowercase(); + if let Some(stripped) = lower.strip_suffix(" eips") { + normalized = stripped.to_owned(); + } + let lower = normalized.to_ascii_lowercase(); + let normalized = lower + .strip_suffix(" for inclusion") + .unwrap_or(&lower) + .trim(); + + match normalized { + "included" => Some(NetworkUpgradeStageKey::Included), + "scheduled" => Some(NetworkUpgradeStageKey::Scheduled), + "considered" => Some(NetworkUpgradeStageKey::Considered), + "proposed" => Some(NetworkUpgradeStageKey::Proposed), + "declined" => Some(NetworkUpgradeStageKey::Declined), + _ => None, + } +} + +fn proposal_number_from_link(dest_url: &str, link_text: &str) -> Option { + proposal_number_from_markdown_path(dest_url).or_else(|| proposal_number_from_text(link_text)) +} + +fn proposal_number_from_markdown_path(dest_url: &str) -> Option { + let path = dest_url.split(['?', '#']).next()?.trim(); + if path.contains("://") || path.starts_with("mailto:") { + return None; + } + + let path = Path::new(path.trim_start_matches('/')); + if path.file_name()? == "index.md" { + return path_component_proposal_number(path.parent()?.file_name()); + } + + flat_proposal_number(path) +} + +fn proposal_number_from_text(link_text: &str) -> Option { + lazy_static! { + static ref PROPOSAL_TEXT_RE: Regex = Regex::new(r"(?i)\b(?:eip|erc)-?([0-9]+)\b").unwrap(); + } + + let captures = PROPOSAL_TEXT_RE.captures(link_text)?; + captures + .get(1)? + .as_str() + .parse::() + .ok() + .and_then(|number| ProposalNumber::from_u32(number).ok()) +} + +fn proposal_record_for_number<'a>( + catalog: &'a ProposalCatalog, + proposal_number: ProposalNumber, + context: &str, +) -> Result<&'a ProposalCatalogRecord, Whatever> { + catalog + .records() + .get(&ProposalCatalogPrefix::Eip.key(proposal_number)) + .with_whatever_context(|| { + format!("network upgrade `{context}` references missing proposal `{proposal_number}`") + }) +} + +fn source_document_for_number<'a>( + catalog: &'a ProposalCatalog, + proposal_number: ProposalNumber, + context: &str, +) -> Result<&'a ProposalCatalogSourceDocument, Whatever> { + catalog + .source_document(ProposalCatalogPrefix::Eip, proposal_number) + .with_whatever_context(|| { + format!("network upgrade `{context}` source proposal `{proposal_number}` was not found") + }) +} + +fn registry_source_number(number: u32) -> Result { + match ProposalNumber::from_u32(number) { + Ok(number) => Ok(number), + Err(()) => { + snafu::whatever!("network upgrade registry source number `{number}` must be positive"); + } + } +} + +fn created_render_sort_key( + source_document: &ProposalCatalogSourceDocument, +) -> Result { + let created = source_document.created().with_whatever_context(|| { + format!( + "network upgrade source `{}` is missing `created`", + source_document.number() + ) + })?; + let date = NaiveDate::parse_from_str(created, "%Y-%m-%d").with_whatever_context(|_| { + format!( + "network upgrade source `{}` has invalid `created` date `{created}`", + source_document.number() + ) + })?; + + Ok(RenderSortKey { + value: i64::from(date.year()) * 10_000 + + i64::from(date.month()) * 100 + + i64::from(date.day()), + source_meta_eip: source_document.number(), + }) +} + +fn validate_sort_orders(selected_sources: &[SelectedNetworkUpgradeSource]) -> Result<(), Whatever> { + let mut sort_orders = BTreeMap::::new(); + for selected_source in selected_sources { + let Some(sort_order) = selected_source.details.sort_order else { + continue; + }; + if let Some(existing_name) = + sort_orders.insert(sort_order, &selected_source.details.display_name) + { + snafu::whatever!( + "network upgrade sort_order `{sort_order}` is used by both `{existing_name}` and `{}`", + selected_source.details.display_name + ); + } + } + + Ok(()) +} + +fn validate_slugs(selected_sources: &[SelectedNetworkUpgradeSource]) -> Result<(), Whatever> { + let mut slugs = BTreeMap::<&str, &str>::new(); + for selected_source in selected_sources { + if let Some(existing_name) = slugs.insert( + selected_source.details.slug.as_str(), + selected_source.details.display_name.as_str(), + ) { + snafu::whatever!( + "network upgrade slug collision `{}` between `{existing_name}` and `{}`", + selected_source.details.slug, + selected_source.details.display_name + ); + } + } + + Ok(()) +} + +fn heading_level_number(level: HeadingLevel) -> u8 { + match level { + HeadingLevel::H1 => 1, + HeadingLevel::H2 => 2, + HeadingLevel::H3 => 3, + HeadingLevel::H4 => 4, + HeadingLevel::H5 => 5, + HeadingLevel::H6 => 6, + } +} + +fn network_upgrade_slug(display_name: &str) -> String { + let mut slug = String::new(); + let mut last_was_separator = false; + + for character in display_name.chars().flat_map(char::to_lowercase) { + if let Some(ascii) = slug_ascii_alphanumeric(character) { + slug.push(ascii); + last_was_separator = false; + } else if !slug.is_empty() && !last_was_separator { + slug.push('-'); + last_was_separator = true; + } + } + + if last_was_separator { + slug.pop(); + } + + slug +} + +fn slug_ascii_alphanumeric(character: char) -> Option { + if character.is_ascii_alphanumeric() { + return Some(character); + } + + // This intentionally small fold covers current hardfork names without + // adding a new direct dependency. Broader Unicode slugging should replace + // it if source names expand beyond simple Latin diacritics. + match character { + 'à' | 'á' | 'â' | 'ã' | 'ä' | 'å' | 'ā' | 'ă' | 'ą' => Some('a'), + 'ç' | 'ć' | 'č' => Some('c'), + 'ď' => Some('d'), + 'è' | 'é' | 'ê' | 'ë' | 'ē' | 'ė' | 'ę' => Some('e'), + 'ì' | 'í' | 'î' | 'ï' | 'ī' | 'į' => Some('i'), + 'ñ' | 'ń' => Some('n'), + 'ò' | 'ó' | 'ô' | 'õ' | 'ö' | 'ø' | 'ō' | 'ő' => Some('o'), + 'ŕ' | 'ř' => Some('r'), + 'ś' | 'š' => Some('s'), + 'ť' => Some('t'), + 'ù' | 'ú' | 'û' | 'ü' | 'ū' | 'ů' | 'ű' => Some('u'), + 'ý' | 'ÿ' => Some('y'), + 'ź' | 'ż' | 'ž' => Some('z'), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use tempfile::TempDir; + + use super::{ + collect_marked_modern_sources, collect_network_upgrades_with_registries, + network_upgrade_slug, normalize_stage_heading, permanent_registry, + transitional_modern_registry, NetworkUpgradeParserMode, NetworkUpgradeRegistrySource, + NetworkUpgradeRegistryUpgrade, NetworkUpgradeSourceBucket, NetworkUpgradeStageKey, + }; + use crate::{ + proposal::ProposalNumber, + proposal_catalog::{collect_proposal_catalog, ProposalCatalog}, + }; + + fn number(value: u32) -> ProposalNumber { + ProposalNumber::from_u32(value).unwrap() + } + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn proposal_markdown( + number: u32, + title: &str, + status: &str, + proposal_type: &str, + category: Option<&str>, + extra_preamble: &str, + body: &str, + ) -> String { + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!( + "---\neip: {number}\ntitle: {title}\nstatus: {status}\ntype: {proposal_type}\n{category}created: 2024-01-01\n{extra_preamble}---\n\n{body}\n" + ) + } + + fn meta_markdown(number: u32, title: &str, created: &str, extra: &str, body: &str) -> String { + format!( + "---\neip: {number}\ntitle: {title}\nstatus: Draft\ntype: Meta\ncreated: {created}\n{extra}---\n\n{body}\n" + ) + } + + fn catalog(files: &[(&str, String)]) -> ProposalCatalog { + let temp = TempDir::new().unwrap(); + for (relative, contents) in files { + write_file(temp.path(), relative, contents); + } + collect_proposal_catalog(temp.path(), None).unwrap() + } + + fn transitional_source( + number: u32, + display_name: &'static str, + ) -> NetworkUpgradeRegistrySource { + NetworkUpgradeRegistrySource { + bucket: NetworkUpgradeSourceBucket::Transitional, + source_meta_eip: number, + parser_mode: NetworkUpgradeParserMode::ModernStages, + upgrades: vec![NetworkUpgradeRegistryUpgrade { + display_name, + slug: None, + sort_order: None, + }], + } + } + + fn permanent_source( + number: u32, + display_name: &'static str, + sort_order: Option, + ) -> NetworkUpgradeRegistrySource { + NetworkUpgradeRegistrySource { + bucket: NetworkUpgradeSourceBucket::Permanent, + source_meta_eip: number, + parser_mode: NetworkUpgradeParserMode::ModernStages, + upgrades: vec![NetworkUpgradeRegistryUpgrade { + display_name, + slug: None, + sort_order, + }], + } + } + + #[test] + fn slug_generation_is_deterministic_and_ascii_folds_known_names() { + assert_eq!(network_upgrade_slug("Hegotá"), "hegota"); + assert_eq!( + network_upgrade_slug("Hegotá"), + network_upgrade_slug("hegota") + ); + assert_eq!( + network_upgrade_slug(" Prague / Electra "), + "prague-electra" + ); + } + + #[test] + fn heading_normalization_supports_known_variants() { + assert_eq!( + normalize_stage_heading("EIPs Scheduled for Inclusion"), + Some(NetworkUpgradeStageKey::Scheduled) + ); + assert_eq!( + normalize_stage_heading("Included EIPs"), + Some(NetworkUpgradeStageKey::Included) + ); + assert_eq!( + normalize_stage_heading("EIPs Included"), + Some(NetworkUpgradeStageKey::Included) + ); + assert_eq!( + normalize_stage_heading("Proposed"), + Some(NetworkUpgradeStageKey::Proposed) + ); + assert_eq!( + normalize_stage_heading("Declined for Inclusion"), + Some(NetworkUpgradeStageKey::Declined) + ); + } + + #[test] + fn active_stage_parsing_handles_glamsterdam_and_hegota_shapes() { + let glamsterdam = meta_markdown( + 7773, + "Hardfork Meta - Glamsterdam", + "2024-09-26", + "", + "### EIPs Scheduled for Inclusion\n\n* [EIP-7732](./07732.md)\n\n### Considered for Inclusion\n\n* [EIP-2780](./02780.md)\n\n### Declined for Inclusion\n\n* [EIP-2926](./02926.md)\n\n### Proposed for Inclusion\n\n* [EIP-7610](./07610.md)\n", + ); + let hegota = meta_markdown( + 8081, + "Hardfork Meta - Hegotá", + "2025-11-11", + "", + "### EIPs Scheduled for Inclusion\n\n### Considered for Inclusion\n\n* [EIP-7805](./07805.md)\n\n### Declined for Inclusion\n\n### Proposed for Inclusion\n", + ); + let catalog = catalog(&[ + ("07773.md", glamsterdam), + ("08081.md", hegota), + ( + "07732.md", + proposal_markdown(7732, "EIP 7732", "Draft", "Standards Track", None, "", ""), + ), + ( + "02780.md", + proposal_markdown(2780, "EIP 2780", "Draft", "Standards Track", None, "", ""), + ), + ( + "02926.md", + proposal_markdown(2926, "EIP 2926", "Draft", "Standards Track", None, "", ""), + ), + ( + "07610.md", + proposal_markdown(7610, "EIP 7610", "Draft", "Standards Track", None, "", ""), + ), + ( + "07805.md", + proposal_markdown(7805, "EIP 7805", "Draft", "Standards Track", None, "", ""), + ), + ]); + + let index = collect_network_upgrades_with_registries( + &catalog, + &[ + transitional_source(7773, "Glamsterdam"), + transitional_source(8081, "Hegotá"), + ], + &[], + ) + .unwrap(); + + let glamsterdam = index + .upgrades + .iter() + .find(|upgrade| upgrade.slug == "glamsterdam") + .unwrap(); + assert_eq!( + glamsterdam + .stages + .iter() + .map(|stage| stage.key.as_str()) + .collect::>(), + ["scheduled", "considered", "proposed", "declined"] + ); + assert_eq!(glamsterdam.stages[0].rows[0].number, number(7732)); + assert_eq!(glamsterdam.stages[3].rows[0].number, number(2926)); + + let hegota = index + .upgrades + .iter() + .find(|upgrade| upgrade.slug == "hegota") + .unwrap(); + assert!(hegota.stages[0].rows.is_empty()); + assert_eq!(hegota.stages[1].rows[0].number, number(7805)); + } + + #[test] + fn final_included_parsing_covers_recent_permanent_hardfork_shapes() { + let dencun = meta_markdown( + 7569, + "Hardfork Meta - Dencun", + "2023-12-01", + "", + "### Included EIPs\n\n* [EIP-1153](./01153.md)\n", + ); + let pectra = meta_markdown( + 7600, + "Hardfork Meta - Pectra", + "2024-01-18", + "", + "### Included EIPs\n\n#### Core EIPs\n\n* [EIP-2537](./02537.md)\n\n#### Other EIPs\n\n* [EIP-7840](./07840.md)\n", + ); + let fusaka = meta_markdown( + 7607, + "Hardfork Meta - Fusaka", + "2024-02-01", + "", + "### Included EIPs\n\n#### Core EIPs\n\n* [EIP-7594](./07594.md)\n\n#### Other EIPs\n\n* [EIP-7892](./07892.md)\n", + ); + let catalog = catalog(&[ + ("07569.md", dencun), + ("07600.md", pectra), + ("07607.md", fusaka), + ( + "01153.md", + proposal_markdown( + 1153, + "Transient storage", + "Final", + "Standards Track", + Some("Core"), + "", + "", + ), + ), + ( + "02537.md", + proposal_markdown( + 2537, + "BLS precompile", + "Final", + "Standards Track", + Some("Core"), + "", + "", + ), + ), + ( + "07594.md", + proposal_markdown( + 7594, + "PeerDAS", + "Final", + "Standards Track", + Some("Core"), + "", + "", + ), + ), + ( + "07840.md", + proposal_markdown( + 7840, + "Blob schedule", + "Review", + "Informational", + None, + "", + "", + ), + ), + ( + "07892.md", + proposal_markdown(7892, "Blob hardforks", "Final", "Meta", None, "", ""), + ), + ]); + + let index = collect_network_upgrades_with_registries( + &catalog, + &[], + &[ + permanent_source(7569, "Dencun", Some(20240313)), + permanent_source(7600, "Pectra", Some(20250507)), + permanent_source(7607, "Fusaka", Some(20251203)), + ], + ) + .unwrap(); + let dencun = index + .upgrades + .iter() + .find(|upgrade| upgrade.slug == "dencun") + .unwrap(); + let pectra = index + .upgrades + .iter() + .find(|upgrade| upgrade.slug == "pectra") + .unwrap(); + let fusaka = index + .upgrades + .iter() + .find(|upgrade| upgrade.slug == "fusaka") + .unwrap(); + let pectra_rows = &pectra.stages[0].rows; + let fusaka_rows = &fusaka.stages[0].rows; + + assert_eq!(dencun.stages[0].key, "included"); + assert_eq!(dencun.stages[0].rows[0].subgroup, None); + assert_eq!(pectra.stages[0].key, "included"); + assert_eq!(pectra_rows[0].subgroup.as_deref(), Some("Core EIPs")); + assert_eq!(pectra_rows[0].category.as_deref(), Some("Core")); + assert_eq!(pectra_rows[1].subgroup.as_deref(), Some("Other EIPs")); + assert_eq!(pectra_rows[1].category, None); + assert_eq!(fusaka_rows[0].subgroup.as_deref(), Some("Core EIPs")); + assert_eq!(fusaka_rows[1].subgroup.as_deref(), Some("Other EIPs")); + } + + #[test] + fn duplicate_stage_headings_error() { + let source = meta_markdown( + 7773, + "Hardfork Meta - Glamsterdam", + "2024-09-26", + "", + "### Included EIPs\n\n### EIPs Included\n", + ); + let catalog = catalog(&[("07773.md", source)]); + let error = collect_network_upgrades_with_registries( + &catalog, + &[transitional_source(7773, "Glamsterdam")], + &[], + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("duplicate network upgrade stage heading")); + assert!(error.contains("Included")); + } + + #[test] + fn links_outside_recognized_stage_sections_are_ignored() { + let source = meta_markdown( + 7773, + "Hardfork Meta - Glamsterdam", + "2024-09-26", + "", + "See [EIP-1](./00001.md).\n\n### Included EIPs\n\n* [EIP-2](./00002.md)\n\n### Activation\n\nSee [EIP-3](./00003.md).\n", + ); + let catalog = catalog(&[ + ("07773.md", source), + ( + "00002.md", + proposal_markdown(2, "EIP 2", "Draft", "Standards Track", None, "", ""), + ), + ]); + + let index = collect_network_upgrades_with_registries( + &catalog, + &[transitional_source(7773, "Glamsterdam")], + &[], + ) + .unwrap(); + + assert_eq!(index.upgrades[0].stages[0].rows.len(), 1); + assert_eq!(index.upgrades[0].stages[0].rows[0].number, number(2)); + } + + #[test] + fn no_markers_still_selects_transitional_registry_sources() { + let source = meta_markdown( + 7773, + "Hardfork Meta - Glamsterdam", + "2024-09-26", + "", + "### Included EIPs\n", + ); + let catalog = catalog(&[("07773.md", source)]); + + assert!(collect_marked_modern_sources(&catalog).unwrap().is_empty()); + let index = collect_network_upgrades_with_registries( + &catalog, + &[transitional_source(7773, "Glamsterdam")], + &[], + ) + .unwrap(); + + assert_eq!( + index.selected_sources[0].source_bucket, + NetworkUpgradeSourceBucket::Transitional + ); + } + + #[test] + fn marked_sources_shadow_transitional_entries_by_source_number() { + let source = meta_markdown( + 7773, + "Hardfork Meta - Glamsterdam", + "2024-09-26", + "network-upgrade: Glamsterdam\n", + "### Included EIPs\n", + ); + let catalog = catalog(&[("07773.md", source)]); + + let index = collect_network_upgrades_with_registries( + &catalog, + &[transitional_source(7773, "Different Name")], + &[], + ) + .unwrap(); + + assert_eq!(index.upgrades.len(), 1); + assert_eq!( + index.selected_sources[0].source_bucket, + NetworkUpgradeSourceBucket::Marked + ); + assert_eq!(index.upgrades[0].display_name, "Glamsterdam"); + } + + #[test] + fn invalid_markers_fail_preprocessor_validation() { + let empty_marker = catalog(&[( + "00001.md", + meta_markdown(1, "Meta", "2024-01-01", "network-upgrade: \n", ""), + )]); + let error = collect_marked_modern_sources(&empty_marker) + .unwrap_err() + .to_string(); + assert!(error.contains("must be non-empty")); + + let non_meta = catalog(&[( + "00002.md", + proposal_markdown( + 2, + "Not Meta", + "Draft", + "Standards Track", + None, + "network-upgrade: Bad\n", + "", + ), + )]); + let error = collect_marked_modern_sources(&non_meta) + .unwrap_err() + .to_string(); + assert!(error.contains("only allowed on `type: Meta`")); + + let erc_meta = catalog(&[( + "00004.md", + proposal_markdown( + 4, + "ERC Meta", + "Draft", + "Meta", + Some("ERC"), + "network-upgrade: Foo\n", + "", + ), + )]); + let error = collect_marked_modern_sources(&erc_meta) + .unwrap_err() + .to_string(); + assert!(error.contains("only supported on EIP Meta proposals")); + + let empty_slug = catalog(&[( + "00003.md", + meta_markdown(3, "Meta", "2024-01-01", "network-upgrade: !!!\n", ""), + )]); + let error = collect_marked_modern_sources(&empty_slug) + .unwrap_err() + .to_string(); + assert!(error.contains("does not derive a stable slug")); + } + + #[test] + fn marker_on_permanent_registry_source_errors() { + let source = meta_markdown( + 7569, + "Hardfork Meta - Dencun", + "2023-12-01", + "network-upgrade: Dencun\n", + "### Included EIPs\n", + ); + let catalog = catalog(&[("07569.md", source)]); + let error = collect_network_upgrades_with_registries( + &catalog, + &[], + &[permanent_source(7569, "Dencun", Some(20240313))], + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("permanent registry source")); + } + + #[test] + fn stage_like_unregistered_meta_eips_are_not_collected() { + let source = meta_markdown( + 8007, + "Glamsterdam Gas Repricings", + "2025-08-21", + "", + "### Considered for Inclusion\n\n* [EIP-2780](./02780.md)\n", + ); + let catalog = catalog(&[ + ("08007.md", source), + ( + "02780.md", + proposal_markdown(2780, "EIP 2780", "Draft", "Standards Track", None, "", ""), + ), + ]); + + let index = collect_network_upgrades_with_registries(&catalog, &[], &[]).unwrap(); + + assert!(index.upgrades.is_empty()); + } + + #[test] + fn duplicate_membership_within_same_hardfork_errors() { + let source = meta_markdown( + 7773, + "Hardfork Meta - Glamsterdam", + "2024-09-26", + "", + "### Scheduled for Inclusion\n\n* [EIP-1](./00001.md)\n\n### Declined for Inclusion\n\n* [EIP-1](./00001.md)\n", + ); + let catalog = catalog(&[ + ("07773.md", source), + ( + "00001.md", + proposal_markdown(1, "EIP 1", "Draft", "Standards Track", None, "", ""), + ), + ]); + + let error = collect_network_upgrades_with_registries( + &catalog, + &[transitional_source(7773, "Glamsterdam")], + &[], + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("lists proposal `1` more than once")); + } + + #[test] + fn cross_hardfork_membership_is_allowed() { + let first = meta_markdown( + 100, + "First", + "2024-01-01", + "", + "### Included EIPs\n\n* [EIP-1](./00001.md)\n", + ); + let second = meta_markdown( + 200, + "Second", + "2024-02-01", + "", + "### Included EIPs\n\n* [EIP-1](./00001.md)\n", + ); + let catalog = catalog(&[ + ("00100.md", first), + ("00200.md", second), + ( + "00001.md", + proposal_markdown(1, "EIP 1", "Draft", "Standards Track", None, "", ""), + ), + ]); + + let index = collect_network_upgrades_with_registries( + &catalog, + &[ + transitional_source(100, "First"), + transitional_source(200, "Second"), + ], + &[], + ) + .unwrap(); + + assert_eq!(index.memberships_by_proposal[&number(1)].len(), 2); + } + + #[test] + fn missing_member_references_error() { + let source = meta_markdown( + 7773, + "Hardfork Meta - Glamsterdam", + "2024-09-26", + "", + "### Included EIPs\n\n* [EIP-1](./00001.md)\n", + ); + let catalog = catalog(&[("07773.md", source)]); + let error = collect_network_upgrades_with_registries( + &catalog, + &[transitional_source(7773, "Glamsterdam")], + &[], + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("references missing proposal `1`")); + } + + #[test] + fn transitional_and_permanent_registries_are_distinct_buckets() { + assert!(transitional_modern_registry().iter().all(|source| { + source.bucket == NetworkUpgradeSourceBucket::Transitional + && source.parser_mode == NetworkUpgradeParserMode::ModernStages + })); + assert!(permanent_registry() + .iter() + .all(|source| source.bucket == NetworkUpgradeSourceBucket::Permanent)); + assert_eq!( + transitional_modern_registry() + .iter() + .map(|source| source.source_meta_eip) + .collect::>(), + [7773, 8081] + ); + assert_eq!( + permanent_registry() + .iter() + .map(|source| source.source_meta_eip) + .collect::>(), + [7569, 7600, 7607] + ); + } + + #[test] + fn slug_collisions_are_detected() { + let first = meta_markdown(100, "First", "2024-01-01", "", "### Included EIPs\n"); + let second = meta_markdown(200, "Second", "2024-02-01", "", "### Included EIPs\n"); + let catalog = catalog(&[("00100.md", first), ("00200.md", second)]); + + let error = collect_network_upgrades_with_registries( + &catalog, + &[ + transitional_source(100, "Same Name"), + transitional_source(200, "Same-Name"), + ], + &[], + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("slug collision `same-name`")); + } + + #[test] + fn transitional_entries_sort_by_created_date_then_meta_number() { + let early = meta_markdown(200, "Early", "2024-01-01", "", "### Included EIPs\n"); + let tie_low = meta_markdown(100, "Tie Low", "2024-02-01", "", "### Included EIPs\n"); + let tie_high = meta_markdown(300, "Tie High", "2024-02-01", "", "### Included EIPs\n"); + let catalog = catalog(&[ + ("00200.md", early), + ("00100.md", tie_low), + ("00300.md", tie_high), + ]); + + let index = collect_network_upgrades_with_registries( + &catalog, + &[ + transitional_source(300, "Tie High"), + transitional_source(200, "Early"), + transitional_source(100, "Tie Low"), + ], + &[], + ) + .unwrap(); + + assert_eq!( + index + .upgrades + .iter() + .map(|upgrade| upgrade.source_meta_eip) + .collect::>(), + [number(200), number(100), number(300)] + ); + } + + #[test] + fn permanent_sort_order_overrides_render_order() { + let first = meta_markdown(100, "First", "2024-01-01", "", "### Included EIPs\n"); + let second = meta_markdown(200, "Second", "2024-02-01", "", "### Included EIPs\n"); + let catalog = catalog(&[("00100.md", first), ("00200.md", second)]); + + let index = collect_network_upgrades_with_registries( + &catalog, + &[], + &[ + permanent_source(100, "First", Some(20)), + permanent_source(200, "Second", Some(10)), + ], + ) + .unwrap(); + + assert_eq!( + index + .upgrades + .iter() + .map(|upgrade| upgrade.display_name.as_str()) + .collect::>(), + ["Second", "First"] + ); + } + + #[test] + fn sort_order_collisions_error_clearly() { + let first = meta_markdown(100, "First", "2024-01-01", "", "### Included EIPs\n"); + let second = meta_markdown(200, "Second", "2024-02-01", "", "### Included EIPs\n"); + let catalog = catalog(&[("00100.md", first), ("00200.md", second)]); + + let error = collect_network_upgrades_with_registries( + &catalog, + &[], + &[ + permanent_source(100, "First", Some(10)), + permanent_source(200, "Second", Some(10)), + ], + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("sort_order `10`")); + assert!(error.contains("First")); + assert!(error.contains("Second")); + } + + #[test] + fn permanent_registry_shape_supports_one_source_to_many_and_empty_members() { + let source = meta_markdown(100, "Backfill", "2024-01-01", "", ""); + let catalog = catalog(&[("00100.md", source)]); + let registry_source = NetworkUpgradeRegistrySource { + bucket: NetworkUpgradeSourceBucket::Permanent, + source_meta_eip: 100, + parser_mode: NetworkUpgradeParserMode::EmptyMembers, + upgrades: vec![ + NetworkUpgradeRegistryUpgrade { + display_name: "First Empty", + slug: Some("first-empty"), + sort_order: Some(1), + }, + NetworkUpgradeRegistryUpgrade { + display_name: "Second Empty", + slug: Some("second-empty"), + sort_order: Some(2), + }, + ], + }; + + let index = + collect_network_upgrades_with_registries(&catalog, &[], &[registry_source]).unwrap(); + + assert_eq!(index.upgrades.len(), 2); + assert!(index + .upgrades + .iter() + .all(|upgrade| upgrade.stages.is_empty())); + } +} diff --git a/src/pipeline.rs b/src/pipeline.rs new file mode 100644 index 0000000..031bfc3 --- /dev/null +++ b/src/pipeline.rs @@ -0,0 +1,613 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Prepared Zola runtime pipeline. + +use std::path::{Path, PathBuf}; + +use snafu::{OptionExt, ResultExt, Whatever}; +use url::Url; + +use crate::{ + config::ServerBinding, + execution::ResolvedExecution, + git, + layout::{mounted_theme_path, output_path, CONTENT_DIR, REPO_DIR}, + markdown, + proposal::OnlyRenderPlan, + proposal_metadata, + serve::{serve_sync_config, DirtyServeWatcher, LocalThemeServeSync}, + zola, +}; + +fn prepare_theme_for_zola( + theme_path: PathBuf, + repo_path: &Path, +) -> Result<(PathBuf, LocalThemeServeSync), Whatever> { + let mounted_theme_dir = mounted_theme_path(repo_path); + git::materialize_working_tree(&theme_path, &mounted_theme_dir) + .whatever_context("unable to materialize workspace-local theme")?; + let theme_index_path = git::index_path(&theme_path) + .whatever_context("unable to resolve workspace-local theme Git index path")?; + + Ok(( + mounted_theme_dir.clone(), + LocalThemeServeSync { + theme_source_root: theme_path, + mounted_theme_dir, + theme_index_path, + }, + )) +} + +fn prepare_runtime_source( + root_path: &Path, + repo_path: &Path, + repository_use: &git::RepositoryUse, + source_materialization: git::SourceMaterialization, +) -> Result<(), Whatever> { + let source = git::Fresh::new( + root_path, + repo_path, + repository_use.clone(), + source_materialization, + ) + .whatever_context("initializing build repo")? + .clone_src() + .whatever_context("cloning source repo")?; + + source + .merge() + .whatever_context("unable to merge ERC/EIP repositories")?; + + Ok(()) +} + +#[derive(Debug)] +pub(crate) struct Prepared { + repo_path: PathBuf, + output_path: PathBuf, + repository_use: git::RepositoryUse, + theme_path: PathBuf, + local_theme_sync: Option, + only_plan: Option, + source_root: PathBuf, + source_materialization: git::SourceMaterialization, + server_binding: ServerBinding, + base_url_override: Option, +} + +impl Prepared { + pub(crate) fn prepare(resolved: ResolvedExecution) -> Result { + zola::find_zola().whatever_context("unable to find suitable zola binary")?; + + let ResolvedExecution { + root_path, + build_path, + repository_use, + theme_path, + only, + source_materialization, + server_binding, + base_url_override, + } = resolved; + let theme_path = + theme_path.whatever_context("Zola runtime requires a workspace-local theme path")?; + + let repo_path = build_path.join(REPO_DIR); + let content_path = repo_path.join(CONTENT_DIR); + let output_path = output_path(&build_path); + + prepare_runtime_source( + &root_path, + &repo_path, + &repository_use, + source_materialization, + )?; + + let only_plan = only + .map(|selected_numbers| OnlyRenderPlan::build(&content_path, selected_numbers)) + .transpose() + .whatever_context("unable to build targeted render plan")?; + proposal_metadata::write_proposal_metadata_json( + &repo_path, + &repository_use.title, + only_plan.as_ref(), + ) + .whatever_context("unable to write proposal metadata JSON")?; + markdown::preprocess(&content_path, only_plan.as_ref()) + .whatever_context("unable to preprocess markdown")?; + if let Some(only_plan) = &only_plan { + only_plan + .prune_content(&content_path) + .whatever_context("unable to prune unselected proposals")?; + } + let (theme_path, local_theme_sync) = prepare_theme_for_zola(theme_path, &repo_path)?; + + Ok(Prepared { + repository_use, + theme_path, + local_theme_sync: Some(local_theme_sync), + repo_path, + output_path, + only_plan, + source_root: root_path, + source_materialization, + server_binding, + base_url_override, + }) + } + + pub(crate) fn build(self) -> Result<(), Whatever> { + let base_url = self + .base_url_override + .as_ref() + .unwrap_or(&self.repository_use.location.base_url); + zola::build( + &self.theme_path, + &self.repo_path, + &self.output_path, + base_url.as_str(), + ) + .whatever_context("zola build failed")?; + Ok(()) + } + + pub(crate) fn serve(self) -> Result<(), Whatever> { + let sync_config = serve_sync_config( + self.source_materialization, + &self.source_root, + &self.repo_path, + self.only_plan.clone(), + self.local_theme_sync.clone(), + ); + let dirty_watcher = if sync_config.has_targets() { + Some( + DirtyServeWatcher::start(sync_config) + .whatever_context("unable to start dirty serve watcher")?, + ) + } else { + None + }; + + let result = zola::serve( + &self.theme_path, + &self.repo_path, + &self.output_path, + &self.server_binding, + self.base_url_override.as_ref(), + ) + .whatever_context("zola serve failed"); + + if let Some(dirty_watcher) = dirty_watcher { + dirty_watcher.stop(); + } + + result + } + + pub(crate) fn check(self) -> Result<(), Whatever> { + zola::check(&self.theme_path, &self.repo_path).whatever_context("zola check failed")?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use clap::Parser; + use git2::{IndexAddOption, Repository, Signature}; + use tempfile::TempDir; + use url::Url; + + use crate::{ + changed, + cli::{Args, ChangedFormat, EditorialCommand, RuntimeOperation}, + config, + editorial::editorial_runtime_execution, + execution::{resolve_execution, ResolvedExecution}, + git::SourceMaterialization, + layout::{mounted_theme_path, theme_config_path, CONTENT_DIR, REPO_DIR}, + network_upgrades::{ + collect_network_upgrades_with_registries, NetworkUpgradeParserMode, + NetworkUpgradeRegistrySource, NetworkUpgradeRegistryUpgrade, + NetworkUpgradeSourceBucket, + }, + proposal_catalog::collect_proposal_catalog, + }; + + use super::{prepare_runtime_source, prepare_theme_for_zola}; + + struct RuntimeWorkspace { + _temp: TempDir, + active_path: PathBuf, + } + + fn write_file(root: &Path, relative: impl AsRef, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn commit_all(repo: &Repository, message: &str) { + let mut index = repo.index().unwrap(); + index + .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + let parents = repo + .head() + .ok() + .and_then(|head| head.target()) + .map(|oid| repo.find_commit(oid).unwrap()) + .into_iter() + .collect::>(); + let parent_refs = parents.iter().collect::>(); + + repo.commit( + Some("HEAD"), + &signature, + &signature, + message, + &tree, + &parent_refs, + ) + .unwrap(); + } + + fn init_repo(path: &Path, files: &[(&str, &str)]) -> Repository { + std::fs::create_dir_all(path).unwrap(); + let repo = Repository::init(path).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + for (relative, contents) in files { + write_file(path, relative, contents); + } + commit_all(&repo, "initial"); + repo + } + + fn file_url(path: &Path) -> Url { + Url::from_directory_path(path).unwrap() + } + + fn repo_manifest_text(repo_id: &str, repository: &Url, siblings: &[(&str, Url)]) -> String { + let mut manifest = format!( + r#" +repo_id = "{repo_id}" + +[production] +repository = "{repository}" +base_url = "https://example.test/{repo_id}/" + +[staging] +repository = "{repository}" +base_url = "https://staging.example.test/{repo_id}/" +"# + ); + + for (sibling_id, sibling_repository) in siblings { + manifest.push_str(&format!( + r#" +[siblings.{sibling_id}.production] +repository = "{sibling_repository}" +base_url = "https://example.test/{sibling_id}/" + +[siblings.{sibling_id}.staging] +repository = "{sibling_repository}" +base_url = "https://staging.example.test/{sibling_id}/" +"# + )); + } + + manifest + } + + fn pipeline_proposal_markdown(number: u32, category: Option<&str>, body: &str) -> String { + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!( + "---\neip: {number}\ntitle: Proposal {number}\nstatus: Draft\ntype: Standards Track\n{category}---\n\n{body}\n" + ) + } + + fn pipeline_meta_markdown(number: u32, created: &str, body: &str) -> String { + format!( + "---\neip: {number}\ntitle: Hardfork Meta {number}\nstatus: Draft\ntype: Meta\ncreated: {created}\n---\n\n{body}\n" + ) + } + + fn runtime_workspace(with_sibling: bool) -> RuntimeWorkspace { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("EIPs"); + let sibling_path = workspace_root.join("ERCs"); + let missing_upstream = file_url(&temp.path().join("missing-upstream")); + let siblings = with_sibling.then(|| ("ERCs", file_url(&sibling_path))); + let siblings = siblings.into_iter().collect::>(); + let manifest = repo_manifest_text("EIPs", &missing_upstream, &siblings); + + write_file(&workspace_root, config::LOCAL_CONFIG_FILE, ""); + std::fs::create_dir_all(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + let _active_repo = init_repo( + &active_path, + &[ + (config::REPO_MANIFEST_FILE, manifest.as_str()), + ("content/00001.md", "active proposal\n"), + ], + ); + + if with_sibling { + let _sibling_repo = init_repo(&sibling_path, &[("content/00002.md", "sibling\n")]); + } + + RuntimeWorkspace { + _temp: temp, + active_path, + } + } + + fn resolved_runtime(workspace: &RuntimeWorkspace, command: &[&str]) -> ResolvedExecution { + let active_path = workspace.active_path.to_str().unwrap(); + let mut arguments = vec!["build-eips", "-C", active_path]; + arguments.extend_from_slice(command); + let args = Args::try_parse_from(arguments).unwrap(); + + resolve_execution(&args).unwrap() + } + + fn prepare_resolved_source(resolved: &ResolvedExecution) -> Result<(), snafu::Whatever> { + std::fs::create_dir_all(&resolved.build_path).unwrap(); + let repo_path = resolved.build_path.join(REPO_DIR); + prepare_runtime_source( + &resolved.root_path, + &repo_path, + &resolved.repository_use, + resolved.source_materialization, + ) + } + + fn prepared_path(resolved: &ResolvedExecution, relative: impl AsRef) -> PathBuf { + resolved.build_path.join(REPO_DIR).join(relative) + } + + #[test] + fn workspace_local_theme_is_materialized_as_mounted_theme_for_zola() { + let temp = TempDir::new().unwrap(); + let theme_root = temp.path().join("workspace/theme"); + init_repo( + &theme_root, + &[ + ("config/zola.toml", "title = 'theme'\n"), + ("templates/index.html", "local theme\n"), + ], + ); + let repo_path = temp.path().join("workspace/.local-build/Core/repo"); + + let (theme_path, sync) = prepare_theme_for_zola(theme_root.clone(), &repo_path).unwrap(); + + let mounted_theme_dir = mounted_theme_path(&repo_path); + assert_eq!(theme_path, mounted_theme_dir); + assert_eq!( + theme_config_path(&mounted_theme_dir), + repo_path.join("themes/eips-theme/config/zola.toml") + ); + assert_eq!( + std::fs::read_to_string(mounted_theme_dir.join("templates/index.html")).unwrap(), + "local theme\n" + ); + assert_eq!(sync.theme_source_root, theme_root); + assert_eq!(sync.mounted_theme_dir, mounted_theme_dir); + assert!(sync.theme_index_path.ends_with(".git/index")); + } + + #[test] + fn prepared_runtime_source_succeeds_with_unreachable_active_upstream() { + for command in [&["build"][..], &["check"][..], &["serve"][..]] { + let workspace = runtime_workspace(false); + let resolved = resolved_runtime(&workspace, command); + + prepare_resolved_source(&resolved).unwrap(); + + assert_eq!( + std::fs::read_to_string(prepared_path(&resolved, "content/00001.md")).unwrap(), + "active proposal\n" + ); + } + } + + #[test] + fn prepared_runtime_source_uses_remote_siblings_without_active_upstream_fetch() { + let workspace = runtime_workspace(true); + let resolved = resolved_runtime(&workspace, &["--remote-siblings", "build"]); + + prepare_resolved_source(&resolved).unwrap(); + + assert_eq!( + std::fs::read_to_string(prepared_path(&resolved, "content/00001.md")).unwrap(), + "active proposal\n" + ); + assert_eq!( + std::fs::read_to_string(prepared_path(&resolved, "content/00002.md")).unwrap(), + "sibling\n" + ); + } + + #[test] + fn remote_environment_runtime_source_prep_keeps_local_active_checkout() { + for command in [ + &["--staging", "build"][..], + &["--production", "check"][..], + &["parity", "serve"][..], + ] { + let workspace = runtime_workspace(false); + let resolved = resolved_runtime(&workspace, command); + + assert_eq!( + resolved.source_materialization, + SourceMaterialization::Clean + ); + prepare_resolved_source(&resolved).unwrap(); + + assert_eq!( + std::fs::read_to_string(prepared_path(&resolved, "content/00001.md")).unwrap(), + "active proposal\n" + ); + } + } + + #[test] + fn changed_still_requires_active_upstream() { + let workspace = runtime_workspace(false); + let resolved = resolved_runtime(&workspace, &["changed"]); + std::fs::create_dir_all(&resolved.build_path).unwrap(); + + let error = changed::run( + &resolved, + &resolved.build_path, + false, + &ChangedFormat::Newline, + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("fetching upstream repo")); + } + + #[test] + fn editorial_check_site_phase_source_prep_does_not_fetch_active_upstream() { + let workspace = runtime_workspace(false); + let active_path = workspace.active_path.to_str().unwrap(); + let args = Args::try_parse_from([ + "build-eips", + "-C", + active_path, + "editorial", + "check", + "--against-upstream", + ]) + .unwrap(); + let resolved = resolve_execution(&args).unwrap(); + let RuntimeOperation::Editorial { + command: EditorialCommand::Check { selectors, .. }, + } = args.operation.runtime_operation().unwrap() + else { + panic!("expected editorial check runtime operation"); + }; + let resolved = editorial_runtime_execution(resolved, &selectors); + + prepare_resolved_source(&resolved).unwrap(); + + assert_eq!( + std::fs::read_to_string(prepared_path(&resolved, "content/00001.md")).unwrap(), + "active proposal\n" + ); + } + + #[test] + fn proposal_catalog_collection_uses_prepared_merged_sources() { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("EIPs"); + let sibling_path = workspace_root.join("ERCs"); + let active_url = file_url(&active_path); + let sibling_url = file_url(&sibling_path); + let manifest = repo_manifest_text("EIPs", &active_url, &[("ERCs", sibling_url)]); + let active_markdown = pipeline_proposal_markdown(1, None, "Active proposal."); + let sibling_markdown = pipeline_proposal_markdown(2, Some("ERC"), "Sibling proposal."); + + write_file(&workspace_root, config::LOCAL_CONFIG_FILE, ""); + std::fs::create_dir_all(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + let _active_repo = init_repo( + &active_path, + &[ + (config::REPO_MANIFEST_FILE, manifest.as_str()), + ("content/00001.md", active_markdown.as_str()), + ], + ); + let _sibling_repo = init_repo( + &sibling_path, + &[("content/00002.md", sibling_markdown.as_str())], + ); + let workspace = RuntimeWorkspace { + _temp: temp, + active_path, + }; + let resolved = resolved_runtime(&workspace, &["build"]); + + prepare_resolved_source(&resolved).unwrap(); + let catalog = + collect_proposal_catalog(&prepared_path(&resolved, CONTENT_DIR), None).unwrap(); + let records = catalog.into_records(); + + assert!(!resolved.root_path.join("content/00002.md").exists()); + assert_eq!(records["erc-2"].title, "Proposal 2"); + } + + #[test] + fn network_upgrade_collection_uses_prepared_merged_sources_without_active_fetch() { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("EIPs"); + let sibling_path = workspace_root.join("ERCs"); + let missing_upstream = file_url(&temp.path().join("missing-upstream")); + let sibling_url = file_url(&sibling_path); + let manifest = repo_manifest_text("EIPs", &missing_upstream, &[("ERCs", sibling_url)]); + let meta_markdown = pipeline_meta_markdown( + 7773, + "2024-09-26", + "### Included EIPs\n\n* [EIP-2](./00002.md)\n", + ); + let sibling_markdown = pipeline_proposal_markdown(2, None, "Sibling proposal."); + + write_file(&workspace_root, config::LOCAL_CONFIG_FILE, ""); + std::fs::create_dir_all(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + let _active_repo = init_repo( + &active_path, + &[ + (config::REPO_MANIFEST_FILE, manifest.as_str()), + ("content/07773.md", meta_markdown.as_str()), + ], + ); + let _sibling_repo = init_repo( + &sibling_path, + &[("content/00002.md", sibling_markdown.as_str())], + ); + let workspace = RuntimeWorkspace { + _temp: temp, + active_path, + }; + let resolved = resolved_runtime(&workspace, &["build"]); + + prepare_resolved_source(&resolved).unwrap(); + let catalog = + collect_proposal_catalog(&prepared_path(&resolved, CONTENT_DIR), None).unwrap(); + let index = collect_network_upgrades_with_registries( + &catalog, + &[NetworkUpgradeRegistrySource { + bucket: NetworkUpgradeSourceBucket::Transitional, + source_meta_eip: 7773, + parser_mode: NetworkUpgradeParserMode::ModernStages, + upgrades: vec![NetworkUpgradeRegistryUpgrade { + display_name: "Glamsterdam", + slug: None, + sort_order: None, + }], + }], + &[], + ) + .unwrap(); + + assert!(!resolved.root_path.join("content/00002.md").exists()); + assert_eq!(index.upgrades[0].stages[0].rows[0].number.get(), 2); + } +} diff --git a/src/preview.rs b/src/preview.rs new file mode 100644 index 0000000..4f91335 --- /dev/null +++ b/src/preview.rs @@ -0,0 +1,263 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +use std::{ + fs::File, + io::ErrorKind, + path::{Component, Path, PathBuf}, +}; + +use log::info; +use snafu::{ResultExt, Whatever}; +use tiny_http::{Header, Method, Request, Response, Server, StatusCode}; + +use crate::config::ServerBinding; + +const INDEX_HTML: &str = "index.html"; + +pub fn serve(output_path: &Path, server_binding: &ServerBinding) -> Result<(), Whatever> { + if !output_path.is_dir() { + snafu::whatever!( + "preview output directory `{}` is missing; run `build-eips build` for this profile first", + output_path.to_string_lossy() + ); + } + + let server = match Server::http((server_binding.host.as_str(), server_binding.port)) { + Ok(server) => server, + Err(error) => { + snafu::whatever!("unable to bind preview server on {server_binding}: {error}") + } + }; + + info!( + "serving static preview from `{}` at http://{server_binding}/", + output_path.to_string_lossy() + ); + + for request in server.incoming_requests() { + handle_request(output_path, request)?; + } + + Ok(()) +} + +fn handle_request(output_path: &Path, request: Request) -> Result<(), Whatever> { + match *request.method() { + Method::Get | Method::Head => {} + _ => { + request + .respond(Response::empty(StatusCode(405))) + .whatever_context("unable to send preview method error response")?; + return Ok(()); + } + } + + let Some(paths) = resolve_request_paths(output_path, request.url()) else { + request + .respond(Response::empty(StatusCode(400))) + .whatever_context("unable to send preview bad request response")?; + return Ok(()); + }; + + let Some((path, file)) = open_preview_asset(paths)? else { + request + .respond(Response::empty(StatusCode(404))) + .whatever_context("unable to send preview not found response")?; + return Ok(()); + }; + + let response = if let Some(value) = content_type(&path) { + Response::from_file(file).with_header(content_type_header(value)) + } else { + Response::from_file(file) + }; + + request + .respond(response) + .with_whatever_context(|e| format!("unable to send preview response: {e}"))?; + + Ok(()) +} + +fn open_preview_asset(paths: Vec) -> Result, Whatever> { + for path in paths { + let file = match File::open(&path) { + Ok(file) => file, + Err(error) + if matches!(error.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => + { + continue; + } + Err(error) => { + snafu::whatever!( + "unable to open preview asset `{}`: {error}", + path.to_string_lossy() + ); + } + }; + + if !file + .metadata() + .with_whatever_context(|e| { + format!( + "unable to inspect preview asset `{}`: {e}", + path.to_string_lossy() + ) + })? + .is_file() + { + continue; + } + + return Ok(Some((path, file))); + } + + Ok(None) +} + +fn resolve_request_paths(output_path: &Path, url: &str) -> Option> { + let raw_path = url.split('?').next().unwrap_or("/"); + let mut resolved = output_path.to_path_buf(); + let mut saw_normal_component = false; + + for component in Path::new(raw_path.trim_start_matches('/')).components() { + match component { + Component::CurDir | Component::RootDir => {} + Component::Normal(component) => { + saw_normal_component = true; + resolved.push(component); + } + Component::ParentDir | Component::Prefix(_) => return None, + } + } + + if raw_path.ends_with('/') || !saw_normal_component { + return Some(vec![resolved.join(INDEX_HTML)]); + } + + if resolved.extension().is_none() { + return Some(vec![resolved.join(INDEX_HTML), resolved]); + } + + Some(vec![resolved]) +} + +fn content_type(path: &Path) -> Option<&'static str> { + match path.extension().and_then(|extension| extension.to_str()) { + Some("css") => Some("text/css; charset=utf-8"), + Some("gif") => Some("image/gif"), + Some("htm" | "html") => Some("text/html; charset=utf-8"), + Some("ico") => Some("image/x-icon"), + Some("jpeg" | "jpg") => Some("image/jpeg"), + Some("js") => Some("application/javascript; charset=utf-8"), + Some("json") => Some("application/json; charset=utf-8"), + Some("mjs") => Some("application/javascript; charset=utf-8"), + Some("png") => Some("image/png"), + Some("svg") => Some("image/svg+xml"), + Some("txt") => Some("text/plain; charset=utf-8"), + Some("webp") => Some("image/webp"), + Some("xml") => Some("application/xml; charset=utf-8"), + _ => None, + } +} + +fn content_type_header(value: &str) -> Header { + Header::from_bytes(b"Content-Type", value.as_bytes()) + .expect("hard-coded content-type headers must be valid") +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use tempfile::TempDir; + + use super::{open_preview_asset, resolve_request_paths, INDEX_HTML}; + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn candidates(root: &Path, url: &str) -> Vec { + resolve_request_paths(root, url).unwrap() + } + + fn selected_path(root: &Path, url: &str) -> Option { + open_preview_asset(candidates(root, url)) + .unwrap() + .map(|(path, _file)| path) + } + + #[test] + fn slash_path_resolves_to_index_candidate() { + let temp = TempDir::new().unwrap(); + let root = temp.path(); + + assert_eq!( + candidates(root, "/foo/"), + vec![root.join("foo").join(INDEX_HTML)] + ); + } + + #[test] + fn extensionless_path_prefers_index_then_file_candidate() { + let temp = TempDir::new().unwrap(); + let root = temp.path(); + + assert_eq!( + candidates(root, "/foo"), + vec![root.join("foo").join(INDEX_HTML), root.join("foo")] + ); + } + + #[test] + fn extension_path_resolves_to_file_candidate() { + let temp = TempDir::new().unwrap(); + let root = temp.path(); + + assert_eq!(candidates(root, "/foo.css"), vec![root.join("foo.css")]); + } + + #[test] + fn parent_traversal_is_rejected() { + let temp = TempDir::new().unwrap(); + + assert!(resolve_request_paths(temp.path(), "/../secret.txt").is_none()); + } + + #[test] + fn extensionless_path_serves_index_when_present() { + let temp = TempDir::new().unwrap(); + let root = temp.path(); + write_file(root, "foo/index.html", "index"); + + assert_eq!( + selected_path(root, "/foo"), + Some(root.join("foo").join(INDEX_HTML)) + ); + } + + #[test] + fn extensionless_path_serves_file_without_index() { + let temp = TempDir::new().unwrap(); + let root = temp.path(); + write_file(root, "foo", "file"); + + assert_eq!(selected_path(root, "/foo"), Some(root.join("foo"))); + } + + #[test] + fn missing_path_returns_no_asset() { + let temp = TempDir::new().unwrap(); + + assert!(selected_path(temp.path(), "/missing").is_none()); + } +} diff --git a/src/print.rs b/src/print.rs index 4220cb2..33b0945 100644 --- a/src/print.rs +++ b/src/print.rs @@ -8,7 +8,7 @@ use eipw_lint::config::DefaultOptions; #[derive(Debug, clap::Args, Clone)] pub struct CmdArgs { - /// Thing to print + /// Linter metadata or configuration output to print what: What, } diff --git a/src/proposal.rs b/src/proposal.rs new file mode 100644 index 0000000..f9392df --- /dev/null +++ b/src/proposal.rs @@ -0,0 +1,1399 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Proposal path classification and targeted render policy. + +use std::{ + collections::{BTreeMap, BTreeSet}, + ffi::OsStr, + fmt, + num::NonZeroU32, + path::{Path, PathBuf}, +}; + +use eipw_preamble::Preamble; +use log::warn; +use serde::{ + de::{self, Unexpected, Visitor}, + Deserialize, Deserializer, Serialize, Serializer, +}; +use snafu::{OptionExt, ResultExt, Whatever}; +use walkdir::WalkDir; + +use crate::layout::CONTENT_DIR; + +/// Positive proposal number used by CLI selectors and `[render].only` config. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ProposalNumber(NonZeroU32); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ProposalNumberParseFailure { + Empty, + NonDigit, + Zero, + Overflow, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum EditorialNumberSelector { + Number(ProposalNumber), + InvalidNumberLike(ProposalNumberParseFailure), + PathLike, +} + +impl ProposalNumber { + fn parse_selector(selector: &str) -> Result { + if selector.is_empty() { + return Err(ProposalNumberParseFailure::Empty); + } + + if !selector.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(ProposalNumberParseFailure::NonDigit); + } + + let number = selector + .parse::() + .map_err(|_| ProposalNumberParseFailure::Overflow)?; + + Self::from_u32(number).map_err(|_| ProposalNumberParseFailure::Zero) + } + + pub(crate) fn parse_cli_selector(selector: &str) -> Result { + Self::parse_selector(selector).map_err(|_| { + format!( + "`{selector}` is not a valid --only selector; expected a positive proposal number" + ) + }) + } + + pub(crate) fn from_u32(number: u32) -> Result { + NonZeroU32::new(number).map(Self).ok_or(()) + } + + pub(crate) fn get(self) -> u32 { + self.0.get() + } +} + +pub(crate) fn classify_editorial_number_selector(selector: &str) -> EditorialNumberSelector { + match ProposalNumber::parse_selector(selector) { + Ok(number) => EditorialNumberSelector::Number(number), + Err(failure) if is_number_like_selector(selector) => { + EditorialNumberSelector::InvalidNumberLike(failure) + } + Err(_) => EditorialNumberSelector::PathLike, + } +} + +fn is_number_like_selector(selector: &str) -> bool { + !selector.is_empty() + && selector + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'+' | b'-' | b',')) +} + +impl fmt::Display for ProposalNumber { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}", self.get()) + } +} + +impl Serialize for ProposalNumber { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_u32(self.get()) + } +} + +impl<'de> Deserialize<'de> for ProposalNumber { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct ProposalNumberVisitor; + + impl Visitor<'_> for ProposalNumberVisitor { + type Value = ProposalNumber; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a positive proposal number") + } + + fn visit_i64(self, value: i64) -> Result + where + E: de::Error, + { + let value = u32::try_from(value).map_err(|_| { + E::invalid_value(Unexpected::Signed(value), &"a positive u32 proposal number") + })?; + ProposalNumber::from_u32(value).map_err(|_| { + E::invalid_value( + Unexpected::Unsigned(u64::from(value)), + &"a non-zero proposal number", + ) + }) + } + + fn visit_u64(self, value: u64) -> Result + where + E: de::Error, + { + let value = u32::try_from(value).map_err(|_| { + E::invalid_value( + Unexpected::Unsigned(value), + &"a positive u32 proposal number", + ) + })?; + ProposalNumber::from_u32(value).map_err(|_| { + E::invalid_value( + Unexpected::Unsigned(u64::from(value)), + &"a non-zero proposal number", + ) + }) + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + Err(E::invalid_type(Unexpected::Str(value), &self)) + } + } + + deserializer.deserialize_any(ProposalNumberVisitor) + } +} + +#[derive(Debug)] +pub(crate) enum ProposalReference<'a> { + Internal(String), + External(&'a str), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ProposalAssetKind { + Static, + Markdown, +} + +impl ProposalAssetKind { + pub(crate) fn from_path(path: &Path) -> Self { + if path.extension().and_then(OsStr::to_str) == Some("md") { + Self::Markdown + } else { + Self::Static + } + } +} + +#[derive(Debug, Clone)] +#[allow(dead_code)] +struct ProposalAssetInventoryEntry { + proposal_number: ProposalNumber, + site: ProposalPublicSite, + asset_relative_path: PathBuf, + kind: ProposalAssetKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ProposalPublicSite { + Eips, + Ercs, +} + +impl ProposalPublicSite { + pub(crate) fn proposal_url(self, proposal_number: ProposalNumber) -> String { + match self { + Self::Eips => format!( + "https://eips.ethereum.org/EIPS/eip-{}", + proposal_number.get() + ), + Self::Ercs => format!( + "https://ercs.ethereum.org/ERCS/erc-{}", + proposal_number.get() + ), + } + } + + fn asset_base_url(self) -> &'static str { + match self { + Self::Eips => "https://eips.ethereum.org", + Self::Ercs => "https://ercs.ethereum.org", + } + } +} + +#[derive(Debug, Clone)] +pub(crate) struct OnlyRenderPlan { + selected_numbers: BTreeSet, + asset_inventory: BTreeMap, + canonical_proposal_numbers: BTreeMap, + markdown_paths_by_number: BTreeMap>, + public_sites_by_number: BTreeMap, + public_urls_by_number: BTreeMap, +} + +impl OnlyRenderPlan { + pub(crate) fn build( + content_root: &Path, + selected_numbers: BTreeSet, + ) -> Result { + let mut plan = Self { + selected_numbers, + asset_inventory: BTreeMap::new(), + canonical_proposal_numbers: BTreeMap::new(), + markdown_paths_by_number: BTreeMap::new(), + public_sites_by_number: BTreeMap::new(), + public_urls_by_number: BTreeMap::new(), + }; + + let entries = std::fs::read_dir(content_root).with_whatever_context(|_| { + format!( + "unable to read materialized content directory `{}`", + content_root.to_string_lossy() + ) + })?; + + for entry in entries { + let entry = entry.with_whatever_context(|_| { + format!( + "unable to read materialized content directory entry in `{}`", + content_root.to_string_lossy() + ) + })?; + let entry_path = entry.path(); + let file_type = entry.file_type().with_whatever_context(|_| { + format!( + "unable to inspect materialized content path `{}`", + entry_path.to_string_lossy() + ) + })?; + + if file_type.is_file() { + let Some(number) = flat_proposal_number(&entry_path) else { + continue; + }; + plan.record_markdown_path(content_root, number, &entry_path)?; + } else if file_type.is_dir() { + let Some(number) = path_component_proposal_number(entry_path.file_name()) else { + continue; + }; + let index_path = entry_path.join("index.md"); + match std::fs::read_to_string(&index_path) { + Ok(contents) => { + plan.record_markdown_contents(content_root, number, &index_path, &contents)? + } + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => {} + Err(error) => { + snafu::whatever!( + "unable to read proposal markdown `{}`: {error}", + index_path.to_string_lossy() + ); + } + } + } + } + + plan.inventory_assets(content_root)?; + + for selected_number in &plan.selected_numbers { + if !plan.markdown_paths_by_number.contains_key(selected_number) { + snafu::whatever!("selected proposal `{selected_number}` was not found"); + } + } + + Ok(plan) + } + + fn record_markdown_path( + &mut self, + content_root: &Path, + proposal_number: ProposalNumber, + markdown_path: &Path, + ) -> Result<(), Whatever> { + let contents = std::fs::read_to_string(markdown_path).with_whatever_context(|_| { + format!( + "unable to read proposal markdown `{}`", + markdown_path.to_string_lossy() + ) + })?; + self.record_markdown_contents(content_root, proposal_number, markdown_path, &contents) + } + + fn record_markdown_contents( + &mut self, + content_root: &Path, + proposal_number: ProposalNumber, + markdown_path: &Path, + contents: &str, + ) -> Result<(), Whatever> { + let relative_path = markdown_path + .strip_prefix(content_root) + .with_whatever_context(|_| { + format!( + "proposal markdown `{}` is outside content root `{}`", + markdown_path.to_string_lossy(), + content_root.to_string_lossy() + ) + })? + .to_path_buf(); + let canonical_path = std::fs::canonicalize(markdown_path).with_whatever_context(|_| { + format!( + "unable to canonicalize proposal markdown `{}`", + markdown_path.to_string_lossy() + ) + })?; + let site = public_site_for_markdown(markdown_path, contents)?; + let public_url = site.proposal_url(proposal_number); + + match self.public_urls_by_number.get(&proposal_number) { + Some(existing_url) if existing_url != &public_url => { + snafu::whatever!( + "proposal `{proposal_number}` has conflicting public URLs `{existing_url}` and `{public_url}`" + ); + } + Some(_) => {} + None => { + self.public_sites_by_number.insert(proposal_number, site); + self.public_urls_by_number + .insert(proposal_number, public_url); + } + } + + self.canonical_proposal_numbers + .insert(canonical_path, proposal_number); + self.markdown_paths_by_number + .entry(proposal_number) + .or_default() + .insert(relative_path); + + Ok(()) + } + + fn inventory_assets(&mut self, content_root: &Path) -> Result<(), Whatever> { + let canon_root = std::fs::canonicalize(content_root).with_whatever_context(|_| { + format!( + "unable to canonicalize content root `{}` for proposal asset inventory", + content_root.to_string_lossy() + ) + })?; + + let proposal_assets = self + .markdown_paths_by_number + .iter() + .flat_map(|(proposal_number, markdown_paths)| { + markdown_paths.iter().map(move |markdown_path| { + ( + *proposal_number, + markdown_path.clone(), + asset_dir_for_markdown_path(markdown_path), + ) + }) + }) + .collect::>(); + + for (proposal_number, markdown_path, asset_dir) in proposal_assets { + let Some(site) = self.public_sites_by_number.get(&proposal_number).copied() else { + continue; + }; + + let absolute_asset_dir = content_root.join(&asset_dir); + for entry in WalkDir::new(&absolute_asset_dir) + .follow_links(true) + .into_iter() + { + let entry = match entry { + Ok(entry) => entry, + Err(error) if missing_asset_dir(&error) => continue, + Err(error) => { + return Err(error).with_whatever_context(|_| { + format!( + "couldn't read proposal asset inventory entry in `{}`", + absolute_asset_dir.to_string_lossy() + ) + }); + } + }; + + if !entry.file_type().is_file() { + continue; + } + + let candidate = match std::fs::canonicalize(entry.path()) { + Ok(candidate) => candidate, + Err(error) => { + warn!( + "unable to canonicalize `{}`: {error}", + entry.path().to_string_lossy() + ); + continue; + } + }; + + if !candidate.starts_with(&canon_root) { + warn!( + "asset `{}` not in root, skipping", + entry.path().to_string_lossy() + ); + continue; + } + + let content_relative_path = entry + .path() + .strip_prefix(content_root) + .with_whatever_context(|_| { + format!( + "proposal asset `{}` for `{}` is outside content root `{}`", + entry.path().to_string_lossy(), + markdown_path.to_string_lossy(), + content_root.to_string_lossy() + ) + })?; + let asset_relative_path = entry + .path() + .strip_prefix(&absolute_asset_dir) + .with_whatever_context(|_| { + format!( + "proposal asset `{}` is outside asset directory `{}`", + entry.path().to_string_lossy(), + absolute_asset_dir.to_string_lossy() + ) + })?; + let entry = ProposalAssetInventoryEntry { + proposal_number, + site, + asset_relative_path: asset_relative_path.to_path_buf(), + kind: ProposalAssetKind::from_path(asset_relative_path), + }; + + self.asset_inventory + .insert(content_relative_path.to_path_buf(), entry); + } + } + + Ok(()) + } + + pub(crate) fn has_proposal_asset(&self, content_relative_asset_path: &Path) -> bool { + self.asset_inventory + .contains_key(content_relative_asset_path) + } + + pub(crate) fn public_url_for_omitted_proposal_asset( + &self, + content_relative_asset_path: &Path, + ) -> Option { + let entry = self.asset_inventory.get(content_relative_asset_path)?; + if self.selected_numbers.contains(&entry.proposal_number) { + return None; + } + + Some(public_asset_url(entry)) + } + + pub(crate) fn is_selected_number(&self, proposal_number: ProposalNumber) -> bool { + self.selected_numbers.contains(&proposal_number) + } + + pub(crate) fn external_url_for_canonical_target( + &self, + canonical_target: &Path, + ) -> Option<&str> { + let proposal_number = self.canonical_proposal_numbers.get(canonical_target)?; + if self.selected_numbers.contains(proposal_number) { + return None; + } + + self.public_urls_by_number + .get(proposal_number) + .map(String::as_str) + } + + pub(crate) fn external_url_for_content_target( + &self, + content_relative_path: &Path, + ) -> Option<&str> { + let proposal_number = proposal_number_from_content_markdown_path(content_relative_path)?; + if self.selected_numbers.contains(&proposal_number) { + return None; + } + + self.public_urls_by_number + .get(&proposal_number) + .map(String::as_str) + } + + pub(crate) fn reference_for_required_number( + &self, + proposal_number: ProposalNumber, + ) -> Result, Whatever> { + if self.selected_numbers.contains(&proposal_number) { + let markdown_path = self + .markdown_paths_by_number + .get(&proposal_number) + .and_then(|paths| paths.iter().next()) + .with_whatever_context(|| { + format!("required selected proposal `{proposal_number}` was not found") + })?; + return Ok(ProposalReference::Internal(format!( + "@/{}", + markdown_path.to_string_lossy() + ))); + } + + let public_url = self + .public_urls_by_number + .get(&proposal_number) + .with_whatever_context(|| { + format!("required proposal `{proposal_number}` was not found") + })?; + Ok(ProposalReference::External(public_url)) + } + + pub(crate) fn should_preprocess_markdown(&self, content_relative_path: &Path) -> bool { + match proposal_number_from_content_markdown_path(content_relative_path) { + Some(proposal_number) => { + self.selected_numbers.contains(&proposal_number) + && self + .markdown_paths_by_number + .get(&proposal_number) + .map(|paths| paths.contains(content_relative_path)) + .unwrap_or(false) + } + None => true, + } + } + + pub(crate) fn should_process_proposal_dir(&self, content_relative_path: &Path) -> bool { + path_component_proposal_number(content_relative_path.file_name()) + .map(|proposal_number| self.selected_numbers.contains(&proposal_number)) + .unwrap_or(true) + } + + pub(crate) fn should_sync_dirty_path(&self, repo_relative_path: &Path) -> bool { + let Ok(content_relative_path) = repo_relative_path.strip_prefix(CONTENT_DIR) else { + return true; + }; + + self.should_sync_content_dirty_path(content_relative_path) + } + + pub(crate) fn is_selected_proposal_markdown_path(&self, repo_relative_path: &Path) -> bool { + let Ok(content_relative_path) = repo_relative_path.strip_prefix(CONTENT_DIR) else { + return false; + }; + + self.is_selected_content_proposal_markdown_path(content_relative_path) + } + + fn is_selected_content_proposal_markdown_path(&self, content_relative_path: &Path) -> bool { + let Some(proposal_number) = + proposal_number_from_content_markdown_path(content_relative_path) + else { + return false; + }; + + self.selected_numbers.contains(&proposal_number) + && self + .markdown_paths_by_number + .get(&proposal_number) + .map(|paths| paths.contains(content_relative_path)) + .unwrap_or(false) + } + + fn should_sync_content_dirty_path(&self, content_relative_path: &Path) -> bool { + if proposal_number_from_content_markdown_path(content_relative_path).is_some() { + return self.is_selected_content_proposal_markdown_path(content_relative_path); + } + + let mut components = content_relative_path.components(); + let Some(first) = components.next() else { + return true; + }; + + path_component_proposal_number(Some(first.as_os_str())) + .map(|proposal_number| self.selected_numbers.contains(&proposal_number)) + .unwrap_or(true) + } + + pub(crate) fn prune_content(&self, content_root: &Path) -> Result<(), Whatever> { + let entries = std::fs::read_dir(content_root).with_whatever_context(|_| { + format!( + "unable to read materialized content directory `{}` for pruning", + content_root.to_string_lossy() + ) + })?; + + for entry in entries { + let entry = entry.with_whatever_context(|_| { + format!( + "unable to read materialized content directory entry in `{}` for pruning", + content_root.to_string_lossy() + ) + })?; + let entry_path = entry.path(); + let file_type = entry.file_type().with_whatever_context(|_| { + format!( + "unable to inspect materialized content path `{}` for pruning", + entry_path.to_string_lossy() + ) + })?; + + if file_type.is_file() { + let Some(number) = flat_proposal_number(&entry_path) else { + continue; + }; + if !self.selected_numbers.contains(&number) { + remove_file_if_present(&entry_path)?; + } + } else if file_type.is_dir() { + let Some(number) = path_component_proposal_number(entry_path.file_name()) else { + continue; + }; + if !self.selected_numbers.contains(&number) { + remove_dir_if_present(&entry_path)?; + } + } + } + + Ok(()) + } +} + +fn missing_asset_dir(error: &walkdir::Error) -> bool { + error.depth() == 0 + && error.io_error().is_some_and(|io_error| { + matches!( + io_error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) + }) +} + +fn asset_dir_for_markdown_path(markdown_path: &Path) -> PathBuf { + if markdown_path.file_name() == Some(OsStr::new("index.md")) { + markdown_path + .parent() + .map(|proposal_dir| proposal_dir.join("assets")) + .expect("index path has proposal parent") + } else { + markdown_path.with_extension("").join("assets") + } +} + +fn remove_file_if_present(path: &Path) -> Result<(), Whatever> { + match std::fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => + { + Ok(()) + } + Err(error) => { + snafu::whatever!( + "unable to prune unselected proposal file `{}`: {error}", + path.to_string_lossy() + ); + } + } +} + +fn remove_dir_if_present(path: &Path) -> Result<(), Whatever> { + match std::fs::remove_dir_all(path) { + Ok(()) => Ok(()), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => + { + Ok(()) + } + Err(error) => { + snafu::whatever!( + "unable to prune unselected proposal directory `{}`: {error}", + path.to_string_lossy() + ); + } + } +} + +pub(crate) fn parse_proposal_preamble<'a>( + markdown_path: &Path, + contents: &'a str, +) -> Result, Whatever> { + let path_lossy = markdown_path.to_string_lossy(); + let (preamble, _) = Preamble::split(contents) + .with_whatever_context(|_| format!("couldn't split preamble for `{path_lossy}`"))?; + Preamble::parse(None, preamble) + .ok() + .with_whatever_context(|| format!("couldn't parse preamble in `{path_lossy}`")) +} + +fn public_site_for_preamble(preamble: &Preamble<'_>) -> ProposalPublicSite { + let is_erc = preamble + .fields() + .any(|field| field.name() == "category" && field.value().trim() == "ERC"); + + if is_erc { + ProposalPublicSite::Ercs + } else { + ProposalPublicSite::Eips + } +} + +pub(crate) fn public_site_for_markdown( + markdown_path: &Path, + contents: &str, +) -> Result { + let preamble = parse_proposal_preamble(markdown_path, contents)?; + Ok(public_site_for_preamble(&preamble)) +} + +#[allow(dead_code)] +fn public_asset_url(entry: &ProposalAssetInventoryEntry) -> String { + let mut url = format!( + "{}/{}/assets", + entry.site.asset_base_url(), + entry.proposal_number.get() + ); + for component in entry.asset_relative_path.components() { + let std::path::Component::Normal(component) = component else { + continue; + }; + url.push('/'); + url.push_str(&percent_encode_url_segment(&component.to_string_lossy())); + } + + if entry.kind == ProposalAssetKind::Markdown { + url.truncate(url.len() - ".md".len()); + url.push('/'); + } + + url +} + +#[allow(dead_code)] +fn percent_encode_url_segment(segment: &str) -> String { + let mut encoded = String::with_capacity(segment.len()); + for byte in segment.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +pub(crate) fn flat_proposal_number(path: &Path) -> Option { + if path.extension().and_then(OsStr::to_str) != Some("md") { + return None; + } + + path_component_proposal_number(path.file_stem()) +} + +pub(crate) fn path_component_proposal_number(component: Option<&OsStr>) -> Option { + let name = component?.to_str()?; + if name.is_empty() || !name.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + + name.parse::() + .ok() + .and_then(|number| NonZeroU32::new(number).map(ProposalNumber)) +} + +pub(crate) fn proposal_number_from_content_markdown_path( + content_relative_path: &Path, +) -> Option { + let mut components = content_relative_path.components(); + let first = components.next()?; + let first_path = Path::new(first.as_os_str()); + + match components.next() { + None => flat_proposal_number(first_path), + Some(component) + if component.as_os_str() == OsStr::new("index.md") && components.next().is_none() => + { + path_component_proposal_number(Some(first.as_os_str())) + } + Some(_) => None, + } +} + +pub(crate) fn is_proposal_path(path: &Path) -> bool { + let Ok(content_relative_path) = path.strip_prefix(CONTENT_DIR) else { + return false; + }; + + proposal_number_from_content_markdown_path(content_relative_path).is_some() +} + +pub(crate) fn resolve_proposal_number_markdown_path( + active_repo_root: &Path, + proposal_number: ProposalNumber, +) -> Result { + let content_root = active_repo_root.join(CONTENT_DIR); + let mut matches = BTreeSet::new(); + let entries = std::fs::read_dir(&content_root).with_whatever_context(|_| { + format!( + "unable to read active repository content directory `{}`", + content_root.to_string_lossy() + ) + })?; + + for entry in entries { + let entry = entry.with_whatever_context(|_| { + format!( + "unable to read active repository content directory entry in `{}`", + content_root.to_string_lossy() + ) + })?; + let entry_path = entry.path(); + let file_type = entry.file_type().with_whatever_context(|_| { + format!( + "unable to inspect active repository content path `{}`", + entry_path.to_string_lossy() + ) + })?; + + if file_type.is_file() { + if flat_proposal_number(&entry_path) == Some(proposal_number) { + matches.insert(PathBuf::from(CONTENT_DIR).join(entry.file_name())); + } + } else if file_type.is_dir() + && path_component_proposal_number(Some(entry.file_name().as_os_str())) + == Some(proposal_number) + { + let index_path = entry_path.join("index.md"); + match std::fs::metadata(&index_path) { + Ok(metadata) if metadata.is_file() => { + matches.insert( + PathBuf::from(CONTENT_DIR) + .join(entry.file_name()) + .join("index.md"), + ); + } + Ok(_) => {} + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => {} + Err(error) => { + snafu::whatever!( + "unable to inspect proposal markdown `{}`: {error}", + index_path.to_string_lossy() + ); + } + } + } + } + + match matches.len() { + 0 => snafu::whatever!( + "proposal `{proposal_number}` was not found in active repository content" + ), + 1 => Ok(matches.into_iter().next().expect("one proposal path")), + _ => { + let paths = matches + .iter() + .map(|path| format!("`{}`", path.to_string_lossy())) + .collect::>() + .join(", "); + snafu::whatever!( + "proposal `{proposal_number}` has more than one markdown path in active repository content: {paths}" + ); + } + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use tempfile::TempDir; + + use super::{ + classify_editorial_number_selector, is_proposal_path, + proposal_number_from_content_markdown_path, resolve_proposal_number_markdown_path, + EditorialNumberSelector, OnlyRenderPlan, ProposalNumber, ProposalNumberParseFailure, + }; + + fn number(value: u32) -> ProposalNumber { + ProposalNumber::from_u32(value).unwrap() + } + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn proposal_markdown(number: u32, category: Option<&str>) -> String { + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!("---\neip: {number}\ntitle: Test\n{category}---\nBody\n") + } + + #[test] + fn proposal_numbers_parse_cli_selectors_strictly() { + assert_eq!( + ProposalNumber::parse_cli_selector("555").unwrap(), + number(555) + ); + assert_eq!( + ProposalNumber::parse_cli_selector("00555").unwrap(), + number(555) + ); + + for selector in ["+555", "0", "-555", "abc", "555,678", "content/00555.md"] { + let error = ProposalNumber::parse_cli_selector(selector).unwrap_err(); + assert_eq!( + error, + format!( + "`{selector}` is not a valid --only selector; expected a positive proposal number" + ) + ); + } + + let error = ProposalNumber::parse_cli_selector("4294967296").unwrap_err(); + assert_eq!( + error, + "`4294967296` is not a valid --only selector; expected a positive proposal number" + ); + } + + #[test] + fn editorial_number_selector_classifier_splits_numbers_invalid_numbers_and_paths() { + assert_eq!( + classify_editorial_number_selector("000555"), + EditorialNumberSelector::Number(number(555)) + ); + + for (selector, expected_failure) in [ + ("0", ProposalNumberParseFailure::Zero), + ("+555", ProposalNumberParseFailure::NonDigit), + ("-555", ProposalNumberParseFailure::NonDigit), + ("555,678", ProposalNumberParseFailure::NonDigit), + ("4294967296", ProposalNumberParseFailure::Overflow), + ] { + assert_eq!( + classify_editorial_number_selector(selector), + EditorialNumberSelector::InvalidNumberLike(expected_failure) + ); + } + + for selector in ["foo", "draft.md", "4a", "draft-4.md", "content/00555.md"] { + assert_eq!( + classify_editorial_number_selector(selector), + EditorialNumberSelector::PathLike + ); + } + } + + #[test] + fn proposal_path_matching_normalizes_numeric_paths() { + assert_eq!( + proposal_number_from_content_markdown_path(Path::new("555.md")), + Some(number(555)) + ); + assert_eq!( + proposal_number_from_content_markdown_path(Path::new("00555.md")), + Some(number(555)) + ); + assert_eq!( + proposal_number_from_content_markdown_path(Path::new("000555/index.md")), + Some(number(555)) + ); + assert!(is_proposal_path(Path::new("content/000555/index.md"))); + assert!(!is_proposal_path(Path::new( + "content/000555/assets/readme.md" + ))); + } + + #[test] + fn proposal_number_resolver_returns_exact_flat_markdown_path() { + for (selector, existing_path) in [ + ("4", "content/4.md"), + ("004", "content/0004.md"), + ("0004", "content/004.md"), + ] { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), existing_path, ""); + + assert_eq!( + resolve_proposal_number_markdown_path( + temp.path(), + ProposalNumber::parse_cli_selector(selector).unwrap(), + ) + .unwrap(), + Path::new(existing_path) + ); + } + } + + #[test] + fn proposal_number_resolver_returns_exact_directory_index_path() { + for (selector, existing_path) in [ + ("4", "content/4/index.md"), + ("004", "content/0004/index.md"), + ("0004", "content/004/index.md"), + ] { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), existing_path, ""); + + assert_eq!( + resolve_proposal_number_markdown_path( + temp.path(), + ProposalNumber::parse_cli_selector(selector).unwrap(), + ) + .unwrap(), + Path::new(existing_path) + ); + } + } + + #[test] + fn proposal_number_resolver_reports_missing_and_ignores_assets_only_dirs() { + let missing = TempDir::new().unwrap(); + write_file(missing.path(), "content/0005.md", ""); + let error = resolve_proposal_number_markdown_path(missing.path(), number(4)) + .unwrap_err() + .to_string(); + assert!(error.contains("proposal `4` was not found in active repository content")); + + let assets_only = TempDir::new().unwrap(); + write_file(assets_only.path(), "content/0004/assets/foo.png", ""); + let error = resolve_proposal_number_markdown_path(assets_only.path(), number(4)) + .unwrap_err() + .to_string(); + assert!(error.contains("proposal `4` was not found in active repository content")); + } + + #[test] + fn proposal_number_resolver_reports_ambiguous_markdown_paths() { + for paths in [ + &["content/4.md", "content/0004/index.md"][..], + &["content/4.md", "content/0004.md"][..], + &["content/4/index.md", "content/0004/index.md"][..], + ] { + let temp = TempDir::new().unwrap(); + for path in paths { + write_file(temp.path(), path, ""); + } + + let error = resolve_proposal_number_markdown_path(temp.path(), number(4)) + .unwrap_err() + .to_string(); + + assert!(error.contains( + "proposal `4` has more than one markdown path in active repository content" + )); + } + } + + #[test] + fn proposal_number_resolver_searches_only_active_repo_content() { + let temp = TempDir::new().unwrap(); + let active_repo = temp.path().join("active"); + let sibling_repo = temp.path().join("sibling"); + write_file(&active_repo, "content/0005.md", ""); + write_file(&sibling_repo, "content/0004.md", ""); + + let error = resolve_proposal_number_markdown_path(&active_repo, number(4)) + .unwrap_err() + .to_string(); + + assert!(error.contains("proposal `4` was not found in active repository content")); + } + + #[test] + fn only_render_plan_requires_selected_markdown_not_assets_only() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555/assets/foo.png", ""); + + let error = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()) + .unwrap_err() + .to_string(); + + assert!(error.contains("selected proposal `555` was not found")); + } + + #[test] + fn only_render_plan_reports_missing_selected_proposal() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + + let error = OnlyRenderPlan::build(content, [number(678)].into_iter().collect()) + .unwrap_err() + .to_string(); + + assert!(error.contains("selected proposal `678` was not found")); + } + + #[test] + fn only_render_plan_records_exact_public_urls_by_category() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", &proposal_markdown(678, Some("ERC"))); + write_file( + content, + "00777.md", + &proposal_markdown(777, Some("Standards Track")), + ); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert_eq!( + plan.public_urls_by_number.get(&number(555)).unwrap(), + "https://eips.ethereum.org/EIPS/eip-555" + ); + assert_eq!( + plan.public_urls_by_number.get(&number(678)).unwrap(), + "https://ercs.ethereum.org/ERCS/erc-678" + ); + assert_eq!( + plan.public_urls_by_number.get(&number(777)).unwrap(), + "https://eips.ethereum.org/EIPS/eip-777" + ); + } + + #[test] + fn only_render_plan_inventories_directory_proposal_assets() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file( + content, + "00678/index.md", + &proposal_markdown(678, Some("ERC")), + ); + write_file(content, "00678/assets/foo.pdf", ""); + write_file(content, "00678/assets/guide.md", ""); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert!(plan.has_proposal_asset(Path::new("00678/assets/foo.pdf"))); + assert!(plan.has_proposal_asset(Path::new("00678/assets/guide.md"))); + assert_eq!( + plan.public_url_for_omitted_proposal_asset(Path::new("00678/assets/foo.pdf")) + .unwrap(), + "https://ercs.ethereum.org/678/assets/foo.pdf" + ); + assert_eq!( + plan.public_url_for_omitted_proposal_asset(Path::new("00678/assets/guide.md")) + .unwrap(), + "https://ercs.ethereum.org/678/assets/guide/" + ); + } + + #[test] + fn only_render_plan_inventories_flat_proposal_assets_when_directory_exists() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", &proposal_markdown(678, None)); + write_file(content, "00678/assets/README.md", ""); + write_file(content, "00678/assets/index.md", ""); + write_file( + content, + "00678/assets/Contract Interactions diagram.svg", + "", + ); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert_eq!( + plan.public_url_for_omitted_proposal_asset(Path::new("00678/assets/README.md")) + .unwrap(), + "https://eips.ethereum.org/678/assets/README/" + ); + assert_eq!( + plan.public_url_for_omitted_proposal_asset(Path::new("00678/assets/index.md")) + .unwrap(), + "https://eips.ethereum.org/678/assets/index/" + ); + assert_eq!( + plan.public_url_for_omitted_proposal_asset(Path::new( + "00678/assets/Contract Interactions diagram.svg" + )) + .unwrap(), + "https://eips.ethereum.org/678/assets/Contract%20Interactions%20diagram.svg" + ); + } + + #[test] + fn only_render_plan_records_flat_proposals_without_assets_as_empty() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", &proposal_markdown(678, None)); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert!(!plan.has_proposal_asset(Path::new("00678/assets/foo.pdf"))); + assert!(plan + .public_url_for_omitted_proposal_asset(Path::new("00678/assets/foo.pdf")) + .is_none()); + } + + #[test] + fn only_render_plan_does_not_inventory_assets_only_numeric_dirs() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678/assets/foo.pdf", ""); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert!(!plan.has_proposal_asset(Path::new("00678/assets/foo.pdf"))); + assert!(plan + .public_url_for_omitted_proposal_asset(Path::new("00678/assets/foo.pdf")) + .is_none()); + } + + #[test] + fn only_render_plan_does_not_construct_public_asset_urls_for_selected_targets() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", &proposal_markdown(678, None)); + write_file(content, "00678/assets/foo.pdf", ""); + + let plan = OnlyRenderPlan::build(content, [number(555), number(678)].into_iter().collect()) + .unwrap(); + + assert!(plan.has_proposal_asset(Path::new("00678/assets/foo.pdf"))); + assert!(plan + .public_url_for_omitted_proposal_asset(Path::new("00678/assets/foo.pdf")) + .is_none()); + } + + #[test] + fn only_render_plan_errors_on_malformed_proposal_before_asset_inventory_policy() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", "not front matter"); + write_file(content, "00678/assets/foo.pdf", ""); + + let error = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()) + .unwrap_err() + .to_string(); + + assert!(error.contains("couldn't split preamble")); + } + + #[test] + fn only_render_plan_does_not_mask_missing_required_targets() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + let error = plan + .reference_for_required_number(number(678)) + .unwrap_err() + .to_string(); + + assert!(error.contains("required proposal `678` was not found")); + } + + #[test] + fn only_render_plan_does_not_mask_malformed_target_front_matter() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", "not front matter"); + + let error = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()) + .unwrap_err() + .to_string(); + + assert!(error.contains("couldn't split preamble")); + } + + #[test] + fn only_render_plan_detects_conflicting_public_urls() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file( + content, + "00555/index.md", + &proposal_markdown(555, Some("ERC")), + ); + + let error = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()) + .unwrap_err() + .to_string(); + + assert!(error.contains("conflicting public URLs")); + } + + #[test] + fn only_render_plan_prunes_unselected_proposal_content_only() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00555/assets/foo.png", ""); + write_file(content, "00678.md", &proposal_markdown(678, None)); + write_file(content, "00777/index.md", &proposal_markdown(777, None)); + write_file(content, "00777/assets/foo.png", ""); + write_file(content, "_index.md", "+++\ntitle = \"Home\"\n+++\n"); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + plan.prune_content(content).unwrap(); + + assert!(content.join("00555.md").is_file()); + assert!(content.join("00555/assets/foo.png").is_file()); + assert!(!content.join("00678.md").exists()); + assert!(!content.join("00777").exists()); + assert!(content.join("_index.md").is_file()); + } + + #[test] + fn only_render_plan_filters_dirty_paths_without_filesystem_state() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", &proposal_markdown(678, None)); + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert!(plan.should_sync_dirty_path(Path::new("content/00555.md"))); + assert!(plan.should_sync_dirty_path(Path::new("content/00555/assets/diagram.png"))); + assert!(plan.should_sync_dirty_path(Path::new("content/_index.md"))); + assert!(plan.should_sync_dirty_path(Path::new(".build-eips.repo.toml"))); + assert!(!plan.should_sync_dirty_path(Path::new("content/00678.md"))); + assert!(!plan.should_sync_dirty_path(Path::new("content/00678/assets/diagram.png"))); + assert!(!plan.should_sync_dirty_path(Path::new("content/00999.md"))); + + assert!(plan.is_selected_proposal_markdown_path(Path::new("content/00555.md"))); + assert!( + !plan.is_selected_proposal_markdown_path(Path::new("content/00555/assets/diagram.png")) + ); + assert!(!plan.is_selected_proposal_markdown_path(Path::new("content/_index.md"))); + assert!(!plan.is_selected_proposal_markdown_path(Path::new("content/00678.md"))); + } +} diff --git a/src/proposal_catalog.rs b/src/proposal_catalog.rs new file mode 100644 index 0000000..426743f --- /dev/null +++ b/src/proposal_catalog.rs @@ -0,0 +1,638 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Shared proposal catalog collection from prepared content sources. + +use std::{ + collections::BTreeMap, + fmt, + path::{Path, PathBuf}, +}; + +use eipw_preamble::Preamble; +use serde::Serialize; +use snafu::{OptionExt, ResultExt, Whatever}; + +use crate::proposal::{ + flat_proposal_number, path_component_proposal_number, public_site_for_markdown, OnlyRenderPlan, + ProposalNumber, ProposalPublicSite, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] +pub(crate) enum ProposalCatalogPrefix { + #[serde(rename = "EIP")] + Eip, + #[serde(rename = "ERC")] + Erc, +} + +impl ProposalCatalogPrefix { + pub(crate) fn key(self, proposal_number: ProposalNumber) -> String { + format!("{self}-{proposal_number}").to_ascii_lowercase() + } +} + +impl From for ProposalCatalogPrefix { + fn from(site: ProposalPublicSite) -> Self { + match site { + ProposalPublicSite::Eips => Self::Eip, + ProposalPublicSite::Ercs => Self::Erc, + } + } +} + +impl fmt::Display for ProposalCatalogPrefix { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Eip => formatter.write_str("EIP"), + Self::Erc => formatter.write_str("ERC"), + } + } +} + +#[derive(Debug, Clone, Serialize)] +pub(crate) struct ProposalCatalogRecord { + pub(crate) number: ProposalNumber, + pub(crate) prefix: ProposalCatalogPrefix, + pub(crate) title: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) description: Option, + pub(crate) status: String, + #[serde(rename = "type")] + pub(crate) proposal_type: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) category: Option, + pub(crate) url: String, +} + +#[derive(Debug, Clone)] +pub(crate) struct ProposalCatalog { + records: BTreeMap, + source_documents: BTreeMap, +} + +impl ProposalCatalog { + pub(crate) fn records(&self) -> &BTreeMap { + &self.records + } + + pub(crate) fn source_documents(&self) -> impl Iterator { + self.source_documents.values() + } + + pub(crate) fn source_document( + &self, + prefix: ProposalCatalogPrefix, + proposal_number: ProposalNumber, + ) -> Option<&ProposalCatalogSourceDocument> { + self.source_documents.get(&prefix.key(proposal_number)) + } + + pub(crate) fn into_records(self) -> BTreeMap { + self.records + } +} + +#[derive(Debug, Clone)] +pub(crate) struct ProposalCatalogSourceDocument { + number: ProposalNumber, + prefix: ProposalCatalogPrefix, + source_path: PathBuf, + preamble_fields: Vec<(String, String)>, + body: String, +} + +impl ProposalCatalogSourceDocument { + pub(crate) fn number(&self) -> ProposalNumber { + self.number + } + + pub(crate) fn prefix(&self) -> ProposalCatalogPrefix { + self.prefix + } + + pub(crate) fn source_path(&self) -> &Path { + &self.source_path + } + + pub(crate) fn field(&self, name: &str) -> Option<&str> { + self.preamble_fields + .iter() + .find(|(field_name, _)| field_name == name) + .map(|(_, value)| value.as_str()) + } + + pub(crate) fn created(&self) -> Option<&str> { + self.field("created") + } + + pub(crate) fn network_upgrade(&self) -> Option<&str> { + self.field("network-upgrade") + } + + pub(crate) fn proposal_type(&self) -> Option<&str> { + self.field("type") + } + + pub(crate) fn body(&self) -> &str { + &self.body + } +} + +#[derive(Debug)] +struct CollectedProposalCatalogRecord { + key: String, + source_path: PathBuf, + record: ProposalCatalogRecord, + source_document: ProposalCatalogSourceDocument, +} + +#[derive(Debug)] +struct ProposalCatalogFields { + title: String, + description: Option, + status: String, + proposal_type: String, + category: Option, +} + +impl ProposalCatalogFields { + fn parse( + markdown_path: &Path, + proposal_number: ProposalNumber, + preamble: &Preamble<'_>, + ) -> Result { + validate_preamble_proposal_number(markdown_path, proposal_number, preamble)?; + + Ok(Self { + title: required_catalog_field(markdown_path, preamble, "title")?, + description: optional_catalog_field(preamble, "description"), + status: required_catalog_field(markdown_path, preamble, "status")?, + proposal_type: required_catalog_field(markdown_path, preamble, "type")?, + category: optional_catalog_field(preamble, "category"), + }) + } +} + +struct ContentRootEntry { + entry_path: PathBuf, + file_type: std::fs::FileType, +} + +pub(crate) fn collect_proposal_catalog( + content_root: &Path, + only_plan: Option<&OnlyRenderPlan>, +) -> Result { + let mut records = BTreeMap::::new(); + + for entry in sorted_content_root_entries(content_root)? { + if entry.file_type.is_file() { + let Some(proposal_number) = flat_proposal_number(&entry.entry_path) else { + continue; + }; + insert_collected_proposal_catalog_record( + &mut records, + collect_proposal_catalog_from_path( + content_root, + proposal_number, + &entry.entry_path, + only_plan, + )?, + )?; + } else if entry.file_type.is_dir() { + let Some(proposal_number) = + path_component_proposal_number(entry.entry_path.file_name()) + else { + continue; + }; + let index_path = entry.entry_path.join("index.md"); + match std::fs::read_to_string(&index_path) { + Ok(contents) => { + insert_collected_proposal_catalog_record( + &mut records, + collect_proposal_catalog_from_contents( + content_root, + proposal_number, + &index_path, + &contents, + only_plan, + )?, + )?; + } + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => {} + Err(error) => { + snafu::whatever!( + "unable to read proposal markdown `{}`: {error}", + index_path.to_string_lossy() + ); + } + } + } + } + + let mut source_documents = BTreeMap::new(); + let records = records + .into_iter() + .map(|(key, metadata)| { + source_documents.insert(key.clone(), metadata.source_document); + (key, metadata.record) + }) + .collect(); + + Ok(ProposalCatalog { + records, + source_documents, + }) +} + +fn sorted_content_root_entries(content_root: &Path) -> Result, Whatever> { + let entries = std::fs::read_dir(content_root).with_whatever_context(|_| { + format!( + "unable to read materialized content directory `{}` for proposal metadata", + content_root.to_string_lossy() + ) + })?; + + let mut entries = entries + .map(|entry| -> Result { + let entry = entry.with_whatever_context(|_| { + format!( + "unable to read materialized content directory entry in `{}` for proposal metadata", + content_root.to_string_lossy() + ) + })?; + let entry_path = entry.path(); + let file_type = entry.file_type().with_whatever_context(|_| { + format!( + "unable to inspect materialized content path `{}` for proposal metadata", + entry_path.to_string_lossy() + ) + })?; + + Ok(ContentRootEntry { + entry_path, + file_type, + }) + }) + .collect::, Whatever>>()?; + entries.sort_by(|left, right| left.entry_path.cmp(&right.entry_path)); + Ok(entries) +} + +fn collect_proposal_catalog_from_path( + content_root: &Path, + proposal_number: ProposalNumber, + markdown_path: &Path, + only_plan: Option<&OnlyRenderPlan>, +) -> Result { + let contents = std::fs::read_to_string(markdown_path).with_whatever_context(|_| { + format!( + "unable to read proposal markdown `{}`", + markdown_path.to_string_lossy() + ) + })?; + collect_proposal_catalog_from_contents( + content_root, + proposal_number, + markdown_path, + &contents, + only_plan, + ) +} + +fn collect_proposal_catalog_from_contents( + content_root: &Path, + proposal_number: ProposalNumber, + markdown_path: &Path, + contents: &str, + only_plan: Option<&OnlyRenderPlan>, +) -> Result { + markdown_path + .strip_prefix(content_root) + .with_whatever_context(|_| { + format!( + "proposal markdown `{}` is outside content root `{}`", + markdown_path.to_string_lossy(), + content_root.to_string_lossy() + ) + })?; + + let site = public_site_for_markdown(markdown_path, contents)?; + let (preamble, body) = split_proposal_source(markdown_path, contents)?; + let prefix = ProposalCatalogPrefix::from(site); + let fields = ProposalCatalogFields::parse(markdown_path, proposal_number, &preamble)?; + let key = prefix.key(proposal_number); + let source_document = ProposalCatalogSourceDocument { + number: proposal_number, + prefix, + source_path: markdown_path.to_path_buf(), + preamble_fields: preamble + .fields() + .map(|field| (field.name().to_owned(), field.value().trim().to_owned())) + .collect(), + body: body.to_owned(), + }; + + Ok(CollectedProposalCatalogRecord { + key, + source_path: markdown_path.to_path_buf(), + record: ProposalCatalogRecord { + number: proposal_number, + prefix, + title: fields.title, + description: fields.description, + status: fields.status, + proposal_type: fields.proposal_type, + category: fields.category, + url: catalog_record_url(proposal_number, site, only_plan), + }, + source_document, + }) +} + +fn split_proposal_source<'a>( + markdown_path: &Path, + contents: &'a str, +) -> Result<(Preamble<'a>, &'a str), Whatever> { + let path_lossy = markdown_path.to_string_lossy(); + let (preamble, body) = Preamble::split(contents) + .with_whatever_context(|_| format!("couldn't split preamble for `{path_lossy}`"))?; + let preamble = Preamble::parse(None, preamble) + .ok() + .with_whatever_context(|| format!("couldn't parse preamble in `{path_lossy}`"))?; + + Ok((preamble, body)) +} + +fn insert_collected_proposal_catalog_record( + records: &mut BTreeMap, + metadata: CollectedProposalCatalogRecord, +) -> Result<(), Whatever> { + if let Some(existing) = records.get(&metadata.key) { + snafu::whatever!( + "duplicate proposal metadata key `{}` from `{}` and `{}`", + metadata.key, + existing.source_path.to_string_lossy(), + metadata.source_path.to_string_lossy() + ); + } + + records.insert(metadata.key.clone(), metadata); + Ok(()) +} + +fn catalog_record_url( + proposal_number: ProposalNumber, + site: ProposalPublicSite, + only_plan: Option<&OnlyRenderPlan>, +) -> String { + if only_plan.is_some_and(|plan| !plan.is_selected_number(proposal_number)) { + site.proposal_url(proposal_number) + } else { + format!("/{}/", proposal_number.get()) + } +} + +fn required_catalog_field( + markdown_path: &Path, + preamble: &Preamble<'_>, + field_name: &str, +) -> Result { + let Some(field) = preamble.by_name(field_name) else { + snafu::whatever!( + "missing required proposal metadata field `{field_name}` in `{}`", + markdown_path.to_string_lossy() + ); + }; + let value = field.value().trim(); + if value.is_empty() { + snafu::whatever!( + "missing required proposal metadata field `{field_name}` in `{}`", + markdown_path.to_string_lossy() + ); + } + + Ok(value.to_owned()) +} + +fn optional_catalog_field(preamble: &Preamble<'_>, field_name: &str) -> Option { + preamble + .by_name(field_name) + .map(|field| field.value().trim()) + .filter(|value| !value.is_empty()) + .map(str::to_owned) +} + +fn validate_preamble_proposal_number( + markdown_path: &Path, + path_proposal_number: ProposalNumber, + preamble: &Preamble<'_>, +) -> Result<(), Whatever> { + for field in preamble + .fields() + .filter(|field| matches!(field.name(), "eip" | "number")) + { + let field_name = field.name(); + let field_value = field.value().trim(); + let parsed = field_value.parse::().with_whatever_context(|_| { + format!( + "couldn't parse proposal number field `{field_name}` in `{}`", + markdown_path.to_string_lossy() + ) + })?; + let Ok(preamble_proposal_number) = ProposalNumber::from_u32(parsed) else { + snafu::whatever!( + "proposal number field `{field_name}` in `{}` must be positive", + markdown_path.to_string_lossy() + ); + }; + + if preamble_proposal_number != path_proposal_number { + snafu::whatever!( + "proposal metadata number mismatch in `{}`: path indicates `{path_proposal_number}`, but `{field_name}` contains `{preamble_proposal_number}`", + markdown_path.to_string_lossy() + ); + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use tempfile::TempDir; + + use super::{collect_proposal_catalog, ProposalCatalogPrefix}; + use crate::proposal::{OnlyRenderPlan, ProposalNumber}; + + fn number(value: u32) -> ProposalNumber { + ProposalNumber::from_u32(value).unwrap() + } + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn catalog_proposal_markdown( + number: u32, + title: &str, + category: Option<&str>, + description: Option<&str>, + ) -> String { + let description = description + .map(|description| format!("description: {description}\n")) + .unwrap_or_default(); + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!( + "---\neip: {number}\ntitle: {title}\n{description}status: Final\ntype: Standards Track\n{category}---\nBody\n" + ) + } + + #[test] + fn proposal_catalog_collects_flat_and_directory_proposals() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "020.md", + &catalog_proposal_markdown(20, "Flat Proposal", None, None), + ); + write_file( + temp.path(), + "21/index.md", + &catalog_proposal_markdown(21, "Directory Proposal", Some("ERC"), Some("Desc")), + ); + + let catalog = collect_proposal_catalog(temp.path(), None).unwrap(); + let records = catalog.into_records(); + + assert_eq!(records.len(), 2); + assert_eq!(records["eip-20"].number, number(20)); + assert_eq!(records["eip-20"].prefix, ProposalCatalogPrefix::Eip); + assert_eq!(records["eip-20"].title, "Flat Proposal"); + assert_eq!(records["eip-20"].url, "/20/"); + assert_eq!(records["erc-21"].number, number(21)); + assert_eq!(records["erc-21"].prefix, ProposalCatalogPrefix::Erc); + assert_eq!(records["erc-21"].description.as_deref(), Some("Desc")); + } + + #[test] + fn proposal_catalog_reports_malformed_preambles() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "001.md", "not front matter\n"); + + let error = collect_proposal_catalog(temp.path(), None) + .unwrap_err() + .to_string(); + + assert!(error.contains("couldn't split preamble")); + assert!(error.contains("001.md")); + } + + #[test] + fn proposal_catalog_reports_path_preamble_number_mismatch() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "001.md", + "---\neip: 2\ntitle: Proposal 1\nstatus: Final\ntype: Standards Track\n---\nBody\n", + ); + + let error = collect_proposal_catalog(temp.path(), None) + .unwrap_err() + .to_string(); + + assert!(error.contains("proposal metadata number mismatch")); + assert!(error.contains("001.md")); + assert!(error.contains("path indicates `1`")); + assert!(error.contains("`eip` contains `2`")); + } + + #[test] + fn proposal_catalog_collects_independently_of_json_writing() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "001.md", + &catalog_proposal_markdown(1, "Proposal 1", None, None), + ); + + let catalog = collect_proposal_catalog(temp.path(), None).unwrap(); + let records = catalog.into_records(); + + assert_eq!(records["eip-1"].status, "Final"); + assert!(!temp + .path() + .join("static/assets/data/proposals.json") + .exists()); + } + + #[test] + fn proposal_catalog_exposes_source_documents_separately_from_records() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "7773.md", + "---\neip: 7773\ntitle: Hardfork Meta\nstatus: Draft\ntype: Meta\ncreated: 2024-09-26\nnetwork-upgrade: Glamsterdam\n---\nBody\n", + ); + + let catalog = collect_proposal_catalog(temp.path(), None).unwrap(); + let records = catalog.records(); + let document = catalog.source_documents().next().unwrap(); + + assert_eq!(records["eip-7773"].title, "Hardfork Meta"); + assert_eq!(document.number(), number(7773)); + assert_eq!(document.prefix(), ProposalCatalogPrefix::Eip); + assert_eq!(document.created(), Some("2024-09-26")); + assert_eq!(document.network_upgrade(), Some("Glamsterdam")); + assert_eq!(document.body(), "Body\n"); + } + + #[test] + fn proposal_catalog_applies_targeted_url_policy() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "020.md", + &catalog_proposal_markdown(20, "Proposal 20", None, None), + ); + write_file( + temp.path(), + "021.md", + &catalog_proposal_markdown(21, "Proposal 21", None, None), + ); + write_file( + temp.path(), + "022.md", + &catalog_proposal_markdown(22, "Proposal 22", Some("ERC"), None), + ); + let plan = OnlyRenderPlan::build(temp.path(), [number(20)].into_iter().collect()).unwrap(); + + let catalog = collect_proposal_catalog(temp.path(), Some(&plan)).unwrap(); + let records = catalog.into_records(); + + assert_eq!(records["eip-20"].url, "/20/"); + assert_eq!( + records["eip-21"].url, + "https://eips.ethereum.org/EIPS/eip-21" + ); + assert_eq!( + records["erc-22"].url, + "https://ercs.ethereum.org/ERCS/erc-22" + ); + } +} diff --git a/src/proposal_metadata.rs b/src/proposal_metadata.rs new file mode 100644 index 0000000..60573e7 --- /dev/null +++ b/src/proposal_metadata.rs @@ -0,0 +1,546 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Proposal metadata JSON generation for theme popovers. + +use std::{ + collections::BTreeMap, + io::Write, + path::{Path, PathBuf}, +}; + +use serde::Serialize; +use snafu::{OptionExt, ResultExt, Whatever}; + +use crate::{ + layout::CONTENT_DIR, + proposal::OnlyRenderPlan, + proposal_catalog::{collect_proposal_catalog, ProposalCatalogPrefix, ProposalCatalogRecord}, +}; + +const PROPOSAL_METADATA_SCHEMA_VERSION: u8 = 1; + +#[derive(Debug, Serialize)] +struct ProposalMetadataIndex { + schema_version: u8, + active_prefix: ProposalCatalogPrefix, + proposals: BTreeMap, +} + +pub(crate) fn write_proposal_metadata_json( + repo_path: &Path, + repository_title: &str, + only_plan: Option<&OnlyRenderPlan>, +) -> Result<(), Whatever> { + let json_path = proposal_metadata_json_path(repo_path); + ensure_proposal_metadata_output_available(&json_path)?; + + let metadata = ProposalMetadataIndex { + schema_version: PROPOSAL_METADATA_SCHEMA_VERSION, + active_prefix: active_proposal_metadata_prefix(repository_title)?, + proposals: collect_proposal_catalog(&repo_path.join(CONTENT_DIR), only_plan)? + .into_records(), + }; + + write_proposal_metadata_file(&json_path, &metadata) +} + +fn proposal_metadata_json_path(repo_path: &Path) -> PathBuf { + repo_path + .join("static") + .join("assets") + .join("data") + .join("proposals.json") +} + +fn ensure_proposal_metadata_output_available(json_path: &Path) -> Result<(), Whatever> { + match std::fs::metadata(json_path) { + Ok(_) => { + snafu::whatever!( + "proposal metadata output `{}` already exists; refusing to overwrite it", + json_path.to_string_lossy() + ); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => { + snafu::whatever!( + "unable to inspect proposal metadata output `{}`: {error}", + json_path.to_string_lossy() + ); + } + } +} + +fn active_proposal_metadata_prefix( + repository_title: &str, +) -> Result { + match repository_title { + "EIPs" => Ok(ProposalCatalogPrefix::Eip), + "ERCs" => Ok(ProposalCatalogPrefix::Erc), + _ => { + snafu::whatever!( + "unsupported active repository title `{repository_title}` for proposal metadata; expected `EIPs` or `ERCs`" + ); + } + } +} + +fn write_proposal_metadata_file( + json_path: &Path, + metadata: &ProposalMetadataIndex, +) -> Result<(), Whatever> { + let parent = json_path.parent().with_whatever_context(|| { + format!( + "proposal metadata output path `{}` has no parent directory", + json_path.to_string_lossy() + ) + })?; + std::fs::create_dir_all(parent).with_whatever_context(|_| { + format!( + "unable to create proposal metadata directory `{}`", + parent.to_string_lossy() + ) + })?; + + let mut file = match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(json_path) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + snafu::whatever!( + "proposal metadata output `{}` already exists; refusing to overwrite it", + json_path.to_string_lossy() + ); + } + Err(error) => { + snafu::whatever!( + "unable to create proposal metadata output `{}`: {error}", + json_path.to_string_lossy() + ); + } + }; + + serde_json::to_writer_pretty(&mut file, metadata).with_whatever_context(|_| { + format!( + "unable to write proposal metadata JSON `{}`", + json_path.to_string_lossy() + ) + })?; + file.write_all(b"\n").with_whatever_context(|_| { + format!( + "unable to finish proposal metadata JSON `{}`", + json_path.to_string_lossy() + ) + })?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use serde_json::{json, Value}; + use tempfile::TempDir; + + use super::{proposal_metadata_json_path, write_proposal_metadata_json}; + use crate::proposal::{OnlyRenderPlan, ProposalNumber}; + + fn number(value: u32) -> ProposalNumber { + ProposalNumber::from_u32(value).unwrap() + } + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn metadata_proposal_markdown( + number: u32, + title: &str, + category: Option<&str>, + description: Option<&str>, + ) -> String { + let description = description + .map(|description| format!("description: {description}\n")) + .unwrap_or_default(); + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!( + "---\neip: {number}\ntitle: {title}\n{description}status: Final\ntype: Standards Track\n{category}---\nBody\n" + ) + } + + fn write_metadata_json( + repo_path: &Path, + repository_title: &str, + only_plan: Option<&OnlyRenderPlan>, + ) -> Value { + write_proposal_metadata_json(repo_path, repository_title, only_plan).unwrap(); + serde_json::from_str( + &std::fs::read_to_string(proposal_metadata_json_path(repo_path)).unwrap(), + ) + .unwrap() + } + + #[test] + fn proposal_metadata_full_build_writes_json() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown( + 20, + "Token Standard", + Some("ERC"), + Some("A standard interface for tokens."), + ), + ); + + let metadata = write_metadata_json(temp.path(), "EIPs", None); + let proposal = &metadata["proposals"]["erc-20"]; + + assert_eq!(metadata["schema_version"], json!(1)); + assert_eq!(metadata["active_prefix"], json!("EIP")); + assert_eq!(proposal["number"], json!(20)); + assert_eq!(proposal["prefix"], json!("ERC")); + assert_eq!(proposal["title"], json!("Token Standard")); + assert_eq!( + proposal["description"], + json!("A standard interface for tokens.") + ); + assert_eq!(proposal["status"], json!("Final")); + assert_eq!(proposal["type"], json!("Standards Track")); + assert_eq!(proposal["category"], json!("ERC")); + assert_eq!(proposal["url"], json!("/20/")); + } + + #[test] + fn proposal_metadata_full_build_records_use_local_urls() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown(20, "Proposal 20", None, None), + ); + + let metadata = write_metadata_json(temp.path(), "EIPs", None); + + assert_eq!(metadata["proposals"]["eip-20"]["url"], json!("/20/")); + } + + #[test] + fn proposal_metadata_targeted_build_includes_pre_prune_proposals() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown(20, "Proposal 20", None, None), + ); + write_file( + temp.path(), + "content/021.md", + &metadata_proposal_markdown(21, "Proposal 21", None, None), + ); + write_file( + temp.path(), + "content/022.md", + &metadata_proposal_markdown(22, "Proposal 22", Some("ERC"), None), + ); + let plan = OnlyRenderPlan::build( + &temp.path().join("content"), + [number(20)].into_iter().collect(), + ) + .unwrap(); + + let metadata = write_metadata_json(temp.path(), "EIPs", Some(&plan)); + let proposals = metadata["proposals"].as_object().unwrap(); + + assert_eq!(proposals.len(), 3); + assert!(proposals.contains_key("eip-20")); + assert!(proposals.contains_key("eip-21")); + assert!(proposals.contains_key("erc-22")); + } + + #[test] + fn proposal_metadata_targeted_selected_records_use_local_urls() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown(20, "Proposal 20", None, None), + ); + write_file( + temp.path(), + "content/021.md", + &metadata_proposal_markdown(21, "Proposal 21", None, None), + ); + let plan = OnlyRenderPlan::build( + &temp.path().join("content"), + [number(20)].into_iter().collect(), + ) + .unwrap(); + + let metadata = write_metadata_json(temp.path(), "EIPs", Some(&plan)); + + assert_eq!(metadata["proposals"]["eip-20"]["url"], json!("/20/")); + } + + #[test] + fn proposal_metadata_targeted_omitted_records_use_public_urls() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown(20, "Proposal 20", None, None), + ); + write_file( + temp.path(), + "content/021.md", + &metadata_proposal_markdown(21, "Proposal 21", None, None), + ); + write_file( + temp.path(), + "content/022.md", + &metadata_proposal_markdown(22, "Proposal 22", Some("ERC"), None), + ); + let plan = OnlyRenderPlan::build( + &temp.path().join("content"), + [number(20)].into_iter().collect(), + ) + .unwrap(); + + let metadata = write_metadata_json(temp.path(), "EIPs", Some(&plan)); + + assert_eq!( + metadata["proposals"]["eip-21"]["url"], + json!("https://eips.ethereum.org/EIPS/eip-21") + ); + assert_eq!( + metadata["proposals"]["erc-22"]["url"], + json!("https://ercs.ethereum.org/ERCS/erc-22") + ); + } + + #[test] + fn proposal_metadata_active_prefix_comes_from_repository_title() { + for (repository_title, expected_prefix) in [("EIPs", "EIP"), ("ERCs", "ERC")] { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/001.md", + &metadata_proposal_markdown(1, "Proposal 1", None, None), + ); + + let metadata = write_metadata_json(temp.path(), repository_title, None); + + assert_eq!(metadata["active_prefix"], json!(expected_prefix)); + } + } + + #[test] + fn proposal_metadata_erc_category_produces_erc_prefix_and_key() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown(20, "Proposal 20", Some("ERC"), None), + ); + + let metadata = write_metadata_json(temp.path(), "EIPs", None); + let proposals = metadata["proposals"].as_object().unwrap(); + + assert!(proposals.contains_key("erc-20")); + assert_eq!(metadata["proposals"]["erc-20"]["prefix"], json!("ERC")); + } + + #[test] + fn proposal_metadata_non_erc_category_and_default_produce_eip_prefix_and_key() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown(20, "Proposal 20", Some("Core"), None), + ); + write_file( + temp.path(), + "content/021.md", + &metadata_proposal_markdown(21, "Proposal 21", None, None), + ); + + let metadata = write_metadata_json(temp.path(), "EIPs", None); + let proposals = metadata["proposals"].as_object().unwrap(); + + assert!(proposals.contains_key("eip-20")); + assert!(proposals.contains_key("eip-21")); + assert_eq!(metadata["proposals"]["eip-20"]["prefix"], json!("EIP")); + assert_eq!(metadata["proposals"]["eip-21"]["prefix"], json!("EIP")); + } + + #[test] + fn proposal_metadata_missing_optional_description_is_omitted() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/001.md", + &metadata_proposal_markdown(1, "Proposal 1", None, None), + ); + + let metadata = write_metadata_json(temp.path(), "EIPs", None); + let proposal = metadata["proposals"]["eip-1"].as_object().unwrap(); + + assert!(!proposal.contains_key("description")); + } + + #[test] + fn proposal_metadata_malformed_preamble_fails_clearly() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/001.md", "not front matter\n"); + + let error = write_proposal_metadata_json(temp.path(), "EIPs", None) + .unwrap_err() + .to_string(); + + assert!(error.contains("couldn't split preamble")); + assert!(error.contains("001.md")); + } + + #[test] + fn proposal_metadata_missing_required_field_fails_clearly() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/001.md", + "---\neip: 1\ntitle: Proposal 1\ntype: Standards Track\n---\nBody\n", + ); + + let error = write_proposal_metadata_json(temp.path(), "EIPs", None) + .unwrap_err() + .to_string(); + + assert!(error.contains("missing required proposal metadata field `status`")); + assert!(error.contains("001.md")); + } + + #[test] + fn proposal_metadata_path_and_preamble_number_mismatch_fails_clearly() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/001.md", + "---\neip: 2\ntitle: Proposal 1\nstatus: Final\ntype: Standards Track\n---\nBody\n", + ); + + let error = write_proposal_metadata_json(temp.path(), "EIPs", None) + .unwrap_err() + .to_string(); + + assert!(error.contains("proposal metadata number mismatch")); + assert!(error.contains("001.md")); + assert!(error.contains("path indicates `1`")); + assert!(error.contains("`eip` contains `2`")); + } + + #[test] + fn proposal_metadata_allows_eip_and_erc_keys_for_same_number() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown(20, "ERC 20", Some("ERC"), None), + ); + write_file( + temp.path(), + "content/20/index.md", + &metadata_proposal_markdown(20, "EIP 20", None, None), + ); + + let metadata = write_metadata_json(temp.path(), "EIPs", None); + let proposals = metadata["proposals"].as_object().unwrap(); + + assert_eq!(proposals.len(), 2); + assert!(proposals.contains_key("eip-20")); + assert!(proposals.contains_key("erc-20")); + } + + #[test] + fn proposal_metadata_duplicate_same_key_fails_with_both_paths() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/020.md", + &metadata_proposal_markdown(20, "ERC 20", Some("ERC"), None), + ); + write_file( + temp.path(), + "content/20/index.md", + &metadata_proposal_markdown(20, "Duplicate ERC 20", Some("ERC"), None), + ); + + let error = write_proposal_metadata_json(temp.path(), "EIPs", None) + .unwrap_err() + .to_string(); + + assert!(error.contains("duplicate proposal metadata key `erc-20`")); + assert!(error.contains("020.md")); + assert!(error.contains("20/index.md")); + } + + #[test] + fn proposal_metadata_existing_output_collision_fails_loudly() { + let temp = TempDir::new().unwrap(); + write_file( + temp.path(), + "content/001.md", + &metadata_proposal_markdown(1, "Proposal 1", None, None), + ); + write_file( + temp.path(), + "static/assets/data/proposals.json", + "{\"existing\":true}\n", + ); + + let error = write_proposal_metadata_json(temp.path(), "EIPs", None) + .unwrap_err() + .to_string(); + + assert!(error.contains("already exists")); + assert_eq!( + std::fs::read_to_string(proposal_metadata_json_path(temp.path())).unwrap(), + "{\"existing\":true}\n" + ); + } + + #[test] + fn proposal_metadata_unsupported_markdown_paths_are_ignored() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/_index.md", "not front matter\n"); + write_file(temp.path(), "content/foo.md", "not front matter\n"); + write_file(temp.path(), "content/001/readme.md", "not front matter\n"); + write_file( + temp.path(), + "content/001/assets/readme.md", + "not front matter\n", + ); + write_file( + temp.path(), + "content/002.md", + &metadata_proposal_markdown(2, "Proposal 2", None, None), + ); + + let metadata = write_metadata_json(temp.path(), "EIPs", None); + let proposals = metadata["proposals"].as_object().unwrap(); + + assert_eq!(proposals.len(), 1); + assert!(proposals.contains_key("eip-2")); + } +} diff --git a/src/serve.rs b/src/serve.rs new file mode 100644 index 0000000..326cb31 --- /dev/null +++ b/src/serve.rs @@ -0,0 +1,817 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Dirty active-repo and local-theme serve synchronization. + +use std::{ + collections::BTreeSet, + ffi::OsStr, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc::{self, RecvTimeoutError}, + Arc, + }, + thread::{self, JoinHandle}, + time::Duration, +}; + +use log::{debug, info, warn}; +use notify::{Event, RecursiveMode, Watcher}; +use snafu::{Report, ResultExt, Whatever}; + +use crate::{git, layout::CONTENT_DIR, markdown, proposal::OnlyRenderPlan}; + +#[derive(Debug)] +pub(crate) struct DirtyServeWatcher { + stop: Arc, + thread: JoinHandle<()>, +} + +#[derive(Debug, Clone)] +struct ActiveRepoServeSync { + source_root: PathBuf, + build_repo_path: PathBuf, + only_plan: Option, +} + +#[derive(Debug, Clone)] +pub(crate) struct LocalThemeServeSync { + pub(crate) theme_source_root: PathBuf, + pub(crate) mounted_theme_dir: PathBuf, + pub(crate) theme_index_path: PathBuf, +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct ServeSyncConfig { + active_repo: Option, + local_theme: Option, +} + +impl ServeSyncConfig { + pub(crate) fn has_targets(&self) -> bool { + self.active_repo.is_some() || self.local_theme.is_some() + } +} + +impl DirtyServeWatcher { + pub(crate) fn start(sync_config: ServeSyncConfig) -> Result { + let stop = Arc::new(AtomicBool::new(false)); + let stop_thread = stop.clone(); + let (ready_tx, ready_rx) = mpsc::channel(); + let thread = + thread::spawn(move || dirty_serve_sync_loop(sync_config, stop_thread, ready_tx)); + + match ready_rx + .recv() + .whatever_context("dirty serve watcher exited before initialization")? + { + Ok(()) => Ok(Self { stop, thread }), + Err(message) => { + stop.store(true, Ordering::Relaxed); + let _ = thread.join(); + snafu::whatever!("{message}"); + } + } + } + + pub(crate) fn stop(self) { + self.stop.store(true, Ordering::Relaxed); + let _ = self.thread.join(); + } +} + +fn path_is_watched_source_path(root_path: &Path, path: &Path) -> bool { + let Ok(relative_path) = path.strip_prefix(root_path) else { + return false; + }; + + relative_path + .components() + .next() + .map(|component| component.as_os_str() != OsStr::new(".git")) + .unwrap_or(false) +} + +fn event_has_watched_source_path(root_path: &Path, event: &Event) -> bool { + event + .paths + .iter() + .any(|path| path_is_watched_source_path(root_path, path)) +} + +fn index_lock_path(index_path: &Path) -> Option { + let file_name = index_path.file_name()?.to_string_lossy(); + Some(index_path.with_file_name(format!("{file_name}.lock"))) +} + +fn event_has_theme_index_path(index_path: &Path, event: &Event) -> bool { + let lock_path = index_lock_path(index_path); + event.paths.iter().any(|path| { + path == index_path + || lock_path + .as_ref() + .map(|lock_path| path == lock_path) + .unwrap_or(false) + }) +} + +fn sync_dirty_serve_state( + source_root: &Path, + build_repo_path: &Path, + only_plan: Option<&OnlyRenderPlan>, + previous_dirty_paths: &mut BTreeSet, +) -> Result<(), Whatever> { + let current_dirty_paths = filter_dirty_paths( + git::working_tree_paths(source_root) + .whatever_context("unable to list tracked dirty paths for dirty serve")?, + only_plan, + ); + + let affected_paths = affected_dirty_paths(previous_dirty_paths, ¤t_dirty_paths); + + if affected_paths.is_empty() { + *previous_dirty_paths = current_dirty_paths; + return Ok(()); + } + + for path in selected_deleted_proposal_markdown_paths(source_root, &affected_paths, only_plan) { + warn!( + "selected proposal path `{}` was removed from the source repo; removing it from the targeted serve build input", + path.to_string_lossy() + ); + } + + git::sync_materialized_paths(source_root, build_repo_path, &affected_paths) + .whatever_context("unable to synchronize tracked paths into the materialized repo")?; + markdown::preprocess_paths( + &build_repo_path.join(CONTENT_DIR), + &affected_paths, + only_plan, + ) + .whatever_context("unable to preprocess synchronized markdown during dirty serve")?; + + info!( + "synchronized {} tracked path(s) into the materialized repo for dirty serve", + affected_paths.len() + ); + + *previous_dirty_paths = current_dirty_paths; + Ok(()) +} + +fn filter_dirty_paths( + dirty_paths: impl IntoIterator, + only_plan: Option<&OnlyRenderPlan>, +) -> BTreeSet { + dirty_paths + .into_iter() + .filter(|path| { + only_plan + .map(|plan| plan.should_sync_dirty_path(path)) + .unwrap_or(true) + }) + .collect() +} + +fn affected_dirty_paths( + previous_dirty_paths: &BTreeSet, + current_dirty_paths: &BTreeSet, +) -> BTreeSet { + previous_dirty_paths + .union(current_dirty_paths) + .cloned() + .collect() +} + +fn selected_deleted_proposal_markdown_paths( + source_root: &Path, + affected_paths: &BTreeSet, + only_plan: Option<&OnlyRenderPlan>, +) -> Vec { + let Some(only_plan) = only_plan else { + return Vec::new(); + }; + + affected_paths + .iter() + .filter(|path| only_plan.is_selected_proposal_markdown_path(path)) + .filter( + |path| match std::fs::symlink_metadata(source_root.join(path)) { + Ok(_) => false, + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => + { + true + } + Err(_) => false, + }, + ) + .cloned() + .collect() +} + +fn capture_active_dirty_paths( + source_root: &Path, + only_plan: Option<&OnlyRenderPlan>, +) -> Result, Whatever> { + Ok(filter_dirty_paths( + git::working_tree_paths(source_root) + .whatever_context("unable to list tracked dirty paths for dirty serve")?, + only_plan, + )) +} + +fn sync_theme_serve_state( + theme_source_root: &Path, + mounted_theme_dir: &Path, + previous_dirty_paths: &mut BTreeSet, +) -> Result<(), Whatever> { + let current_dirty_paths: BTreeSet<_> = git::tracked_working_tree_paths(theme_source_root) + .whatever_context("unable to list tracked dirty paths for local theme serve")? + .into_iter() + .collect(); + + let affected_paths: BTreeSet<_> = previous_dirty_paths + .union(¤t_dirty_paths) + .cloned() + .collect(); + + if affected_paths.is_empty() { + *previous_dirty_paths = current_dirty_paths; + return Ok(()); + } + + git::sync_working_tree_paths(theme_source_root, mounted_theme_dir, &affected_paths) + .whatever_context("unable to synchronize tracked local theme paths")?; + + info!( + "synchronized {} tracked path(s) into the mounted local theme for serve", + affected_paths.len() + ); + + *previous_dirty_paths = current_dirty_paths; + Ok(()) +} + +fn watch_theme_index( + watcher: &mut notify::RecommendedWatcher, + theme_index_path: &Path, +) -> Result<(), String> { + let file_result = watcher.watch(theme_index_path, RecursiveMode::NonRecursive); + let Some(parent) = theme_index_path.parent() else { + return file_result.map_err(|file_error| { + format!( + "unable to watch local theme Git index `{}`: {file_error}", + theme_index_path.to_string_lossy() + ) + }); + }; + let parent_result = watcher.watch(parent, RecursiveMode::NonRecursive); + + match (file_result, parent_result) { + (Ok(()), Ok(())) => Ok(()), + (Ok(()), Err(parent_error)) => { + debug!( + "unable to watch local theme Git index parent `{}`: {parent_error}", + parent.to_string_lossy() + ); + Ok(()) + } + (Err(file_error), Ok(())) => { + debug!( + "using local theme Git index parent watch for `{}` after file watch failed: {file_error}", + theme_index_path.to_string_lossy() + ); + Ok(()) + } + (Err(file_error), Err(parent_error)) => Err(format!( + "unable to watch local theme Git index `{}`: {file_error}; fallback watch on `{}` also failed: {parent_error}", + theme_index_path.to_string_lossy(), + parent.to_string_lossy() + )), + } +} + +fn dirty_serve_sync_loop( + sync_config: ServeSyncConfig, + stop: Arc, + ready_tx: mpsc::Sender>, +) { + let (event_tx, event_rx) = mpsc::channel(); + let mut watcher = match notify::recommended_watcher(move |result| { + let _ = event_tx.send(result); + }) { + Ok(watcher) => watcher, + Err(error) => { + let _ = ready_tx.send(Err(format!("unable to start dirty serve watcher: {error}"))); + return; + } + }; + + if let Some(active_repo) = &sync_config.active_repo { + if let Err(error) = watcher.watch(&active_repo.source_root, RecursiveMode::Recursive) { + let _ = ready_tx.send(Err(format!( + "unable to watch `{}` for dirty serve changes: {error}", + active_repo.source_root.to_string_lossy() + ))); + return; + } + } + + if let Some(local_theme) = &sync_config.local_theme { + if let Err(error) = watcher.watch(&local_theme.theme_source_root, RecursiveMode::Recursive) + { + let _ = ready_tx.send(Err(format!( + "unable to watch local theme `{}` for serve changes: {error}", + local_theme.theme_source_root.to_string_lossy() + ))); + return; + } + + if let Err(message) = watch_theme_index(&mut watcher, &local_theme.theme_index_path) { + let _ = ready_tx.send(Err(message)); + return; + } + } + + let mut previous_active_dirty_paths: BTreeSet<_> = match &sync_config.active_repo { + Some(active_repo) => match capture_active_dirty_paths( + &active_repo.source_root, + active_repo.only_plan.as_ref(), + ) { + Ok(paths) => paths, + Err(error) => { + let _ = ready_tx.send(Err(format!( + "unable to capture initial dirty serve state: {}", + Report::from_error(error) + ))); + return; + } + }, + None => BTreeSet::new(), + }; + + let mut previous_theme_dirty_paths: BTreeSet<_> = match &sync_config.local_theme { + Some(local_theme) => { + if let Err(error) = git::materialize_working_tree( + &local_theme.theme_source_root, + &local_theme.mounted_theme_dir, + ) { + let _ = ready_tx.send(Err(format!( + "unable to synchronize initial local theme state after watcher setup: {}", + Report::from_error(error) + ))); + return; + } + + match git::tracked_working_tree_paths(&local_theme.theme_source_root) { + Ok(paths) => paths.into_iter().collect(), + Err(error) => { + let _ = ready_tx.send(Err(format!( + "unable to capture initial local theme dirty state: {}", + Report::from_error(error) + ))); + return; + } + } + } + None => BTreeSet::new(), + }; + + if let Some(active_repo) = &sync_config.active_repo { + info!( + "watching `{}` for dirty serve changes", + active_repo.source_root.to_string_lossy() + ); + } + if let Some(local_theme) = &sync_config.local_theme { + info!( + "watching `{}` for local theme serve changes", + local_theme.theme_source_root.to_string_lossy() + ); + } + let _ = ready_tx.send(Ok(())); + + while !stop.load(Ordering::Relaxed) { + let first_event = match event_rx.recv_timeout(Duration::from_millis(250)) { + Ok(event) => Some(event), + Err(RecvTimeoutError::Timeout) => None, + Err(RecvTimeoutError::Disconnected) => break, + }; + + let Some(first_event) = first_event else { + continue; + }; + + let mut saw_active_event = false; + let mut saw_theme_event = false; + + match first_event { + Ok(event) => { + if let Some(active_repo) = &sync_config.active_repo { + saw_active_event |= + event_has_watched_source_path(&active_repo.source_root, &event); + } + if let Some(local_theme) = &sync_config.local_theme { + saw_theme_event |= + event_has_watched_source_path(&local_theme.theme_source_root, &event) + || event_has_theme_index_path(&local_theme.theme_index_path, &event); + } + } + Err(error) => { + warn!("filesystem watcher error: {error}"); + } + } + + loop { + match event_rx.recv_timeout(Duration::from_millis(75)) { + Ok(Ok(event)) => { + if let Some(active_repo) = &sync_config.active_repo { + saw_active_event |= + event_has_watched_source_path(&active_repo.source_root, &event); + } + if let Some(local_theme) = &sync_config.local_theme { + saw_theme_event |= + event_has_watched_source_path(&local_theme.theme_source_root, &event) + || event_has_theme_index_path( + &local_theme.theme_index_path, + &event, + ); + } + } + Ok(Err(error)) => warn!("filesystem watcher error: {error}"), + Err(RecvTimeoutError::Timeout) => break, + Err(RecvTimeoutError::Disconnected) => return, + } + } + + if saw_active_event { + if let Some(active_repo) = &sync_config.active_repo { + if let Err(error) = sync_dirty_serve_state( + &active_repo.source_root, + &active_repo.build_repo_path, + active_repo.only_plan.as_ref(), + &mut previous_active_dirty_paths, + ) { + warn!( + "unable to synchronize dirty serve changes: {}", + Report::from_error(error) + ); + } + } + } + + if saw_theme_event { + if let Some(local_theme) = &sync_config.local_theme { + if let Err(error) = sync_theme_serve_state( + &local_theme.theme_source_root, + &local_theme.mounted_theme_dir, + &mut previous_theme_dirty_paths, + ) { + warn!( + "unable to synchronize local theme serve changes: {}", + Report::from_error(error) + ); + } + } + } + } +} + +pub(crate) fn serve_sync_config( + source_materialization: git::SourceMaterialization, + source_root: &Path, + repo_path: &Path, + only_plan: Option, + local_theme_sync: Option, +) -> ServeSyncConfig { + ServeSyncConfig { + active_repo: (source_materialization == git::SourceMaterialization::Dirty).then(|| { + ActiveRepoServeSync { + source_root: source_root.to_path_buf(), + build_repo_path: repo_path.to_path_buf(), + only_plan, + } + }), + local_theme: local_theme_sync, + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeSet, + path::{Path, PathBuf}, + }; + + use git2::{IndexAddOption, Repository, Signature}; + use notify::{Event, EventKind}; + use tempfile::TempDir; + + use crate::{ + git::SourceMaterialization, + proposal::{OnlyRenderPlan, ProposalNumber}, + }; + + use super::{ + affected_dirty_paths, event_has_theme_index_path, filter_dirty_paths, + selected_deleted_proposal_markdown_paths, serve_sync_config, sync_dirty_serve_state, + LocalThemeServeSync, + }; + + fn fake_theme_sync(root: &Path) -> LocalThemeServeSync { + LocalThemeServeSync { + theme_source_root: root.join("theme"), + mounted_theme_dir: root.join("repo/themes/eips-theme"), + theme_index_path: root.join("theme/.git/index"), + } + } + + fn number(value: u32) -> ProposalNumber { + ProposalNumber::from_u32(value).unwrap() + } + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn commit_all(repo: &Repository, message: &str) { + let mut index = repo.index().unwrap(); + index + .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + let parents = repo + .head() + .ok() + .and_then(|head| head.target()) + .map(|oid| repo.find_commit(oid).unwrap()) + .into_iter() + .collect::>(); + let parent_refs = parents.iter().collect::>(); + + repo.commit( + Some("HEAD"), + &signature, + &signature, + message, + &tree, + &parent_refs, + ) + .unwrap(); + } + + fn init_repo(root: &Path, files: &[(&str, &str)]) -> Repository { + std::fs::create_dir_all(root).unwrap(); + let repo = Repository::init(root).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + for (relative, contents) in files { + write_file(root, relative, contents); + } + commit_all(&repo, "initial"); + repo + } + + fn proposal_markdown(value: u32, extra_preamble: &str, body: &str) -> String { + format!("---\neip: {value}\ntitle: Proposal {value}\n{extra_preamble}---\n{body}\n") + } + + fn only_plan(root: &Path) -> OnlyRenderPlan { + let content = root.join("content"); + write_file(&content, "00555.md", &proposal_markdown(555, "", "Body")); + write_file(&content, "00678.md", &proposal_markdown(678, "", "Body")); + OnlyRenderPlan::build(&content, [number(555)].into_iter().collect()).unwrap() + } + + fn paths(paths: &[&str]) -> BTreeSet { + paths.iter().map(PathBuf::from).collect() + } + + fn rendered_body(path: &Path) -> String { + let contents = std::fs::read_to_string(path).unwrap(); + contents.split_once("\n+++\n").unwrap().1.to_owned() + } + + fn dirty_sync_fixture() -> (TempDir, PathBuf, PathBuf, OnlyRenderPlan) { + let temp = TempDir::new().unwrap(); + let source = temp.path().join("source"); + let build = temp.path().join("build/repo"); + let selected = proposal_markdown(555, "requires: 678\n", "See [EIP-678](/00678.md)."); + let unselected = proposal_markdown(678, "", "Unselected."); + init_repo( + &source, + &[ + ("content/00555.md", selected.as_str()), + ("content/00555/assets/diagram.png", "selected image\n"), + ("content/00678.md", unselected.as_str()), + ("content/00678/assets/diagram.png", "unselected image\n"), + ( + "content/_index.md", + "---\ntitle: Home\n---\nSee [EIP-678](/00678.md).\n", + ), + ], + ); + let plan = + OnlyRenderPlan::build(&source.join("content"), [number(555)].into_iter().collect()) + .unwrap(); + init_repo( + &build, + &[ + ("content/00555.md", selected.as_str()), + ("content/00555/assets/diagram.png", "selected image\n"), + ( + "content/_index.md", + "---\ntitle: Home\n---\nSee [EIP-678](/00678.md).\n", + ), + ], + ); + (temp, source, build, plan) + } + + #[test] + fn local_theme_index_events_trigger_rescan() { + let index_path = PathBuf::from("/workspace/theme/.git/index"); + let index_event = Event::new(EventKind::Any).add_path(index_path.clone()); + let lock_event = + Event::new(EventKind::Any).add_path(PathBuf::from("/workspace/theme/.git/index.lock")); + let unrelated_event = + Event::new(EventKind::Any).add_path(PathBuf::from("/workspace/theme/.git/config")); + + assert!(event_has_theme_index_path(&index_path, &index_event)); + assert!(event_has_theme_index_path(&index_path, &lock_event)); + assert!(!event_has_theme_index_path(&index_path, &unrelated_event)); + } + + #[test] + fn local_serve_syncs_theme_and_dirty_active_repo() { + let temp = TempDir::new().unwrap(); + let plan = only_plan(temp.path()); + + let sync_config = serve_sync_config( + SourceMaterialization::Dirty, + &temp.path().join("Core"), + &temp.path().join(".local-build/Core/repo"), + Some(plan), + Some(fake_theme_sync(temp.path())), + ); + + assert!(sync_config.active_repo.is_some()); + assert!(sync_config + .active_repo + .as_ref() + .unwrap() + .only_plan + .is_some()); + assert!(sync_config.local_theme.is_some()); + } + + #[test] + fn clean_local_serve_keeps_theme_sync_but_disables_active_repo_dirty_sync() { + let temp = TempDir::new().unwrap(); + + let sync_config = serve_sync_config( + SourceMaterialization::Clean, + &temp.path().join("Core"), + &temp.path().join(".local-build/Core/repo"), + None, + Some(fake_theme_sync(temp.path())), + ); + + assert!(sync_config.active_repo.is_none()); + assert!(sync_config.local_theme.is_some()); + } + + #[test] + fn only_dirty_path_filter_runs_before_union_and_keeps_selected_deletions() { + let temp = TempDir::new().unwrap(); + let plan = only_plan(temp.path()); + let previous_raw = paths(&[ + "content/00555.md", + "content/00678.md", + "content/00678/assets/diagram.png", + ]); + let current_raw = paths(&["content/00555/assets/diagram.png", "content/00999.md"]); + + let previous_filtered = filter_dirty_paths(previous_raw, Some(&plan)); + let current_filtered = filter_dirty_paths(current_raw, Some(&plan)); + let affected = affected_dirty_paths(&previous_filtered, ¤t_filtered); + + assert_eq!( + affected, + paths(&["content/00555.md", "content/00555/assets/diagram.png"]) + ); + } + + #[test] + fn selected_deleted_proposal_markdown_paths_reports_only_selected_markdown_deletions() { + let (_temp, source, _build, plan) = dirty_sync_fixture(); + std::fs::remove_file(source.join("content/00555.md")).unwrap(); + std::fs::remove_file(source.join("content/00555/assets/diagram.png")).unwrap(); + std::fs::remove_file(source.join("content/_index.md")).unwrap(); + + let affected_paths = paths(&[ + "content/00555.md", + "content/00555/assets/diagram.png", + "content/_index.md", + "content/00678.md", + ]); + + assert_eq!( + selected_deleted_proposal_markdown_paths(&source, &affected_paths, Some(&plan)), + vec![PathBuf::from("content/00555.md")] + ); + assert!( + selected_deleted_proposal_markdown_paths(&source, &affected_paths, None).is_empty() + ); + } + + #[test] + fn only_dirty_sync_does_not_reintroduce_unselected_markdown_or_assets() { + let (_temp, source, build, plan) = dirty_sync_fixture(); + write_file( + &source, + "content/00678.md", + &proposal_markdown(678, "", "Dirty unselected."), + ); + write_file( + &source, + "content/00678/assets/diagram.png", + "dirty unselected image\n", + ); + let mut previous_dirty_paths = BTreeSet::new(); + + sync_dirty_serve_state(&source, &build, Some(&plan), &mut previous_dirty_paths).unwrap(); + + assert!(!build.join("content/00678.md").exists()); + assert!(!build.join("content/00678/assets/diagram.png").exists()); + assert!(previous_dirty_paths.is_empty()); + } + + #[test] + fn only_dirty_sync_copies_selected_assets_without_markdown_preprocessing() { + let (_temp, source, build, plan) = dirty_sync_fixture(); + write_file( + &source, + "content/00555/assets/diagram.png", + "dirty selected image\n", + ); + let mut previous_dirty_paths = BTreeSet::new(); + + sync_dirty_serve_state(&source, &build, Some(&plan), &mut previous_dirty_paths).unwrap(); + + assert_eq!( + std::fs::read_to_string(build.join("content/00555/assets/diagram.png")).unwrap(), + "dirty selected image\n" + ); + assert!(previous_dirty_paths.contains(Path::new("content/00555/assets/diagram.png"))); + } + + #[test] + fn only_dirty_sync_preprocesses_selected_and_retained_markdown_with_plan() { + let (_temp, source, build, plan) = dirty_sync_fixture(); + write_file( + &source, + "content/00555.md", + &proposal_markdown(555, "requires: 678\n", "Dirty [EIP-678](/00678.md)."), + ); + write_file( + &source, + "content/_index.md", + "---\ntitle: Home\n---\nDirty [EIP-678](/00678.md).\n", + ); + let mut previous_dirty_paths = BTreeSet::new(); + + sync_dirty_serve_state(&source, &build, Some(&plan), &mut previous_dirty_paths).unwrap(); + + let selected = std::fs::read_to_string(build.join("content/00555.md")).unwrap(); + let index_body = rendered_body(&build.join("content/_index.md")); + assert!(selected.contains("https://eips.ethereum.org/EIPS/eip-678")); + assert!(index_body.contains("https://eips.ethereum.org/EIPS/eip-678")); + } + + #[test] + fn only_dirty_sync_propagates_selected_proposal_deletion() { + let (_temp, source, build, plan) = dirty_sync_fixture(); + std::fs::remove_file(source.join("content/00555.md")).unwrap(); + let mut previous_dirty_paths = BTreeSet::new(); + + sync_dirty_serve_state(&source, &build, Some(&plan), &mut previous_dirty_paths).unwrap(); + + assert!(!build.join("content/00555.md").exists()); + assert!(previous_dirty_paths.contains(Path::new("content/00555.md"))); + } +} diff --git a/src/tests.rs b/src/tests.rs new file mode 100644 index 0000000..f45eba9 --- /dev/null +++ b/src/tests.rs @@ -0,0 +1,814 @@ +#![cfg(test)] + +// Cross-domain behavior tests live here; see src/README.md for module test ownership. + +use std::{ + collections::BTreeSet, + path::{Path, PathBuf}, +}; + +use clap::Parser; +use git2::{IndexAddOption, Repository, Signature}; +use snafu::Report; +use tempfile::TempDir; +use url::Url; + +use crate::{ + cli::{Args, EditorialCommand, EditorialSelectorArgs, Operation, RuntimeOperation}, + config::{self, LoadedWorkspaceConfig}, + editorial::editorial_targets_from_source, + execution::{ + resolve_execution, resolve_execution_settings, validate_non_execution_command_flags, + ExecutionSettings, ResolvedExecution, SelectedSource, + }, + layout::{BUILD_DIR, CONTENT_DIR, REPO_DIR}, + markdown, + proposal::{OnlyRenderPlan, ProposalNumber}, +}; + +fn parse_args(arguments: &[&str]) -> Args { + Args::try_parse_from(arguments).unwrap() +} + +fn settings_for( + arguments: &[&str], + sibling_ids: &[&str], + workspace_config: Option<&LoadedWorkspaceConfig>, +) -> ExecutionSettings { + let args = parse_args(arguments); + let sibling_ids = sibling_ids + .iter() + .map(|sibling_id| (*sibling_id).to_owned()) + .collect::>(); + + resolve_execution_settings(&args, &sibling_ids, workspace_config).unwrap() +} + +fn assert_settings( + arguments: &[&str], + sibling_ids: &[&str], + workspace_config: Option<&LoadedWorkspaceConfig>, + expected: ExecutionSettings, +) { + assert_eq!( + settings_for(arguments, sibling_ids, workspace_config), + expected + ); +} + +fn file_url(path: &Path) -> Url { + Url::from_directory_path(path).unwrap() +} + +fn write_file(root: &Path, relative: impl AsRef, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); +} + +fn commit_all(repo: &Repository, message: &str) { + let mut index = repo.index().unwrap(); + index + .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + let parents = repo + .head() + .ok() + .and_then(|head| head.target()) + .map(|oid| repo.find_commit(oid).unwrap()) + .into_iter() + .collect::>(); + let parent_refs = parents.iter().collect::>(); + + repo.commit( + Some("HEAD"), + &signature, + &signature, + message, + &tree, + &parent_refs, + ) + .unwrap(); +} + +fn init_repo(path: &Path, files: &[(&str, &str)]) -> Repository { + std::fs::create_dir_all(path).unwrap(); + let repo = Repository::init(path).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + for (relative, contents) in files { + write_file(path, relative, contents); + } + commit_all(&repo, "initial"); + repo +} + +fn append_and_commit(repo: &Repository, root: &Path, files: &[(&str, &str)], message: &str) { + for (relative, contents) in files { + write_file(root, relative, contents); + } + commit_all(repo, message); +} + +fn proposal_markdown(number: u32, category: Option<&str>, body: &str) -> String { + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!("---\neip: {number}\ntitle: Proposal {number}\n{category}---\n{body}\n") +} + +fn materialize_resolved_repo(resolved: &ResolvedExecution) -> PathBuf { + let repo_path = resolved.build_path.join(REPO_DIR); + crate::git::Fresh::new( + &resolved.root_path, + &repo_path, + resolved.repository_use.clone(), + resolved.source_materialization, + ) + .unwrap() + .clone_src() + .unwrap() + .fetch_upstream() + .unwrap() + .merge() + .unwrap(); + + repo_path +} + +fn preprocess_and_prune_only(repo_path: &Path, selected: BTreeSet) { + let content_path = repo_path.join(CONTENT_DIR); + let plan = OnlyRenderPlan::build(&content_path, selected).unwrap(); + markdown::preprocess(&content_path, Some(&plan)).unwrap(); + plan.prune_content(&content_path).unwrap(); +} + +fn repo_manifest_text(repo_id: &str, repository: &Url, siblings: &[(&str, Url)]) -> String { + let mut manifest = format!( + r#" +repo_id = "{repo_id}" + +[production] +repository = "{repository}" +base_url = "https://example.test/{repo_id}/" + +[staging] +repository = "{repository}" +base_url = "https://staging.example.test/{repo_id}/" +"# + ); + + for (sibling_id, sibling_repository) in siblings { + manifest.push_str(&format!( + r#" +[siblings.{sibling_id}.production] +repository = "{sibling_repository}" +base_url = "https://example.test/{sibling_id}/" + +[siblings.{sibling_id}.staging] +repository = "{sibling_repository}" +base_url = "https://staging.example.test/{sibling_id}/" +"# + )); + } + + manifest +} + +fn write_repo_manifest_file(path: &Path, repo_id: &str, upstream: &Url, siblings: &[(&str, Url)]) { + write_file( + path, + config::REPO_MANIFEST_FILE, + &repo_manifest_text(repo_id, upstream, siblings), + ); +} + +fn write_manifest_repo( + path: &Path, + repo_id: &str, + upstream: &Url, + siblings: &[(&str, Url)], +) -> Repository { + let repo = init_repo(path, &[("content/0001.md", "# Proposal\n")]); + write_repo_manifest_file(path, repo_id, upstream, siblings); + commit_all(&repo, "add repo manifest"); + repo +} + +#[test] +fn command_groups_route_separately_from_parity() { + let init = parse_args(&["build-eips", "init", "/tmp/workspace"]); + let doctor = parse_args(&["build-eips", "doctor"]); + let editorial_lint = parse_args(&["build-eips", "editorial", "lint", "--working-tree"]); + let editorial_check = parse_args(&["build-eips", "editorial", "check", "--working-tree"]); + + assert!(matches!(init.operation, Operation::Init { .. })); + assert!(matches!(doctor.operation, Operation::Doctor)); + assert!(matches!( + editorial_lint.operation.runtime_operation(), + Some(RuntimeOperation::Editorial { + command: EditorialCommand::Lint { .. } + }) + )); + assert!(matches!( + editorial_check.operation.runtime_operation(), + Some(RuntimeOperation::Editorial { + command: EditorialCommand::Check { .. } + }) + )); + assert!(validate_non_execution_command_flags(&editorial_lint).is_ok()); +} + +#[test] +fn downstream_ci_changed_forms_do_not_need_workspace_config() { + for (arguments, expected_staging) in [ + (&["build-eips", "--staging", "changed"][..], true), + (&["build-eips", "--production", "changed"][..], false), + ] { + assert_settings( + arguments, + &["ERCs"], + None, + ExecutionSettings { + build_root: None, + staging: expected_staging, + allow_dirty: false, + sibling: SelectedSource::Remote, + }, + ); + } +} + +#[test] +fn downstream_ci_zola_forms_require_workspace_local_theme_config() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + let active_path = active_path.to_string_lossy().to_string(); + + for arguments in [ + &["build-eips", "--staging", "build"][..], + &["build-eips", "--production", "build"][..], + &[ + "build-eips", + "--staging", + "editorial", + "check", + "--against-upstream", + ][..], + &["build-eips", "parity", "check"][..], + ] { + let mut cli_arguments = vec!["build-eips", "-C", active_path.as_str()]; + cli_arguments.extend_from_slice(&arguments[1..]); + let args = parse_args(&cli_arguments); + let message = resolve_execution(&args).unwrap_err().to_string(); + + assert!(message.contains("requires a workspace config with a local theme")); + assert!(message.contains("build-eips init ")); + } +} + +#[test] +fn manifest_identity_drives_runtime_resolution() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + std::fs::create_dir(workspace.path().join(config::DEFAULT_THEME_DIR)).unwrap(); + write_file( + workspace.path(), + config::LOCAL_CONFIG_FILE, + &config::default_workspace_config_text(), + ); + let build_root = workspace.path().join("build-root"); + let args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "--build-root", + build_root.to_str().unwrap(), + "parity", + "build", + ]); + + let resolved = resolve_execution(&args).unwrap(); + + assert_eq!(resolved.repository_use.title, "Core"); + assert_eq!(resolved.repository_use.location.repository, active_url); + assert!(resolved.repository_use.other_repos.is_empty()); + assert_eq!(resolved.build_path, build_root); +} + +#[test] +fn execution_commands_discover_workspace_config_from_active_repo_root() { + let workspace = TempDir::new().unwrap(); + let workspace_root = workspace.path().join("workspace"); + let active_path = workspace_root.join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + std::fs::create_dir(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + write_file( + &workspace_root, + config::LOCAL_CONFIG_FILE, + &config::default_workspace_config_text(), + ); + let args = parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "build"]); + + let resolved = resolve_execution(&args).unwrap(); + + assert_eq!( + resolved.build_path, + workspace_root + .join(config::DEFAULT_BUILD_ROOT_BASE) + .join("Core") + ); + assert_eq!( + resolved.source_materialization, + crate::git::SourceMaterialization::Dirty + ); +} + +#[test] +fn build_root_override_wins_with_workspace_config() { + let workspace = TempDir::new().unwrap(); + let workspace_root = workspace.path().join("workspace"); + let active_path = workspace_root.join("Core"); + let active_url = file_url(&active_path); + let build_root = workspace.path().join("override-build-root"); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + std::fs::create_dir(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + write_file( + &workspace_root, + config::LOCAL_CONFIG_FILE, + &config::default_workspace_config_text(), + ); + let args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "--build-root", + build_root.to_str().unwrap(), + "build", + ]); + + let resolved = resolve_execution(&args).unwrap(); + + assert_eq!(resolved.build_path, build_root); +} + +#[test] +fn non_workspace_runtime_path_falls_back_to_active_repo_build_dir() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + let args = parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "changed"]); + + let resolved = resolve_execution(&args).unwrap(); + + assert_eq!(resolved.build_path, active_path.join(BUILD_DIR)); +} + +#[test] +fn non_theme_runtime_commands_resolve_without_workspace_config() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + let active_path = active_path.to_string_lossy().to_string(); + + for command in ["changed", "clean", "preview"] { + let args = parse_args(&["build-eips", "-C", active_path.as_str(), command]); + let resolved = resolve_execution(&args).unwrap(); + + assert!(resolved.theme_path.is_none()); + } +} + +#[test] +fn unknown_repo_without_manifest_or_legacy_identity_errors() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Unknown"); + init_repo(&active_path, &[("content/0001.md", "# Proposal\n")]); + let args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "parity", + "build", + ]); + + let error = resolve_execution(&args).unwrap_err(); + let message = error.to_string(); + + assert!(message.contains(config::REPO_MANIFEST_FILE)); + assert!(message.contains("legacy EIPs/ERCs identity fallback")); +} + +#[test] +fn malformed_repo_manifest_does_not_fall_back_to_legacy_identity() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Malformed"); + init_repo(&active_path, &[("content/0001.md", "# Proposal\n")]); + write_file(&active_path, config::REPO_MANIFEST_FILE, "repo_id = ["); + let args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "parity", + "build", + ]); + + let message = Report::from_error(resolve_execution(&args).unwrap_err()).to_string(); + + assert!(message.contains("unable to load repo manifest")); + assert!(!message.contains("legacy EIPs/ERCs identity fallback")); +} + +#[test] +fn workspace_local_sibling_mode_is_all_or_nothing() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let eips_path = workspace.path().join("EIPs"); + init_repo(&eips_path, &[("content/0002.md", "# EIP\n")]); + let siblings = vec![ + ("EIPs", file_url(&eips_path)), + ("ERCs", file_url(&workspace.path().join("remotes/ERCs"))), + ]; + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &siblings); + std::fs::create_dir(workspace.path().join(config::DEFAULT_THEME_DIR)).unwrap(); + std::fs::write( + workspace.path().join(config::LOCAL_CONFIG_FILE), + config::default_workspace_config_text(), + ) + .unwrap(); + let args = parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "build"]); + + let error = resolve_execution(&args).unwrap_err(); + let message = error.to_string(); + + assert!(message.contains("requires all declared sibling repos")); + assert!(message.contains("ERCs")); +} + +#[test] +fn workspace_local_sources_resolve_from_standard_layout() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let eips_path = workspace.path().join("EIPs"); + let ercs_path = workspace.path().join("ERCs"); + init_repo(&eips_path, &[("content/0002.md", "# EIP\n")]); + init_repo(&ercs_path, &[("content/0003.md", "# ERC\n")]); + std::fs::create_dir(workspace.path().join(config::DEFAULT_THEME_DIR)).unwrap(); + let siblings = vec![ + ("EIPs", file_url(&eips_path)), + ("ERCs", file_url(&ercs_path)), + ]; + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &siblings); + std::fs::write( + workspace.path().join(config::LOCAL_CONFIG_FILE), + config::default_workspace_config_text(), + ) + .unwrap(); + let args = parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "build"]); + + let resolved = resolve_execution(&args).unwrap(); + + assert_eq!( + resolved.theme_path.as_deref(), + Some(workspace.path().join(config::DEFAULT_THEME_DIR).as_path()) + ); + assert_eq!( + resolved.repository_use.other_repos["EIPs"], + file_url(&eips_path) + ); + assert_eq!( + resolved.repository_use.other_repos["ERCs"], + file_url(&ercs_path) + ); +} + +#[test] +fn manifest_driven_multi_repo_build_and_editorial_flows_resolve_siblings() { + let temp = TempDir::new().unwrap(); + let upstream_path = temp.path().join("upstream/Core"); + init_repo( + &upstream_path, + &[("content/0001.md", "# Original proposal\n")], + ); + let upstream_url = file_url(&upstream_path); + + let active_path = temp.path().join("workspace/Core"); + std::fs::create_dir_all(active_path.parent().unwrap()).unwrap(); + std::fs::create_dir(temp.path().join("workspace/theme")).unwrap(); + write_file( + active_path.parent().unwrap(), + config::LOCAL_CONFIG_FILE, + &config::default_workspace_config_text(), + ); + git2::build::RepoBuilder::new() + .clone(upstream_url.as_str(), &active_path) + .unwrap(); + let active_repo = Repository::open(&active_path).unwrap(); + + let eips_path = temp.path().join("remotes/EIPs"); + init_repo(&eips_path, &[("content/0002.md", "# EIP sibling\n")]); + let ercs_path = temp.path().join("remotes/ERCs"); + init_repo(&ercs_path, &[("content/0003.md", "# ERC sibling\n")]); + let siblings = vec![ + ("EIPs", file_url(&eips_path)), + ("ERCs", file_url(&ercs_path)), + ]; + write_repo_manifest_file(&active_path, "Core", &upstream_url, &siblings); + append_and_commit( + &active_repo, + &active_path, + &[("content/0001.md", "# Updated proposal\n")], + "local proposal update", + ); + let build_root = temp.path().join("build-root"); + let args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "--build-root", + build_root.to_str().unwrap(), + "parity", + "build", + ]); + let resolved = resolve_execution(&args).unwrap(); + + assert_eq!(resolved.repository_use.other_repos.len(), 2); + + let repo_path = resolved.build_path.join(REPO_DIR); + let source = crate::git::Fresh::new( + &resolved.root_path, + &repo_path, + resolved.repository_use.clone(), + resolved.source_materialization, + ) + .unwrap() + .clone_src() + .unwrap() + .fetch_upstream() + .unwrap(); + + let selectors = EditorialSelectorArgs { + paths: Vec::::new(), + batch: None, + working_tree: false, + against_upstream: true, + }; + let targets = editorial_targets_from_source(&selectors, &resolved, Some(&source)).unwrap(); + + source.merge().unwrap(); + + assert!(repo_path.join("content/0002.md").is_file()); + assert!(repo_path.join("content/0003.md").is_file()); + + assert_eq!(targets, vec![PathBuf::from("content/0001.md")]); +} + +#[test] +fn only_build_selection_can_come_from_workspace_local_sibling_after_merge() { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("Core"); + let sibling_path = workspace_root.join("ERCs"); + let active_url = file_url(&active_path); + + let active_555 = proposal_markdown(555, None, "Active proposal."); + let active_repo = init_repo(&active_path, &[("content/00555.md", active_555.as_str())]); + let sibling_678 = proposal_markdown(678, Some("ERC"), "Sibling proposal."); + init_repo(&sibling_path, &[("content/00678.md", sibling_678.as_str())]); + write_repo_manifest_file( + &active_path, + "Core", + &active_url, + &[("ERCs", file_url(&sibling_path))], + ); + commit_all(&active_repo, "add manifest"); + std::fs::create_dir(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + write_file( + &workspace_root, + config::LOCAL_CONFIG_FILE, + &config::default_workspace_config_text(), + ); + + let args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "build", + "--only", + "678", + ]); + let resolved = resolve_execution(&args).unwrap(); + let repo_path = materialize_resolved_repo(&resolved); + let selected = resolved.only.clone().unwrap(); + + preprocess_and_prune_only(&repo_path, selected); + + assert!(repo_path.join("content/00678.md").is_file()); + assert!(!repo_path.join("content/00555.md").exists()); +} + +#[test] +fn normal_build_after_only_restores_full_materialized_content_tree() { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("Core"); + let active_url = file_url(&active_path); + let selected_555 = proposal_markdown(555, None, "Selected proposal."); + let unselected_678 = proposal_markdown(678, Some("ERC"), "Unselected proposal."); + let active_repo = init_repo( + &active_path, + &[ + ("content/00555.md", selected_555.as_str()), + ("content/00678.md", unselected_678.as_str()), + ], + ); + write_repo_manifest_file(&active_path, "Core", &active_url, &[]); + commit_all(&active_repo, "add manifest"); + std::fs::create_dir(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + write_file( + &workspace_root, + config::LOCAL_CONFIG_FILE, + &config::default_workspace_config_text(), + ); + let build_root = temp.path().join("build-root"); + + let only_args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "--build-root", + build_root.to_str().unwrap(), + "build", + "--only", + "555", + ]); + let only_resolved = resolve_execution(&only_args).unwrap(); + let repo_path = materialize_resolved_repo(&only_resolved); + preprocess_and_prune_only(&repo_path, only_resolved.only.clone().unwrap()); + + assert!(repo_path.join("content/00555.md").is_file()); + assert!(!repo_path.join("content/00678.md").exists()); + + let normal_args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "--build-root", + build_root.to_str().unwrap(), + "build", + ]); + let normal_resolved = resolve_execution(&normal_args).unwrap(); + assert!(normal_resolved.only.is_none()); + let restored_repo_path = materialize_resolved_repo(&normal_resolved); + markdown::preprocess(&restored_repo_path.join(CONTENT_DIR), None).unwrap(); + + assert!(restored_repo_path.join("content/00555.md").is_file()); + assert!(restored_repo_path.join("content/00678.md").is_file()); +} + +#[test] +fn serve_applies_render_only_config_in_phase_two() { + let workspace = TempDir::new().unwrap(); + let workspace_root = workspace.path().join("workspace"); + let active_path = workspace_root.join("Core"); + let active_url = file_url(&active_path); + let active_555 = proposal_markdown(555, None, "Active proposal."); + let active_repo = init_repo(&active_path, &[("content/00555.md", active_555.as_str())]); + write_repo_manifest_file(&active_path, "Core", &active_url, &[]); + commit_all(&active_repo, "add manifest"); + std::fs::create_dir(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + write_file( + &workspace_root, + config::LOCAL_CONFIG_FILE, + r#" +[render] +only = [555] +"#, + ); + + let args = parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "serve"]); + let resolved = resolve_execution(&args).unwrap(); + + assert_eq!( + resolved + .only + .unwrap() + .into_iter() + .map(|number| number.get()) + .collect::>(), + vec![555] + ); +} + +#[test] +fn only_serve_startup_uses_build_filtering() { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("Core"); + let active_url = file_url(&active_path); + let selected_555 = proposal_markdown(555, None, "Selected proposal."); + let unselected_678 = proposal_markdown(678, Some("ERC"), "Unselected proposal."); + let active_repo = init_repo( + &active_path, + &[ + ("content/00555.md", selected_555.as_str()), + ("content/00678.md", unselected_678.as_str()), + ], + ); + write_repo_manifest_file(&active_path, "Core", &active_url, &[]); + commit_all(&active_repo, "add manifest"); + std::fs::create_dir(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + write_file( + &workspace_root, + config::LOCAL_CONFIG_FILE, + &config::default_workspace_config_text(), + ); + + let args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "serve", + "--only", + "555", + ]); + let resolved = resolve_execution(&args).unwrap(); + let repo_path = materialize_resolved_repo(&resolved); + preprocess_and_prune_only(&repo_path, resolved.only.clone().unwrap()); + + assert!(repo_path.join("content/00555.md").is_file()); + assert!(!repo_path.join("content/00678.md").exists()); +} + +#[test] +fn normal_serve_after_only_restores_full_materialized_content_tree() { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("Core"); + let active_url = file_url(&active_path); + let selected_555 = proposal_markdown(555, None, "Selected proposal."); + let unselected_678 = proposal_markdown(678, Some("ERC"), "Unselected proposal."); + let active_repo = init_repo( + &active_path, + &[ + ("content/00555.md", selected_555.as_str()), + ("content/00678.md", unselected_678.as_str()), + ], + ); + write_repo_manifest_file(&active_path, "Core", &active_url, &[]); + commit_all(&active_repo, "add manifest"); + std::fs::create_dir(workspace_root.join(config::DEFAULT_THEME_DIR)).unwrap(); + write_file( + &workspace_root, + config::LOCAL_CONFIG_FILE, + &config::default_workspace_config_text(), + ); + let build_root = temp.path().join("build-root"); + + let only_args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "--build-root", + build_root.to_str().unwrap(), + "serve", + "--only", + "555", + ]); + let only_resolved = resolve_execution(&only_args).unwrap(); + let repo_path = materialize_resolved_repo(&only_resolved); + preprocess_and_prune_only(&repo_path, only_resolved.only.clone().unwrap()); + + assert!(repo_path.join("content/00555.md").is_file()); + assert!(!repo_path.join("content/00678.md").exists()); + + let normal_args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "--build-root", + build_root.to_str().unwrap(), + "serve", + ]); + let normal_resolved = resolve_execution(&normal_args).unwrap(); + assert!(normal_resolved.only.is_none()); + let restored_repo_path = materialize_resolved_repo(&normal_resolved); + markdown::preprocess(&restored_repo_path.join(CONTENT_DIR), None).unwrap(); + + assert!(restored_repo_path.join("content/00555.md").is_file()); + assert!(restored_repo_path.join("content/00678.md").is_file()); +} diff --git a/src/workspace.rs b/src/workspace.rs new file mode 100644 index 0000000..bd76562 --- /dev/null +++ b/src/workspace.rs @@ -0,0 +1,1138 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Local workspace setup and diagnostics. + +use std::{ + fmt, + fs::OpenOptions, + io::{ErrorKind, Write}, + path::{Path, PathBuf}, +}; + +use log::info; +use snafu::{Report, ResultExt, Whatever}; +use url::Url; + +use crate::{ + cli::Args, + config::{self, LoadedRepoManifest, LoadedWorkspaceConfig}, + context::{load_workspace_command_context, resolve_input_path, root}, + git, + identity::ActiveRepoIdentity, +}; + +const WORKSPACE_THEME_URL: &str = "https://github.com/eips-wg/theme.git"; +const PROPOSAL_TEMPLATE_URL: &str = "https://github.com/eips-wg/template.git"; +const PLATFORM_PREPROCESSOR_URL: &str = "https://github.com/eips-wg/preprocessor.git"; +const PLATFORM_EIPW_URL: &str = "https://github.com/ethereum/eipw.git"; +const WORKSPACE_DOC_FILE: &str = "WORKSPACE.md"; + +#[derive(Debug, Clone, Copy)] +enum DoctorStatus { + Ok, + Warn, + Fail, +} + +#[derive(Debug, Default)] +struct DoctorReport { + warnings: usize, + failures: usize, +} + +struct WorkspaceInitRepositories<'a> { + theme: &'a Url, + template: &'a Url, + preprocessor: &'a Url, + eipw: &'a Url, +} + +impl fmt::Display for DoctorStatus { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let label = match self { + Self::Ok => "ok", + Self::Warn => "warn", + Self::Fail => "fail", + }; + + f.write_str(label) + } +} + +impl DoctorReport { + fn record(&mut self, status: DoctorStatus, message: impl AsRef) { + match status { + DoctorStatus::Ok => (), + DoctorStatus::Warn => self.warnings += 1, + DoctorStatus::Fail => self.failures += 1, + } + + println!("[{status}] {}", message.as_ref()); + } +} + +fn command_path(command: &str) -> Option { + let path = std::env::var_os("PATH")?; + + #[cfg(not(windows))] + let candidates = [command.to_owned()]; + + #[cfg(windows)] + { + use std::ffi::OsString; + + let mut candidates = vec![command.to_owned()]; + let command = OsString::from(command); + let path_exts = std::env::var_os("PATHEXT") + .unwrap_or_default() + .to_string_lossy() + .split(';') + .filter(|ext| !ext.is_empty()) + .map(|ext| format!("{}{}", command.to_string_lossy(), ext)) + .collect::>(); + candidates.extend(path_exts); + std::env::split_paths(&path).find_map(|entry| { + candidates + .iter() + .map(|candidate| entry.join(candidate)) + .find(|candidate| candidate.is_file()) + }) + } + + #[cfg(not(windows))] + { + std::env::split_paths(&path).find_map(|entry| { + candidates + .iter() + .map(|candidate| entry.join(candidate)) + .find(|candidate| candidate.is_file()) + }) + } +} + +fn check_workspace_repo( + report: &mut DoctorReport, + workspace_root: &Path, + name: &str, +) -> Option { + let path = workspace_root.join(name); + match git2::Repository::open(&path) { + Ok(_) => { + report.record( + DoctorStatus::Ok, + format!( + "found workspace repo `{}` at `{}`", + name, + path.to_string_lossy() + ), + ); + Some(path) + } + Err(_) if !path.exists() => { + report.record( + DoctorStatus::Fail, + format!( + "expected workspace repo `{}` at `{}`", + name, + path.to_string_lossy() + ), + ); + None + } + Err(_) => { + report.record( + DoctorStatus::Fail, + format!( + "expected `{}` to be a git repository at `{}`", + name, + path.to_string_lossy() + ), + ); + None + } + } +} + +fn check_sibling_manifest_id( + report: &mut DoctorReport, + sibling_path: &Path, + expected_repo_id: &str, +) { + match LoadedRepoManifest::load(sibling_path) { + Ok(Some(manifest)) if manifest.manifest().repo_id == expected_repo_id => report.record( + DoctorStatus::Ok, + format!("sibling `{expected_repo_id}` manifest repo_id matches workspace key"), + ), + Ok(Some(manifest)) => report.record( + DoctorStatus::Fail, + format!( + "sibling `{expected_repo_id}` manifest declares repo_id `{}`", + manifest.manifest().repo_id + ), + ), + Ok(None) => (), + Err(error) => report.record( + DoctorStatus::Fail, + format!( + "sibling `{expected_repo_id}` repo manifest could not be loaded: {}", + Report::from_error(error) + ), + ), + } +} + +fn check_tool(report: &mut DoctorReport, command: &str, why: &str) -> bool { + match command_path(command) { + Some(path) => { + report.record( + DoctorStatus::Ok, + format!( + "found required tool `{}` at `{}`", + command, + path.to_string_lossy() + ), + ); + true + } + None => { + report.record( + DoctorStatus::Fail, + format!("missing required tool `{}`: {}", command, why), + ); + false + } + } +} + +#[cfg(windows)] +fn check_default_windows_build_eips_path(report: &mut DoctorReport) { + let Some(local_app_data) = std::env::var_os("LOCALAPPDATA") else { + return; + }; + + let install_dir = PathBuf::from(local_app_data).join("build-eips").join("bin"); + let build_eips_path = install_dir.join("build-eips.exe"); + + if build_eips_path.is_file() { + report.record( + DoctorStatus::Warn, + format!( + "found build-eips at the default user-local install path `{}`, but `{}` is not on PATH", + build_eips_path.to_string_lossy(), + install_dir.to_string_lossy() + ), + ); + } +} + +#[cfg(not(windows))] +fn check_default_windows_build_eips_path(_report: &mut DoctorReport) {} + +#[cfg(not(windows))] +fn check_optional_download_tool(report: &mut DoctorReport) { + let curl = command_path("curl"); + let wget = command_path("wget"); + + record_optional_download_tool(report, curl.as_deref(), wget.as_deref()); +} + +#[cfg(not(windows))] +fn record_optional_download_tool( + report: &mut DoctorReport, + curl: Option<&Path>, + wget: Option<&Path>, +) { + match (curl, wget) { + (Some(path), _) => report.record( + DoctorStatus::Ok, + format!( + "found front-door download helper `curl` at `{}`", + path.to_string_lossy() + ), + ), + (None, Some(path)) => report.record( + DoctorStatus::Ok, + format!( + "found front-door download helper `wget` at `{}`", + path.to_string_lossy() + ), + ), + (None, None) => report.record( + DoctorStatus::Warn, + "missing both `curl` and `wget`; `scripts/dev-setup` will not be able to download a release binary", + ), + } +} + +#[cfg(not(windows))] +fn check_front_door_archive_tool(report: &mut DoctorReport) { + let tar = command_path("tar"); + record_front_door_archive_tool(report, tar.as_deref()); +} + +#[cfg(not(windows))] +fn record_front_door_archive_tool(report: &mut DoctorReport, tar: Option<&Path>) { + match tar { + Some(path) => report.record( + DoctorStatus::Ok, + format!( + "found front-door archive tool `tar` at `{}`", + path.to_string_lossy() + ), + ), + None => report.record( + DoctorStatus::Warn, + "missing `tar`; `scripts/dev-setup` will not be able to unpack the release binary", + ), + } +} + +#[cfg(not(windows))] +fn check_front_door_setup_tools(report: &mut DoctorReport) { + check_optional_download_tool(report); + check_front_door_archive_tool(report); +} + +#[cfg(windows)] +fn check_front_door_setup_tools(_report: &mut DoctorReport) {} + +fn collect_doctor_report(args: &Args, check_tools: bool) -> Result { + let context = load_workspace_command_context(args)?; + let mut report = DoctorReport::default(); + let (root_path, active_repo) = match root(args) { + Ok(root_path) => match ActiveRepoIdentity::load(&root_path) { + Ok(active_repo) => { + report.record( + DoctorStatus::Ok, + format!( + "identified active repo `{}` from {}", + active_repo.repo_id(), + active_repo.source_description() + ), + ); + if let Some(manifest) = active_repo.manifest() { + report.record( + DoctorStatus::Ok, + format!( + "repo manifest parses at `{}`", + manifest.manifest_path().to_string_lossy() + ), + ); + } + (Some(root_path), Some(active_repo)) + } + Err(error) => { + report.record( + DoctorStatus::Fail, + format!("active repo identity could not be resolved: {error}"), + ); + (Some(root_path), None) + } + }, + Err(error) => { + report.record( + DoctorStatus::Fail, + format!( + "active repo root could not be resolved: {}", + Report::from_error(error) + ), + ); + (None, None) + } + }; + + match context.config_path.as_ref() { + Some(path) => report.record( + DoctorStatus::Ok, + format!( + "found workspace config candidate `{}`", + path.to_string_lossy() + ), + ), + None => report.record( + DoctorStatus::Fail, + format!( + "could not find `{}` while searching upward from `{}`", + config::LOCAL_CONFIG_FILE, + context.search_from.to_string_lossy() + ), + ), + } + + let parsed_config = context + .config_path + .as_deref() + .map(LoadedWorkspaceConfig::from_path) + .transpose(); + + match parsed_config { + Ok(Some(config)) => { + report.record( + DoctorStatus::Ok, + format!( + "workspace config parses at `{}`", + config.config_path().to_string_lossy() + ), + ); + + let workspace_root = config.workspace_root(); + if workspace_root.is_dir() { + report.record( + DoctorStatus::Ok, + format!( + "workspace root exists at `{}`", + workspace_root.to_string_lossy() + ), + ); + } else { + report.record( + DoctorStatus::Fail, + format!( + "workspace root is missing at `{}`", + workspace_root.to_string_lossy() + ), + ); + } + + if let (Some(root_path), Some(active_repo)) = (root_path.as_ref(), active_repo.as_ref()) + { + let expected_root = workspace_root.join(active_repo.repo_id()); + if root_path == &expected_root { + report.record( + DoctorStatus::Ok, + format!( + "active repo `{}` is checked out at `{}`", + active_repo.repo_id(), + expected_root.to_string_lossy() + ), + ); + } else { + report.record( + DoctorStatus::Fail, + format!( + "active repo `{}` should be checked out at `{}`, found `{}`", + active_repo.repo_id(), + expected_root.to_string_lossy(), + root_path.to_string_lossy() + ), + ); + } + + check_workspace_repo(&mut report, workspace_root, active_repo.repo_id()); + for sibling_id in active_repo.sibling_ids() { + if let Some(sibling_path) = + check_workspace_repo(&mut report, workspace_root, &sibling_id) + { + check_sibling_manifest_id(&mut report, &sibling_path, &sibling_id); + } + } + } else { + report.record( + DoctorStatus::Warn, + "workspace repo layout checks were skipped because active repo identity was unavailable", + ); + } + + check_workspace_repo(&mut report, workspace_root, config::DEFAULT_THEME_DIR); + } + Err(error) => { + report.record( + DoctorStatus::Fail, + format!( + "workspace config could not be parsed: {}", + Report::from_error(error) + ), + ); + } + Ok(None) => (), + } + + if check_tools { + if !check_tool( + &mut report, + "build-eips", + "workspace bootstrap and build-eips commands expect `build-eips` on PATH", + ) { + check_default_windows_build_eips_path(&mut report); + } + check_tool( + &mut report, + "git", + "workspace bootstrap and build-eips commands expect git to be available", + ); + check_tool( + &mut report, + "zola", + "build, check, and serve commands need a working zola binary", + ); + check_front_door_setup_tools(&mut report); + } + + Ok(report) +} + +pub(crate) fn doctor_workspace(args: &Args) -> Result<(), Whatever> { + let report = collect_doctor_report(args, true)?; + + if report.failures > 0 { + snafu::whatever!("doctor found {} failing check(s)", report.failures); + } + + Ok(()) +} + +pub(crate) fn init_workspace( + args: &Args, + workspace_root: PathBuf, + include_template: bool, + platform_dev: bool, +) -> Result<(), Whatever> { + let theme_repository = Url::parse(WORKSPACE_THEME_URL) + .whatever_context("invalid workspace theme repository URL")?; + let template_repository = Url::parse(PROPOSAL_TEMPLATE_URL) + .whatever_context("invalid proposal template repository URL")?; + let preprocessor_repository = Url::parse(PLATFORM_PREPROCESSOR_URL) + .whatever_context("invalid platform preprocessor repository URL")?; + let eipw_repository = + Url::parse(PLATFORM_EIPW_URL).whatever_context("invalid platform eipw repository URL")?; + let repositories = WorkspaceInitRepositories { + theme: &theme_repository, + template: &template_repository, + preprocessor: &preprocessor_repository, + eipw: &eipw_repository, + }; + + init_workspace_with_repositories( + args, + workspace_root, + include_template, + platform_dev, + &repositories, + ) +} + +fn init_workspace_with_repositories( + args: &Args, + workspace_root: PathBuf, + include_template: bool, + platform_dev: bool, + repositories: &WorkspaceInitRepositories<'_>, +) -> Result<(), Whatever> { + let root_path = root(args)?; + let active_repo = ActiveRepoIdentity::load(&root_path)?; + let workspace_root = resolve_input_path(&workspace_root)?; + std::fs::create_dir_all(&workspace_root) + .whatever_context("unable to create workspace root directory")?; + let workspace_root = workspace_root + .canonicalize() + .whatever_context("unable to canonicalize workspace root directory")?; + + // Workspace init is a local-dev bootstrap path, so it intentionally uses staging URLs. + let repository_use = active_repo.repository_use(true)?; + + let expected_root = workspace_root.join(&repository_use.title); + if root_path != expected_root { + snafu::whatever!( + "init expects the active repository at `{}`, found `{}`", + expected_root.to_string_lossy(), + root_path.to_string_lossy(), + ); + } + + for (sibling_id, sibling_url) in repository_use.other_repos { + git::clone_missing_repo(sibling_url.as_str(), &workspace_root.join(&sibling_id)) + .with_whatever_context(|_| { + format!("unable to clone workspace sibling repo `{sibling_id}`") + })?; + } + + git::clone_missing_repo( + repositories.theme.as_str(), + &workspace_root.join(config::DEFAULT_THEME_DIR), + ) + .whatever_context("unable to clone workspace theme repo")?; + + if include_template { + git::clone_missing_repo( + repositories.template.as_str(), + &workspace_root.join("template"), + ) + .whatever_context("unable to clone workspace template repo")?; + } + + if platform_dev { + git::clone_missing_repo( + repositories.preprocessor.as_str(), + &workspace_root.join("preprocessor"), + ) + .whatever_context("unable to clone workspace preprocessor repo")?; + git::clone_missing_repo(repositories.eipw.as_str(), &workspace_root.join("eipw")) + .whatever_context("unable to clone workspace eipw repo")?; + } + + std::fs::create_dir_all(workspace_root.join(config::DEFAULT_BUILD_ROOT_BASE)) + .whatever_context("unable to create local build root")?; + + let config_path = workspace_root.join(config::LOCAL_CONFIG_FILE); + match OpenOptions::new() + .write(true) + .create_new(true) + .open(&config_path) + { + Ok(mut config_file) => { + config_file + .write_all(config::default_workspace_config_text().as_bytes()) + .whatever_context("unable to write workspace config")?; + } + Err(error) if error.kind() == ErrorKind::AlreadyExists => { + info!( + "leaving existing workspace config `{}` in place", + config_path.to_string_lossy() + ); + } + Err(error) => { + return Err(error).whatever_context("unable to write workspace config"); + } + } + + write_workspace_doc(&workspace_root)?; + + Ok(()) +} + +fn workspace_doc_text() -> &'static str { + include_str!("workspace_doc.md") +} + +fn write_workspace_doc(workspace_root: &Path) -> Result<(), Whatever> { + let doc_path = workspace_root.join(WORKSPACE_DOC_FILE); + std::fs::write(&doc_path, workspace_doc_text()).with_whatever_context(|_| { + format!( + "unable to write workspace document `{}`", + doc_path.to_string_lossy() + ) + })?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use clap::Parser; + use git2::{IndexAddOption, Repository, Signature}; + use tempfile::TempDir; + use url::Url; + + use crate::{ + cli::{Args, Operation}, + config::{self, LoadedWorkspaceConfig}, + }; + + use super::{ + collect_doctor_report, init_workspace_with_repositories, workspace_doc_text, + WorkspaceInitRepositories, WORKSPACE_DOC_FILE, WORKSPACE_THEME_URL, + }; + + fn parse_args(arguments: &[&str]) -> Args { + Args::try_parse_from(arguments).unwrap() + } + + fn file_url(path: &Path) -> Url { + Url::from_directory_path(path).unwrap() + } + + fn write_file(root: &Path, relative: impl AsRef, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn commit_all(repo: &Repository, message: &str) { + let mut index = repo.index().unwrap(); + index + .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + let parents = repo + .head() + .ok() + .and_then(|head| head.target()) + .map(|oid| repo.find_commit(oid).unwrap()) + .into_iter() + .collect::>(); + let parent_refs = parents.iter().collect::>(); + + repo.commit( + Some("HEAD"), + &signature, + &signature, + message, + &tree, + &parent_refs, + ) + .unwrap(); + } + + fn init_repo(path: &Path, files: &[(&str, &str)]) -> Repository { + std::fs::create_dir_all(path).unwrap(); + let repo = Repository::init(path).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + for (relative, contents) in files { + write_file(path, relative, contents); + } + commit_all(&repo, "initial"); + repo + } + + fn repo_manifest_text(repo_id: &str, repository: &Url, siblings: &[(&str, Url)]) -> String { + let mut manifest = format!( + r#" +repo_id = "{repo_id}" + +[production] +repository = "{repository}" +base_url = "https://example.test/{repo_id}/" + +[staging] +repository = "{repository}" +base_url = "https://staging.example.test/{repo_id}/" +"# + ); + + for (sibling_id, sibling_repository) in siblings { + manifest.push_str(&format!( + r#" +[siblings.{sibling_id}.production] +repository = "{sibling_repository}" +base_url = "https://example.test/{sibling_id}/" + +[siblings.{sibling_id}.staging] +repository = "{sibling_repository}" +base_url = "https://staging.example.test/{sibling_id}/" +"# + )); + } + + manifest + } + + fn write_repo_manifest_file( + path: &Path, + repo_id: &str, + upstream: &Url, + siblings: &[(&str, Url)], + ) { + write_file( + path, + config::REPO_MANIFEST_FILE, + &repo_manifest_text(repo_id, upstream, siblings), + ); + } + + fn write_manifest_repo( + path: &Path, + repo_id: &str, + upstream: &Url, + siblings: &[(&str, Url)], + ) -> Repository { + let repo = init_repo(path, &[("content/0001.md", "# Proposal\n")]); + write_repo_manifest_file(path, repo_id, upstream, siblings); + commit_all(&repo, "add repo manifest"); + repo + } + + fn init_workspace_source_repo(remotes_root: &Path, name: &str) -> Url { + let path = remotes_root.join(name); + init_repo(&path, &[("README.md", "init test repo\n")]); + file_url(&path) + } + + fn workspace_init_test_repository_urls(remotes_root: &Path) -> (Url, Url, Url, Url) { + ( + init_workspace_source_repo(remotes_root, "theme"), + init_workspace_source_repo(remotes_root, "template"), + init_workspace_source_repo(remotes_root, "preprocessor"), + init_workspace_source_repo(remotes_root, "eipw"), + ) + } + + fn run_workspace_init_for_docs( + existing_doc: Option<&str>, + existing_config: Option<&str>, + ) -> (TempDir, std::path::PathBuf) { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let remotes_root = temp.path().join("remotes"); + let (theme_url, template_url, preprocessor_url, eipw_url) = + workspace_init_test_repository_urls(&remotes_root); + let repositories = WorkspaceInitRepositories { + theme: &theme_url, + template: &template_url, + preprocessor: &preprocessor_url, + eipw: &eipw_url, + }; + let active_path = workspace_root.join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + + if let Some(contents) = existing_doc { + write_file(&workspace_root, WORKSPACE_DOC_FILE, contents); + } + if let Some(contents) = existing_config { + write_file(&workspace_root, config::LOCAL_CONFIG_FILE, contents); + } + + let init_args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "init", + workspace_root.to_str().unwrap(), + ]); + + init_workspace_with_repositories( + &init_args, + workspace_root.clone(), + false, + false, + &repositories, + ) + .unwrap(); + + (temp, workspace_root) + } + + #[test] + fn workspace_theme_url_is_bootstrap_metadata() { + assert_eq!( + Url::parse(WORKSPACE_THEME_URL).unwrap().as_str(), + "https://github.com/eips-wg/theme.git" + ); + } + + #[cfg(not(windows))] + #[test] + fn front_door_setup_tool_records_posix_helper_warnings() { + let mut report = super::DoctorReport::default(); + + super::record_optional_download_tool(&mut report, None, None); + super::record_front_door_archive_tool(&mut report, None); + + assert_eq!(report.warnings, 2); + assert_eq!(report.failures, 0); + } + + #[cfg(not(windows))] + #[test] + fn front_door_setup_tool_accepts_posix_helpers() { + let mut report = super::DoctorReport::default(); + let tool_path = Path::new("/usr/bin/tool"); + + super::record_optional_download_tool(&mut report, Some(tool_path), None); + super::record_front_door_archive_tool(&mut report, Some(tool_path)); + + assert_eq!(report.warnings, 0); + assert_eq!(report.failures, 0); + } + + #[cfg(windows)] + #[test] + fn front_door_setup_tools_skip_posix_helpers_on_windows() { + let mut report = super::DoctorReport::default(); + + super::check_front_door_setup_tools(&mut report); + + assert_eq!(report.warnings, 0); + assert_eq!(report.failures, 0); + } + + #[test] + fn workspace_doc_text_mentions_required_workspace_reference_content() { + let text = workspace_doc_text(); + + for expected in [ + ".build-eips.toml", + ".local-build", + "build-eips init", + "build-eips doctor", + "build-eips build", + "build-eips serve", + "build-eips preview", + "build-eips editorial check", + "[render]", + "only = [", + "--only", + "--remote-siblings", + "--base-url", + ] { + assert!( + text.contains(expected), + "workspace document text should contain `{expected}`" + ); + } + + assert!(text.ends_with('\n')); + } + + #[test] + fn workspace_init_writes_workspace_doc() { + let (_temp, workspace_root) = run_workspace_init_for_docs(None, None); + + let doc = std::fs::read_to_string(workspace_root.join(WORKSPACE_DOC_FILE)).unwrap(); + assert_eq!(doc, workspace_doc_text()); + } + + #[test] + fn workspace_init_overwrites_existing_workspace_doc() { + let existing_doc = "Old workspace docs\n"; + let (_temp, workspace_root) = run_workspace_init_for_docs(Some(existing_doc), None); + + let doc = std::fs::read_to_string(workspace_root.join(WORKSPACE_DOC_FILE)).unwrap(); + assert_ne!(doc, existing_doc); + assert_eq!(doc, workspace_doc_text()); + } + + #[test] + fn workspace_init_leaves_existing_config_without_render_unchanged() { + let existing_config = "[server]\nhost = \"127.0.0.1\"\nport = 1111\n"; + let (_temp, workspace_root) = run_workspace_init_for_docs(None, Some(existing_config)); + + assert_eq!( + std::fs::read_to_string(workspace_root.join(config::LOCAL_CONFIG_FILE)).unwrap(), + existing_config + ); + } + + fn assert_workspace_init_optional_repos( + workspace_root: &Path, + expect_template: bool, + expect_platform_dev: bool, + ) { + assert!(Repository::open(workspace_root.join(config::DEFAULT_THEME_DIR)).is_ok()); + assert_eq!( + Repository::open(workspace_root.join("template")).is_ok(), + expect_template + ); + assert_eq!( + Repository::open(workspace_root.join("preprocessor")).is_ok(), + expect_platform_dev + ); + assert_eq!( + Repository::open(workspace_root.join("eipw")).is_ok(), + expect_platform_dev + ); + } + + fn assert_workspace_init_and_doctor_for_siblings(sibling_ids: &[&str]) { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let remotes_root = temp.path().join("remotes"); + let (theme_url, template_url, preprocessor_url, eipw_url) = + workspace_init_test_repository_urls(&remotes_root); + let repositories = WorkspaceInitRepositories { + theme: &theme_url, + template: &template_url, + preprocessor: &preprocessor_url, + eipw: &eipw_url, + }; + + let sibling_repositories = sibling_ids + .iter() + .map(|sibling_id| { + let sibling_id = *sibling_id; + let sibling_path = remotes_root.join(sibling_id); + let sibling_url = file_url(&sibling_path); + write_manifest_repo(&sibling_path, sibling_id, &sibling_url, &[]); + (sibling_id.to_owned(), sibling_url) + }) + .collect::>(); + let sibling_manifest_entries = sibling_repositories + .iter() + .map(|(repo_id, url)| (repo_id.as_str(), url.clone())) + .collect::>(); + let active_path = workspace_root.join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &sibling_manifest_entries); + let init_args = parse_args(&[ + "build-eips", + "-C", + active_path.to_str().unwrap(), + "init", + workspace_root.to_str().unwrap(), + ]); + + init_workspace_with_repositories( + &init_args, + workspace_root.clone(), + false, + false, + &repositories, + ) + .unwrap(); + + assert!(workspace_root.join(config::LOCAL_CONFIG_FILE).is_file()); + assert_workspace_init_optional_repos(&workspace_root, false, false); + for sibling_id in sibling_ids { + assert!(Repository::open(workspace_root.join(sibling_id)).is_ok()); + } + + let doctor_args = + parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "doctor"]); + let report = collect_doctor_report(&doctor_args, false).unwrap(); + + assert_eq!(report.failures, 0); + } + + fn assert_workspace_init_optional_clone_behavior( + flags: &[&str], + expect_template: bool, + expect_platform_dev: bool, + ) { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let remotes_root = temp.path().join("remotes"); + let (theme_url, template_url, preprocessor_url, eipw_url) = + workspace_init_test_repository_urls(&remotes_root); + let repositories = WorkspaceInitRepositories { + theme: &theme_url, + template: &template_url, + preprocessor: &preprocessor_url, + eipw: &eipw_url, + }; + let active_path = workspace_root.join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + let active_path = active_path.to_string_lossy(); + let workspace_root_arg = workspace_root.to_string_lossy(); + let mut arguments = vec![ + "build-eips", + "-C", + active_path.as_ref(), + "init", + workspace_root_arg.as_ref(), + ]; + arguments.extend_from_slice(flags); + let init_args = parse_args(&arguments); + let Operation::Init { + path: workspace_root_path, + template, + platform_dev, + } = init_args.operation.clone() + else { + panic!("expected init command"); + }; + + assert_eq!(template, expect_template); + assert_eq!(platform_dev, expect_platform_dev); + + init_workspace_with_repositories( + &init_args, + workspace_root_path, + template, + platform_dev, + &repositories, + ) + .unwrap(); + + assert_workspace_init_optional_repos(&workspace_root, expect_template, expect_platform_dev); + } + + #[test] + fn workspace_init_and_doctor_cover_zero_one_and_many_siblings() { + assert_workspace_init_and_doctor_for_siblings(&[]); + assert_workspace_init_and_doctor_for_siblings(&["ERCs"]); + assert_workspace_init_and_doctor_for_siblings(&["EIPs", "ERCs"]); + } + + #[test] + fn default_workspace_init_clones_required_repos_only() { + assert_workspace_init_optional_clone_behavior(&[], false, false); + } + + #[test] + fn workspace_init_template_clones_template_only_as_optional_repo() { + assert_workspace_init_optional_clone_behavior(&["--template"], true, false); + } + + #[test] + fn workspace_init_platform_dev_clones_platform_repos_only_as_optional_repos() { + assert_workspace_init_optional_clone_behavior(&["--platform-dev"], false, true); + } + + #[test] + fn workspace_init_template_and_platform_dev_clone_all_optional_repos() { + assert_workspace_init_optional_clone_behavior( + &["--template", "--platform-dev"], + true, + true, + ); + } + + #[test] + fn workspace_doctor_missing_config_reports_one_failure_without_skip_warning() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + let args = parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "doctor"]); + + let report = collect_doctor_report(&args, false).unwrap(); + + assert_eq!(report.failures, 1); + assert_eq!(report.warnings, 0); + } + + #[test] + fn workspace_doctor_parse_failed_config_reports_one_failure_without_skip_warning() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + std::fs::write(workspace.path().join(config::LOCAL_CONFIG_FILE), "[").unwrap(); + let args = parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "doctor"]); + + let report = collect_doctor_report(&args, false).unwrap(); + + assert_eq!(report.failures, 1); + assert_eq!(report.warnings, 0); + } + + #[test] + fn workspace_doctor_removed_config_fields_report_parse_failure_check() { + let workspace = TempDir::new().unwrap(); + let active_path = workspace.path().join("Core"); + let active_url = file_url(&active_path); + write_manifest_repo(&active_path, "Core", &active_url, &[]); + let config_path = workspace.path().join(config::LOCAL_CONFIG_FILE); + std::fs::write( + &config_path, + r#" +build_root_base = ".local-build" +default_profile = "local" + +[profiles.local] +staging = true +"#, + ) + .unwrap(); + let error = LoadedWorkspaceConfig::from_path(&config_path).unwrap_err(); + assert!(matches!(error, config::WorkspaceError::Parse { .. })); + let args = parse_args(&["build-eips", "-C", active_path.to_str().unwrap(), "doctor"]); + + let report = collect_doctor_report(&args, false).unwrap(); + + assert_eq!(report.failures, 1); + assert_eq!(report.warnings, 0); + } +} diff --git a/src/workspace_doc.md b/src/workspace_doc.md new file mode 100644 index 0000000..d2216be --- /dev/null +++ b/src/workspace_doc.md @@ -0,0 +1,269 @@ +# build-eips Workspace + +This directory is a local multi-repo workspace for building, serving, previewing, and validating EIPs/ERCs with the shared theme and proposal sibling repos. + +The workspace keeps the proposal repos, theme repo, generated build output, and local workspace settings in one predictable layout. Run commands from an active proposal repo such as `EIPs/` or `ERCs/`, or from this workspace root with `-C EIPs` or `-C ERCs`. + +## Workspace Layout + +After running a setup script, the minimal operational workspace should look like this: + +```text +EIPs-project/ +├── .build-eips.toml +├── WORKSPACE.md +├── .local-build/ +├── EIPs/ +├── ERCs/ +└── theme/ +``` + +Optional setup flags can add more repos: + +```text +EIPs-project/ +├── template/ # --template +├── preprocessor/ # --platform-dev +└── eipw/ # --platform-dev +``` + +- `.build-eips.toml`: workspace settings. +- `WORKSPACE.md`: generated workspace guide. +- `.local-build/`: generated build output and materialized repositories. +- `EIPs/` and `ERCs/`: proposal source repositories. +- `theme/`: workspace-local Zola theme required by build, serve, and check commands. +- `template/`: optional proposal template repository. +- `preprocessor/`: optional local `build-eips` development checkout. +- `eipw/`: optional local `eipw` development checkout. + +If the optional repos are missing, rerun build-eips init with the needed flags. + +From an active proposal repo: + +```sh +build-eips init .. --template +build-eips init .. --platform-dev +build-eips init .. --template --platform-dev +``` + +From the workspace root: + +```sh +build-eips -C EIPs init . --template +build-eips -C EIPs init . --platform-dev +build-eips -C EIPs init . --template --platform-dev +``` + +## Requirements And Troubleshooting + +Local workspace commands require these tools on `PATH`: + +- Git +- `build-eips` +- Zola 0.22.1 + +Git must be installed separately. The setup scripts locate or install `build-eips` and Zola, add locally installed tool directories to `PATH` for the current shell session, and print guidance for making those `PATH` changes permanent. + +Run `build-eips doctor` after setup and whenever a command cannot find a repo, config file, theme, or required tool: + +```sh +build-eips doctor +``` + +From the workspace root, anchor the command through an active proposal repo: + +```sh +build-eips -C EIPs doctor +build-eips -C ERCs doctor +``` + +`build-eips doctor` checks: + +- required tools: Git, `build-eips`, and Zola 0.22.1 +- the active proposal repo manifest +- `.build-eips.toml` +- workspace-local sibling proposal repos +- workspace-local `theme/` +- optional setup helper tools used by setup scripts + +If a fresh shell cannot find `build-eips` or Zola, rerun the setup script or apply the permanent `PATH` guidance printed by the setup script. + +If a sibling repo, `theme/`, or optional platform repo is missing, rerun `build-eips init` with the needed flags. + +If `theme/` or `preprocessor/` setup cannot create the default `EIPs/` checkout, check that Git is installed and that the workspace `EIPs/` path does not already exist as a non-git directory. Set `ACTIVE_REPO_ROOT` when you want setup to use an existing ERCs or custom proposal repo checkout. + +If `build-eips doctor` reports that Zola is missing or too old, rerun the setup script to install the supported Zola version. + +### Build And Serve Locally + +Build the full static site, then preview that built output: + +```bash +build-eips build +build-eips preview +``` + +`preview` serves the last output written by `build`. Run `build` again before `preview` when you want to inspect fresh output. + +Use `serve` when you want a live development server that livereloads changes instead of a reusable build output: + +```bash +build-eips serve +``` + +`serve` runs a fresh temporary site build each time it is invoked (without using `build`), starts a local development server, and watches tracked local edits. Its output cannot be reused by `preview`. + +Use `check` to quickly validate whether the site will build cleanly without producing the full built site: + +```bash +build-eips check +``` + +By default, `check`, `build`, and `serve` use the local workspace in dirty mode, which includes tracked working-tree edits from this repo. `preview` serves the last output written by `build`. Use `--clean` when you want to ignore tracked local proposal edits for one command: + +```bash +build-eips check --clean +build-eips build --clean +build-eips serve --clean +``` + +For staging, production, parity, and remote-sibling modes, see `../WORKSPACE.md`. + +### Local Settings + +Local build settings live in `../.build-eips.toml`, which the setup script generates. Use that workspace file to change the local server address or local site URL: + +```toml +[server] +host = "127.0.0.1" +port = 1111 + +[site] +base_url = "http://127.0.0.1:1111" +``` + +`serve` and `preview` use `[server]` for the local bind address. `build` and `serve` use `[site].base_url` when generating links. + +CLI flags such as `--host`, `--port`, and `--base-url` override the workspace config for one run: + +```bash +build-eips serve --host 0.0.0.0 --port 3000 --base-url http://127.0.0.1:3000 +``` + +### Render Specific Proposals Only + +Full local `build` and `serve` runs can take time because they process every proposal file. When you want to quickly test a single proposal or a specific batch, add a list of desired proposal numbers to the workspace `.build-eips.toml`: + +```toml +[render] +only = [555, 678] +``` + +Add one or more proposal numbers in `[render].only`, separated by commas. It's empty by default, but whenever it is populated, the regular `build` and `serve` commands render only those proposal pages. Links and references to excluded proposals are rewritten to the canonical public site. + +Use CLI `--only` when you want a one-run target list; it also overrides any proposals in `[render].only` for that run: + +```bash +build-eips serve --only 555 +build-eips build --only 555 +build-eips build --only 555 678 +``` + +Multiple proposal numbers in the CLI are space-separated; no commas. + +### Editorial Validation + +Use editorial commands to validate proposal files before opening or updating a pull request. + +- `editorial lint` runs targeted `eipw` proposal-rule checks. +- `editorial check` runs `editorial lint`, then checks that the selected proposal changes will not prevent the full site from building cleanly. + +Check one or more specific proposals by number: + +```bash +build-eips editorial check 1 +build-eips editorial check 1 123 +``` + +For the closest match to PR CI, use `editorial check` against the proposal files changed versus upstream: + +```bash +build-eips --staging editorial check --against-upstream --format github +``` + +Both commands accept the same selector modes: + +* proposal numbers or repo-relative proposal paths for explicit targets +* `--working-tree` for tracked dirty proposal files +* `--against-upstream` for proposal files changed versus the upstream merge-base +* `--batch ` for a repeatable target list + +They also accept `eipw` options such as `--format github`. + +Use a batch file when you want to lint or check the same proposal set repeatedly. A batch file is a plain text file with one proposal number per line: + +```txt +1 +7949 +``` + +```bash +build-eips editorial lint --batch ../editor-batch.txt +build-eips editorial check --batch ../editor-batch.txt +``` + +### Source And Output Overrides + +Workspace-local sources come from the standard workspace layout. The local theme is `workspace/theme`, and local sibling repos are `workspace/` from the active repo manifest. + +Use `--remote-siblings` when you need to force remote sibling proposal sources for a single command. + +Use global `--build-root ` when you want a separate prepared repo and output directory, for example to compare two builds side by side. The path replaces the default `.local-build/` location for each command where you pass it, so use the same `--build-root` value when serving or previewing builds. + +Example: + +```bash +build-eips -C /work/EIPs-project/EIPs --build-root /tmp/eips-local build --base-url http://127.0.0.1:1111 +build-eips -C /work/EIPs-project/EIPs --build-root /tmp/eips-staging --staging build --base-url http://127.0.0.1:1112 + +build-eips -C /work/EIPs-project/EIPs --build-root /tmp/eips-local preview --port 1111 +build-eips -C /work/EIPs-project/EIPs --build-root /tmp/eips-staging preview --port 1112 + +# Or using serve +build-eips -C /work/EIPs-project/EIPs --build-root /tmp/eips-local serve --port 1111 +build-eips -C /work/EIPs-project/EIPs --build-root /tmp/eips-staging --staging serve --port 1112 +``` + +### Remote And Parity Modes + +Use remote modes when you want a clean render of the local active checkout with staging or production environment metadata and remote sibling proposal sources. + +`--staging` and `--production` use the local active checkout, reject dirty active-repo edits, and select remote sibling sources plus staging or production environment metadata: + +```sh +build-eips --staging check +build-eips --staging build +build-eips --staging serve + +build-eips --production check +build-eips --production build +build-eips --production serve +``` + +`parity` is the built-in clean staging path for checking whether the local active checkout behaves like the staging environment: + +```sh +build-eips parity check +build-eips parity build +build-eips parity serve +``` + +Use `--remote-siblings` when you want to keep the active proposal repo local, but resolve sibling proposal repos from the configured remote environment: + +```sh +build-eips --remote-siblings check +build-eips --remote-siblings build +build-eips --remote-siblings serve +``` + +Remote environment commands and `parity` use the local active checkout, but do not use local dirty proposal edits. They still use the workspace-local `theme/`, so check out the theme commit or branch you want before running them. diff --git a/src/zola.rs b/src/zola.rs index fc65638..eb185af 100644 --- a/src/zola.rs +++ b/src/zola.rs @@ -15,7 +15,10 @@ use semver::Version; use snafu::{ensure, Backtrace, IntoError, Report, ResultExt, Snafu}; use url::Url; -use crate::{cache::Cache, git}; +use crate::{ + config::ServerBinding, + layout::{mounted_theme_path, theme_config_path}, +}; const MINIMUM_VERSION: Version = Version::new(0, 22, 1); @@ -38,8 +41,11 @@ fn symlink_dir(original: &Path, link: &Path) -> Result<(), std::io::Error> { } fn force_symlink_dir(original: &Path, link: &Path) -> Result<(), std::io::Error> { - match std::fs::remove_file(link) { - Ok(()) => (), + match std::fs::symlink_metadata(link) { + Ok(metadata) if metadata.file_type().is_dir() && !metadata.file_type().is_symlink() => { + std::fs::remove_dir_all(link)?; + } + Ok(_) => std::fs::remove_file(link)?, Err(e) if e.kind() == ErrorKind::NotFound => (), Err(e) => return Err(e), } @@ -47,6 +53,19 @@ fn force_symlink_dir(original: &Path, link: &Path) -> Result<(), std::io::Error> symlink_dir(original, link) } +fn mount_theme(theme_dir: &Path, project_path: &Path) -> Result { + let mounted_theme_path = mounted_theme_path(project_path); + if theme_dir == mounted_theme_path { + return Ok(mounted_theme_path); + } + + if let Some(parent) = mounted_theme_path.parent() { + std::fs::create_dir_all(parent)?; + } + force_symlink_dir(theme_dir, &mounted_theme_path)?; + Ok(mounted_theme_path) +} + #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("could not find zola binary (requires at least version {MINIMUM_VERSION})"))] @@ -71,11 +90,6 @@ pub enum Error { backtrace: Backtrace, source: std::io::Error, }, - #[snafu(context(false))] - Git { - #[snafu(backtrace)] - source: git::Error, - }, } pub fn find_zola() -> Result<(), Error> { @@ -96,21 +110,14 @@ pub fn find_zola() -> Result<(), Error> { Ok(()) } -pub fn check( - theme_repo: &str, - theme_rev: &str, - cache: &Cache, - project_path: &Path, -) -> Result<(), Error> { +pub fn check(theme_dir: &Path, project_path: &Path) -> Result<(), Error> { let args = ["check", "--drafts", "--skip-external-links"]; - spawn_log(theme_repo, theme_rev, cache, project_path, args)?; + spawn_log(theme_dir, project_path, args)?; Ok(()) } pub fn build( - theme_repo: &str, - theme_rev: &str, - cache: &Cache, + theme_dir: &Path, project_path: &Path, output_path: &Path, base_url: &str, @@ -120,7 +127,7 @@ pub fn build( .map(OsString::from) .into_iter() .chain(std::iter::once(output_path.into())); - spawn_log(theme_repo, theme_rev, cache, project_path, args)?; + spawn_log(theme_dir, project_path, args)?; if let Ok(url) = Url::from_file_path(output_path) { info!("HTML output to: {}", url); } @@ -128,23 +135,50 @@ pub fn build( } pub fn serve( - theme_repo: &str, - theme_rev: &str, - cache: &Cache, + theme_dir: &Path, project_path: &Path, output_path: &Path, + server_binding: &ServerBinding, + base_url_override: Option<&Url>, ) -> Result<(), Error> { // TODO: Properly kill the child process when we receive ctrl-c. - warn!("live reloading is not implemented"); remove_output(output_path); - let args = ["serve", "--drafts", "-o"] - .map(OsString::from) - .into_iter() - .chain(std::iter::once(output_path.into())); - spawn_log(theme_repo, theme_rev, cache, project_path, args)?; + let args = serve_args(server_binding, output_path, base_url_override); + spawn_log(theme_dir, project_path, args)?; Ok(()) } +fn serve_args( + server_binding: &ServerBinding, + output_path: &Path, + base_url_override: Option<&Url>, +) -> Vec { + let mut args = [ + "serve", + "--drafts", + "--fast", + "--force", + "--interface", + server_binding.host.as_str(), + "--port", + ] + .map(OsString::from) + .to_vec(); + + args.push(OsString::from(server_binding.port.to_string())); + + if let Some(base_url) = base_url_override { + args.extend([ + OsString::from("-u"), + OsString::from(base_url.as_str()), + OsString::from("--no-port-append"), + ]); + } + + args.extend([OsString::from("-o"), output_path.as_os_str().to_os_string()]); + args +} + fn remove_output(output_path: &Path) { if let Err(e) = std::fs::remove_dir_all(output_path) { debug!( @@ -154,13 +188,7 @@ fn remove_output(output_path: &Path) { } } -fn spawn_log( - theme_repo: &str, - theme_rev: &str, - cache: &Cache, - project_path: &Path, - args: U, -) -> Result<(), Error> +fn spawn_log(theme_dir: &Path, project_path: &Path, args: U) -> Result<(), Error> where U: IntoIterator, I: Into, @@ -173,18 +201,9 @@ where find_zola()?; - let theme_dir = cache.repo(theme_repo, theme_rev)?; - - let mut themes_dir = project_path.join("themes"); - if let Err(e) = std::fs::create_dir(&themes_dir) { - debug!("got while creating themes dir: {}", Report::from_error(e)); - } - themes_dir.push("eips-theme"); - force_symlink_dir(&theme_dir, &themes_dir).context(FsSnafu { path: &themes_dir })?; - - let config_path: PathBuf = [&theme_dir, Path::new("config"), Path::new("zola.toml")] - .iter() - .collect(); + let mounted_theme_path = + mount_theme(theme_dir, project_path).context(FsSnafu { path: theme_dir })?; + let config_path = theme_config_path(&mounted_theme_path); let prefix = [OsString::from("-c"), config_path.into()].into_iter(); let args = prefix.chain(args.into_iter().map(Into::into)); @@ -221,3 +240,188 @@ where Ok(()) } + +#[cfg(test)] +mod tests { + use std::{ + ffi::OsString, + fs, + path::{Path, PathBuf}, + process::{Command, ExitStatus}, + }; + + use crate::{ + config::ServerBinding, + layout::{mounted_theme_path, theme_config_path}, + }; + use tempfile::TempDir; + + use super::{find_zola, mount_theme, serve_args}; + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).unwrap(); + } + fs::write(path, contents).unwrap(); + } + + fn zola_build_status(project_root: &Path) -> ExitStatus { + Command::new("zola") + .arg("build") + .arg("--drafts") + .current_dir(project_root) + .status() + .unwrap() + } + + #[test] + fn zola_rejects_missing_internal_links_but_accepts_external_links() { + if find_zola().is_err() { + eprintln!("skipping zola link behavior fixture because zola is not installed"); + return; + } + + let temp = TempDir::new().unwrap(); + let internal = temp.path().join("internal"); + write_file( + &internal, + "config.toml", + "base_url = \"https://example.test\"\n", + ); + write_file( + &internal, + "content/_index.md", + "+++\ntitle = \"Internal\"\n+++\n[Missing](@/missing.md)\n", + ); + let external = temp.path().join("external"); + write_file( + &external, + "config.toml", + "base_url = \"https://example.test\"\n", + ); + write_file( + &external, + "content/_index.md", + "+++\ntitle = \"External\"\n+++\n[External](https://eips.ethereum.org/EIPS/eip-1)\n", + ); + + assert!(!zola_build_status(&internal).success()); + assert!(zola_build_status(&external).success()); + } + + #[test] + fn serve_args_include_configured_interface_and_port() { + let server_binding = ServerBinding { + host: "0.0.0.0".to_owned(), + port: 8080, + }; + + assert_eq!( + serve_args(&server_binding, Path::new("/tmp/build-output"), None), + vec![ + OsString::from("serve"), + OsString::from("--drafts"), + OsString::from("--fast"), + OsString::from("--force"), + OsString::from("--interface"), + OsString::from("0.0.0.0"), + OsString::from("--port"), + OsString::from("8080"), + OsString::from("-o"), + OsString::from("/tmp/build-output"), + ] + ); + } + + #[test] + fn serve_args_include_base_url_override_when_present() { + let server_binding = ServerBinding { + host: "127.0.0.1".to_owned(), + port: 1111, + }; + let base_url = "http://127.0.0.1:1111".parse().unwrap(); + + assert_eq!( + serve_args( + &server_binding, + Path::new("/tmp/build-output"), + Some(&base_url) + ), + vec![ + OsString::from("serve"), + OsString::from("--drafts"), + OsString::from("--fast"), + OsString::from("--force"), + OsString::from("--interface"), + OsString::from("127.0.0.1"), + OsString::from("--port"), + OsString::from("1111"), + OsString::from("-u"), + OsString::from("http://127.0.0.1:1111/"), + OsString::from("--no-port-append"), + OsString::from("-o"), + OsString::from("/tmp/build-output"), + ] + ); + } + + #[test] + fn mounted_theme_paths_are_under_project_themes_directory() { + let project_path = PathBuf::from("/tmp/project"); + let mounted_theme = mounted_theme_path(&project_path); + + assert_eq!( + mounted_theme, + PathBuf::from("/tmp/project/themes/eips-theme") + ); + assert_eq!( + theme_config_path(&mounted_theme), + PathBuf::from("/tmp/project/themes/eips-theme/config/zola.toml") + ); + } + + #[test] + fn mount_theme_does_not_symlink_mounted_local_theme_onto_itself() { + let temp = TempDir::new().unwrap(); + let project_path = temp.path().join("repo"); + let mounted_theme = mounted_theme_path(&project_path); + fs::create_dir_all(mounted_theme.join("config")).unwrap(); + fs::write(mounted_theme.join("config/zola.toml"), "title = 'local'\n").unwrap(); + + let result = mount_theme(&mounted_theme, &project_path).unwrap(); + + assert_eq!(result, mounted_theme); + assert!(mounted_theme.join("config/zola.toml").is_file()); + assert!(!fs::symlink_metadata(&mounted_theme) + .unwrap() + .file_type() + .is_symlink()); + } + + #[cfg(target_family = "unix")] + #[test] + fn theme_mount_replaces_prior_real_mounted_theme_directory() { + let temp = TempDir::new().unwrap(); + let project_path = temp.path().join("repo"); + let source_theme = temp.path().join("source-theme"); + fs::create_dir_all(source_theme.join("config")).unwrap(); + fs::write(source_theme.join("config/zola.toml"), "title = 'source'\n").unwrap(); + + let mounted_theme = mounted_theme_path(&project_path); + fs::create_dir_all(&mounted_theme).unwrap(); + fs::write(mounted_theme.join("stale.txt"), "stale").unwrap(); + + let result = mount_theme(&source_theme, &project_path).unwrap(); + + assert_eq!(result, mounted_theme); + assert!(fs::symlink_metadata(&mounted_theme) + .unwrap() + .file_type() + .is_symlink()); + assert_eq!( + fs::read_to_string(theme_config_path(&mounted_theme)).unwrap(), + "title = 'source'\n" + ); + } +}