diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 26f5f0b..0000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Publish crates to crates.io - -on: - workflow_dispatch: - inputs: - ic-certified-assets: - description: "Publish ic-certified-assets" - type: boolean - default: false - -jobs: - publish: - name: Publish selected crates - runs-on: ubuntu-24.04 - if: >- - inputs.ic-certified-assets - - permissions: - contents: read - id-token: write # Required for trusted publishing via OIDC - - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Install Rust toolchain - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 - with: - cache: false - - - name: Authenticate with crates.io - id: auth - uses: rust-lang/crates-io-auth-action@b7e9a28eded4986ec6b1fa40eeee8f8f165559ec # v1 - - - name: Publish ic-certified-assets - if: inputs.ic-certified-assets - run: cargo publish -p ic-certified-assets - env: - CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} diff --git a/AGENT.md b/AGENT.md new file mode 100644 index 0000000..e835acd --- /dev/null +++ b/AGENT.md @@ -0,0 +1,34 @@ +# Agent guide + +Development principles for working in this repo. Documentation for users lives in [`docs/`](docs/). + +## Testing + +Tests are organized around three components. Each runs independently. + +### Canister (`canister-core`) + +- **Location**: [`crates/canister-core/src/tests.rs`](crates/canister-core/src/tests.rs) +- **Run**: `cargo test -p canister-core` + +`canister-core` is the library crate behind `canister`. Its unit tests cover all canister behaviors using a mock system context — no live replica needed: asset CRUD, encoding selection, HTTP semantics, certification, permissions, stable state, and streaming. + +**Add tests here when** you change anything inside `canister-core`: new canister endpoints, modified serving logic, certification changes, permission rules, or upgrade/downgrade behavior. + +### Plugin (`sync-core`) + +- **Location**: inline `#[cfg(test)]` modules in each [`crates/sync-core/src/`](crates/sync-core/src/)`*.rs` file +- **Run**: `cargo test -p sync-core` + +`sync-core` is the library crate behind `sync-plugin`. It has no WASI dependency and compiles natively. Its unit tests cover all sync business logic: directory scanning, MIME detection and encoding, operation diffing, batch sequencing, canister API calls and pagination, and authorization. + +**Add tests here when** you change any sync logic: how files are discovered, how encodings are chosen, how diffs are computed, how batch operations are sequenced, or how permissions are managed. Prefer this over E2E for new logic — tests are fast and require no infrastructure. + +### End-to-end (`e2e`) + +- **Location**: [`crates/e2e/`](crates/e2e/) +- **Run**: `cargo test -p e2e` + +E2E tests verify that the canister and plugin work correctly together through the `icp` CLI against a live local replica. Covers the basic sync workflow: deploy, no-op re-sync, content update, deletion, and multi-directory sync. + +**Add tests here when** you introduce a new top-level workflow or change how the plugin integrates with the CLI or canister in a way that unit tests cannot exercise — for example, a new deploy mode or wire-protocol changes. Keep this suite small; unit tests are preferred for logic coverage. diff --git a/Cargo.lock b/Cargo.lock index 9ba8feb..7fe7883 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -122,24 +122,6 @@ dependencies = [ "wait-timeout", ] -[[package]] -name = "assets-sync" -version = "0.1.0" -dependencies = [ - "brotli", - "candid", - "flate2", - "hex", - "http", - "mime", - "mime_guess", - "serde", - "serde_bytes", - "sha2 0.11.0", - "tempfile", - "url", -] - [[package]] name = "atomic-waker" version = "1.1.2" @@ -409,12 +391,38 @@ version = "0.0.0" dependencies = [ "candid", "candid_parser", + "canister-core", "ic-cdk", - "ic-certified-assets", "serde", "serde_cbor", ] +[[package]] +name = "canister-core" +version = "0.0.0" +dependencies = [ + "anyhow", + "base64 0.22.1", + "candid", + "hex", + "http", + "ic-cdk", + "ic-certification 3.2.0", + "ic-certification-testing", + "ic-crypto-tree-hash", + "ic-http-certification 3.2.0", + "ic-representation-independent-hash 3.2.0", + "ic-response-verification", + "ic-response-verification-test-utils", + "itertools 0.14.0", + "num-traits", + "percent-encoding", + "serde", + "serde_bytes", + "serde_cbor", + "sha2 0.11.0", +] + [[package]] name = "cc" version = "1.2.62" @@ -815,7 +823,7 @@ dependencies = [ [[package]] name = "e2e" -version = "0.1.0" +version = "0.0.0" dependencies = [ "assert_cmd", "candid", @@ -1373,32 +1381,6 @@ dependencies = [ "wasm-bindgen-console-logger", ] -[[package]] -name = "ic-certified-assets" -version = "0.3.0" -dependencies = [ - "anyhow", - "base64 0.22.1", - "candid", - "hex", - "http", - "ic-cdk", - "ic-certification 3.2.0", - "ic-certification-testing", - "ic-crypto-tree-hash", - "ic-http-certification 3.2.0", - "ic-representation-independent-hash 3.2.0", - "ic-response-verification", - "ic-response-verification-test-utils", - "itertools 0.14.0", - "num-traits", - "percent-encoding", - "serde", - "serde_bytes", - "serde_cbor", - "sha2 0.11.0", -] - [[package]] name = "ic-crypto-internal-bls12-381-type" version = "0.9.0" @@ -2429,16 +2411,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "plugin" -version = "0.1.0" -dependencies = [ - "assets-sync", - "candid", - "serde", - "wit-bindgen 0.57.1", -] - [[package]] name = "potential_utf" version = "0.1.5" @@ -3113,6 +3085,34 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync-core" +version = "0.0.0" +dependencies = [ + "brotli", + "candid", + "flate2", + "hex", + "http", + "mime", + "mime_guess", + "serde", + "serde_bytes", + "sha2 0.11.0", + "tempfile", + "url", +] + +[[package]] +name = "sync-plugin" +version = "0.0.0" +dependencies = [ + "candid", + "serde", + "sync-core", + "wit-bindgen 0.57.1", +] + [[package]] name = "sync_wrapper" version = "1.0.2" diff --git a/Cargo.toml b/Cargo.toml index 30a7bc8..e612643 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,14 @@ [workspace] -members = ["assets-sync", "canister", "e2e", "ic-certified-assets", "plugin"] +members = ["crates/*"] resolver = "2" [workspace.package] +version = "0.0.0" authors = ["DFINITY Stiftung "] edition = "2021" repository = "https://github.com/dfinity/certified-assets" license = "Apache-2.0" +publish = false [workspace.dependencies] anyhow = "1.0.56" diff --git a/README.md b/README.md index a129f2e..5ed6f01 100644 --- a/README.md +++ b/README.md @@ -1,77 +1,3 @@ # Certified Assets -Two Wasm modules, each backed by a library crate: - -``` - ┌────────────────────┐ ┌────────────────────┐ - │ canister/ │ │ plugin/ │ - │ deployed to ICP │ │ loaded by icp-cli │ - └─────────┬──────────┘ └─────────┬──────────┘ - │ wraps │ wraps - ┌─────────▼──────────┐ ┌─────────▼──────────┐ - │ ic-certified- │ │ assets-sync/ │ - │ assets/ │ │ │ - └────────────────────┘ └────────────────────┘ -``` - -## [`canister/`](canister/) - -The ICP assets canister — a deployable WebAssembly canister that serves certified static assets over HTTP. It wraps `ic-certified-assets` and exposes the canister interface. - -### [`ic-certified-assets/`](ic-certified-assets/) - -The core business logic library. Handles asset storage, certification (response verification), streaming, and access control. `canister` depends on this crate; it can also be embedded in other canisters. - -## [`plugin/`](plugin/) - -A thin `icp-cli` sync plugin. Delegates all sync logic to `assets-sync`. - -### [`assets-sync/`](assets-sync/) - -Platform-agnostic library implementing the asset sync logic: directory scanning, MIME detection, content encoding, canister diffing, and the `CanisterCall` trait that abstracts the transport layer. - -## Redirects, rewrites, and custom error pages - -The canister honours a Netlify-style `_redirects` file at the root of the project's input directory. (The sync plugin only accepts one source directory — see [`plugin/README.md`](plugin/README.md#scope) for why.) Each line is ` `, where `` is one of `{200, 301, 302, 307, 308, 404, 410}`: - -```text -/old-page /new-page 301 # 3xx redirect (Location header) -/external https://example.com/ 302 -/about /about.html 200 # 200 rewrite (serve target's body) -/blog/* /blog/index.html 200 # subtree rewrite -/missing /404.html 404 # custom error page -``` - -The file is consumed by the plugin and lowered to certified canister-side rules — a real asset at the rule's `from` path always wins. Ahead of the user's `_redirects`, the plugin auto-synthesises Cloudflare's [`auto-trailing-slash`](https://developers.cloudflare.com/workers/static-assets/routing/advanced/html-handling/#automatic-trailing-slashes-default) rule set for every `.html` asset, so `/foo.html` is reachable as `/foo`, `/bar/index.html` is reachable as `/bar/`, and the non-canonical forms (`/foo/`, `/foo/index`, etc.) 307 to the canonical URL. User-declared rules apply to paths the html-handling defaults don't claim — e.g. a SPA-style `/* /404.html 404` catch-all only fires for paths with no matching HTML asset. See [`plugin/README.md`](plugin/README.md#redirects) for the full reference and migration notes. - -## Per-path response headers - -The canister also honours a Netlify-style `_headers` file at the root of the project's input directory. Each block is a non-indented `` line followed by one or more indented `Header-Name: value` lines: - -```text -/_astro/* - Cache-Control: public, max-age=31536000, immutable - -/* - X-Frame-Options: DENY - X-Robots-Tag: noindex -``` - -`` is an absolute path with optional `*` wildcards — `/about` is exact, `/_astro/*` is a subtree, `/*.md` matches any `.md` file at any depth. A single `*` matches any sequence including `/` and empty; `**` is not supported (redundant) and neither is `:placeholder`. All matching rules apply per the Cloudflare Pages / Netlify semantics — same-name values across rules concatenate with `, ` (RFC 7230), with `Set-Cookie` carved out (RFC 6265). `Content-Type` is recognised but routed to the asset's stored media type instead of the appended response headers — see below. See [`plugin/README.md`](plugin/README.md#headers) for the full reference and reject list. - -### `Content-Type` overrides - -The canister derives a `Content-Type` for every asset from its media type and certifies it as part of the response. To override what `mime_guess::from_path` picks (or to add a `charset` parameter), set `Content-Type:` inside any `_headers` block: - -```text -/*.md - Content-Type: text/markdown; charset=utf-8 - -/*.did - Content-Type: text/plain; charset=utf-8 - -/llms.txt - Content-Type: text/plain; charset=utf-8 -``` - -The plugin extracts `Content-Type` and feeds it into `CreateAssetArguments.content_type` rather than appending it as a response header — so the canister emits exactly one `Content-Type` per response, no duplicates. Other headers in the same block continue to flow through `headers` as usual. `Content-Type` is single-valued, so when multiple blocks match the same asset the first matching `Content-Type` wins (other matching rules still contribute their non-`Content-Type` headers as normal). +An ICP assets canister and `icp-cli` sync plugin for serving certified static assets. diff --git a/TEST.md b/TEST.md deleted file mode 100644 index 7211a06..0000000 --- a/TEST.md +++ /dev/null @@ -1,30 +0,0 @@ -# Testing Guide - -Tests are organized around three components. Each runs independently. - -## Canister (`ic-certified-assets`) - -**Location**: `ic-certified-assets/src/tests.rs` -**Run**: `cargo test -p ic-certified-assets` - -`ic-certified-assets` is the library crate behind `canister/`. Its unit tests cover all canister behaviors using a mock system context — no live replica needed: asset CRUD, encoding selection, HTTP semantics, certification, permissions, stable state, and streaming. - -**Add tests here when** you change anything inside `ic-certified-assets`: new canister endpoints, modified serving logic, certification changes, permission rules, or upgrade/downgrade behavior. - -## Plugin (`assets-sync`) - -**Location**: Inline `#[cfg(test)]` modules in each `assets-sync/src/*.rs` file -**Run**: `cargo test -p assets-sync` - -`assets-sync` is the library crate behind `plugin/`. It has no WASI dependency and compiles natively. Its unit tests cover all sync business logic: directory scanning, MIME detection and encoding, operation diffing, batch sequencing, canister API calls and pagination, and authorization. - -**Add tests here when** you change any sync logic: how files are discovered, how encodings are chosen, how diffs are computed, how batch operations are sequenced, or how permissions are managed. Prefer this over E2E for new logic — tests are fast and require no infrastructure. - -## End-to-End (`e2e`) - -**Location**: `e2e/` -**Run**: `cargo test -p e2e` - -E2E tests verify that the canister and plugin work correctly together through the `icp` CLI against a live local replica. Covers the basic sync workflow: deploy, no-op re-sync, content update, deletion, and multi-directory sync. - -**Add tests here when** you introduce a new top-level workflow or change how the plugin integrates with the CLI or canister in a way that unit tests cannot exercise — for example, a new deploy mode or wire-protocol changes. Keep this suite small; unit tests are preferred for logic coverage. diff --git a/assets-sync/tests/OPTIMIZATIONS.md b/assets-sync/tests/OPTIMIZATIONS.md deleted file mode 100644 index 5fb8b28..0000000 --- a/assets-sync/tests/OPTIMIZATIONS.md +++ /dev/null @@ -1,119 +0,0 @@ -# Deferred sync-performance optimizations - -Working notes for performance work identified by [`bench_sync.rs`](bench_sync.rs) -but not yet shipped. Each entry says what to do, the measured or expected -impact, and what's blocking us from doing it now. - -Run the bench to get current baseline numbers: - -```sh -cargo test --release --test bench_sync -- --ignored --nocapture --test-threads=1 -``` - -## 1. Inline trailing chunk into `commit_batch` - -`SetAssetContentArguments` has a `last_chunk: Option>` field -([canister.rs](../src/canister.rs)) that the canister accepts as content -shipped *inside* `commit_batch` rather than via a separate `create_chunks` -call. Today we always set it to `None` -([sync.rs](../src/sync.rs), `build_operations` — `last_chunk: None`). - -**What to do.** For each encoding, if the final chunk is sub-`MAX_CHUNK_SIZE`, -move it out of the upload-pending list and into the `SetAssetContent` op's -`last_chunk`. Budget the total bytes carried this way against the ingress -message limit — `dfx`'s `ic-asset` uses `MAX_CHUNK_SIZE / 2` -(`~950 KB`) as the ceiling. - -**Expected impact** (from the bench, post chunk-batching baseline): - -| fixture | `create_chunks` now | with `last_chunk` | -|---|---:|---:| -| many_tiny (1000 × 1 KB) | 1 | **0** | -| many_small (100 × 5 KB) | 1 | **0** | -| few_medium (10 × 2 MB) | 12 | **10** (drops 2 trailing-chunk calls) | -| one_huge (1 × 50 MB) | 28 | **27** (drops the trailing-chunk call) | - -For small/medium projects the entire upload disappears into `commit_batch` and -there is no `create_chunks` round-trip at all. - -**Why deferred.** Orthogonal to chunk batching; deserves its own PR so we can -measure the delta cleanly. Some care needed around the inline-budget -accounting: `last_chunk` bytes count toward the `commit_batch` arg size, -which has the same ~2 MB ingress limit, so the budget interacts with the -"split `commit_batch`" work below. - -## 2. Split `commit_batch` into bounded sub-batches - -Today we commit every operation in one `commit_batch` call -([sync.rs](../src/sync.rs), Phase 3). For projects with thousands of operations -(or with `last_chunk` inlining adding bytes per op), the request may exceed -the ~2 MB ingress message limit and the call will fail outright. - -**What to do.** Mirror the SDK's approach -([dfx sync.rs:253-290](../../../sdk/src/canisters/frontend/ic-asset/src/sync.rs#L253-L290)): -split the operations list into batches of at most ~500 ops or ~1.5 MB of -header bytes, commit each batch, send a final empty-ops `commit_batch` to -finalize. - -**Expected impact.** Correctness/scaling, not raw speed. At today's typical -project sizes it's a no-op; at 10k+ assets or with `last_chunk` enabled it -prevents outright failure. - -**Why deferred.** No project we know of currently hits the limit. Should land -together with `last_chunk` inlining since the two share the ingress-size -budget. - -## 3. Eliminate / batch per-asset `get_asset_properties` - -[sync.rs](../src/sync.rs) (`sync`, Phase 1) issues one query call per surviving -asset to read its current `(max_age, headers, allow_raw_access)`. N round-trips -for N assets, even though most projects have property drift on zero or one -asset. - -**What to do.** Two options: - -- **Bulk endpoint.** Add `get_asset_properties_bulk(keys) -> Vec` - to the canister and use it. One round-trip total. -- **Always emit `SetAssetProperties`.** Skip the query entirely and emit a - `SetAssetProperties` op for every surviving asset with project-desired - values. Idempotent; trades N query calls for at most N ops in a batch we - were already going to send. - -**Expected impact.** Eliminates one round-trip per asset. - -**Why deferred.** Query calls are much cheaper than update calls on the IC -(no consensus), so for typical projects this is unlikely to dominate -wall-clock. Worth profiling a real deployment first to confirm before adding -either of the above. - -## 4. Host-side concurrent calls - -The plugin compiles to a `wasm32-wasip2` component. WASI Preview 2 is -single-threaded — `std::thread`, `rayon`, and async runtimes are all -unavailable inside the component. So even though the SDK's async fan-out -gives `dfx deploy` 50× concurrency on chunk uploads, that lever isn't -reachable from inside the current plugin model. - -**What to do.** Extend the WIT interface -([`plugin/wit/sync-plugin.wit`](../../plugin/wit/sync-plugin.wit)) with a -batch-call import — something like -`canister-call-batch(requests: list) -> list` — and -implement it on the host (native Rust, tokio-based) so multiple update calls -fire concurrently. The plugin would still drive everything synchronously, -but each batch import would expand into N concurrent calls under the hood. - -**Expected impact.** Where this helps: - -- `one_huge` (1 × 50 MB): 28 MAX-sized `create_chunks` calls. Chunk packing - can't help (chunks already fill the call). Concurrency could fan these out. - -Where this doesn't help (because we already collapsed to 1 call): - -- `many_tiny`, `many_small`: nothing left to parallelize. - -**Why deferred.** This is a plugin-host interface change, not an -`assets-sync` change. The chunk-batching work in this PR already removed the -majority of the call-count gap — large-file uploads remain the only regime -where concurrency would still pay off, and even there only by a constant -factor. Do this only after measuring against a real replica and confirming -large-file uploads are the bottleneck. diff --git a/canister/README.md b/canister/README.md deleted file mode 100644 index c454b0f..0000000 --- a/canister/README.md +++ /dev/null @@ -1 +0,0 @@ -# The Assets Canister diff --git a/ic-certified-assets/Cargo.toml b/crates/canister-core/Cargo.toml similarity index 78% rename from ic-certified-assets/Cargo.toml rename to crates/canister-core/Cargo.toml index bd6907a..87798b7 100644 --- a/ic-certified-assets/Cargo.toml +++ b/crates/canister-core/Cargo.toml @@ -1,15 +1,11 @@ [package] -name = "ic-certified-assets" -version = "0.3.0" +name = "canister-core" +version.workspace = true authors.workspace = true edition.workspace = true repository.workspace = true license.workspace = true -description = "Rust support for asset certification." -documentation = "https://docs.rs/ic-certified-assets" -categories = ["wasm", "filesystem", "data-structures"] -keywords = ["internet-computer", "dfinity"] -rust-version = "1.84.0" +publish.workspace = true [dependencies] base64.workspace = true diff --git a/ic-certified-assets/src/asset.rs b/crates/canister-core/src/asset.rs similarity index 100% rename from ic-certified-assets/src/asset.rs rename to crates/canister-core/src/asset.rs diff --git a/ic-certified-assets/src/batch.rs b/crates/canister-core/src/batch.rs similarity index 100% rename from ic-certified-assets/src/batch.rs rename to crates/canister-core/src/batch.rs diff --git a/ic-certified-assets/src/certification.rs b/crates/canister-core/src/certification.rs similarity index 100% rename from ic-certified-assets/src/certification.rs rename to crates/canister-core/src/certification.rs diff --git a/ic-certified-assets/src/http.rs b/crates/canister-core/src/http.rs similarity index 100% rename from ic-certified-assets/src/http.rs rename to crates/canister-core/src/http.rs diff --git a/ic-certified-assets/src/lib.rs b/crates/canister-core/src/lib.rs similarity index 100% rename from ic-certified-assets/src/lib.rs rename to crates/canister-core/src/lib.rs diff --git a/ic-certified-assets/src/nested_tree.rs b/crates/canister-core/src/nested_tree.rs similarity index 100% rename from ic-certified-assets/src/nested_tree.rs rename to crates/canister-core/src/nested_tree.rs diff --git a/ic-certified-assets/src/rc_bytes.rs b/crates/canister-core/src/rc_bytes.rs similarity index 100% rename from ic-certified-assets/src/rc_bytes.rs rename to crates/canister-core/src/rc_bytes.rs diff --git a/ic-certified-assets/src/redirect.rs b/crates/canister-core/src/redirect.rs similarity index 100% rename from ic-certified-assets/src/redirect.rs rename to crates/canister-core/src/redirect.rs diff --git a/ic-certified-assets/src/stable.rs b/crates/canister-core/src/stable.rs similarity index 100% rename from ic-certified-assets/src/stable.rs rename to crates/canister-core/src/stable.rs diff --git a/ic-certified-assets/src/state.rs b/crates/canister-core/src/state.rs similarity index 100% rename from ic-certified-assets/src/state.rs rename to crates/canister-core/src/state.rs diff --git a/ic-certified-assets/src/state_hash.rs b/crates/canister-core/src/state_hash.rs similarity index 100% rename from ic-certified-assets/src/state_hash.rs rename to crates/canister-core/src/state_hash.rs diff --git a/ic-certified-assets/src/system_context.rs b/crates/canister-core/src/system_context.rs similarity index 100% rename from ic-certified-assets/src/system_context.rs rename to crates/canister-core/src/system_context.rs diff --git a/ic-certified-assets/src/tests.rs b/crates/canister-core/src/tests.rs similarity index 100% rename from ic-certified-assets/src/tests.rs rename to crates/canister-core/src/tests.rs diff --git a/ic-certified-assets/src/types.rs b/crates/canister-core/src/types.rs similarity index 100% rename from ic-certified-assets/src/types.rs rename to crates/canister-core/src/types.rs diff --git a/ic-certified-assets/src/url.rs b/crates/canister-core/src/url.rs similarity index 100% rename from ic-certified-assets/src/url.rs rename to crates/canister-core/src/url.rs diff --git a/canister/Cargo.toml b/crates/canister/Cargo.toml similarity index 64% rename from canister/Cargo.toml rename to crates/canister/Cargo.toml index e763493..366dfc3 100644 --- a/canister/Cargo.toml +++ b/crates/canister/Cargo.toml @@ -1,21 +1,18 @@ [package] name = "canister" -version = "0.0.0" +version.workspace = true authors.workspace = true edition.workspace = true repository.workspace = true license.workspace = true -description = "Assets Canister for IC." -categories = ["wasm"] -keywords = ["internet-computer", "dfinity"] -publish = false +publish.workspace = true [lib] path = "src/lib.rs" crate-type = ["cdylib", "rlib"] [dependencies] -ic-certified-assets = { path = "../ic-certified-assets" } +canister-core = { path = "../canister-core" } ic-cdk.workspace = true candid.workspace = true serde.workspace = true diff --git a/canister/assets.did b/crates/canister/assets.did similarity index 100% rename from canister/assets.did rename to crates/canister/assets.did diff --git a/canister/src/lib.rs b/crates/canister/src/lib.rs similarity index 72% rename from canister/src/lib.rs rename to crates/canister/src/lib.rs index 954a302..bff7af7 100644 --- a/canister/src/lib.rs +++ b/crates/canister/src/lib.rs @@ -1,8 +1,7 @@ mod state; use candid::Principal; -use ic_cdk::{init, post_upgrade, pre_upgrade, query, update}; -use ic_certified_assets::{ +use canister_core::{ asset::{AssetDetails, EncodedAsset}, can_commit, can_prepare, certification::AssetKey, @@ -20,195 +19,196 @@ use ic_certified_assets::{ StateInfo, StoreArg, UnsetAssetContentArguments, }, }; +use ic_cdk::{init, post_upgrade, pre_upgrade, query, update}; use crate::state::{load_stable_state, save_stable_state}; #[init] fn init(args: Option) { - ic_certified_assets::init(args); + canister_core::init(args); } #[pre_upgrade] fn pre_upgrade() { - let stable_state = ic_certified_assets::pre_upgrade(); + let stable_state = canister_core::pre_upgrade(); save_stable_state(&stable_state).expect("failed to serialize stable state"); } #[post_upgrade] fn post_upgrade(args: Option) { let stable_state = load_stable_state().expect("failed to deserialize stable state"); - ic_certified_assets::post_upgrade(stable_state, args); + canister_core::post_upgrade(stable_state, args); } #[cfg(target_family = "wasm")] #[used] #[unsafe(link_section = "icp:public supported_certificate_versions")] -static CERTIFICATE_VERSIONS: [u8; 3] = ic_certified_assets::SUPPORTED_CERTIFICATE_VERSIONS; +static CERTIFICATE_VERSIONS: [u8; 3] = canister_core::SUPPORTED_CERTIFICATE_VERSIONS; // Query methods #[query] fn api_version() -> u16 { - ic_certified_assets::api_version() + canister_core::api_version() } #[query] fn retrieve(key: AssetKey) -> RcBytes { - ic_certified_assets::retrieve(key) + canister_core::retrieve(key) } #[query] fn get(arg: GetArg) -> EncodedAsset { - ic_certified_assets::get(arg) + canister_core::get(arg) } #[query] fn get_chunk(arg: GetChunkArg) -> GetChunkResponse { - ic_certified_assets::get_chunk(arg) + canister_core::get_chunk(arg) } #[query] fn list(request: ListRequest) -> Vec { - ic_certified_assets::list(request) + canister_core::list(request) } #[query] fn certified_tree() -> CertifiedTree { - ic_certified_assets::certified_tree() + canister_core::certified_tree() } #[query] fn http_request(req: HttpRequest) -> HttpResponse { - ic_certified_assets::http_request(req) + canister_core::http_request(req) } #[query] fn http_request_streaming_callback( token: StreamingCallbackToken, ) -> Option { - ic_certified_assets::http_request_streaming_callback(token) + canister_core::http_request_streaming_callback(token) } #[query] fn get_asset_properties(key: AssetKey) -> AssetProperties { - ic_certified_assets::get_asset_properties(key) + canister_core::get_asset_properties(key) } #[query] fn get_state_info() -> StateInfo { - ic_certified_assets::get_state_info() + canister_core::get_state_info() } #[query] fn get_redirect_rules() -> Vec { - ic_certified_assets::get_redirect_rules() + canister_core::get_redirect_rules() } // Update methods #[update(guard = "is_manager_or_controller")] fn authorize(other: Principal) { - ic_certified_assets::authorize(other) + canister_core::authorize(other) } #[update(guard = "is_manager_or_controller")] fn grant_permission(arg: GrantPermissionArguments) { - ic_certified_assets::grant_permission(arg) + canister_core::grant_permission(arg) } #[update] async fn deauthorize(other: Principal) { - ic_certified_assets::deauthorize(other).await + canister_core::deauthorize(other).await } #[update] async fn revoke_permission(arg: RevokePermissionArguments) { - ic_certified_assets::revoke_permission(arg).await + canister_core::revoke_permission(arg).await } #[update] fn list_authorized() -> Vec { - ic_certified_assets::list_authorized() + canister_core::list_authorized() } #[update] fn list_permitted(arg: ListPermittedArguments) -> Vec { - ic_certified_assets::list_permitted(arg) + canister_core::list_permitted(arg) } #[update(guard = "is_controller")] async fn take_ownership() { - ic_certified_assets::take_ownership().await + canister_core::take_ownership().await } #[update(guard = "can_commit")] fn store(arg: StoreArg) { - ic_certified_assets::store(arg) + canister_core::store(arg) } #[update(guard = "can_prepare")] fn create_batch() -> CreateBatchResponse { - ic_certified_assets::create_batch() + canister_core::create_batch() } #[update(guard = "can_prepare")] fn create_chunks(arg: CreateChunksArg) -> CreateChunksResponse { - ic_certified_assets::create_chunks(arg) + canister_core::create_chunks(arg) } #[update(guard = "can_commit")] fn create_asset(arg: CreateAssetArguments) { - ic_certified_assets::create_asset(arg) + canister_core::create_asset(arg) } #[update(guard = "can_commit")] fn set_asset_content(arg: SetAssetContentArguments) { - ic_certified_assets::set_asset_content(arg) + canister_core::set_asset_content(arg) } #[update(guard = "can_commit")] fn unset_asset_content(arg: UnsetAssetContentArguments) { - ic_certified_assets::unset_asset_content(arg) + canister_core::unset_asset_content(arg) } #[update(guard = "can_commit")] fn delete_asset(arg: DeleteAssetArguments) { - ic_certified_assets::delete_asset(arg) + canister_core::delete_asset(arg) } #[update(guard = "can_commit")] fn clear() { - ic_certified_assets::clear() + canister_core::clear() } #[update(guard = "can_commit")] async fn commit_batch(arg: CommitBatchArguments) { - ic_certified_assets::commit_batch(arg).await + canister_core::commit_batch(arg).await } #[update] async fn compute_state_hash() -> Option { - ic_certified_assets::compute_state_hash().await + canister_core::compute_state_hash().await } #[update(guard = "can_prepare")] fn delete_batch(arg: DeleteBatchArguments) { - ic_certified_assets::delete_batch(arg) + canister_core::delete_batch(arg) } #[update(guard = "can_commit")] fn set_asset_properties(arg: SetAssetPropertiesArguments) { - ic_certified_assets::set_asset_properties(arg) + canister_core::set_asset_properties(arg) } #[update(guard = "can_prepare")] fn get_configuration() -> ConfigurationResponse { - ic_certified_assets::get_configuration() + canister_core::get_configuration() } #[update(guard = "can_commit")] fn configure(arg: ConfigureArguments) { - ic_certified_assets::configure(arg) + canister_core::configure(arg) } ic_cdk::export_candid!(); diff --git a/canister/src/state.rs b/crates/canister/src/state.rs similarity index 95% rename from canister/src/state.rs rename to crates/canister/src/state.rs index 5ff78a9..3ae1db7 100644 --- a/canister/src/state.rs +++ b/crates/canister/src/state.rs @@ -1,5 +1,5 @@ +use canister_core::StableState; use ic_cdk::stable; -use ic_certified_assets::StableState; pub fn save_stable_state(stable_state: &StableState) -> Result<(), serde_cbor::Error> { let mut stable_writer = stable::StableWriter::default(); diff --git a/e2e/Cargo.toml b/crates/e2e/Cargo.toml similarity index 75% rename from e2e/Cargo.toml rename to crates/e2e/Cargo.toml index 8edf4e1..c2de19f 100644 --- a/e2e/Cargo.toml +++ b/crates/e2e/Cargo.toml @@ -1,9 +1,11 @@ [package] name = "e2e" -version = "0.1.0" +version.workspace = true +authors.workspace = true edition.workspace = true +repository.workspace = true license.workspace = true -publish = false +publish.workspace = true [dependencies] assert_cmd.workspace = true diff --git a/e2e/build.rs b/crates/e2e/build.rs similarity index 69% rename from e2e/build.rs rename to crates/e2e/build.rs index c1805e7..7f5ad4c 100644 --- a/e2e/build.rs +++ b/crates/e2e/build.rs @@ -2,15 +2,19 @@ use std::{env, path::Path, path::PathBuf, process::Command}; fn main() { println!("cargo:rerun-if-changed=../canister/src"); - println!("cargo:rerun-if-changed=../ic-certified-assets/src"); - println!("cargo:rerun-if-changed=../plugin/src"); - println!("cargo:rerun-if-changed=../assets-sync/src"); + println!("cargo:rerun-if-changed=../canister-core/src"); + println!("cargo:rerun-if-changed=../sync-plugin/src"); + println!("cargo:rerun-if-changed=../sync-core/src"); let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); - let workspace_root = manifest_dir.parent().expect("e2e/ must have a parent"); + // crates/e2e -> crates -> workspace root + let workspace_root = manifest_dir + .parent() + .and_then(Path::parent) + .expect("crates/e2e/ must have a workspace root two levels up"); build_wasm(workspace_root, "canister", "wasm32-unknown-unknown"); - build_wasm(workspace_root, "plugin", "wasm32-wasip2"); + build_wasm(workspace_root, "sync-plugin", "wasm32-wasip2"); println!( "cargo:rustc-env=CANISTER_WASM={}", @@ -21,7 +25,7 @@ fn main() { println!( "cargo:rustc-env=PLUGIN_WASM={}", workspace_root - .join("target/wasm32-wasip2/release/plugin.wasm") + .join("target/wasm32-wasip2/release/sync_plugin.wasm") .display() ); } diff --git a/e2e/src/lib.rs b/crates/e2e/src/lib.rs similarity index 100% rename from e2e/src/lib.rs rename to crates/e2e/src/lib.rs diff --git a/e2e/tests/fixture/basic/dist/index.html b/crates/e2e/tests/fixture/basic/dist/index.html similarity index 100% rename from e2e/tests/fixture/basic/dist/index.html rename to crates/e2e/tests/fixture/basic/dist/index.html diff --git a/e2e/tests/fixture/basic/dist/style.css b/crates/e2e/tests/fixture/basic/dist/style.css similarity index 100% rename from e2e/tests/fixture/basic/dist/style.css rename to crates/e2e/tests/fixture/basic/dist/style.css diff --git a/e2e/tests/fixture/basic/icp.yaml b/crates/e2e/tests/fixture/basic/icp.yaml similarity index 100% rename from e2e/tests/fixture/basic/icp.yaml rename to crates/e2e/tests/fixture/basic/icp.yaml diff --git a/e2e/tests/fixture/headers-content-type/dist/_headers b/crates/e2e/tests/fixture/headers-content-type/dist/_headers similarity index 100% rename from e2e/tests/fixture/headers-content-type/dist/_headers rename to crates/e2e/tests/fixture/headers-content-type/dist/_headers diff --git a/e2e/tests/fixture/headers-content-type/dist/ic.did b/crates/e2e/tests/fixture/headers-content-type/dist/ic.did similarity index 100% rename from e2e/tests/fixture/headers-content-type/dist/ic.did rename to crates/e2e/tests/fixture/headers-content-type/dist/ic.did diff --git a/e2e/tests/fixture/headers-content-type/dist/index.html b/crates/e2e/tests/fixture/headers-content-type/dist/index.html similarity index 100% rename from e2e/tests/fixture/headers-content-type/dist/index.html rename to crates/e2e/tests/fixture/headers-content-type/dist/index.html diff --git a/e2e/tests/fixture/headers-content-type/icp.yaml b/crates/e2e/tests/fixture/headers-content-type/icp.yaml similarity index 100% rename from e2e/tests/fixture/headers-content-type/icp.yaml rename to crates/e2e/tests/fixture/headers-content-type/icp.yaml diff --git a/e2e/tests/fixture/headers/dist/_astro/app.js b/crates/e2e/tests/fixture/headers/dist/_astro/app.js similarity index 100% rename from e2e/tests/fixture/headers/dist/_astro/app.js rename to crates/e2e/tests/fixture/headers/dist/_astro/app.js diff --git a/e2e/tests/fixture/headers/dist/_headers b/crates/e2e/tests/fixture/headers/dist/_headers similarity index 100% rename from e2e/tests/fixture/headers/dist/_headers rename to crates/e2e/tests/fixture/headers/dist/_headers diff --git a/e2e/tests/fixture/headers/dist/index.html b/crates/e2e/tests/fixture/headers/dist/index.html similarity index 100% rename from e2e/tests/fixture/headers/dist/index.html rename to crates/e2e/tests/fixture/headers/dist/index.html diff --git a/e2e/tests/fixture/headers/icp.yaml b/crates/e2e/tests/fixture/headers/icp.yaml similarity index 100% rename from e2e/tests/fixture/headers/icp.yaml rename to crates/e2e/tests/fixture/headers/icp.yaml diff --git a/e2e/tests/fixture/html-handling-with-catchall/dist/404.html b/crates/e2e/tests/fixture/html-handling-with-catchall/dist/404.html similarity index 100% rename from e2e/tests/fixture/html-handling-with-catchall/dist/404.html rename to crates/e2e/tests/fixture/html-handling-with-catchall/dist/404.html diff --git a/e2e/tests/fixture/html-handling-with-catchall/dist/_redirects b/crates/e2e/tests/fixture/html-handling-with-catchall/dist/_redirects similarity index 100% rename from e2e/tests/fixture/html-handling-with-catchall/dist/_redirects rename to crates/e2e/tests/fixture/html-handling-with-catchall/dist/_redirects diff --git a/e2e/tests/fixture/html-handling-with-catchall/dist/blog/index.html b/crates/e2e/tests/fixture/html-handling-with-catchall/dist/blog/index.html similarity index 100% rename from e2e/tests/fixture/html-handling-with-catchall/dist/blog/index.html rename to crates/e2e/tests/fixture/html-handling-with-catchall/dist/blog/index.html diff --git a/e2e/tests/fixture/html-handling-with-catchall/dist/foo.html b/crates/e2e/tests/fixture/html-handling-with-catchall/dist/foo.html similarity index 100% rename from e2e/tests/fixture/html-handling-with-catchall/dist/foo.html rename to crates/e2e/tests/fixture/html-handling-with-catchall/dist/foo.html diff --git a/e2e/tests/fixture/html-handling-with-catchall/dist/index.html b/crates/e2e/tests/fixture/html-handling-with-catchall/dist/index.html similarity index 100% rename from e2e/tests/fixture/html-handling-with-catchall/dist/index.html rename to crates/e2e/tests/fixture/html-handling-with-catchall/dist/index.html diff --git a/e2e/tests/fixture/html-handling-with-catchall/icp.yaml b/crates/e2e/tests/fixture/html-handling-with-catchall/icp.yaml similarity index 100% rename from e2e/tests/fixture/html-handling-with-catchall/icp.yaml rename to crates/e2e/tests/fixture/html-handling-with-catchall/icp.yaml diff --git a/e2e/tests/fixture/html-handling/dist/blog/index.html b/crates/e2e/tests/fixture/html-handling/dist/blog/index.html similarity index 100% rename from e2e/tests/fixture/html-handling/dist/blog/index.html rename to crates/e2e/tests/fixture/html-handling/dist/blog/index.html diff --git a/e2e/tests/fixture/html-handling/dist/foo.html b/crates/e2e/tests/fixture/html-handling/dist/foo.html similarity index 100% rename from e2e/tests/fixture/html-handling/dist/foo.html rename to crates/e2e/tests/fixture/html-handling/dist/foo.html diff --git a/e2e/tests/fixture/html-handling/dist/index.html b/crates/e2e/tests/fixture/html-handling/dist/index.html similarity index 100% rename from e2e/tests/fixture/html-handling/dist/index.html rename to crates/e2e/tests/fixture/html-handling/dist/index.html diff --git a/e2e/tests/fixture/html-handling/icp.yaml b/crates/e2e/tests/fixture/html-handling/icp.yaml similarity index 100% rename from e2e/tests/fixture/html-handling/icp.yaml rename to crates/e2e/tests/fixture/html-handling/icp.yaml diff --git a/e2e/tests/fixture/multi-dir/dist-a/page.html b/crates/e2e/tests/fixture/multi-dir/dist-a/page.html similarity index 100% rename from e2e/tests/fixture/multi-dir/dist-a/page.html rename to crates/e2e/tests/fixture/multi-dir/dist-a/page.html diff --git a/e2e/tests/fixture/multi-dir/dist-b/app.js b/crates/e2e/tests/fixture/multi-dir/dist-b/app.js similarity index 100% rename from e2e/tests/fixture/multi-dir/dist-b/app.js rename to crates/e2e/tests/fixture/multi-dir/dist-b/app.js diff --git a/e2e/tests/fixture/multi-dir/icp.yaml b/crates/e2e/tests/fixture/multi-dir/icp.yaml similarity index 100% rename from e2e/tests/fixture/multi-dir/icp.yaml rename to crates/e2e/tests/fixture/multi-dir/icp.yaml diff --git a/e2e/tests/fixture/redirects/dist/404.html b/crates/e2e/tests/fixture/redirects/dist/404.html similarity index 100% rename from e2e/tests/fixture/redirects/dist/404.html rename to crates/e2e/tests/fixture/redirects/dist/404.html diff --git a/e2e/tests/fixture/redirects/dist/410.html b/crates/e2e/tests/fixture/redirects/dist/410.html similarity index 100% rename from e2e/tests/fixture/redirects/dist/410.html rename to crates/e2e/tests/fixture/redirects/dist/410.html diff --git a/e2e/tests/fixture/redirects/dist/_redirects b/crates/e2e/tests/fixture/redirects/dist/_redirects similarity index 100% rename from e2e/tests/fixture/redirects/dist/_redirects rename to crates/e2e/tests/fixture/redirects/dist/_redirects diff --git a/e2e/tests/fixture/redirects/dist/about.html b/crates/e2e/tests/fixture/redirects/dist/about.html similarity index 100% rename from e2e/tests/fixture/redirects/dist/about.html rename to crates/e2e/tests/fixture/redirects/dist/about.html diff --git a/e2e/tests/fixture/redirects/dist/blog/index.html b/crates/e2e/tests/fixture/redirects/dist/blog/index.html similarity index 100% rename from e2e/tests/fixture/redirects/dist/blog/index.html rename to crates/e2e/tests/fixture/redirects/dist/blog/index.html diff --git a/e2e/tests/fixture/redirects/dist/new.html b/crates/e2e/tests/fixture/redirects/dist/new.html similarity index 100% rename from e2e/tests/fixture/redirects/dist/new.html rename to crates/e2e/tests/fixture/redirects/dist/new.html diff --git a/e2e/tests/fixture/redirects/icp.yaml b/crates/e2e/tests/fixture/redirects/icp.yaml similarity index 100% rename from e2e/tests/fixture/redirects/icp.yaml rename to crates/e2e/tests/fixture/redirects/icp.yaml diff --git a/e2e/tests/headers.rs b/crates/e2e/tests/headers.rs similarity index 100% rename from e2e/tests/headers.rs rename to crates/e2e/tests/headers.rs diff --git a/e2e/tests/headers_content_type.rs b/crates/e2e/tests/headers_content_type.rs similarity index 100% rename from e2e/tests/headers_content_type.rs rename to crates/e2e/tests/headers_content_type.rs diff --git a/e2e/tests/redirects.rs b/crates/e2e/tests/redirects.rs similarity index 99% rename from e2e/tests/redirects.rs rename to crates/e2e/tests/redirects.rs index 92ac6be..bd23b3e 100644 --- a/e2e/tests/redirects.rs +++ b/crates/e2e/tests/redirects.rs @@ -81,7 +81,7 @@ fn redirect_rules_honoured() { /// Without a user-supplied `_redirects`, the plugin auto-synthesises /// Cloudflare's `auto-trailing-slash` rule set for every HTML asset (see -/// `assets-sync::html_handling`). This test deploys an HTML-only fixture +/// `sync-core::html_handling`). This test deploys an HTML-only fixture /// and walks the full CF table for each of the three asset shapes: /// root index, directory index, and non-index HTML file. /// diff --git a/e2e/tests/sync.rs b/crates/e2e/tests/sync.rs similarity index 100% rename from e2e/tests/sync.rs rename to crates/e2e/tests/sync.rs diff --git a/assets-sync/Cargo.toml b/crates/sync-core/Cargo.toml similarity index 64% rename from assets-sync/Cargo.toml rename to crates/sync-core/Cargo.toml index 3b4f06e..865c6fc 100644 --- a/assets-sync/Cargo.toml +++ b/crates/sync-core/Cargo.toml @@ -1,8 +1,11 @@ [package] -name = "assets-sync" -version = "0.1.0" -edition = "2021" -publish = false +name = "sync-core" +version.workspace = true +authors.workspace = true +edition.workspace = true +repository.workspace = true +license.workspace = true +publish.workspace = true [dependencies] brotli.workspace = true diff --git a/assets-sync/src/canister.rs b/crates/sync-core/src/canister.rs similarity index 100% rename from assets-sync/src/canister.rs rename to crates/sync-core/src/canister.rs diff --git a/assets-sync/src/content.rs b/crates/sync-core/src/content.rs similarity index 100% rename from assets-sync/src/content.rs rename to crates/sync-core/src/content.rs diff --git a/assets-sync/src/glob.rs b/crates/sync-core/src/glob.rs similarity index 100% rename from assets-sync/src/glob.rs rename to crates/sync-core/src/glob.rs diff --git a/assets-sync/src/headers.rs b/crates/sync-core/src/headers.rs similarity index 99% rename from assets-sync/src/headers.rs rename to crates/sync-core/src/headers.rs index 1ab4901..ce5bb91 100644 --- a/assets-sync/src/headers.rs +++ b/crates/sync-core/src/headers.rs @@ -8,7 +8,7 @@ //! `Content-Type` is parsed structurally onto [`HeaderRule::content_type`] //! instead of accumulating in `headers`: the canister stores it as asset //! metadata that drives encoder selection and certification, not as an -//! appended response header. See HEADERS.md for the full reject list. +//! appended response header. See `docs/headers.md` for the full reject list. use crate::glob::KeyPattern; use crate::strip_comment; @@ -69,7 +69,7 @@ struct OpenBlock { /// rules are concatenated with `, ` per RFC 7230 §3.2.2, with `Set-Cookie` /// carved out per RFC 6265 §3 (kept as separate entries). The returned Vec is /// stable-sorted by lowercased header name so multi-valued headers preserve -/// their declaration order — see the determinism guarantee in HEADERS.md. +/// their declaration order — see the determinism guarantee in `docs/headers.md`. /// /// `Content-Type` is never present in the output — it routes through /// [`content_type_for`] into the asset's stored `content_type` metadata. diff --git a/assets-sync/src/html_handling.rs b/crates/sync-core/src/html_handling.rs similarity index 96% rename from assets-sync/src/html_handling.rs rename to crates/sync-core/src/html_handling.rs index f652ef9..adcd76b 100644 --- a/assets-sync/src/html_handling.rs +++ b/crates/sync-core/src/html_handling.rs @@ -32,11 +32,13 @@ //! for `/foo.html` still serves the asset directly with a 200 rather than the //! 307 Cloudflare would emit. We synthesise the rules anyway so the ruleset //! reflects the full table and self-activates if that precedence ever changes; -//! the README documents the gap for users who care about strict URL +//! `docs/redirects.md` documents the gap for users who care about strict URL //! canonicalisation. //! -//! Synthesised rules are appended **after** the user's `_redirects`, so any -//! user-declared rule with the same `from` wins via declaration order. +//! Synthesised rules are prepended **before** the user's `_redirects` (see +//! `sync.rs`), so the html-handling defaults win at the exact paths they cover +//! and user rules catch what's left. A user-declared rule with the same `from` +//! as a synthesised rule is therefore shadowed by the synth rule. use crate::canister::{RedirectRule, RulePattern}; diff --git a/assets-sync/src/lib.rs b/crates/sync-core/src/lib.rs similarity index 100% rename from assets-sync/src/lib.rs rename to crates/sync-core/src/lib.rs diff --git a/assets-sync/src/redirects.rs b/crates/sync-core/src/redirects.rs similarity index 100% rename from assets-sync/src/redirects.rs rename to crates/sync-core/src/redirects.rs diff --git a/assets-sync/src/scan.rs b/crates/sync-core/src/scan.rs similarity index 100% rename from assets-sync/src/scan.rs rename to crates/sync-core/src/scan.rs diff --git a/assets-sync/src/sync.rs b/crates/sync-core/src/sync.rs similarity index 100% rename from assets-sync/src/sync.rs rename to crates/sync-core/src/sync.rs diff --git a/assets-sync/tests/bench_sync.rs b/crates/sync-core/tests/bench_sync.rs similarity index 97% rename from assets-sync/tests/bench_sync.rs rename to crates/sync-core/tests/bench_sync.rs index 3778fe5..98d0cd2 100644 --- a/assets-sync/tests/bench_sync.rs +++ b/crates/sync-core/tests/bench_sync.rs @@ -15,16 +15,16 @@ //! //! Tests are `#[ignore]`'d so they don't slow down the regular suite. -use assets_sync::canister::{AssetDetails, AssetProperties, CallType, CanisterCall, RedirectRule}; -use assets_sync::sync::sync; use candid::{CandidType, Decode, Encode, Nat, Principal}; use serde::Deserialize; use std::cell::{Cell, RefCell}; use std::collections::BTreeMap; use std::path::Path; +use sync_core::canister::{AssetDetails, AssetProperties, CallType, CanisterCall, RedirectRule}; +use sync_core::sync::sync; // Wire-compatible mirrors of the response types defined privately in -// assets_sync::canister. Same field name → same Candid encoding. +// sync_core::canister. Same field name → same Candid encoding. #[derive(CandidType)] struct CreateBatchOk { batch_id: Nat, diff --git a/crates/sync-plugin/Cargo.toml b/crates/sync-plugin/Cargo.toml new file mode 100644 index 0000000..6fa2dc9 --- /dev/null +++ b/crates/sync-plugin/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "sync-plugin" +version.workspace = true +authors.workspace = true +edition.workspace = true +repository.workspace = true +license.workspace = true +publish.workspace = true + +[lib] +crate-type = ["cdylib"] + +[dependencies] +sync-core = { path = "../sync-core" } +candid.workspace = true +serde.workspace = true +wit-bindgen = { workspace = true, features = ["realloc"] } diff --git a/plugin/build.rs b/crates/sync-plugin/build.rs similarity index 100% rename from plugin/build.rs rename to crates/sync-plugin/build.rs diff --git a/plugin/src/lib.rs b/crates/sync-plugin/src/lib.rs similarity index 94% rename from plugin/src/lib.rs rename to crates/sync-plugin/src/lib.rs index b8c01fa..23201c9 100644 --- a/plugin/src/lib.rs +++ b/crates/sync-plugin/src/lib.rs @@ -8,9 +8,9 @@ wit_bindgen::generate!({ path: "wit/sync-plugin.wit", }); -use assets_sync::canister::{CallType, CanisterCall}; use candid::{CandidType, Decode, Encode}; use serde::de::DeserializeOwned; +use sync_core::canister::{CallType, CanisterCall}; use crate::icp::sync_plugin::types as ty; @@ -52,7 +52,7 @@ impl Guest for Plugin { "sync plugin: starting for canister {} (environment: {})", input.canister_id, input.environment ); - let summary = assets_sync::sync::sync( + let summary = sync_core::sync::sync( &WasiCall, &input.dirs, &input.identity_principal, diff --git a/plugin/wit/sync-plugin.wit b/crates/sync-plugin/wit/sync-plugin.wit similarity index 100% rename from plugin/wit/sync-plugin.wit rename to crates/sync-plugin/wit/sync-plugin.wit diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..34e2236 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,13 @@ +# Architecture + +This repo builds two WebAssembly modules — the assets **canister** (deployed to ICP) and the `icp-cli` sync **plugin** — each split into a logic library plus a thin wasm wrapper. All crates live under [`crates/`](../crates/): + +| Crate | Kind | Role | +|-------|------|------| +| [`canister-core`](../crates/canister-core/) | library | Asset storage, certification (response verification), streaming, and access control. Can also be embedded in other canisters. | +| [`canister`](../crates/canister/) | `cdylib`, `wasm32-unknown-unknown` | Thin wrapper that exposes `canister-core` as the deployable ICP assets canister. | +| [`sync-core`](../crates/sync-core/) | library | Platform-agnostic sync logic: directory scanning, MIME detection, content encoding, canister diffing, and the `CanisterCall` trait that abstracts the transport layer. | +| [`sync-plugin`](../crates/sync-plugin/) | `cdylib`, `wasm32-wasip2` | Thin `icp-cli` sync plugin that wraps `sync-core`. | +| [`e2e`](../crates/e2e/) | tests | End-to-end tests driving the canister and plugin together through the `icp` CLI. | + +The `*-core` crates hold the logic; the matching wrapper is just the wasm build target — `canister` wraps `canister-core`, `sync-plugin` wraps `sync-core`. diff --git a/ic-certified-assets/CHANGELOG.md b/ic-certified-assets/CHANGELOG.md deleted file mode 100644 index 0924d04..0000000 --- a/ic-certified-assets/CHANGELOG.md +++ /dev/null @@ -1,156 +0,0 @@ -# Changelog -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [Unreleased] - -### Changed - -- **BREAKING**: Implement `serde::Serialize` and `serde::Deserialize` for stable state structures: - - Moved all stable state structures to the `stable_machine::v1` module, renaming them to `StableStateV1`, `StableConfigurationV1`, `StableStatePermissionsV1`, `StableAssetV1`, `StableAssetEncodingV1` - - Removed `StableState` struct - - Introduced new `state_machine::v2` module with the new `StableStateV2` struct, which implements `serde::Serialize` and `serde::Deserialize`. This allows to serialize and deserialize the state using serde-compatible libraries, such as `serde_cbor`. - - Added conversion between legacy `StableStateV1` and new `StableStateV2` structs - - `pre_upgrade()` now returns `StableStateV2` instead of `StableStateV1` - - `post_upgrade()` now accepts `StableStateV2` parameter instead of `StableStateV1` - - Removed `estimate_size()` methods from the `StableStateV1`, `StableConfigurationV1`, `StableStatePermissionsV1`, `StableAssetV1`, `StableAssetEncodingV1` structs -- **BREAKING**: Use `BTreeMap` instead of `HashMap` for headers to guarantee deterministic ordering. - - Changed `StableAssetV2.headers` to use `BTreeMap` instead of `HashMap` - - Changed `Asset.headers` to use `BTreeMap` instead of `HashMap` - - Changed `CreateAssetArguments.headers` to use `BTreeMap` instead of `HashMap` - - Changed `AssetProperties.headers` to use `BTreeMap` instead of `HashMap` - - Changed `SetAssetPropertiesArguments.headers` to use `BTreeMap` instead of `HashMap` -- **BREAKING**: Sets the `ic_env` cookie for html files, which contains the root key and the canister environment variables that are prefixed with `PUBLIC_`. Please note that this version of the `ic-certified-assets` is only compatible with PocketIC **v10** and above. -- **BREAKING**: Use `SystemContext` instead of multiple arguments for `State` methods. - - Changed `State.store()` to accept `&SystemContext` instead of `time` - - Changed `State.create_batch()` to accept `&SystemContext` instead of `now` - - Changed `State.create_chunk()` to accept `&SystemContext` instead of `now` - - Changed `State.create_chunks()` to accept `&SystemContext` instead of `now` - - Changed `State.set_asset_content()` to accept `&SystemContext` instead of `now` - - Changed `State.commit_batch()` to accept `&SystemContext` instead of `now` - - Changed `State.commit_proposed_batch()` to accept `&SystemContext` instead of `now` - -#### Migration guide - -To migrate canisters that use the `ic-certified-assets` library to the new serde-serializable stable state: - -1. Upgrade to the latest `ic-certified-assets` which exports `StableStateV2` and implements `serde::{Serialize, Deserialize}` for stable state types. - -2. Choose a serde-compatible library to serialize and deserialize the stable state, such as [`serde_cbor`](https://crates.io/crates/serde_cbor), and add it to your canister's dependencies. - -3. Update the upgrade hooks to persist the new serialized state in stable memory and keep backward compatibility with existing deployments that stored Candid: - ```rust - // In this example, the serde-compatible library of choice is `serde_cbor`. - - use ic_cdk::stable; - use ic_certified_assets::{StableStateV1, StableStateV2, types::AssetCanisterArgs}; - - pub fn save_stable_state(stable_state: &StableStateV2) -> Result<(), serde_cbor::Error> { - let mut stable_writer = stable::StableWriter::default(); - serde_cbor::to_writer(&mut stable_writer, stable_state) - } - - pub fn is_candid_stable_state() -> bool { - let mut maybe_magic_bytes = vec![0u8; 4]; - stable::stable_read(0, &mut maybe_magic_bytes); - maybe_magic_bytes == b"DIDL" - } - - pub fn load_candid_stable_state() -> Result { - let (stable_state,) = ic_cdk::storage::stable_restore()?; - Ok(stable_state) - } - - pub fn load_stable_state() -> Result { - let stable_reader = stable::StableReader::default(); - from_reader_ignore_trailing_data(stable_reader) - } - - fn from_reader_ignore_trailing_data(reader: R) -> Result - where - T: serde::de::DeserializeOwned, - R: std::io::Read, - { - let mut deserializer = serde_cbor::de::Deserializer::from_reader(reader); - let value = serde::de::Deserialize::deserialize(&mut deserializer)?; - // we do not call deserializer.end() here - // because we want to ignore trailing data loaded from stable memory - Ok(value) - } - - #[ic_cdk::pre_upgrade] - fn pre_upgrade() { - let stable_state = ic_certified_assets::pre_upgrade(); - save_stable_state(&stable_state).expect("failed to serialize stable state"); - } - - #[ic_cdk::post_upgrade] - fn post_upgrade(args: Option) { - let stable_state = if is_candid_stable_state() { - // backward compatibility - load_candid_stable_state() - .expect("failed to restore candid stable state") - .into() - } else { - load_stable_state().expect("failed to deserialize stable state") - }; - ic_certified_assets::post_upgrade(stable_state, args); - } - ``` - -This way, you maintain backward compatibility with the existing deployment of your asset canister, which was using Candid to save and load the stable state. An implementation reference can be found in the [Asset Canister source code](https://github.com/dfinity/sdk/tree/master/src/canisters/frontend/ic-frontend-canister). - -## [0.3.0] - 2025-06-26 - -### Added - -- The stored state can now be directly accessed with `with_state` -- Asset permissions can be set in the initialization parameters when installing or upgrading -- Added bulk operations for chunk uploading and committing -- Added support for response verification v2 certificate expressions -- Added configurable upload limits for chunks and batches -- Added an API for proposing, validating, and committing asset change batches, for use in SNSes -- Added functions for getting and setting asset properties -- Added `list_authorized` and `deauthorize` functions -- Added `take_ownership` function for clearing the ACL -- Added a more fine-grained permission system to the ACL - -### Changed - -- Exported methods are now declared explicitly with the `export_canister_methods!` macro. The implementations of these methods are now public and can be invoked explicitly -- Converted `list_permitted` to an update call -- Domain redirection now prefers `icp0.io` over `ic0.app` -- Authorized users can no longer authorize other users - -## [0.2.5] - 2022-08-22 -### Added -- Support for asset caching based on [ETag](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag) -- Automatic redirection of all traffic from `.raw.ic0.app` domain to `.ic0.app` - -## [0.2.4] - 2022-07-12 -### Fixed -- headers field in Candid spec accepts mmultiple HTTP headers - -## [0.2.3] - 2022-07-06 -### Added -- Support for setting custom HTTP headers on asset creation - -## [0.2.2] - 2022-05-12 -### Fixed -- Parse and produce ETag headers with quotes around the hash - -## [0.2.1] - 2022-05-12 -### Fixed -- Make StableState public again - -## [0.2.0] - 2022-05-11 -### Added -- Support for asset caching based on [ETag](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag) -- Support for asset caching based on [max-age](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control) -- Automatic redirection of all traffic from `.raw.ic0.app` domain to `.ic0.app` - -## [0.1.0] - 2022-02-02 -### Added -- First release diff --git a/ic-certified-assets/PERMISSIONS.md b/ic-certified-assets/PERMISSIONS.md deleted file mode 100644 index 73e83a1..0000000 --- a/ic-certified-assets/PERMISSIONS.md +++ /dev/null @@ -1,156 +0,0 @@ -# Assets Canister Permission System - -## Overview - -The canister uses a three-tier permission model stored as three independent -`BTreeSet` collections in the canister state. Controllers (set at -the platform level by `ic0.msg_caller_is_controller`) always pass every -permission check regardless of the explicit permission lists. - ---- - -## Permission Types - -Defined in `src/types.rs`: - -``` -Permission::Commit — create/update/delete assets, commit batches -Permission::Prepare — create batches and chunks -Permission::ManagePermissions — grant and revoke permissions for others -``` - -**Hierarchy:** A principal with `Commit` implicitly has `Prepare` as well -(implemented in `State::can`). The reverse is not true. - ---- - -## Storage - -`src/state_machine/mod.rs` — three fields on `State`: - -```rust -commit_principals: BTreeSet -prepare_principals: BTreeSet -manage_permissions_principals: BTreeSet -``` - -These are serialised to `StableStatePermissionsV2` on upgrade and restored in -`impl From for State`. Canisters upgraded from the legacy -`authorized` field treat all principals in that list as having `Commit`. - ---- - -## Authorization Helpers (`src/lib.rs`) - -| Helper | Succeeds when | -|--------|---------------| -| `can(permission)` | caller's principal is in the list **or** is a controller (`Commit` also implies `Prepare`) | -| `has_permission_or_is_controller(p)` | principal is in list **or** is a controller | -| `is_manager_or_controller()` | `ManagePermissions` list or controller | -| `is_controller()` | platform-level controller only | - ---- - -## Operation → Required Permission - -| Canister method | Minimum permission | -|---|---| -| `store` | `Commit` | -| `create_batch` | `Prepare` (or `Commit`) | -| `create_chunks` | `Prepare` (or `Commit`) | -| `create_asset` | `Commit` | -| `set_asset_content` | `Commit` | -| `unset_asset_content` | `Commit` | -| `delete_asset` | `Commit` | -| `clear` | `Commit` | -| `commit_batch` | `Commit` | -| `delete_batch` | `Prepare` (or `Commit`) | -| `set_asset_properties` | `Commit` | -| `configure` | `Commit` | -| `get_configuration` | `Prepare` (or `Commit`) | -| `authorize` | `ManagePermissions` or controller | -| `grant_permission` | `ManagePermissions` or controller | -| `deauthorize` | `Commit` (self) or controller (others) | -| `revoke_permission` | self: any listed permission; others: `ManagePermissions` or controller | -| `take_ownership` | controller only | -| `list_authorized` / `list_permitted` | none (public) | -| `get` / `get_chunk` / `list` / `api_version` | none (public) | - ---- - -## Initialisation - -`init()` in `src/lib.rs`: - -1. Clears all state. -2. Grants the **message caller** `Commit` permission — so whoever deploys the - canister gets `Commit` by default. -3. Optionally applies `InitArgs::set_permissions` to override the lists - completely (used for programmatic deployments that want a custom initial - permission set). - -**Implication for proxy-deployed canisters:** when `icp deploy --proxy` creates -the assets canister, the proxy canister is the message caller at `init` time, so -the proxy gets `Commit`. The user's signing identity is not granted any -permission. The proxy is also the sole controller of the new canister. - ---- - -## Proxy Mode Problem and Recommended Solution - -### Problem - -When the sync plugin is invoked after `icp deploy --proxy`: - -1. The assets canister was created by the proxy → the **proxy canister** holds - `Commit` permission and is the controller. -2. The sync plugin currently sends all calls with `direct: true` → calls are - signed by the **user's identity**. -3. `create_batch`, `create_chunks`, and `commit_batch` all require `Commit`; - the user's identity has none → every upload call is rejected. - -### Solution (permission bootstrap in proxy mode) - -Before starting the upload flow, the plugin should: - -1. **Detect proxy mode** — `SyncExecInput::proxy_canister_id` is `Some`. - -2. **Check current permissions** — call `list_permitted(Commit)` as a direct - query (no proxy needed; this method is public). Parse the returned - `Vec` and check whether `input.identity_principal` is present. - -3. **Grant if missing** — if the user's principal is not in the list, make a - **proxy call** (`direct: false`) to `grant_permission` with: - ``` - GrantPermissionArguments { - to_principal: identity_principal, - permission: Permission::Commit, - } - ``` - The proxy canister is a controller of the assets canister, so - `has_permission_or_is_controller(ManagePermissions)` succeeds for it. - -4. **Proceed normally** — all subsequent upload calls remain `direct: true` - (signed by the user's identity, which now has `Commit`). - -### Why `Commit` is sufficient - -The V2 batch upload flow (`create_batch` → `create_chunks` → `commit_batch`) -requires only `Commit`. Granting `Commit` to the user's identity is therefore -the minimal permission needed and does not expose `ManagePermissions` to the -user. - -### Candid reference - -```candid -// Public read — no permission required; update (not query) call -list_permitted : (ListPermitted) -> (vec principal); - -// Requires ManagePermissions or controller -grant_permission : (GrantPermission) -> (); - -// Types -type Permission = variant { Commit; Prepare; ManagePermissions }; -type ListPermitted = record { permission : Permission }; -type GrantPermission = record { to_principal : principal; permission : Permission }; -``` diff --git a/ic-certified-assets/README.md b/ic-certified-assets/README.md deleted file mode 100644 index 5f89f83..0000000 --- a/ic-certified-assets/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Certified Assets Library - -Rust support for asset certification. - -Certified assets can also be served from any Rust canister by including this library. diff --git a/plugin/Cargo.toml b/plugin/Cargo.toml deleted file mode 100644 index 3526386..0000000 --- a/plugin/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "plugin" -version = "0.1.0" -edition = "2021" -publish = false - -[lib] -crate-type = ["cdylib"] - -[dependencies] -assets-sync = { path = "../assets-sync" } -candid.workspace = true -serde.workspace = true -wit-bindgen = { workspace = true, features = ["realloc"] } diff --git a/plugin/README.md b/plugin/README.md deleted file mode 100644 index 764f26b..0000000 --- a/plugin/README.md +++ /dev/null @@ -1,215 +0,0 @@ -# icp-cli sync plugin - -This crate is the sync plugin invoked by `icp-cli` for assets canister syncing. - -## What - -`icp-cli` added a new plugin system for canister syncing (operations that always run after canister module installation/upgrade). - -This crate is an implementation of it that is supposed to be used with the [`assets canister`](../canister/) only. - -It is a WASM component module that only access the functionalities exposed by the icp-cli plugin runtime. - -It exports an `exec()` function which executes operations similar to the `sync()` function of the public `ic-asset` crate: -- Read access to one or more directories (assets to be uploaded) -- Sync them to a deployed assets canister - -## How - -The runtime side of the plugin system lives in [`github.com/dfinity/icp-cli`](https://github.com/dfinity/icp-cli): -- The runtime crate: `crates/icp-sync-plugin` -- An example: `examples/icp-sync-plugin` - -All sync logic (directory scanning, MIME detection, content encoding, canister diffing, and per-endpoint call wrappers) lives in the [`assets-sync`](../assets-sync/) library crate. This plugin crate is a thin WASI/WIT wrapper: it implements the `CanisterCall` trait (`WasiCall`) on top of the host's `canister-call` import, then delegates to `assets_sync::sync::sync()`. - -The protocol-level pieces (Candid types, batch/chunk upload flow, content encoding) were ported from the `ic-asset` crate in [`github.com/dfinity/sdk`](https://github.com/dfinity/sdk) (`src/canisters/frontend/ic-asset`). The transport layer was rewritten on top of the host's `canister-call` import. - -## Build - -```sh -cargo build -p plugin --target wasm32-wasip2 --release -``` - -The output WASM lands at `../target/wasm32-wasip2/release/plugin.wasm` — the path that [`../example/icp.yaml`](../example/icp.yaml) references. - -## Scope - -The current implementation supports the V2 protocol of the assets canister (transactional batch API): -- The manifest's `dirs:` setting must list **exactly one** directory. The plugin rejects the sync before any canister call if zero or multiple entries are given — the assets canister owns the URL space below `/`, and a single tree keeps key collisions and `_redirects` precedence unambiguous. -- Walks that directory; dotfiles are skipped. -- Detects the MIME type of each file and computes encodings: `gzip` for all `text/*`, `*/javascript`, and `*/html` types (only if the compressed output is smaller), `identity` for everything. -- Diffs against `list_assets()`: skips encodings already in place (matched by sha256), unsets encodings that are stale, and deletes assets that have been removed or whose `content_type` changed. -- Reads `_redirects` at the root of the input directory and replaces the canister's ruleset in the same batch (see "Redirects" below). -- Reads `_headers` at the root of the input directory, resolves per-asset header lists, and routes them through `CreateAssetArguments.headers` and `SetAssetProperties` (see "Headers" below). -- Opens a transaction (`create_batch`), uploads each content chunk via `create_chunks` (one chunk per call, 1.9 MB max), then commits all operations atomically with a single `commit_batch` call. -- In normal mode all canister calls use `direct: true`. In proxy mode (when a `proxy_canister_id` is provided by the host) the plugin first ensures the signing identity has `Commit` permission, routing a `grant_permission` call through the proxy (which is the canister's controller) if needed, then proceeds with direct calls. - -The plugin calls `api_version` first and aborts if the canister advertises anything below 2. - -## Redirects - -The plugin reads a Netlify-style `_redirects` file at the root of the input directory (`dirs:` must list exactly one). The file itself is **not** uploaded as an asset — it's consumed by the plugin and lowered into canister-side rules. Each non-empty, non-comment line: - -``` - -``` - -- `` — absolute path. A trailing `/*` makes the rule match every URL whose path starts with the prefix (subtree). Anywhere else, `*` is an error. -- `` — absolute path for `200` rewrites and `4xx` custom error pages. For `3xx` rules it may also be a fully-qualified URL (sent in the `Location` header). -- `` — required integer, one of `{200, 301, 302, 307, 308, 404, 410}`. Unlike Netlify, there is no default; the explicit number keeps the intent (rewrite vs. redirect vs. error) unambiguous. -- Lines starting with `#` and blank lines are ignored. Inline `# comments` at the end of a rule line are stripped before parsing. - -### Examples - -```text -# 3xx — issue a Location-header redirect -/old-page /new-page 301 -/external https://example.com/ 302 - -# 200 — rewrite: serve the target asset's body at the source URL -/about /about.html 200 -/blog/* /blog/index.html 200 - -# 4xx — custom error page (serves the target asset's body with the override status) -/missing /404.html 404 -/gone /tombstone.html 410 -``` - -A real asset at the rule's `from` path always wins — a file at `/about.html` is served at `/about.html` regardless of any rule. For 200 rewrites and 4xx error pages, the target asset's existing `Content-Type` and certified headers are inherited verbatim; the rule itself can't override them. - -Rule order is significant: the plugin sends rules in declaration order, and the canister returns the first match. Replacing the file is a full replace-all operation — if you remove `_redirects` between deploys, the plugin emits an empty `SetRedirectRules` op so the canister clears its ruleset. - -### Default HTML handling (auto-synthesised) - -Ahead of the user's `_redirects`, the plugin auto-synthesises Cloudflare's [`auto-trailing-slash`](https://developers.cloudflare.com/workers/static-assets/routing/advanced/html-handling/#automatic-trailing-slashes-default) rule set for every `.html` asset in the project. Synthesised rules are prepended **before** the user's rules, so the html-handling defaults claim the exact paths they cover (`/foo`, `/foo/`, `/foo/index`, `/foo/index.html`, `/bar/`, `/bar`, etc.) and user rules apply to whatever paths are left. The most common use of `_redirects` after this — a SPA-style `/* /404.html 404` catch-all — works as expected: it fires only for paths no HTML asset claims. - -This ordering is also a correctness requirement: the IC HTTP gateway's response verifier rejects a wildcard expression path response (`["http_expr", "<*>"]`) when a potential exact expression path (`["http_expr", , "<$>"]`) exists in the certified tree. Putting synth first ensures the canister always returns the Exact-path response for paths the html-handling defaults certify, which keeps the verifier happy. A user rule placed at the same `from` as a synthesised rule (e.g. `/foo /foo.html 200` when `/foo.html` already exists) is therefore dead — the synth rule wins. To override html-handling for a particular page, remove its `.html` source and serve it under a non-HTML key. - -Given two assets `/foo.html` and `/bar/index.html`, the synthesised rules behave per Cloudflare's table: - -| Request | Effective response | -|-------------------|----------------------------------------------| -| `/foo` | `200`, body of `/foo.html` | -| `/foo.html` | `200`, body of `/foo.html` (see note) | -| `/foo/` | `307` → `/foo` | -| `/foo/index` | `307` → `/foo` | -| `/foo/index.html` | `307` → `/foo` | -| `/bar/` | `200`, body of `/bar/index.html` | -| `/bar` | `307` → `/bar/` | -| `/bar.html` | `307` → `/bar` (client chains to `/bar/`) | -| `/bar/index` | `307` → `/bar` (client chains to `/bar/`) | -| `/bar/index.html` | `200`, body of `/bar/index.html` (see note) | - -**Note on the two inert rows.** Cloudflare emits `307` for both `/foo.html → /foo` and `/bar/index.html → /bar/` (URL canonicalisation). The asset canister matches direct asset lookups before rules, so the asset at those keys wins and the synthesised 307 is shadowed. Requests to the literal `.html` URL still 200-serve the asset rather than redirecting. The synthesised rules are kept in the rule list so the table activates automatically if that precedence ever changes; users who care about strict canonicalisation can omit the `.html` source by other means (e.g. excluding it from the input directory). - -### Migration from built-in aliasing - -Earlier versions of this canister implicitly served `/foo.html` at `/foo` and `/foo/index.html` at both `/foo/` and `/foo`. Equivalent routing is now produced by auto-synthesis (above) for every `.html` asset, so most projects need no migration — the `_redirects` file is optional. - -The one pre-existing pattern that is **not** covered by auto-synthesis is the SPA fallback (subtree → single HTML), since it requires a wildcard match across paths that don't correspond to an asset. Declare it explicitly: - -```text -# SPA fallback (formerly served implicitly when no other asset matched) -/* /index.html 200 -``` - -The canister's built-in aliasing (and the `is_aliased` field on `set_asset_properties`) have been retired in favour of `_redirects` + plugin-side synthesis. The plugin no longer reads any project-side config file — only `_redirects` at the root of the input directory is consulted. The canister will drop the `is_aliased` field in a follow-up cleanup. - -### Unsupported syntax - -- `:splat` and `:placeholder` substitution in `` — deferred (see the design plan's tier-3 follow-up). -- Netlify's `!` force suffix on status codes — files always win over rules at the same path; remove the conflicting asset instead. -- Country/role conditions and query-string matching — out of scope. -- Inline headers as a fourth field — headers are configured in a separate `_headers` file (see "Headers" below). - -Parse errors abort the sync with the offending file path and 1-based line number, before any canister call is issued. - -## Headers - -The plugin reads a Netlify-style `_headers` file at the root of the input directory (`dirs:` must list exactly one). The file itself is **not** uploaded as an asset — it's consumed by the plugin and lowered into per-asset header lists. Each non-indented `` line opens a block; subsequent indented lines (1+ spaces or tabs) are `Header-Name: value` entries belonging to the block. Blank lines close the block: - -```text -/_astro/* - Cache-Control: public, max-age=31536000, immutable - X-Content-Type-Options: nosniff - -/* - X-Frame-Options: DENY - X-Robots-Tag: noindex - -/api - Cache-Control: no-store -``` - -- `` — absolute path. A trailing `/*` makes it a subtree match (`/*` alone matches every key). Anywhere else, `*` is an error. -- Header lines must follow a `` block; an indented line at the top of the file or after a blank-line boundary is an error. -- Lines starting with `#` and blank lines are ignored. Inline `# comments` at the end of a line are stripped before parsing. - -### Precedence - -When multiple rules match the same key, **all** matching rules apply — there is no "more specific overrides" rule, and exact vs subtree patterns are not ranked. Same-name values across matching rules are concatenated with `, ` per [RFC 7230 §3.2.2](https://datatracker.ietf.org/doc/html/rfc7230#section-3.2.2): - -```text -/* - X-Robots-Tag: noindex - -/admin/* - X-Robots-Tag: nofollow -``` - -`/admin/page` sees `X-Robots-Tag: noindex, nofollow`. Semantically conflicting concatenations (`Cache-Control: public, no-store`) are the user's responsibility to avoid. - -`Set-Cookie` is the one exception per [RFC 6265 §3](https://datatracker.ietf.org/doc/html/rfc6265#section-3) — multiple `Set-Cookie` lines, whether from a single rule or across rules, stay as separate header entries instead of being comma-folded. - -The plugin stable-sorts the resolved list by lowercased header name before sending so wire order is a deterministic function of header content. `Set-Cookie` entries get grouped together but preserve declaration order within the group ([RFC 6265 §5.3](https://datatracker.ietf.org/doc/html/rfc6265#section-5.3) makes the *last* same-name cookie win, so the group order is load-bearing). - -### What gets touched - -- **New assets**: the resolved header list is passed in `CreateAssetArguments.headers`. -- **Existing assets**: drift is detected byte-for-byte against canister-stored headers; mismatches emit a `SetAssetProperties` op with the new list. A `_headers`-only edit propagates without re-uploading content. -- **3xx redirects** (rules in `_redirects` with status 301/302/307/308) synthesize their own response, so they don't inherit asset headers. The plugin populates `RedirectRule.headers` for these from any `_headers` rule whose pattern matches the redirect's `from`. 200 / 4xx rules borrow headers from the target asset, so no plumbing is needed there. - -### `Content-Type` overrides - -`Content-Type` is recognised inside a `_headers` block, but it routes to the asset's stored media type instead of the appended response headers. This is the only way to override the `mime_guess::from_path` default (or to add a `charset` parameter) — the canister always derives a single, certified `Content-Type` for every asset, and an appended header would produce a duplicate on the wire. - -```text -/*.md - Content-Type: text/markdown; charset=utf-8 - Cache-Control: public, max-age=300 -``` - -The plugin extracts `Content-Type` and feeds it into `CreateAssetArguments.content_type`; other headers in the block continue to flow through `headers` as usual. `Content-Type` is single-valued, so when multiple blocks match the same asset the first matching `Content-Type` wins (other matching rules still contribute their non-`Content-Type` headers). Editing a `Content-Type` and redeploying triggers delete-then-recreate on the canister to keep the certified type in sync. - -Validation: - -- The value must parse as a MIME type via the `mime` crate. -- A duplicate `Content-Type:` line within the same block is rejected. -- Empty value is rejected. - -### Validation and unsupported syntax - -Header names and values are validated via the `http` crate's `HeaderName` / `HeaderValue` (rejects CR/LF, so no header injection). Rejected with a 1-based line number: - -- Mid-path wildcards in `` (e.g. `/foo/*/bar`) — not supported. -- `:splat` / `:placeholder` substitution in header values — deferred. -- Patterns like `/*.html` or `/blog/:slug` — deferred (see the design plan's tier-3 follow-up). -- Missing colon, blank header name, or value containing CR/LF. -- A `` block with no header lines under it (likely a typo). - -Parsing aborts at the first bad line so users fix issues one at a time. - -## TODO - -- [x] **Asset properties update** — emit `SetAssetProperties` ops for assets whose canister-side properties drifted from the plugin's defaults. `get_asset_properties` is called after `list_assets` to collect the current `AssetProperties` for every canister asset, and `update_properties` resets any non-default `max_age`, `headers`, or `allow_raw_access`. (`is_aliased` is no longer carried; the canister will drop it in a follow-up cleanup PR.) - -- [x] **Header representation: `Map` → `Vec<(name, value)>`** — `Option>` / `Option>` were replaced with a list of pairs across `ic-certified-assets` (`Asset`, `AssetProperties`, `CreateAssetArguments`, `SetAssetPropertiesArguments`, `build_headers`, `evidence::hash_headers`) and `assets-sync` (`canister::AssetProperties`, `update_properties`). Candid wire type `vec record { text; text }` is already a list, so this was a Rust-only change. Lets users carry multiple `Set-Cookie` values. Evidence hashing stable-sorts by lowercased name only so same-name groups preserve declaration order. - -- [ ] **`commit_batch` chunking** — split operations across multiple `commit_batch` calls to stay within the ~2 MB ICP ingress message limit, matching `ic-asset` behaviour. - - Replace the single `commit_batch` call in `sync()` with a `commit_in_stages` helper modelled on `ic-asset/src/sync.rs::commit_in_stages`. - - Implement `create_commit_batches(ops)` that splits the operation list using two limits: 500 ops per batch (cert-tree work) and 1.5 MB of accumulated header data per batch (header maps dominate ingress size). Header size is the sum of key+value lengths across all `CreateAsset` and `SetAssetProperties` operations in the batch. - - Each intermediate batch is committed with `batch_id = 0`; after all operation batches are committed, a final call with the real `batch_id` and an empty operations list closes the batch. - -- [ ] **Multi-chunk upload via `create_chunks`** — send multiple small chunks per canister call instead of one, reducing round-trips for projects with many small files. - - Add a `create_chunks` wrapper in `canister.rs` that calls the canister's `create_chunks` method (takes `batch_id` and `Vec>`, returns `Vec` chunk IDs). - - In `sync()`, replace the single-chunk loop with a batching uploader: accumulate chunks up to `MAX_CHUNK_SIZE` (1.9 MB) of total payload per call, then flush via `create_chunks`. The last chunk of each file can be sent inline as `last_chunk` in `SetAssetContentArguments` rather than as a separate chunk ID, saving one round-trip per file.