diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index 1d91edc..0000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -* @TheDancingDeveloper-org/migration-maintainers diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 74cb186..0000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: 2 -updates: - - package-ecosystem: cargo - directory: / - schedule: - interval: weekly - open-pull-requests-limit: 10 - groups: - rust-minor: - update-types: [minor, patch] - - - package-ecosystem: github-actions - directory: / - schedule: - interval: weekly - open-pull-requests-limit: 5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 59191d9..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: CI - -on: - push: - branches: [main] - pull_request: - workflow_dispatch: - -permissions: - contents: read - -jobs: - rust: - # Dependabot metadata PRs do not consume the self-hosted validation runner. - if: github.event_name != 'pull_request' || github.event.pull_request.user.type != 'Bot' - runs-on: [self-hosted] - steps: - - uses: actions/checkout@v7 - - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt,clippy - - run: cargo fmt --all -- --check - - run: cargo check --workspace --locked - - run: cargo test --workspace --locked - - run: cargo clippy --workspace --all-targets --locked -- -D warnings diff --git a/.github/workflows/policy.yml b/.github/workflows/policy.yml deleted file mode 100644 index 8e9b508..0000000 --- a/.github/workflows/policy.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Migration policy - -on: - push: - branches: [main] - pull_request: - workflow_dispatch: - -permissions: - contents: read - -jobs: - policy: - # Dependabot metadata PRs do not consume the self-hosted policy runner. - if: github.event_name != 'pull_request' || github.event.pull_request.user.type != 'Bot' - runs-on: [self-hosted] - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: Verify migration policies - shell: bash - run: bash ci/migration-policy.sh diff --git a/.github/workflows/runner-policy.yml b/.github/workflows/runner-policy.yml deleted file mode 100644 index 65a1c1f..0000000 --- a/.github/workflows/runner-policy.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Copy this file to .github/workflows/runner-policy.yml in every repository. -# -# It is deliberately tiny: all logic lives in the reusable workflow in -# github-policy, so the rule can be changed in one place. After adding it, make -# `runner-policy` a required status check on the repository's protected branch -# — without that it reports but does not block. -name: Runner policy - -on: - pull_request: - push: - branches: [main] - -jobs: - runner-policy: - uses: TheDancingDeveloper-org/github-policy/.github/workflows/runner-policy-reusable.yml@main \ No newline at end of file diff --git a/.woodpecker.yml b/.woodpecker.yml deleted file mode 100644 index 5be3013..0000000 --- a/.woodpecker.yml +++ /dev/null @@ -1,105 +0,0 @@ -when: - - event: push - branch: main - - event: manual - - event: pull_request - -steps: - - name: fmt - image: rust:1.94-bookworm - environment: - RUSTC_WRAPPER: sccache - SCCACHE_REDIS: "redis://100.92.54.45:6380" - commands: - - apt-get install -y --no-install-recommends sccache >/dev/null 2>&1 || true - - rustup component add rustfmt - - cargo fmt --all -- --check - - - name: check - image: rust:1.94-bookworm - environment: - RUSTC_WRAPPER: sccache - SCCACHE_REDIS: "redis://100.92.54.45:6380" - commands: - - apt-get install -y --no-install-recommends sccache >/dev/null 2>&1 || true - - cargo check --workspace - - - name: test - image: rust:1.94-bookworm - environment: - RUSTC_WRAPPER: sccache - SCCACHE_REDIS: "redis://100.92.54.45:6380" - commands: - - apt-get install -y --no-install-recommends sccache >/dev/null 2>&1 || true - - cargo test --workspace - - - name: clippy - image: rust:1.94-bookworm - environment: - RUSTC_WRAPPER: sccache - SCCACHE_REDIS: "redis://100.92.54.45:6380" - commands: - - apt-get install -y --no-install-recommends sccache >/dev/null 2>&1 || true - - rustup component add clippy - - cargo clippy --workspace -- -D warnings - - - name: publish - image: rust:1.94-bookworm - environment: - GIT_AUTH_TOKEN: - from_secret: git_auth_token - commands: - - printf '[registries.forgejo]\nindex = "sparse+https://repo.indexarr.net/api/packages/indexarr/cargo/"\ncredential-provider = "cargo:token"\n\n[registry]\ndefault = "forgejo"\n' > $CARGO_HOME/config.toml - - printf '[registries.forgejo]\ntoken = "Bearer %s"\n' "$GIT_AUTH_TOKEN" > $CARGO_HOME/credentials.toml - - cargo publish --allow-dirty 2>&1 || echo "Publish skipped (version may already exist)" - when: - - event: push - branch: main - - - name: publish-crates-io - image: rust:1.94-bookworm - environment: - CARGO_CRATES_IO_TOKEN: - from_secret: cargo_crates_io_token - commands: - - printf '[registries.crates-io]\ntoken = "%s"\n' "$CARGO_CRATES_IO_TOKEN" > $CARGO_HOME/credentials.toml - - sed -i 's/, registry = "forgejo"//g; s/registry = "forgejo", //g' Cargo.toml - - rm -f Cargo.lock - - cargo publish --registry crates-io --allow-dirty --no-verify 2>&1 || echo "crates.io publish skipped (version may already exist)" - when: - - event: push - branch: main - - # trigger-renovate step removed — woodpecker_token secret not configured, - # which errored the pipeline at secret-validation time. Re-add when provisioned. - - # push-github step removed — gh_release_token secret missing errors - # pipeline at validation. Re-add once provisioned. - - - name: discord-success - image: alpine/curl - environment: - DISCORD_WEBHOOK_URL: - from_secret: discord_webhook_url - commands: - - | - curl -s -X POST "$DISCORD_WEBHOOK_URL" \ - -H "Content-Type: application/json" \ - -d "{\"embeds\":[{\"title\":\"nzb-decode published\",\"description\":\"Published to Forgejo registry.\",\"color\":3066993}]}" - when: - - event: push - branch: main - status: [success] - - - name: discord-failure - image: alpine/curl - environment: - DISCORD_WEBHOOK_URL: - from_secret: discord_webhook_url - commands: - - | - curl -s -X POST "$DISCORD_WEBHOOK_URL" \ - -H "Content-Type: application/json" \ - -d "{\"embeds\":[{\"title\":\"nzb-decode build failed\",\"description\":\"Check CI logs for details\",\"color\":15158332}]}" - when: - - status: [failure] diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 70c5da5..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,43 +0,0 @@ -# nzb-decode - -yEnc decoding, file assembly, and article cache for NZB download clients. - -**Version:** 0.1.0 | **Edition:** Rust 2024 | **License:** MIT - -## This Is a Shared Library - -### Consumed By - -| App | Via | Tag | -|-----|-----|-----| -| rustnzbd | git | v0.1.0 | -| Arz | git | v0.1.0 | -| NGMS | vendored (path) | — | - -### Depends On - -- **rust-yenc-simd** (crates.io, v0.1) — SIMD-accelerated yEnc decoder - -## Public API - -```rust -pub mod assembler; // FileAssembler, AssemblerError -pub mod cache; // ArticleCache, CacheKey, CacheStats, CacheError -pub mod yenc; // YencDecodeResult, decode_yenc - -pub use assembler::FileAssembler; -pub use cache::ArticleCache; -pub use yenc::{YencDecodeResult, decode_yenc}; -``` - -### Key Types - -- **`FileAssembler`** — writes decoded articles into output files with lock-free concurrent writes (pwrite). Tracks per-file progress with atomic bitmap. -- **`ArticleCache`** — bounded in-memory cache with disk spill to temp files. LRU eviction with stats (hits, misses, spills). Key: `(job_id, file_id, segment_number)`. -- **`decode_yenc(&[u8]) -> Result`** — decodes raw NNTP article data, returns decoded bytes + metadata (filename, part info, CRC32). - -## Key Dependencies - -- yenc-simd (SIMD yEnc decoding) -- crc32fast (checksums) -- tokio, parking_lot diff --git a/Cargo.lock b/Cargo.lock index ad1ad58..f8bb4a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,42 +4,21 @@ version = 4 [[package]] name = "anyhow" -version = "1.0.104" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" - -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - -[[package]] -name = "bit-set" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" [[package]] name = "bytes" -version = "1.12.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cfg-if" @@ -56,6 +35,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -68,44 +53,85 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.5.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] -name = "fnv" -version = "1.0.7" +name = "foldhash" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" [[package]] name = "getrandom" -version = "0.3.4" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", "libc", - "r-efi 5.3.0", + "r-efi", "wasip2", + "wasip3", ] [[package]] -name = "getrandom" -version = "0.4.3" +name = "hashbrown" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "cfg-if", - "libc", - "r-efi 6.0.0", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", ] +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "libc" -version = "0.2.189" +version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "linux-raw-sys" @@ -123,23 +149,25 @@ dependencies = [ ] [[package]] -name = "num-traits" -version = "0.2.19" +name = "log" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "nzb-decode" -version = "0.1.3" +version = "0.1.2" dependencies = [ "anyhow", "bytes", "crc32fast", "parking_lot", - "proptest", "tempfile", "thiserror", "tokio", @@ -183,107 +211,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] -name = "ppv-lite86" -version = "0.2.21" +name = "prettyplease" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ - "zerocopy", + "proc-macro2", + "syn", ] [[package]] name = "proc-macro2" -version = "1.0.107" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] -[[package]] -name = "proptest" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" -dependencies = [ - "bit-set", - "bit-vec", - "bitflags", - "num-traits", - "rand", - "rand_chacha", - "rand_xorshift", - "regex-syntax", - "rusty-fork", - "tempfile", - "unarray", -] - -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - [[package]] name = "quote" -version = "1.0.47" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - [[package]] name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" -dependencies = [ - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom 0.3.4", -] - -[[package]] -name = "rand_xorshift" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" -dependencies = [ - "rand_core", -] - [[package]] name = "redox_syscall" version = "0.5.18" @@ -293,12 +253,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "regex-syntax" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" - [[package]] name = "rustix" version = "1.1.4" @@ -312,18 +266,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "rusty-fork" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" -dependencies = [ - "fnv", - "quick-error", - "tempfile", - "wait-timeout", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -331,27 +273,64 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] -name = "smallvec" -version = "1.15.2" +name = "semver" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] -name = "syn" -version = "2.0.119" +name = "serde" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "unicode-ident", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", ] +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + [[package]] name = "syn" -version = "3.0.3" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -365,7 +344,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom", "once_cell", "rustix", "windows-sys", @@ -373,29 +352,29 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.19" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.19" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn", ] [[package]] name = "tokio" -version = "1.53.1" +version = "1.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "f66bf9585cda4b724d3e78ab34b73fb2bbaba9011b9bfdf69dc836382ea13b8c" dependencies = [ "pin-project-lite", ] @@ -419,7 +398,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -431,12 +410,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "unarray" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" - [[package]] name = "unicode-ident" version = "1.0.24" @@ -444,23 +417,63 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] -name = "wait-timeout" -version = "0.2.1" +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" dependencies = [ - "libc", + "wit-bindgen", ] [[package]] -name = "wasip2" -version = "1.0.4+wasi-0.2.12" +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -478,36 +491,104 @@ dependencies = [ [[package]] name = "wit-bindgen" -version = "0.57.1" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] [[package]] -name = "yenc-simd" -version = "0.1.1" +name = "wit-bindgen-core" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33092857e41bb8d4868df26b7706277a3b1ea9ef44e100b95a85401371a94d7" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" dependencies = [ - "crc32fast", - "thiserror", + "anyhow", + "heck", + "wit-parser", ] [[package]] -name = "zerocopy" -version = "0.8.55" +name = "wit-bindgen-rust" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ - "zerocopy-derive", + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", ] [[package]] -name = "zerocopy-derive" -version = "0.8.55" +name = "wit-bindgen-rust-macro" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" dependencies = [ + "anyhow", + "prettyplease", "proc-macro2", "quote", - "syn 2.0.119", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", ] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "yenc-simd" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9af469088d22edb5c63bd83b7d9f005441330721fbbc122ff393cbcdcaa16a52" +dependencies = [ + "crc32fast", + "thiserror", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml index 13394f8..ab7451c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nzb-decode" -version = "0.1.3" +version = "0.1.4" edition = "2024" description = "yEnc decoding, file assembly, and article cache for NZB download clients" license = "MIT" @@ -10,6 +10,7 @@ readme = "README.md" [dependencies] bytes = "1" crc32fast = "1" +md-5 = "0.11" yenc-simd = "0.1" tracing = "0.1" anyhow = "1" diff --git a/ci/migration-policy.sh b/ci/migration-policy.sh deleted file mode 100755 index e269fd2..0000000 --- a/ci/migration-policy.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -forbidden="$(printf '%s%s%s' 'sp' 'roo' 'ty')" -failures=0 -fail() { printf 'migration policy: %s\n' "$1" >&2; failures=$((failures + 1)); } -if git grep -Iqi -- "$forbidden" HEAD --; then - git grep -Ini -- "$forbidden" HEAD -- >&2 || true - fail 'forbidden legacy identifier found in tracked content' -fi -while IFS= read -r -d '' path; do - if [[ "${path,,}" == *"$forbidden"* ]]; then - printf '%s\n' "$path" >&2 - fail 'forbidden legacy identifier found in tracked path' - fi -done < <(git ls-files -z) -workflow_matches="$(git grep -nE 'runs-on:[[:space:]]*(ubuntu|windows|macos)-|runs-on:[[:space:]]*.*(ubuntu|windows|macos|arm).*latest|runs-on:[[:space:]]*\$\{\{' HEAD -- '.github/workflows/*.yml' '.github/workflows/*.yaml' || true)" -if [[ -n "$workflow_matches" ]]; then - printf '%s\n' "$workflow_matches" >&2 - fail 'workflow selects a hosted or unresolved dynamic runner' -fi -if [[ -n "${GITHUB_BASE_REF:-}" ]]; then - # Fetch the base ref with full history (not --depth=1): a shallow base ref - # grafts away its ancestry, so `origin/BASE..HEAD` can no longer tell that - # the base tip's ancestors are shared with HEAD. On a branch that has merged - # the base in (e.g. after "Update branch"), that made historical base commits - # reappear in the range as false positives. - git fetch --no-tags origin "$GITHUB_BASE_REF" >/dev/null 2>&1 || true - # Scope to commits this PR actually introduces: everything since the branch - # diverged from the base (merge-base), which excludes commits merged in from - # the base itself. - base="$(git merge-base "origin/$GITHUB_BASE_REF" HEAD 2>/dev/null || true)" - commits="$(git rev-list --reverse "${base:+$base..}HEAD" 2>/dev/null || git rev-list --reverse HEAD~1..HEAD 2>/dev/null || git rev-list --reverse HEAD)" -elif [[ "${GITHUB_EVENT_NAME:-}" == push && "${GITHUB_BEFORE:-}" != 0000000000000000000000000000000000000000 ]]; then - commits="$(git rev-list --reverse "${GITHUB_BEFORE:-}..HEAD" 2>/dev/null || git rev-list --reverse HEAD~1..HEAD 2>/dev/null || git rev-list --reverse HEAD)" -else - commits="$(git rev-list --reverse HEAD~1..HEAD 2>/dev/null || git rev-list --reverse HEAD)" -fi -while IFS= read -r commit; do - [[ -n "$commit" ]] || continue - metadata="$(git show -s --format='%an%n%ae%n%cn%n%ce%n%B' "$commit")" - if grep -qi -- "$forbidden" <<<"$metadata"; then - printf '%s\n' "$commit" >&2 - fail 'forbidden legacy identifier found in commit metadata' - fi -done <<<"$commits" -(( failures == 0 )) || exit 1 -printf 'migration policy passed\n' diff --git a/src/lib.rs b/src/lib.rs index 23649a4..029fa23 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,8 +7,10 @@ pub mod assembler; pub mod cache; +pub mod slice_map; pub mod yenc; pub use assembler::FileAssembler; pub use cache::ArticleCache; +pub use slice_map::{FileSlices, SliceDigest, SliceLayout, SliceState, SliceVerifier}; pub use yenc::{YencDecodeResult, decode_yenc}; diff --git a/src/slice_map.rs b/src/slice_map.rs new file mode 100644 index 0000000..0572331 --- /dev/null +++ b/src/slice_map.rs @@ -0,0 +1,315 @@ +//! In-flight PAR2 slice verification (WI-144 core). +//! +//! As decoded articles land in the assembler, the bytes that make up each PAR2 +//! recovery slice arrive too — out of order, at arbitrary offsets, from many +//! workers. This module turns those writes into a per-slice verdict +//! (`Verified` / `Damaged` / `Unknown`) so the post-download PAR2 verify pass +//! can be skipped for a clean job (WI-147) and damage can be compared to +//! recovery blocks during the download (WI-145). +//! +//! MD5 is not associative, so a slice cannot be hashed incrementally from +//! out-of-order fragments. Instead each slice tracks how many of its bytes +//! have been written; once a slice's whole byte range is filled the caller +//! reads it back (from the assembler's file, page-cache-hot) and calls +//! [`SliceVerifier::verify`], which pads the final partial slice with zeros — +//! matching how PAR2 IFSC checksums are computed — and compares MD5 and CRC32. +//! +//! The verifier holds only PAR2 geometry as plain data ([`SliceLayout`]), so +//! this crate needs no PAR2 parser: the download engine builds the layout from +//! a parsed `rust-par2` file set and the NZB-file-to-PAR2-file mapping. + +use crc32fast::Hasher as Crc32; +use md5::{Digest, Md5}; + +/// A 16-byte MD5 digest. +pub type Md5Hash = [u8; 16]; + +/// The verdict for one slice. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SliceState { + /// Not yet fully written, or written but not yet checked. + Unknown, + /// Fully written and its MD5 and CRC32 match the PAR2 checksums. + Verified, + /// Fully written but a checksum mismatched, or a byte is known missing. + Damaged, +} + +/// The expected checksums for one slice, from a PAR2 IFSC packet. +#[derive(Debug, Clone, Copy)] +pub struct SliceDigest { + pub md5: Md5Hash, + pub crc32: u32, +} + +/// One file's slice checksums, in slice order, with the file's total size. +#[derive(Debug, Clone)] +pub struct FileSlices { + pub size: u64, + pub slices: Vec, +} + +/// PAR2 geometry the verifier needs, in the recovery set's file order. Built by +/// the caller from a parsed `rust-par2` `Par2FileSet` (its `slice_size`, +/// `file_order`, and each file's IFSC `slices`). +#[derive(Debug, Clone)] +pub struct SliceLayout { + pub slice_size: u64, + pub files: Vec, +} + +#[derive(Debug)] +struct SliceCell { + file_index: usize, + /// Byte offset of this slice within its file. + start: u64, + /// Actual byte length (the last slice of a file may be shorter). + len: u64, + /// Bytes written into this slice's range so far. + filled: u64, + expected: SliceDigest, + state: SliceState, +} + +/// Tracks slice fill as writes arrive and verifies each slice once complete. +#[derive(Debug)] +pub struct SliceVerifier { + slice_size: u64, + cells: Vec, + /// Global index of the first slice of each file, plus a trailing sentinel + /// equal to `cells.len()` so `[file_index]..[file_index + 1]` is the file's + /// slice range for any valid `file_index`. + file_bounds: Vec, +} + +fn overlap(a0: u64, a1: u64, b0: u64, b1: u64) -> u64 { + a1.min(b1).saturating_sub(a0.max(b0)) +} + +impl SliceVerifier { + /// Build a verifier for a PAR2 set. Every slice starts `Unknown`. + pub fn new(layout: &SliceLayout) -> Self { + let mut cells = Vec::new(); + let mut file_bounds = Vec::with_capacity(layout.files.len() + 1); + for (file_index, file) in layout.files.iter().enumerate() { + file_bounds.push(cells.len()); + for (i, digest) in file.slices.iter().enumerate() { + let start = i as u64 * layout.slice_size; + let len = file.size.saturating_sub(start).min(layout.slice_size); + cells.push(SliceCell { + file_index, + start, + len, + filled: 0, + expected: *digest, + state: SliceState::Unknown, + }); + } + } + file_bounds.push(cells.len()); + Self { + slice_size: layout.slice_size, + cells, + file_bounds, + } + } + + /// Record that `len` bytes were written to `file_index` at `offset`. + /// Returns the global indices of any slices that became fully filled with + /// this write and are ready to [`verify`](Self::verify). + pub fn mark_written(&mut self, file_index: usize, offset: u64, len: u64) -> Vec { + let mut completed = Vec::new(); + let Some(&start_g) = self.file_bounds.get(file_index) else { + return completed; + }; + let end_g = self.file_bounds[file_index + 1]; + let (w0, w1) = (offset, offset + len); + for gi in start_g..end_g { + let cell = &mut self.cells[gi]; + if cell.state != SliceState::Unknown { + continue; + } + let ov = overlap(cell.start, cell.start + cell.len, w0, w1); + if ov == 0 { + continue; + } + cell.filled = (cell.filled + ov).min(cell.len); + if cell.filled == cell.len && cell.len > 0 { + completed.push(gi); + } + } + completed + } + + /// Which file and byte range a global slice covers, so the caller can read + /// the slice's bytes back to verify it. Returns `(file_index, offset, len)`. + pub fn slice_range(&self, global_index: usize) -> Option<(usize, u64, u64)> { + self.cells + .get(global_index) + .map(|c| (c.file_index, c.start, c.len)) + } + + /// Verify a completed slice against its PAR2 checksums. `bytes` is the + /// slice's actual bytes (`len` from [`slice_range`](Self::slice_range)); + /// they are zero-padded to the slice size before hashing, as PAR2 does. + pub fn verify(&mut self, global_index: usize, bytes: &[u8]) -> SliceState { + let slice_size = self.slice_size as usize; + let Some(cell) = self.cells.get_mut(global_index) else { + return SliceState::Unknown; + }; + let mut padded = bytes.to_vec(); + padded.resize(slice_size, 0); + + let md5: Md5Hash = Md5::digest(&padded).into(); + let mut crc = Crc32::new(); + crc.update(&padded); + let crc = crc.finalize(); + + cell.state = if md5 == cell.expected.md5 && crc == cell.expected.crc32 { + SliceState::Verified + } else { + SliceState::Damaged + }; + cell.state + } + + /// Mark a slice damaged without hashing — used when the ledger confirms an + /// article overlapping the slice is missing. + pub fn mark_damaged(&mut self, global_index: usize) { + if let Some(cell) = self.cells.get_mut(global_index) { + cell.state = SliceState::Damaged; + } + } + + /// The verdict of every slice, in global index order. + pub fn states(&self) -> Vec { + self.cells.iter().map(|c| c.state).collect() + } + + /// Total number of slices in the set. + pub fn total_slices(&self) -> usize { + self.cells.len() + } + + /// Number of slices currently marked `Damaged`. + pub fn damaged_count(&self) -> usize { + self.cells + .iter() + .filter(|c| c.state == SliceState::Damaged) + .count() + } + + /// True once no slice is still `Unknown` — the map is decided. + pub fn is_complete(&self) -> bool { + self.cells.iter().all(|c| c.state != SliceState::Unknown) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn digest_of(slice: &[u8], slice_size: usize) -> SliceDigest { + let mut padded = slice.to_vec(); + padded.resize(slice_size, 0); + let md5: Md5Hash = Md5::digest(&padded).into(); + let mut crc = Crc32::new(); + crc.update(&padded); + SliceDigest { + md5, + crc32: crc.finalize(), + } + } + + /// Build a one-file layout plus the payload for `size` bytes at + /// `slice_size`, with correct per-slice checksums. + fn one_file(size: usize, slice_size: usize) -> (SliceLayout, Vec) { + let payload: Vec = (0..size).map(|i| (i % 251) as u8).collect(); + let slices = payload + .chunks(slice_size) + .map(|c| digest_of(c, slice_size)) + .collect(); + let layout = SliceLayout { + slice_size: slice_size as u64, + files: vec![FileSlices { + size: size as u64, + slices, + }], + }; + (layout, payload) + } + + #[test] + fn in_order_writes_verify_all_slices() { + let (layout, payload) = one_file(250, 100); // slices of 100, 100, 50 + let mut v = SliceVerifier::new(&layout); + assert_eq!(v.total_slices(), 3); + + // Three articles, one per slice, in order. + for (i, chunk) in payload.chunks(100).enumerate() { + let completed = v.mark_written(0, (i * 100) as u64, chunk.len() as u64); + assert_eq!(completed, vec![i]); + let (_f, off, len) = v.slice_range(i).unwrap(); + assert_eq!( + v.verify(i, &payload[off as usize..(off + len) as usize]), + SliceState::Verified + ); + } + assert!(v.is_complete()); + assert_eq!(v.damaged_count(), 0); + assert_eq!(v.states(), vec![SliceState::Verified; 3]); + } + + #[test] + fn article_spanning_two_slices_and_out_of_order() { + let (layout, payload) = one_file(300, 100); + let mut v = SliceVerifier::new(&layout); + + // A write covering [50, 250) straddles slices 0, 1 and 2 but completes + // only the fully covered middle slice 1. + assert_eq!(v.mark_written(0, 50, 200), vec![1]); + // Fill the head of slice 0 and the tail of slice 2, out of order. + assert_eq!(v.mark_written(0, 250, 50), vec![2]); + assert_eq!(v.mark_written(0, 0, 50), vec![0]); + + for gi in 0..3 { + let (_f, off, len) = v.slice_range(gi).unwrap(); + assert_eq!( + v.verify(gi, &payload[off as usize..(off + len) as usize]), + SliceState::Verified + ); + } + assert!(v.is_complete()); + } + + #[test] + fn corrupt_bytes_mark_slice_damaged() { + let (layout, payload) = one_file(200, 100); + let mut v = SliceVerifier::new(&layout); + + v.mark_written(0, 0, 100); + let mut corrupt = payload[0..100].to_vec(); + corrupt[0] ^= 0xff; + assert_eq!(v.verify(0, &corrupt), SliceState::Damaged); + assert_eq!(v.damaged_count(), 1); + assert!(!v.is_complete()); // slice 1 still Unknown + } + + #[test] + fn partial_fill_does_not_complete() { + let (layout, _payload) = one_file(200, 100); + let mut v = SliceVerifier::new(&layout); + // Only half of slice 0. + assert!(v.mark_written(0, 0, 50).is_empty()); + assert_eq!(v.states()[0], SliceState::Unknown); + } + + #[test] + fn mark_damaged_sets_state_directly() { + let (layout, _payload) = one_file(150, 100); + let mut v = SliceVerifier::new(&layout); + v.mark_damaged(1); + assert_eq!(v.states()[1], SliceState::Damaged); + assert_eq!(v.damaged_count(), 1); + } +}