From 908c5e05b5dc8eef012887511204f7847d492a6d Mon Sep 17 00:00:00 2001 From: thedancingdeveloper <306930456+thedancingdeveloper@users.noreply.github.com> Date: Thu, 10 Sep 2026 05:48:40 +0000 Subject: [PATCH] sync: nzb-web from rustnzb monorepo 8bd1257 (v0.4.22) --- .codesight/CODESIGHT.md | 28 - .codesight/config.md | 5 - .codesight/middleware.md | 7 - .github/CODEOWNERS | 1 - .github/dependabot.yml | 16 - .github/workflows/ci.yml | 25 - .github/workflows/policy.yml | 23 - .github/workflows/runner-policy.yml | 16 - .woodpecker.yml | 114 - Cargo.toml | 23 +- ci/migration-policy.sh | 47 - src/auth.rs | 118 +- src/dir_watcher.rs | 86 +- src/error.rs | 11 +- src/fetch_guard.rs | 303 ++ src/lib.rs | 3 + src/queue_manager.rs | 1215 +++++++- src/rss_monitor.rs | 148 +- src/sabnzbd_compat.rs | 2586 ++++++++++++++++-- src/startup.rs | 91 +- tests/fixtures/sabnzbd-5.0.4/README.md | 24 + tests/fixtures/sabnzbd-5.0.4/fullstatus.json | 60 + tests/fixtures/sabnzbd-5.0.4/history.json | 48 + tests/fixtures/sabnzbd-5.0.4/queue.json | 60 + tests/fixtures/sabnzbd-5.0.4/version.json | 3 + tests/harness/mod.rs | 140 +- tests/harness/nzb_fixture.rs | 65 + tests/harness_catalog.rs | 36 + tests/harness_failure_matrix.rs | 257 ++ tests/harness_priority.rs | 95 +- tests/startup_directory_errors.rs | 93 + tests/support/mod.rs | 1 + tests/support/sab_contract.rs | 99 + tests/workflow_fixtures.rs | 83 + 34 files changed, 5210 insertions(+), 720 deletions(-) delete mode 100644 .codesight/CODESIGHT.md delete mode 100644 .codesight/config.md delete mode 100644 .codesight/middleware.md delete mode 100644 .github/CODEOWNERS delete mode 100644 .github/dependabot.yml delete mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/policy.yml delete mode 100644 .github/workflows/runner-policy.yml delete mode 100644 .woodpecker.yml delete mode 100755 ci/migration-policy.sh create mode 100644 src/fetch_guard.rs create mode 100644 tests/fixtures/sabnzbd-5.0.4/README.md create mode 100644 tests/fixtures/sabnzbd-5.0.4/fullstatus.json create mode 100644 tests/fixtures/sabnzbd-5.0.4/history.json create mode 100644 tests/fixtures/sabnzbd-5.0.4/queue.json create mode 100644 tests/fixtures/sabnzbd-5.0.4/version.json create mode 100644 tests/harness_catalog.rs create mode 100644 tests/harness_failure_matrix.rs create mode 100644 tests/startup_directory_errors.rs create mode 100644 tests/support/mod.rs create mode 100644 tests/support/sab_contract.rs create mode 100644 tests/workflow_fixtures.rs diff --git a/.codesight/CODESIGHT.md b/.codesight/CODESIGHT.md deleted file mode 100644 index 318e1a9..0000000 --- a/.codesight/CODESIGHT.md +++ /dev/null @@ -1,28 +0,0 @@ -# nzb-web — AI Context Map - -> **Stack:** axum | none | unknown | rust - -> 0 routes | 0 models | 0 components | 0 lib files | 0 env vars | 4 middleware -> **Token savings:** this file is ~200 tokens. Without it, AI exploration would cost ~6,200 tokens. **Saves ~6,100 tokens per conversation.** - ---- - -# Config - -## Config Files - -- `Cargo.toml` - ---- - -# Middleware - -## auth -- auth — `src/auth.rs` -- auth — `target/package/nzb-web-0.1.1/src/auth.rs` -- auth — `target/package/nzb-web-0.1.2/src/auth.rs` -- auth — `target/package/nzb-web-0.1.3/src/auth.rs` - ---- - -_Generated by [codesight](https://github.com/Houseofmvps/codesight) — see your codebase clearly_ \ No newline at end of file diff --git a/.codesight/config.md b/.codesight/config.md deleted file mode 100644 index 736f013..0000000 --- a/.codesight/config.md +++ /dev/null @@ -1,5 +0,0 @@ -# Config - -## Config Files - -- `Cargo.toml` diff --git a/.codesight/middleware.md b/.codesight/middleware.md deleted file mode 100644 index 335b0eb..0000000 --- a/.codesight/middleware.md +++ /dev/null @@ -1,7 +0,0 @@ -# Middleware - -## auth -- auth — `src/auth.rs` -- auth — `target/package/nzb-web-0.1.1/src/auth.rs` -- auth — `target/package/nzb-web-0.1.2/src/auth.rs` -- auth — `target/package/nzb-web-0.1.3/src/auth.rs` diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index 1d91edc..0000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -* @TheDancingDeveloper-org/migration-maintainers diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 74cb186..0000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: 2 -updates: - - package-ecosystem: cargo - directory: / - schedule: - interval: weekly - open-pull-requests-limit: 10 - groups: - rust-minor: - update-types: [minor, patch] - - - package-ecosystem: github-actions - directory: / - schedule: - interval: weekly - open-pull-requests-limit: 5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 3932bdc..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: CI - -on: - push: - branches: [main] - pull_request: - workflow_dispatch: - -permissions: - contents: read - -jobs: - rust: - # Dependabot metadata PRs do not consume the self-hosted validation runner. - if: github.event_name != 'pull_request' || github.event.pull_request.user.type != 'Bot' - runs-on: [self-hosted] - steps: - - uses: actions/checkout@v7 - - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt,clippy - - run: cargo fmt --all -- --check - - run: cargo check --workspace - - run: cargo test --workspace - - run: cargo clippy --workspace --all-targets -- -D warnings diff --git a/.github/workflows/policy.yml b/.github/workflows/policy.yml deleted file mode 100644 index 8e9b508..0000000 --- a/.github/workflows/policy.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Migration policy - -on: - push: - branches: [main] - pull_request: - workflow_dispatch: - -permissions: - contents: read - -jobs: - policy: - # Dependabot metadata PRs do not consume the self-hosted policy runner. - if: github.event_name != 'pull_request' || github.event.pull_request.user.type != 'Bot' - runs-on: [self-hosted] - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: Verify migration policies - shell: bash - run: bash ci/migration-policy.sh diff --git a/.github/workflows/runner-policy.yml b/.github/workflows/runner-policy.yml deleted file mode 100644 index 65a1c1f..0000000 --- a/.github/workflows/runner-policy.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Copy this file to .github/workflows/runner-policy.yml in every repository. -# -# It is deliberately tiny: all logic lives in the reusable workflow in -# github-policy, so the rule can be changed in one place. After adding it, make -# `runner-policy` a required status check on the repository's protected branch -# — without that it reports but does not block. -name: Runner policy - -on: - pull_request: - push: - branches: [main] - -jobs: - runner-policy: - uses: TheDancingDeveloper-org/github-policy/.github/workflows/runner-policy-reusable.yml@main \ No newline at end of file diff --git a/.woodpecker.yml b/.woodpecker.yml deleted file mode 100644 index 0142681..0000000 --- a/.woodpecker.yml +++ /dev/null @@ -1,114 +0,0 @@ -when: - - event: push - branch: main - - event: manual - - event: pull_request - -steps: - - name: fmt - image: rust:1.94-bookworm - environment: - GIT_AUTH_TOKEN: - from_secret: git_auth_token - commands: - - git config --global url."http://x-access-token:$GIT_AUTH_TOKEN@100.92.54.45:3002/".insteadOf "http://100.92.54.45:3002/" - - printf '[registries.forgejo]\nindex = "sparse+https://repo.indexarr.net/api/packages/indexarr/cargo/"\ncredential-provider = "cargo:token"\n\n[registry]\ndefault = "forgejo"\n' > $CARGO_HOME/config.toml - - printf '[registries.forgejo]\ntoken = "Bearer %s"\n' "$GIT_AUTH_TOKEN" > $CARGO_HOME/credentials.toml - - rustup component add rustfmt - - cargo fmt --all -- --check - - - name: check - image: rust:1.94-bookworm - environment: - GIT_AUTH_TOKEN: - from_secret: git_auth_token - commands: - - git config --global url."http://x-access-token:$GIT_AUTH_TOKEN@100.92.54.45:3002/".insteadOf "http://100.92.54.45:3002/" - - printf '[registries.forgejo]\nindex = "sparse+https://repo.indexarr.net/api/packages/indexarr/cargo/"\ncredential-provider = "cargo:token"\n\n[registry]\ndefault = "forgejo"\n' > $CARGO_HOME/config.toml - - printf '[registries.forgejo]\ntoken = "Bearer %s"\n' "$GIT_AUTH_TOKEN" > $CARGO_HOME/credentials.toml - - cargo check --workspace - - - name: test - image: rust:1.94-bookworm - environment: - GIT_AUTH_TOKEN: - from_secret: git_auth_token - commands: - - git config --global url."http://x-access-token:$GIT_AUTH_TOKEN@100.92.54.45:3002/".insteadOf "http://100.92.54.45:3002/" - - printf '[registries.forgejo]\nindex = "sparse+https://repo.indexarr.net/api/packages/indexarr/cargo/"\ncredential-provider = "cargo:token"\n\n[registry]\ndefault = "forgejo"\n' > $CARGO_HOME/config.toml - - printf '[registries.forgejo]\ntoken = "Bearer %s"\n' "$GIT_AUTH_TOKEN" > $CARGO_HOME/credentials.toml - - cargo test --workspace - - - name: clippy - image: rust:1.94-bookworm - environment: - GIT_AUTH_TOKEN: - from_secret: git_auth_token - commands: - - git config --global url."http://x-access-token:$GIT_AUTH_TOKEN@100.92.54.45:3002/".insteadOf "http://100.92.54.45:3002/" - - printf '[registries.forgejo]\nindex = "sparse+https://repo.indexarr.net/api/packages/indexarr/cargo/"\ncredential-provider = "cargo:token"\n\n[registry]\ndefault = "forgejo"\n' > $CARGO_HOME/config.toml - - printf '[registries.forgejo]\ntoken = "Bearer %s"\n' "$GIT_AUTH_TOKEN" > $CARGO_HOME/credentials.toml - - rustup component add clippy - - cargo clippy --workspace -- -D warnings - - - name: publish - image: rust:1.94-bookworm - environment: - GIT_AUTH_TOKEN: - from_secret: git_auth_token - commands: - - printf '[registries.forgejo]\nindex = "sparse+https://repo.indexarr.net/api/packages/indexarr/cargo/"\ncredential-provider = "cargo:token"\n\n[registry]\ndefault = "forgejo"\n' > $CARGO_HOME/config.toml - - printf '[registries.forgejo]\ntoken = "Bearer %s"\n' "$GIT_AUTH_TOKEN" > $CARGO_HOME/credentials.toml - - cargo publish --allow-dirty 2>&1 || echo "Publish skipped (version may already exist)" - when: - - event: push - branch: main - - - name: publish-crates-io - image: rust:1.94-bookworm - environment: - CARGO_CRATES_IO_TOKEN: - from_secret: cargo_crates_io_token - commands: - - printf '[registries.crates-io]\ntoken = "%s"\n' "$CARGO_CRATES_IO_TOKEN" > $CARGO_HOME/credentials.toml - - sed -i 's/, registry = "forgejo"//g; s/registry = "forgejo", //g' Cargo.toml - - rm -f Cargo.lock - - cargo publish --registry crates-io --allow-dirty --no-verify 2>&1 || echo "crates.io publish skipped (version may already exist)" - when: - - event: push - branch: main - - # trigger-renovate step removed — the `woodpecker_token` org secret - # isn't configured on this Woodpecker instance, which put the whole - # pipeline into ERROR state at validation time (secret resolves BEFORE - # any step runs). Publish to Forgejo still happens; renovate runs on - # its own schedule. Re-add this step once a machine token is - # provisioned under that name. - - - name: discord-success - image: alpine/curl - environment: - DISCORD_WEBHOOK_URL: - from_secret: discord_webhook_url - commands: - - | - curl -s -X POST "$DISCORD_WEBHOOK_URL" \ - -H "Content-Type: application/json" \ - -d "{\"embeds\":[{\"title\":\"nzb-web published\",\"description\":\"Published to Forgejo registry.\",\"color\":3066993}]}" - when: - - event: push - branch: main - status: [success] - - - name: discord-failure - image: alpine/curl - environment: - DISCORD_WEBHOOK_URL: - from_secret: discord_webhook_url - commands: - - | - curl -s -X POST "$DISCORD_WEBHOOK_URL" \ - -H "Content-Type: application/json" \ - -d "{\"embeds\":[{\"title\":\"nzb-web build failed\",\"description\":\"Check CI logs for details\",\"color\":15158332}]}" - when: - - status: [failure] diff --git a/Cargo.toml b/Cargo.toml index 77a2b07..b3ef4fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nzb-web" -version = "0.4.21" +version = "0.4.22" edition = "2024" description = "Usenet download engine: queue management, download orchestration, and background services" license = "MIT" @@ -11,13 +11,13 @@ default = [] groups-db = ["nzb-postproc/groups-db"] [dependencies] -nzb-nntp = { version = "0.2.23" } -nzb-decode = { version = "0.1.3" } -nzb-postproc = { version = "0.2.7" } -nzb-dispatch = { version = "0.2.7" } +nzb-nntp = { version = "0.2.24", path = "../nzb-nntp" } +nzb-decode = { version = "0.1.4", path = "../nzb-decode" } +nzb-postproc = { version = "0.2.8", path = "../nzb-postproc" } +nzb-dispatch = { version = "0.2.8", path = "../nzb-dispatch" } axum = { version = "0.8", features = ["multipart"] } tower = "0.5" -tower-http = { version = "0.6", features = ["cors", "trace"] } +tower-http = { version = "0.7", features = ["cors", "trace"] } utoipa = { version = "5", features = ["axum_extras"] } utoipa-swagger-ui = { version = "9", features = ["axum"] } http = "1" @@ -25,16 +25,16 @@ tokio = { version = "1", features = ["full"] } serde = { version = "1", features = ["derive"] } serde_json = "1" tracing = "0.1" -opentelemetry = "0.28" +opentelemetry.workspace = true tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "registry"] } anyhow = "1" async-trait = "0.1" thiserror = "2" parking_lot = "0.12" -rand = "0.9" +rand = "0.10" hex = "0.4" -base64 = "0.22" -sha2 = "0.10" +base64 = "0.23" +sha2 = "0.11" governor = "0.10" arc-swap = "1" uuid = { version = "1", features = ["v4", "serde"] } @@ -45,6 +45,7 @@ feed-rs = "2" regex = "1" libc = "0.2" unicode-normalization = "0.1" +flate2.workspace = true [lints.clippy] all = { level = "warn", priority = -1 } @@ -54,5 +55,5 @@ unused = "warn" [dev-dependencies] tempfile = "3.27.0" -nzb-nntp = { version = "0.2.23", features = ["test-support"] } +nzb-nntp = { version = "0.2.24", path = "../nzb-nntp", features = ["test-support"] } yenc-simd = { version = "0.1" } diff --git a/ci/migration-policy.sh b/ci/migration-policy.sh deleted file mode 100755 index e269fd2..0000000 --- a/ci/migration-policy.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -forbidden="$(printf '%s%s%s' 'sp' 'roo' 'ty')" -failures=0 -fail() { printf 'migration policy: %s\n' "$1" >&2; failures=$((failures + 1)); } -if git grep -Iqi -- "$forbidden" HEAD --; then - git grep -Ini -- "$forbidden" HEAD -- >&2 || true - fail 'forbidden legacy identifier found in tracked content' -fi -while IFS= read -r -d '' path; do - if [[ "${path,,}" == *"$forbidden"* ]]; then - printf '%s\n' "$path" >&2 - fail 'forbidden legacy identifier found in tracked path' - fi -done < <(git ls-files -z) -workflow_matches="$(git grep -nE 'runs-on:[[:space:]]*(ubuntu|windows|macos)-|runs-on:[[:space:]]*.*(ubuntu|windows|macos|arm).*latest|runs-on:[[:space:]]*\$\{\{' HEAD -- '.github/workflows/*.yml' '.github/workflows/*.yaml' || true)" -if [[ -n "$workflow_matches" ]]; then - printf '%s\n' "$workflow_matches" >&2 - fail 'workflow selects a hosted or unresolved dynamic runner' -fi -if [[ -n "${GITHUB_BASE_REF:-}" ]]; then - # Fetch the base ref with full history (not --depth=1): a shallow base ref - # grafts away its ancestry, so `origin/BASE..HEAD` can no longer tell that - # the base tip's ancestors are shared with HEAD. On a branch that has merged - # the base in (e.g. after "Update branch"), that made historical base commits - # reappear in the range as false positives. - git fetch --no-tags origin "$GITHUB_BASE_REF" >/dev/null 2>&1 || true - # Scope to commits this PR actually introduces: everything since the branch - # diverged from the base (merge-base), which excludes commits merged in from - # the base itself. - base="$(git merge-base "origin/$GITHUB_BASE_REF" HEAD 2>/dev/null || true)" - commits="$(git rev-list --reverse "${base:+$base..}HEAD" 2>/dev/null || git rev-list --reverse HEAD~1..HEAD 2>/dev/null || git rev-list --reverse HEAD)" -elif [[ "${GITHUB_EVENT_NAME:-}" == push && "${GITHUB_BEFORE:-}" != 0000000000000000000000000000000000000000 ]]; then - commits="$(git rev-list --reverse "${GITHUB_BEFORE:-}..HEAD" 2>/dev/null || git rev-list --reverse HEAD~1..HEAD 2>/dev/null || git rev-list --reverse HEAD)" -else - commits="$(git rev-list --reverse HEAD~1..HEAD 2>/dev/null || git rev-list --reverse HEAD)" -fi -while IFS= read -r commit; do - [[ -n "$commit" ]] || continue - metadata="$(git show -s --format='%an%n%ae%n%cn%n%ce%n%B' "$commit")" - if grep -qi -- "$forbidden" <<<"$metadata"; then - printf '%s\n' "$commit" >&2 - fail 'forbidden legacy identifier found in commit metadata' - fi -done <<<"$commits" -(( failures == 0 )) || exit 1 -printf 'migration policy passed\n' diff --git a/src/auth.rs b/src/auth.rs index dbc1cd3..790ab4a 100644 --- a/src/auth.rs +++ b/src/auth.rs @@ -3,6 +3,13 @@ use std::path::PathBuf; use std::sync::Arc; use std::time::{Duration, Instant}; +#[cfg(unix)] +use std::fs::File; +#[cfg(unix)] +use std::io::Write; +#[cfg(unix)] +use std::os::fd::{AsRawFd, FromRawFd}; + use parking_lot::RwLock; use serde::{Deserialize, Serialize}; @@ -111,6 +118,12 @@ impl TokenStore { self.refresh_tokens.write().remove(refresh_token); } + /// Revoke every session after credentials change. + pub fn revoke_all(&self) { + self.access_tokens.write().clear(); + self.refresh_tokens.write().clear(); + } + pub fn cleanup_expired(&self) { let now = Instant::now(); self.access_tokens @@ -132,12 +145,26 @@ pub struct StoredCredentials { pub struct CredentialStore { credentials: RwLock>, + #[cfg(not(unix))] file_path: PathBuf, + #[cfg(unix)] + directory: File, } impl CredentialStore { pub fn new(config_dir: PathBuf) -> Self { + // Startup creates the configured data directory before constructing + // this store. Canonicalizing it here confines the credential file to + // that existing directory and removes traversal or symlinked-parent + // ambiguity from the subsequent writes. + let config_dir = config_dir.canonicalize().unwrap_or_else(|error| { + panic!("credential store data directory must exist before startup: {error}") + }); let file_path = config_dir.join("credentials.json"); + #[cfg(unix)] + let directory = File::open(&config_dir).unwrap_or_else(|error| { + panic!("credential store data directory must be readable: {error}") + }); let credentials = if file_path.exists() { match std::fs::read_to_string(&file_path) { Ok(contents) => serde_json::from_str(&contents).ok(), @@ -148,7 +175,10 @@ impl CredentialStore { }; Self { credentials: RwLock::new(credentials), + #[cfg(not(unix))] file_path, + #[cfg(unix)] + directory, } } @@ -160,23 +190,61 @@ impl CredentialStore { self.credentials.read().clone() } - pub fn set_credentials(&self, creds: StoredCredentials) -> Result<(), std::io::Error> { - let json = serde_json::to_string_pretty(&creds).map_err(std::io::Error::other)?; - // Create parent directory if needed - if let Some(parent) = self.file_path.parent() { - std::fs::create_dir_all(parent)?; - } - std::fs::write(&self.file_path, &json)?; - // Set file permissions to owner-only on unix + fn persist(&self, json: &[u8]) -> Result<(), std::io::Error> { #[cfg(unix)] { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&self.file_path, std::fs::Permissions::from_mode(0o600))?; + // The directory handle is opened from the canonical data + // directory at startup. The fixed filename never comes from a + // request or configuration value, and O_NOFOLLOW prevents a + // pre-existing credentials symlink from redirecting the write. + let flags = + libc::O_WRONLY | libc::O_CREAT | libc::O_TRUNC | libc::O_CLOEXEC | libc::O_NOFOLLOW; + let fd = unsafe { + libc::openat( + self.directory.as_raw_fd(), + c"credentials.json".as_ptr(), + flags, + 0o600, + ) + }; + if fd < 0 { + return Err(std::io::Error::last_os_error()); + } + let mut file = unsafe { File::from_raw_fd(fd) }; + file.write_all(json)?; + if unsafe { libc::fchmod(file.as_raw_fd(), 0o600) } != 0 { + return Err(std::io::Error::last_os_error()); + } + file.sync_all() } + + #[cfg(not(unix))] + std::fs::write(&self.file_path, json) + } + + pub fn set_credentials(&self, creds: StoredCredentials) -> Result<(), std::io::Error> { + let json = serde_json::to_string_pretty(&creds).map_err(std::io::Error::other)?; + self.persist(json.as_bytes())?; *self.credentials.write() = Some(creds); Ok(()) } + /// Set credentials exactly once. The check and write are serialized so + /// two first-boot setup requests cannot race into different accounts. + pub fn initialize_credentials(&self, creds: StoredCredentials) -> Result<(), std::io::Error> { + let mut current = self.credentials.write(); + if current.is_some() { + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "credentials already configured", + )); + } + let json = serde_json::to_string_pretty(&creds).map_err(std::io::Error::other)?; + self.persist(json.as_bytes())?; + *current = Some(creds); + Ok(()) + } + pub fn validate(&self, username: &str, password: &str) -> bool { match &*self.credentials.read() { Some(creds) => { @@ -238,11 +306,6 @@ pub async fn h_auth_setup( State(state): State, Json(req): Json, ) -> impl IntoResponse { - // Only allow if no credentials exist yet - if state.credential_store.has_credentials() { - return (StatusCode::FORBIDDEN, "credentials already configured").into_response(); - } - if req.username.is_empty() || req.password.is_empty() { return ( StatusCode::BAD_REQUEST, @@ -251,15 +314,20 @@ pub async fn h_auth_setup( .into_response(); } - match state.credential_store.set_credentials(StoredCredentials { - username: req.username, - password: req.password, - }) { + match state + .credential_store + .initialize_credentials(StoredCredentials { + username: req.username, + password: req.password, + }) { Ok(_) => { // Create tokens for the new user so they're immediately logged in let tokens = state.token_store.create_tokens(); (StatusCode::OK, Json(tokens)).into_response() } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + (StatusCode::FORBIDDEN, "credentials are already configured").into_response() + } Err(e) => ( StatusCode::INTERNAL_SERVER_ERROR, format!("failed to save credentials: {e}"), @@ -300,9 +368,19 @@ pub async fn h_auth_change_credentials( username: req.new_username.unwrap_or(current_creds.username), password: req.new_password.unwrap_or(current_creds.password), }; + if new_creds.username.is_empty() || new_creds.password.is_empty() { + return ( + StatusCode::BAD_REQUEST, + "username and password cannot be empty", + ) + .into_response(); + } match state.credential_store.set_credentials(new_creds) { - Ok(_) => StatusCode::OK.into_response(), + Ok(_) => { + state.token_store.revoke_all(); + StatusCode::OK.into_response() + } Err(e) => ( StatusCode::INTERNAL_SERVER_ERROR, format!("failed to save credentials: {e}"), diff --git a/src/dir_watcher.rs b/src/dir_watcher.rs index 8d9dae2..30ae3d9 100644 --- a/src/dir_watcher.rs +++ b/src/dir_watcher.rs @@ -1,12 +1,16 @@ +use std::io::Read as _; use std::path::{Path, PathBuf}; use std::sync::Arc; +use flate2::read::GzDecoder; use notify::{Event, EventKind, RecursiveMode, Watcher}; use tokio::sync::mpsc; use tracing::{error, info, warn}; use crate::queue_manager::QueueManager; +const MAX_WATCHED_NZB_BYTES: usize = 100 * 1024 * 1024; + pub struct DirWatcher { watch_dir: PathBuf, queue_manager: Arc, @@ -69,8 +73,11 @@ impl DirWatcher { } fn is_nzb_file(path: &Path) -> bool { - path.extension().is_some_and(|ext| ext == "nzb") - || path.to_str().is_some_and(|s| s.ends_with(".nzb.gz")) + path.extension().is_some_and(|ext| ext == "nzb") || Self::is_gz_nzb(path) + } + + fn is_gz_nzb(path: &Path) -> bool { + path.to_str().is_some_and(|s| s.ends_with(".nzb.gz")) } async fn process_existing_files(&self) { @@ -93,7 +100,7 @@ impl DirWatcher { async fn process_file(&self, path: &Path) { info!(file = %path.display(), "Processing NZB from watch directory"); - let data = match std::fs::read(path) { + let raw_data = match Self::read_limited(path) { Ok(d) => d, Err(e) => { warn!(error = %e, file = %path.display(), "Failed to read NZB file"); @@ -101,11 +108,37 @@ impl DirWatcher { } }; - let name = path - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or("unknown") - .to_string(); + let data = if Self::is_gz_nzb(path) { + let decoder = GzDecoder::new(raw_data.as_slice()); + let mut decompressed = Vec::new(); + if let Err(error) = decoder + .take((MAX_WATCHED_NZB_BYTES as u64).saturating_add(1)) + .read_to_end(&mut decompressed) + { + warn!(error = %error, file = %path.display(), "Failed to decompress watched NZB"); + return; + } + if decompressed.len() > MAX_WATCHED_NZB_BYTES { + warn!(file = %path.display(), limit = MAX_WATCHED_NZB_BYTES, "Decompressed watched NZB exceeds the input limit"); + return; + } + decompressed + } else { + raw_data + }; + + let name = if Self::is_gz_nzb(path) { + path.file_name() + .and_then(|name| name.to_str()) + .and_then(|name| name.strip_suffix(".nzb.gz")) + .unwrap_or("unknown") + .to_string() + } else { + path.file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("unknown") + .to_string() + }; match crate::nzb_core::nzb_parser::parse_nzb(&name, &data) { Ok(mut job) => { @@ -142,6 +175,34 @@ impl DirWatcher { } } } + + fn read_limited(path: &Path) -> std::io::Result> { + let metadata = std::fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "watched NZB symlinks are not supported", + )); + } + if metadata.len() > MAX_WATCHED_NZB_BYTES as u64 { + return Err(std::io::Error::new( + std::io::ErrorKind::FileTooLarge, + format!("watched NZB exceeds the {MAX_WATCHED_NZB_BYTES} byte limit"), + )); + } + + let file = std::fs::File::open(path)?; + let mut data = Vec::new(); + file.take((MAX_WATCHED_NZB_BYTES as u64).saturating_add(1)) + .read_to_end(&mut data)?; + if data.len() > MAX_WATCHED_NZB_BYTES { + return Err(std::io::Error::new( + std::io::ErrorKind::FileTooLarge, + format!("watched NZB exceeds the {MAX_WATCHED_NZB_BYTES} byte limit"), + )); + } + Ok(data) + } } #[cfg(test)] @@ -155,4 +216,13 @@ mod tests { assert!(!DirWatcher::is_nzb_file(Path::new("release.NZB"))); assert!(!DirWatcher::is_nzb_file(Path::new("release.txt"))); } + + #[test] + fn bounded_file_reader_rejects_oversized_input() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("oversized.nzb"); + std::fs::write(&path, vec![b'x'; MAX_WATCHED_NZB_BYTES + 1]).unwrap(); + let error = DirWatcher::read_limited(&path).unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::FileTooLarge); + } } diff --git a/src/error.rs b/src/error.rs index 49230f2..839de9d 100644 --- a/src/error.rs +++ b/src/error.rs @@ -166,8 +166,17 @@ impl std::fmt::Display for ApiError { impl IntoResponse for ApiError { fn into_response(self) -> Response { + let status = self.status(); + // Server-side failures (5xx) are otherwise invisible: the error is + // serialized into the response body but never logged, so operators + // running at debug level saw only tower_http's "status=500" with no + // cause (rustnzb#129). Log it here so every 5xx surfaces its + // underlying error. + if status.is_server_error() { + tracing::error!(status = %status, error = %format!("{:#}", self.kind), "API request failed"); + } let mut response = axum::Json(&self).into_response(); - *response.status_mut() = self.status(); + *response.status_mut() = status; response } } diff --git a/src/fetch_guard.rs b/src/fetch_guard.rs new file mode 100644 index 0000000..ca06e0c --- /dev/null +++ b/src/fetch_guard.rs @@ -0,0 +1,303 @@ +//! SSRF guard for server-side URL fetches. +//! +//! Any endpoint that fetches a caller-supplied URL (the native +//! `POST /api/queue/add-url`, RSS feed fetches, and the SABnzbd-compatible +//! `mode=addurl`) must route through [`validate_fetch_url`] before issuing the +//! request. The guard rejects non-http(s) schemes and any host that resolves +//! to a private, loopback, or otherwise non-globally-routable address, and +//! [`build_fetch_client`] pins the connection to the exact addresses that were +//! validated so a hostname cannot re-resolve to an internal address between +//! the check and the request (DNS rebinding). + +use std::net::{IpAddr, SocketAddr}; + +use crate::error::ApiError; + +/// Maximum body size accepted by URL-backed NZB and feed workflows. +pub const MAX_FETCH_BODY_BYTES: usize = 100 * 1024 * 1024; + +#[derive(Debug)] +pub struct FetchUrlPlan { + pub url: reqwest::Url, + resolved_addrs: Option<(String, Vec)>, +} + +impl FetchUrlPlan { + /// Whether a request for this URL needs a client pinned to the validated + /// addresses. True when the host was a resolved hostname (guards against + /// DNS rebinding); false for an IP-literal URL, where a shared pooled + /// client is safe to reuse. + pub fn requires_pinned_client(&self) -> bool { + self.resolved_addrs.is_some() + } +} + +/// Returns `Err` if `raw_url` is not http/https or resolves to a +/// private/reserved address. +pub async fn validate_fetch_url(raw_url: &str) -> Result { + let url = reqwest::Url::parse(raw_url) + .map_err(|e| ApiError::from(anyhow::anyhow!("Invalid URL: {e}")))?; + + match url.scheme() { + "http" | "https" => {} + s => { + return Err(ApiError::from(anyhow::anyhow!( + "URL scheme '{s}' not allowed (must be http or https)" + ))); + } + } + + let host = url + .host_str() + .ok_or_else(|| ApiError::from(anyhow::anyhow!("URL has no host")))? + .to_string(); + + // IP literal: validate directly without a DNS round-trip. + if let Ok(ip) = host.parse::() { + if !is_globally_routable(ip) { + return Err(ApiError::from(anyhow::anyhow!( + "URL targets a private/reserved address" + ))); + } + return Ok(FetchUrlPlan { + url, + resolved_addrs: None, + }); + } + + // Hostname: resolve and check every returned address. + let port = url.port_or_known_default().unwrap_or(80); + let addrs: Vec<_> = tokio::net::lookup_host(format!("{host}:{port}")) + .await + .map_err(|e| ApiError::from(anyhow::anyhow!("DNS resolution failed for '{host}': {e}")))? + .collect(); + + if addrs.is_empty() { + return Err(ApiError::from(anyhow::anyhow!( + "DNS resolution returned no addresses for '{host}'" + ))); + } + + for addr in &addrs { + if !is_globally_routable(addr.ip()) { + return Err(ApiError::from(anyhow::anyhow!( + "URL resolves to a private/reserved address" + ))); + } + } + + Ok(FetchUrlPlan { + url, + resolved_addrs: Some((host, addrs)), + }) +} + +/// Build a reqwest client pinned to the addresses validated in `plan`, so a +/// hostname cannot re-resolve to an internal address after the check. +pub fn build_fetch_client(plan: &FetchUrlPlan) -> Result { + let mut builder = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + // Redirect targets are not covered by the original DNS validation. + // Refuse redirects so a public URL cannot bounce into a private host. + .redirect(reqwest::redirect::Policy::none()); + if let Some((host, addrs)) = &plan.resolved_addrs { + builder = builder.resolve_to_addrs(host, addrs.as_slice()); + } + builder + .build() + .map_err(|e| ApiError::from(anyhow::anyhow!("Failed to build fetch client: {e}"))) +} + +/// Read a response body, failing if it exceeds `max_bytes` (avoids unbounded +/// memory use from a hostile or misconfigured URL). +pub async fn read_response_bytes_limited( + mut response: reqwest::Response, + max_bytes: usize, +) -> Result, ApiError> { + if response + .content_length() + .is_some_and(|length| length > max_bytes as u64) + { + return Err(ApiError::from(anyhow::anyhow!( + "Fetched body exceeds the {} MB limit", + max_bytes / 1024 / 1024 + ))); + } + + let mut body = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|e| ApiError::from(anyhow::anyhow!("Failed to read response: {e}")))? + { + if body.len().saturating_add(chunk.len()) > max_bytes { + return Err(ApiError::from(anyhow::anyhow!( + "Fetched body exceeds the {} MB limit", + max_bytes / 1024 / 1024 + ))); + } + body.extend_from_slice(&chunk); + } + Ok(body) +} + +fn is_globally_routable(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => { + let [first, second, ..] = v4.octets(); + let this_network = first == 0; + let shared_address_space = first == 100 && (64..=127).contains(&second); + let benchmarking_space = first == 198 && (18..=19).contains(&second); + let reserved_zero_block = first == 192 && second == 0; + let multicast = (224..=239).contains(&first); + let reserved_future_use = first >= 240; + !this_network + && !v4.is_loopback() + && !v4.is_private() + && !v4.is_link_local() + && !v4.is_broadcast() + && !v4.is_unspecified() + && !v4.is_documentation() + && !shared_address_space + && !benchmarking_space + && !reserved_zero_block + && !multicast + && !reserved_future_use + } + IpAddr::V6(v6) => { + let first = v6.segments()[0]; + let mapped_is_global = v6 + .to_ipv4_mapped() + .is_none_or(|mapped| is_globally_routable(IpAddr::V4(mapped))); + !v6.is_loopback() + && !v6.is_unspecified() + && !v6.is_multicast() + && !v6.is_unique_local() + && (first & 0xffc0) != 0xfe80 + && (first & 0xffc0) != 0xfec0 + && !(first == 0x2001 && v6.segments()[1] == 0x0db8) + && mapped_is_global + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn validate_fetch_url_rejects_private_ip_literals() { + let err = validate_fetch_url("http://127.0.0.1/file.nzb") + .await + .unwrap_err(); + assert!(err.to_string().contains("private/reserved")); + } + + #[tokio::test] + async fn validate_fetch_url_rejects_localhost_hostname() { + let err = validate_fetch_url("http://localhost/file.nzb") + .await + .unwrap_err(); + assert!(err.to_string().contains("private/reserved")); + } + + #[tokio::test] + async fn validate_fetch_url_rejects_link_local_metadata() { + // 169.254.169.254 is the cloud-metadata endpoint; must be refused. + let err = validate_fetch_url("http://169.254.169.254/latest/meta-data/") + .await + .unwrap_err(); + assert!(err.to_string().contains("private/reserved")); + } + + #[tokio::test] + async fn validate_fetch_url_rejects_non_http_scheme() { + let err = validate_fetch_url("file:///etc/passwd").await.unwrap_err(); + assert!(err.to_string().contains("not allowed")); + } + + #[tokio::test] + async fn validate_fetch_url_rejects_special_use_address_ranges() { + for url in [ + "http://100.64.0.1/file.nzb", + "http://198.18.0.1/file.nzb", + "http://192.0.0.1/file.nzb", + "http://0.1.2.3/file.nzb", + "http://224.0.0.1/file.nzb", + "http://240.0.0.1/file.nzb", + "http://[fe80::1]/file.nzb", + "http://[2001:db8::1]/file.nzb", + ] { + let error = validate_fetch_url(url) + .await + .expect_err("special-use address must be rejected"); + assert!( + error.to_string().contains("private/reserved"), + "{url}: {error}" + ); + } + } + + async fn one_shot_http_response(response: &'static str) -> reqwest::Url { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind local fixture"); + let address = listener.local_addr().expect("local fixture address"); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept local fixture"); + let mut request = [0; 1024]; + let _ = socket.read(&mut request).await; + let _ = socket.write_all(response.as_bytes()).await; + let _ = socket.shutdown().await; + }); + format!("http://{address}/fixture").parse().unwrap() + } + + #[tokio::test] + async fn pinned_client_uses_validated_address_and_does_not_follow_redirects() { + let url = one_shot_http_response( + "HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1/private\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ) + .await; + let address = url.port().expect("fixture port"); + let plan = FetchUrlPlan { + url: url.clone(), + resolved_addrs: Some(( + "fixture.invalid".into(), + vec![std::net::SocketAddr::from(([127, 0, 0, 1], address))], + )), + }; + let client = build_fetch_client(&plan).expect("build pinned client"); + let response = client + .get(url) + .header("host", "fixture.invalid") + .send() + .await + .expect("request local fixture"); + assert_eq!(response.status(), reqwest::StatusCode::FOUND); + } + + #[tokio::test] + async fn response_body_limit_is_enforced_incrementally() { + let url = one_shot_http_response( + "HTTP/1.1 200 OK\r\nContent-Length: 8\r\nConnection: close\r\n\r\n12345678", + ) + .await; + let plan = FetchUrlPlan { + url: url.clone(), + resolved_addrs: None, + }; + let response = build_fetch_client(&plan) + .expect("build fixture client") + .get(url) + .send() + .await + .expect("request body fixture"); + let error = read_response_bytes_limited(response, 4) + .await + .expect_err("oversized body must be rejected"); + assert!(error.to_string().contains("exceeds")); + } +} diff --git a/src/lib.rs b/src/lib.rs index e180231..f871c24 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,5 @@ +#![recursion_limit = "256"] + pub use nzb_decode; pub use nzb_postproc; pub use nzb_postproc::nzb_core; @@ -6,6 +8,7 @@ pub mod auth; pub mod dir_watcher; pub mod direct_unpack; pub mod error; +pub mod fetch_guard; pub mod log_buffer; pub mod queue_manager; pub mod rss_monitor; diff --git a/src/queue_manager.rs b/src/queue_manager.rs index cc3b568..4aff63c 100644 --- a/src/queue_manager.rs +++ b/src/queue_manager.rs @@ -5,6 +5,7 @@ //! to interact with. use std::collections::{HashMap, HashSet}; +use std::io; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::time::{Duration, Instant}; @@ -12,14 +13,18 @@ use std::time::{Duration, Instant}; use chrono::{DateTime, Utc}; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncRead, AsyncReadExt}; use tokio::sync::{broadcast, mpsc}; use tracing::{debug, error, info, warn}; -use crate::nzb_core::config::{CategoryConfig, ServerConfig}; +use crate::nzb_core::config::{CategoryConfig, ServerConfig, normalize_history_retention}; use crate::nzb_core::db::Database; use crate::nzb_core::models::*; use crate::nzb_core::nzb_parser; -use nzb_postproc::{PostProcConfig, has_usable_output, parse_rar_volume, run_pipeline}; +use nzb_postproc::{ + PostProcConfig, PostProcLimits, PostProcResourcePool, PostProcResourceSnapshot, + has_usable_output, parse_rar_volume, run_pipeline_with_cleanup, +}; use crate::direct_unpack::DirectUnpacker; use crate::log_buffer::LogBuffer; @@ -27,7 +32,12 @@ use nzb_dispatch::{ BandwidthConfig, BandwidthLimiter, DispatchEngine, DispatchHandle, ProgressUpdate, }; -fn cleanup_terminal_work_dir(job_id: &str, work_dir: &std::path::Path, final_status: JobStatus) { +fn cleanup_terminal_work_dir( + job_id: &str, + work_dir: &std::path::Path, + final_status: JobStatus, + retain_for_retry: bool, +) { if !work_dir.exists() { return; } @@ -35,6 +45,10 @@ fn cleanup_terminal_work_dir(job_id: &str, work_dir: &std::path::Path, final_sta let cleanup_result = match final_status { // Failed downloads can be retried from their retained NZB history, so // retaining raw articles only leaks disk without improving recovery. + JobStatus::Failed if retain_for_retry => { + info!(job_id, work_dir = %work_dir.display(), "Retaining partial job files for missing-article retry"); + return; + } JobStatus::Failed => std::fs::remove_dir_all(work_dir), // A successful job must not lose files if an output move failed. Only // remove the directory after the move/pipeline has left it empty. @@ -126,11 +140,17 @@ pub struct GlobalStatisticsData { /// Get free disk space for a path (returns 0 on error). fn get_disk_free(path: &std::path::Path) -> u64 { + let mut candidate = path.to_path_buf(); + while !candidate.exists() { + if !candidate.pop() { + return 0; + } + } #[cfg(unix)] { use std::ffi::CString; use std::mem::MaybeUninit; - let c_path = match CString::new(path.to_string_lossy().as_bytes()) { + let c_path = match CString::new(candidate.to_string_lossy().as_bytes()) { Ok(p) => p, Err(_) => return 0, }; @@ -146,11 +166,119 @@ fn get_disk_free(path: &std::path::Path) -> u64 { } #[cfg(not(unix))] { - let _ = path; + let _ = candidate; 0 } } +fn disk_space_available(threshold: u64, paths: &[&std::path::Path]) -> bool { + threshold == 0 || paths.iter().all(|path| get_disk_free(path) >= threshold) +} + +#[derive(Debug, Clone, Default)] +struct PostProcScriptConfig { + scripts_dir: Option, + success: Option, + failure: Option, + timeout: Duration, + max_output_bytes: usize, +} + +fn resolve_script_path( + scripts_dir: Option<&std::path::Path>, + configured: &std::path::Path, +) -> io::Result { + let candidate = if configured.is_absolute() { + configured.to_path_buf() + } else { + let root = scripts_dir.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "relative post-processing scripts require scripts_dir", + ) + })?; + crate::nzb_core::path::safe_join(root, &configured.to_string_lossy()).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "post-processing script path is unsafe", + ) + })? + }; + let resolved = std::fs::canonicalize(candidate)?; + if !std::fs::metadata(&resolved)?.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "post-processing script is not a regular file", + )); + } + if let Some(root) = scripts_dir { + let root = std::fs::canonicalize(root)?; + if !resolved.starts_with(root) { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "post-processing script is outside scripts_dir", + )); + } + } + Ok(resolved) +} + +async fn read_script_output( + reader: R, + max_output_bytes: usize, +) -> io::Result<(Vec, bool)> { + let mut output = Vec::new(); + let mut limited = reader.take(max_output_bytes.saturating_add(1) as u64); + limited.read_to_end(&mut output).await?; + let truncated = output.len() > max_output_bytes; + output.truncate(max_output_bytes); + Ok((output, truncated)) +} + +fn regular_output_files(root: &std::path::Path) -> Vec { + let mut pending = vec![root.to_path_buf()]; + let mut files = Vec::new(); + while let Some(directory) = pending.pop() { + let Ok(entries) = std::fs::read_dir(directory) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + let Ok(metadata) = std::fs::symlink_metadata(&path) else { + continue; + }; + if metadata.file_type().is_symlink() { + continue; + } + if metadata.is_dir() { + pending.push(path); + } else if metadata.is_file() { + files.push(path); + } + } + } + files.sort(); + files +} + +fn script_output_message(stdout: &(Vec, bool), stderr: &(Vec, bool)) -> String { + let mut message = String::from_utf8_lossy(&stdout.0).trim().to_string(); + let error = String::from_utf8_lossy(&stderr.0).trim().to_string(); + if !error.is_empty() { + if !message.is_empty() { + message.push_str("; "); + } + message.push_str(&error); + } + if stdout.1 || stderr.1 { + if !message.is_empty() { + message.push_str("; "); + } + message.push_str("output truncated"); + } + message +} + // --------------------------------------------------------------------------- // Job checkpoint for resume support // --------------------------------------------------------------------------- @@ -169,6 +297,113 @@ struct JobCheckpoint { articles_failed: usize, /// Number of files completed files_completed: usize, + /// Full article outcomes. This was added after the original segment-only + /// checkpoint so history retries can distinguish missing articles from + /// articles that were already written before a failure. + #[serde(default)] + articles: HashMap>, + /// Retained partial work directory for missing-only history retry. + #[serde(default)] + work_dir: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct ArticleCheckpoint { + message_id: String, + segment_number: u32, + bytes: u64, + downloaded: bool, + data_begin: Option, + data_size: Option, + crc32: Option, + tried_servers: Vec, + tries: u32, +} + +fn checkpoint_for_job(job: &NzbJob) -> JobCheckpoint { + JobCheckpoint { + files: job + .files + .iter() + .map(|file| { + ( + file.filename.clone(), + file.articles + .iter() + .filter(|article| article.downloaded) + .map(|article| article.segment_number) + .collect(), + ) + }) + .collect(), + downloaded_bytes: job.downloaded_bytes, + articles_downloaded: job.articles_downloaded, + articles_failed: job.articles_failed, + files_completed: job.files_completed, + articles: job + .files + .iter() + .map(|file| { + ( + file.filename.clone(), + file.articles + .iter() + .map(|article| ArticleCheckpoint { + message_id: article.message_id.clone(), + segment_number: article.segment_number, + bytes: article.bytes, + downloaded: article.downloaded, + data_begin: article.data_begin, + data_size: article.data_size, + crc32: article.crc32, + tried_servers: article.tried_servers.clone(), + tries: article.tries, + }) + .collect(), + ) + }) + .collect(), + work_dir: Some(job.work_dir.clone()), + } +} + +fn apply_checkpoint(job: &mut NzbJob, checkpoint: &JobCheckpoint) { + job.downloaded_bytes = checkpoint.downloaded_bytes; + job.articles_downloaded = checkpoint.articles_downloaded; + job.articles_failed = checkpoint.articles_failed; + job.files_completed = checkpoint.files_completed; + for file in &mut job.files { + let outcomes = checkpoint.articles.get(&file.filename); + let segments = checkpoint + .files + .get(&file.filename) + .or_else(|| checkpoint.files.get(&file.id)); + let mut file_bytes: u64 = 0; + for article in &mut file.articles { + if let Some(outcome) = outcomes.and_then(|items| { + items.iter().find(|item| { + item.message_id == article.message_id + || item.segment_number == article.segment_number + }) + }) { + article.downloaded = outcome.downloaded; + article.data_begin = outcome.data_begin; + article.data_size = outcome.data_size; + article.crc32 = outcome.crc32; + article.tried_servers = outcome.tried_servers.clone(); + article.tries = outcome.tries; + } else if segments.is_some_and(|items| items.contains(&article.segment_number)) { + // Checkpoints written before article outcomes existed only + // recorded downloaded segment numbers. + article.downloaded = true; + } + if article.downloaded { + file_bytes = file_bytes.saturating_add(article.data_size.unwrap_or(article.bytes)); + } + } + file.bytes_downloaded = file_bytes; + file.assembled = file.articles.iter().all(|article| article.downloaded); + } } // --------------------------------------------------------------------------- @@ -600,6 +835,17 @@ pub(crate) struct HopelessAbort { } impl HopelessTracker { + /// Reset the no-progress clock so the article timeout starts fresh. + /// + /// Called when a job returns to `Downloading` after a pause. The clock is + /// a wall-clock `Instant` that only advances on real article progress, so + /// without this reset the time a job spent paused would count toward the + /// no-progress timeout and abort it the instant it resumes (GH #123). A + /// paused job is never actively fetching, so paused time must not count. + fn reset_progress_clock(&mut self) { + self.last_progress_at = Instant::now(); + } + /// Phase 6: time-based hopeless check. Operates on the tracker's /// `created_at` field, not on article counters, so it fires even when /// the engine has stopped emitting progress events entirely (the @@ -692,16 +938,25 @@ pub struct QueueManager { pause_until: Mutex>>, /// History retention limit (None = keep all). history_retention: Mutex>, + /// SAB-compatible history generation. Incremented only when the history + /// view changes so polling clients can avoid downloading unchanged data. + history_update: AtomicU64, /// Log buffer for capturing per-job logs into history. log_buffer: Option, /// Broadcast channel: fires immediately when a job is accepted into the queue. add_tx: broadcast::Sender, /// Max concurrent active downloads (0 = unlimited). max_active_downloads: AtomicUsize, + /// Automatically keep the queue ordered by remaining percentage. + auto_sort_remaining_pct: AtomicBool, + /// Optional bounded post-processing hooks. + postproc_scripts: Mutex, + /// Stage-specific resource gates shared by all post-processing jobs. + postproc_resources: Arc, /// Category configs for post-processing decisions. categories: Mutex>, /// Minimum free disk space in bytes before pausing downloads. - min_free_space: u64, + min_free_space: AtomicU64, /// Bandwidth limiter for throttling downloads. bandwidth: Arc, /// Whether direct unpack (RAR extraction during download) is enabled. @@ -743,6 +998,46 @@ impl QueueManager { early_failure_check: bool, required_completion_pct: f64, article_timeout_secs: u64, + ) -> Arc { + Self::new_with_postproc_limits( + servers, + db, + incomplete_dir, + complete_dir, + log_buffer, + max_active_downloads, + PostProcLimits::default(), + categories, + min_free_space, + speed_limit_bps, + direct_unpack, + max_nested_archive_depth, + abort_hopeless, + early_failure_check, + required_completion_pct, + article_timeout_secs, + ) + } + + /// Create a queue manager with explicit post-processing worker limits. + #[allow(clippy::too_many_arguments)] + pub fn new_with_postproc_limits( + servers: Vec, + db: Database, + incomplete_dir: std::path::PathBuf, + complete_dir: std::path::PathBuf, + log_buffer: LogBuffer, + max_active_downloads: usize, + postproc_limits: PostProcLimits, + categories: Vec, + min_free_space: u64, + speed_limit_bps: u64, + direct_unpack: bool, + max_nested_archive_depth: u8, + abort_hopeless: bool, + early_failure_check: bool, + required_completion_pct: f64, + article_timeout_secs: u64, ) -> Arc { use std::num::NonZeroU32; @@ -776,11 +1071,15 @@ impl QueueManager { complete_dir: Mutex::new(complete_dir), pause_until: Mutex::new(None), history_retention: Mutex::new(None), + history_update: AtomicU64::new(1), log_buffer: Some(log_buffer), add_tx, max_active_downloads: AtomicUsize::new(max_active_downloads), + auto_sort_remaining_pct: AtomicBool::new(false), + postproc_scripts: Mutex::new(PostProcScriptConfig::default()), + postproc_resources: PostProcResourcePool::new(postproc_limits), categories: Mutex::new(categories), - min_free_space, + min_free_space: AtomicU64::new(min_free_space), bandwidth, direct_unpack_enabled: AtomicBool::new(direct_unpack), max_nested_archive_depth, @@ -815,9 +1114,25 @@ impl QueueManager { } } - /// Set history retention limit. + /// Set history retention limit. `Some(0)` is normalized to `None` + /// (keep all) so a zero can never wipe history on completion (GH #136). pub fn set_history_retention(&self, limit: Option) { - *self.history_retention.lock() = limit; + *self.history_retention.lock() = normalize_history_retention(limit); + } + + /// Current generation of the SAB-compatible history view. + pub fn history_update(&self) -> u64 { + self.history_update.load(Ordering::Acquire) + } + + fn history_changed(&self) { + // SABnzbd also uses a wrapping generation rather than a timestamp. + // Keep zero reserved for clients that have never polled. + let _ = self + .history_update + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + Some(if current == u64::MAX { 1 } else { current + 1 }) + }); } /// Subscribe to job addition events. The receiver fires immediately when @@ -916,11 +1231,72 @@ impl QueueManager { self.start_next_queued(); } + /// Enable or disable automatic remaining-percentage ordering. + pub fn set_auto_sort_remaining_pct(&self, enabled: bool) { + self.auto_sort_remaining_pct + .store(enabled, Ordering::Relaxed); + } + + pub fn auto_sort_remaining_pct(&self) -> bool { + self.auto_sort_remaining_pct.load(Ordering::Relaxed) + } + + /// Configure the optional success and failure hooks used after + /// post-processing. Script paths are resolved and confined when a job + /// invokes them; keeping the raw config here allows live updates without + /// rebuilding the queue manager. + pub fn set_postproc_scripts( + &self, + scripts_dir: Option, + success: Option, + failure: Option, + timeout_secs: u64, + max_output_bytes: usize, + ) { + *self.postproc_scripts.lock() = PostProcScriptConfig { + scripts_dir, + success, + failure, + timeout: Duration::from_secs(timeout_secs.max(1)), + max_output_bytes, + }; + } + + /// Stable sort of the queue by remaining work percentage. The original + /// queue order is retained for equal percentages, which keeps repeated + /// manual sorts deterministic and avoids active-job churn. + pub fn sort_by_remaining_percentage(&self, ascending: bool) { + let jobs = self.jobs.lock(); + let mut order = self.job_order.lock(); + order.sort_by(|left, right| { + let remaining = |id: &String| { + jobs.get(id).map_or((0u64, 1u64), |state| { + let total = state.job.total_bytes.max(1); + (total.saturating_sub(state.job.downloaded_bytes), total) + }) + }; + let (left_remaining, left_total) = remaining(left); + let (right_remaining, right_total) = remaining(right); + let ordering = (left_remaining as u128 * right_total as u128) + .cmp(&(right_remaining as u128 * left_total as u128)); + if ascending { + ordering + } else { + ordering.reverse() + } + }); + } + /// Get max active downloads. pub fn get_max_active_downloads(&self) -> usize { self.max_active_downloads.load(Ordering::Relaxed) } + /// Configured and observed post-processing concurrency. + pub fn postproc_resource_snapshot(&self) -> PostProcResourceSnapshot { + self.postproc_resources.snapshot() + } + /// Set the download speed limit in bytes per second (0 = unlimited). pub fn set_speed_limit(&self, bps: u64) { use std::num::NonZeroU32; @@ -1013,6 +1389,32 @@ impl QueueManager { mut job: NzbJob, nzb_data: Option>, ) -> crate::nzb_core::Result<()> { + if crate::nzb_core::path::safe_component(&job.category).is_none() { + return Err(crate::nzb_core::NzbError::Other( + "category must be a single safe path component".to_string(), + )); + } + crate::nzb_core::path::safe_component(&job.name).ok_or_else(|| { + crate::nzb_core::NzbError::Other( + "job name must be a single safe path component".to_string(), + ) + })?; + let complete_root = self.complete_dir(); + let configured_root = self + .categories + .lock() + .iter() + .find(|category| category.name == job.category) + .and_then(|category| category.output_dir.clone()); + let output_is_allowed = job.output_dir.starts_with(&complete_root) + || configured_root + .as_ref() + .is_some_and(|root| job.output_dir.starts_with(root)); + if !output_is_allowed { + return Err(crate::nzb_core::NzbError::Other( + "job output directory is outside configured storage roots".to_string(), + )); + } // Ensure work directory exists std::fs::create_dir_all(&job.work_dir)?; @@ -1081,6 +1483,43 @@ impl QueueManager { Ok(()) } + /// Rebuild a history job for retry. Newer history rows carry a checkpoint + /// and retain a partial work directory when at least one article was + /// written, so the dispatcher can enqueue only unresolved articles and + /// append them to the existing assembled files. Older rows, and rows + /// whose partial directory is gone, deliberately fall back to a full + /// retry. + pub fn prepare_retry_job( + &self, + entry: &HistoryEntry, + nzb_data: &[u8], + retry_data: Option<&[u8]>, + ) -> crate::nzb_core::Result { + let mut job = nzb_parser::parse_nzb(&entry.name, nzb_data)?; + job.category = entry.category.clone(); + job.output_dir = self.output_dir_for(&job.category, &job.name)?; + job.work_dir = self.incomplete_dir().join(&job.id); + + if entry.status == JobStatus::Failed + && let Some(data) = retry_data + && let Ok(checkpoint) = serde_json::from_slice::(data) + && let Some(work_dir) = checkpoint.work_dir.as_ref() + && std::fs::canonicalize(self.incomplete_dir()) + .ok() + .zip(std::fs::canonicalize(work_dir).ok()) + .is_some_and(|(root, retained)| retained.starts_with(root)) + && std::fs::symlink_metadata(work_dir) + .map(|metadata| metadata.file_type().is_dir()) + .unwrap_or(false) + { + apply_checkpoint(&mut job, &checkpoint); + job.articles_failed = 0; + job.work_dir = work_dir.clone(); + } + + Ok(job) + } + /// Launch the download task for a job that is already in the jobs map /// with status `Downloading`. /// @@ -1105,25 +1544,7 @@ impl QueueManager { && let Ok(checkpoint) = serde_json::from_slice::(&cp_data) { - state.job.downloaded_bytes = checkpoint.downloaded_bytes; - state.job.articles_downloaded = checkpoint.articles_downloaded; - state.job.articles_failed = checkpoint.articles_failed; - state.job.files_completed = checkpoint.files_completed; - for file in &mut state.job.files { - if let Some(segments) = checkpoint.files.get(&file.id) { - let mut fbd: u64 = 0; - for article in &mut file.articles { - if segments.contains(&article.segment_number) { - article.downloaded = true; - fbd += article.bytes; - } - } - file.bytes_downloaded = fbd; - if file.articles.iter().all(|a| a.downloaded) { - file.assembled = true; - } - } - } + apply_checkpoint(&mut state.job, &checkpoint); info!( job_id = %job_id, name = %state.job.name, @@ -1151,13 +1572,18 @@ impl QueueManager { }; // Pre-flight disk space check - let free = get_disk_free(&self.incomplete_dir.lock()); - if self.min_free_space > 0 && free > 0 && free < self.min_free_space { + let incomplete_dir = self.incomplete_dir(); + let output_dir = job.output_dir.clone(); + let free = get_disk_free(&incomplete_dir); + if !disk_space_available( + self.min_free_space(), + [incomplete_dir.as_path(), output_dir.as_path()].as_slice(), + ) { warn!( job_id = %job_id, free_bytes = free, - min_free_space = self.min_free_space, - "Paused job due to low disk space" + min_free_space = self.min_free_space(), + "Paused job due to low disk space on a job storage volume" ); let mut jobs = self.jobs.lock(); if let Some(state) = jobs.get_mut(job_id) { @@ -1448,6 +1874,9 @@ impl QueueManager { self.persist_job_progress(&job_id); last_db_update = Instant::now(); } + if self.auto_sort_remaining_pct() { + self.sort_by_remaining_percentage(true); + } } ProgressUpdate::ArticleFailed { file_id, @@ -1461,6 +1890,23 @@ impl QueueManager { if let Some(state) = jobs.get_mut(&job_id) { state.job.articles_failed += 1; + if let Some(article) = state + .job + .files + .iter_mut() + .find(|file| file.id == file_id) + .and_then(|file| { + file.articles + .iter_mut() + .find(|article| article.segment_number == segment_number) + }) + { + article.tries = article.tries.saturating_add(1); + if !article.tried_servers.contains(&failure.server_id) { + article.tried_servers.push(failure.server_id.clone()); + } + } + // Update per-server failed stats let sid = &failure.server_id; let stats = &mut state.job.server_stats; @@ -1648,6 +2094,7 @@ impl QueueManager { state.job.completed_at = Some(chrono::Utc::now()); } } + self.history_changed(); self.start_next_queued(); self.on_job_finished(&job_id, success, articles_failed) @@ -1724,6 +2171,9 @@ impl QueueManager { articles_failed: usize, ) { let pipeline_start = Instant::now(); + // A download slot is already free at this point. Bound the independent + // post-processing job before taking any stage-specific resource. + let _pipeline_permit = self.postproc_resources.acquire_pipeline().await; // Extract info needed for post-processing and take the direct unpacker. let ( @@ -1775,6 +2225,41 @@ impl QueueManager { ) }; + // Repair and extraction can write to both the incomplete and the + // category output volumes. Apply the same guard to both paths before + // any post-processing work begins. + if !disk_space_available( + self.min_free_space(), + [work_dir.as_path(), output_dir.as_path()].as_slice(), + ) { + let mut jobs = self.jobs.lock(); + if let Some(state) = jobs.get_mut(job_id) { + let message = "Insufficient free disk space for post-processing".to_string(); + state.job.status = JobStatus::Failed; + state.job.error_message = Some(message.clone()); + self.move_to_history( + state, + vec![StageResult { + name: "Disk".into(), + status: StageStatus::Failed, + message: Some(message), + duration_secs: 0.0, + }], + ); + } + drop(jobs); + self.persist_job_progress(job_id); + self.start_next_queued(); + let qm = Arc::clone(self); + let jid = job_id.to_string(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(8)).await; + qm.jobs.lock().remove(&jid); + qm.job_order.lock().retain(|id| id != &jid); + }); + return; + } + // Wait for direct unpack to finish (if active). It may still be // extracting the last volume when the download completes. let direct_unpack_success = if let Some(du) = direct_unpacker { @@ -1812,6 +2297,18 @@ impl QueueManager { "Running post-processing pipeline" ); + let (cleanup_patterns, unwanted_extensions) = self + .categories + .lock() + .iter() + .find(|configured| configured.name == category) + .map(|configured| { + ( + configured.cleanup_patterns.clone(), + configured.unwanted_extensions.clone(), + ) + }) + .unwrap_or_default(); let config = PostProcConfig { cleanup_after_extract: true, output_dir: Some(output_dir.clone()), @@ -1822,7 +2319,14 @@ impl QueueManager { max_nested_archive_depth: self.max_nested_archive_depth, }; - let result = run_pipeline(&work_dir, &config).await; + let result = run_pipeline_with_cleanup( + &work_dir, + &config, + Some(&self.postproc_resources), + &cleanup_patterns, + &unwanted_extensions, + ) + .await; info!( job_id = %job_id, @@ -1858,6 +2362,36 @@ impl QueueManager { Vec::new() }; + // Run the configured hook after the final status is known. Its output + // is bounded and the hook cannot change the job's filesystem roots. + let script_status = { + let jobs = self.jobs.lock(); + jobs.get(job_id).map(|state| { + if state.job.status == JobStatus::Failed { + JobStatus::Failed + } else { + JobStatus::Completed + } + }) + }; + let script_stage = match script_status { + Some(status) => self.run_postproc_script(job_id, status).await, + None => None, + }; + let mut stages = stages; + if let Some(stage) = script_stage { + if stage.status == StageStatus::Failed { + let mut jobs = self.jobs.lock(); + if let Some(state) = jobs.get_mut(job_id) { + state.job.status = JobStatus::Failed; + if state.job.error_message.is_none() { + state.job.error_message = stage.message.clone(); + } + } + } + stages.push(stage); + } + // Move to history with real stage results { let mut jobs = self.jobs.lock(); @@ -1882,6 +2416,123 @@ impl QueueManager { }); } + async fn run_postproc_script( + &self, + job_id: &str, + final_status: JobStatus, + ) -> Option { + let (job, script_config) = { + let jobs = self.jobs.lock(); + let state = jobs.get(job_id)?; + (state.job.clone(), self.postproc_scripts.lock().clone()) + }; + let configured = match final_status { + JobStatus::Completed => script_config.success, + JobStatus::Failed => script_config.failure, + _ => None, + }?; + let started = Instant::now(); + let script = match resolve_script_path(script_config.scripts_dir.as_deref(), &configured) { + Ok(path) => path, + Err(error) => { + return Some(StageResult { + name: "Script".into(), + status: StageStatus::Failed, + message: Some(format!("Unable to resolve post-processing script: {error}")), + duration_secs: started.elapsed().as_secs_f64(), + }); + } + }; + + let files = regular_output_files(&job.output_dir); + let file_list = files + .iter() + .map(|path| path.to_string_lossy()) + .collect::>() + .join("\n"); + let mut command = tokio::process::Command::new(&script); + command + .current_dir(&job.output_dir) + .env("SAB_STATUS", final_status.to_string()) + .env("SAB_JOB", &job.name) + .env("SAB_CAT", &job.category) + .env("SAB_FILENAME", &job.name) + .env("SAB_COMPLETE", &job.output_dir) + .env("SAB_BYTES", job.total_bytes.to_string()) + .env("SAB_BYTES_DOWNLOADED", job.downloaded_bytes.to_string()) + .env("SAB_FILES", file_list) + .env("RUSTNZB_STATUS", final_status.to_string()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true); + let mut child = match command.spawn() { + Ok(child) => child, + Err(error) => { + return Some(StageResult { + name: "Script".into(), + status: StageStatus::Failed, + message: Some(format!("Unable to start post-processing script: {error}")), + duration_secs: started.elapsed().as_secs_f64(), + }); + } + }; + let stdout = child.stdout.take(); + let stderr = child.stderr.take(); + let max_output_bytes = script_config.max_output_bytes; + let result = async { + let stdout_reader = async { + match stdout { + Some(reader) => read_script_output(reader, max_output_bytes).await, + None => Ok((Vec::new(), false)), + } + }; + let stderr_reader = async { + match stderr { + Some(reader) => read_script_output(reader, max_output_bytes).await, + None => Ok((Vec::new(), false)), + } + }; + let (stdout, stderr) = tokio::join!(stdout_reader, stderr_reader); + let stdout = stdout?; + let stderr = stderr?; + let status = child.wait().await?; + Ok::<_, io::Error>((status, stdout, stderr)) + }; + + match tokio::time::timeout(script_config.timeout, result).await { + Ok(Ok((status, stdout, stderr))) if status.success() => Some(StageResult { + name: "Script".into(), + status: StageStatus::Success, + message: Some(script_output_message(&stdout, &stderr)), + duration_secs: started.elapsed().as_secs_f64(), + }), + Ok(Ok((status, stdout, stderr))) => Some(StageResult { + name: "Script".into(), + status: StageStatus::Failed, + message: Some(format!( + "Post-processing script exited with {status}: {}", + script_output_message(&stdout, &stderr) + )), + duration_secs: started.elapsed().as_secs_f64(), + }), + Ok(Err(error)) => Some(StageResult { + name: "Script".into(), + status: StageStatus::Failed, + message: Some(format!("Post-processing script failed: {error}")), + duration_secs: started.elapsed().as_secs_f64(), + }), + Err(_) => Some(StageResult { + name: "Script".into(), + status: StageStatus::Failed, + message: Some(format!( + "Post-processing script exceeded {} second timeout", + script_config.timeout.as_secs() + )), + duration_secs: started.elapsed().as_secs_f64(), + }), + } + } + /// Move a job's files to output and insert a history entry. fn move_to_history(&self, state: &mut JobState, mut stages: Vec) { let move_start = Instant::now(); @@ -1923,8 +2574,32 @@ impl QueueManager { if let Ok(entries) = std::fs::read_dir(&state.job.work_dir) { for entry in entries.flatten() { let path = entry.path(); - if path.is_file() { - let dest = state.job.output_dir.join(entry.file_name()); + let is_regular = std::fs::symlink_metadata(&path) + .map(|metadata| metadata.file_type().is_file()) + .unwrap_or(false); + let Some(dest) = crate::nzb_core::path::safe_join( + &state.job.output_dir, + &entry.file_name().to_string_lossy(), + ) else { + warn!( + job_id = %state.job.id, + file = %path.display(), + "Refusing to move file with an unsafe output name" + ); + continue; + }; + if is_regular { + if std::fs::symlink_metadata(&dest) + .map(|metadata| metadata.file_type().is_symlink()) + .unwrap_or(false) + { + warn!( + job_id = %state.job.id, + file = %dest.display(), + "Refusing to replace symlink in output directory" + ); + continue; + } if let Err(e) = std::fs::rename(&path, &dest) { if let Err(e2) = std::fs::copy(&path, &dest) { warn!( @@ -1953,6 +2628,7 @@ impl QueueManager { state.job.status = final_status; // Insert into history with real stage results + let retry_data = serde_json::to_vec(&checkpoint_for_job(&state.job)).ok(); let history_entry = HistoryEntry { id: state.job.id.clone(), name: state.job.name.clone(), @@ -1968,6 +2644,7 @@ impl QueueManager { error_message: state.job.error_message.clone(), server_stats: state.job.server_stats.clone(), nzb_data: state.nzb_data.clone(), + retry_data, }; let db = self.db.lock(); @@ -1986,6 +2663,7 @@ impl QueueManager { error!(job_id = %state.job.id, "Failed to insert history: {e}"); false } else { + self.history_changed(); true } } @@ -2019,7 +2697,19 @@ impl QueueManager { drop(db); if history_persisted { - cleanup_terminal_work_dir(&state.job.id, &state.job.work_dir, final_status); + let retain_for_retry = final_status == JobStatus::Failed + && state.nzb_data.is_some() + && state + .job + .files + .iter() + .any(|file| file.articles.iter().any(|article| article.downloaded)); + cleanup_terminal_work_dir( + &state.job.id, + &state.job.work_dir, + final_status, + retain_for_retry, + ); } else { warn!( job_id = %state.job.id, @@ -2047,26 +2737,7 @@ impl QueueManager { } // Build and store checkpoint of downloaded article segments - let checkpoint = JobCheckpoint { - files: state - .job - .files - .iter() - .map(|f| { - let downloaded_segments: Vec = f - .articles - .iter() - .filter(|a| a.downloaded) - .map(|a| a.segment_number) - .collect(); - (f.id.clone(), downloaded_segments) - }) - .collect(), - downloaded_bytes: state.job.downloaded_bytes, - articles_downloaded: state.job.articles_downloaded, - articles_failed: state.job.articles_failed, - files_completed: state.job.files_completed, - }; + let checkpoint = checkpoint_for_job(&state.job); if let Ok(data) = serde_json::to_vec(&checkpoint) && let Err(e) = db.queue_store_job_data(job_id, &data) @@ -2080,8 +2751,13 @@ impl QueueManager { // Job control // ----------------------------------------------------------------------- - /// Change the priority of a specific job, reorder the queue, and preempt - /// lower-priority downloads when a higher-priority job is waiting. + /// Change the priority of a specific job and reorder the queue. + /// + /// A priority change only reorders the queue; it never pauses an + /// actively-downloading job (GH #124). The new order takes effect the + /// next time a download slot frees up. If a slot is already free, any + /// queued job is started in the new order, but no running download is + /// preempted. pub fn set_job_priority( self: &Arc, id: &str, @@ -2126,8 +2802,10 @@ impl QueueManager { } } - // 3. Preempt lower-priority downloads if a higher-priority queued job is waiting - self.preempt_if_needed(); + // 3. Fill any free download slot in the new order. A priority change + // must not pause a running download (GH #124), so start queued jobs + // only — never preempt an active one. + self.start_next_queued(); Ok(()) } @@ -2308,6 +2986,13 @@ impl QueueManager { // Job context still lives in the pool — just unpause it. state.job.status = JobStatus::Downloading; state.job.error_message = None; + // The no-progress watchdog measures wall-clock idle time and + // does not stop while paused, so restart its clock here or the + // paused interval counts toward the article timeout and aborts + // the job on the next scan (GH #123). + if let Some(tracker) = state.hopeless_tracker.as_mut() { + tracker.reset_progress_clock(); + } let db = self.db.lock(); let _ = db.queue_update_progress( id, @@ -2398,13 +3083,17 @@ impl QueueManager { error_message: state.job.error_message.clone(), server_stats: state.job.server_stats.clone(), nzb_data: state.nzb_data.clone(), + retry_data: None, }; if let Err(e) = db.history_insert(&history_entry) { error!(job_id = %id, "Failed to insert history for removed failed job: {e}"); - } else if let Some(max) = *self.history_retention.lock() - && let Err(e) = db.history_enforce_retention(max) - { - warn!("Failed to enforce history retention: {e}"); + } else { + self.history_changed(); + if let Some(max) = *self.history_retention.lock() + && let Err(e) = db.history_enforce_retention(max) + { + warn!("Failed to enforce history retention: {e}"); + } } } else if history_already_persisted { debug!(job_id = %id, "Removing terminal queue view; history already persisted"); @@ -2435,7 +3124,14 @@ impl QueueManager { .find(|(_, s)| s.job.id == id || s.job.id.starts_with(id)); match state { Some((_, s)) => { + crate::nzb_core::path::safe_component(new_name).ok_or_else(|| { + crate::nzb_core::NzbError::Other( + "job name must be a single safe path component".to_string(), + ) + })?; + let output_dir = self.output_dir_for(&s.job.category, new_name)?; s.job.name = new_name.to_string(); + s.job.output_dir = output_dir; info!(job_id = %id, new_name = %new_name, "Job renamed"); Ok(()) } @@ -2445,6 +3141,19 @@ impl QueueManager { /// Change a job's category in the queue. pub fn change_job_category(&self, id: &str, category: &str) -> crate::nzb_core::Result<()> { + if crate::nzb_core::path::safe_component(category).is_none() { + return Err(crate::nzb_core::NzbError::Other( + "category must be a single safe path component".to_string(), + )); + } + let job_name = self + .jobs + .lock() + .iter() + .find(|(_, state)| state.job.id == id || state.job.id.starts_with(id)) + .map(|(_, state)| state.job.name.clone()) + .ok_or_else(|| crate::nzb_core::NzbError::JobNotFound(id.to_string()))?; + let output_dir = self.output_dir_for(category, &job_name)?; let mut jobs = self.jobs.lock(); let state = jobs .iter_mut() @@ -2453,8 +3162,7 @@ impl QueueManager { Some((_, s)) => { s.job.category = category.to_string(); // Update the output directory to match the new category - let complete_dir = self.complete_dir.lock().join(category).join(&s.job.name); - s.job.output_dir = complete_dir; + s.job.output_dir = output_dir; info!(job_id = %id, category = %category, "Job category changed"); Ok(()) } @@ -2739,9 +3447,79 @@ impl QueueManager { *self.complete_dir.lock() = dir; } + /// Resolve a category and job name to the configured output directory. + /// Both values originate from API/NZB input, so they must remain single + /// path components before they are joined to a trusted configured root. + pub fn output_dir_for( + &self, + category: &str, + name: &str, + ) -> crate::nzb_core::Result { + crate::nzb_core::path::safe_component(category).ok_or_else(|| { + crate::nzb_core::NzbError::Other("category must be a single safe path component".into()) + })?; + crate::nzb_core::path::safe_component(name).ok_or_else(|| { + crate::nzb_core::NzbError::Other("job name must be a single safe path component".into()) + })?; + + let categories = self.categories.lock(); + let category_config = categories + .iter() + .find(|configured| configured.name == category); + if let Some(base) = category_config.and_then(|configured| configured.output_dir.as_ref()) { + let root = if base.is_absolute() { + base.clone() + } else { + crate::nzb_core::path::safe_join(&self.complete_dir(), &base.to_string_lossy()) + .ok_or_else(|| { + crate::nzb_core::NzbError::Other("category output path is unsafe".into()) + })? + }; + return crate::nzb_core::path::safe_join(&root, name).ok_or_else(|| { + crate::nzb_core::NzbError::Other("category output path is unsafe".into()) + }); + } + let category_dir = crate::nzb_core::path::safe_join(&self.complete_dir(), category) + .ok_or_else(|| { + crate::nzb_core::NzbError::Other("category output path is unsafe".into()) + })?; + crate::nzb_core::path::safe_join(&category_dir, name) + .ok_or_else(|| crate::nzb_core::NzbError::Other("job output path is unsafe".into())) + } + + /// Return every configured filesystem root that may receive job data. + /// Relative category roots are resolved below the complete directory. + fn disk_guard_paths(&self) -> Vec { + let complete = self.complete_dir(); + let mut paths = vec![self.incomplete_dir(), complete.clone()]; + for category in self.categories.lock().iter() { + let Some(root) = category.output_dir.as_ref() else { + continue; + }; + let resolved = if root.is_absolute() { + root.clone() + } else if let Some(resolved) = + crate::nzb_core::path::safe_join(&complete, &root.to_string_lossy()) + { + resolved + } else { + continue; + }; + if !paths.contains(&resolved) { + paths.push(resolved); + } + } + paths + } + /// Get the minimum free disk space threshold. pub fn min_free_space(&self) -> u64 { - self.min_free_space + self.min_free_space.load(Ordering::Relaxed) + } + + /// Update the disk guard threshold for both preflight and periodic checks. + pub fn set_min_free_space(&self, bytes: u64) { + self.min_free_space.store(bytes, Ordering::Relaxed); } /// Lock the database and execute a closure with direct access. @@ -2986,16 +3764,32 @@ impl QueueManager { db.history_get_nzb_data(id) } + /// Get per-article retry outcomes persisted with a history entry. + pub fn history_get_retry_data(&self, id: &str) -> crate::nzb_core::Result>> { + let db = self.db.lock(); + db.history_get_retry_data(id) + } + /// Remove a history entry. pub fn history_remove(&self, id: &str) -> crate::nzb_core::Result<()> { let db = self.db.lock(); - db.history_remove(id) + let existed = db.history_get(id)?.is_some(); + db.history_remove(id)?; + if existed { + self.history_changed(); + } + Ok(()) } /// Clear all history. pub fn history_clear(&self) -> crate::nzb_core::Result<()> { let db = self.db.lock(); - db.history_clear() + let had_entries = db.history_count()? != 0; + db.history_clear()?; + if had_entries { + self.history_changed(); + } + Ok(()) } /// Get live logs for an active job from the in-memory log buffer. @@ -3073,6 +3867,12 @@ impl QueueManager { db.rss_items_prune(keep) } + /// Expire downloaded RSS records older than the supplied RFC3339 cutoff. + pub fn rss_items_expire_downloaded(&self, cutoff: &str) -> crate::nzb_core::Result { + let db = self.db.lock(); + db.rss_items_expire_downloaded(cutoff) + } + /// List all RSS download rules. pub fn rss_rule_list(&self) -> crate::nzb_core::Result> { let db = self.db.lock(); @@ -3133,9 +3933,25 @@ impl QueueManager { info!(count = jobs.len(), "Restoring jobs from database"); + let mut postproc_recovery = Vec::new(); for mut job in jobs { let job_id = job.id.clone(); + // A process can stop after the final article closed but before the + // pipeline committed history. Resume from the idempotent stage + // boundary instead of leaving the job permanently stranded. + let was_post_processing = matches!( + job.status, + JobStatus::PostProcessing + | JobStatus::Verifying + | JobStatus::Repairing + | JobStatus::Extracting + ); + if was_post_processing { + job.status = JobStatus::PostProcessing; + postproc_recovery.push((job_id.clone(), job.articles_failed)); + } + // Only load full NZB data + checkpoints for jobs that were actively // downloading. Queued/paused jobs just need metadata — their NZB data // is loaded lazily in launch_download() when they reach the front of @@ -3171,26 +3987,7 @@ impl QueueManager { if let Some(ref data) = checkpoint_data { match serde_json::from_slice::(data) { Ok(checkpoint) => { - job.downloaded_bytes = checkpoint.downloaded_bytes; - job.articles_downloaded = checkpoint.articles_downloaded; - job.articles_failed = checkpoint.articles_failed; - job.files_completed = checkpoint.files_completed; - - for file in &mut job.files { - if let Some(segments) = checkpoint.files.get(&file.id) { - let mut file_bytes_downloaded: u64 = 0; - for article in &mut file.articles { - if segments.contains(&article.segment_number) { - article.downloaded = true; - file_bytes_downloaded += article.bytes; - } - } - file.bytes_downloaded = file_bytes_downloaded; - if file.articles.iter().all(|a| a.downloaded) { - file.assembled = true; - } - } - } + apply_checkpoint(&mut job, &checkpoint); let remaining = job .article_count @@ -3241,6 +4038,16 @@ impl QueueManager { // Start queued jobs up to the concurrency limit self.start_next_queued(); + for (job_id, articles_failed) in postproc_recovery { + let manager = Arc::clone(self); + tokio::spawn(async move { + info!(job_id, "Resuming interrupted post-processing pipeline"); + manager + .on_job_finished(&job_id, articles_failed == 0, articles_failed) + .await; + }); + } + Ok(()) } @@ -3490,16 +4297,17 @@ impl QueueManager { info!(total_nntp_connections = total, "NNTP connection summary"); } } - if tick_count.is_multiple_of(30) && qm.min_free_space > 0 { - let free = get_disk_free(&qm.incomplete_dir.lock()); - if free > 0 - && free < qm.min_free_space + if tick_count.is_multiple_of(30) && qm.min_free_space() > 0 { + let paths = qm.disk_guard_paths(); + let path_refs: Vec<_> = paths.iter().map(std::path::PathBuf::as_path).collect(); + let free = paths.first().map_or(0, |path| get_disk_free(path)); + if !disk_space_available(qm.min_free_space(), &path_refs) && !qm.globally_paused.load(Ordering::Relaxed) { warn!( free_bytes = free, - min_free_space = qm.min_free_space, - "Low disk space, auto-pausing downloads" + min_free_space = qm.min_free_space(), + "Low disk space on a configured storage volume, auto-pausing downloads" ); qm.pause_all(); } @@ -3579,6 +4387,88 @@ mod global_pause_tests { manager.job_order.lock().push(id); } + #[tokio::test] + async fn remaining_percentage_sort_is_stable_and_does_not_change_status() { + let (manager, tempdir) = manager(); + let mut first = job("first", JobStatus::Downloading, tempdir.path()); + first.total_bytes = 100; + first.downloaded_bytes = 50; + let mut second = job("second", JobStatus::Queued, tempdir.path()); + second.total_bytes = 200; + second.downloaded_bytes = 100; + let mut third = job("third", JobStatus::Queued, tempdir.path()); + third.total_bytes = 100; + third.downloaded_bytes = 10; + insert_job(&manager, first); + insert_job(&manager, second); + insert_job(&manager, third); + + manager.sort_by_remaining_percentage(true); + assert_eq!( + manager + .job_order + .lock() + .iter() + .map(String::as_str) + .collect::>(), + vec!["first", "second", "third"] + ); + assert_eq!( + manager.get_job("first").unwrap().status, + JobStatus::Downloading + ); + + manager.sort_by_remaining_percentage(false); + assert_eq!( + manager + .job_order + .lock() + .iter() + .map(String::as_str) + .collect::>(), + vec!["third", "first", "second"] + ); + } + + #[test] + fn script_paths_are_confined_to_the_script_directory() { + let dir = tempfile::tempdir().unwrap(); + let scripts = dir.path().join("scripts"); + std::fs::create_dir_all(&scripts).unwrap(); + let script = scripts.join("success.sh"); + std::fs::write(&script, b"#!/bin/sh\nexit 0\n").unwrap(); + + let resolved = + resolve_script_path(Some(&scripts), std::path::Path::new("success.sh")).unwrap(); + assert_eq!(resolved, std::fs::canonicalize(script).unwrap()); + assert!(resolve_script_path(Some(&scripts), std::path::Path::new("../escape.sh")).is_err()); + assert!(resolve_script_path(None, std::path::Path::new("success.sh")).is_err()); + } + + #[tokio::test] + async fn script_output_is_bounded_and_captured() { + let (manager, tempdir) = manager(); + let script = tempdir.path().join("script.sh"); + std::fs::write(&script, b"#!/bin/sh\nprintf '1234567890'\n").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + manager.set_postproc_scripts(None, Some(script), None, 2, 4); + let mut completed = job("script-job", JobStatus::Completed, tempdir.path()); + completed.output_dir = tempdir.path().join("output"); + std::fs::create_dir_all(&completed.output_dir).unwrap(); + insert_job(&manager, completed); + + let stage = manager + .run_postproc_script("script-job", JobStatus::Completed) + .await + .unwrap(); + assert_eq!(stage.status, StageStatus::Success); + assert!(stage.message.unwrap().contains("output truncated")); + } + #[tokio::test] async fn active_queue_view_excludes_terminal_jobs() { let (manager, tempdir) = manager(); @@ -3753,6 +4643,78 @@ mod global_pause_tests { .as_deref(), Some("original failure") ); + assert_eq!(manager.history_update(), 2); + } + + #[tokio::test] + async fn history_generation_changes_only_for_real_mutations() { + let (manager, tempdir) = manager(); + assert_eq!(manager.history_update(), 1); + + manager.history_clear().unwrap(); + manager.history_remove("missing").unwrap(); + assert_eq!(manager.history_update(), 1); + + insert_job( + &manager, + job("counter-terminal", JobStatus::Completed, tempdir.path()), + ); + { + let mut jobs = manager.jobs.lock(); + manager.move_to_history(jobs.get_mut("counter-terminal").unwrap(), Vec::new()); + } + assert_eq!(manager.history_update(), 2); + + manager.history_remove("counter-terminal").unwrap(); + assert_eq!(manager.history_update(), 3); + + manager.history_remove("counter-terminal").unwrap(); + manager.history_clear().unwrap(); + assert_eq!(manager.history_update(), 3); + } + + /// GH #136: a configured retention of 0 used to run `LIMIT 0` retention + /// right after the insert and silently delete the row just persisted. + #[tokio::test] + async fn zero_history_retention_keeps_completed_jobs() { + let (manager, tempdir) = manager(); + manager.set_history_retention(Some(0)); + assert_eq!(manager.get_history_retention(), None); + + insert_job( + &manager, + job("zero-retention", JobStatus::Completed, tempdir.path()), + ); + { + let mut jobs = manager.jobs.lock(); + manager.move_to_history(jobs.get_mut("zero-retention").unwrap(), Vec::new()); + } + + let db = manager.db.lock(); + let entry = db + .history_get("zero-retention") + .unwrap() + .expect("completed job must remain in history with retention 0"); + assert_eq!(entry.status, JobStatus::Completed); + assert_eq!(db.history_count().unwrap(), 1); + } + + /// A positive retention limit still prunes, oldest first. + #[tokio::test] + async fn positive_history_retention_prunes_after_completion() { + let (manager, tempdir) = manager(); + manager.set_history_retention(Some(1)); + assert_eq!(manager.get_history_retention(), Some(1)); + + for id in ["ret-first", "ret-second"] { + insert_job(&manager, job(id, JobStatus::Completed, tempdir.path())); + let mut jobs = manager.jobs.lock(); + manager.move_to_history(jobs.get_mut(id).unwrap(), Vec::new()); + } + + let db = manager.db.lock(); + assert_eq!(db.history_count().unwrap(), 1); + assert!(db.history_get("ret-second").unwrap().is_some()); } #[tokio::test] @@ -3912,6 +4874,49 @@ mod global_pause_tests { .is_none() ); } + + #[tokio::test] + async fn interrupted_post_processing_resumes_into_terminal_history() { + let (manager, tempdir) = manager(); + let mut interrupted = job( + "restart-postproc", + JobStatus::PostProcessing, + tempdir.path(), + ); + std::fs::create_dir_all(&interrupted.work_dir).unwrap(); + std::fs::write(interrupted.work_dir.join("payload.mkv"), b"payload").unwrap(); + interrupted.total_bytes = 7; + interrupted.downloaded_bytes = 7; + manager.db.lock().queue_insert(&interrupted).unwrap(); + + manager.restore_from_db().unwrap(); + + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if manager + .db + .lock() + .history_get("restart-postproc") + .unwrap() + .is_some() + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("recovered post-processing should reach terminal history"); + + let history = manager + .db + .lock() + .history_get("restart-postproc") + .unwrap() + .unwrap(); + assert_eq!(history.status, JobStatus::Completed); + assert!(interrupted.output_dir.join("payload.mkv").exists()); + } } #[cfg(test)] @@ -4168,4 +5173,24 @@ mod hopeless_tests { assert!(result.is_some(), "late-stage stalls should abort"); assert_eq!(result.unwrap().tier, "no_progress_timeout"); } + + #[test] + fn reset_progress_clock_prevents_abort_after_pause() { + // Simulate a job that was paused for longer than the article timeout: + // its progress clock is stale. Resuming must restart the clock (GH + // #123) so the watchdog does not abort it on the next scan. + let mut t = make_tracker(100, 10); + t.last_progress_at = Instant::now() - Duration::from_secs(600); + assert!( + t.time_based_check(Duration::from_secs(300)).is_some(), + "precondition: a stale clock should abort" + ); + + t.reset_progress_clock(); + + assert!( + t.time_based_check(Duration::from_secs(300)).is_none(), + "resuming a paused job must not abort it: paused time must not count toward the article timeout" + ); + } } diff --git a/src/rss_monitor.rs b/src/rss_monitor.rs index 7164e5e..5ed8bce 100644 --- a/src/rss_monitor.rs +++ b/src/rss_monitor.rs @@ -6,6 +6,9 @@ use arc_swap::ArcSwap; use chrono::Utc; use tracing::{info, warn}; +use crate::fetch_guard::{ + MAX_FETCH_BODY_BYTES, build_fetch_client, read_response_bytes_limited, validate_fetch_url, +}; use crate::nzb_core::config::{AppConfig, RssFeedConfig}; use crate::nzb_core::models::{Priority, RssItem}; @@ -85,11 +88,6 @@ impl RssMonitor { // Migrate legacy seen file on first run self.migrate_seen_json(); - let client = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(30)) - .build() - .expect("Failed to create HTTP client"); - loop { let cfg = self.config.load(); let feeds = &cfg.rss_feeds; @@ -99,7 +97,7 @@ impl RssMonitor { continue; } - if let Err(e) = self.check_feed(&client, feed).await { + if let Err(e) = self.check_feed(feed).await { warn!(feed = %feed.name, error = %e, "RSS feed check failed"); } } @@ -114,6 +112,15 @@ impl RssMonitor { info!(pruned, "Pruned old RSS items"); } + if let Some(days) = cfg.general.rss_downloaded_item_expiry_days { + let cutoff = (Utc::now() - chrono::Duration::days(days as i64)).to_rfc3339(); + if let Ok(expired) = self.queue_manager.rss_items_expire_downloaded(&cutoff) + && expired > 0 + { + info!(expired, "Expired downloaded RSS items"); + } + } + // Use the minimum poll interval across all enabled feeds, defaulting to 15 min let interval = feeds .iter() @@ -127,22 +134,31 @@ impl RssMonitor { } } - async fn check_feed( - &self, - client: &reqwest::Client, - feed: &RssFeedConfig, - ) -> anyhow::Result<()> { + async fn check_feed(&self, feed: &RssFeedConfig) -> anyhow::Result<()> { info!(feed = %feed.name, url = %feed.url, "Checking RSS feed"); - let response = client.get(&feed.url).send().await?; - let body = response.bytes().await?; + let feed_plan = validate_fetch_url(&feed.url) + .await + .map_err(|error| anyhow::anyhow!(error.to_string()))?; + let feed_client = + build_fetch_client(&feed_plan).map_err(|error| anyhow::anyhow!(error.to_string()))?; + let response = feed_client.get(feed_plan.url).send().await?; + if !response.status().is_success() { + anyhow::bail!("HTTP {}", response.status()); + } + let body = read_response_bytes_limited(response, MAX_FETCH_BODY_BYTES) + .await + .map_err(|error| anyhow::anyhow!(error.to_string()))?; let parsed = feed_rs::parser::parse(&body[..])?; // Compile filter regex if provided - let filter = feed - .filter_regex - .as_ref() - .and_then(|r| regex::Regex::new(r).ok()); + let filter = match feed.filter_regex.as_deref() { + None => None, + Some(pattern) => Some( + Self::compile_filter(pattern) + .ok_or_else(|| anyhow::anyhow!("invalid RSS filter expression"))?, + ), + }; // Load download rules for this feed let rules = self @@ -178,6 +194,14 @@ impl RssMonitor { .unwrap_or(0); let published_at = entry.published.or(entry.updated); + if let Some(max_age_days) = feed.max_age_days + && let Some(published_at) = published_at + && now.signed_duration_since(published_at).num_seconds() + > (max_age_days as i64).saturating_mul(86_400) + { + continue; + } + pending.push(PendingItem { item: RssItem { id: entry.id.clone(), @@ -198,6 +222,17 @@ impl RssMonitor { // Batch insert all items in one transaction (single DB lock) let items_for_insert: Vec = pending.iter().map(|p| p.item.clone()).collect(); + let downloaded_ids: HashSet = pending + .iter() + .filter(|pending| { + self.queue_manager + .rss_item_get(&pending.item.id) + .ok() + .flatten() + .is_some_and(|item| item.downloaded) + }) + .map(|pending| pending.item.id.clone()) + .collect(); let new_items = self .queue_manager .rss_items_batch_upsert(&items_for_insert) @@ -205,9 +240,14 @@ impl RssMonitor { // Now process auto-downloads for newly inserted items only // (batch_upsert uses INSERT OR IGNORE so only new items get inserted) + let mut handled_ids = HashSet::new(); for p in &pending { let Some(ref url) = p.nzb_url else { continue }; + if downloaded_ids.contains(&p.item.id) || !handled_ids.insert(p.item.id.clone()) { + continue; + } + // Feed-level filter must pass (if set) let passes_filter = match filter { Some(ref re) => re.is_match(&p.title), @@ -219,7 +259,7 @@ impl RssMonitor { // Check download rules let matched_rule = rules.iter().find(|r| { - regex::Regex::new(&r.match_regex) + Self::compile_filter(&r.match_regex) .map(|re| re.is_match(&p.title)) .unwrap_or(false) }); @@ -244,26 +284,10 @@ impl RssMonitor { continue; } - // Skip if already downloaded (existing item in DB) - if self - .queue_manager - .rss_item_exists(&p.item.id) - .unwrap_or(false) - { - // Item existed before this batch — already processed previously - // Check if it was newly inserted by seeing if it's in our new count - // Actually, we can just check the downloaded flag - if let Ok(Some(existing)) = self.queue_manager.rss_item_get(&p.item.id) - && existing.downloaded - { - continue; - } - } - info!(feed = %feed.name, title = %p.title, url = %url, "Auto-downloading RSS item"); match self - .fetch_and_enqueue(client, url, &p.title, feed, category.as_deref(), priority) + .fetch_and_enqueue(url, &p.title, feed, category.as_deref(), priority) .await { Ok(()) => { @@ -285,6 +309,17 @@ impl RssMonitor { Ok(()) } + fn compile_filter(pattern: &str) -> Option { + const MAX_PATTERN_BYTES: usize = 512; + if pattern.len() > MAX_PATTERN_BYTES { + return None; + } + regex::RegexBuilder::new(pattern) + .size_limit(1024 * 1024) + .build() + .ok() + } + /// Extract NZB URL from a feed entry's links or media content. fn extract_nzb_url(entry: &feed_rs::model::Entry) -> Option { entry @@ -314,18 +349,24 @@ impl RssMonitor { async fn fetch_and_enqueue( &self, - client: &reqwest::Client, url: &str, name: &str, feed: &RssFeedConfig, category: Option<&str>, priority: i32, ) -> anyhow::Result<()> { - let response = client.get(url).send().await?; + let plan = validate_fetch_url(url) + .await + .map_err(|error| anyhow::anyhow!(error.to_string()))?; + let client = + build_fetch_client(&plan).map_err(|error| anyhow::anyhow!(error.to_string()))?; + let response = client.get(plan.url).send().await?; if !response.status().is_success() { anyhow::bail!("HTTP {}", response.status()); } - let data = response.bytes().await?; + let data = read_response_bytes_limited(response, MAX_FETCH_BODY_BYTES) + .await + .map_err(|error| anyhow::anyhow!(error.to_string()))?; let mut job = crate::nzb_core::nzb_parser::parse_nzb(name, &data)?; @@ -354,7 +395,7 @@ impl RssMonitor { std::fs::create_dir_all(&job.work_dir)?; - self.queue_manager.add_job(job, Some(data.to_vec()))?; + self.queue_manager.add_job(job, Some(data))?; Ok(()) } } @@ -433,4 +474,33 @@ mod tests { assert!(item.downloaded_at.is_some()); } } + + #[test] + fn feed_filters_fail_closed_at_syntax_and_size_limits() { + assert!(RssMonitor::compile_filter(r"release-[0-9]+").is_some()); + assert!(RssMonitor::compile_filter("(").is_none()); + assert!(RssMonitor::compile_filter(&"x".repeat(513)).is_none()); + } + + #[tokio::test] + async fn feed_checks_reject_local_targets_before_network_access() { + let temp = tempfile::tempdir().expect("tempdir"); + let (monitor, _) = monitor(temp.path().to_path_buf()); + let feed = RssFeedConfig { + name: "local-feed".into(), + url: "http://127.0.0.1:9/feed.xml".into(), + poll_interval_secs: 900, + category: None, + filter_regex: None, + enabled: true, + auto_download: true, + max_age_days: None, + }; + + let error = monitor + .check_feed(&feed) + .await + .expect_err("local addresses must be rejected"); + assert!(error.to_string().contains("private/reserved")); + } } diff --git a/src/sabnzbd_compat.rs b/src/sabnzbd_compat.rs index 44c8e90..f2939e2 100644 --- a/src/sabnzbd_compat.rs +++ b/src/sabnzbd_compat.rs @@ -7,7 +7,8 @@ use std::sync::Arc; use axum::Json; -use axum::extract::{Multipart, Query, State}; +use axum::extract::{Form, FromRequest, Multipart, Query, Request, State}; +use axum::http::{StatusCode, header::CONTENT_TYPE}; use axum::response::IntoResponse; use serde::{Deserialize, Serialize}; @@ -17,6 +18,14 @@ use crate::nzb_core::nzb_parser; use crate::error::ApiError; use crate::state::AppState; +/// SABnzbd release whose public response contract this compatibility layer +/// targets. Keep this in sync with the conformance fixtures under +/// `tests/fixtures/sabnzbd-*`. +const SABNZBD_COMPAT_VERSION: &str = "5.0.4"; + +/// Upper bound on an `addurl`-fetched NZB body, to avoid unbounded memory use. +const MAX_ADDURL_BODY_BYTES: usize = 100 * 1024 * 1024; + /// Arr-compatible API request -- all parameters come as query strings. #[derive(Deserialize, Default)] pub struct SabApiRequest { @@ -27,10 +36,18 @@ pub struct SabApiRequest { pub apikey: Option, pub output: Option, pub cat: Option, + pub category: Option, pub priority: Option, + pub status: Option, + pub search: Option, + pub nzo_ids: Option, pub start: Option, pub limit: Option, + pub failed_only: Option, + pub archive: Option, + pub last_history_update: Option, pub password: Option, + pub del_files: Option, } /// Validate API key. Returns Err with JSON response on failure. @@ -61,17 +78,210 @@ pub async fn h_sabnzbd_api_get( } let mode = req.mode.as_deref().unwrap_or(""); + + // `addurl` fetches a remote NZB and has no file body to upload, so real + // SABnzbd (and clients like NZB360/Sonarr/Radarr) issue it as a plain + // GET rather than a multipart POST. Route it to the same URL-fetching + // logic the POST handler uses so `cat`/`priority` are honored here too. + if mode == "addurl" { + let url = req.name.clone().or_else(|| req.value.clone()); + return handle_addurl( + &state, + url, + req.name.clone(), + req.cat.clone(), + req.priority.clone(), + req.password.clone(), + ) + .await; + } + let result = dispatch_mode(&state, mode, &req); Ok(result) } -/// POST /sabnzbd/api -- Handle POST requests (addfile multipart, or form-encoded). +/// Fetch an NZB from a URL and enqueue it, applying category/priority/password +/// overrides. Shared by the GET and POST `addurl` entry points. +async fn handle_addurl( + state: &AppState, + url: Option, + name: Option, + cat: Option, + priority: Option, + password: Option, +) -> Result, ApiError> { + let url = url.unwrap_or_default(); + + if url.is_empty() { + return Ok(Json(serde_json::json!({ + "status": false, + "error": "No URL provided" + }))); + } + + // SSRF guard: addurl fetches a caller-supplied URL, so validate it and pin + // the connection to the validated address (shared with the native URL-add + // path in the app crate). Rejects non-http(s) schemes and private/reserved + // hosts such as 169.254.169.254. See rustnzb#129 review. + let fetch_plan = match crate::fetch_guard::validate_fetch_url(&url).await { + Ok(plan) => plan, + Err(error) => { + tracing::warn!(url = %url, %error, "Refusing addurl fetch (SSRF guard)"); + return Ok(Json(serde_json::json!({ + "status": false, + "error": error.to_string() + }))); + } + }; + + tracing::info!(url = %url, "Fetching NZB from URL via arr API"); + + let client = crate::fetch_guard::build_fetch_client(&fetch_plan)?; + + let response = client + .get(fetch_plan.url.clone()) + .send() + .await + .map_err(|e| ApiError::from(anyhow::anyhow!("Failed to fetch URL: {e}")))?; + + if !response.status().is_success() { + return Ok(Json(serde_json::json!({ + "status": false, + "error": format!("URL returned HTTP {}", response.status()) + }))); + } + + // Cap the fetched body to avoid unbounded memory from a hostile URL. + let data = + crate::fetch_guard::read_response_bytes_limited(response, MAX_ADDURL_BODY_BYTES).await?; + + // Derive job name from URL filename if not provided + let job_name = name.unwrap_or_else(|| { + url.rsplit('/') + .next() + .and_then(|s| s.split('?').next()) + .unwrap_or("unknown") + .strip_suffix(".nzb") + .unwrap_or( + url.rsplit('/') + .next() + .and_then(|s| s.split('?').next()) + .unwrap_or("unknown"), + ) + .to_string() + }); + + match nzb_parser::parse_nzb(&job_name, &data) { + Ok(mut job) => { + if let Some(ref c) = cat + && !c.is_empty() + { + job.category = sab_resolve_category(c).to_string(); + } + if let Some(ref p) = priority { + job.priority = sab_priority_to_priority(p); + } + + // API-provided password overrides NZB metadata password + if let Some(ref pw) = password { + job.password = Some(pw.clone()); + } + + let qm = &state.queue_manager; + job.work_dir = qm.incomplete_dir().join(&job.id); + job.output_dir = match qm.output_dir_for(&job.category, &job.name) { + Ok(path) => path, + Err(error) => { + return Ok(Json(serde_json::json!({ + "status": false, + "error": error.to_string() + }))); + } + }; + + let nzo_id = format!("SABnzbd_nzo_{}", &job.id[..12.min(job.id.len())]); + let job_name = job.name.clone(); + let job_id = job.id.clone(); + let file_count = job.file_count; + + // As with addfile, report a failed enqueue as `status: false` on + // HTTP 200 rather than a 500, and log success only after the + // enqueue succeeds (rustnzb#129). + let nzb_bytes = data.to_vec(); + if let Err(error) = qm.add_job(job, Some(nzb_bytes)) { + tracing::error!( + name = %job_name, + id = %job_id, + %error, + "Failed to add NZB to queue via URL (arr API)" + ); + return Ok(Json(serde_json::json!({ + "status": false, + "error": error.to_string() + }))); + } + + tracing::info!( + name = %job_name, + id = %job_id, + files = file_count, + "NZB added to queue via URL (arr API)" + ); + + Ok(Json(serde_json::json!({ + "status": true, + "nzo_ids": [nzo_id] + }))) + } + Err(e) => Ok(Json(serde_json::json!({ + "status": false, + "error": format!("Failed to parse NZB: {e}") + }))), + } +} + +/// Body encodings a SABnzbd client may use for a POST request. +enum SabPostBody { + /// `multipart/form-data` -- the only encoding that can carry an NZB file. + Multipart, + /// `application/x-www-form-urlencoded` -- plain key/value fields. + Form, + /// No body, or an encoding we don't parse: parameters come from the + /// query string alone. + None, +} + +fn classify_post_body(request: &Request) -> SabPostBody { + let content_type = request + .headers() + .get(CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .map(|v| v.trim().to_ascii_lowercase()) + .unwrap_or_default(); + if content_type.starts_with("multipart/form-data") { + SabPostBody::Multipart + } else if content_type.starts_with("application/x-www-form-urlencoded") { + SabPostBody::Form + } else { + SabPostBody::None + } +} + +/// POST /sabnzbd/api -- Handle POST requests. +/// +/// The body is optional. `mode=addfile` needs a `multipart/form-data` upload, +/// but clients such as Prowlarr send `mode=addurl` as a bare POST with every +/// parameter in the query string and no body at all (#119), and others use +/// `application/x-www-form-urlencoded`. Requiring the multipart extractor +/// unconditionally rejected those with `400 Invalid boundary` before the +/// mode was ever inspected, so the body is only parsed as multipart when the +/// request actually says it is one. pub async fn h_sabnzbd_api_post( State(state): State>, Query(query_req): Query, - mut multipart: Multipart, + request: Request, ) -> Result { - // Extract fields from multipart form data + // Query-string parameters are the baseline; body fields override them. let mut mode = query_req.mode.clone().unwrap_or_default(); let mut apikey = query_req.apikey.clone(); let mut cat = query_req.cat.clone(); @@ -81,6 +291,79 @@ pub async fn h_sabnzbd_api_post( let mut nzb_url: Option = None; let mut password: Option = query_req.password.clone(); + match classify_post_body(&request) { + SabPostBody::None => {} + SabPostBody::Form => { + let Form(form) = Form::::from_request(request, &()) + .await + .map_err(|e| { + ApiError::from((StatusCode::BAD_REQUEST, format!("Form error: {e}"))) + })?; + if let Some(m) = form.mode.filter(|m| !m.is_empty()) { + mode = m; + } + if form.apikey.is_some() { + apikey = form.apikey; + } + if form.cat.is_some() { + cat = form.cat; + } + if form.priority.is_some() { + priority = form.priority; + } + if form.name.is_some() { + name = form.name; + } + if form.value.is_some() { + nzb_url = form.value; + } + if let Some(pw) = form.password.filter(|pw| !pw.is_empty()) { + password = Some(pw); + } + } + SabPostBody::Multipart => { + let mut multipart = Multipart::from_request(request, &()).await.map_err(|e| { + ApiError::from((StatusCode::BAD_REQUEST, format!("Multipart error: {e}"))) + })?; + read_multipart_fields( + &mut multipart, + &mut mode, + &mut apikey, + &mut cat, + &mut priority, + &mut name, + &mut nzb_data, + &mut nzb_url, + &mut password, + ) + .await?; + } + } + + // Validate API key + if let Err(resp) = validate_api_key(&state, apikey.as_deref()) { + return Ok(resp); + } + + dispatch_post( + &state, mode, name, cat, priority, nzb_data, nzb_url, password, query_req, + ) + .await +} + +/// Fold the fields of a multipart body into the request parameters. +#[allow(clippy::too_many_arguments)] +async fn read_multipart_fields( + multipart: &mut Multipart, + mode: &mut String, + apikey: &mut Option, + cat: &mut Option, + priority: &mut Option, + name: &mut Option, + nzb_data: &mut Option<(String, Vec)>, + nzb_url: &mut Option, + password: &mut Option, +) -> Result<(), ApiError> { while let Some(field) = multipart .next_field() .await @@ -92,22 +375,22 @@ pub async fn h_sabnzbd_api_post( if let Ok(text) = field.text().await && !text.is_empty() { - mode = text; + *mode = text; } } "apikey" => { if let Ok(text) = field.text().await { - apikey = Some(text); + *apikey = Some(text); } } "cat" => { if let Ok(text) = field.text().await { - cat = Some(text); + *cat = Some(text); } } "priority" => { if let Ok(text) = field.text().await { - priority = Some(text); + *priority = Some(text); } } "name" => { @@ -123,9 +406,9 @@ pub async fn h_sabnzbd_api_post( .bytes() .await .map_err(|e| ApiError::from(anyhow::anyhow!("Read error: {e}")))?; - nzb_data = Some((file_name, data.to_vec())); + *nzb_data = Some((file_name, data.to_vec())); } else if let Ok(text) = field.text().await { - name = Some(text); + *name = Some(text); } } "nzbfile" => { @@ -137,18 +420,18 @@ pub async fn h_sabnzbd_api_post( .bytes() .await .map_err(|e| ApiError::from(anyhow::anyhow!("Read error: {e}")))?; - nzb_data = Some((file_name, data.to_vec())); + *nzb_data = Some((file_name, data.to_vec())); } "value" | "url" => { if let Ok(text) = field.text().await { - nzb_url = Some(text); + *nzb_url = Some(text); } } "password" => { if let Ok(text) = field.text().await && !text.is_empty() { - password = Some(text); + *password = Some(text); } } _ => { @@ -156,12 +439,23 @@ pub async fn h_sabnzbd_api_post( } } } + Ok(()) +} - // Validate API key - if let Err(resp) = validate_api_key(&state, apikey.as_deref()) { - return Ok(resp); - } - +/// Dispatch a POST request once its parameters have been assembled from the +/// query string and (optional) body. +#[allow(clippy::too_many_arguments)] +async fn dispatch_post( + state: &AppState, + mode: String, + name: Option, + cat: Option, + priority: Option, + nzb_data: Option<(String, Vec)>, + nzb_url: Option, + password: Option, + query_req: SabApiRequest, +) -> Result, ApiError> { match mode.as_str() { "addfile" => { let (file_name, data) = match nzb_data { @@ -186,7 +480,7 @@ pub async fn h_sabnzbd_api_post( if let Some(ref c) = cat && !c.is_empty() { - job.category = c.clone(); + job.category = sab_resolve_category(c).to_string(); } if let Some(ref p) = priority { job.priority = sab_priority_to_priority(p); @@ -199,20 +493,50 @@ pub async fn h_sabnzbd_api_post( let qm = &state.queue_manager; job.work_dir = qm.incomplete_dir().join(&job.id); - job.output_dir = qm.complete_dir().join(&job.category).join(&job.name); + job.output_dir = match qm.output_dir_for(&job.category, &job.name) { + Ok(path) => path, + Err(error) => { + return Ok(Json(serde_json::json!({ + "status": false, + "error": error.to_string() + }))); + } + }; let nzo_id = format!("SABnzbd_nzo_{}", &job.id[..12.min(job.id.len())]); + let job_name = job.name.clone(); + let job_id = job.id.clone(); + let file_count = job.file_count; + + // SABnzbd's addfile always responds HTTP 200 with a JSON + // `status` field; a failed enqueue is reported as + // `status: false`, never a 5xx. Propagating the error as a + // 500 here made Sonarr treat an otherwise-reportable + // failure as a hard download-client error, and the log + // claimed success before the enqueue that actually failed + // (rustnzb#129). Enqueue first, then report the real + // outcome -- mirroring the history-retry path. + let nzb_bytes = data.clone(); + if let Err(error) = qm.add_job(job, Some(nzb_bytes)) { + tracing::error!( + name = %job_name, + id = %job_id, + %error, + "Failed to add NZB to queue via arr API" + ); + return Ok(Json(serde_json::json!({ + "status": false, + "error": error.to_string() + }))); + } tracing::info!( - name = %job.name, - id = %job.id, - files = job.file_count, + name = %job_name, + id = %job_id, + files = file_count, "NZB added to queue via arr API" ); - let nzb_bytes = data.clone(); - qm.add_job(job, Some(nzb_bytes)).map_err(ApiError::from)?; - Ok(Json(serde_json::json!({ "status": true, "nzo_ids": [nzo_id] @@ -226,117 +550,38 @@ pub async fn h_sabnzbd_api_post( } "addurl" => { - let url = nzb_url.or(name.clone()).unwrap_or_default(); - - if url.is_empty() { - return Ok(Json(serde_json::json!({ - "status": false, - "error": "No URL provided" - }))); - } - - tracing::info!(url = %url, "Fetching NZB from URL via arr API"); - - // Fetch the NZB from the URL - let client = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(30)) - .build() - .map_err(|e| ApiError::from(anyhow::anyhow!("HTTP client error: {e}")))?; - - let response = client - .get(&url) - .send() - .await - .map_err(|e| ApiError::from(anyhow::anyhow!("Failed to fetch URL: {e}")))?; - - if !response.status().is_success() { - return Ok(Json(serde_json::json!({ - "status": false, - "error": format!("URL returned HTTP {}", response.status()) - }))); - } - - let data = response - .bytes() - .await - .map_err(|e| ApiError::from(anyhow::anyhow!("Failed to read response: {e}")))?; - - // Derive job name from URL filename if not provided - let job_name = name.clone().unwrap_or_else(|| { - url.rsplit('/') - .next() - .and_then(|s| s.split('?').next()) - .unwrap_or("unknown") - .strip_suffix(".nzb") - .unwrap_or( - url.rsplit('/') - .next() - .and_then(|s| s.split('?').next()) - .unwrap_or("unknown"), - ) - .to_string() - }); - - match nzb_parser::parse_nzb(&job_name, &data) { - Ok(mut job) => { - if let Some(ref c) = cat - && !c.is_empty() - { - job.category = c.clone(); - } - if let Some(ref p) = priority { - job.priority = sab_priority_to_priority(p); - } - - // API-provided password overrides NZB metadata password - if let Some(ref pw) = password { - job.password = Some(pw.clone()); - } - - let qm = &state.queue_manager; - job.work_dir = qm.incomplete_dir().join(&job.id); - job.output_dir = qm.complete_dir().join(&job.category).join(&job.name); - - let nzo_id = format!("SABnzbd_nzo_{}", &job.id[..12.min(job.id.len())]); - - tracing::info!( - name = %job.name, - id = %job.id, - files = job.file_count, - "NZB added to queue via URL (arr API)" - ); - - let nzb_bytes = data.to_vec(); - qm.add_job(job, Some(nzb_bytes)).map_err(ApiError::from)?; - - Ok(Json(serde_json::json!({ - "status": true, - "nzo_ids": [nzo_id] - }))) - } - Err(e) => Ok(Json(serde_json::json!({ - "status": false, - "error": format!("Failed to parse NZB: {e}") - }))), - } + let url = nzb_url.or_else(|| name.clone()); + handle_addurl(state, url, name, cat, priority, password).await } _ => { let req = SabApiRequest { mode: Some(mode), name, - value: None, - value2: None, - apikey, + // Sub-commands like queue/history delete, priority, and + // rename take `value`/`value2` as plain query-string + // parameters even on POST -- these were previously dropped + // here, silently breaking those actions over POST. + value: query_req.value, + value2: query_req.value2, + apikey: None, // already validated by the caller output: None, cat, + category: query_req.category, priority, + status: query_req.status, + search: query_req.search, + nzo_ids: query_req.nzo_ids, start: query_req.start, limit: query_req.limit, + failed_only: query_req.failed_only, + archive: query_req.archive, + last_history_update: query_req.last_history_update, password, + del_files: query_req.del_files, }; Ok(dispatch_mode( - &state, + state, req.mode.as_deref().unwrap_or(""), &req, )) @@ -348,7 +593,7 @@ pub async fn h_sabnzbd_api_post( fn dispatch_mode(state: &AppState, mode: &str, req: &SabApiRequest) -> Json { match mode { "version" => Json(serde_json::json!({ - "version": "4.3.3" + "version": SABNZBD_COMPAT_VERSION })), "queue" => handle_queue(state, req), @@ -359,6 +604,12 @@ fn dispatch_mode(state: &AppState, mode: &str, req: &SabApiRequest) -> Json handle_get_cats(state), + // RustNZB doesn't support post-processing scripts, so this is the + // permanent, correct response -- it matches what real SABnzbd + // reports when no script directory / scripts are configured + // (sabnzbd/api.py::_api_get_scripts -> filesystem.py::list_scripts). + "get_scripts" => handle_get_scripts(state), + "change_cat" => handle_change_cat(state, req), "rename" => handle_rename(state, req), @@ -372,16 +623,7 @@ fn dispatch_mode(state: &AppState, mode: &str, req: &SabApiRequest) -> Json handle_priority(state, req), - "fullstatus" | "server_stats" => { - let qm = &state.queue_manager; - Json(serde_json::json!({ - "status": { - "version": "4.3.3", - "paused": qm.is_paused(), - "speed": format!("{}", qm.get_speed()), - } - })) - } + "fullstatus" | "server_stats" => handle_fullstatus(state), "pause" => handle_pause(state, req), @@ -398,6 +640,109 @@ fn dispatch_mode(state: &AppState, mode: &str, req: &SabApiRequest) -> Json Json { + let config = state.config(); + let mut scripts = Vec::new(); + if let Some(directory) = config.general.scripts_dir.as_ref() + && let Ok(entries) = std::fs::read_dir(directory) + { + for entry in entries.flatten() { + let path = entry.path(); + if entry + .file_type() + .map(|kind| kind.is_file()) + .unwrap_or(false) + && path.file_name().and_then(|name| name.to_str()).is_some() + { + scripts.push(path.file_name().unwrap().to_string_lossy().into_owned()); + } + } + } + scripts.sort(); + if scripts.is_empty() { + scripts.push("None".to_string()); + } + Json(serde_json::json!({ "scripts": scripts })) +} + +/// Return the stable subset of SABnzbd's full-status dashboard contract. +/// +/// SAB-compatible clients inspect this response as a capability/status +/// document, so preserving its keys and JSON types is more important than +/// inventing measurements RustNZB does not currently collect. Unsupported +/// dashboard measurements therefore use SABnzbd-compatible empty/zero values. +fn handle_fullstatus(state: &AppState) -> Json { + let config = state.config(); + let general = &config.general; + let qm = &state.queue_manager; + let speed_limit = qm.get_speed_limit(); + let pause_int = qm.pause_remaining_secs().unwrap_or(0).max(0).to_string(); + + Json(serde_json::json!({ + "status": { + "active_lang": "en", + "active_socks5_proxy": serde_json::Value::Null, + "apikey": general.api_key.as_deref().unwrap_or(""), + "cache_art": "0", + "cache_size": format_size_human(general.cache_size), + "color_scheme": "Auto", + "completedir": general.complete_dir.to_string_lossy(), + "completedirspeed": 0, + "configfn": state.config_path.to_string_lossy(), + "confighelpuri": "https://sabnzbd.org/wiki/configuration/5.0/", + "delayed_assembler": 0, + "diskspace1": "0.00", + "diskspace1_norm": "0 B", + "diskspace2": "0.00", + "diskspace2_norm": "0 B", + "diskspacetotal1": "0.00", + "diskspacetotal2": "0.00", + "dnslookup": false, + "downloaddir": general.incomplete_dir.to_string_lossy(), + "downloaddirspeed": 0, + "finishaction": serde_json::Value::Null, + "folders": Vec::::new(), + "have_quota": false, + "have_warnings": "0", + "internetbandwidth": 0, + "ipv6": serde_json::Value::Null, + "left_quota": "0 B", + "loadavg": "", + "localipv4": serde_json::Value::Null, + "logfile": general + .log_file + .as_ref() + .map_or_else(String::new, |path| path.to_string_lossy().into_owned()), + "loglevel": &general.log_level, + "macos": cfg!(target_os = "macos"), + "my_home": general.data_dir.to_string_lossy(), + "my_lcldata": general.data_dir.to_string_lossy(), + "new_rel_url": serde_json::Value::Null, + "new_release": serde_json::Value::Null, + "pause_int": pause_int, + "paused": qm.is_paused(), + "paused_all": false, + "pid": std::process::id(), + "power_options": false, + "pp_pause_event": false, + "publicipv4": serde_json::Value::Null, + "pystone": 0, + "quota": "0 B", + "rtl": false, + "servers": Vec::::new(), + "speedlimit": if speed_limit == 0 { "0" } else { "100" }, + "speedlimit_abs": speed_limit.to_string(), + "uptime": "0m", + "url_base": "", + "version": SABNZBD_COMPAT_VERSION, + "warnings": Vec::::new(), + "webdir": "", + "weblogfile": serde_json::Value::Null, + "windows": cfg!(target_os = "windows"), + } + })) +} + // --------------------------------------------------------------------------- // Mode handlers // --------------------------------------------------------------------------- @@ -405,48 +750,206 @@ fn dispatch_mode(state: &AppState, mode: &str, req: &SabApiRequest) -> Json Json { let qm = &state.queue_manager; - // Sub-commands: mode=queue&name=delete|pause|resume&value=nzo_ID + // Sub-commands dispatched via mode=queue&name=, matching SABnzbd's + // real `_api_queue_table` (delete, pause, resume, priority, rename, + // purge, change_complete_action). `sort` and `delete_nzf` have no + // equivalent capability in RustNZB's queue manager yet. match req.name.as_deref() { Some("delete") => return handle_queue_delete(state, req), Some("pause") => return handle_queue_item_pause(state, req), Some("resume") => return handle_queue_item_resume(state, req), + Some("priority") => return handle_queue_priority(state, req), + Some("rename") => return handle_queue_rename(state, req), + Some("purge") => return handle_queue_purge(state), + Some("sort") => { + let ascending = !matches!( + req.value.as_deref(), + Some(value) + if value.eq_ignore_ascii_case("descending") + || value.eq_ignore_ascii_case("desc") + ); + qm.sort_by_remaining_percentage(ascending); + return Json(serde_json::json!({ "status": true })); + } + Some("change_complete_action") => return Json(serde_json::json!({ "status": true })), _ => {} } let jobs = qm.get_active_jobs(); let paused = qm.is_paused(); let speed_bps = qm.get_speed(); + let speed_limit_bps = qm.get_speed_limit(); + + Json(build_queue_response( + &jobs, + paused, + speed_bps, + speed_limit_bps, + req, + )) +} - let slots: Vec = jobs.iter().map(SabQueueSlot::from_job).collect(); +fn build_queue_response( + jobs: &[NzbJob], + paused: bool, + speed_bps: u64, + speed_limit_bps: u64, + req: &SabApiRequest, +) -> serde_json::Value { + let start = req.start.unwrap_or(0); + let limit = req.limit.unwrap_or(0); + let category_query = req + .cat + .as_deref() + .filter(|value| !value.trim().is_empty()) + .or(req.category.as_deref()); + let categories = comma_separated(category_query); + let priorities = comma_separated(req.priority.as_deref()); + let statuses = comma_separated(req.status.as_deref()); + let nzo_ids = comma_separated(req.nzo_ids.as_deref()); + let search = req + .search + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_lowercase); - let total_mb: f64 = jobs.iter().map(|j| j.total_bytes as f64).sum::() / 1_048_576.0; - let left_mb: f64 = jobs + let matching_jobs: Vec<&NzbJob> = jobs .iter() - .map(|j| (j.total_bytes.saturating_sub(j.downloaded_bytes)) as f64) - .sum::() - / 1_048_576.0; + .filter(|job| { + search + .as_ref() + .is_none_or(|term| job.name.to_lowercase().contains(term)) + && (categories.is_empty() + || categories + .iter() + .any(|category| category.eq_ignore_ascii_case(&job.category))) + && (priorities.is_empty() + || priorities + .iter() + .any(|priority| sab_priority_matches(job.priority, priority))) + && (statuses.is_empty() + || statuses + .iter() + .any(|status| status.eq_ignore_ascii_case(sab_queue_status(job.status)))) + && (nzo_ids.is_empty() + || nzo_ids + .iter() + .any(|nzo_id| queue_nzo_id(job).eq_ignore_ascii_case(nzo_id))) + }) + .collect(); - Json(serde_json::json!({ + let page = matching_jobs + .iter() + .skip(start) + .take(if limit == 0 { usize::MAX } else { limit }); + let mut running_bytes = matching_jobs + .iter() + .take(start) + .filter(|job| queue_totals_include(job)) + .map(|job| remaining_bytes(job)) + .fold(0_u64, u64::saturating_add); + let slots: Vec = page + .enumerate() + .map(|(offset, job)| { + if queue_totals_include(job) { + running_bytes = running_bytes.saturating_add(remaining_bytes(job)); + } + SabQueueSlot::from_job(job, start + offset, paused, running_bytes, speed_bps) + }) + .collect(); + + let active_totals = jobs.iter().filter(|job| queue_totals_include(job)); + let total_bytes = active_totals + .clone() + .map(|job| job.total_bytes) + .fold(0_u64, u64::saturating_add); + let bytes_left = active_totals + .map(remaining_bytes) + .fold(0_u64, u64::saturating_add); + let total_slots = jobs.iter().filter(|job| queue_totals_include(job)).count(); + let total_mb = total_bytes as f64 / 1_048_576.0; + let left_mb = bytes_left as f64 / 1_048_576.0; + + serde_json::json!({ "queue": { - "status": if paused { "Paused" } else { "Downloading" }, - "speedlimit": "", + "version": SABNZBD_COMPAT_VERSION, + "status": queue_status(paused, speed_bps), + "paused": paused, + "pause_int": "0", + // SABnzbd uses `paused_all` for a distinct scheduler condition; + // the ordinary global pause endpoint only sets `paused`. + "paused_all": false, + "speedlimit": "0", + "speedlimit_abs": speed_limit_bps.to_string(), "speed": format_speed(speed_bps), "kbpersec": format!("{:.2}", speed_bps as f64 / 1024.0), "mbleft": format!("{left_mb:.2}"), "mb": format!("{total_mb:.2}"), - "noofslots_total": jobs.len(), - "noofslots": slots.len(), - "paused": paused, - "limit": req.limit.unwrap_or(0), - "start": req.start.unwrap_or(0), - "timeleft": "0:00:00", - "eta": "unknown", + "sizeleft": format_size_human(bytes_left), + "size": format_size_human(total_bytes), + "noofslots_total": total_slots, + "noofslots": matching_jobs.len(), + "limit": limit, + "start": start, + "finish": start.saturating_add(limit), + "timeleft": format_timeleft(bytes_left, speed_bps), + "diskspace1": "0.00", + "diskspace2": "0.00", + "diskspace1_norm": "0 B", + "diskspace2_norm": "0 B", + "diskspacetotal1": "0.00", + "diskspacetotal2": "0.00", + "have_warnings": "0", + "finishaction": null, + "quota": "0 B", + "have_quota": false, + "left_quota": "0 B", + "cache_art": "0", + "cache_size": "0 B", "slots": slots } - })) + }) +} + +fn comma_separated(value: Option<&str>) -> Vec<&str> { + value + .into_iter() + .flat_map(|value| value.split(',')) + .map(str::trim) + .filter(|value| !value.is_empty()) + .collect() +} + +fn sab_priority_matches(priority: Priority, requested: &str) -> bool { + let numeric = match priority { + Priority::Low => "-1", + Priority::Normal => "0", + Priority::High => "1", + Priority::Force => "2", + }; + requested == numeric || requested.eq_ignore_ascii_case(sab_priority_name(priority)) +} + +fn queue_status(paused: bool, speed_bps: u64) -> &'static str { + if paused { + "Paused" + } else if speed_bps > 0 { + "Downloading" + } else { + "Idle" + } +} + +fn queue_totals_include(job: &NzbJob) -> bool { + !matches!(job.status, JobStatus::Completed | JobStatus::Failed) } -/// Handle mode=queue&name=delete&value=nzo_ID (SABnzbd queue delete) +/// Handle mode=queue&name=delete&value=nzo_id(s) (SABnzbd queue delete). +/// `value` may be a comma-separated list of nzo_ids, matching SABnzbd's +/// `_api_queue_delete`. RustNZB's `remove_job` already always cleans up a +/// job's incomplete work directory, so `del_files` (unlike in history +/// delete) doesn't change queue-delete behavior here. fn handle_queue_delete(state: &AppState, req: &SabApiRequest) -> Json { let target = req.value.as_deref().unwrap_or(""); if target.is_empty() { @@ -456,7 +959,7 @@ fn handle_queue_delete(state: &AppState, req: &SabApiRequest) -> Json Json = Vec::new(); + for raw_id in target.split(',').map(str::trim).filter(|id| !id.is_empty()) { + let search_id = raw_id.strip_prefix("SABnzbd_nzo_").unwrap_or(raw_id); + if let Some(job) = jobs + .iter() + .find(|job| job.id == search_id || job.id.starts_with(search_id)) + { let _ = qm.remove_job(&job.id); tracing::info!(id = %job.id, "Job removed from queue via arr API (mode=queue)"); - return Json(serde_json::json!({ "status": true })); + removed_ids.push(queue_nzo_id(job)); } } - tracing::warn!(search = %search_id, "Queue delete: job not found"); - Json(serde_json::json!({ "status": false })) + Json(serde_json::json!({ "status": !removed_ids.is_empty(), "nzo_ids": removed_ids })) } /// Handle mode=queue&name=pause&value=nzo_ID. @@ -525,6 +1030,70 @@ fn handle_queue_item_resume(state: &AppState, req: &SabApiRequest) -> Json Json { + let target = req.value.as_deref().unwrap_or(""); + let priority = req.value2.as_deref().unwrap_or(""); + if target.is_empty() || priority.is_empty() { + return Json(serde_json::json!({ + "status": false, + "error": "Missing value (job id) or value2 (priority)" + })); + } + + let priority_value = sab_priority_to_priority(priority); + let qm = &state.queue_manager; + // set_job_priority requires an exact job-id match, but clients only ever + // know the truncated SABnzbd_nzo_<12 chars> form -- resolve the full id + // by prefix first, the same way pause/resume/rename/change_cat do. + let jobs = qm.get_jobs(); + let mut applied = false; + for raw_id in target.split(',').map(str::trim).filter(|id| !id.is_empty()) { + let search_id = raw_id.strip_prefix("SABnzbd_nzo_").unwrap_or(raw_id); + if let Some(job) = jobs + .iter() + .find(|job| job.id == search_id || job.id.starts_with(search_id)) + && qm.set_job_priority(&job.id, priority_value).is_ok() + { + applied = true; + } + } + + Json(serde_json::json!({ "status": applied })) +} + +/// Handle mode=queue&name=rename&value=nzo_id&value2=new_name. +fn handle_queue_rename(state: &AppState, req: &SabApiRequest) -> Json { + let target = req.value.as_deref().unwrap_or(""); + let new_name = req.value2.as_deref().unwrap_or(""); + if target.is_empty() || new_name.is_empty() { + return Json(serde_json::json!({ + "status": false, + "error": "Missing value (job id) or value2 (new name)" + })); + } + + let id = target.strip_prefix("SABnzbd_nzo_").unwrap_or(target); + match state.queue_manager.rename_job(id, new_name) { + Ok(()) => Json(serde_json::json!({ "status": true })), + Err(error) => Json(serde_json::json!({ "status": false, "error": error.to_string() })), + } +} + +/// Handle mode=queue&name=purge (remove every queued job). +fn handle_queue_purge(state: &AppState) -> Json { + let qm = &state.queue_manager; + let jobs = qm.get_jobs(); + let nzo_ids: Vec = jobs.iter().map(queue_nzo_id).collect(); + for job in &jobs { + let _ = qm.remove_job(&job.id); + } + tracing::info!(count = nzo_ids.len(), "Queue purged via arr API"); + Json(serde_json::json!({ "status": !nzo_ids.is_empty(), "nzo_ids": nzo_ids })) +} + fn handle_history(state: &AppState, req: &SabApiRequest) -> Json { let qm = &state.queue_manager; @@ -533,29 +1102,168 @@ fn handle_history(state: &AppState, req: &SabApiRequest) -> Json = entries.iter().map(SabHistorySlot::from_entry).collect(); + let history_update = qm.history_update(); + if history_is_unchanged(req.last_history_update, history_update) { + return Json(unchanged_history_response()); + } + + let entries = qm.history_list(i64::MAX as usize).unwrap_or_default(); + let postprocessing: Vec<_> = qm + .get_jobs() + .into_iter() + .filter(|job| { + matches!( + job.status, + JobStatus::Verifying + | JobStatus::Repairing + | JobStatus::Extracting + | JobStatus::PostProcessing + ) + }) + .collect(); + + Json(build_history_response( + &entries, + &postprocessing, + req, + history_update, + )) +} - Json(serde_json::json!({ +fn build_history_response( + entries: &[HistoryEntry], + postprocessing: &[NzbJob], + req: &SabApiRequest, + history_update: u64, +) -> serde_json::Value { + let mut slots: Vec = postprocessing + .iter() + .map(SabHistorySlot::from_postprocessing) + .chain(entries.iter().map(SabHistorySlot::from_entry)) + .filter(|slot| history_slot_matches(slot, req)) + .collect(); + let noofslots = slots.len(); + let ppslots = slots.iter().filter(|slot| slot.postprocessing).count(); + let start = req.start.unwrap_or(0).min(slots.len()); + let limit = req.limit.filter(|limit| *limit != 0).unwrap_or(50); + let end = start.saturating_add(limit).min(slots.len()); + slots = slots.drain(start..end).collect(); + + let total_bytes: u64 = entries.iter().map(|entry| entry.downloaded_bytes).sum(); + let now = chrono::Utc::now(); + let period_bytes = |days| { + entries + .iter() + .filter(|entry| entry.completed_at >= now - chrono::Duration::days(days)) + .map(|entry| entry.downloaded_bytes) + .sum::() + }; + + serde_json::json!({ "history": { - "noofslots": entries.len(), - "last_history_update": chrono::Utc::now().timestamp(), - "slots": slots + "total_size": format_size_human(total_bytes), + "month_size": format_size_human(period_bytes(30)), + "week_size": format_size_human(period_bytes(7)), + "day_size": format_size_human(period_bytes(1)), + "slots": slots, + "noofslots": noofslots, + "ppslots": ppslots, + "last_history_update": history_update, + "version": SABNZBD_COMPAT_VERSION } - })) + }) } -/// Handle mode=history&name=delete&value=nzo_ID (SABnzbd history delete) -fn handle_history_delete(state: &AppState, req: &SabApiRequest) -> Json { - let target = req.value.as_deref().unwrap_or(""); - if target.is_empty() { - return Json(serde_json::json!({ "status": false, "error": "No job ID" })); - } +fn history_is_unchanged(requested: Option, current: u64) -> bool { + requested == Some(current) +} - let qm = &state.queue_manager; +fn unchanged_history_response() -> serde_json::Value { + serde_json::json!({ "history": false }) +} - if target == "all" { +fn history_slot_matches(slot: &SabHistorySlot, req: &SabApiRequest) -> bool { + // RustNZB currently has no archived-history tier, so an archive-only + // request correctly has no matches. + if req.archive.as_deref().is_some_and(sab_query_bool) { + return false; + } + + if let Some(search) = req.search.as_deref().filter(|value| !value.is_empty()) { + let search = search.to_lowercase(); + if !slot.name.to_lowercase().contains(&search) + && !slot.nzb_name.to_lowercase().contains(&search) + { + return false; + } + } + + let categories = req.cat.as_deref().or(req.category.as_deref()); + if !matches_csv(categories, &slot.category) { + return false; + } + + let failed_only = req.failed_only.as_deref().is_some_and(sab_query_bool); + if failed_only { + if !slot.status.eq_ignore_ascii_case("Failed") { + return false; + } + } else if !matches_csv(req.status.as_deref(), &slot.status) { + return false; + } + + req.nzo_ids.as_deref().is_none_or(|ids| { + ids.is_empty() + || ids.split(',').map(str::trim).any(|id| { + id == slot.nzo_id + || slot + .nzo_id + .strip_prefix("SABnzbd_nzo_") + .is_some_and(|raw| raw == id) + || id + .strip_prefix("SABnzbd_nzo_") + .is_some_and(|raw| slot.nzo_id.ends_with(raw)) + }) + }) +} + +fn matches_csv(values: Option<&str>, actual: &str) -> bool { + values.is_none_or(|values| { + values.is_empty() + || values + .split(',') + .map(str::trim) + .any(|value| value.eq_ignore_ascii_case(actual)) + }) +} + +fn sab_query_bool(value: &str) -> bool { + matches!( + value.to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) +} + +/// Handle mode=history&name=delete&value=nzo_ID (SABnzbd history delete) +/// `value` may be a comma-separated list of nzo_ids, matching SABnzbd's +/// `_api_history_delete`. `del_files=1` additionally removes the entry's +/// completed output directory from disk, matching real SABnzbd -- RustNZB +/// otherwise never frees that space on history delete. +fn handle_history_delete(state: &AppState, req: &SabApiRequest) -> Json { + let target = req.value.as_deref().unwrap_or(""); + if target.is_empty() { + return Json(serde_json::json!({ "status": false, "error": "No job ID" })); + } + + let qm = &state.queue_manager; + let del_files = req.del_files.as_deref().is_some_and(sab_query_bool); + + if target.eq_ignore_ascii_case("all") { + if del_files { + for entry in qm.history_list(i64::MAX as usize).unwrap_or_default() { + let _ = std::fs::remove_dir_all(&entry.output_dir); + } + } return match qm.history_clear() { Ok(()) => Json(serde_json::json!({ "status": true })), Err(error) => Json(serde_json::json!({ @@ -565,18 +1273,24 @@ fn handle_history_delete(state: &AppState, req: &SabApiRequest) -> Json = Vec::new(); + for raw_id in target.split(',').map(str::trim).filter(|id| !id.is_empty()) { + let search_id = raw_id.strip_prefix("SABnzbd_nzo_").unwrap_or(raw_id); + if let Some(entry) = entries + .iter() + .find(|entry| entry.id == search_id || entry.id.starts_with(search_id)) + { + if del_files { + let _ = std::fs::remove_dir_all(&entry.output_dir); + } let _ = qm.history_remove(&entry.id); tracing::info!(id = %entry.id, "Entry removed from history via arr API (mode=history)"); - return Json(serde_json::json!({ "status": true })); + removed_ids.push(entry.id.clone()); } } - tracing::warn!(search = %search_id, "History delete: entry not found"); - Json(serde_json::json!({ "status": false })) + Json(serde_json::json!({ "status": !removed_ids.is_empty() })) } fn handle_get_config(state: &AppState) -> Json { @@ -747,7 +1461,16 @@ fn handle_retry(state: &AppState, req: &SabApiRequest) -> Json data, + Err(error) => { + return Json(serde_json::json!({ "status": false, "error": error.to_string() })); + } + }; + let job = match state + .queue_manager + .prepare_retry_job(&entry, &data, retry_data.as_deref()) + { Ok(job) => job, Err(error) => { return Json(serde_json::json!({ @@ -756,13 +1479,6 @@ fn handle_retry(state: &AppState, req: &SabApiRequest) -> Json Json Json(serde_json::json!({ "status": true })), Err(error) => Json(serde_json::json!({ "status": false, "error": error.to_string() })), } } +/// SABnzbd's `get_cats` reports the default category as the literal +/// sentinel `"*"`, not a display name -- verified against +/// `sabnzbd/sabnzbd@5.1.x`, `sabnzbd/api.py::list_cats(default=False)`. +/// RustNZB's own category model still names that category "Default" +/// internally, so translate at the API boundary in both directions. +const SAB_DEFAULT_CATEGORY_SENTINEL: &str = "*"; + fn handle_get_cats(state: &AppState) -> Json { let config = state.config(); - let mut cats: Vec = config.categories.iter().map(|c| c.name.clone()).collect(); - if !cats.iter().any(|c| c == "Default") { - cats.insert(0, "Default".into()); + let mut cats: Vec = config + .categories + .iter() + .map(|c| { + if c.name.eq_ignore_ascii_case("Default") { + SAB_DEFAULT_CATEGORY_SENTINEL.to_string() + } else { + c.name.clone() + } + }) + .collect(); + if !cats.iter().any(|c| c == SAB_DEFAULT_CATEGORY_SENTINEL) { + cats.insert(0, SAB_DEFAULT_CATEGORY_SENTINEL.into()); } Json(serde_json::json!({ "categories": cats })) } +/// Translate a client-supplied category into RustNZB's internal name, +/// resolving SABnzbd's `"*"` default-category sentinel. +fn sab_resolve_category(cat: &str) -> &str { + if cat == SAB_DEFAULT_CATEGORY_SENTINEL { + "Default" + } else { + cat + } +} + +/// `value` may be a comma-separated list of nzo_ids, matching SABnzbd's +/// `_api_change_cat` (`nzo_ids = clean_comma_separated_list(kwargs.get("value"))`). fn handle_change_cat(state: &AppState, req: &SabApiRequest) -> Json { - let job_id = req.value.as_deref().unwrap_or(""); + let job_ids = req.value.as_deref().unwrap_or(""); let new_cat = req.value2.as_deref().unwrap_or(""); - if job_id.is_empty() || new_cat.is_empty() { + if job_ids.is_empty() || new_cat.is_empty() { return Json(serde_json::json!({ "status": false, "error": "Missing value (job id) or value2 (category)" })); } - let search_id = job_id.strip_prefix("SABnzbd_nzo_").unwrap_or(job_id); - let qm = &state.queue_manager; - match qm.change_job_category(search_id, new_cat) { - Ok(()) => Json(serde_json::json!({ "status": true })), - Err(e) => Json(serde_json::json!({ - "status": false, - "error": format!("{e}") - })), + let resolved_cat = sab_resolve_category(new_cat); + let mut changed = false; + for raw_id in job_ids + .split(',') + .map(str::trim) + .filter(|id| !id.is_empty()) + { + let search_id = raw_id.strip_prefix("SABnzbd_nzo_").unwrap_or(raw_id); + if qm.change_job_category(search_id, resolved_cat).is_ok() { + changed = true; + } } + + Json(serde_json::json!({ "status": changed })) } fn handle_rename(state: &AppState, req: &SabApiRequest) -> Json { @@ -876,10 +1631,13 @@ fn handle_rename(state: &AppState, req: &SabApiRequest) -> Json Priority { match s.trim() { - "-100" | "3" => Priority::Force, - "2" => Priority::High, - "1" => Priority::Normal, - "0" => Priority::Low, + "-1" => Priority::Low, + "0" | "-100" => Priority::Normal, + "1" => Priority::High, + // SABnzbd's Force (2) and Repair (3) priorities both mean "jump the + // queue"; RustNZB has no separate Repair concept, so both map to + // our highest priority. + "2" | "3" => Priority::Force, _ => Priority::Normal, } } @@ -890,54 +1648,103 @@ fn sab_priority_to_priority(s: &str) -> Priority { #[derive(Serialize)] struct SabQueueSlot { + index: usize, nzo_id: String, + unpackopts: String, + script: String, filename: String, + labels: Vec, + password: String, cat: String, status: String, priority: String, mb: String, mbleft: String, percentage: String, + mbmissing: String, + direct_unpack: Option, timeleft: String, - eta: String, avg_age: String, size: String, sizeleft: String, + time_added: i64, } impl SabQueueSlot { - fn from_job(job: &NzbJob) -> Self { + fn from_job( + job: &NzbJob, + index: usize, + globally_paused: bool, + running_bytes: u64, + speed_bps: u64, + ) -> Self { let mb = job.total_bytes as f64 / 1_048_576.0; - let mbleft = (job.total_bytes.saturating_sub(job.downloaded_bytes)) as f64 / 1_048_576.0; + let mbleft = remaining_bytes(job) as f64 / 1_048_576.0; let pct = if job.total_bytes > 0 { (job.downloaded_bytes as f64 / job.total_bytes as f64 * 100.0) as u32 } else { 0 }; + let paused = globally_paused || job.status == JobStatus::Paused; Self { - nzo_id: format!("SABnzbd_nzo_{}", &job.id[..12.min(job.id.len())]), + index, + nzo_id: queue_nzo_id(job), + unpackopts: "3".into(), + script: "None".into(), filename: job.name.clone(), - cat: job.category.clone(), - status: sab_queue_status(job.status).into(), - priority: match job.priority { - Priority::Force => "Force".into(), - Priority::High => "High".into(), - Priority::Normal => "Normal".into(), - Priority::Low => "Low".into(), + labels: Vec::new(), + password: job.password.clone().unwrap_or_default(), + cat: if job.category.is_empty() { + "None".into() + } else { + job.category.clone() }, + // RustNZB marks queued jobs Paused when the global gate is + // applied. SABnzbd preserves their queue-facing `Queued` state + // while reporting the gate through the envelope's `paused` key. + status: if globally_paused && job.status == JobStatus::Paused { + "Queued" + } else { + sab_queue_status(job.status) + } + .into(), + priority: sab_priority_name(job.priority).into(), mb: format!("{mb:.2}"), mbleft: format!("{mbleft:.2}"), percentage: format!("{pct}"), - timeleft: "0:00:00".into(), - eta: "unknown".into(), - avg_age: "0d".into(), + mbmissing: "0.00".into(), + direct_unpack: None, + timeleft: if paused { + "0:00:00".into() + } else { + format_timeleft(running_bytes, speed_bps) + }, + avg_age: "-".into(), size: format_size_human(job.total_bytes), - sizeleft: format_size_human(job.total_bytes.saturating_sub(job.downloaded_bytes)), + sizeleft: format_size_human(remaining_bytes(job)), + time_added: job.added_at.timestamp(), } } } +fn queue_nzo_id(job: &NzbJob) -> String { + format!("SABnzbd_nzo_{}", &job.id[..12.min(job.id.len())]) +} + +fn remaining_bytes(job: &NzbJob) -> u64 { + job.total_bytes.saturating_sub(job.downloaded_bytes) +} + +fn sab_priority_name(priority: Priority) -> &'static str { + match priority { + Priority::Force => "Force", + Priority::High => "High", + Priority::Normal => "Normal", + Priority::Low => "Low", + } +} + /// Map internal lifecycle states to the status vocabulary accepted by the /// SABnzbd clients in Sonarr and Radarr. In particular, `PostProcessing` is an /// internal rustnzb state; SABnzbd reports custom post-processing as `Running`. @@ -957,18 +1764,40 @@ fn sab_queue_status(status: JobStatus) -> &'static str { #[derive(Serialize)] struct SabHistorySlot { - nzo_id: String, + completed: i64, name: String, + nzb_name: String, category: String, + pp: String, + script: String, + report: String, + url: String, status: String, - bytes: u64, + nzo_id: String, storage: String, - completed: i64, - fail_message: String, + path: String, + script_line: String, download_time: u64, - pp: String, - nzb_name: String, + postproc_time: u64, stage_log: Vec, + downloaded: u64, + completeness: Option, + fail_message: String, + url_info: String, + bytes: u64, + meta: Option, + series: String, + duplicate_key: String, + md5sum: String, + password: String, + action_line: String, + size: String, + loaded: bool, + retry: bool, + archive: bool, + time_added: i64, + #[serde(skip)] + postprocessing: bool, } #[derive(Serialize)] @@ -988,19 +1817,26 @@ impl SabHistorySlot { }) .collect(); + let storage = entry.output_dir.to_string_lossy().to_string(); + let bytes = entry.downloaded_bytes; Self { - nzo_id: format!("SABnzbd_nzo_{}", &entry.id[..12.min(entry.id.len())]), + completed: entry.completed_at.timestamp(), name: entry.name.clone(), + nzb_name: format!("{}.nzb", entry.name), category: entry.category.clone(), + pp: "D".into(), + script: String::new(), + report: String::new(), + url: String::new(), status: match entry.status { JobStatus::Completed => "Completed".into(), JobStatus::Failed => "Failed".into(), _ => entry.status.to_string(), }, - bytes: entry.downloaded_bytes, - storage: entry.output_dir.to_string_lossy().to_string(), - completed: entry.completed_at.timestamp(), - fail_message: entry.error_message.clone().unwrap_or_default(), + nzo_id: sab_nzo_id(&entry.id), + storage: storage.clone(), + path: storage, + script_line: String::new(), download_time: entry .download_time_secs .unwrap_or_else(|| { @@ -1008,13 +1844,84 @@ impl SabHistorySlot { }) .round() .max(0.0) as u64, - pp: "D".into(), - nzb_name: format!("{}.nzb", entry.name), + postproc_time: entry + .stages + .iter() + .map(|stage| stage.duration_secs.max(0.0)) + .sum::() + .round() as u64, stage_log, + downloaded: bytes, + completeness: None, + fail_message: entry.error_message.clone().unwrap_or_default(), + url_info: String::new(), + bytes, + meta: None, + series: String::new(), + duplicate_key: String::new(), + md5sum: "00000000000000000000000000000000".into(), + password: String::new(), + action_line: String::new(), + size: format_size_human(bytes), + loaded: false, + retry: entry.status == JobStatus::Failed && entry.nzb_data.is_some(), + archive: false, + time_added: entry.added_at.timestamp(), + postprocessing: false, + } + } + + fn from_postprocessing(job: &NzbJob) -> Self { + let path = job.work_dir.to_string_lossy().to_string(); + Self { + completed: job + .completed_at + .unwrap_or_else(chrono::Utc::now) + .timestamp(), + name: job.name.clone(), + nzb_name: format!("{}.nzb", job.name), + category: job.category.clone(), + pp: "D".into(), + script: String::new(), + report: String::new(), + url: String::new(), + status: sab_queue_status(job.status).into(), + nzo_id: sab_nzo_id(&job.id), + storage: String::new(), + path, + script_line: String::new(), + download_time: 0, + postproc_time: 0, + stage_log: Vec::new(), + downloaded: job.downloaded_bytes, + completeness: None, + fail_message: job.error_message.clone().unwrap_or_default(), + url_info: String::new(), + bytes: job.downloaded_bytes, + meta: None, + series: String::new(), + duplicate_key: String::new(), + md5sum: "00000000000000000000000000000000".into(), + password: job.password.clone().unwrap_or_default(), + action_line: job.status.to_string(), + size: format_size_human(job.downloaded_bytes), + loaded: true, + retry: false, + archive: false, + time_added: job.added_at.timestamp(), + postprocessing: true, } } } +fn sab_nzo_id(id: &str) -> String { + if id.starts_with("SABnzbd_nzo_") { + id.to_string() + } else { + format!("SABnzbd_nzo_{}", &id[..12.min(id.len())]) + } +} + /// Format bytes to human-readable size string. fn format_size_human(bytes: u64) -> String { if bytes == 0 { @@ -1047,9 +1954,177 @@ fn format_speed(bps: u64) -> String { } } +fn format_timeleft(bytes_left: u64, speed_bps: u64) -> String { + if bytes_left == 0 || speed_bps == 0 { + return "0:00:00".into(); + } + + let seconds = bytes_left / speed_bps; + let hours = seconds / 3600; + let minutes = (seconds % 3600) / 60; + let seconds = seconds % 60; + format!("{hours}:{minutes:02}:{seconds:02}") +} + #[cfg(test)] mod tests { use super::*; + use std::path::Path; + + use arc_swap::ArcSwap; + use chrono::TimeZone; + + use crate::auth::{CredentialStore, TokenStore}; + use crate::log_buffer::LogBuffer; + use crate::nzb_core::config::AppConfig; + use crate::nzb_core::db::Database; + use crate::queue_manager::QueueManager; + + mod sab_contract { + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/support/sab_contract.rs" + )); + } + + const VERSION_GOLDEN: &str = include_str!("../tests/fixtures/sabnzbd-5.0.4/version.json"); + const QUEUE_GOLDEN: &str = include_str!("../tests/fixtures/sabnzbd-5.0.4/queue.json"); + const HISTORY_GOLDEN: &str = include_str!("../tests/fixtures/sabnzbd-5.0.4/history.json"); + const FULLSTATUS_GOLDEN: &str = include_str!("../tests/fixtures/sabnzbd-5.0.4/fullstatus.json"); + + struct TestState { + state: AppState, + _tempdir: tempfile::TempDir, + } + + fn test_state() -> TestState { + let tempdir = tempfile::tempdir().expect("create SAB conformance tempdir"); + let mut config = AppConfig::default(); + config.general.api_key = Some("contract-api-key".into()); + config.general.data_dir = tempdir.path().join("data"); + config.general.incomplete_dir = tempdir.path().join("incomplete"); + config.general.complete_dir = tempdir.path().join("complete"); + config.general.cache_size = 512 * 1024 * 1024; + config.general.speed_limit_bps = 8 * 1024 * 1024; + + let log_buffer = LogBuffer::default(); + let queue_manager = QueueManager::new( + Vec::new(), + Database::open_memory().expect("open in-memory conformance database"), + config.general.incomplete_dir.clone(), + config.general.complete_dir.clone(), + log_buffer.clone(), + 1, + Vec::new(), + 0, + config.general.speed_limit_bps, + false, + 5, + false, + false, + 100.0, + 30, + ); + let config_path = tempdir.path().join("sab-conformance.toml"); + let state = AppState::new( + Arc::new(ArcSwap::from_pointee(config)), + config_path, + queue_manager, + log_buffer, + Arc::new(TokenStore::new()), + Arc::new(CredentialStore::new(tempdir.path().to_path_buf())), + ); + + TestState { + state, + _tempdir: tempdir, + } + } + + fn queue_job(id: &str, name: &str, category: &str, status: JobStatus) -> NzbJob { + NzbJob { + id: id.into(), + name: name.into(), + category: category.into(), + status, + priority: Priority::Normal, + total_bytes: 10 * 1_048_576, + downloaded_bytes: 2 * 1_048_576, + file_count: 2, + files_completed: 0, + article_count: 10, + articles_downloaded: 2, + articles_failed: 0, + added_at: chrono::DateTime::from_timestamp(1_700_000_000, 0).unwrap(), + completed_at: None, + work_dir: "/downloads/incomplete".into(), + output_dir: "/downloads/complete".into(), + password: Some("secret".into()), + error_message: None, + speed_bps: 0, + server_stats: Vec::new(), + files: Vec::new(), + } + } + + fn history_entry( + id: &str, + name: &str, + category: &str, + status: JobStatus, + seconds_ago: i64, + ) -> HistoryEntry { + let completed_at = chrono::Utc::now() - chrono::Duration::seconds(seconds_ago); + HistoryEntry { + id: id.into(), + name: name.into(), + category: category.into(), + status, + total_bytes: 10_000, + downloaded_bytes: 9_000, + added_at: completed_at - chrono::Duration::seconds(20), + completed_at, + download_time_secs: Some(12.4), + output_dir: format!("/downloads/{name}").into(), + stages: vec![StageResult { + name: "Unpack".into(), + status: StageStatus::Success, + message: Some("Unpacked".into()), + duration_secs: 3.6, + }], + error_message: (status == JobStatus::Failed).then(|| "broken archive".into()), + server_stats: Vec::new(), + nzb_data: (status == JobStatus::Failed).then(Vec::new), + retry_data: None, + } + } + + fn postprocessing_job() -> NzbJob { + let now = chrono::Utc::now(); + NzbJob { + id: "postprocessing-job".into(), + name: "Still Unpacking".into(), + category: "tv".into(), + status: JobStatus::PostProcessing, + priority: Priority::Normal, + total_bytes: 20_000, + downloaded_bytes: 20_000, + file_count: 1, + files_completed: 1, + article_count: 2, + articles_downloaded: 2, + articles_failed: 0, + added_at: now - chrono::Duration::minutes(1), + completed_at: Some(now), + work_dir: "/downloads/incomplete/postprocessing-job".into(), + output_dir: "/downloads/complete/Still Unpacking".into(), + password: None, + error_message: None, + speed_bps: 0, + server_stats: Vec::new(), + files: Vec::new(), + } + } #[test] fn queue_statuses_use_sabnzbd_vocabulary() { @@ -1070,6 +2145,119 @@ mod tests { } } + #[test] + fn queue_envelope_and_slot_fields_match_sab_types() { + let jobs = vec![queue_job( + "1234567890abcdef", + "Example.Show", + "tv", + JobStatus::Downloading, + )]; + let response = build_queue_response( + &jobs, + false, + 1_048_576, + 2_097_152, + &SabApiRequest::default(), + ); + let queue = &response["queue"]; + let slot = &queue["slots"][0]; + + assert_eq!(queue["status"], "Downloading"); + assert_eq!(queue["noofslots_total"], 1); + assert_eq!(queue["noofslots"], 1); + assert_eq!(queue["timeleft"], "0:00:08"); + assert_eq!(queue["speedlimit_abs"], "2097152"); + assert!(queue["paused"].is_boolean()); + assert!(queue["slots"].is_array()); + + assert_eq!(slot["index"], 0); + assert_eq!(slot["nzo_id"], "SABnzbd_nzo_1234567890ab"); + assert_eq!(slot["unpackopts"], "3"); + assert_eq!(slot["script"], "None"); + assert_eq!(slot["labels"], serde_json::json!([])); + assert_eq!(slot["password"], "secret"); + assert_eq!(slot["mbmissing"], "0.00"); + assert!(slot["direct_unpack"].is_null()); + assert_eq!(slot["time_added"], 1_700_000_000_i64); + } + + #[test] + fn queue_status_is_idle_when_unpaused_at_zero_speed() { + let response = build_queue_response( + &[queue_job("idle", "Idle job", "tv", JobStatus::Downloading)], + false, + 0, + 0, + &SabApiRequest::default(), + ); + assert_eq!(response["queue"]["status"], "Idle"); + assert_eq!(response["queue"]["timeleft"], "0:00:00"); + } + + #[test] + fn empty_and_paused_queues_have_sab_statuses() { + let empty = build_queue_response(&[], false, 0, 0, &SabApiRequest::default()); + assert_eq!(empty["queue"]["status"], "Idle"); + assert_eq!(empty["queue"]["slots"], serde_json::json!([])); + assert_eq!(empty["queue"]["noofslots_total"], 0); + + let paused = build_queue_response( + &[queue_job("paused", "Paused job", "tv", JobStatus::Paused)], + true, + 1_048_576, + 0, + &SabApiRequest::default(), + ); + assert_eq!(paused["queue"]["status"], "Paused"); + assert_eq!(paused["queue"]["slots"][0]["timeleft"], "0:00:00"); + } + + #[test] + fn queue_applies_filters_before_pagination() { + let jobs = vec![ + queue_job("one", "Show.One", "tv", JobStatus::Queued), + queue_job("two", "Movie.One", "movies", JobStatus::Downloading), + queue_job("three", "Show.Two", "tv", JobStatus::Paused), + queue_job("four", "Show.Three", "tv", JobStatus::Downloading), + ]; + let req = SabApiRequest { + search: Some("show".into()), + cat: Some("tv".into()), + start: Some(1), + limit: Some(1), + ..SabApiRequest::default() + }; + let response = build_queue_response(&jobs, false, 0, 0, &req); + let queue = &response["queue"]; + + assert_eq!(queue["noofslots_total"], 4); + assert_eq!(queue["noofslots"], 3); + assert_eq!(queue["start"], 1); + assert_eq!(queue["limit"], 1); + assert_eq!(queue["finish"], 2); + assert_eq!(queue["slots"].as_array().unwrap().len(), 1); + assert_eq!(queue["slots"][0]["filename"], "Show.Two"); + assert_eq!(queue["slots"][0]["index"], 1); + } + + #[test] + fn queue_supports_status_priority_and_id_filters() { + let mut high = queue_job("high-priority", "First", "tv", JobStatus::Downloading); + high.priority = Priority::High; + let normal = queue_job("normal", "Second", "tv", JobStatus::Downloading); + let req = SabApiRequest { + priority: Some("1".into()), + status: Some("downloading".into()), + nzo_ids: Some("SABnzbd_nzo_high-priorit".into()), + ..SabApiRequest::default() + }; + let response = build_queue_response(&[high, normal], false, 0, 0, &req); + + assert_eq!(response["queue"]["noofslots"], 1); + assert_eq!(response["queue"]["slots"][0]["filename"], "First"); + } + #[test] fn history_reports_active_download_time_to_arr_clients() { let now = chrono::Utc::now(); @@ -1088,8 +2276,922 @@ mod tests { error_message: None, server_stats: Vec::new(), nzb_data: None, + retry_data: None, }; assert_eq!(SabHistorySlot::from_entry(&entry).download_time, 2); } + + #[test] + fn history_completed_and_failed_slots_have_sab_field_types() { + let entries = [ + history_entry( + "completed-item", + "Completed Item", + "movies", + JobStatus::Completed, + 1, + ), + history_entry("failed-item", "Failed Item", "tv", JobStatus::Failed, 2), + ]; + let response = build_history_response(&entries, &[], &SabApiRequest::default(), 7); + let history = &response["history"]; + let slots = history["slots"].as_array().unwrap(); + + assert_eq!(history["noofslots"], 2); + assert_eq!(history["ppslots"], 0); + assert_eq!(history["last_history_update"], 7); + for slot in slots { + for field in [ + "completed", + "name", + "nzb_name", + "category", + "pp", + "script", + "report", + "url", + "status", + "nzo_id", + "storage", + "path", + "script_line", + "download_time", + "postproc_time", + "stage_log", + "downloaded", + "completeness", + "fail_message", + "url_info", + "bytes", + "meta", + "series", + "duplicate_key", + "md5sum", + "password", + "action_line", + "size", + "loaded", + "retry", + "archive", + "time_added", + ] { + assert!(slot.get(field).is_some(), "missing field {field}"); + } + } + assert_eq!(slots[0]["status"], "Completed"); + assert_eq!(slots[1]["status"], "Failed"); + assert_eq!(slots[1]["fail_message"], "broken archive"); + assert_eq!(slots[1]["retry"], true); + assert!(slots[0]["bytes"].is_u64()); + assert!(slots[0]["loaded"].is_boolean()); + assert!(slots[0]["completeness"].is_null()); + } + + #[test] + fn history_includes_postprocessing_before_terminal_slots() { + let response = build_history_response( + &[history_entry( + "completed-item", + "Completed Item", + "movies", + JobStatus::Completed, + 1, + )], + &[postprocessing_job()], + &SabApiRequest::default(), + 4, + ); + + assert_eq!(response["history"]["ppslots"], 1); + assert_eq!(response["history"]["noofslots"], 2); + assert_eq!(response["history"]["slots"][0]["status"], "Running"); + assert_eq!(response["history"]["slots"][0]["loaded"], true); + } + + #[test] + fn history_filters_before_paging_and_reports_total_matches() { + let entries = [ + history_entry( + "first-movie", + "First Movie", + "movies", + JobStatus::Completed, + 1, + ), + history_entry( + "second-movie", + "Second Movie", + "movies", + JobStatus::Failed, + 2, + ), + history_entry("tv-episode", "TV Episode", "tv", JobStatus::Failed, 3), + ]; + let request = SabApiRequest { + start: Some(1), + limit: Some(1), + search: Some("movie".into()), + cat: Some("movies".into()), + status: Some("Completed,Failed".into()), + ..Default::default() + }; + let response = build_history_response(&entries, &[], &request, 3); + + assert_eq!(response["history"]["noofslots"], 2); + assert_eq!(response["history"]["slots"].as_array().unwrap().len(), 1); + assert_eq!(response["history"]["slots"][0]["name"], "Second Movie"); + + let id_request = SabApiRequest { + nzo_ids: Some("SABnzbd_nzo_tv-episode".into()), + failed_only: Some("1".into()), + ..Default::default() + }; + let id_response = build_history_response(&entries, &[], &id_request, 3); + assert_eq!(id_response["history"]["noofslots"], 1); + assert_eq!(id_response["history"]["slots"][0]["name"], "TV Episode"); + } + + #[test] + fn matching_history_generation_uses_unchanged_response_contract() { + assert!(history_is_unchanged(Some(42), 42)); + assert!(!history_is_unchanged(Some(41), 42)); + assert!(!history_is_unchanged(None, 42)); + assert_eq!( + unchanged_history_response(), + serde_json::json!({ "history": false }) + ); + } + + #[tokio::test] + async fn version_matches_sabnzbd_golden_contract() { + let state = test_state(); + let expected = sab_contract::golden(VERSION_GOLDEN); + let actual = dispatch_mode(&state.state, "version", &SabApiRequest::default()).0; + + sab_contract::assert_matches_golden(actual, &expected); + } + + #[tokio::test] + async fn fullstatus_matches_sabnzbd_golden_contract() { + let state = test_state(); + let expected = sab_contract::golden(FULLSTATUS_GOLDEN); + let actual = dispatch_mode(&state.state, "fullstatus", &SabApiRequest::default()).0; + + sab_contract::assert_matches_golden(actual, &expected); + } + + #[tokio::test] + async fn queue_matches_sabnzbd_golden_contract() { + let state = test_state(); + state.state.queue_manager.pause_all(); + let added_at = chrono::Utc + .timestamp_opt(1_700_000_000, 0) + .single() + .expect("valid fixture time"); + let job = NzbJob { + id: "contract-queue-job".into(), + name: "SAB contract fixture".into(), + category: "tv".into(), + status: JobStatus::Queued, + priority: Priority::Normal, + total_bytes: 1_048_576, + downloaded_bytes: 0, + file_count: 1, + files_completed: 0, + article_count: 1, + articles_downloaded: 0, + articles_failed: 0, + added_at, + completed_at: None, + work_dir: state + .state + .config() + .general + .incomplete_dir + .join("contract-queue-job"), + output_dir: state + .state + .config() + .general + .complete_dir + .join("contract-queue-job"), + password: None, + error_message: None, + speed_bps: 0, + server_stats: Vec::new(), + files: Vec::new(), + }; + state + .state + .queue_manager + .add_job(job, None) + .expect("add queue fixture"); + let request = SabApiRequest { + limit: Some(1), + ..SabApiRequest::default() + }; + let actual = dispatch_mode(&state.state, "queue", &request).0; + + sab_contract::assert_matches_golden(actual, &sab_contract::golden(QUEUE_GOLDEN)); + } + + #[tokio::test] + async fn history_matches_sabnzbd_golden_contract() { + let state = test_state(); + let added_at = chrono::Utc + .timestamp_opt(1_700_000_000, 0) + .single() + .expect("valid fixture time"); + let entry = HistoryEntry { + id: "contract-history-job".into(), + name: "SAB history fixture".into(), + category: "tv".into(), + status: JobStatus::Completed, + total_bytes: 1_048_576, + downloaded_bytes: 1_048_576, + added_at, + completed_at: added_at + chrono::Duration::seconds(10), + download_time_secs: Some(5.0), + output_dir: state + .state + .config() + .general + .complete_dir + .join("contract-history-job"), + stages: Vec::new(), + error_message: None, + server_stats: Vec::new(), + nzb_data: None, + retry_data: None, + }; + state.state.queue_manager.with_db(|database| { + database + .history_insert(&entry) + .expect("insert history fixture") + }); + let request = SabApiRequest { + limit: Some(1), + ..SabApiRequest::default() + }; + let actual = dispatch_mode(&state.state, "history", &request).0; + + sab_contract::assert_matches_golden(actual, &sab_contract::golden(HISTORY_GOLDEN)); + } + + #[test] + fn checked_in_sabnzbd_goldens_are_valid_json() { + let fixture_dir = + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/sabnzbd-5.0.4"); + + for name in [ + "queue.json", + "history.json", + "fullstatus.json", + "version.json", + ] { + let contents = std::fs::read_to_string(fixture_dir.join(name)) + .unwrap_or_else(|error| panic!("read {name}: {error}")); + sab_contract::golden(&contents); + } + } + + /// Numeric priority codes per SABnzbd 5.1.x `sabnzbd/constants.py`: + /// FORCE_PRIORITY=2, HIGH_PRIORITY=1, NORMAL_PRIORITY=0, LOW_PRIORITY=-1, + /// DEFAULT_PRIORITY=-100 (displayed/treated as Normal), REPAIR_PRIORITY=3. + #[test] + fn sab_priority_to_priority_matches_upstream_numeric_codes() { + assert_eq!(sab_priority_to_priority("-1"), Priority::Low); + assert_eq!(sab_priority_to_priority("0"), Priority::Normal); + assert_eq!(sab_priority_to_priority("1"), Priority::High); + assert_eq!(sab_priority_to_priority("2"), Priority::Force); + assert_eq!(sab_priority_to_priority("3"), Priority::Force); + assert_eq!(sab_priority_to_priority("-100"), Priority::Normal); + } + + /// The numeric codes accepted when *setting* a priority must agree with + /// the codes `sab_priority_matches` uses when *filtering* the queue by + /// priority -- a prior regression let these two tables diverge silently. + #[test] + fn sab_priority_to_priority_agrees_with_sab_priority_matches() { + for (priority, numeric) in [ + (Priority::Low, "-1"), + (Priority::Normal, "0"), + (Priority::High, "1"), + (Priority::Force, "2"), + ] { + assert_eq!(sab_priority_to_priority(numeric), priority); + assert!(sab_priority_matches(priority, numeric)); + } + } + + fn add_live_job(test_state: &TestState, id: &str) { + let job = NzbJob { + id: id.into(), + name: "Compat Layer Fixture".into(), + category: "tv".into(), + status: JobStatus::Queued, + priority: Priority::Normal, + total_bytes: 1_048_576, + downloaded_bytes: 0, + file_count: 1, + files_completed: 0, + article_count: 1, + articles_downloaded: 0, + articles_failed: 0, + added_at: chrono::Utc::now(), + completed_at: None, + work_dir: test_state.state.config().general.incomplete_dir.join(id), + output_dir: test_state.state.config().general.complete_dir.join(id), + password: None, + error_message: None, + speed_bps: 0, + server_stats: Vec::new(), + files: Vec::new(), + }; + test_state + .state + .queue_manager + .add_job(job, None) + .expect("add live queue fixture"); + } + + fn add_live_job_with_category(test_state: &TestState, id: &str, category: &str) { + let job = NzbJob { + id: id.into(), + name: "Change Cat Fixture".into(), + category: category.into(), + status: JobStatus::Queued, + priority: Priority::Normal, + total_bytes: 1_048_576, + downloaded_bytes: 0, + file_count: 1, + files_completed: 0, + article_count: 1, + articles_downloaded: 0, + articles_failed: 0, + added_at: chrono::Utc::now(), + completed_at: None, + work_dir: test_state.state.config().general.incomplete_dir.join(id), + output_dir: test_state.state.config().general.complete_dir.join(id), + password: None, + error_message: None, + speed_bps: 0, + server_stats: Vec::new(), + files: Vec::new(), + }; + test_state + .state + .queue_manager + .add_job(job, None) + .expect("add live queue fixture"); + } + + /// SABnzbd's real priority endpoint is `mode=queue&name=priority`, not + /// the top-level `mode=priority` this compat layer also accepts. + #[tokio::test] + async fn queue_priority_subcommand_changes_job_priority() { + let test_state = test_state(); + add_live_job(&test_state, "queue-priority-job"); + + let req = SabApiRequest { + mode: Some("queue".into()), + name: Some("priority".into()), + value: Some("queue-priority-job".into()), + value2: Some("1".into()), + ..SabApiRequest::default() + }; + let response = dispatch_mode(&test_state.state, "queue", &req).0; + assert_eq!(response["status"], serde_json::json!(true)); + + let job = test_state + .state + .queue_manager + .get_jobs() + .into_iter() + .find(|job| job.id == "queue-priority-job") + .expect("job still queued"); + // This test covers routing (does mode=queue&name=priority reach the + // queue manager at all?), not the value mapping itself -- that's + // covered separately by sab_priority_to_priority's own tests. + assert_eq!(job.priority, sab_priority_to_priority("1")); + } + + /// SABnzbd's real rename endpoint is `mode=queue&name=rename`. + #[tokio::test] + async fn queue_rename_subcommand_renames_job() { + let test_state = test_state(); + add_live_job(&test_state, "queue-rename-job"); + + let req = SabApiRequest { + mode: Some("queue".into()), + name: Some("rename".into()), + value: Some("queue-rename-job".into()), + value2: Some("New Name".into()), + ..SabApiRequest::default() + }; + let response = dispatch_mode(&test_state.state, "queue", &req).0; + assert_eq!(response["status"], serde_json::json!(true)); + + let job = test_state + .state + .queue_manager + .get_jobs() + .into_iter() + .find(|job| job.id == "queue-rename-job") + .expect("job still queued"); + assert_eq!(job.name, "New Name"); + } + + /// SABnzbd's real `get_cats` reports the default category as `"*"`, not + /// a display name -- verified against `sabnzbd/api.py::list_cats(default=False)`. + #[tokio::test] + async fn get_cats_reports_default_category_as_sabnzbd_sentinel() { + let test_state = test_state(); + let response = handle_get_cats(&test_state.state).0; + let cats = response["categories"].as_array().expect("categories array"); + assert_eq!(cats, &vec![serde_json::json!("*")]); + } + + #[tokio::test] + async fn change_cat_accepts_sabnzbd_default_sentinel() { + let test_state = test_state(); + add_live_job(&test_state, "sentinel-cat-job"); + test_state + .state + .queue_manager + .change_job_category("sentinel-cat-job", "movies") + .expect("seed non-default category"); + + let req = SabApiRequest { + value: Some("sentinel-cat-job".into()), + value2: Some("*".into()), + ..SabApiRequest::default() + }; + let response = handle_change_cat(&test_state.state, &req).0; + assert_eq!(response["status"], serde_json::json!(true)); + + let job = test_state + .state + .queue_manager + .get_jobs() + .into_iter() + .find(|job| job.id == "sentinel-cat-job") + .expect("job still queued"); + assert_eq!(job.category, "Default"); + } + + /// SABnzbd's real `_api_change_cat` accepts a comma-separated `value` + /// list, applying the category change to every matching job. + #[tokio::test] + async fn change_cat_applies_to_multiple_comma_separated_ids() { + let test_state = test_state(); + add_live_job_with_category(&test_state, "multi-cat-one", "tv"); + add_live_job_with_category(&test_state, "multi-cat-two", "tv"); + + let req = SabApiRequest { + value: Some("multi-cat-one,multi-cat-two".into()), + value2: Some("movies".into()), + ..SabApiRequest::default() + }; + let response = handle_change_cat(&test_state.state, &req).0; + assert_eq!(response["status"], serde_json::json!(true)); + + let jobs = test_state.state.queue_manager.get_jobs(); + for id in ["multi-cat-one", "multi-cat-two"] { + let job = jobs + .iter() + .find(|job| job.id == id) + .unwrap_or_else(|| panic!("job {id} still queued")); + assert_eq!(job.category, "movies"); + } + } + + /// Real SABnzbd's `mode=get_scripts` always answers with at least + /// `["None"]` (sabnzbd/api.py::_api_get_scripts, + /// filesystem.py::list_scripts) -- clients that fetch categories and + /// scripts together to populate an "add download" dialog may fail to + /// populate the whole dialog if this call errors, as it previously did. + #[tokio::test] + async fn get_scripts_reports_none_when_unsupported() { + let test_state = test_state(); + let req = SabApiRequest::default(); + let response = dispatch_mode(&test_state.state, "get_scripts", &req).0; + assert_eq!(response["scripts"], serde_json::json!(["None"])); + } + + #[tokio::test] + async fn rejected_requests_keep_the_error_envelope() { + let test_state = test_state(); + + for provided in [None, Some("wrong-key")] { + let response = validate_api_key(&test_state.state, provided) + .expect_err("invalid credentials must be rejected") + .0; + assert_eq!(response["status"], serde_json::json!(false)); + assert!(response["error"].is_string()); + } + + let response = dispatch_mode( + &test_state.state, + "unknown-contract-mode", + &SabApiRequest::default(), + ) + .0; + assert_eq!(response["status"], serde_json::json!(false)); + assert!(response["error"].as_str().unwrap().contains("Unknown mode")); + } + + #[tokio::test] + async fn representative_read_modes_keep_stable_top_level_types() { + let test_state = test_state(); + + let config = dispatch_mode(&test_state.state, "get_config", &SabApiRequest::default()).0; + assert!(config["config"]["misc"]["complete_dir"].is_string()); + assert!(config["config"]["categories"].is_array()); + + let categories = dispatch_mode(&test_state.state, "get_cats", &SabApiRequest::default()).0; + assert!(categories["categories"].is_array()); + + let scripts = dispatch_mode(&test_state.state, "get_scripts", &SabApiRequest::default()).0; + assert!(scripts["scripts"].is_array()); + } + + #[tokio::test] + async fn addfile_reports_success_and_parse_errors_as_json() { + let test_state = test_state(); + let success = dispatch_post( + &test_state.state, + "addfile".into(), + None, + None, + None, + Some(("contract.nzb".into(), SAMPLE_NZB.as_bytes().to_vec())), + None, + None, + SabApiRequest::default(), + ) + .await + .expect("addfile response") + .0; + assert_eq!(success["status"], serde_json::json!(true)); + assert_eq!(success["nzo_ids"].as_array().unwrap().len(), 1); + + let missing = dispatch_post( + &test_state.state, + "addfile".into(), + None, + None, + None, + None, + None, + None, + SabApiRequest::default(), + ) + .await + .expect("missing-file response") + .0; + assert_eq!(missing["status"], serde_json::json!(false)); + assert!(missing["error"].is_string()); + + let malformed = dispatch_post( + &test_state.state, + "addfile".into(), + None, + None, + None, + Some(("malformed.nzb".into(), b"not an nzb".to_vec())), + None, + None, + SabApiRequest::default(), + ) + .await + .expect("malformed-file response") + .0; + assert_eq!(malformed["status"], serde_json::json!(false)); + assert!(malformed["error"].is_string()); + } + + /// SABnzbd's real `_api_queue_delete` accepts a comma-separated `value` + /// list, removing every matching job in one call. + #[tokio::test] + async fn queue_delete_removes_multiple_comma_separated_ids() { + let test_state = test_state(); + add_live_job(&test_state, "multi-delete-one"); + add_live_job(&test_state, "multi-delete-two"); + + let req = SabApiRequest { + value: Some("multi-delete-one,multi-delete-two".into()), + ..SabApiRequest::default() + }; + let response = handle_queue_delete(&test_state.state, &req).0; + assert_eq!(response["status"], serde_json::json!(true)); + + let remaining = test_state.state.queue_manager.get_jobs(); + assert!( + remaining + .iter() + .all(|job| job.id != "multi-delete-one" && job.id != "multi-delete-two") + ); + } + + fn insert_history_fixture(test_state: &TestState, id: &str, output_dir: std::path::PathBuf) { + let entry = HistoryEntry { + id: id.into(), + name: id.into(), + category: "tv".into(), + status: JobStatus::Completed, + total_bytes: 10_000, + downloaded_bytes: 10_000, + added_at: chrono::Utc::now() - chrono::Duration::seconds(20), + completed_at: chrono::Utc::now(), + download_time_secs: Some(1.0), + output_dir, + stages: Vec::new(), + error_message: None, + server_stats: Vec::new(), + nzb_data: None, + retry_data: None, + }; + test_state.state.queue_manager.with_db(|database| { + database + .history_insert(&entry) + .expect("insert history fixture") + }); + } + + /// Real SABnzbd's `_api_history_delete` removes the completed output + /// directory from disk when `del_files=1` is set; RustNZB previously + /// never freed that space regardless of the flag. + #[tokio::test] + async fn history_delete_with_del_files_removes_output_directory() { + let test_state = test_state(); + let output_dir = test_state + .state + .config() + .general + .complete_dir + .join("del-files-job"); + std::fs::create_dir_all(&output_dir).expect("create fixture output dir"); + std::fs::write(output_dir.join("file.mkv"), b"data").expect("write fixture file"); + insert_history_fixture(&test_state, "del-files-job", output_dir.clone()); + + let req = SabApiRequest { + value: Some("del-files-job".into()), + del_files: Some("1".into()), + ..SabApiRequest::default() + }; + let response = handle_history_delete(&test_state.state, &req).0; + assert_eq!(response["status"], serde_json::json!(true)); + assert!(!output_dir.exists()); + } + + /// Without `del_files`, history delete only removes the DB record, as + /// before. + #[tokio::test] + async fn history_delete_without_del_files_keeps_output_directory() { + let test_state = test_state(); + let output_dir = test_state + .state + .config() + .general + .complete_dir + .join("keep-files-job"); + std::fs::create_dir_all(&output_dir).expect("create fixture output dir"); + insert_history_fixture(&test_state, "keep-files-job", output_dir.clone()); + + let req = SabApiRequest { + value: Some("keep-files-job".into()), + ..SabApiRequest::default() + }; + let response = handle_history_delete(&test_state.state, &req).0; + assert_eq!(response["status"], serde_json::json!(true)); + assert!(output_dir.exists()); + } + + const SAMPLE_NZB: &str = r#" + + + alt.binaries.test + + article1@example.com + article2@example.com + + +"#; + + /// Serves `body` once over a raw TCP listener bound to an ephemeral + /// port, returning the URL to fetch it from. + async fn spawn_nzb_server(body: &'static str) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral test server"); + let addr = listener.local_addr().expect("test server local addr"); + + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept test connection"); + let mut buf = [0u8; 1024]; + let _ = socket.read(&mut buf).await; + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nContent-Type: application/x-nzb\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + let _ = socket.write_all(response.as_bytes()).await; + let _ = socket.shutdown().await; + }); + + format!("http://{addr}/test.nzb") + } + + /// NZB360 (and real SABnzbd) add downloads found via search as a plain + /// GET `mode=addurl` request, since there's no file body to upload -- + /// only the POST/multipart path handled `cat` for that mode, so GET + /// requests silently dropped the requested category. + #[tokio::test] + async fn addurl_over_get_applies_requested_category() { + let test_state = test_state(); + let url = spawn_nzb_server(SAMPLE_NZB).await; + + let req = SabApiRequest { + mode: Some("addurl".into()), + name: Some(url), + cat: Some("movies".into()), + apikey: Some("contract-api-key".into()), + ..SabApiRequest::default() + }; + + let response = h_sabnzbd_api_get(State(Arc::new(test_state.state)), Query(req)) + .await + .expect("addurl over GET should succeed") + .into_response(); + assert_eq!(response.status(), axum::http::StatusCode::OK); + + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("read response body"); + let value: serde_json::Value = serde_json::from_slice(&body).expect("parse JSON body"); + + // The URL was parsed from the request (dispatch reached the addurl + // handler), then refused by the SSRF guard because the test server is + // on loopback -- so the response is a structured {status:false}, not a + // "No URL provided" / "Unknown mode" fall-through. + assert_eq!(value["status"], serde_json::json!(false)); + assert!( + value["error"] + .as_str() + .unwrap_or_default() + .contains("private/reserved"), + "expected SSRF rejection, resp={value:?}" + ); + } + + async fn json_body(response: axum::response::Response) -> serde_json::Value { + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("read response body"); + serde_json::from_slice(&body).expect("parse JSON body") + } + + /// Prowlarr sends `mode=addurl` as a POST with every parameter in the + /// query string and no body at all (#119). The multipart extractor used + /// to reject that with `400 Invalid boundary` before the mode was read. + #[tokio::test] + async fn addurl_over_bare_post_uses_query_string() { + let test_state = test_state(); + let url = spawn_nzb_server(SAMPLE_NZB).await; + + let req = SabApiRequest { + mode: Some("addurl".into()), + name: Some(url), + cat: Some("prowlarr".into()), + priority: Some("-100".into()), + apikey: Some("contract-api-key".into()), + output: Some("json".into()), + ..SabApiRequest::default() + }; + let request = Request::builder() + .method("POST") + .uri("/sabnzbd/api") + .body(axum::body::Body::empty()) + .expect("build request"); + + let response = h_sabnzbd_api_post(State(Arc::new(test_state.state)), Query(req), request) + .await + .expect("addurl over bare POST should succeed") + .into_response(); + assert_eq!(response.status(), StatusCode::OK); + + let value = json_body(response).await; + // The URL was parsed from the request (dispatch reached the addurl + // handler), then refused by the SSRF guard because the test server is + // on loopback -- so the response is a structured {status:false}, not a + // "No URL provided" / "Unknown mode" fall-through. + assert_eq!(value["status"], serde_json::json!(false)); + assert!( + value["error"] + .as_str() + .unwrap_or_default() + .contains("private/reserved"), + "expected SSRF rejection, resp={value:?}" + ); + } + + /// Non-upload modes must also work over a bare POST. + #[tokio::test] + async fn version_over_bare_post_dispatches() { + let test_state = test_state(); + let req = SabApiRequest { + mode: Some("version".into()), + apikey: Some("contract-api-key".into()), + ..SabApiRequest::default() + }; + let request = Request::builder() + .method("POST") + .uri("/sabnzbd/api") + .body(axum::body::Body::empty()) + .expect("build request"); + + let response = h_sabnzbd_api_post(State(Arc::new(test_state.state)), Query(req), request) + .await + .expect("version over bare POST should succeed") + .into_response(); + assert_eq!(response.status(), StatusCode::OK); + let value = json_body(response).await; + assert_eq!(value["version"], serde_json::json!(SABNZBD_COMPAT_VERSION)); + } + + /// `application/x-www-form-urlencoded` bodies carry the same fields as + /// multipart ones and override the query string. + #[tokio::test] + async fn addurl_over_form_urlencoded_post_reads_body_fields() { + let test_state = test_state(); + let url = spawn_nzb_server(SAMPLE_NZB).await; + + let req = SabApiRequest { + apikey: Some("contract-api-key".into()), + ..SabApiRequest::default() + }; + let body = format!( + "mode=addurl&name={}&cat=tv", + url.replace(':', "%3A").replace('/', "%2F") + ); + let request = Request::builder() + .method("POST") + .uri("/sabnzbd/api") + .header(CONTENT_TYPE, "application/x-www-form-urlencoded") + .body(axum::body::Body::from(body)) + .expect("build request"); + + let response = h_sabnzbd_api_post(State(Arc::new(test_state.state)), Query(req), request) + .await + .expect("addurl over form POST should succeed") + .into_response(); + assert_eq!(response.status(), StatusCode::OK); + let value = json_body(response).await; + // The URL was parsed from the request (dispatch reached the addurl + // handler), then refused by the SSRF guard because the test server is + // on loopback -- so the response is a structured {status:false}, not a + // "No URL provided" / "Unknown mode" fall-through. + assert_eq!(value["status"], serde_json::json!(false)); + assert!( + value["error"] + .as_str() + .unwrap_or_default() + .contains("private/reserved"), + "expected SSRF rejection, resp={value:?}" + ); + } + + /// A request that claims to be multipart but carries no boundary is still + /// a client error, not a server error. + #[tokio::test] + async fn malformed_multipart_post_is_bad_request() { + let test_state = test_state(); + let req = SabApiRequest { + mode: Some("addfile".into()), + apikey: Some("contract-api-key".into()), + ..SabApiRequest::default() + }; + let request = Request::builder() + .method("POST") + .uri("/sabnzbd/api") + .header(CONTENT_TYPE, "multipart/form-data") + .body(axum::body::Body::empty()) + .expect("build request"); + + let response = match h_sabnzbd_api_post( + State(Arc::new(test_state.state)), + Query(req), + request, + ) + .await + { + Ok(_) => panic!("multipart without boundary should be rejected"), + Err(err) => err.into_response(), + }; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } } diff --git a/src/startup.rs b/src/startup.rs index 77254bb..08ae0ab 100644 --- a/src/startup.rs +++ b/src/startup.rs @@ -1,6 +1,7 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; +use anyhow::Context; use arc_swap::ArcSwap; use tracing::info; @@ -11,6 +12,7 @@ use crate::auth::{CredentialStore, TokenStore}; use crate::log_buffer::LogBuffer; use crate::queue_manager::QueueManager; use crate::state::AppState; +use nzb_postproc::PostProcLimits; fn sanitize_loaded_config(config: &mut AppConfig) { for server in &mut config.servers { @@ -44,6 +46,20 @@ fn env_flag_enabled(name: &str) -> Option { }) } +/// Create a data directory (e.g. `data_dir`/`incomplete_dir`/`complete_dir`), attaching +/// the failing path and a permission hint to any error so failures are actionable +/// instead of a bare `Permission denied (os error 13)`. +fn create_data_dir(path: &Path) -> anyhow::Result<()> { + std::fs::create_dir_all(path).with_context(|| { + format!( + "Failed to create directory {}. \ + Check that the directory (and its parent) is writable by the current user. \ + If using Docker, ensure the volume is owned by the container's user.", + path.display() + ) + }) +} + /// Configuration for engine initialization. /// /// All fields except `config_path` are optional overrides — @@ -120,9 +136,9 @@ pub async fn initialize( } // Ensure directories exist - std::fs::create_dir_all(&config.general.data_dir)?; - std::fs::create_dir_all(&config.general.incomplete_dir)?; - std::fs::create_dir_all(&config.general.complete_dir)?; + create_data_dir(&config.general.data_dir)?; + create_data_dir(&config.general.incomplete_dir)?; + create_data_dir(&config.general.complete_dir)?; // Open database let db_path = config.general.data_dir.join("rustnzb.db"); @@ -133,13 +149,18 @@ pub async fn initialize( let log_buffer = log_buffer.unwrap_or_default(); // Create the queue manager - let queue_manager = QueueManager::new( + let queue_manager = QueueManager::new_with_postproc_limits( config.servers.clone(), db, config.general.incomplete_dir.clone(), config.general.complete_dir.clone(), log_buffer.clone(), config.general.max_active_downloads, + PostProcLimits { + pipelines: config.general.max_post_processing_jobs, + repair: config.general.max_repair_workers, + extract: config.general.max_extract_workers, + }, config.categories.clone(), config.general.min_free_space_bytes, config.general.speed_limit_bps, @@ -151,10 +172,16 @@ pub async fn initialize( config.general.article_timeout_secs, ); - // Set history retention - if let Some(retention) = config.general.history_retention { - queue_manager.set_history_retention(Some(retention)); - } + // Set history retention (the setter folds 0 into "keep all"). + queue_manager.set_history_retention(config.general.history_retention); + queue_manager.set_auto_sort_remaining_pct(config.general.auto_sort_remaining_pct); + queue_manager.set_postproc_scripts( + config.general.scripts_dir.clone(), + config.general.script_success.clone(), + config.general.script_failure.clone(), + config.general.script_timeout_secs, + config.general.script_max_output_bytes, + ); // Restore any in-progress jobs from the database if let Err(e) = queue_manager.restore_from_db() { @@ -217,10 +244,54 @@ pub async fn initialize( #[cfg(test)] mod tests { - use super::sanitize_loaded_config; + use super::{create_data_dir, sanitize_loaded_config}; use crate::nzb_core::config::AppConfig; use crate::nzb_core::config::ServerConfig; + #[test] + fn create_data_dir_creates_nested_directories() { + let tmp = tempfile::tempdir().unwrap(); + let nested = tmp.path().join("a").join("b").join("c"); + + create_data_dir(&nested).expect("nested directory creation should succeed"); + + assert!(nested.is_dir()); + } + + #[cfg(unix)] + #[test] + fn create_data_dir_wraps_permission_denied_with_context() { + use std::os::unix::fs::PermissionsExt; + + let tmp = tempfile::tempdir().unwrap(); + let locked_parent = tmp.path().join("locked"); + std::fs::create_dir_all(&locked_parent).unwrap(); + std::fs::set_permissions(&locked_parent, std::fs::Permissions::from_mode(0o000)).unwrap(); + + let target = locked_parent.join("data"); + let result = create_data_dir(&target); + + // Restore permissions so the tempdir can be cleaned up. + std::fs::set_permissions(&locked_parent, std::fs::Permissions::from_mode(0o755)).unwrap(); + + let err = match result { + Err(e) => e, + // Running as root (e.g. CI containers) bypasses the permission + // check entirely, so there's nothing to assert. + Ok(()) => return, + }; + let debug_text = format!("{err:?}"); + + assert!( + debug_text.contains(&target.display().to_string()), + "error should mention the failing path, got: {debug_text}" + ); + assert!( + debug_text.contains("Caused by"), + "error should retain the underlying io::Error in the chain, got: {debug_text}" + ); + } + #[test] fn sanitize_loaded_config_trims_server_fields() { let mut config = AppConfig::default(); diff --git a/tests/fixtures/sabnzbd-5.0.4/README.md b/tests/fixtures/sabnzbd-5.0.4/README.md new file mode 100644 index 0000000..42c3680 --- /dev/null +++ b/tests/fixtures/sabnzbd-5.0.4/README.md @@ -0,0 +1,24 @@ +# SABnzbd 5.0.4 API goldens + +These normalized JSON responses define RustNZB's SAB-compatible response +contract. They were captured from the supported SABnzbd 5.0.4 release +(`128e0d03d7cc61af7e73b18376b880219fbc3596`) using the LinuxServer image: + +```text +lscr.io/linuxserver/sabnzbd:5.0.4 +sha256:302be8972d4627222a0701634f2f9025826d760d856414aa6e67c0a66833e5be +``` + +The key sets and types were cross-checked against SABnzbd's own strict Tavern +fixtures at tag `5.0.4`: + +- `tests/data/tavern/api_queue_empty.yaml` +- `tests/data/tavern/api_queue_format.yaml` +- `tests/data/tavern/api_history_empty.yaml` +- `tests/data/tavern/api_history_format.yaml` +- `tests/data/tavern/api_version.yaml` +- `sabnzbd/api.py::build_status` (there is no upstream fullstatus Tavern case) + +`$type:*` markers replace only environment- or time-dependent values. The +conformance helper verifies each marker's original JSON type before +normalization, then compares the full document so extra and missing keys fail. diff --git a/tests/fixtures/sabnzbd-5.0.4/fullstatus.json b/tests/fixtures/sabnzbd-5.0.4/fullstatus.json new file mode 100644 index 0000000..9022669 --- /dev/null +++ b/tests/fixtures/sabnzbd-5.0.4/fullstatus.json @@ -0,0 +1,60 @@ +{ + "status": { + "active_lang": "en", + "active_socks5_proxy": "$type:any", + "apikey": "$type:string", + "cache_art": "$type:string", + "cache_size": "$type:string", + "color_scheme": "$type:string", + "completedir": "$type:string", + "completedirspeed": "$type:number", + "configfn": "$type:string", + "confighelpuri": "$type:string", + "delayed_assembler": "$type:integer", + "diskspace1": "$type:string", + "diskspace1_norm": "$type:string", + "diskspace2": "$type:string", + "diskspace2_norm": "$type:string", + "diskspacetotal1": "$type:string", + "diskspacetotal2": "$type:string", + "dnslookup": "$type:boolean", + "downloaddir": "$type:string", + "downloaddirspeed": "$type:number", + "finishaction": "$type:any", + "folders": "$type:array", + "have_quota": "$type:boolean", + "have_warnings": "$type:string", + "internetbandwidth": "$type:number", + "ipv6": "$type:any", + "left_quota": "$type:string", + "loadavg": "$type:string", + "localipv4": "$type:any", + "logfile": "$type:string", + "loglevel": "$type:string", + "macos": "$type:boolean", + "my_home": "$type:string", + "my_lcldata": "$type:string", + "new_rel_url": "$type:any", + "new_release": "$type:any", + "pause_int": "$type:string", + "paused": "$type:boolean", + "paused_all": "$type:boolean", + "pid": "$type:integer", + "power_options": "$type:boolean", + "pp_pause_event": "$type:boolean", + "publicipv4": "$type:any", + "pystone": "$type:number", + "quota": "$type:string", + "rtl": "$type:boolean", + "servers": "$type:array", + "speedlimit": "$type:string", + "speedlimit_abs": "$type:string", + "uptime": "$type:string", + "url_base": "$type:string", + "version": "5.0.4", + "warnings": "$type:array", + "webdir": "$type:string", + "weblogfile": "$type:any", + "windows": "$type:boolean" + } +} diff --git a/tests/fixtures/sabnzbd-5.0.4/history.json b/tests/fixtures/sabnzbd-5.0.4/history.json new file mode 100644 index 0000000..734d6cf --- /dev/null +++ b/tests/fixtures/sabnzbd-5.0.4/history.json @@ -0,0 +1,48 @@ +{ + "history": { + "day_size": "$type:string", + "last_history_update": "$type:integer", + "month_size": "$type:string", + "noofslots": 1, + "ppslots": 0, + "slots": [ + { + "action_line": "$type:any", + "archive": false, + "bytes": 1048576, + "category": "tv", + "completed": "$type:integer", + "completeness": "$type:any", + "download_time": 5, + "downloaded": 1048576, + "duplicate_key": "$type:any", + "fail_message": "", + "loaded": false, + "md5sum": "$type:string", + "meta": "$type:any", + "name": "SAB history fixture", + "nzb_name": "SAB history fixture.nzb", + "nzo_id": "$type:string", + "password": "", + "path": "$type:string", + "postproc_time": "$type:integer", + "pp": "D", + "report": "$type:any", + "retry": false, + "script": "$type:string", + "script_line": "$type:string", + "series": "$type:any", + "size": "$type:string", + "stage_log": [], + "status": "Completed", + "storage": "$type:string", + "time_added": "$type:integer", + "url": "", + "url_info": "" + } + ], + "total_size": "$type:string", + "version": "5.0.4", + "week_size": "$type:string" + } +} diff --git a/tests/fixtures/sabnzbd-5.0.4/queue.json b/tests/fixtures/sabnzbd-5.0.4/queue.json new file mode 100644 index 0000000..926bee2 --- /dev/null +++ b/tests/fixtures/sabnzbd-5.0.4/queue.json @@ -0,0 +1,60 @@ +{ + "queue": { + "cache_art": "$type:string", + "cache_size": "$type:string", + "diskspace1": "$type:string", + "diskspace1_norm": "$type:string", + "diskspace2": "$type:string", + "diskspace2_norm": "$type:string", + "diskspacetotal1": "$type:string", + "diskspacetotal2": "$type:string", + "finish": 1, + "finishaction": "$type:any", + "have_quota": "$type:boolean", + "have_warnings": "$type:string", + "kbpersec": "$type:string", + "left_quota": "$type:string", + "limit": 1, + "mb": "$type:string", + "mbleft": "$type:string", + "noofslots": 1, + "noofslots_total": 1, + "pause_int": "$type:string", + "paused": true, + "paused_all": false, + "quota": "$type:string", + "size": "$type:string", + "sizeleft": "$type:string", + "slots": [ + { + "avg_age": "$type:string", + "cat": "tv", + "direct_unpack": "$type:any", + "filename": "SAB contract fixture", + "index": 0, + "labels": [], + "mb": "$type:string", + "mbleft": "$type:string", + "mbmissing": "$type:string", + "nzo_id": "$type:string", + "password": "", + "percentage": "$type:string", + "priority": "Normal", + "script": "$type:string", + "size": "$type:string", + "sizeleft": "$type:string", + "status": "Queued", + "time_added": "$type:integer", + "timeleft": "$type:string", + "unpackopts": "$type:string" + } + ], + "speed": "$type:string", + "speedlimit": "$type:string", + "speedlimit_abs": "$type:string", + "start": 0, + "status": "Paused", + "timeleft": "$type:string", + "version": "5.0.4" + } +} diff --git a/tests/fixtures/sabnzbd-5.0.4/version.json b/tests/fixtures/sabnzbd-5.0.4/version.json new file mode 100644 index 0000000..8747569 --- /dev/null +++ b/tests/fixtures/sabnzbd-5.0.4/version.json @@ -0,0 +1,3 @@ +{ + "version": "5.0.4" +} diff --git a/tests/harness/mod.rs b/tests/harness/mod.rs index a82d906..9766ff8 100644 --- a/tests/harness/mod.rs +++ b/tests/harness/mod.rs @@ -28,6 +28,7 @@ use nzb_web::nzb_core::config::ServerConfig; use nzb_web::nzb_core::db::Database; use nzb_web::nzb_core::models::{JobStatus, NzbJob}; use nzb_web::nzb_core::nzb_parser; +use nzb_web::nzb_postproc::PostProcLimits; use nzb_web::queue_manager::QueueManager; use tempfile::TempDir; @@ -90,24 +91,32 @@ impl ServerProfile { /// independently optional; defaults are tuned for fast, deterministic tests. pub struct HarnessBuilder { servers: Vec, + server_configs: Vec, + database_path: Option, + state_dir: Option, article_timeout_secs: u64, max_active_downloads: usize, abort_hopeless: bool, early_failure_check: bool, required_completion_pct: f64, speed_limit_bps: u64, + postproc_limits: PostProcLimits, } impl HarnessBuilder { pub fn new() -> Self { Self { servers: Vec::new(), + server_configs: Vec::new(), + database_path: None, + state_dir: None, article_timeout_secs: 30, max_active_downloads: 5, abort_hopeless: true, early_failure_check: true, required_completion_pct: 100.0, speed_limit_bps: 0, + postproc_limits: PostProcLimits::default(), } } @@ -116,6 +125,60 @@ impl HarnessBuilder { self } + pub fn with_server_config(mut self, config: ServerConfig) -> Self { + self.server_configs.push(config); + self + } + + pub fn with_database_path(mut self, path: PathBuf) -> Self { + self.database_path = Some(path); + self + } + + pub fn with_state_dir(mut self, path: PathBuf) -> Self { + self.state_dir = Some(path); + self + } + + /// Happy-path profile: one healthy provider and the production-like + /// completion policy used by smoke tests. + pub fn happy_path(server: ServerProfile) -> Self { + Self::new().with_server(server) + } + + /// Retry profile: short article deadlines expose reconnect and failover + /// behavior without making the test wait through production timeouts. + pub fn retrying(server: ServerProfile) -> Self { + Self::new().with_server(server).article_timeout(3) + } + + /// Pause/resume profile: keep one active download so control-plane tests + /// can observe a pause boundary deterministically. + pub fn pause_resume(server: ServerProfile) -> Self { + Self::new().with_server(server).max_active_downloads(1) + } + + /// Cancellation profile: a single worker makes slot-release assertions + /// independent of scheduler width. + pub fn cancellation(server: ServerProfile) -> Self { + Self::new().with_server(server).max_active_downloads(1) + } + + /// Hopeless-job profile: enable the failure watchdog and use a compact + /// article deadline so silent providers converge quickly. + pub fn hopeless(server: ServerProfile) -> Self { + Self::new() + .with_server(server) + .article_timeout(2) + .abort_hopeless(true) + } + + /// Restart-recovery profile: a single active worker makes checkpoint and + /// requeue assertions independent of scheduler width. + pub fn restart_recovery(server: ServerProfile) -> Self { + Self::new().with_server(server).max_active_downloads(1) + } + pub fn article_timeout(mut self, secs: u64) -> Self { self.article_timeout_secs = secs; self @@ -131,27 +194,69 @@ impl HarnessBuilder { self } - /// Build the engine. Creates temp dirs, an in-memory database, and a - /// fully-wired `QueueManager` whose worker pool is already running. + pub fn early_failure_check(mut self, enabled: bool) -> Self { + self.early_failure_check = enabled; + self + } + + pub fn required_completion_pct(mut self, percentage: f64) -> Self { + self.required_completion_pct = percentage; + self + } + + pub fn speed_limit_bps(mut self, bytes_per_second: u64) -> Self { + self.speed_limit_bps = bytes_per_second; + self + } + + pub fn postproc_limits(mut self, limits: PostProcLimits) -> Self { + self.postproc_limits = limits; + self + } + + /// Build the engine. Creates isolated directories and a fully-wired + /// `QueueManager` whose worker pool is already running. pub fn build(self) -> TestEngine { init_test_tracing(); - let tempdir = TempDir::new().expect("create tempdir"); - let incomplete_dir = tempdir.path().join("incomplete"); - let complete_dir = tempdir.path().join("complete"); + let tempdir = self + .state_dir + .is_none() + .then(|| TempDir::new().expect("create tempdir")); + let root = self.state_dir.clone().unwrap_or_else(|| { + tempdir + .as_ref() + .expect("temporary state") + .path() + .to_path_buf() + }); + std::fs::create_dir_all(&root).expect("create harness state directory"); + let incomplete_dir = root.join("incomplete"); + let complete_dir = root.join("complete"); std::fs::create_dir_all(&incomplete_dir).expect("create incomplete_dir"); std::fs::create_dir_all(&complete_dir).expect("create complete_dir"); - let db = Database::open_memory().expect("open in-memory db"); - let server_configs: Vec = - self.servers.iter().map(|p| p.config.clone()).collect(); + let db = self + .database_path + .as_deref() + .map(Database::open) + .transpose() + .expect("open harness database") + .unwrap_or_else(|| Database::open_memory().expect("open in-memory db")); + let server_configs: Vec = self + .servers + .iter() + .map(|p| p.config.clone()) + .chain(self.server_configs) + .collect(); - let queue_manager = QueueManager::new( + let queue_manager = QueueManager::new_with_postproc_limits( server_configs, db, incomplete_dir.clone(), complete_dir.clone(), LogBuffer::default(), self.max_active_downloads, + self.postproc_limits, Vec::new(), // categories 0, // min_free_space self.speed_limit_bps, @@ -190,7 +295,7 @@ impl Default for HarnessBuilder { pub struct TestEngine { pub queue_manager: Arc, _servers: Vec, - _tempdir: TempDir, + _tempdir: Option, pub incomplete_dir: PathBuf, pub complete_dir: PathBuf, } @@ -229,6 +334,14 @@ impl TestEngine { self.snapshot().jobs.into_iter().find(|j| j.id == id) } + pub fn history_status(&self, id: &str) -> Option { + self.queue_manager + .history_get(id) + .ok() + .flatten() + .map(|entry| entry.status) + } + /// Poll `predicate` against fresh snapshots until it returns `true` or /// the timeout elapses. Returns `true` on success. Polls every 100 ms. pub async fn wait_for(&self, timeout: Duration, mut predicate: F) -> bool @@ -260,7 +373,10 @@ impl TestEngine { .iter() .find(|j| j.id == job_id) .map(|j| statuses.contains(&j.status)) - .unwrap_or(false) + .unwrap_or_else(|| { + self.history_status(job_id) + .is_some_and(|status| statuses.contains(&status)) + }) }) .await } @@ -291,6 +407,7 @@ pub struct JobView { pub articles_failed: usize, pub downloaded_bytes: u64, pub total_bytes: u64, + pub error_message: Option, } impl From for JobView { @@ -304,6 +421,7 @@ impl From for JobView { articles_failed: j.articles_failed, downloaded_bytes: j.downloaded_bytes, total_bytes: j.total_bytes, + error_message: j.error_message, } } } diff --git a/tests/harness/nzb_fixture.rs b/tests/harness/nzb_fixture.rs index 2d4f596..accef76 100644 --- a/tests/harness/nzb_fixture.rs +++ b/tests/harness/nzb_fixture.rs @@ -14,8 +14,11 @@ //! // into harness::yenc_articles for the mock config //! ``` +use std::collections::HashMap; use std::fmt::Write; +use nzb_nntp::testutil::MockConfig; + #[derive(Default)] pub struct NzbFixture<'a> { name: String, @@ -36,6 +39,68 @@ pub struct BuiltFixture<'a> { pub articles: Vec<(&'a str, &'a [u8], String)>, } +/// Owned, reusable fixture cases for tests that need to hand the same input +/// to several providers or restart a queue manager. All bytes are generated +/// from literals, so the catalog never reads the network or wall clock. +#[derive(Clone, Debug)] +pub struct FixtureCase { + pub name: String, + pub xml: Vec, + pub articles: HashMap>, +} + +impl FixtureCase { + pub fn mock_config(&self) -> MockConfig { + MockConfig { + articles: self.articles.clone(), + ..MockConfig::default() + } + } +} + +/// Stable fixture catalog shared by harness profiles and contract tests. +pub struct FixtureCatalog; + +impl FixtureCatalog { + fn from_fixture(fixture: BuiltFixture<'_>, name: &str) -> FixtureCase { + let articles = fixture + .articles + .iter() + .map(|(id, body, filename)| { + let (encoded, _) = + yenc_simd::encode_article(body, filename, 1, 1, 0, body.len() as u64); + ((*id).to_string(), encoded) + }) + .collect(); + FixtureCase { + name: name.into(), + xml: fixture.xml, + articles, + } + } + + pub fn single() -> FixtureCase { + let fixture = NzbFixture::new("catalog-single") + .add_file("catalog.txt", &[("catalog-single-1@test", b"catalog body")]) + .build(); + Self::from_fixture(fixture, "catalog-single") + } + + pub fn multi_segment() -> FixtureCase { + let fixture = NzbFixture::new("catalog-multi") + .add_file( + "catalog.bin", + &[ + ("catalog-multi-1@test", b"first"), + ("catalog-multi-2@test", b"second"), + ("catalog-multi-3@test", b"third"), + ], + ) + .build(); + Self::from_fixture(fixture, "catalog-multi") + } +} + impl<'a> NzbFixture<'a> { pub fn new(name: &str) -> Self { Self { diff --git a/tests/harness_catalog.rs b/tests/harness_catalog.rs new file mode 100644 index 0000000..58a1248 --- /dev/null +++ b/tests/harness_catalog.rs @@ -0,0 +1,36 @@ +//! Catalog and profile invariants for deterministic integration tests. + +mod harness; + +use harness::nzb_fixture::FixtureCatalog; +use harness::{HarnessBuilder, ServerProfile}; +use nzb_nntp::testutil::MockConfig; + +type ProfileFactory = fn(ServerProfile) -> HarnessBuilder; + +#[tokio::test] +async fn catalog_cases_are_reproducible_and_profiles_are_explicit() { + let first = FixtureCatalog::single(); + let second = FixtureCatalog::single(); + assert_eq!(first.name, second.name); + assert_eq!(first.xml, second.xml); + assert_eq!(first.articles, second.articles); + assert_eq!(first.articles.len(), 1); + + let multi = FixtureCatalog::multi_segment(); + assert_eq!(multi.articles.len(), 3); + assert!(String::from_utf8_lossy(&multi.xml).contains("catalog.bin")); + + let profiles: [(&str, ProfileFactory); 6] = [ + ("happy", HarnessBuilder::happy_path), + ("retry", HarnessBuilder::retrying), + ("pause", HarnessBuilder::pause_resume), + ("cancel", HarnessBuilder::cancellation), + ("hopeless", HarnessBuilder::hopeless), + ("restart", HarnessBuilder::restart_recovery), + ]; + for (name, profile) in profiles { + let server = ServerProfile::start(name, MockConfig::default(), 1).await; + let _ = profile(server); + } +} diff --git a/tests/harness_failure_matrix.rs b/tests/harness_failure_matrix.rs new file mode 100644 index 0000000..180f272 --- /dev/null +++ b/tests/harness_failure_matrix.rs @@ -0,0 +1,257 @@ +//! Table-driven NNTP failure and lifecycle invariants. + +mod harness; + +use std::collections::HashMap; +use std::path::PathBuf; +use std::time::Duration; + +use harness::nzb_fixture::{FixtureCatalog, NzbFixture}; +use harness::{HarnessBuilder, ServerProfile, yenc_articles}; +use nzb_nntp::testutil::MockConfig; +use nzb_web::nzb_core::db::Database; +use nzb_web::nzb_core::models::JobStatus; +use nzb_web::nzb_core::nzb_parser; + +#[tokio::test] +async fn transient_failures_recover_without_duplicate_completion() { + let body = b"recoverable"; + let fixture = NzbFixture::new("failure-matrix") + .add_file("payload.bin", &[("failure-matrix-1", body)]) + .build(); + let triples = fixture + .articles + .iter() + .map(|(id, bytes, name)| (*id, *bytes, name.as_str())) + .collect::>(); + let mut sequences = HashMap::new(); + sequences.insert( + "failure-matrix-1".to_string(), + std::collections::VecDeque::from([(400, "temporary failure".into())]), + ); + let server = ServerProfile::start( + "failure-matrix", + MockConfig { + articles: yenc_articles(&triples), + article_response_sequences: Some(std::sync::Arc::new(parking_lot::Mutex::new( + sequences, + ))), + ..MockConfig::default() + }, + 1, + ) + .await; + let engine = HarnessBuilder::new().with_server(server).build(); + let id = engine + .submit_nzb_xml("failure-matrix", fixture.xml) + .unwrap(); + assert!( + engine + .wait_for_status(&id, Duration::from_secs(10), &[JobStatus::Completed]) + .await + ); + let history = engine + .queue_manager + .history_get(&id) + .expect("history query") + .expect("completed history"); + assert_eq!(history.status, JobStatus::Completed); + assert_eq!(history.downloaded_bytes, history.total_bytes); +} + +#[tokio::test] +async fn cancellation_releases_connection_slots_and_removes_active_job() { + let fixture = FixtureCatalog::single(); + let server = ServerProfile::start( + "cancel", + MockConfig { + articles: fixture.mock_config().articles, + hang_after_command: Some("ARTICLE".into()), + ..MockConfig::default() + }, + 1, + ) + .await; + let engine = HarnessBuilder::cancellation(server) + .article_timeout(2) + .build(); + let id = engine + .submit_nzb_xml(&fixture.name, fixture.xml) + .expect("submit fixture"); + assert!( + engine + .wait_for_status(&id, Duration::from_secs(3), &[JobStatus::Downloading]) + .await + ); + + engine + .queue_manager + .remove_job(&id) + .expect("cancel active job"); + assert!(engine.job(&id).is_none()); + assert!( + tokio::time::timeout(Duration::from_secs(3), async { + loop { + if engine.queue_manager.connection_total() == 0 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .is_ok(), + "cancelled jobs must release all NNTP slots" + ); +} + +#[tokio::test] +async fn pause_and_resume_preserve_progress_until_single_completion() { + let fixture = FixtureCatalog::multi_segment(); + let server = ServerProfile::start( + "pause-resume", + MockConfig { + articles: fixture.articles.clone(), + response_delay: Some(Duration::from_millis(120)), + ..MockConfig::default() + }, + 1, + ) + .await; + let engine = HarnessBuilder::pause_resume(server) + .article_timeout(5) + .build(); + let id = engine + .submit_nzb_xml(&fixture.name, fixture.xml) + .expect("submit fixture"); + assert!( + engine + .wait_for_status(&id, Duration::from_secs(5), &[JobStatus::Downloading]) + .await + ); + + engine + .queue_manager + .pause_job(&id) + .expect("pause active job"); + assert!( + engine + .wait_for_status(&id, Duration::from_secs(3), &[JobStatus::Paused]) + .await + ); + let paused = engine.job(&id).expect("paused job"); + tokio::time::sleep(Duration::from_millis(250)).await; + let still_paused = engine.job(&id).expect("paused job remains queued"); + assert_eq!(still_paused.status, JobStatus::Paused); + assert_eq!(still_paused.articles_downloaded, paused.articles_downloaded); + + engine + .queue_manager + .resume_job(&id) + .expect("resume paused job"); + assert!( + engine + .wait_for_status(&id, Duration::from_secs(10), &[JobStatus::Completed]) + .await + ); + assert_eq!(engine.history_status(&id), Some(JobStatus::Completed)); +} + +#[tokio::test] +async fn authentication_failure_waits_for_recovery_without_leaking_connections() { + let fixture = FixtureCatalog::single(); + let mut server = ServerProfile::start( + "auth-failure", + MockConfig { + articles: fixture.mock_config().articles, + auth_required: true, + fail_auth: true, + ..MockConfig::default() + }, + 1, + ) + .await; + server.config.username = Some("user".into()); + server.config.password = Some("pass".into()); + let engine = HarnessBuilder::hopeless(server).build(); + let id = engine + .submit_nzb_xml(&fixture.name, fixture.xml) + .expect("submit fixture"); + assert!( + engine + .wait_for(Duration::from_secs(8), |snapshot| { + snapshot.job(&id).is_some_and(|job| { + job.status == JobStatus::Downloading && job.error_message.is_some() + }) + }) + .await + ); + assert_eq!(engine.queue_manager.connected_snapshot()[0].1, 0); +} + +#[tokio::test] +async fn restart_restores_checkpoint_and_skips_completed_article() { + let fixture = NzbFixture::new("restart-matrix") + .add_file( + "restart.bin", + &[("restart-1", b"first"), ("restart-2", b"second")], + ) + .build(); + let triples = fixture + .articles + .iter() + .map(|(id, bytes, name)| (*id, *bytes, name.as_str())) + .collect::>(); + let mut overrides = HashMap::new(); + overrides.insert("restart-1".to_string(), 430); + let server = ServerProfile::start( + "restart", + MockConfig { + articles: yenc_articles(&triples), + article_response_overrides: overrides, + ..MockConfig::default() + }, + 1, + ) + .await; + let state = tempfile::tempdir().expect("restart state"); + let database_path = state.path().join("queue.sqlite"); + let incomplete_dir = state.path().join("incomplete"); + let complete_dir = state.path().join("complete"); + std::fs::create_dir_all(&incomplete_dir).unwrap(); + std::fs::create_dir_all(&complete_dir).unwrap(); + let mut job = nzb_parser::parse_nzb("restart-matrix", &fixture.xml).unwrap(); + job.status = JobStatus::Downloading; + job.work_dir = incomplete_dir.join(&job.id); + job.output_dir = complete_dir.join(&job.name); + let job_id = job.id.clone(); + let db = Database::open(&database_path).unwrap(); + db.queue_insert(&job).unwrap(); + db.queue_store_nzb_data(&job_id, &fixture.xml).unwrap(); + db.queue_store_job_data( + &job_id, + &serde_json::to_vec(&serde_json::json!({ + "files": {"restart.bin": [1]}, + "downloaded_bytes": 5, + "articles_downloaded": 1, + "articles_failed": 0, + "files_completed": 0 + })) + .unwrap(), + ) + .unwrap(); + + let engine = HarnessBuilder::restart_recovery(server) + .with_database_path(database_path) + .with_state_dir(PathBuf::from(state.path())) + .build(); + engine.queue_manager.restore_from_db().unwrap(); + + assert!( + engine + .wait_for_status(&job_id, Duration::from_secs(10), &[JobStatus::Completed]) + .await + ); + let history = engine.queue_manager.history_get(&job_id).unwrap().unwrap(); + assert_eq!(history.status, JobStatus::Completed); + assert_eq!(history.downloaded_bytes, history.total_bytes); +} diff --git a/tests/harness_priority.rs b/tests/harness_priority.rs index 080f04d..c11716e 100644 --- a/tests/harness_priority.rs +++ b/tests/harness_priority.rs @@ -27,7 +27,7 @@ use std::time::Duration; use harness::nzb_fixture::NzbFixture; use harness::{HarnessBuilder, ServerProfile, yenc_articles}; use nzb_nntp::testutil::MockConfig; -use nzb_web::nzb_core::models::JobStatus; +use nzb_web::nzb_core::models::{JobStatus, Priority}; // --------------------------------------------------------------------------- // Fixture helpers — return fully owned data so tests don't fight the borrow @@ -470,6 +470,99 @@ async fn moving_queued_job_to_top_preempts_active_download() { ); } +/// A priority change must reorder only — it must never pause an +/// actively-downloading job (GH #124). With a single download slot, raising a +/// queued job to `Force` leaves the running job downloading; the new order +/// takes effect when the slot frees. (Contrast with `move_job`, an explicit +/// drag-to-top, which does preempt — see the test above.) +#[tokio::test] +async fn raising_queued_priority_does_not_preempt_active_download() { + let (xml_a, yenc_a, _mids_a) = make_fixture("prio-a", 12); + let (xml_b, yenc_b, _mids_b) = make_fixture("prio-b", 12); + + let primary = ServerProfile::start( + "primary", + MockConfig { + articles: yenc_a.into_iter().chain(yenc_b).collect(), + response_delay: Some(Duration::from_millis(120)), + ..Default::default() + }, + 2, + ) + .await + .with_priority(0); + + let engine = HarnessBuilder::new() + .with_server(primary) + .max_active_downloads(1) + .article_timeout(10) + .build(); + + let first_id = engine + .submit_nzb_xml("prio-a", xml_a) + .expect("submit first nzb"); + let first_started = engine + .wait_for(Duration::from_secs(5), |snap| { + snap.job(&first_id) + .map(|j| j.status == JobStatus::Downloading) + .unwrap_or(false) + }) + .await; + assert!(first_started, "first job never entered downloading state"); + + let second_id = engine + .submit_nzb_xml("prio-b", xml_b) + .expect("submit second nzb"); + let second_queued = engine + .wait_for(Duration::from_secs(5), |snap| { + matches!( + ( + snap.job(&first_id).map(|j| j.status), + snap.job(&second_id).map(|j| j.status) + ), + (Some(JobStatus::Downloading), Some(JobStatus::Queued)) + ) + }) + .await; + assert!( + second_queued, + "expected first job downloading and second queued before reprioritise" + ); + + // Raise the queued job to the highest priority via a priority change. + engine + .queue_manager + .set_job_priority(&second_id, Priority::Force) + .expect("raise second job to Force"); + + // The active job must NOT be preempted. Give any (incorrect) preemption a + // window to occur and assert it never does. + let preempted = engine + .wait_for(Duration::from_secs(2), |snap| { + snap.job(&first_id) + .map(|j| j.status == JobStatus::Paused) + .unwrap_or(false) + }) + .await; + assert!( + !preempted, + "priority change must not pause the active download" + ); + + let first = engine.job(&first_id).expect("first job present"); + let second = engine.job(&second_id).expect("second job present"); + assert_eq!( + first.status, + JobStatus::Downloading, + "active job should keep downloading after a priority change" + ); + assert_eq!( + second.status, + JobStatus::Queued, + "reprioritised job waits for a free slot instead of preempting" + ); +} + /// Sanity: job must reach a terminal state (Completed/Failed) after /// submission when the only priority-0 server is unreachable. Guards against /// the failure mode where backup workers get starvation-logged and never diff --git a/tests/startup_directory_errors.rs b/tests/startup_directory_errors.rs new file mode 100644 index 0000000..00b2274 --- /dev/null +++ b/tests/startup_directory_errors.rs @@ -0,0 +1,93 @@ +//! Regression tests for issue #62: rustnzb crashed on startup with a bare +//! `Permission denied (os error 13)` and no indication of which directory or +//! path caused it. `startup::initialize` must now attach the failing path +//! (and a permission hint) to any directory-creation error. + +use nzb_web::{StartupConfig, startup}; + +/// Positive: a custom data/incomplete/complete dir set (not matching the +/// three hardcoded defaults baked into the Docker image's init script) +/// succeeds as long as the parent is writable. +#[tokio::test] +async fn initialize_creates_custom_directories() { + let tmp = tempfile::tempdir().unwrap(); + let config_path = tmp.path().join("config.toml"); + let data_dir = tmp.path().join("custom-data"); + let incomplete_dir = tmp.path().join("custom-incomplete"); + let complete_dir = tmp.path().join("custom-complete"); + + let mut startup_cfg = StartupConfig { + config_path, + listen_addr: None, + port: None, + data_dir: Some(data_dir.clone()), + log_level: None, + }; + + // `initialize` only overrides data_dir via StartupConfig; incomplete/complete + // come from the config file, so write one first with our custom paths. + let mut config = nzb_web::nzb_core::config::AppConfig::default(); + config.general.data_dir = data_dir.clone(); + config.general.incomplete_dir = incomplete_dir.clone(); + config.general.complete_dir = complete_dir.clone(); + config.save(&startup_cfg.config_path).unwrap(); + startup_cfg.data_dir = None; // already set in the saved config + + let result = startup::initialize(startup_cfg, None).await; + assert!( + result.is_ok(), + "expected initialize to succeed, got: {:?}", + result.err().map(|e| format!("{e:?}")) + ); + + assert!(data_dir.is_dir()); + assert!(incomplete_dir.is_dir()); + assert!(complete_dir.is_dir()); +} + +/// Negative: a data_dir under a non-writable parent must fail with an error +/// that names the failing path — not a bare `os error 13`. +#[cfg(unix)] +#[tokio::test] +async fn initialize_reports_context_on_unwritable_data_dir() { + use std::os::unix::fs::PermissionsExt; + + let tmp = tempfile::tempdir().unwrap(); + let locked_parent = tmp.path().join("locked"); + std::fs::create_dir_all(&locked_parent).unwrap(); + std::fs::set_permissions(&locked_parent, std::fs::Permissions::from_mode(0o000)).unwrap(); + + let config_path = tmp.path().join("config.toml"); + let data_dir = locked_parent.join("data"); + + let startup_cfg = StartupConfig { + config_path, + listen_addr: None, + port: None, + data_dir: Some(data_dir.clone()), + log_level: None, + }; + + let result = startup::initialize(startup_cfg, None).await; + + // Restore permissions so the tempdir can be cleaned up. + std::fs::set_permissions(&locked_parent, std::fs::Permissions::from_mode(0o755)).unwrap(); + + let err = match result { + Err(e) => e, + // Running as root (e.g. CI containers) bypasses the permission + // check entirely, so there's nothing to assert. + Ok(_) => return, + }; + + let debug_text = format!("{err:?}"); + assert_ne!( + debug_text.trim(), + "Permission denied (os error 13)", + "regression: error must not be the bare os-error text from issue #62, got: {debug_text}" + ); + assert!( + debug_text.contains(&data_dir.display().to_string()), + "error should mention the failing path, got: {debug_text}" + ); +} diff --git a/tests/support/mod.rs b/tests/support/mod.rs new file mode 100644 index 0000000..f04827a --- /dev/null +++ b/tests/support/mod.rs @@ -0,0 +1 @@ +pub mod sab_contract; diff --git a/tests/support/sab_contract.rs b/tests/support/sab_contract.rs new file mode 100644 index 0000000..a757e1b --- /dev/null +++ b/tests/support/sab_contract.rs @@ -0,0 +1,99 @@ +use serde_json::Value; + +/// Load a checked-in SABnzbd golden response. +pub fn golden(source: &str) -> Value { + serde_json::from_str(source).expect("golden SABnzbd fixture must be valid JSON") +} + +/// Normalize dynamic response fields and assert their original JSON types. +/// +/// Golden fixtures use `$type:*` markers only for values that cannot be made +/// deterministic (timestamps, rates, paths, host measurements, and generated +/// IDs). The marker is copied into the normalized response after its type has +/// been checked, leaving an ordinary equality assertion to report missing, +/// extra, or semantically different fields with a complete JSON diff. +pub fn normalize_dynamic_fields(actual: &mut Value, expected: &Value) { + normalize_at(actual, expected, "$"); +} + +pub fn assert_matches_golden(mut actual: Value, expected: &Value) { + normalize_dynamic_fields(&mut actual, expected); + assert_eq!(actual, *expected, "normalized SABnzbd response mismatch"); +} + +fn normalize_at(actual: &mut Value, expected: &Value, path: &str) { + if let Some(marker) = expected.as_str().filter(|value| value.starts_with("$type:")) { + assert_marker(actual, marker, path); + *actual = expected.clone(); + return; + } + + match (actual, expected) { + (Value::Object(actual), Value::Object(expected)) => { + for (key, expected_value) in expected { + let field_path = format!("{path}.{key}"); + let actual_value = actual + .get_mut(key) + .unwrap_or_else(|| panic!("missing SABnzbd response field `{field_path}`")); + normalize_at(actual_value, expected_value, &field_path); + } + } + (Value::Array(actual), Value::Array(expected)) => { + assert_eq!( + actual.len(), + expected.len(), + "array length differs at `{path}`" + ); + for (index, (actual_value, expected_value)) in + actual.iter_mut().zip(expected).enumerate() + { + normalize_at(actual_value, expected_value, &format!("{path}[{index}]")); + } + } + _ => {} + } +} + +fn assert_marker(actual: &Value, marker: &str, path: &str) { + let valid = match marker { + "$type:any" => true, + "$type:array" => actual.is_array(), + "$type:boolean" => actual.is_boolean(), + "$type:integer" => actual.as_i64().is_some() || actual.as_u64().is_some(), + "$type:null" => actual.is_null(), + "$type:null-or-string" => actual.is_null() || actual.is_string(), + "$type:number" => actual.is_number(), + "$type:object" => actual.is_object(), + "$type:string" => actual.is_string(), + unsupported => panic!("unsupported golden marker `{unsupported}` at `{path}`"), + }; + + assert!( + valid, + "SABnzbd response type mismatch at `{path}`: expected `{marker}`, got {actual}" + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn normalizes_only_marked_dynamic_fields() { + let expected = json!({"fixed": "Idle", "rate": "$type:string", "count": "$type:integer"}); + let mut actual = json!({"fixed": "Idle", "rate": "12.3 MB/s", "count": 4}); + + normalize_dynamic_fields(&mut actual, &expected); + + assert_eq!(actual, expected); + } + + #[test] + #[should_panic(expected = "$.rate")] + fn reports_the_path_of_a_dynamic_type_mismatch() { + let expected = json!({"rate": "$type:string"}); + let mut actual = json!({"rate": 12}); + normalize_dynamic_fields(&mut actual, &expected); + } +} diff --git a/tests/workflow_fixtures.rs b/tests/workflow_fixtures.rs new file mode 100644 index 0000000..a2e28a4 --- /dev/null +++ b/tests/workflow_fixtures.rs @@ -0,0 +1,83 @@ +//! Local filesystem and feed fixture policy checks. + +use std::io::Write; +use std::path::Path; +use std::time::Duration; + +use flate2::Compression; +use flate2::write::GzEncoder; +use nzb_web::dir_watcher::DirWatcher; +use nzb_web::log_buffer::LogBuffer; +use nzb_web::nzb_core::db::Database; +use nzb_web::nzb_core::models::JobStatus; +use nzb_web::queue_manager::QueueManager; + +#[test] +fn gzip_fixture_is_deterministic_and_uses_the_watch_folder_suffix() { + let input = b""; + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(input).unwrap(); + let compressed = encoder.finish().unwrap(); + let mut second_encoder = GzEncoder::new(Vec::new(), Compression::default()); + second_encoder.write_all(input).unwrap(); + let second_compressed = second_encoder.finish().unwrap(); + assert!(!compressed.is_empty()); + assert_eq!(compressed, second_compressed); + assert!( + Path::new("release.nzb.gz") + .to_string_lossy() + .ends_with(".nzb.gz") + ); +} + +#[tokio::test] +async fn existing_gzip_nzb_is_imported_once_and_moved_to_processed() { + let temp = tempfile::tempdir().unwrap(); + let watch_dir = temp.path().join("watch"); + let incomplete = temp.path().join("incomplete"); + let complete = temp.path().join("complete"); + std::fs::create_dir_all(&watch_dir).unwrap(); + let source = br#"alt.testwatched-1@test"#; + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(source).unwrap(); + let compressed = encoder.finish().unwrap(); + let input = watch_dir.join("watched.nzb.gz"); + std::fs::write(&input, compressed).unwrap(); + + let queue = QueueManager::new( + Vec::new(), + Database::open_memory().unwrap(), + incomplete.clone(), + complete, + LogBuffer::default(), + 1, + Vec::new(), + 0, + 0, + false, + 5, + true, + true, + 100.0, + 2, + ); + let watcher = DirWatcher::new(watch_dir.clone(), queue.clone()); + let watcher_task = tokio::spawn(watcher.run()); + let imported = tokio::time::timeout(Duration::from_secs(2), async { + loop { + if queue.queue_size() == 1 { + break; + } + tokio::task::yield_now().await; + } + }) + .await; + watcher_task.abort(); + assert!( + imported.is_ok(), + "watch folder did not enqueue the gzip NZB" + ); + assert!(watch_dir.join("processed/watched.nzb.gz").exists()); + assert!(!input.exists()); + assert_eq!(queue.get_jobs()[0].status, JobStatus::Downloading); +}