From 70afb678b74e571d9e46743d973eddf895a2acd3 Mon Sep 17 00:00:00 2001 From: Grayson Hieb Date: Sun, 9 Aug 2026 12:21:19 -0600 Subject: [PATCH 01/10] fix(db): cascade library and user cleanup --- ...igence_artifact_source_library_cascade.sql | 259 +++++ crates/ferrex-core/tests/library_deletion.rs | 991 ++++++++++++++++++ 2 files changed, 1250 insertions(+) create mode 100644 crates/ferrex-core/migrations/012_intelligence_artifact_source_library_cascade.sql create mode 100644 crates/ferrex-core/tests/library_deletion.rs diff --git a/crates/ferrex-core/migrations/012_intelligence_artifact_source_library_cascade.sql b/crates/ferrex-core/migrations/012_intelligence_artifact_source_library_cascade.sql new file mode 100644 index 00000000..01236d9e --- /dev/null +++ b/crates/ferrex-core/migrations/012_intelligence_artifact_source_library_cascade.sql @@ -0,0 +1,259 @@ +-- Keep provenance edges and scoped collections consistent when their owning +-- source, library, or user is deleted. +-- +-- The original SET NULL actions contradicted CHECK constraints that require +-- the selected source/owner/scope reference to remain non-null. PostgreSQL +-- therefore rejected deletes instead of completing the owning cascade. An +-- edge or scoped row cannot remain meaningful without that referenced owner, +-- so delete it with the owner rather than manufacturing an invalid NULL. + +CREATE SCHEMA IF NOT EXISTS ferrex; + +DO $$ +DECLARE + app_schema text; +BEGIN + SELECT n.nspname + INTO app_schema + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relname = 'libraries' + AND c.relkind IN ('r', 'p') + AND n.nspname IN ('ferrex', 'public') + ORDER BY CASE WHEN n.nspname = 'ferrex' THEN 0 ELSE 1 END + LIMIT 1; + + IF app_schema IS NULL THEN + app_schema := 'ferrex'; + END IF; + + PERFORM set_config('search_path', format('%I, public', app_schema), false); +END $$; + +ALTER TABLE intelligence_artifact_sources + DROP CONSTRAINT IF EXISTS intelligence_artifact_sources_source_library_id_fkey; + +ALTER TABLE intelligence_artifact_sources + DROP CONSTRAINT IF EXISTS intelligence_artifact_sources_source_media_context_id_fkey; + +ALTER TABLE intelligence_artifact_sources + DROP CONSTRAINT IF EXISTS intelligence_artifact_sources_source_search_document_id_fkey; + +ALTER TABLE intelligence_artifact_sources + DROP CONSTRAINT IF EXISTS intelligence_artifact_sources_source_artifact_id_fkey; + +ALTER TABLE intelligence_artifact_sources + DROP CONSTRAINT IF EXISTS intelligence_artifact_sources_source_run_id_fkey; + +ALTER TABLE intelligence_artifact_sources + DROP CONSTRAINT IF EXISTS intelligence_artifact_sources_source_tool_call_id_fkey; + +ALTER TABLE intelligence_artifact_sources + DROP CONSTRAINT IF EXISTS intelligence_artifact_sources_source_transcript_source_id_fkey; + +ALTER TABLE intelligence_artifact_sources + ADD CONSTRAINT intelligence_artifact_sources_source_library_id_fkey + FOREIGN KEY (source_library_id) + REFERENCES libraries(id) + ON DELETE CASCADE; + +ALTER TABLE intelligence_artifact_sources + ADD CONSTRAINT intelligence_artifact_sources_source_media_context_id_fkey + FOREIGN KEY (source_media_context_id) + REFERENCES intelligence_media_context(id) + ON DELETE CASCADE; + +ALTER TABLE intelligence_artifact_sources + ADD CONSTRAINT intelligence_artifact_sources_source_search_document_id_fkey + FOREIGN KEY (source_search_document_id) + REFERENCES intelligence_search_documents(id) + ON DELETE CASCADE; + +ALTER TABLE intelligence_artifact_sources + ADD CONSTRAINT intelligence_artifact_sources_source_artifact_id_fkey + FOREIGN KEY (source_artifact_id) + REFERENCES intelligence_artifacts(id) + ON DELETE CASCADE; + +ALTER TABLE intelligence_artifact_sources + ADD CONSTRAINT intelligence_artifact_sources_source_run_id_fkey + FOREIGN KEY (source_run_id) + REFERENCES intelligence_runs(id) + ON DELETE CASCADE; + +ALTER TABLE intelligence_artifact_sources + ADD CONSTRAINT intelligence_artifact_sources_source_tool_call_id_fkey + FOREIGN KEY (source_tool_call_id) + REFERENCES intelligence_tool_calls(id) + ON DELETE CASCADE; + +ALTER TABLE intelligence_artifact_sources + ADD CONSTRAINT intelligence_artifact_sources_source_transcript_source_id_fkey + FOREIGN KEY (source_transcript_source_id) + REFERENCES transcript_sources(id) + ON DELETE CASCADE; + +COMMENT ON CONSTRAINT intelligence_artifact_sources_source_library_id_fkey + ON intelligence_artifact_sources IS + 'Delete provenance edges when their source library is deleted; media edges cannot remain valid with a NULL source_library_id.'; + +COMMENT ON CONSTRAINT intelligence_artifact_sources_source_media_context_id_fkey + ON intelligence_artifact_sources IS + 'Delete provenance edges with their media-context source; media-context edges require a non-null source reference.'; + +COMMENT ON CONSTRAINT intelligence_artifact_sources_source_search_document_id_fkey + ON intelligence_artifact_sources IS + 'Delete provenance edges with their search-document source; search-document edges require a non-null source reference.'; + +COMMENT ON CONSTRAINT intelligence_artifact_sources_source_artifact_id_fkey + ON intelligence_artifact_sources IS + 'Delete provenance edges with their related-artifact source; artifact edges require a non-null source reference.'; + +COMMENT ON CONSTRAINT intelligence_artifact_sources_source_run_id_fkey + ON intelligence_artifact_sources IS + 'Delete provenance edges with their run source; run edges require a non-null source reference.'; + +COMMENT ON CONSTRAINT intelligence_artifact_sources_source_tool_call_id_fkey + ON intelligence_artifact_sources IS + 'Delete provenance edges with their tool-call source; tool-call edges require a non-null source reference.'; + +COMMENT ON CONSTRAINT intelligence_artifact_sources_source_transcript_source_id_fkey + ON intelligence_artifact_sources IS + 'Delete provenance edges with their transcript source; transcript-source edges require a non-null source reference.'; + +ALTER TABLE collection_definitions + DROP CONSTRAINT IF EXISTS collection_definitions_owner_user_id_fkey; + +ALTER TABLE collection_definitions + DROP CONSTRAINT IF EXISTS collection_definitions_library_id_fkey; + +ALTER TABLE collection_definitions + ADD CONSTRAINT collection_definitions_owner_user_id_fkey + FOREIGN KEY (owner_user_id) + REFERENCES users(id) + ON DELETE CASCADE; + +ALTER TABLE collection_definitions + ADD CONSTRAINT collection_definitions_library_id_fkey + FOREIGN KEY (library_id) + REFERENCES libraries(id) + ON DELETE CASCADE; + +COMMENT ON CONSTRAINT collection_definitions_owner_user_id_fkey + ON collection_definitions IS + 'Delete user-owned collection definitions with their owner; owner_type=user requires owner_user_id.'; + +COMMENT ON CONSTRAINT collection_definitions_library_id_fkey + ON collection_definitions IS + 'Delete library-scoped collection definitions with their library; scope=library requires library_id.'; + +ALTER TABLE collection_shelf_placements + DROP CONSTRAINT IF EXISTS collection_shelf_placements_scope_user_id_fkey; + +ALTER TABLE collection_shelf_placements + DROP CONSTRAINT IF EXISTS collection_shelf_placements_scope_library_id_fkey; + +ALTER TABLE collection_shelf_placements + ADD CONSTRAINT collection_shelf_placements_scope_user_id_fkey + FOREIGN KEY (scope_user_id) + REFERENCES users(id) + ON DELETE CASCADE; + +ALTER TABLE collection_shelf_placements + ADD CONSTRAINT collection_shelf_placements_scope_library_id_fkey + FOREIGN KEY (scope_library_id) + REFERENCES libraries(id) + ON DELETE CASCADE; + +COMMENT ON CONSTRAINT collection_shelf_placements_scope_user_id_fkey + ON collection_shelf_placements IS + 'Delete user-scoped shelf placements with their user; placement_scope=user requires scope_user_id.'; + +COMMENT ON CONSTRAINT collection_shelf_placements_scope_library_id_fkey + ON collection_shelf_placements IS + 'Delete library-scoped shelf placements with their library; placement_scope=library requires scope_library_id.'; + +ALTER TABLE sync_sessions + DROP CONSTRAINT IF EXISTS sync_sessions_host_id_fkey; + +ALTER TABLE sync_sessions + ADD CONSTRAINT sync_sessions_host_id_fkey + FOREIGN KEY (host_id) + REFERENCES users(id) + ON DELETE CASCADE; + +COMMENT ON CONSTRAINT sync_sessions_host_id_fkey + ON sync_sessions IS + 'Delete sync sessions with their required host user.'; + +ALTER TABLE sync_participants + DROP CONSTRAINT IF EXISTS sync_participants_user_id_fkey; + +ALTER TABLE sync_participants + ADD CONSTRAINT sync_participants_user_id_fkey + FOREIGN KEY (user_id) + REFERENCES users(id) + ON DELETE CASCADE; + +COMMENT ON CONSTRAINT sync_participants_user_id_fkey + ON sync_participants IS + 'Delete sync participation with its required user; session deletion already cascades through session_id.'; + +ALTER TABLE sync_session_history + DROP CONSTRAINT IF EXISTS sync_session_history_user_id_fkey; + +ALTER TABLE sync_session_history + ADD CONSTRAINT sync_session_history_user_id_fkey + FOREIGN KEY (user_id) + REFERENCES users(id) + ON DELETE SET NULL; + +COMMENT ON CONSTRAINT sync_session_history_user_id_fkey + ON sync_session_history IS + 'Preserve session audit history while clearing its optional actor when that user is deleted.'; + +ALTER TABLE user_permissions + DROP CONSTRAINT IF EXISTS user_permissions_granted_by_fkey; + +ALTER TABLE user_permissions + ADD CONSTRAINT user_permissions_granted_by_fkey + FOREIGN KEY (granted_by) + REFERENCES users(id) + ON DELETE SET NULL; + +COMMENT ON CONSTRAINT user_permissions_granted_by_fkey + ON user_permissions IS + 'Preserve permission assignments while clearing their optional grantor when that user is deleted.'; + +ALTER TABLE user_roles + DROP CONSTRAINT IF EXISTS user_roles_granted_by_fkey; + +ALTER TABLE user_roles + ADD CONSTRAINT user_roles_granted_by_fkey + FOREIGN KEY (granted_by) + REFERENCES users(id) + ON DELETE SET NULL; + +COMMENT ON CONSTRAINT user_roles_granted_by_fkey + ON user_roles IS + 'Preserve role assignments while clearing their optional grantor when that user is deleted.'; + +DELETE FROM user_episode_state AS episode_state +WHERE NOT EXISTS ( + SELECT 1 + FROM users + WHERE users.id = episode_state.user_id +); + +ALTER TABLE user_episode_state + DROP CONSTRAINT IF EXISTS user_episode_state_user_id_fkey; + +ALTER TABLE user_episode_state + ADD CONSTRAINT user_episode_state_user_id_fkey + FOREIGN KEY (user_id) + REFERENCES users(id) + ON DELETE CASCADE; + +COMMENT ON CONSTRAINT user_episode_state_user_id_fkey + ON user_episode_state IS + 'Delete episode watch state with its owning user; pre-existing orphan state is removed before this constraint is installed.'; diff --git a/crates/ferrex-core/tests/library_deletion.rs b/crates/ferrex-core/tests/library_deletion.rs new file mode 100644 index 00000000..85e96e91 --- /dev/null +++ b/crates/ferrex-core/tests/library_deletion.rs @@ -0,0 +1,991 @@ +//! PostgreSQL library/user-deletion integrity regressions. + +use ferrex_core::database::{ + repositories::{ + library::PostgresLibraryRepository, users::PostgresUsersRepository, + }, + repository_ports::{library::LibraryRepository, users::UsersRepository}, +}; +use ferrex_core::types::LibraryId; +use sqlx::PgPool; +use uuid::Uuid; + +async fn seed_library(pool: &PgPool, id: Uuid, name: &str) { + sqlx::query( + r#" + INSERT INTO libraries (id, name, library_type, paths) + VALUES ($1, $2, 'movies', ARRAY[$3]::varchar[]) + "#, + ) + .bind(id) + .bind(name) + .bind(format!("/fixtures/{name}")) + .execute(pool) + .await + .expect("seed library"); +} + +async fn seed_user(pool: &PgPool, id: Uuid, username: &str) { + sqlx::query( + r#" + INSERT INTO users (id, username, display_name) + VALUES ($1, $2, $3) + "#, + ) + .bind(id) + .bind(username) + .bind(format!("Deletion fixture {username}")) + .execute(pool) + .await + .expect("seed user"); +} + +async fn seed_artifact( + pool: &PgPool, + id: Uuid, + library_id: Option, + user_id: Option, + title: &str, +) { + let scope = if user_id.is_some() { "user" } else { "global" }; + sqlx::query( + r#" + INSERT INTO intelligence_artifacts ( + id, artifact_kind, scope, status, library_id, user_id, title, + content_hash + ) + VALUES ($1, 'note', $2, 'active', $3, $4, $5, $6) + "#, + ) + .bind(id) + .bind(scope) + .bind(library_id) + .bind(user_id) + .bind(title) + .bind("1".repeat(64)) + .execute(pool) + .await + .expect("seed artifact"); +} + +async fn seed_media_context( + pool: &PgPool, + id: Uuid, + library_id: Uuid, + user_id: Option, + media_id: Uuid, +) { + sqlx::query( + r#" + INSERT INTO intelligence_media_context ( + id, library_id, user_id, media_id, media_type, context_kind, + title, content_hash + ) + VALUES ($1, $2, $3, $4, 'movie', 'metadata', $5, $6) + "#, + ) + .bind(id) + .bind(library_id) + .bind(user_id) + .bind(media_id) + .bind(format!("Context {id}")) + .bind("2".repeat(64)) + .execute(pool) + .await + .expect("seed media context"); +} + +async fn seed_search_document( + pool: &PgPool, + id: Uuid, + library_id: Uuid, + user_id: Option, + media_id: Uuid, +) { + sqlx::query( + r#" + INSERT INTO intelligence_search_documents ( + id, library_id, user_id, media_id, media_type, document_kind, + title, search_text, content_hash + ) + VALUES ($1, $2, $3, $4, 'movie', 'combined', $5, $6, $7) + "#, + ) + .bind(id) + .bind(library_id) + .bind(user_id) + .bind(media_id) + .bind(format!("Document {id}")) + .bind("bounded deletion regression document") + .bind("3".repeat(64)) + .execute(pool) + .await + .expect("seed search document"); +} + +async fn seed_transcript_source( + pool: &PgPool, + id: Uuid, + library_id: Uuid, + media_id: Uuid, +) { + let media_file_id = Uuid::now_v7(); + sqlx::query( + r#" + INSERT INTO media_files ( + id, library_id, media_id, media_type, file_path, filename, file_size + ) + VALUES ($1, $2, $3, 'movie', $4, $5, 1) + "#, + ) + .bind(media_file_id) + .bind(library_id) + .bind(media_id) + .bind(format!("/fixtures/{media_file_id}.mkv")) + .bind(format!("{media_file_id}.mkv")) + .execute(pool) + .await + .expect("seed transcript media file"); + + sqlx::query( + r#" + INSERT INTO transcript_sources ( + id, library_id, media_id, media_type, media_file_id, source_kind, + status, language_code, source_key, source_content_hash + ) + VALUES ($1, $2, $3, 'movie', $4, 'manual', 'active', 'und', $5, $6) + "#, + ) + .bind(id) + .bind(library_id) + .bind(media_id) + .bind(media_file_id) + .bind(format!("manual:{id}")) + .bind("4".repeat(64)) + .execute(pool) + .await + .expect("seed transcript source"); +} + +async fn seed_library_scoped_collection( + pool: &PgPool, + collection_id: Uuid, + placement_id: Uuid, + library_id: Uuid, +) { + sqlx::query( + r#" + INSERT INTO collection_definitions ( + id, stable_key, title, owner_type, scope, library_id + ) + VALUES ($1, $2, $3, 'system', 'library', $4) + "#, + ) + .bind(collection_id) + .bind(format!("library-delete-{collection_id}")) + .bind("Library deletion collection") + .bind(library_id) + .execute(pool) + .await + .expect("seed library-scoped collection"); + + sqlx::query( + r#" + INSERT INTO collection_shelf_placements ( + id, collection_id, surface, shelf_key, placement_scope, + placement_scope_key, scope_library_id, position, position_key + ) + VALUES ($1, $2, 'library', $3, 'library', $4, $5, 0, 0) + "#, + ) + .bind(placement_id) + .bind(collection_id) + .bind(format!("library-delete-{collection_id}")) + .bind(format!("library:{library_id}")) + .bind(library_id) + .execute(pool) + .await + .expect("seed library-scoped shelf placement"); +} + +async fn seed_user_scoped_collection( + pool: &PgPool, + collection_id: Uuid, + placement_id: Uuid, + user_id: Uuid, +) { + sqlx::query( + r#" + INSERT INTO collection_definitions ( + id, stable_key, title, owner_type, owner_user_id, scope + ) + VALUES ($1, $2, $3, 'user', $4, 'user') + "#, + ) + .bind(collection_id) + .bind(format!("user-delete-{collection_id}")) + .bind("User deletion collection") + .bind(user_id) + .execute(pool) + .await + .expect("seed user-owned collection"); + + sqlx::query( + r#" + INSERT INTO collection_shelf_placements ( + id, collection_id, surface, shelf_key, placement_scope, + placement_scope_key, scope_user_id, position, position_key + ) + VALUES ($1, $2, 'home', $3, 'user', $4, $5, 0, 0) + "#, + ) + .bind(placement_id) + .bind(collection_id) + .bind(format!("user-delete-{collection_id}")) + .bind(format!("user:{user_id}")) + .bind(user_id) + .execute(pool) + .await + .expect("seed user-scoped shelf placement"); +} + +async fn seed_intelligence_run_and_tool_call( + pool: &PgPool, + run_id: Uuid, + tool_call_id: Uuid, +) { + sqlx::query( + r#" + INSERT INTO intelligence_runs (id, run_kind, status) + VALUES ($1, 'search', 'queued') + "#, + ) + .bind(run_id) + .execute(pool) + .await + .expect("seed intelligence run"); + + sqlx::query( + r#" + INSERT INTO intelligence_tool_calls ( + id, run_id, sequence, tool_kind, tool_name, status + ) + VALUES ($1, $2, 0, 'search', 'deletion-regression', 'queued') + "#, + ) + .bind(tool_call_id) + .bind(run_id) + .execute(pool) + .await + .expect("seed intelligence tool call"); +} + +#[sqlx::test(migrator = "ferrex_core::MIGRATOR")] +async fn deleting_library_removes_its_provenance_edges_without_deleting_artifact( + pool: PgPool, +) { + let deleted_library_id = Uuid::now_v7(); + let retained_library_id = Uuid::now_v7(); + let artifact_id = Uuid::now_v7(); + + seed_library(&pool, deleted_library_id, "delete-source-library").await; + seed_library(&pool, retained_library_id, "retain-source-library").await; + + sqlx::query( + r#" + INSERT INTO intelligence_artifacts ( + id, artifact_kind, scope, status, title, content_hash + ) + VALUES ($1, 'note', 'global', 'active', $2, $3) + "#, + ) + .bind(artifact_id) + .bind("Cross-library artifact") + .bind("0".repeat(64)) + .execute(&pool) + .await + .expect("seed global artifact"); + + for (source_ordinal, source_library_id) in + [(0_i32, deleted_library_id), (1_i32, retained_library_id)] + { + sqlx::query( + r#" + INSERT INTO intelligence_artifact_sources ( + artifact_id, + source_ordinal, + source_kind, + source_library_id, + source_media_id, + source_media_type + ) + VALUES ($1, $2, 'media', $3, $4, 'movie') + "#, + ) + .bind(artifact_id) + .bind(source_ordinal) + .bind(source_library_id) + .bind(Uuid::now_v7()) + .execute(&pool) + .await + .expect("seed media provenance edge"); + } + + PostgresLibraryRepository::new(pool.clone()) + .delete_library(LibraryId(deleted_library_id)) + .await + .expect("library deletion should cascade through provenance edges"); + + let deleted_library_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM libraries WHERE id = $1", + ) + .bind(deleted_library_id) + .fetch_one(&pool) + .await + .expect("count deleted library"); + assert_eq!(deleted_library_count, 0); + + let remaining_source_libraries = sqlx::query_scalar::<_, Uuid>( + r#" + SELECT source_library_id + FROM intelligence_artifact_sources + WHERE artifact_id = $1 + ORDER BY source_ordinal + "#, + ) + .bind(artifact_id) + .fetch_all(&pool) + .await + .expect("load remaining provenance edges"); + assert_eq!(remaining_source_libraries, vec![retained_library_id]); + + let artifact_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM intelligence_artifacts WHERE id = $1", + ) + .bind(artifact_id) + .fetch_one(&pool) + .await + .expect("count preserved artifact"); + assert_eq!(artifact_count, 1); +} + +#[sqlx::test(migrator = "ferrex_core::MIGRATOR")] +async fn deleting_run_and_tool_call_removes_their_provenance_edges( + pool: PgPool, +) { + let artifact_id = Uuid::now_v7(); + let run_id = Uuid::now_v7(); + let tool_call_id = Uuid::now_v7(); + + seed_artifact( + &pool, + artifact_id, + None, + None, + "Preserved run provenance owner", + ) + .await; + seed_intelligence_run_and_tool_call(&pool, run_id, tool_call_id).await; + + sqlx::query( + r#" + INSERT INTO intelligence_artifact_sources ( + artifact_id, source_ordinal, source_kind, source_run_id + ) + VALUES ($1, 0, 'run', $2) + "#, + ) + .bind(artifact_id) + .bind(run_id) + .execute(&pool) + .await + .expect("seed run provenance"); + + sqlx::query( + r#" + INSERT INTO intelligence_artifact_sources ( + artifact_id, source_ordinal, source_kind, source_tool_call_id + ) + VALUES ($1, 1, 'tool_call', $2) + "#, + ) + .bind(artifact_id) + .bind(tool_call_id) + .execute(&pool) + .await + .expect("seed tool-call provenance"); + + sqlx::query("DELETE FROM intelligence_tool_calls WHERE id = $1") + .bind(tool_call_id) + .execute(&pool) + .await + .expect("delete intelligence tool call"); + + let remaining_ordinals = sqlx::query_scalar::<_, i32>( + r#" + SELECT source_ordinal + FROM intelligence_artifact_sources + WHERE artifact_id = $1 + ORDER BY source_ordinal + "#, + ) + .bind(artifact_id) + .fetch_all(&pool) + .await + .expect("load provenance after tool-call deletion"); + assert_eq!(remaining_ordinals, vec![0]); + + sqlx::query("DELETE FROM intelligence_runs WHERE id = $1") + .bind(run_id) + .execute(&pool) + .await + .expect("delete intelligence run"); + + let remaining_source_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM intelligence_artifact_sources WHERE artifact_id = $1", + ) + .bind(artifact_id) + .fetch_one(&pool) + .await + .expect("count provenance after run deletion"); + assert_eq!(remaining_source_count, 0); + + let artifact_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM intelligence_artifacts WHERE id = $1", + ) + .bind(artifact_id) + .fetch_one(&pool) + .await + .expect("count preserved run provenance owner"); + assert_eq!(artifact_count, 1); +} + +#[sqlx::test(migrator = "ferrex_core::MIGRATOR")] +async fn deleting_library_cascades_indirect_provenance_and_scoped_collections( + pool: PgPool, +) { + let library_id = Uuid::now_v7(); + let owner_artifact_id = Uuid::now_v7(); + let related_artifact_id = Uuid::now_v7(); + let media_id = Uuid::now_v7(); + let media_context_id = Uuid::now_v7(); + let search_document_id = Uuid::now_v7(); + let transcript_source_id = Uuid::now_v7(); + let collection_id = Uuid::now_v7(); + let placement_id = Uuid::now_v7(); + + seed_library(&pool, library_id, "cascade-all-library-edges").await; + seed_artifact( + &pool, + owner_artifact_id, + None, + None, + "Preserved cross-library artifact", + ) + .await; + seed_artifact( + &pool, + related_artifact_id, + Some(library_id), + None, + "Deleted library artifact", + ) + .await; + seed_media_context(&pool, media_context_id, library_id, None, media_id) + .await; + seed_search_document(&pool, search_document_id, library_id, None, media_id) + .await; + seed_transcript_source(&pool, transcript_source_id, library_id, media_id) + .await; + seed_library_scoped_collection( + &pool, + collection_id, + placement_id, + library_id, + ) + .await; + + sqlx::query( + r#" + INSERT INTO intelligence_artifact_sources ( + artifact_id, source_ordinal, source_kind, source_library_id, + source_media_id, source_media_type + ) + VALUES ($1, 0, 'media', $2, $3, 'movie') + "#, + ) + .bind(owner_artifact_id) + .bind(library_id) + .bind(media_id) + .execute(&pool) + .await + .expect("seed direct library provenance"); + + sqlx::query( + r#" + INSERT INTO intelligence_artifact_sources ( + artifact_id, source_ordinal, source_kind, source_media_context_id + ) + VALUES ($1, 1, 'media_context', $2) + "#, + ) + .bind(owner_artifact_id) + .bind(media_context_id) + .execute(&pool) + .await + .expect("seed media-context provenance"); + + sqlx::query( + r#" + INSERT INTO intelligence_artifact_sources ( + artifact_id, source_ordinal, source_kind, + source_search_document_id + ) + VALUES ($1, 2, 'search_document', $2) + "#, + ) + .bind(owner_artifact_id) + .bind(search_document_id) + .execute(&pool) + .await + .expect("seed search-document provenance"); + + sqlx::query( + r#" + INSERT INTO intelligence_artifact_sources ( + artifact_id, source_ordinal, source_kind, source_artifact_id + ) + VALUES ($1, 3, 'artifact', $2) + "#, + ) + .bind(owner_artifact_id) + .bind(related_artifact_id) + .execute(&pool) + .await + .expect("seed related-artifact provenance"); + + sqlx::query( + r#" + INSERT INTO intelligence_artifact_sources ( + artifact_id, source_ordinal, source_kind, + source_transcript_source_id + ) + VALUES ($1, 4, 'transcript_source', $2) + "#, + ) + .bind(owner_artifact_id) + .bind(transcript_source_id) + .execute(&pool) + .await + .expect("seed transcript-source provenance"); + + let seeded_source_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM intelligence_artifact_sources WHERE artifact_id = $1", + ) + .bind(owner_artifact_id) + .fetch_one(&pool) + .await + .expect("count seeded provenance"); + assert_eq!(seeded_source_count, 5); + + PostgresLibraryRepository::new(pool.clone()) + .delete_library(LibraryId(library_id)) + .await + .expect("library deletion should cascade every owned edge and scope"); + + let remaining_sources = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM intelligence_artifact_sources WHERE artifact_id = $1", + ) + .bind(owner_artifact_id) + .fetch_one(&pool) + .await + .expect("count remaining provenance"); + assert_eq!(remaining_sources, 0); + + let owner_artifact_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM intelligence_artifacts WHERE id = $1", + ) + .bind(owner_artifact_id) + .fetch_one(&pool) + .await + .expect("count preserved owner artifact"); + assert_eq!(owner_artifact_count, 1); + + let collection_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM collection_definitions WHERE id = $1", + ) + .bind(collection_id) + .fetch_one(&pool) + .await + .expect("count library-scoped collection"); + assert_eq!(collection_count, 0); + + let placement_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM collection_shelf_placements WHERE id = $1", + ) + .bind(placement_id) + .fetch_one(&pool) + .await + .expect("count library-scoped shelf placement"); + assert_eq!(placement_count, 0); +} + +#[sqlx::test(migrator = "ferrex_core::MIGRATOR")] +async fn deleting_user_cascades_owned_provenance_and_scoped_collections( + pool: PgPool, +) { + let library_id = Uuid::now_v7(); + let deleted_user_id = Uuid::now_v7(); + let retained_user_id = Uuid::now_v7(); + let owner_artifact_id = Uuid::now_v7(); + let related_artifact_id = Uuid::now_v7(); + let media_id = Uuid::now_v7(); + let media_context_id = Uuid::now_v7(); + let search_document_id = Uuid::now_v7(); + let collection_id = Uuid::now_v7(); + let placement_id = Uuid::now_v7(); + let deleted_host_session_id = Uuid::now_v7(); + let retained_host_session_id = Uuid::now_v7(); + let history_id = Uuid::now_v7(); + let permission_id = Uuid::now_v7(); + let role_id = Uuid::now_v7(); + + seed_library(&pool, library_id, "retain-user-delete-library").await; + seed_user(&pool, deleted_user_id, "deletecascadeuser").await; + seed_user(&pool, retained_user_id, "retaincascadeuser").await; + seed_artifact( + &pool, + owner_artifact_id, + None, + None, + "Preserved global artifact", + ) + .await; + seed_artifact( + &pool, + related_artifact_id, + Some(library_id), + Some(deleted_user_id), + "Deleted user artifact", + ) + .await; + seed_media_context( + &pool, + media_context_id, + library_id, + Some(deleted_user_id), + media_id, + ) + .await; + seed_search_document( + &pool, + search_document_id, + library_id, + Some(deleted_user_id), + media_id, + ) + .await; + seed_user_scoped_collection( + &pool, + collection_id, + placement_id, + deleted_user_id, + ) + .await; + + for (session_id, room_code, host_id) in [ + (deleted_host_session_id, "DEL001", deleted_user_id), + (retained_host_session_id, "KEEP01", retained_user_id), + ] { + sqlx::query( + r#" + INSERT INTO sync_sessions ( + id, room_code, host_id, expires_at, media_uuid, media_type + ) + VALUES ($1, $2, $3, now() + interval '1 hour', $4, 0) + "#, + ) + .bind(session_id) + .bind(room_code) + .bind(host_id) + .bind(Uuid::now_v7()) + .execute(&pool) + .await + .expect("seed sync session"); + } + + for (session_id, user_id) in [ + (deleted_host_session_id, retained_user_id), + (retained_host_session_id, deleted_user_id), + ] { + sqlx::query( + "INSERT INTO sync_participants (session_id, user_id) VALUES ($1, $2)", + ) + .bind(session_id) + .bind(user_id) + .execute(&pool) + .await + .expect("seed sync participant"); + } + + sqlx::query( + r#" + INSERT INTO sync_session_history ( + id, session_id, event_type, event_data, user_id + ) + VALUES ($1, $2, 'participant_left', '{}'::jsonb, $3) + "#, + ) + .bind(history_id) + .bind(retained_host_session_id) + .bind(deleted_user_id) + .execute(&pool) + .await + .expect("seed sync-session history"); + + sqlx::query( + r#" + INSERT INTO permissions (id, name, category) + VALUES ($1, $2, 'deletion-regression') + "#, + ) + .bind(permission_id) + .bind(format!("deletion-regression-{permission_id}")) + .execute(&pool) + .await + .expect("seed permission"); + + sqlx::query( + r#" + INSERT INTO user_permissions ( + user_id, permission_id, granted, granted_by + ) + VALUES ($1, $2, true, $3) + "#, + ) + .bind(retained_user_id) + .bind(permission_id) + .bind(deleted_user_id) + .execute(&pool) + .await + .expect("seed permission grant"); + + sqlx::query( + r#" + INSERT INTO roles (id, name, is_system) + VALUES ($1, $2, false) + "#, + ) + .bind(role_id) + .bind(format!("del-{role_id}")) + .execute(&pool) + .await + .expect("seed role"); + + sqlx::query( + r#" + INSERT INTO user_roles (user_id, role_id, granted_by) + VALUES ($1, $2, $3) + "#, + ) + .bind(retained_user_id) + .bind(role_id) + .bind(deleted_user_id) + .execute(&pool) + .await + .expect("seed role grant"); + + sqlx::query( + r#" + INSERT INTO user_episode_state ( + user_id, tmdb_series_id, season_number, episode_number, + position, duration, last_watched + ) + VALUES ($1, 12345, 1, 1, 30.0, 60.0, 1) + "#, + ) + .bind(deleted_user_id) + .execute(&pool) + .await + .expect("seed episode watch state"); + + sqlx::query( + r#" + INSERT INTO intelligence_artifact_sources ( + artifact_id, source_ordinal, source_kind, source_media_context_id + ) + VALUES ($1, 0, 'media_context', $2) + "#, + ) + .bind(owner_artifact_id) + .bind(media_context_id) + .execute(&pool) + .await + .expect("seed user media-context provenance"); + + sqlx::query( + r#" + INSERT INTO intelligence_artifact_sources ( + artifact_id, source_ordinal, source_kind, + source_search_document_id + ) + VALUES ($1, 1, 'search_document', $2) + "#, + ) + .bind(owner_artifact_id) + .bind(search_document_id) + .execute(&pool) + .await + .expect("seed user search-document provenance"); + + sqlx::query( + r#" + INSERT INTO intelligence_artifact_sources ( + artifact_id, source_ordinal, source_kind, source_artifact_id + ) + VALUES ($1, 2, 'artifact', $2) + "#, + ) + .bind(owner_artifact_id) + .bind(related_artifact_id) + .execute(&pool) + .await + .expect("seed user related-artifact provenance"); + + PostgresUsersRepository::new(pool.clone()) + .delete_user(deleted_user_id) + .await + .expect("user deletion should cascade every owned edge and scope"); + + let remaining_sources = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM intelligence_artifact_sources WHERE artifact_id = $1", + ) + .bind(owner_artifact_id) + .fetch_one(&pool) + .await + .expect("count remaining user provenance"); + assert_eq!(remaining_sources, 0); + + let owner_artifact_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM intelligence_artifacts WHERE id = $1", + ) + .bind(owner_artifact_id) + .fetch_one(&pool) + .await + .expect("count preserved global artifact"); + assert_eq!(owner_artifact_count, 1); + + let retained_user_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM users WHERE id = $1", + ) + .bind(retained_user_id) + .fetch_one(&pool) + .await + .expect("count retained user"); + assert_eq!(retained_user_count, 1); + + let retained_library_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM libraries WHERE id = $1", + ) + .bind(library_id) + .fetch_one(&pool) + .await + .expect("count retained library"); + assert_eq!(retained_library_count, 1); + + let collection_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM collection_definitions WHERE id = $1", + ) + .bind(collection_id) + .fetch_one(&pool) + .await + .expect("count user-owned collection"); + assert_eq!(collection_count, 0); + + let placement_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM collection_shelf_placements WHERE id = $1", + ) + .bind(placement_id) + .fetch_one(&pool) + .await + .expect("count user-scoped shelf placement"); + assert_eq!(placement_count, 0); + + let deleted_host_session_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM sync_sessions WHERE id = $1", + ) + .bind(deleted_host_session_id) + .fetch_one(&pool) + .await + .expect("count deleted host session"); + assert_eq!(deleted_host_session_count, 0); + + let retained_host_session_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM sync_sessions WHERE id = $1", + ) + .bind(retained_host_session_id) + .fetch_one(&pool) + .await + .expect("count retained host session"); + assert_eq!(retained_host_session_count, 1); + + let stale_participant_count = sqlx::query_scalar::<_, i64>( + r#" + SELECT count(*) + FROM sync_participants + WHERE user_id = $1 OR session_id = $2 + "#, + ) + .bind(deleted_user_id) + .bind(deleted_host_session_id) + .fetch_one(&pool) + .await + .expect("count stale sync participants"); + assert_eq!(stale_participant_count, 0); + + let history_actor = sqlx::query_scalar::<_, Option>( + "SELECT user_id FROM sync_session_history WHERE id = $1", + ) + .bind(history_id) + .fetch_one(&pool) + .await + .expect("load retained sync-session history actor"); + assert_eq!(history_actor, None); + + let permission_grantor = sqlx::query_scalar::<_, Option>( + r#" + SELECT granted_by + FROM user_permissions + WHERE user_id = $1 AND permission_id = $2 + "#, + ) + .bind(retained_user_id) + .bind(permission_id) + .fetch_one(&pool) + .await + .expect("load retained permission grantor"); + assert_eq!(permission_grantor, None); + + let role_grantor = sqlx::query_scalar::<_, Option>( + r#" + SELECT granted_by + FROM user_roles + WHERE user_id = $1 AND role_id = $2 + "#, + ) + .bind(retained_user_id) + .bind(role_id) + .fetch_one(&pool) + .await + .expect("load retained role grantor"); + assert_eq!(role_grantor, None); + + let episode_state_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM user_episode_state WHERE user_id = $1", + ) + .bind(deleted_user_id) + .fetch_one(&pool) + .await + .expect("count deleted episode state"); + assert_eq!(episode_state_count, 0); +} From 3001000363383523e34d04e875ca71c6687fd417 Mon Sep 17 00:00:00 2001 From: Grayson Hieb Date: Sun, 9 Aug 2026 12:21:41 -0600 Subject: [PATCH 02/10] fix(scan): make orchestration and maintenance durable --- ...44d08b1b1d6b258076b7c1e5be01fa177725.json} | 17 +- ...ce79ddacee672c0c54fa981814a1f426da45c.json | 12 + ...243b186772940ee7fe717dff522578edcd627.json | 22 - ...843e4505db60a2133193709069793595b17f5.json | 44 + ...47bb9e1f822a714171868779b24b0628198a7.json | 14 + ...949fce59e54ea17278ed81e508a104cb1d202.json | 22 + ...1719ded381158f0f4f9b5eed62c7288a9bc19.json | 12 + ...e7c5ce8f129fbc4cffff0304dc78b0aebb115.json | 12 + ...58926044c7a6df889cb88d01aa99d727de1b2.json | 22 + ...a98efd8be663e9fdc1889241943ab4dad38de.json | 12 + ...944cc09243e40e6faa55d1eb962259dc65d54.json | 22 + ...32f6a6c267387d8504cd923a22a38843505ff.json | 12 + ...8d6f2542077c285de41902311c761de6ce2ab.json | 29 + ...da0a00981de305b0e54073dc432898b8299b7.json | 12 + ...ab08e7d092a559c1f478c87b5c4d35596634f.json | 12 + ...92150c47f69949e44359f5e91d3ae1ca6cdfe.json | 12 + ...2b5a80df83664af7c2e7fa42e0ca1935f9909.json | 22 + ...52af8e30e4e183857832cdd1fe43471066c31.json | 12 + ...c9b69e2ba173c4cd34792a784025c89fc459b.json | 12 + ...75c31a27c59f2e9a8f6e334ef82be0d348f86.json | 12 + ...62eacc0b76e05f37aa369286c80f633c79fe4.json | 12 + ...fe5843aba502c022d7cede17ad0875174b7b.json} | 4 +- ...764419bc906f060cb163cdd898dd328b2b578.json | 22 + ...4d88b3536859002d3af56beb2cd687ed426fe.json | 12 + ...59f108ec0acfb6e4ec124b14ccb4749ca2e73.json | 12 + ...0c1886d15cb8bbfa36efcfc4b1959a1462d2e.json | 127 + ...75446e5d2613af83a5858a85e32cdafcc12eb.json | 12 + ...3607558256bb48bcd1cc53c02867862d70ad3.json | 12 + ...787b616bdad8454a1a1b65b95e11701f4051b.json | 20 + ...def642f3d66a24ab03b30391d252e80046958.json | 12 + ...4836245b7a33cebb6719b2becd3c306e8445a.json | 12 + ...548ad54efb87d9d39fb4b89a5527918be2d2c.json | 12 + ...67965726eea767b1e4aa9e79b23c6d575f7e.json} | 4 +- ...93b1ad4fff0b64ff642afa48ce5f080aa89ec.json | 12 + ...402ca48a6ee9461a92db1316168888d5d18b5.json | 22 + ...2fde283f3e0021b4f27357f178a80ae8769aa.json | 12 + ...dd1003bcd6eec6e8eebb5bad2daf30f93287.json} | 4 +- ...c5213c39c615f5cf541fd8d4ec553cea89c9e.json | 15 + ...96ce3440471cf5f0cab26e82eab263781e33.json} | 4 +- ...3738e6ea7f42be4ea0f46e8a93c282174ded9.json | 38 - ...d400c3dd27ee74dd047fd017b3b6b1ed1bf77.json | 14 + ...575d3f1d7a056e247607126cd5b86fb2f6cdb.json | 12 + ...5864f20a4b060475af6b8455978e10db9c7bd.json | 12 + ...b992fc2689263b5cc1c3b82d254116250a358.json | 22 + ...2f0ed6a357f1487c49c9c5688a48eb1d2ed80.json | 77 + ...a3717e3e5c1b32a93416ee6c0f717d520dc5c.json | 15 + ...9e539cb6b8b3031412dc5914869254cb01efa.json | 12 + ...b770c40120655caa08b878e5207a63ba60f20.json | 12 + ...31212f02ab8b9534d3545b624a7391be54046.json | 22 - ...6be7e3e0b049662532af318eae887d7ba5661.json | 12 + .../013_orchestrator_job_enrollments.sql | 49 + .../014_library_maintenance_operations.sql | 43 + crates/ferrex-core/src/api/routes.rs | 1 + crates/ferrex-core/src/api/scan.rs | 37 + crates/ferrex-core/src/api/types/admin.rs | 60 + crates/ferrex-core/src/api/types/library.rs | 53 +- crates/ferrex-core/src/api/types/mod.rs | 8 +- .../src/domain/scan/actors/library.rs | 513 +++- .../src/domain/scan/fs_watch/mod.rs | 877 +++++-- .../domain/scan/orchestration/dispatcher.rs | 556 ++++- .../src/domain/scan/orchestration/enqueuer.rs | 46 + .../src/domain/scan/orchestration/events.rs | 13 + .../domain/scan/orchestration/maintenance.rs | 121 +- .../src/domain/scan/orchestration/mod.rs | 5 +- .../domain/scan/orchestration/persistence.rs | 1343 +++++++---- .../src/domain/scan/orchestration/queue.rs | 128 +- .../scan/orchestration/runtime/README.md | 1 + .../scan/orchestration/runtime/event_bus.rs | 41 + .../scan/orchestration/runtime/supervisor.rs | 588 ++++- .../scan/orchestration/runtime/task_graph.rs | 1075 ++++++--- .../src/domain/scan/orchestration/scan_run.rs | 390 +++- .../domain/scan/orchestration/scheduler.rs | 313 ++- .../tests/support/scanner_runtime.rs | 3 + crates/ferrex-model/src/scan.rs | 7 +- .../src/handlers/admin/dev_handlers.rs | 553 ++++- .../src/handlers/media/handle_library.rs | 524 ++++- .../src/handlers/scan/handle_scan.rs | 1 + .../src/infra/orchestration/maintenance.rs | 527 ++++- .../src/infra/orchestration/mod.rs | 77 +- .../src/infra/scan/movie_batch_notifier.rs | 9 + .../src/infra/scan/scan_manager.rs | 2053 +++++++++++++++-- crates/ferrex-server/src/routes/v1.rs | 47 +- .../tests/mobile_media_flatbuffers.rs | 8 + .../tests/scan_completion_regression.rs | 6 +- .../content/docs/operator/configuration.md | 13 +- 85 files changed, 9482 insertions(+), 1543 deletions(-) rename .sqlx/{query-43c6d42f218e18ee4ddb897e22a2b2c074bc70482c12fb15167962764f279b4e.json => query-020580dea4853a889361d12c872e44d08b1b1d6b258076b7c1e5be01fa177725.json} (59%) create mode 100644 .sqlx/query-03a14b4d4fc4606a098cc7f547dce79ddacee672c0c54fa981814a1f426da45c.json delete mode 100644 .sqlx/query-044cbd90ca9d36112461a275d2e243b186772940ee7fe717dff522578edcd627.json create mode 100644 .sqlx/query-08a72054edf54e6ae8290278fa9843e4505db60a2133193709069793595b17f5.json create mode 100644 .sqlx/query-0b9bd5a5623e75299ba08110ba547bb9e1f822a714171868779b24b0628198a7.json create mode 100644 .sqlx/query-10d5a59fb3692590a19b836fc05949fce59e54ea17278ed81e508a104cb1d202.json create mode 100644 .sqlx/query-286c0be8258d77892a7e3d8bcb61719ded381158f0f4f9b5eed62c7288a9bc19.json create mode 100644 .sqlx/query-30117d6189bea05be2b7bbd40e2e7c5ce8f129fbc4cffff0304dc78b0aebb115.json create mode 100644 .sqlx/query-32ec3fd949c758c63e014294a8e58926044c7a6df889cb88d01aa99d727de1b2.json create mode 100644 .sqlx/query-38953d2583c82e2e4028fe0e943a98efd8be663e9fdc1889241943ab4dad38de.json create mode 100644 .sqlx/query-405fecdb9accc2e8e64da77b1ff944cc09243e40e6faa55d1eb962259dc65d54.json create mode 100644 .sqlx/query-4282f759559fa6735c125d6b13c32f6a6c267387d8504cd923a22a38843505ff.json create mode 100644 .sqlx/query-44718c0fb07d9c3cb90d53a782d8d6f2542077c285de41902311c761de6ce2ab.json create mode 100644 .sqlx/query-46b3aa7b6551e0d3d1d505cd44dda0a00981de305b0e54073dc432898b8299b7.json create mode 100644 .sqlx/query-4a2402588740a7191d62ee749bcab08e7d092a559c1f478c87b5c4d35596634f.json create mode 100644 .sqlx/query-56d7eb86c33b7edaef567d014e792150c47f69949e44359f5e91d3ae1ca6cdfe.json create mode 100644 .sqlx/query-596550aa56d20e18f0a03d89b942b5a80df83664af7c2e7fa42e0ca1935f9909.json create mode 100644 .sqlx/query-5a0c7532b7d6d050f8622d9965c52af8e30e4e183857832cdd1fe43471066c31.json create mode 100644 .sqlx/query-5eca03ae857930ca4b0946c92fac9b69e2ba173c4cd34792a784025c89fc459b.json create mode 100644 .sqlx/query-63e45ddac711f6cd18d3d48382b75c31a27c59f2e9a8f6e334ef82be0d348f86.json create mode 100644 .sqlx/query-64694c91bfbe00227865a96d6fc62eacc0b76e05f37aa369286c80f633c79fe4.json rename .sqlx/{query-95db679bd28a9d5933b1016c614a13569ec7faee16e0a5a23a8cd3c7150939e0.json => query-665a4d6652b31b60137a595c0e68fe5843aba502c022d7cede17ad0875174b7b.json} (89%) create mode 100644 .sqlx/query-715016252d78119ae1bd9d96166764419bc906f060cb163cdd898dd328b2b578.json create mode 100644 .sqlx/query-73c5a1e01e63c79af33aba255154d88b3536859002d3af56beb2cd687ed426fe.json create mode 100644 .sqlx/query-8632d9e90621148e02347da3af059f108ec0acfb6e4ec124b14ccb4749ca2e73.json create mode 100644 .sqlx/query-8736f06a0ebb67d924cbb9315940c1886d15cb8bbfa36efcfc4b1959a1462d2e.json create mode 100644 .sqlx/query-8a26b8cebe4954b0359be55bc1275446e5d2613af83a5858a85e32cdafcc12eb.json create mode 100644 .sqlx/query-8d19a5d0f4bfc1e5848e3d207a63607558256bb48bcd1cc53c02867862d70ad3.json create mode 100644 .sqlx/query-902d7952e5a13dfe76d96fefee0787b616bdad8454a1a1b65b95e11701f4051b.json create mode 100644 .sqlx/query-91533cd5e162a8f08d0e86268e0def642f3d66a24ab03b30391d252e80046958.json create mode 100644 .sqlx/query-93df08dc012720ecad6d68be61c4836245b7a33cebb6719b2becd3c306e8445a.json create mode 100644 .sqlx/query-994160052da8e5589f26e940f82548ad54efb87d9d39fb4b89a5527918be2d2c.json rename .sqlx/{query-fa8f5d7e0b651edaf1d3abd5ba6514d874b11868d45ab52317fb4a9a152a8fa3.json => query-9a74012ab92ac38f70d50f5c738e67965726eea767b1e4aa9e79b23c6d575f7e.json} (82%) create mode 100644 .sqlx/query-9b38caf4bf0f6ee00d5a0426dd393b1ad4fff0b64ff642afa48ce5f080aa89ec.json create mode 100644 .sqlx/query-9dfef18700bd6cccc20d0a8346f402ca48a6ee9461a92db1316168888d5d18b5.json create mode 100644 .sqlx/query-b1a9b19bf03fd716439cc5540022fde283f3e0021b4f27357f178a80ae8769aa.json rename .sqlx/{query-19f736de007613eb96945862856b5238e2804ec689013b36da9a7d5855e7adc1.json => query-ba7c8fdb4ff7463758dd671e38b6dd1003bcd6eec6e8eebb5bad2daf30f93287.json} (61%) create mode 100644 .sqlx/query-bbb29cc7b7b85ff74ea3ded1e90c5213c39c615f5cf541fd8d4ec553cea89c9e.json rename .sqlx/{query-5cf4c625c65c1e46ea1a46fd1843f6316120cad4c4507b206f24a028b7a9efde.json => query-bdacab6d82b49ddaa6ff1e5a512896ce3440471cf5f0cab26e82eab263781e33.json} (85%) delete mode 100644 .sqlx/query-c514145d3cfc647f16aedf696a93738e6ea7f42be4ea0f46e8a93c282174ded9.json create mode 100644 .sqlx/query-c72161ee1c548c8f7da4cb942ead400c3dd27ee74dd047fd017b3b6b1ed1bf77.json create mode 100644 .sqlx/query-d41c6024a08e93bbf19b266c41a575d3f1d7a056e247607126cd5b86fb2f6cdb.json create mode 100644 .sqlx/query-ddcb9088f8c30cc37e3416ea1115864f20a4b060475af6b8455978e10db9c7bd.json create mode 100644 .sqlx/query-dedd637c1b70c020debaaea0344b992fc2689263b5cc1c3b82d254116250a358.json create mode 100644 .sqlx/query-e029d0e62d55c2357c7d6ecfc572f0ed6a357f1487c49c9c5688a48eb1d2ed80.json create mode 100644 .sqlx/query-e2b975691701fcbbc33ee1b72e5a3717e3e5c1b32a93416ee6c0f717d520dc5c.json create mode 100644 .sqlx/query-e4d26a9ceb3e549c933e8d293399e539cb6b8b3031412dc5914869254cb01efa.json create mode 100644 .sqlx/query-e7c499c0e1bd3714d2c7ae146f4b770c40120655caa08b878e5207a63ba60f20.json delete mode 100644 .sqlx/query-eea7450568fe35d45cbdc489e1431212f02ab8b9534d3545b624a7391be54046.json create mode 100644 .sqlx/query-fbf3849da2c49f9ed0517ea90496be7e3e0b049662532af318eae887d7ba5661.json create mode 100644 crates/ferrex-core/migrations/013_orchestrator_job_enrollments.sql create mode 100644 crates/ferrex-core/migrations/014_library_maintenance_operations.sql diff --git a/.sqlx/query-43c6d42f218e18ee4ddb897e22a2b2c074bc70482c12fb15167962764f279b4e.json b/.sqlx/query-020580dea4853a889361d12c872e44d08b1b1d6b258076b7c1e5be01fa177725.json similarity index 59% rename from .sqlx/query-43c6d42f218e18ee4ddb897e22a2b2c074bc70482c12fb15167962764f279b4e.json rename to .sqlx/query-020580dea4853a889361d12c872e44d08b1b1d6b258076b7c1e5be01fa177725.json index 3201e592..ebc394d8 100644 --- a/.sqlx/query-43c6d42f218e18ee4ddb897e22a2b2c074bc70482c12fb15167962764f279b4e.json +++ b/.sqlx/query-020580dea4853a889361d12c872e44d08b1b1d6b258076b7c1e5be01fa177725.json @@ -1,16 +1,16 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE library_scan_runs\n SET status = COALESCE($2::varchar, status),\n completed_items = $3,\n total_items = $4,\n retrying_items = $5,\n dead_lettered_items = $6,\n current_path = $7,\n sequence = GREATEST(sequence, $8),\n updated_at = NOW()\n WHERE scan_id = $1\n AND status::text IN ('pending','running','paused')\n RETURNING\n scan_id,\n library_id,\n mode as \"mode!\",\n run_key as \"run_key!\",\n correlation_id,\n status as \"status!\",\n completed_items as \"completed_items!\",\n total_items as \"total_items!\",\n retrying_items as \"retrying_items!\",\n dead_lettered_items as \"dead_lettered_items!\",\n current_path,\n last_error,\n sequence as \"sequence!\",\n started_at,\n terminal_at,\n created_at,\n updated_at\n ", + "query": "\n SELECT\n scan_id as \"scan_id!\",\n library_id as \"library_id!\",\n mode as \"mode!\",\n run_key as \"run_key!\",\n correlation_id,\n status as \"status!\",\n completed_items as \"completed_items!\",\n total_items as \"total_items!\",\n retrying_items as \"retrying_items!\",\n dead_lettered_items as \"dead_lettered_items!\",\n current_path,\n last_error,\n sequence as \"sequence!\",\n started_at,\n terminal_at,\n created_at,\n updated_at\n FROM library_scan_runs\n WHERE scan_id = $1\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "scan_id", + "name": "scan_id!", "type_info": "Uuid" }, { "ordinal": 1, - "name": "library_id", + "name": "library_id!", "type_info": "Uuid" }, { @@ -91,14 +91,7 @@ ], "parameters": { "Left": [ - "Uuid", - "Varchar", - "Int8", - "Int8", - "Int8", - "Int8", - "Text", - "Int8" + "Uuid" ] }, "nullable": [ @@ -121,5 +114,5 @@ false ] }, - "hash": "43c6d42f218e18ee4ddb897e22a2b2c074bc70482c12fb15167962764f279b4e" + "hash": "020580dea4853a889361d12c872e44d08b1b1d6b258076b7c1e5be01fa177725" } diff --git a/.sqlx/query-03a14b4d4fc4606a098cc7f547dce79ddacee672c0c54fa981814a1f426da45c.json b/.sqlx/query-03a14b4d4fc4606a098cc7f547dce79ddacee672c0c54fa981814a1f426da45c.json new file mode 100644 index 00000000..3ff9096c --- /dev/null +++ b/.sqlx/query-03a14b4d4fc4606a098cc7f547dce79ddacee672c0c54fa981814a1f426da45c.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM library_maintenance_operations", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "03a14b4d4fc4606a098cc7f547dce79ddacee672c0c54fa981814a1f426da45c" +} diff --git a/.sqlx/query-044cbd90ca9d36112461a275d2e243b186772940ee7fe717dff522578edcd627.json b/.sqlx/query-044cbd90ca9d36112461a275d2e243b186772940ee7fe717dff522578edcd627.json deleted file mode 100644 index 0ff6b56a..00000000 --- a/.sqlx/query-044cbd90ca9d36112461a275d2e243b186772940ee7fe717dff522578edcd627.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO orchestrator_jobs (\n id, library_id, kind, payload, priority, state,\n attempts, available_at, lease_owner, lease_id, lease_expires_at,\n dedupe_key, dependency_key, correlation_id,\n last_error, created_at, updated_at\n )\n VALUES ($1, $2, $3, $4, $5, $6, 0, NOW(), NULL, NULL, NULL, $7, $8, $9, NULL, NOW(), NOW())\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Uuid", - "Int2", - "Jsonb", - "Int2", - "Varchar", - "Text", - "Text", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "044cbd90ca9d36112461a275d2e243b186772940ee7fe717dff522578edcd627" -} diff --git a/.sqlx/query-08a72054edf54e6ae8290278fa9843e4505db60a2133193709069793595b17f5.json b/.sqlx/query-08a72054edf54e6ae8290278fa9843e4505db60a2133193709069793595b17f5.json new file mode 100644 index 00000000..9903baac --- /dev/null +++ b/.sqlx/query-08a72054edf54e6ae8290278fa9843e4505db60a2133193709069793595b17f5.json @@ -0,0 +1,44 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n kind,\n library_id,\n priority,\n COUNT(*) FILTER (\n WHERE state = 'ready' AND available_at <= NOW()\n )::bigint AS \"ready!\",\n COUNT(*) FILTER (\n WHERE state = 'leased' AND lease_expires_at > NOW()\n )::bigint AS \"leased!\"\n FROM orchestrator_jobs\n WHERE (state = 'ready' AND available_at <= NOW())\n OR (state = 'leased' AND lease_expires_at > NOW())\n GROUP BY kind, library_id, priority\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind", + "type_info": "Int2" + }, + { + "ordinal": 1, + "name": "library_id", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 3, + "name": "ready!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "leased!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + null, + null + ] + }, + "hash": "08a72054edf54e6ae8290278fa9843e4505db60a2133193709069793595b17f5" +} diff --git a/.sqlx/query-0b9bd5a5623e75299ba08110ba547bb9e1f822a714171868779b24b0628198a7.json b/.sqlx/query-0b9bd5a5623e75299ba08110ba547bb9e1f822a714171868779b24b0628198a7.json new file mode 100644 index 00000000..f1c0acc6 --- /dev/null +++ b/.sqlx/query-0b9bd5a5623e75299ba08110ba547bb9e1f822a714171868779b24b0628198a7.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE auth_sessions\n SET device_session_id = NULL\n WHERE user_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "0b9bd5a5623e75299ba08110ba547bb9e1f822a714171868779b24b0628198a7" +} diff --git a/.sqlx/query-10d5a59fb3692590a19b836fc05949fce59e54ea17278ed81e508a104cb1d202.json b/.sqlx/query-10d5a59fb3692590a19b836fc05949fce59e54ea17278ed81e508a104cb1d202.json new file mode 100644 index 00000000..8b3174a5 --- /dev/null +++ b/.sqlx/query-10d5a59fb3692590a19b836fc05949fce59e54ea17278ed81e508a104cb1d202.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH inserted_job AS (\n INSERT INTO orchestrator_jobs (\n id, library_id, kind, payload, priority, state,\n attempts, available_at, lease_owner, lease_id, lease_expires_at,\n dedupe_key, dependency_key, correlation_id, last_error,\n created_at, updated_at\n )\n VALUES ($1, $2, $3, $4, $5, $6, 0, NOW(), NULL, NULL, NULL, $7, $8, $9, NULL, NOW(), NOW())\n RETURNING id\n )\n INSERT INTO orchestrator_job_enrollments (correlation_id, job_id)\n SELECT $9, id\n FROM inserted_job\n WHERE $9 IS NOT NULL\n ON CONFLICT (correlation_id, job_id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid", + "Int2", + "Jsonb", + "Int2", + "Varchar", + "Text", + "Text", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "10d5a59fb3692590a19b836fc05949fce59e54ea17278ed81e508a104cb1d202" +} diff --git a/.sqlx/query-286c0be8258d77892a7e3d8bcb61719ded381158f0f4f9b5eed62c7288a9bc19.json b/.sqlx/query-286c0be8258d77892a7e3d8bcb61719ded381158f0f4f9b5eed62c7288a9bc19.json new file mode 100644 index 00000000..1e789f0c --- /dev/null +++ b/.sqlx/query-286c0be8258d77892a7e3d8bcb61719ded381158f0f4f9b5eed62c7288a9bc19.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM user_episode_state", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "286c0be8258d77892a7e3d8bcb61719ded381158f0f4f9b5eed62c7288a9bc19" +} diff --git a/.sqlx/query-30117d6189bea05be2b7bbd40e2e7c5ce8f129fbc4cffff0304dc78b0aebb115.json b/.sqlx/query-30117d6189bea05be2b7bbd40e2e7c5ce8f129fbc4cffff0304dc78b0aebb115.json new file mode 100644 index 00000000..4179ff98 --- /dev/null +++ b/.sqlx/query-30117d6189bea05be2b7bbd40e2e7c5ce8f129fbc4cffff0304dc78b0aebb115.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM user_view_history", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "30117d6189bea05be2b7bbd40e2e7c5ce8f129fbc4cffff0304dc78b0aebb115" +} diff --git a/.sqlx/query-32ec3fd949c758c63e014294a8e58926044c7a6df889cb88d01aa99d727de1b2.json b/.sqlx/query-32ec3fd949c758c63e014294a8e58926044c7a6df889cb88d01aa99d727de1b2.json new file mode 100644 index 00000000..2f10f21a --- /dev/null +++ b/.sqlx/query-32ec3fd949c758c63e014294a8e58926044c7a6df889cb88d01aa99d727de1b2.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT metadata as \"metadata!\" FROM library_scan_runs WHERE scan_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "metadata!", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "32ec3fd949c758c63e014294a8e58926044c7a6df889cb88d01aa99d727de1b2" +} diff --git a/.sqlx/query-38953d2583c82e2e4028fe0e943a98efd8be663e9fdc1889241943ab4dad38de.json b/.sqlx/query-38953d2583c82e2e4028fe0e943a98efd8be663e9fdc1889241943ab4dad38de.json new file mode 100644 index 00000000..53b5de6c --- /dev/null +++ b/.sqlx/query-38953d2583c82e2e4028fe0e943a98efd8be663e9fdc1889241943ab4dad38de.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM intelligence_runs", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "38953d2583c82e2e4028fe0e943a98efd8be663e9fdc1889241943ab4dad38de" +} diff --git a/.sqlx/query-405fecdb9accc2e8e64da77b1ff944cc09243e40e6faa55d1eb962259dc65d54.json b/.sqlx/query-405fecdb9accc2e8e64da77b1ff944cc09243e40e6faa55d1eb962259dc65d54.json new file mode 100644 index 00000000..7b761860 --- /dev/null +++ b/.sqlx/query-405fecdb9accc2e8e64da77b1ff944cc09243e40e6faa55d1eb962259dc65d54.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtextextended($1::uuid::text, 0))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "405fecdb9accc2e8e64da77b1ff944cc09243e40e6faa55d1eb962259dc65d54" +} diff --git a/.sqlx/query-4282f759559fa6735c125d6b13c32f6a6c267387d8504cd923a22a38843505ff.json b/.sqlx/query-4282f759559fa6735c125d6b13c32f6a6c267387d8504cd923a22a38843505ff.json new file mode 100644 index 00000000..81aa52a5 --- /dev/null +++ b/.sqlx/query-4282f759559fa6735c125d6b13c32f6a6c267387d8504cd923a22a38843505ff.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE user_permissions SET granted_by = NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "4282f759559fa6735c125d6b13c32f6a6c267387d8504cd923a22a38843505ff" +} diff --git a/.sqlx/query-44718c0fb07d9c3cb90d53a782d8d6f2542077c285de41902311c761de6ce2ab.json b/.sqlx/query-44718c0fb07d9c3cb90d53a782d8d6f2542077c285de41902311c761de6ce2ab.json new file mode 100644 index 00000000..521a7b51 --- /dev/null +++ b/.sqlx/query-44718c0fb07d9c3cb90d53a782d8d6f2542077c285de41902311c761de6ce2ab.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH mergeable_job AS MATERIALIZED (\n SELECT id\n FROM orchestrator_jobs\n WHERE id = $2\n AND state IN ('ready','deferred','leased')\n FOR UPDATE\n ), enrollment AS (\n INSERT INTO orchestrator_job_enrollments (correlation_id, job_id)\n SELECT $1::uuid, id\n FROM mergeable_job\n WHERE $1::uuid IS NOT NULL\n ON CONFLICT (correlation_id, job_id) DO NOTHING\n RETURNING job_id\n )\n SELECT\n EXISTS(SELECT 1 FROM mergeable_job) AS \"mergeable!\",\n (SELECT COUNT(*)::bigint FROM enrollment) AS \"enrollment_count!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "mergeable!", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "enrollment_count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Uuid" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "44718c0fb07d9c3cb90d53a782d8d6f2542077c285de41902311c761de6ce2ab" +} diff --git a/.sqlx/query-46b3aa7b6551e0d3d1d505cd44dda0a00981de305b0e54073dc432898b8299b7.json b/.sqlx/query-46b3aa7b6551e0d3d1d505cd44dda0a00981de305b0e54073dc432898b8299b7.json new file mode 100644 index 00000000..090d3455 --- /dev/null +++ b/.sqlx/query-46b3aa7b6551e0d3d1d505cd44dda0a00981de305b0e54073dc432898b8299b7.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM persons", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "46b3aa7b6551e0d3d1d505cd44dda0a00981de305b0e54073dc432898b8299b7" +} diff --git a/.sqlx/query-4a2402588740a7191d62ee749bcab08e7d092a559c1f478c87b5c4d35596634f.json b/.sqlx/query-4a2402588740a7191d62ee749bcab08e7d092a559c1f478c87b5c4d35596634f.json new file mode 100644 index 00000000..3d96eb1a --- /dev/null +++ b/.sqlx/query-4a2402588740a7191d62ee749bcab08e7d092a559c1f478c87b5c4d35596634f.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM sync_session_history", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "4a2402588740a7191d62ee749bcab08e7d092a559c1f478c87b5c4d35596634f" +} diff --git a/.sqlx/query-56d7eb86c33b7edaef567d014e792150c47f69949e44359f5e91d3ae1ca6cdfe.json b/.sqlx/query-56d7eb86c33b7edaef567d014e792150c47f69949e44359f5e91d3ae1ca6cdfe.json new file mode 100644 index 00000000..cf9fda4b --- /dev/null +++ b/.sqlx/query-56d7eb86c33b7edaef567d014e792150c47f69949e44359f5e91d3ae1ca6cdfe.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM intelligence_search_documents", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "56d7eb86c33b7edaef567d014e792150c47f69949e44359f5e91d3ae1ca6cdfe" +} diff --git a/.sqlx/query-596550aa56d20e18f0a03d89b942b5a80df83664af7c2e7fa42e0ca1935f9909.json b/.sqlx/query-596550aa56d20e18f0a03d89b942b5a80df83664af7c2e7fa42e0ca1935f9909.json new file mode 100644 index 00000000..b18d3f54 --- /dev/null +++ b/.sqlx/query-596550aa56d20e18f0a03d89b942b5a80df83664af7c2e7fa42e0ca1935f9909.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT (\n (SELECT COUNT(*) FROM movie_references WHERE library_id = ANY($1))\n + (SELECT COUNT(*) FROM series WHERE library_id = ANY($1))\n + (SELECT COUNT(*) FROM season_references WHERE library_id = ANY($1))\n + (\n SELECT COUNT(*)\n FROM episode_references er\n JOIN series s ON s.id = er.series_id\n WHERE s.library_id = ANY($1)\n )\n )::bigint AS \"count!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "596550aa56d20e18f0a03d89b942b5a80df83664af7c2e7fa42e0ca1935f9909" +} diff --git a/.sqlx/query-5a0c7532b7d6d050f8622d9965c52af8e30e4e183857832cdd1fe43471066c31.json b/.sqlx/query-5a0c7532b7d6d050f8622d9965c52af8e30e4e183857832cdd1fe43471066c31.json new file mode 100644 index 00000000..9c523bae --- /dev/null +++ b/.sqlx/query-5a0c7532b7d6d050f8622d9965c52af8e30e4e183857832cdd1fe43471066c31.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM tmdb_image_variants", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "5a0c7532b7d6d050f8622d9965c52af8e30e4e183857832cdd1fe43471066c31" +} diff --git a/.sqlx/query-5eca03ae857930ca4b0946c92fac9b69e2ba173c4cd34792a784025c89fc459b.json b/.sqlx/query-5eca03ae857930ca4b0946c92fac9b69e2ba173c4cd34792a784025c89fc459b.json new file mode 100644 index 00000000..25abbd99 --- /dev/null +++ b/.sqlx/query-5eca03ae857930ca4b0946c92fac9b69e2ba173c4cd34792a784025c89fc459b.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM intelligence_artifact_sources", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "5eca03ae857930ca4b0946c92fac9b69e2ba173c4cd34792a784025c89fc459b" +} diff --git a/.sqlx/query-63e45ddac711f6cd18d3d48382b75c31a27c59f2e9a8f6e334ef82be0d348f86.json b/.sqlx/query-63e45ddac711f6cd18d3d48382b75c31a27c59f2e9a8f6e334ef82be0d348f86.json new file mode 100644 index 00000000..f698f3b0 --- /dev/null +++ b/.sqlx/query-63e45ddac711f6cd18d3d48382b75c31a27c59f2e9a8f6e334ef82be0d348f86.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM user_completed_media", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "63e45ddac711f6cd18d3d48382b75c31a27c59f2e9a8f6e334ef82be0d348f86" +} diff --git a/.sqlx/query-64694c91bfbe00227865a96d6fc62eacc0b76e05f37aa369286c80f633c79fe4.json b/.sqlx/query-64694c91bfbe00227865a96d6fc62eacc0b76e05f37aa369286c80f633c79fe4.json new file mode 100644 index 00000000..d5bcbe2d --- /dev/null +++ b/.sqlx/query-64694c91bfbe00227865a96d6fc62eacc0b76e05f37aa369286c80f633c79fe4.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM cached_images", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "64694c91bfbe00227865a96d6fc62eacc0b76e05f37aa369286c80f633c79fe4" +} diff --git a/.sqlx/query-95db679bd28a9d5933b1016c614a13569ec7faee16e0a5a23a8cd3c7150939e0.json b/.sqlx/query-665a4d6652b31b60137a595c0e68fe5843aba502c022d7cede17ad0875174b7b.json similarity index 89% rename from .sqlx/query-95db679bd28a9d5933b1016c614a13569ec7faee16e0a5a23a8cd3c7150939e0.json rename to .sqlx/query-665a4d6652b31b60137a595c0e68fe5843aba502c022d7cede17ad0875174b7b.json index e8d548e3..4cd4ecb5 100644 --- a/.sqlx/query-95db679bd28a9d5933b1016c614a13569ec7faee16e0a5a23a8cd3c7150939e0.json +++ b/.sqlx/query-665a4d6652b31b60137a595c0e68fe5843aba502c022d7cede17ad0875174b7b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id, priority, available_at, state, attempts, correlation_id\n FROM orchestrator_jobs\n WHERE dedupe_key = $1\n AND state IN ('ready','deferred','leased')\n ORDER BY created_at ASC\n LIMIT 1\n ", + "query": "\n SELECT id, priority, available_at, state, attempts, correlation_id\n FROM orchestrator_jobs\n WHERE dedupe_key = $1\n AND state IN ('ready','deferred','leased')\n ORDER BY created_at ASC\n LIMIT 1\n FOR UPDATE\n ", "describe": { "columns": [ { @@ -48,5 +48,5 @@ true ] }, - "hash": "95db679bd28a9d5933b1016c614a13569ec7faee16e0a5a23a8cd3c7150939e0" + "hash": "665a4d6652b31b60137a595c0e68fe5843aba502c022d7cede17ad0875174b7b" } diff --git a/.sqlx/query-715016252d78119ae1bd9d96166764419bc906f060cb163cdd898dd328b2b578.json b/.sqlx/query-715016252d78119ae1bd9d96166764419bc906f060cb163cdd898dd328b2b578.json new file mode 100644 index 00000000..6bfdd963 --- /dev/null +++ b/.sqlx/query-715016252d78119ae1bd9d96166764419bc906f060cb163cdd898dd328b2b578.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH inserted_job AS (\n INSERT INTO orchestrator_jobs (\n id, library_id, kind, payload, priority, state,\n attempts, available_at, lease_owner, lease_id, lease_expires_at,\n dedupe_key, dependency_key, correlation_id,\n last_error, created_at, updated_at\n )\n VALUES ($1, $2, $3, $4, $5, $6, 0, NOW(), NULL, NULL, NULL, $7, $8, $9, NULL, NOW(), NOW())\n RETURNING id\n )\n INSERT INTO orchestrator_job_enrollments (correlation_id, job_id)\n SELECT $9, id\n FROM inserted_job\n WHERE $9 IS NOT NULL\n ON CONFLICT (correlation_id, job_id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid", + "Int2", + "Jsonb", + "Int2", + "Varchar", + "Text", + "Text", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "715016252d78119ae1bd9d96166764419bc906f060cb163cdd898dd328b2b578" +} diff --git a/.sqlx/query-73c5a1e01e63c79af33aba255154d88b3536859002d3af56beb2cd687ed426fe.json b/.sqlx/query-73c5a1e01e63c79af33aba255154d88b3536859002d3af56beb2cd687ed426fe.json new file mode 100644 index 00000000..242b1899 --- /dev/null +++ b/.sqlx/query-73c5a1e01e63c79af33aba255154d88b3536859002d3af56beb2cd687ed426fe.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM login_attempts", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "73c5a1e01e63c79af33aba255154d88b3536859002d3af56beb2cd687ed426fe" +} diff --git a/.sqlx/query-8632d9e90621148e02347da3af059f108ec0acfb6e4ec124b14ccb4749ca2e73.json b/.sqlx/query-8632d9e90621148e02347da3af059f108ec0acfb6e4ec124b14ccb4749ca2e73.json new file mode 100644 index 00000000..68e853f1 --- /dev/null +++ b/.sqlx/query-8632d9e90621148e02347da3af059f108ec0acfb6e4ec124b14ccb4749ca2e73.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM sync_sessions", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "8632d9e90621148e02347da3af059f108ec0acfb6e4ec124b14ccb4749ca2e73" +} diff --git a/.sqlx/query-8736f06a0ebb67d924cbb9315940c1886d15cb8bbfa36efcfc4b1959a1462d2e.json b/.sqlx/query-8736f06a0ebb67d924cbb9315940c1886d15cb8bbfa36efcfc4b1959a1462d2e.json new file mode 100644 index 00000000..96dab146 --- /dev/null +++ b/.sqlx/query-8736f06a0ebb67d924cbb9315940c1886d15cb8bbfa36efcfc4b1959a1462d2e.json @@ -0,0 +1,127 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE library_scan_runs\n SET status = COALESCE($2::varchar, status),\n completed_items = $3,\n total_items = $4,\n retrying_items = $5,\n dead_lettered_items = $6,\n current_path = $7,\n sequence = GREATEST(sequence, $8),\n metadata = jsonb_set(\n jsonb_set(\n metadata,\n '{tracked_job_ids}',\n (\n SELECT COALESCE(\n jsonb_agg(value ORDER BY value),\n '[]'::jsonb\n )\n FROM (\n SELECT DISTINCT value\n FROM jsonb_array_elements(\n COALESCE(\n metadata -> 'tracked_job_ids',\n '[]'::jsonb\n ) || $9::jsonb\n ) AS ids(value)\n ) AS unique_ids\n ),\n true\n ),\n '{seed_completed}',\n to_jsonb(\n (\n CASE\n WHEN jsonb_typeof(metadata -> 'seed_completed') = 'boolean'\n THEN (metadata ->> 'seed_completed')::boolean\n ELSE false\n END\n ) OR $10::boolean\n ),\n true\n ),\n updated_at = NOW()\n WHERE scan_id = $1\n AND status::text IN ('pending','running','paused')\n RETURNING\n scan_id as \"scan_id!\",\n library_id as \"library_id!\",\n mode as \"mode!\",\n run_key as \"run_key!\",\n correlation_id,\n status as \"status!\",\n completed_items as \"completed_items!\",\n total_items as \"total_items!\",\n retrying_items as \"retrying_items!\",\n dead_lettered_items as \"dead_lettered_items!\",\n current_path,\n last_error,\n sequence as \"sequence!\",\n started_at,\n terminal_at,\n created_at,\n updated_at\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "scan_id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "library_id!", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "mode!", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "run_key!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "correlation_id", + "type_info": "Uuid" + }, + { + "ordinal": 5, + "name": "status!", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "completed_items!", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "total_items!", + "type_info": "Int8" + }, + { + "ordinal": 8, + "name": "retrying_items!", + "type_info": "Int8" + }, + { + "ordinal": 9, + "name": "dead_lettered_items!", + "type_info": "Int8" + }, + { + "ordinal": 10, + "name": "current_path", + "type_info": "Text" + }, + { + "ordinal": 11, + "name": "last_error", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "sequence!", + "type_info": "Int8" + }, + { + "ordinal": 13, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 14, + "name": "terminal_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 15, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 16, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Int8", + "Int8", + "Int8", + "Int8", + "Text", + "Int8", + "Jsonb", + "Bool" + ] + }, + "nullable": [ + false, + false, + false, + true, + false, + false, + false, + false, + false, + false, + true, + true, + false, + false, + true, + false, + false + ] + }, + "hash": "8736f06a0ebb67d924cbb9315940c1886d15cb8bbfa36efcfc4b1959a1462d2e" +} diff --git a/.sqlx/query-8a26b8cebe4954b0359be55bc1275446e5d2613af83a5858a85e32cdafcc12eb.json b/.sqlx/query-8a26b8cebe4954b0359be55bc1275446e5d2613af83a5858a85e32cdafcc12eb.json new file mode 100644 index 00000000..6c5ef891 --- /dev/null +++ b/.sqlx/query-8a26b8cebe4954b0359be55bc1275446e5d2613af83a5858a85e32cdafcc12eb.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM intelligence_media_context", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "8a26b8cebe4954b0359be55bc1275446e5d2613af83a5858a85e32cdafcc12eb" +} diff --git a/.sqlx/query-8d19a5d0f4bfc1e5848e3d207a63607558256bb48bcd1cc53c02867862d70ad3.json b/.sqlx/query-8d19a5d0f4bfc1e5848e3d207a63607558256bb48bcd1cc53c02867862d70ad3.json new file mode 100644 index 00000000..8fabed6b --- /dev/null +++ b/.sqlx/query-8d19a5d0f4bfc1e5848e3d207a63607558256bb48bcd1cc53c02867862d70ad3.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM setup_claims", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "8d19a5d0f4bfc1e5848e3d207a63607558256bb48bcd1cc53c02867862d70ad3" +} diff --git a/.sqlx/query-902d7952e5a13dfe76d96fefee0787b616bdad8454a1a1b65b95e11701f4051b.json b/.sqlx/query-902d7952e5a13dfe76d96fefee0787b616bdad8454a1a1b65b95e11701f4051b.json new file mode 100644 index 00000000..4502f08c --- /dev/null +++ b/.sqlx/query-902d7952e5a13dfe76d96fefee0787b616bdad8454a1a1b65b95e11701f4051b.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*)::bigint AS \"count!\" FROM auth_sessions", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "902d7952e5a13dfe76d96fefee0787b616bdad8454a1a1b65b95e11701f4051b" +} diff --git a/.sqlx/query-91533cd5e162a8f08d0e86268e0def642f3d66a24ab03b30391d252e80046958.json b/.sqlx/query-91533cd5e162a8f08d0e86268e0def642f3d66a24ab03b30391d252e80046958.json new file mode 100644 index 00000000..252b2cfb --- /dev/null +++ b/.sqlx/query-91533cd5e162a8f08d0e86268e0def642f3d66a24ab03b30391d252e80046958.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM rate_limit_state", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "91533cd5e162a8f08d0e86268e0def642f3d66a24ab03b30391d252e80046958" +} diff --git a/.sqlx/query-93df08dc012720ecad6d68be61c4836245b7a33cebb6719b2becd3c306e8445a.json b/.sqlx/query-93df08dc012720ecad6d68be61c4836245b7a33cebb6719b2becd3c306e8445a.json new file mode 100644 index 00000000..eb7727e9 --- /dev/null +++ b/.sqlx/query-93df08dc012720ecad6d68be61c4836245b7a33cebb6719b2becd3c306e8445a.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM auth_events", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "93df08dc012720ecad6d68be61c4836245b7a33cebb6719b2becd3c306e8445a" +} diff --git a/.sqlx/query-994160052da8e5589f26e940f82548ad54efb87d9d39fb4b89a5527918be2d2c.json b/.sqlx/query-994160052da8e5589f26e940f82548ad54efb87d9d39fb4b89a5527918be2d2c.json new file mode 100644 index 00000000..70681cf1 --- /dev/null +++ b/.sqlx/query-994160052da8e5589f26e940f82548ad54efb87d9d39fb4b89a5527918be2d2c.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE user_roles SET granted_by = NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "994160052da8e5589f26e940f82548ad54efb87d9d39fb4b89a5527918be2d2c" +} diff --git a/.sqlx/query-fa8f5d7e0b651edaf1d3abd5ba6514d874b11868d45ab52317fb4a9a152a8fa3.json b/.sqlx/query-9a74012ab92ac38f70d50f5c738e67965726eea767b1e4aa9e79b23c6d575f7e.json similarity index 82% rename from .sqlx/query-fa8f5d7e0b651edaf1d3abd5ba6514d874b11868d45ab52317fb4a9a152a8fa3.json rename to .sqlx/query-9a74012ab92ac38f70d50f5c738e67965726eea767b1e4aa9e79b23c6d575f7e.json index c1d5240d..46bdb265 100644 --- a/.sqlx/query-fa8f5d7e0b651edaf1d3abd5ba6514d874b11868d45ab52317fb4a9a152a8fa3.json +++ b/.sqlx/query-9a74012ab92ac38f70d50f5c738e67965726eea767b1e4aa9e79b23c6d575f7e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO orchestrator_jobs (\n id, library_id, kind, payload, priority, state,\n attempts, available_at, lease_owner, lease_id, lease_expires_at,\n dedupe_key, dependency_key, correlation_id, last_error,\n created_at, updated_at\n )\n VALUES ($1, $2, $3, $4, $5, $6, 0, NOW(), NULL, NULL, NULL, $7, $8, $9, NULL, NOW(), NOW())\n ", + "query": "\n INSERT INTO orchestrator_jobs (\n id, library_id, kind, payload, priority, state,\n attempts, available_at, lease_owner, lease_id, lease_expires_at,\n dedupe_key, dependency_key, correlation_id, last_error,\n created_at, updated_at\n )\n VALUES ($1, $2, $3, $4, $5, $6, 0, NOW(), NULL, NULL, NULL, $7, $8, $9, NULL, NOW(), NOW())\n ON CONFLICT DO NOTHING\n ", "describe": { "columns": [], "parameters": { @@ -18,5 +18,5 @@ }, "nullable": [] }, - "hash": "fa8f5d7e0b651edaf1d3abd5ba6514d874b11868d45ab52317fb4a9a152a8fa3" + "hash": "9a74012ab92ac38f70d50f5c738e67965726eea767b1e4aa9e79b23c6d575f7e" } diff --git a/.sqlx/query-9b38caf4bf0f6ee00d5a0426dd393b1ad4fff0b64ff642afa48ce5f080aa89ec.json b/.sqlx/query-9b38caf4bf0f6ee00d5a0426dd393b1ad4fff0b64ff642afa48ce5f080aa89ec.json new file mode 100644 index 00000000..56afc83a --- /dev/null +++ b/.sqlx/query-9b38caf4bf0f6ee00d5a0426dd393b1ad4fff0b64ff642afa48ce5f080aa89ec.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM auth_security_settings", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "9b38caf4bf0f6ee00d5a0426dd393b1ad4fff0b64ff642afa48ce5f080aa89ec" +} diff --git a/.sqlx/query-9dfef18700bd6cccc20d0a8346f402ca48a6ee9461a92db1316168888d5d18b5.json b/.sqlx/query-9dfef18700bd6cccc20d0a8346f402ca48a6ee9461a92db1316168888d5d18b5.json new file mode 100644 index 00000000..1d5a541f --- /dev/null +++ b/.sqlx/query-9dfef18700bd6cccc20d0a8346f402ca48a6ee9461a92db1316168888d5d18b5.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH deleted AS (\n DELETE FROM libraries\n WHERE id = $1\n RETURNING\n id,\n name,\n library_type,\n paths,\n scan_interval_minutes,\n enabled,\n auto_scan,\n watch_for_changes,\n analyze_on_scan,\n max_retry_attempts,\n movie_ref_batch_size,\n created_at\n )\n INSERT INTO libraries (\n id,\n name,\n library_type,\n paths,\n scan_interval_minutes,\n enabled,\n auto_scan,\n watch_for_changes,\n analyze_on_scan,\n max_retry_attempts,\n movie_ref_batch_size,\n created_at,\n updated_at\n )\n SELECT\n id,\n name,\n library_type,\n paths,\n scan_interval_minutes,\n enabled,\n auto_scan,\n watch_for_changes,\n analyze_on_scan,\n max_retry_attempts,\n movie_ref_batch_size,\n created_at,\n NOW()\n FROM deleted\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "9dfef18700bd6cccc20d0a8346f402ca48a6ee9461a92db1316168888d5d18b5" +} diff --git a/.sqlx/query-b1a9b19bf03fd716439cc5540022fde283f3e0021b4f27357f178a80ae8769aa.json b/.sqlx/query-b1a9b19bf03fd716439cc5540022fde283f3e0021b4f27357f178a80ae8769aa.json new file mode 100644 index 00000000..194a0545 --- /dev/null +++ b/.sqlx/query-b1a9b19bf03fd716439cc5540022fde283f3e0021b4f27357f178a80ae8769aa.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM collection_sources", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "b1a9b19bf03fd716439cc5540022fde283f3e0021b4f27357f178a80ae8769aa" +} diff --git a/.sqlx/query-19f736de007613eb96945862856b5238e2804ec689013b36da9a7d5855e7adc1.json b/.sqlx/query-ba7c8fdb4ff7463758dd671e38b6dd1003bcd6eec6e8eebb5bad2daf30f93287.json similarity index 61% rename from .sqlx/query-19f736de007613eb96945862856b5238e2804ec689013b36da9a7d5855e7adc1.json rename to .sqlx/query-ba7c8fdb4ff7463758dd671e38b6dd1003bcd6eec6e8eebb5bad2daf30f93287.json index b2e31d65..f5ee75f9 100644 --- a/.sqlx/query-19f736de007613eb96945862856b5238e2804ec689013b36da9a7d5855e7adc1.json +++ b/.sqlx/query-ba7c8fdb4ff7463758dd671e38b6dd1003bcd6eec6e8eebb5bad2daf30f93287.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n WITH next AS (\n SELECT id\n FROM orchestrator_jobs\n WHERE state = 'ready'\n AND kind = $1\n AND available_at <= NOW()\n AND library_id = $2\n AND priority = $3\n ORDER BY available_at, attempts, created_at\n FOR UPDATE SKIP LOCKED\n LIMIT 1\n ), fallback AS (\n SELECT id\n FROM orchestrator_jobs\n WHERE state = 'ready'\n AND kind = $1\n AND available_at <= NOW()\n AND NOT EXISTS (SELECT 1 FROM next)\n ORDER BY available_at, attempts, created_at\n FOR UPDATE SKIP LOCKED\n LIMIT 1\n )\n SELECT j.id, j.payload, j.priority, j.attempts,\n j.available_at, j.dedupe_key, j.dependency_key,\n j.correlation_id, j.created_at\n FROM orchestrator_jobs j\n JOIN (\n SELECT id FROM next\n UNION ALL\n SELECT id FROM fallback\n LIMIT 1\n ) pick ON pick.id = j.id\n ", + "query": "\n WITH next AS (\n SELECT id\n FROM orchestrator_jobs\n WHERE state = 'ready'\n AND kind = $1\n AND available_at <= NOW()\n AND library_id = $2\n AND priority = $3\n ORDER BY available_at, attempts, created_at\n FOR UPDATE SKIP LOCKED\n LIMIT 1\n )\n SELECT j.id, j.payload, j.priority, j.attempts,\n j.available_at, j.dedupe_key, j.dependency_key,\n j.correlation_id, j.created_at\n FROM orchestrator_jobs j\n JOIN next ON next.id = j.id\n ", "describe": { "columns": [ { @@ -68,5 +68,5 @@ false ] }, - "hash": "19f736de007613eb96945862856b5238e2804ec689013b36da9a7d5855e7adc1" + "hash": "ba7c8fdb4ff7463758dd671e38b6dd1003bcd6eec6e8eebb5bad2daf30f93287" } diff --git a/.sqlx/query-bbb29cc7b7b85ff74ea3ded1e90c5213c39c615f5cf541fd8d4ec553cea89c9e.json b/.sqlx/query-bbb29cc7b7b85ff74ea3ded1e90c5213c39c615f5cf541fd8d4ec553cea89c9e.json new file mode 100644 index 00000000..b47a0ccb --- /dev/null +++ b/.sqlx/query-bbb29cc7b7b85ff74ea3ded1e90c5213c39c615f5cf541fd8d4ec553cea89c9e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO orchestrator_job_enrollments (correlation_id, job_id)\n VALUES ($1, $2)\n ON CONFLICT (correlation_id, job_id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "bbb29cc7b7b85ff74ea3ded1e90c5213c39c615f5cf541fd8d4ec553cea89c9e" +} diff --git a/.sqlx/query-5cf4c625c65c1e46ea1a46fd1843f6316120cad4c4507b206f24a028b7a9efde.json b/.sqlx/query-bdacab6d82b49ddaa6ff1e5a512896ce3440471cf5f0cab26e82eab263781e33.json similarity index 85% rename from .sqlx/query-5cf4c625c65c1e46ea1a46fd1843f6316120cad4c4507b206f24a028b7a9efde.json rename to .sqlx/query-bdacab6d82b49ddaa6ff1e5a512896ce3440471cf5f0cab26e82eab263781e33.json index 8be5edd6..1451086b 100644 --- a/.sqlx/query-5cf4c625c65c1e46ea1a46fd1843f6316120cad4c4507b206f24a028b7a9efde.json +++ b/.sqlx/query-bdacab6d82b49ddaa6ff1e5a512896ce3440471cf5f0cab26e82eab263781e33.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id, priority, state, attempts, correlation_id\n FROM orchestrator_jobs\n WHERE dedupe_key = $1\n AND state IN ('ready','deferred','leased')\n ORDER BY created_at ASC\n LIMIT 1\n ", + "query": "\n SELECT id, priority, state, attempts, correlation_id\n FROM orchestrator_jobs\n WHERE dedupe_key = $1\n AND state IN ('ready','deferred','leased')\n ORDER BY created_at ASC\n LIMIT 1\n FOR UPDATE\n ", "describe": { "columns": [ { @@ -42,5 +42,5 @@ true ] }, - "hash": "5cf4c625c65c1e46ea1a46fd1843f6316120cad4c4507b206f24a028b7a9efde" + "hash": "bdacab6d82b49ddaa6ff1e5a512896ce3440471cf5f0cab26e82eab263781e33" } diff --git a/.sqlx/query-c514145d3cfc647f16aedf696a93738e6ea7f42be4ea0f46e8a93c282174ded9.json b/.sqlx/query-c514145d3cfc647f16aedf696a93738e6ea7f42be4ea0f46e8a93c282174ded9.json deleted file mode 100644 index 94cf6f5b..00000000 --- a/.sqlx/query-c514145d3cfc647f16aedf696a93738e6ea7f42be4ea0f46e8a93c282174ded9.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT kind, library_id, priority, COUNT(*)::bigint AS ready\n FROM orchestrator_jobs\n WHERE state = 'ready'\n GROUP BY kind, library_id, priority\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "kind", - "type_info": "Int2" - }, - { - "ordinal": 1, - "name": "library_id", - "type_info": "Uuid" - }, - { - "ordinal": 2, - "name": "priority", - "type_info": "Int2" - }, - { - "ordinal": 3, - "name": "ready", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - null - ] - }, - "hash": "c514145d3cfc647f16aedf696a93738e6ea7f42be4ea0f46e8a93c282174ded9" -} diff --git a/.sqlx/query-c72161ee1c548c8f7da4cb942ead400c3dd27ee74dd047fd017b3b6b1ed1bf77.json b/.sqlx/query-c72161ee1c548c8f7da4cb942ead400c3dd27ee74dd047fd017b3b6b1ed1bf77.json new file mode 100644 index 00000000..3ffc5aee --- /dev/null +++ b/.sqlx/query-c72161ee1c548c8f7da4cb942ead400c3dd27ee74dd047fd017b3b6b1ed1bf77.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE auth_device_sessions\n SET first_authenticated_by = user_id\n WHERE user_id = $1\n AND first_authenticated_by <> user_id\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "c72161ee1c548c8f7da4cb942ead400c3dd27ee74dd047fd017b3b6b1ed1bf77" +} diff --git a/.sqlx/query-d41c6024a08e93bbf19b266c41a575d3f1d7a056e247607126cd5b86fb2f6cdb.json b/.sqlx/query-d41c6024a08e93bbf19b266c41a575d3f1d7a056e247607126cd5b86fb2f6cdb.json new file mode 100644 index 00000000..94608ff7 --- /dev/null +++ b/.sqlx/query-d41c6024a08e93bbf19b266c41a575d3f1d7a056e247607126cd5b86fb2f6cdb.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM user_watch_progress", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "d41c6024a08e93bbf19b266c41a575d3f1d7a056e247607126cd5b86fb2f6cdb" +} diff --git a/.sqlx/query-ddcb9088f8c30cc37e3416ea1115864f20a4b060475af6b8455978e10db9c7bd.json b/.sqlx/query-ddcb9088f8c30cc37e3416ea1115864f20a4b060475af6b8455978e10db9c7bd.json new file mode 100644 index 00000000..8c9165d8 --- /dev/null +++ b/.sqlx/query-ddcb9088f8c30cc37e3416ea1115864f20a4b060475af6b8455978e10db9c7bd.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM collection_shelf_placements", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "ddcb9088f8c30cc37e3416ea1115864f20a4b060475af6b8455978e10db9c7bd" +} diff --git a/.sqlx/query-dedd637c1b70c020debaaea0344b992fc2689263b5cc1c3b82d254116250a358.json b/.sqlx/query-dedd637c1b70c020debaaea0344b992fc2689263b5cc1c3b82d254116250a358.json new file mode 100644 index 00000000..3fbe2de4 --- /dev/null +++ b/.sqlx/query-dedd637c1b70c020debaaea0344b992fc2689263b5cc1c3b82d254116250a358.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT library_id\n FROM library_maintenance_operations\n WHERE operation_id = $1\n AND operation = 'reset'\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "library_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "dedd637c1b70c020debaaea0344b992fc2689263b5cc1c3b82d254116250a358" +} diff --git a/.sqlx/query-e029d0e62d55c2357c7d6ecfc572f0ed6a357f1487c49c9c5688a48eb1d2ed80.json b/.sqlx/query-e029d0e62d55c2357c7d6ecfc572f0ed6a357f1487c49c9c5688a48eb1d2ed80.json new file mode 100644 index 00000000..d39f150c --- /dev/null +++ b/.sqlx/query-e029d0e62d55c2357c7d6ecfc572f0ed6a357f1487c49c9c5688a48eb1d2ed80.json @@ -0,0 +1,77 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT job.id, job.kind, job.state, job.attempts, job.payload,\n job.dedupe_key, job.correlation_id, job.last_error,\n job.created_at, job.updated_at\n FROM orchestrator_jobs job\n WHERE job.id = ANY($2::uuid[])\n OR EXISTS (\n SELECT 1\n FROM orchestrator_job_enrollments enrollment\n WHERE enrollment.correlation_id = $1\n AND enrollment.job_id = job.id\n )\n ORDER BY job.created_at ASC, job.id ASC\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "kind", + "type_info": "Int2" + }, + { + "ordinal": 2, + "name": "state", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "attempts", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "payload", + "type_info": "Jsonb" + }, + { + "ordinal": 5, + "name": "dedupe_key", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "correlation_id", + "type_info": "Uuid" + }, + { + "ordinal": 7, + "name": "last_error", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "UuidArray" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + true, + false, + false + ] + }, + "hash": "e029d0e62d55c2357c7d6ecfc572f0ed6a357f1487c49c9c5688a48eb1d2ed80" +} diff --git a/.sqlx/query-e2b975691701fcbbc33ee1b72e5a3717e3e5c1b32a93416ee6c0f717d520dc5c.json b/.sqlx/query-e2b975691701fcbbc33ee1b72e5a3717e3e5c1b32a93416ee6c0f717d520dc5c.json new file mode 100644 index 00000000..0949a390 --- /dev/null +++ b/.sqlx/query-e2b975691701fcbbc33ee1b72e5a3717e3e5c1b32a93416ee6c0f717d520dc5c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO library_maintenance_operations (\n operation_id,\n library_id,\n operation\n )\n VALUES ($1, $2, 'reset')\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "e2b975691701fcbbc33ee1b72e5a3717e3e5c1b32a93416ee6c0f717d520dc5c" +} diff --git a/.sqlx/query-e4d26a9ceb3e549c933e8d293399e539cb6b8b3031412dc5914869254cb01efa.json b/.sqlx/query-e4d26a9ceb3e549c933e8d293399e539cb6b8b3031412dc5914869254cb01efa.json new file mode 100644 index 00000000..d9576766 --- /dev/null +++ b/.sqlx/query-e4d26a9ceb3e549c933e8d293399e539cb6b8b3031412dc5914869254cb01efa.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM collection_definitions", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "e4d26a9ceb3e549c933e8d293399e539cb6b8b3031412dc5914869254cb01efa" +} diff --git a/.sqlx/query-e7c499c0e1bd3714d2c7ae146f4b770c40120655caa08b878e5207a63ba60f20.json b/.sqlx/query-e7c499c0e1bd3714d2c7ae146f4b770c40120655caa08b878e5207a63ba60f20.json new file mode 100644 index 00000000..7e4efdc2 --- /dev/null +++ b/.sqlx/query-e7c499c0e1bd3714d2c7ae146f4b770c40120655caa08b878e5207a63ba60f20.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM security_audit_log", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "e7c499c0e1bd3714d2c7ae146f4b770c40120655caa08b878e5207a63ba60f20" +} diff --git a/.sqlx/query-eea7450568fe35d45cbdc489e1431212f02ab8b9534d3545b624a7391be54046.json b/.sqlx/query-eea7450568fe35d45cbdc489e1431212f02ab8b9534d3545b624a7391be54046.json deleted file mode 100644 index 7d5e3656..00000000 --- a/.sqlx/query-eea7450568fe35d45cbdc489e1431212f02ab8b9534d3545b624a7391be54046.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO orchestrator_jobs (\n id, library_id, kind, payload, priority, state,\n attempts, available_at, lease_owner, lease_id, lease_expires_at,\n dedupe_key, dependency_key, correlation_id, last_error,\n created_at, updated_at\n )\n VALUES ($1, $2, $3, $4, $5, $6, 0, NOW(), NULL, NULL, NULL, $7, $8, $9, NULL, NOW(), NOW())\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Uuid", - "Int2", - "Jsonb", - "Int2", - "Varchar", - "Text", - "Text", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "eea7450568fe35d45cbdc489e1431212f02ab8b9534d3545b624a7391be54046" -} diff --git a/.sqlx/query-fbf3849da2c49f9ed0517ea90496be7e3e0b049662532af318eae887d7ba5661.json b/.sqlx/query-fbf3849da2c49f9ed0517ea90496be7e3e0b049662532af318eae887d7ba5661.json new file mode 100644 index 00000000..815f24a9 --- /dev/null +++ b/.sqlx/query-fbf3849da2c49f9ed0517ea90496be7e3e0b049662532af318eae887d7ba5661.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM intelligence_artifacts", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "fbf3849da2c49f9ed0517ea90496be7e3e0b049662532af318eae887d7ba5661" +} diff --git a/crates/ferrex-core/migrations/013_orchestrator_job_enrollments.sql b/crates/ferrex-core/migrations/013_orchestrator_job_enrollments.sql new file mode 100644 index 00000000..5610c586 --- /dev/null +++ b/crates/ferrex-core/migrations/013_orchestrator_job_enrollments.sql @@ -0,0 +1,49 @@ +-- Enroll durable queue jobs into every scan run that owns their outcome. +-- +-- A job can be shared by multiple runs through dedupe merging, so correlation +-- on the job row itself is not sufficient ownership state. This many-to-many +-- table makes accepted and merged enqueue outcomes independently durable. + +CREATE SCHEMA IF NOT EXISTS ferrex; + +DO $$ +DECLARE + app_schema text; +BEGIN + SELECT n.nspname + INTO app_schema + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relname = 'orchestrator_jobs' + AND c.relkind IN ('r', 'p') + AND n.nspname IN ('ferrex', 'public') + ORDER BY CASE WHEN n.nspname = 'ferrex' THEN 0 ELSE 1 END + LIMIT 1; + + IF app_schema IS NULL THEN + app_schema := 'ferrex'; + END IF; + + PERFORM set_config('search_path', format('%I, public', app_schema), false); +END $$; + +CREATE TABLE IF NOT EXISTS orchestrator_job_enrollments ( + correlation_id uuid NOT NULL, + job_id uuid NOT NULL + REFERENCES orchestrator_jobs(id) + ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (correlation_id, job_id) +); + +CREATE INDEX IF NOT EXISTS idx_job_enrollments_job_id + ON orchestrator_job_enrollments USING btree (job_id); + +INSERT INTO orchestrator_job_enrollments (correlation_id, job_id) +SELECT correlation_id, id +FROM orchestrator_jobs +WHERE correlation_id IS NOT NULL +ON CONFLICT (correlation_id, job_id) DO NOTHING; + +COMMENT ON TABLE orchestrator_job_enrollments IS + 'Authoritative many-to-many ownership between scan correlations and accepted or merged orchestrator jobs.'; diff --git a/crates/ferrex-core/migrations/014_library_maintenance_operations.sql b/crates/ferrex-core/migrations/014_library_maintenance_operations.sql new file mode 100644 index 00000000..c51f9bec --- /dev/null +++ b/crates/ferrex-core/migrations/014_library_maintenance_operations.sql @@ -0,0 +1,43 @@ +-- Durable idempotency keys for destructive library maintenance commands. +-- +-- Reset preserves the library row's identity by deleting and recreating it in +-- one transaction. This independent operation record survives that delete so +-- a lost HTTP response can be replayed without clearing the library twice. + +CREATE SCHEMA IF NOT EXISTS ferrex; + +DO $$ +DECLARE + app_schema text; +BEGIN + SELECT n.nspname + INTO app_schema + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relname = 'libraries' + AND c.relkind IN ('r', 'p') + AND n.nspname IN ('ferrex', 'public') + ORDER BY CASE WHEN n.nspname = 'ferrex' THEN 0 ELSE 1 END + LIMIT 1; + + IF app_schema IS NULL THEN + app_schema := 'ferrex'; + END IF; + + PERFORM set_config('search_path', format('%I, public', app_schema), false); +END $$; + +CREATE TABLE IF NOT EXISTS library_maintenance_operations ( + operation_id uuid PRIMARY KEY, + library_id uuid NOT NULL, + operation varchar(32) NOT NULL, + completed_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT library_maintenance_operations_kind_check + CHECK (operation IN ('reset')) +); + +CREATE INDEX IF NOT EXISTS idx_library_maintenance_operations_library + ON library_maintenance_operations (library_id, completed_at DESC); + +COMMENT ON TABLE library_maintenance_operations IS + 'Completed idempotent destructive library commands. Intentionally has no library FK so reset records survive the delete/recreate transaction.'; diff --git a/crates/ferrex-core/src/api/routes.rs b/crates/ferrex-core/src/api/routes.rs index 3369fc6f..73b42fae 100644 --- a/crates/ferrex-core/src/api/routes.rs +++ b/crates/ferrex-core/src/api/routes.rs @@ -168,6 +168,7 @@ pub mod v1 { pub mod libraries { pub const COLLECTION: &str = v1_path!("/libraries"); pub const ITEM: &str = v1_path!("/libraries/{id}"); + pub const RESET: &str = v1_path!("/libraries/{id}/reset"); pub const DISCOVERY: &str = v1_path!("/libraries/{id}/discovery"); pub const MEDIA: &str = v1_path!("/libraries/{id}/media"); pub const SORTED_IDS: &str = v1_path!("/libraries/{id}/sorted-ids"); diff --git a/crates/ferrex-core/src/api/scan.rs b/crates/ferrex-core/src/api/scan.rs index e7c38082..e1e60aba 100644 --- a/crates/ferrex-core/src/api/scan.rs +++ b/crates/ferrex-core/src/api/scan.rs @@ -119,6 +119,14 @@ pub struct IncrementalScanStatusView { pub stale_cursor_libraries: u64, pub stale_cursors: u64, pub oldest_cursor_staleness_ms: Option, + /// Number of maintenance planning passes that deferred eligible root + /// folders because the per-library discovery bound was reached. + #[serde(default)] + pub root_discovery_truncated_sweeps: u64, + /// Total eligible root folders deferred by bounded maintenance planning + /// since this server process started. + #[serde(default)] + pub root_discovery_deferred_entries: u64, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -229,6 +237,8 @@ pub struct MaintenanceConfigView { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct LeaseConfigView { pub lease_ttl_secs: i64, + #[serde(default)] + pub dispatch_timeout_ms: u64, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -254,3 +264,30 @@ pub struct BudgetConfigView { #[serde(default)] pub transcript_extraction_limit: usize, } + +#[cfg(test)] +mod tests { + use super::IncrementalScanStatusView; + + #[test] + fn incremental_scan_metrics_expose_root_discovery_truncation() { + let status = IncrementalScanStatusView { + root_discovery_truncated_sweeps: 3, + root_discovery_deferred_entries: 769, + ..IncrementalScanStatusView::default() + }; + + let encoded = serde_json::to_value(&status).expect("serialize metrics"); + assert_eq!(encoded["root_discovery_truncated_sweeps"], 3); + assert_eq!(encoded["root_discovery_deferred_entries"], 769); + + let mut legacy = encoded; + let object = legacy.as_object_mut().expect("metrics object"); + object.remove("root_discovery_truncated_sweeps"); + object.remove("root_discovery_deferred_entries"); + let decoded: IncrementalScanStatusView = + serde_json::from_value(legacy).expect("deserialize legacy metrics"); + assert_eq!(decoded.root_discovery_truncated_sweeps, 0); + assert_eq!(decoded.root_discovery_deferred_entries, 0); + } +} diff --git a/crates/ferrex-core/src/api/types/admin.rs b/crates/ferrex-core/src/api/types/admin.rs index 2952c627..7b624221 100644 --- a/crates/ferrex-core/src/api/types/admin.rs +++ b/crates/ferrex-core/src/api/types/admin.rs @@ -1,5 +1,65 @@ use serde::{Deserialize, Serialize}; +/// Authenticated administrative database-reset request. +/// +/// The confirmation value intentionally travels with every request so callers +/// cannot trigger destructive maintenance through an empty POST. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResetDatabaseRequest { + /// Reset all users and authentication data. + pub reset_users: bool, + /// Reset all libraries and their library-owned data. + pub reset_libraries: bool, + /// Reset media entries. Library deletion currently owns the media cascade. + pub reset_media: bool, + /// Destructive-operation confirmation phrase. + pub confirmation: String, +} + +impl ResetDatabaseRequest { + /// Build the full wipe requested by the player dashboard's Clear All Data action. + pub fn clear_all_data() -> Self { + Self { + reset_users: true, + reset_libraries: true, + reset_media: true, + confirmation: "RESET_DATABASE".to_string(), + } + } +} + +/// Counts returned after an administrative database reset. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResetDatabaseResult { + /// Number of users deleted. + pub users_deleted: usize, + /// Number of sessions deleted. + pub sessions_deleted: usize, + /// Number of roles restored to their default definitions. + pub roles_reset: usize, + /// Number of libraries deleted. + pub libraries_deleted: usize, + /// Number of media items deleted directly. + pub media_deleted: usize, + /// Number of watch-status entries deleted directly. + pub watch_status_deleted: usize, +} + +#[cfg(test)] +mod reset_database_tests { + use super::ResetDatabaseRequest; + + #[test] + fn clear_all_data_requests_every_destructive_reset() { + let request = ResetDatabaseRequest::clear_all_data(); + + assert!(request.reset_users); + assert!(request.reset_libraries); + assert!(request.reset_media); + assert_eq!(request.confirmation, "RESET_DATABASE"); + } +} + /// Request parameters accepted by the media root browser endpoint. /// /// `path` is expected to be a relative POSIX-style path anchored at the server's diff --git a/crates/ferrex-core/src/api/types/library.rs b/crates/ferrex-core/src/api/types/library.rs index 5c5de03f..39ed3106 100644 --- a/crates/ferrex-core/src/api/types/library.rs +++ b/crates/ferrex-core/src/api/types/library.rs @@ -16,6 +16,8 @@ use crate::types::media::{ }; use crate::types::media_id::MediaID; +use super::scan::ScanCommandAcceptedResponse; + /// Lightweight payload of library media used by UI clients #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "rkyv", derive(Archive, RkyvSerialize, RkyvDeserialize))] @@ -173,6 +175,10 @@ pub struct CreateLibraryRequest { pub auto_scan: bool, #[serde(default = "default_enabled")] pub watch_for_changes: bool, + #[serde(default)] + pub analyze_on_scan: bool, + #[serde(default = "default_max_retry_attempts")] + pub max_retry_attempts: u32, #[serde(default = "default_movie_ref_batch_size")] pub movie_ref_batch_size: u32, #[serde(default = "default_start_scan")] @@ -188,9 +194,33 @@ pub struct UpdateLibraryRequest { pub enabled: Option, pub auto_scan: Option, pub watch_for_changes: Option, + pub analyze_on_scan: Option, + pub max_retry_attempts: Option, pub movie_ref_batch_size: Option, } +/// Idempotent request to clear a library's owned data while preserving its +/// identity and configuration, then start a fresh scan. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResetLibraryRequest { + pub operation_id: Uuid, +} + +impl Default for ResetLibraryRequest { + fn default() -> Self { + Self { + operation_id: Uuid::now_v7(), + } + } +} + +/// Result of a server-side library reset. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResetLibraryResult { + pub library_id: LibraryId, + pub scan: Option, +} + fn default_scan_interval() -> u32 { 60 } @@ -207,6 +237,10 @@ fn default_movie_ref_batch_size() -> u32 { 250 } +fn default_max_retry_attempts() -> u32 { + 3 +} + #[cfg(test)] mod tests { use super::*; @@ -225,6 +259,8 @@ mod tests { assert!(request.auto_scan); assert!(request.watch_for_changes); assert_eq!(request.scan_interval_minutes, 60); + assert!(!request.analyze_on_scan); + assert_eq!(request.max_retry_attempts, 3); } #[test] @@ -236,7 +272,9 @@ mod tests { "paths": ["/mnt/nas/movies"], "auto_scan": false, "watch_for_changes": false, - "scan_interval_minutes": 240 + "scan_interval_minutes": 240, + "analyze_on_scan": true, + "max_retry_attempts": 7 }"#, ) .expect("create request should deserialize"); @@ -244,6 +282,8 @@ mod tests { assert!(!request.auto_scan); assert!(!request.watch_for_changes); assert_eq!(request.scan_interval_minutes, 240); + assert!(request.analyze_on_scan); + assert_eq!(request.max_retry_attempts, 7); } #[test] @@ -251,12 +291,21 @@ mod tests { let request: UpdateLibraryRequest = serde_json::from_str( r#"{ "auto_scan": false, - "watch_for_changes": false + "watch_for_changes": false, + "analyze_on_scan": true, + "max_retry_attempts": 5 }"#, ) .expect("update request should deserialize"); assert_eq!(request.auto_scan, Some(false)); assert_eq!(request.watch_for_changes, Some(false)); + assert_eq!(request.analyze_on_scan, Some(true)); + assert_eq!(request.max_retry_attempts, Some(5)); + } + + #[test] + fn reset_library_request_generates_an_operation_id() { + assert_ne!(ResetLibraryRequest::default().operation_id, Uuid::nil()); } } diff --git a/crates/ferrex-core/src/api/types/mod.rs b/crates/ferrex-core/src/api/types/mod.rs index e2497d43..9ac9a187 100644 --- a/crates/ferrex-core/src/api/types/mod.rs +++ b/crates/ferrex-core/src/api/types/mod.rs @@ -45,8 +45,8 @@ pub use library::{ FetchMediaRequest, LibraryMediaCache, LibraryMediaResponse, ManualMatchRequest, MovieReferenceBatchBlob, MovieReferenceBatchBundleResponse, MovieReferenceBatchResponse, - SeriesBundleBlob, SeriesBundleBundleResponse, SeriesBundleResponse, - UpdateLibraryRequest, + ResetLibraryRequest, ResetLibraryResult, SeriesBundleBlob, + SeriesBundleBundleResponse, SeriesBundleResponse, UpdateLibraryRequest, }; pub use media::{ ImageData, ImageManifestItem, ImageManifestRequest, ImageManifestResponse, @@ -92,8 +92,8 @@ pub mod player { FetchMediaRequest, LibraryMediaCache, LibraryMediaResponse, ManualMatchRequest, MovieReferenceBatchBlob, MovieReferenceBatchBundleResponse, MovieReferenceBatchResponse, - SeriesBundleBlob, SeriesBundleBundleResponse, SeriesBundleResponse, - UpdateLibraryRequest, + ResetLibraryRequest, ResetLibraryResult, SeriesBundleBlob, + SeriesBundleBundleResponse, SeriesBundleResponse, UpdateLibraryRequest, }; pub use super::media::{ ImageData, ImageManifestItem, ImageManifestRequest, diff --git a/crates/ferrex-core/src/domain/scan/actors/library.rs b/crates/ferrex-core/src/domain/scan/actors/library.rs index a0db37cd..0edcf7e5 100644 --- a/crates/ferrex-core/src/domain/scan/actors/library.rs +++ b/crates/ferrex-core/src/domain/scan/actors/library.rs @@ -23,7 +23,7 @@ use crate::domain::scan::orchestration::{ events::JobEventPublisher, job::{ DedupeKey, EnqueueRequest, JobHandle, JobId, JobPayload, JobPriority, - MetadataEnrichJob, ScanReason, + MetadataEnrichJob, }, queue::QueueService, scan_cursor::normalize_path, @@ -321,6 +321,16 @@ where } } + fn release_tracked_job(&mut self, dedupe_key: &DedupeKey) { + let _ = self.state.release_job(dedupe_key); + if let DedupeKey::FolderScan { + candidate: MediaCandidate { path_norm, .. }, + } = dedupe_key + { + self.state.mark_scan_inactive(path_norm); + } + } + async fn enqueue_folder_scan( &mut self, request: EnqueueRequest, @@ -352,20 +362,6 @@ where return Ok(vec![LibraryActorEvent::JobThrottled { dedupe_key }]); } - // For bulk seeding we bypass the in-memory outstanding throttle so we can - // enqueue all folders into the persistent queue up-front. The scheduler - // will regulate actual execution concurrency. - if !matches!(reason, ScanReason::BulkSeed) { - let outstanding_limit_reached = self.state.outstanding_jobs.len() - >= self.config.max_outstanding_jobs - && !self.state.outstanding_jobs.contains_key(&dedupe_key); - if outstanding_limit_reached { - return Ok(vec![LibraryActorEvent::JobThrottled { - dedupe_key, - }]); - } - } - // Record outstanding and mark active; orchestrator will enqueue from the returned event. self.state.record_job(IssuedJobRecord { dedupe_key: dedupe_key.clone(), @@ -452,9 +448,6 @@ where events: Vec, correlation_id: Option, ) -> Result> { - if self.state.is_bulk_scanning { - return Ok(vec![]); - } let mut responses = Vec::new(); let Some(root_path) = self.config.root_path(root) else { @@ -480,7 +473,7 @@ where }) .collect(); - let plan = plan_fs_event_burst(FsEventPlanningInput { + let mut plan = plan_fs_event_burst(FsEventPlanningInput { library: &self.config.library, root: planning_root, events: planning_events, @@ -492,6 +485,38 @@ where }) .await?; + // Overflow planning and ordinary change planning can identify the same + // folder in one burst. Keep admission atomic and let the durable queue + // dedupe across separate commands. + let mut planned_folders = HashSet::new(); + plan.requests.retain(|request| match &request.payload { + JobPayload::FolderScan(job) => planned_folders + .insert(job.context.folder_path_norm().to_string()), + _ => true, + }); + + // A watcher batch is acknowledged as a unit. Check every target before + // recording any of them so an active folder makes the whole batch + // retryable instead of silently dropping only the conflicting path. + let active_folders: Vec<_> = plan + .requests + .iter() + .filter_map(|request| match &request.payload { + JobPayload::FolderScan(job) => { + let folder = job.context.folder_path_norm(); + self.state.is_scan_active(folder).then(|| folder.to_owned()) + } + _ => None, + }) + .collect(); + if !active_folders.is_empty() { + return Err(crate::error::MediaError::ConcurrencyLimit(format!( + "filesystem event targets already being scanned for library {}: {}", + self.config.library.id, + active_folders.join(", ") + ))); + } + if plan.dropped_events > 0 { warn!( target: "scan::events", @@ -542,8 +567,55 @@ where ) -> Result> { if self.state.is_paused { match command { + LibraryActorCommand::Start { + mode: StartMode::Maintenance, + correlation_id, + } => { + if self.state.is_bulk_scanning + && self.state.current_correlation != correlation_id + { + warn!( + library_id = %self.config.library.id, + current_correlation = ?self.state.current_correlation, + maintenance_correlation = ?correlation_id, + "ignoring stale maintenance transition for newer bulk scan" + ); + return Ok(vec![]); + } + // Scan finalization uses Maintenance as the actor's durable + // reset transition. It must work while paused so a + // pause/cancel sequence cannot leave stale active folders + // suppressing every later bulk seed. + self.state.is_paused = false; + self.state.is_bulk_scanning = false; + self.state.current_correlation = correlation_id; + self.state.outstanding_jobs.clear(); + self.state.active_folder_scans.clear(); + self.state.roots.clear(); + for root in self.config.roots() { + self.state.roots.insert( + root.root_id, + LibraryRootState { + last_scan_at: None, + is_watching: true, + }, + ); + } + Ok(vec![]) + } LibraryActorCommand::Resume => { self.state.is_paused = false; + for root_state in self.state.roots.values_mut() { + root_state.is_watching = true; + } + Ok(vec![]) + } + LibraryActorCommand::JobCompleted { dedupe_key, .. } + | LibraryActorCommand::JobFailed { dedupe_key, .. } => { + // Pausing admission does not pause already leased workers. + // Consume their terminal notifications so Resume cannot + // retain phantom active-folder conflicts. + self.release_tracked_job(&dedupe_key); Ok(vec![]) } LibraryActorCommand::Shutdown => { @@ -552,74 +624,105 @@ where self.state.current_correlation = None; Ok(vec![]) } - _ => Ok(vec![]), // Ignore other commands when paused + LibraryActorCommand::FsEvents { .. } => { + Err(crate::error::MediaError::ConcurrencyLimit(format!( + "filesystem event admission paused for library {}", + self.config.library.id + ))) + } + LibraryActorCommand::Start { + mode: StartMode::Bulk | StartMode::Resume, + .. + } => Err(crate::error::MediaError::ConcurrencyLimit(format!( + "scan admission paused for library {}", + self.config.library.id + ))), + LibraryActorCommand::Pause => Ok(vec![]), } } else { match command { LibraryActorCommand::Start { mode, correlation_id, - } => { - self.state.current_correlation = correlation_id; - match mode { - StartMode::Bulk => { - self.state.is_bulk_scanning = true; - // Initialize root states and seed bulk folders - for root in self.config.roots() { - self.state.roots.insert( - root.root_id, - LibraryRootState { - last_scan_at: None, - is_watching: true, - }, - ); - } - self.seed_bulk_folders(mode, correlation_id).await + } => match mode { + StartMode::Bulk => { + self.state.current_correlation = correlation_id; + self.state.is_bulk_scanning = true; + // Initialize root states and seed bulk folders + for root in self.config.roots() { + self.state.roots.insert( + root.root_id, + LibraryRootState { + last_scan_at: None, + is_watching: true, + }, + ); } - StartMode::Maintenance | StartMode::Resume => { - self.state.is_bulk_scanning = false; - // Initialize roots for watching only - for root in self.config.roots() { - self.state.roots.insert( - root.root_id, - LibraryRootState { - last_scan_at: None, - is_watching: true, - }, - ); - } - Ok(vec![]) + self.seed_bulk_folders(mode, correlation_id).await + } + StartMode::Maintenance => { + if self.state.is_bulk_scanning + && self.state.current_correlation != correlation_id + { + warn!( + library_id = %self.config.library.id, + current_correlation = ?self.state.current_correlation, + maintenance_correlation = ?correlation_id, + "ignoring stale maintenance transition for newer bulk scan" + ); + return Ok(vec![]); } + + let reset_bulk_state = self.state.is_bulk_scanning; + self.state.current_correlation = correlation_id; + self.state.is_bulk_scanning = false; + if reset_bulk_state { + self.state.outstanding_jobs.clear(); + self.state.active_folder_scans.clear(); + } + // Initialize roots for watching only. + for root in self.config.roots() { + self.state.roots.insert( + root.root_id, + LibraryRootState { + last_scan_at: None, + is_watching: true, + }, + ); + } + Ok(vec![]) } - } + StartMode::Resume => { + self.state.current_correlation = correlation_id; + self.state.is_bulk_scanning = false; + // Initialize roots for watching only. + for root in self.config.roots() { + self.state.roots.insert( + root.root_id, + LibraryRootState { + last_scan_at: None, + is_watching: true, + }, + ); + } + Ok(vec![]) + } + }, LibraryActorCommand::FsEvents { root, events, correlation_id, } => self.handle_fs_events(root, events, correlation_id).await, LibraryActorCommand::JobCompleted { dedupe_key, .. } => { - let _ = self.state.release_job(&dedupe_key); - if let DedupeKey::FolderScan { - candidate: MediaCandidate { path_norm, .. }, - } = &dedupe_key - { - self.state.mark_scan_inactive(path_norm); - } + self.release_tracked_job(&dedupe_key); Ok(vec![]) } LibraryActorCommand::JobFailed { dedupe_key, .. } => { - let _ = self.state.release_job(&dedupe_key); - if let DedupeKey::FolderScan { - candidate: MediaCandidate { path_norm, .. }, - } = &dedupe_key - { - self.state.mark_scan_inactive(path_norm); - } + self.release_tracked_job(&dedupe_key); Ok(vec![]) } LibraryActorCommand::Pause => { self.state.is_paused = true; - self.state.current_correlation = None; for root_state in self.state.roots.values_mut() { root_state.is_watching = false; } @@ -916,7 +1019,8 @@ mod tests { } #[tokio::test] - async fn fs_watch_events_are_ignored_during_bulk_scan() -> Result<()> { + async fn fs_watch_events_added_during_bulk_scan_are_enqueued() -> Result<()> + { let temp = tempfile::tempdir().unwrap(); let root = temp.path().to_path_buf(); let queue = Arc::new(RecordingQueue::default()); @@ -953,23 +1057,17 @@ mod tests { }) .await?; - // While a bulk scan is underway watcher bursts are ignored—the bulk seed - // has already enqueued every folder, so any queued work here would be - // redundant and could reintroduce the incomplete-state bug these guards - // were meant to avoid. - assert!( - responses.iter().all(|event| !matches!( - event, - LibraryActorEvent::EnqueueFolderScan { .. } - )), - "fs events during bulk should not enqueue additional scans" + assert_eq!( + enqueued_folder_paths(&responses), + vec![normalize_path(&folder)?], + "folders created after bulk enumeration must still be scanned" ); Ok(()) } #[tokio::test] - async fn bulk_seed_and_watcher_burst_share_one_active_folder_scan() + async fn watcher_burst_for_active_folder_is_retryable_without_state_change() -> Result<()> { let temp = tempfile::tempdir().unwrap(); let root = temp.path().to_path_buf(); @@ -1026,21 +1124,74 @@ mod tests { )?, ]; - let responses = actor + let before_outstanding: HashSet<_> = + actor.state.outstanding_jobs.keys().cloned().collect(); + let before_active = actor.state.active_folder_scans.clone(); + let error = actor .handle_command(LibraryActorCommand::FsEvents { root: LibraryRootsId(0), events: burst, correlation_id: Some(scan_id), }) - .await?; + .await + .expect_err( + "an active folder must make the watcher burst retryable", + ); assert!( - enqueued_folder_paths(&responses).is_empty(), - "watcher bursts during bulk must not enqueue duplicate folder scans" + matches!(error, MediaError::ConcurrencyLimit(_)), + "active folder conflicts should use the retryable concurrency error" ); - assert_eq!(actor.state.outstanding_jobs.len(), 1); - assert_eq!(actor.state.active_folder_scans.len(), 1); - assert!(actor.state.is_scan_active(&seeded_folder_norm)); + assert_eq!( + actor + .state + .outstanding_jobs + .keys() + .cloned() + .collect::>(), + before_outstanding + ); + assert_eq!(actor.state.active_folder_scans, before_active); + + Ok(()) + } + + #[tokio::test] + async fn watcher_overflow_bypasses_actor_outstanding_limit() -> Result<()> { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().to_path_buf(); + let folder_count = 40usize; + for index in 0..folder_count { + std::fs::create_dir_all(root.join(format!("movie-{index:03}"))) + .unwrap(); + } + + let queue = Arc::new(RecordingQueue::default()); + let publisher = Arc::new(RecordingPublisher::default()); + let mut actor = make_actor( + Arc::clone(&queue), + root.clone(), + Arc::clone(&publisher), + ); + assert!(actor.config.max_outstanding_jobs < folder_count); + let library_id = actor.config.library.id; + + let responses = actor + .handle_command(LibraryActorCommand::FsEvents { + root: LibraryRootsId(0), + events: vec![make_event( + &root, + FileSystemEventKind::Overflow, + library_id, + None, + )?], + correlation_id: None, + }) + .await?; + + assert_eq!(enqueued_folder_paths(&responses).len(), folder_count); + assert_eq!(actor.state.outstanding_jobs.len(), folder_count); + assert_eq!(actor.state.active_folder_scans.len(), folder_count); Ok(()) } @@ -1092,6 +1243,184 @@ mod tests { Ok(()) } + #[tokio::test] + async fn maintenance_resets_paused_actor_for_subsequent_bulk_scan() + -> Result<()> { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().to_path_buf(); + let seeded_folder = root.join("cancelled-movie"); + std::fs::create_dir_all(&seeded_folder).unwrap(); + let seeded_folder_norm = normalize_path(&seeded_folder)?; + + let queue = Arc::new(RecordingQueue::default()); + let publisher = Arc::new(RecordingPublisher::default()); + let mut actor = + make_actor(Arc::clone(&queue), root, Arc::clone(&publisher)); + let bulk_correlation = Uuid::now_v7(); + + let first_bulk = actor + .handle_command(LibraryActorCommand::Start { + mode: StartMode::Bulk, + correlation_id: Some(bulk_correlation), + }) + .await?; + assert_eq!( + enqueued_folder_paths(&first_bulk), + vec![seeded_folder_norm.clone()] + ); + + actor.handle_command(LibraryActorCommand::Pause).await?; + assert!(actor.state.is_paused); + assert!(actor.state.is_scan_active(&seeded_folder_norm)); + + actor + .handle_command(LibraryActorCommand::Start { + mode: StartMode::Maintenance, + correlation_id: Some(bulk_correlation), + }) + .await?; + assert!(!actor.state.is_paused); + assert!(!actor.state.is_bulk_scanning); + assert!(actor.state.outstanding_jobs.is_empty()); + assert!(actor.state.active_folder_scans.is_empty()); + assert!(actor.state.roots.values().all(|root| root.is_watching)); + + let second_bulk = actor + .handle_command(LibraryActorCommand::Start { + mode: StartMode::Bulk, + correlation_id: Some(Uuid::now_v7()), + }) + .await?; + assert_eq!( + enqueued_folder_paths(&second_bulk), + vec![seeded_folder_norm], + "bulk admission must recover after paused scan finalization" + ); + + Ok(()) + } + + #[tokio::test] + async fn paused_actor_consumes_terminal_jobs_before_resume() -> Result<()> { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().to_path_buf(); + let seeded_folder = root.join("paused-worker-movie"); + std::fs::create_dir_all(&seeded_folder).unwrap(); + let media_file = seeded_folder.join("feature.mkv"); + std::fs::write(&media_file, b"fixture").unwrap(); + let seeded_folder_norm = normalize_path(&seeded_folder)?; + + let queue = Arc::new(RecordingQueue::default()); + let publisher = Arc::new(RecordingPublisher::default()); + let mut actor = + make_actor(Arc::clone(&queue), root, Arc::clone(&publisher)); + let library_id = actor.config.library.id; + + let _ = actor + .handle_command(LibraryActorCommand::Start { + mode: StartMode::Bulk, + correlation_id: Some(Uuid::now_v7()), + }) + .await?; + actor.handle_command(LibraryActorCommand::Pause).await?; + actor + .handle_command(LibraryActorCommand::JobCompleted { + job_id: JobId::new(), + dedupe_key: DedupeKey::FolderScan { + candidate: MediaCandidate::new( + library_id, + seeded_folder_norm.clone(), + ), + }, + }) + .await?; + + assert!(actor.state.is_paused); + assert!(actor.state.outstanding_jobs.is_empty()); + assert!(actor.state.active_folder_scans.is_empty()); + + actor.handle_command(LibraryActorCommand::Resume).await?; + let watcher_events = actor + .handle_command(LibraryActorCommand::FsEvents { + root: LibraryRootsId(0), + events: vec![make_event( + &media_file, + FileSystemEventKind::Modified, + library_id, + None, + )?], + correlation_id: None, + }) + .await?; + assert_eq!( + enqueued_folder_paths(&watcher_events), + vec![seeded_folder_norm] + ); + + Ok(()) + } + + #[tokio::test] + async fn stale_maintenance_does_not_clobber_newer_bulk_scan() -> Result<()> + { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().to_path_buf(); + let seeded_folder = root.join("newer-bulk-movie"); + std::fs::create_dir_all(&seeded_folder).unwrap(); + let seeded_folder_norm = normalize_path(&seeded_folder)?; + + let queue = Arc::new(RecordingQueue::default()); + let publisher = Arc::new(RecordingPublisher::default()); + let mut actor = + make_actor(Arc::clone(&queue), root, Arc::clone(&publisher)); + let library_id = actor.config.library.id; + let old_bulk = Uuid::now_v7(); + let new_bulk = Uuid::now_v7(); + + let _ = actor + .handle_command(LibraryActorCommand::Start { + mode: StartMode::Bulk, + correlation_id: Some(old_bulk), + }) + .await?; + actor + .handle_command(LibraryActorCommand::JobCompleted { + job_id: JobId::new(), + dedupe_key: DedupeKey::FolderScan { + candidate: MediaCandidate::new( + library_id, + seeded_folder_norm.clone(), + ), + }, + }) + .await?; + + let newer_events = actor + .handle_command(LibraryActorCommand::Start { + mode: StartMode::Bulk, + correlation_id: Some(new_bulk), + }) + .await?; + assert_eq!( + enqueued_folder_paths(&newer_events), + vec![seeded_folder_norm.clone()] + ); + + actor + .handle_command(LibraryActorCommand::Start { + mode: StartMode::Maintenance, + correlation_id: Some(old_bulk), + }) + .await?; + + assert!(actor.state.is_bulk_scanning); + assert_eq!(actor.state.current_correlation, Some(new_bulk)); + assert!(actor.state.is_scan_active(&seeded_folder_norm)); + assert_eq!(actor.state.outstanding_jobs.len(), 1); + + Ok(()) + } + #[tokio::test] async fn post_bulk_maintenance_start_does_not_duplicate_folder_jobs() -> Result<()> { @@ -1111,11 +1440,12 @@ mod tests { Arc::clone(&publisher), ); let library_id = actor.config.library.id; + let bulk_correlation = Uuid::now_v7(); let start_events = actor .handle_command(LibraryActorCommand::Start { mode: StartMode::Bulk, - correlation_id: Some(Uuid::now_v7()), + correlation_id: Some(bulk_correlation), }) .await?; assert_eq!( @@ -1140,7 +1470,7 @@ mod tests { let first_maintenance = actor .handle_command(LibraryActorCommand::Start { mode: StartMode::Maintenance, - correlation_id: Some(Uuid::now_v7()), + correlation_id: Some(bulk_correlation), }) .await?; let duplicate_maintenance = actor @@ -1182,18 +1512,15 @@ mod tests { ); assert_eq!(actor.state.active_folder_scans.len(), 1); - let duplicate_responses = actor + let duplicate_error = actor .handle_command(LibraryActorCommand::FsEvents { root: LibraryRootsId(0), events: burst, correlation_id: None, }) - .await?; - assert!(enqueued_folder_paths(&duplicate_responses).is_empty()); - assert!(duplicate_responses.iter().any(|event| matches!( - event, - LibraryActorEvent::JobThrottled { .. } - ))); + .await + .expect_err("active folder should defer the whole watcher burst"); + assert!(matches!(duplicate_error, MediaError::ConcurrencyLimit(_))); let active_paths: HashSet<_> = actor.state.active_folder_scans.iter().cloned().collect(); assert_eq!(active_paths, HashSet::from([seeded_folder_norm])); diff --git a/crates/ferrex-core/src/domain/scan/fs_watch/mod.rs b/crates/ferrex-core/src/domain/scan/fs_watch/mod.rs index 3c0b781c..041fbc50 100644 --- a/crates/ferrex-core/src/domain/scan/fs_watch/mod.rs +++ b/crates/ferrex-core/src/domain/scan/fs_watch/mod.rs @@ -18,7 +18,7 @@ use crate::{ types::ids::LibraryId, }; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::env; use std::fmt; use std::path::{Component, Path, PathBuf}; @@ -35,7 +35,7 @@ use notify::{ use sha2::{Digest, Sha256}; use tokio::sync::{RwLock, mpsc}; use tokio::task::{JoinHandle, spawn_blocking}; -use tokio::time::{Duration, timeout}; +use tokio::time::{Duration, MissedTickBehavior, interval}; use tracing::{error, info, trace, warn}; use uuid::Uuid; @@ -258,25 +258,6 @@ impl FsWatchService { ); drop(guard); - if let Err(err) = replay_unacknowledged_events( - library_id, - &resolved_roots, - Arc::clone(&self.observer), - Arc::clone(&self.command_executor), - self.event_bus.clone(), - &self.consumer_group, - self.config.max_batch_events, - ) - .await - { - if let Some(watch) = - self.libraries.write().await.remove(&library_id) - { - watch.shutdown(); - } - return Err(err); - } - let libraries = Arc::clone(&self.libraries); let observer = Arc::clone(&self.observer); let watcher_roots = resolved_roots.clone(); @@ -382,25 +363,6 @@ impl FsWatchService { ); drop(guard); - if let Err(err) = replay_unacknowledged_events( - library_id, - &resolved_roots, - Arc::clone(&self.observer), - Arc::clone(&self.command_executor), - self.event_bus.clone(), - &self.consumer_group, - self.config.max_batch_events, - ) - .await - { - if let Some(watch) = - self.libraries.write().await.remove(&library_id) - { - watch.shutdown(); - } - return Err(err); - } - drop(tx); Ok(()) @@ -602,34 +564,49 @@ fn spawn_watch_loop( tokio::spawn(async move { let mut pending: HashMap> = HashMap::new(); + let replay_cadence = config + .poll_interval + .min(Duration::from_secs(30)) + .max(config.debounce_window); + let mut replay_tick = interval(replay_cadence); + replay_tick.set_missed_tick_behavior(MissedTickBehavior::Skip); loop { - let msg = if pending.is_empty() { - rx.recv().await - } else { - match timeout(config.debounce_window, rx.recv()).await { - Ok(msg) => msg, - Err(_) => { - if let Err(err) = flush_pending( - Arc::clone(&observer), - library_id, - &mut pending, - &command_executor, - event_bus.clone(), - &consumer_group, - ) - .await - { - observer.on_error(library_id, &err.to_string()); - } - continue; + let debounce = tokio::time::sleep(config.debounce_window); + tokio::pin!(debounce); + + let msg = tokio::select! { + msg = rx.recv() => msg, + _ = &mut debounce, if !pending.is_empty() => { + if let Err(err) = flush_pending( + library_id, + &mut pending, + &command_executor, + event_bus.clone(), + &consumer_group, + ).await { + observer.on_error(library_id, &err.to_string()); } + continue; + } + _ = replay_tick.tick(), if event_bus.is_some() => { + if let Err(err) = replay_unacknowledged_events( + library_id, + &roots, + &mut pending, + &command_executor, + event_bus.clone(), + &consumer_group, + config.max_batch_events, + ).await { + observer.on_error(library_id, &err.to_string()); + } + continue; } }; let Some(msg) = msg else { if let Err(err) = flush_pending( - Arc::clone(&observer), library_id, &mut pending, &command_executor, @@ -660,50 +637,20 @@ fn spawn_watch_loop( event_id, event: normalized.event, }; - - if matches!( - pending_event.event.kind, - FileSystemEventKind::Overflow - ) { - if let Err(err) = dispatch_events( - Arc::clone(&observer), - library_id, - &command_executor, - event_bus.clone(), - &consumer_group, - root_id, - vec![pending_event], - ) - .await - { - observer.on_error( - library_id, - &err.to_string(), - ); - } - continue; - } - let entry = pending.entry(root_id).or_default(); entry.push(pending_event); - if entry.len() >= config.max_batch_events { - let events = std::mem::take(entry); - if let Err(err) = dispatch_events( - Arc::clone(&observer), + if entry.len() >= config.max_batch_events + && let Err(err) = flush_pending( library_id, + &mut pending, &command_executor, event_bus.clone(), &consumer_group, - root_id, - events, ) .await - { - observer.on_error( - library_id, - &err.to_string(), - ); - } + { + observer + .on_error(library_id, &err.to_string()); } } Ok(PersistedWatchEvent::Duplicate) => { @@ -735,26 +682,12 @@ fn spawn_watch_loop( Ok(PersistedWatchEvent::Dispatch { event_id, }) => { - let pending_event = PendingWatchEvent { - event_id, - event: normalized.event, - }; - if let Err(err) = dispatch_events( - Arc::clone(&observer), - library_id, - &command_executor, - event_bus.clone(), - &consumer_group, - root_id, - vec![pending_event], - ) - .await - { - observer.on_error( - library_id, - &err.to_string(), - ); - } + pending.entry(root_id).or_default().push( + PendingWatchEvent { + event_id, + event: normalized.event, + }, + ); } Ok(PersistedWatchEvent::Duplicate) => { trace!( @@ -769,6 +702,17 @@ fn spawn_watch_loop( } } } + if let Err(err) = flush_pending( + library_id, + &mut pending, + &command_executor, + event_bus.clone(), + &consumer_group, + ) + .await + { + observer.on_error(library_id, &err.to_string()); + } }; } } @@ -776,57 +720,56 @@ fn spawn_watch_loop( }) } -async fn flush_pending( - observer: Arc, +async fn flush_pending( library_id: LibraryId, pending: &mut HashMap>, command_executor: &Arc, event_bus: Option>, consumer_group: &str, ) -> Result<()> { - let mut batches = Vec::new(); - for (root_id, events) in pending.iter_mut() { - if events.is_empty() { - continue; + let root_ids: Vec<_> = pending.keys().copied().collect(); + let mut first_error = None; + + for root_id in root_ids { + let result = match pending.get_mut(&root_id) { + Some(events) if !events.is_empty() => { + dispatch_events( + library_id, + command_executor, + event_bus.clone(), + consumer_group, + root_id, + events, + ) + .await + } + _ => Ok(()), + }; + if let Err(err) = result + && first_error.is_none() + { + first_error = Some(err); } - let drained = std::mem::take(events); - batches.push((*root_id, drained)); } + pending.retain(|_, events| !events.is_empty()); - for (root_id, events) in batches { - dispatch_events( - Arc::clone(&observer), - library_id, - command_executor, - event_bus.clone(), - consumer_group, - root_id, - events, - ) - .await?; - } - pending.clear(); - Ok(()) + first_error.map_or(Ok(()), Err) } -async fn dispatch_events( - observer: Arc, +async fn dispatch_events( library_id: LibraryId, command_executor: &Arc, event_bus: Option>, consumer_group: &str, root_id: LibraryRootsId, - events: Vec, + events: &mut Vec, ) -> Result<()> { if events.is_empty() { return Ok(()); } - let mut pending_events = events; - let mut actor_events: Vec = pending_events - .iter() - .map(|pending| pending.event.clone()) - .collect(); + let mut actor_events: Vec = + events.iter().map(|pending| pending.event.clone()).collect(); let correlation_hint = actor_events .iter() @@ -847,21 +790,24 @@ async fn dispatch_events( correlation_id: correlation_hint, }; - if let Err(err) = command_executor + command_executor .execute_library_command(library_id, command) - .await - { - observer.on_error(library_id, &err.to_string()); - return Err(err); - } + .await?; if let Some(event_bus) = event_bus { - for event_id in pending_events - .drain(..) - .filter_map(|pending| pending.event_id) - { - event_bus.ack(consumer_group, event_id).await?; + let mut acknowledged = 0usize; + for pending in events.iter() { + if let Some(event_id) = pending.event_id + && let Err(err) = event_bus.ack(consumer_group, event_id).await + { + events.drain(..acknowledged); + return Err(err); + } + acknowledged += 1; } + events.drain(..acknowledged); + } else { + events.clear(); } Ok(()) @@ -937,11 +883,11 @@ fn file_system_event_kind(kind: &FileWatchEventType) -> FileSystemEventKind { } } -async fn replay_unacknowledged_events( +async fn replay_unacknowledged_events( library_id: LibraryId, roots: &[(LibraryRootsId, PathBuf)], - observer: Arc, - command_executor: Arc, + pending: &mut HashMap>, + command_executor: &Arc, event_bus: Option>, consumer_group: &str, batch_limit: usize, @@ -952,48 +898,37 @@ async fn replay_unacknowledged_events( let limit = batch_limit.clamp(1, i32::MAX as usize) as i32; loop { - let events = + let records = event_bus.get_unprocessed_events(library_id, limit).await?; - if events.is_empty() { + if records.is_empty() { return Ok(()); } + let page_is_full = records.len() == limit as usize; + let mut known_event_ids: HashSet<_> = pending + .values() + .flat_map(|events| events.iter().filter_map(|event| event.event_id)) + .collect(); - let mut current_root: Option = None; - let mut batch = Vec::new(); - - for record in events { - let pending = replay_record_to_pending(&record, roots)?; - let root_id = LibraryRootsId(record.library_root_id as u16); - if let Some(active_root) = current_root { - if active_root != root_id { - dispatch_events( - Arc::clone(&observer), - library_id, - &command_executor, - Some(Arc::clone(&event_bus)), - consumer_group, - active_root, - std::mem::take(&mut batch), - ) - .await?; - } + for record in records { + if !known_event_ids.insert(record.id) { + continue; } - - current_root = Some(root_id); - batch.push(pending); + let pending_event = replay_record_to_pending(&record, roots)?; + let root_id = LibraryRootsId(record.library_root_id as u16); + pending.entry(root_id).or_default().push(pending_event); } - if let Some(root_id) = current_root { - dispatch_events( - Arc::clone(&observer), - library_id, - &command_executor, - Some(Arc::clone(&event_bus)), - consumer_group, - root_id, - batch, - ) - .await?; + flush_pending( + library_id, + pending, + command_executor, + Some(Arc::clone(&event_bus)), + consumer_group, + ) + .await?; + + if !page_is_full { + return Ok(()); } } } @@ -1028,7 +963,7 @@ fn replay_record_to_pending( path_key: record.path_key.clone(), fingerprint: record.fingerprint.clone(), path: PathBuf::from(&record.file_path), - old_path: record.old_path.as_ref().map(|path| PathBuf::from(path)), + old_path: record.old_path.as_ref().map(PathBuf::from), kind: file_system_event_kind(&record.event_type), occurred_at: record.detected_at, }, @@ -1266,6 +1201,99 @@ fn resolve_roots( .collect() } +fn is_network_filesystem_type(filesystem_type: &str) -> bool { + matches!(filesystem_type, "cifs" | "smb3" | "nfs" | "nfs4") +} + +fn effective_watch_strategy( + configured: WatchStrategy, + network_filesystem_type: Option<&str>, +) -> WatchStrategy { + if configured == WatchStrategy::Auto + && network_filesystem_type.is_some_and(is_network_filesystem_type) + { + WatchStrategy::Poll + } else { + configured + } +} + +#[cfg(target_os = "linux")] +fn decode_mountinfo_path(encoded: &str) -> PathBuf { + let bytes = encoded.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut index = 0usize; + + while index < bytes.len() { + if bytes[index] == b'\\' && index + 3 < bytes.len() { + let digits = &bytes[index + 1..index + 4]; + if digits.iter().all(|digit| matches!(digit, b'0'..=b'7')) { + let value = (digits[0] - b'0') * 64 + + (digits[1] - b'0') * 8 + + (digits[2] - b'0'); + decoded.push(value); + index += 4; + continue; + } + } + + decoded.push(bytes[index]); + index += 1; + } + + PathBuf::from(String::from_utf8_lossy(&decoded).into_owned()) +} + +#[cfg(target_os = "linux")] +fn network_filesystem_type_from_mountinfo( + mountinfo: &str, + path: &Path, +) -> Option { + let mut best_match: Option<(usize, String)> = None; + + for line in mountinfo.lines() { + let Some((mount_fields, filesystem_fields)) = line.split_once(" - ") + else { + continue; + }; + let Some(mount_point) = mount_fields.split_whitespace().nth(4) else { + continue; + }; + let Some(filesystem_type) = filesystem_fields.split_whitespace().next() + else { + continue; + }; + let mount_path = decode_mountinfo_path(mount_point); + if !path.starts_with(&mount_path) { + continue; + } + + let specificity = mount_path.components().count(); + if best_match + .as_ref() + .is_none_or(|(best_specificity, _)| specificity > *best_specificity) + { + best_match = Some((specificity, filesystem_type.to_owned())); + } + } + + best_match + .map(|(_, filesystem_type)| filesystem_type) + .filter(|filesystem_type| is_network_filesystem_type(filesystem_type)) +} + +#[cfg(target_os = "linux")] +fn network_filesystem_type(path: &Path) -> Option { + let resolved = std::fs::canonicalize(path).unwrap_or_else(|_| path.into()); + let mountinfo = std::fs::read_to_string("/proc/self/mountinfo").ok()?; + network_filesystem_type_from_mountinfo(&mountinfo, &resolved) +} + +#[cfg(not(target_os = "linux"))] +fn network_filesystem_type(_path: &Path) -> Option { + None +} + fn init_watchers( config: FsWatchConfig, watcher_roots: Vec<(LibraryRootsId, PathBuf)>, @@ -1273,7 +1301,28 @@ fn init_watchers( ) -> Result> { let mut watchers = Vec::with_capacity(watcher_roots.len()); for (_root_id, root_path) in &watcher_roots { - match config.strategy { + let network_filesystem = network_filesystem_type(root_path); + let strategy = effective_watch_strategy( + config.strategy, + network_filesystem.as_deref(), + ); + if let Some(filesystem_type) = network_filesystem.as_deref() { + match config.strategy { + WatchStrategy::Auto => warn!( + path = %root_path.display(), + filesystem_type, + "network filesystem detected; auto watch strategy selected polling" + ), + WatchStrategy::Native => warn!( + path = %root_path.display(), + filesystem_type, + "network filesystem detected with forced native watch strategy; notifications may be unreliable; configure polling for reliable change detection" + ), + WatchStrategy::Poll => {} + } + } + + match strategy { WatchStrategy::Native => { let watcher = build_native_watcher(root_path, watcher_tx.clone())?; @@ -1442,6 +1491,7 @@ mod tests { use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::Arc; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Duration; use async_trait::async_trait; @@ -1453,24 +1503,28 @@ mod tests { use tokio::time; use uuid::Uuid; + #[cfg(target_os = "linux")] + use super::network_filesystem_type_from_mountinfo; use super::{ EVENT_VERSION, FsWatchConfig, FsWatchService, NoopFsWatchObserver, - WatchConfig, WatchMessage, WatchStrategy, encode_hash, + WatchConfig, WatchMessage, WatchStrategy, effective_watch_strategy, + encode_hash, }; use crate::database::traits::{FileWatchEvent, FileWatchEventType}; + use crate::domain::scan::MediaCandidate; use crate::domain::scan::fs_watch::event_bus::FileChangeEventBus; use crate::domain::scan::orchestration::lease::DequeueRequest; use crate::domain::scan::orchestration::scan_cursor::normalize_path; use crate::domain::scan::orchestration::{ - CorrelationCache, DefaultLibraryActor, DependencyKey, DispatchStatus, - EnqueueRequest, FileSystemEvent, FileSystemEventKind, FolderScanJob, - InMemoryBudget, InProcJobEventBus, JobDispatcher, JobEvent, - JobEventPayload, JobHandle, JobId, JobKind, JobLease, JobPayload, - JobPriority, LeaseExpiryScanner, LeaseId, LeaseRenewal, + CorrelationCache, DedupeKey, DefaultLibraryActor, DependencyKey, + DispatchStatus, EnqueueRequest, FileSystemEvent, FileSystemEventKind, + FolderScanJob, InMemoryBudget, InProcJobEventBus, JobDispatcher, + JobEvent, JobEventPayload, JobHandle, JobId, JobKind, JobLease, + JobPayload, JobPriority, LeaseExpiryScanner, LeaseId, LeaseRenewal, LibraryActorCommand, LibraryActorConfig, LibraryActorHandle, LibraryCommandExecutor, LibraryRootsId, NoopActorObserver, OrchestratorConfig, OrchestratorRuntime, OrchestratorRuntimeBuilder, - QueueService, ScanReason, + QueueService, ScanReason, StartMode, }; use crate::error::{MediaError, Result}; use crate::types::{ @@ -1482,10 +1536,12 @@ mod tests { #[test] fn fs_watch_config_preserves_forced_poll_strategy() { - let mut watch = WatchConfig::default(); - watch.strategy = WatchStrategy::Poll; - watch.poll_interval_ms = 2_500; - watch.poll_backoff_max_ms = 42_000; + let watch = WatchConfig { + strategy: WatchStrategy::Poll, + poll_interval_ms: 2_500, + poll_backoff_max_ms: 42_000, + ..WatchConfig::default() + }; let config = FsWatchConfig::from(watch); @@ -1494,6 +1550,55 @@ mod tests { assert_eq!(config.poll_backoff_max, Duration::from_millis(42_000)); } + #[test] + fn auto_watch_strategy_prefers_polling_for_cifs_and_nfs() { + for filesystem_type in ["cifs", "smb3", "nfs", "nfs4"] { + assert_eq!( + effective_watch_strategy( + WatchStrategy::Auto, + Some(filesystem_type), + ), + WatchStrategy::Poll + ); + } + + assert_eq!( + effective_watch_strategy(WatchStrategy::Auto, Some("ext4")), + WatchStrategy::Auto + ); + assert_eq!( + effective_watch_strategy(WatchStrategy::Native, Some("nfs4")), + WatchStrategy::Native + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn mountinfo_detection_uses_longest_mount_and_decodes_paths() { + let mountinfo = concat!( + "20 1 0:1 / / rw - ext4 /dev/root rw\n", + "21 20 0:2 / /mnt/TV\\040Shows rw - nfs4 host:/shows rw\n", + "22 21 0:3 / /mnt/TV\\040Shows/local rw - ext4 /dev/loop0 rw\n", + ); + + assert_eq!( + network_filesystem_type_from_mountinfo( + mountinfo, + Path::new("/mnt/TV Shows/Series/Example"), + ) + .as_deref(), + Some("nfs4") + ); + assert_eq!( + network_filesystem_type_from_mountinfo( + mountinfo, + Path::new("/mnt/TV Shows/local/Movie"), + ), + None, + "a nested local mount must override its network parent" + ); + } + #[derive(Clone, Debug)] struct RecordedRequest { job: FolderScanJob, @@ -1505,6 +1610,8 @@ mod tests { struct RecordingQueue { records: Arc>>, accepted_by_dedupe: Arc>>, + fail_next_enqueue: Arc, + enqueue_attempts: Arc, } impl std::fmt::Debug for RecordingQueue { @@ -1524,11 +1631,25 @@ mod tests { async fn records(&self) -> Vec { self.records.lock().await.clone() } + + fn fail_next_enqueue(&self) { + self.fail_next_enqueue.store(true, Ordering::SeqCst); + } + + fn enqueue_attempts(&self) -> usize { + self.enqueue_attempts.load(Ordering::SeqCst) + } } #[async_trait] impl QueueService for RecordingQueue { async fn enqueue(&self, request: EnqueueRequest) -> Result { + self.enqueue_attempts.fetch_add(1, Ordering::SeqCst); + if self.fail_next_enqueue.swap(false, Ordering::SeqCst) { + return Err(MediaError::Internal( + "transient in-memory enqueue failure".into(), + )); + } let payload = request.payload.clone(); let dedupe_key = request.dedupe_key().to_string(); @@ -1649,6 +1770,8 @@ mod tests { struct MemoryFileChangeEventBus { events: Arc>>, acked: Arc>>, + fail_next_ack: Arc, + fail_next_replay: Arc, } impl MemoryFileChangeEventBus { @@ -1664,6 +1787,14 @@ mod tests { self.events.lock().await.clear(); self.acked.lock().await.clear(); } + + fn fail_next_ack(&self) { + self.fail_next_ack.store(true, Ordering::SeqCst); + } + + fn fail_next_replay(&self) { + self.fail_next_replay.store(true, Ordering::SeqCst); + } } #[async_trait] @@ -1680,6 +1811,11 @@ mod tests { } async fn ack(&self, _group: &str, event_id: Uuid) -> Result<()> { + if self.fail_next_ack.swap(false, Ordering::SeqCst) { + return Err(MediaError::Internal( + "transient in-memory fs watch ack failure".into(), + )); + } let mut events = self.events.lock().await; let event = events .iter_mut() @@ -1701,6 +1837,11 @@ mod tests { library_id: LibraryId, limit: i32, ) -> Result> { + if self.fail_next_replay.swap(false, Ordering::SeqCst) { + return Err(MediaError::Internal( + "transient in-memory fs watch replay failure".into(), + )); + } let mut events: Vec<_> = self .events .lock() @@ -2116,6 +2257,218 @@ mod tests { Ok(()) } + #[tokio::test] + async fn fs_watch_service_does_not_ack_active_path_until_retry_succeeds() + -> Result<()> { + let tmp = tempdir().unwrap(); + let root = tmp.path().to_path_buf(); + let movie_dir = root.join("Active Movie"); + std::fs::create_dir_all(&movie_dir).unwrap(); + let media_path = movie_dir.join("feature.mkv"); + std::fs::write(&media_path, b"movie").unwrap(); + + let harness = runtime_harness(root.clone()).await?; + harness + .runtime + .submit_library_command_and_wait( + harness.library_id, + LibraryActorCommand::Start { + mode: StartMode::Bulk, + correlation_id: Some(Uuid::now_v7()), + }, + ) + .await?; + assert_eq!(wait_for_records(&harness.queue, 1).await.len(), 1); + + let bus = Arc::new(MemoryFileChangeEventBus::default()); + let event_bus: Arc = bus.clone(); + let command_executor: Arc = + harness.runtime.clone(); + let service: FsWatchService = FsWatchService::with_event_bus( + FsWatchConfig { + debounce_window: Duration::from_millis(15), + max_batch_events: 16, + strategy: WatchStrategy::Auto, + poll_interval: Duration::from_secs(1), + poll_backoff_max: Duration::from_secs(5), + }, + Arc::new(NoopFsWatchObserver), + command_executor, + event_bus, + ); + service + .register_library_for_test( + harness.library_id, + vec![(LibraryRootsId(0), root)], + ) + .await?; + service + .send_watch_message_for_test( + harness.library_id, + WatchMessage::Event( + Event::new(EventKind::Modify(ModifyKind::Data( + DataChange::Content, + ))) + .add_path(media_path), + ), + ) + .await?; + + let durable = wait_for_published_events(&bus, 1).await; + time::sleep(Duration::from_millis(75)).await; + assert!(!bus.events().await[0].processed); + assert!(bus.acked().await.is_empty()); + + harness + .runtime + .submit_library_command_and_wait( + harness.library_id, + LibraryActorCommand::JobCompleted { + job_id: JobId::new(), + dedupe_key: DedupeKey::FolderScan { + candidate: MediaCandidate::new( + harness.library_id, + normalize_path(&movie_dir)?, + ), + }, + }, + ) + .await?; + + let acked = wait_for_acked_events(&bus, 1).await; + assert_eq!(acked[0].id, durable[0].id); + service.unregister_library(harness.library_id).await; + Ok(()) + } + + #[tokio::test] + async fn fs_watch_service_does_not_ack_paused_actor_until_resume() + -> Result<()> { + let tmp = tempdir().unwrap(); + let root = tmp.path().to_path_buf(); + let movie_dir = root.join("Paused Movie"); + std::fs::create_dir_all(&movie_dir).unwrap(); + let media_path = movie_dir.join("feature.mkv"); + std::fs::write(&media_path, b"movie").unwrap(); + + let harness = runtime_harness(root.clone()).await?; + harness + .runtime + .submit_library_command_and_wait( + harness.library_id, + LibraryActorCommand::Pause, + ) + .await?; + + let bus = Arc::new(MemoryFileChangeEventBus::default()); + let event_bus: Arc = bus.clone(); + let command_executor: Arc = + harness.runtime.clone(); + let service: FsWatchService = FsWatchService::with_event_bus( + FsWatchConfig { + debounce_window: Duration::from_millis(15), + max_batch_events: 16, + strategy: WatchStrategy::Auto, + poll_interval: Duration::from_secs(1), + poll_backoff_max: Duration::from_secs(5), + }, + Arc::new(NoopFsWatchObserver), + command_executor, + event_bus, + ); + service + .register_library_for_test( + harness.library_id, + vec![(LibraryRootsId(0), root)], + ) + .await?; + service + .send_watch_message_for_test( + harness.library_id, + WatchMessage::Event( + Event::new(EventKind::Create(CreateKind::File)) + .add_path(media_path), + ), + ) + .await?; + + let durable = wait_for_published_events(&bus, 1).await; + time::sleep(Duration::from_millis(75)).await; + assert!(!bus.events().await[0].processed); + assert!(bus.acked().await.is_empty()); + + harness + .runtime + .submit_library_command_and_wait( + harness.library_id, + LibraryActorCommand::Resume, + ) + .await?; + let acked = wait_for_acked_events(&bus, 1).await; + assert_eq!(acked[0].id, durable[0].id); + + service.unregister_library(harness.library_id).await; + Ok(()) + } + + #[tokio::test] + async fn fs_watch_service_retries_after_enqueue_failure_and_acks_once() + -> Result<()> { + let tmp = tempdir().unwrap(); + let root = tmp.path().to_path_buf(); + let movie_dir = root.join("Retry Enqueue Movie"); + std::fs::create_dir_all(&movie_dir).unwrap(); + let media_path = movie_dir.join("feature.mkv"); + std::fs::write(&media_path, b"movie").unwrap(); + + let harness = runtime_harness(root.clone()).await?; + harness.queue.fail_next_enqueue(); + let bus = Arc::new(MemoryFileChangeEventBus::default()); + let event_bus: Arc = bus.clone(); + let command_executor: Arc = + harness.runtime.clone(); + let service: FsWatchService = FsWatchService::with_event_bus( + FsWatchConfig { + debounce_window: Duration::from_millis(15), + max_batch_events: 16, + strategy: WatchStrategy::Auto, + poll_interval: Duration::from_secs(1), + poll_backoff_max: Duration::from_secs(5), + }, + Arc::new(NoopFsWatchObserver), + command_executor, + event_bus, + ); + service + .register_library_for_test( + harness.library_id, + vec![(LibraryRootsId(0), root)], + ) + .await?; + service + .send_watch_message_for_test( + harness.library_id, + WatchMessage::Event( + Event::new(EventKind::Create(CreateKind::File)) + .add_path(media_path), + ), + ) + .await?; + + let durable = wait_for_published_events(&bus, 1).await; + let records = wait_for_records(&harness.queue, 1).await; + let acked = wait_for_acked_events(&bus, 1).await; + assert_eq!(records.len(), 1); + assert_eq!(harness.queue.enqueue_attempts(), 2); + assert_eq!(acked[0].id, durable[0].id); + time::sleep(Duration::from_millis(50)).await; + assert_eq!(harness.queue.records().await.len(), 1); + assert_eq!(bus.acked().await, vec![durable[0].id]); + + service.unregister_library(harness.library_id).await; + Ok(()) + } + #[tokio::test] async fn fs_watch_service_replays_unacked_events_on_register() -> Result<()> { @@ -2161,6 +2514,114 @@ mod tests { Ok(()) } + #[tokio::test] + async fn fs_watch_service_recovers_from_transient_initial_replay_failure() + -> Result<()> { + let tmp = tempdir().unwrap(); + let root = tmp.path().to_path_buf(); + let movie_dir = root.join("Durable Replay"); + std::fs::create_dir_all(&movie_dir).unwrap(); + let media_path = movie_dir.join("feature.mkv"); + std::fs::write(&media_path, b"movie").unwrap(); + + let library_id = LibraryId::new(); + let bus = Arc::new(MemoryFileChangeEventBus::default()); + let stored = durable_event( + library_id, + &root, + &media_path, + FileWatchEventType::Created, + )?; + assert!(bus.publish(stored.clone()).await?); + bus.fail_next_replay(); + + let event_bus: Arc = bus.clone(); + let executor = RecordingCommandExecutor::default(); + let command_executor: Arc = + Arc::new(executor.clone()); + let service: FsWatchService = FsWatchService::with_event_bus( + FsWatchConfig { + debounce_window: Duration::from_millis(10), + max_batch_events: 16, + strategy: WatchStrategy::Auto, + poll_interval: Duration::from_millis(40), + poll_backoff_max: Duration::from_secs(1), + }, + Arc::new(NoopFsWatchObserver), + command_executor, + event_bus, + ); + + service + .register_library_for_test( + library_id, + vec![(LibraryRootsId(0), root)], + ) + .await?; + assert_eq!(service.runtime_snapshot().await.registered_libraries, 1); + + let events = wait_for_acked_events(&bus, 1).await; + assert_eq!(events[0].id, stored.id); + assert_eq!(executor.commands().await.len(), 1); + service.unregister_library(library_id).await; + Ok(()) + } + + #[tokio::test] + async fn fs_watch_service_retries_retained_batch_after_ack_failure() + -> Result<()> { + let tmp = tempdir().unwrap(); + let root = tmp.path().to_path_buf(); + let movie_dir = root.join("Durable Ack"); + std::fs::create_dir_all(&movie_dir).unwrap(); + let media_path = movie_dir.join("feature.mkv"); + std::fs::write(&media_path, b"movie").unwrap(); + + let library_id = LibraryId::new(); + let bus = Arc::new(MemoryFileChangeEventBus::default()); + bus.fail_next_ack(); + let event_bus: Arc = bus.clone(); + let executor = RecordingCommandExecutor::default(); + let command_executor: Arc = + Arc::new(executor.clone()); + let service: FsWatchService = FsWatchService::with_event_bus( + FsWatchConfig { + debounce_window: Duration::from_millis(15), + max_batch_events: 16, + strategy: WatchStrategy::Auto, + poll_interval: Duration::from_secs(1), + poll_backoff_max: Duration::from_secs(5), + }, + Arc::new(NoopFsWatchObserver), + command_executor, + event_bus, + ); + + service + .register_library_for_test( + library_id, + vec![(LibraryRootsId(0), root)], + ) + .await?; + service + .send_watch_message_for_test( + library_id, + WatchMessage::Event( + Event::new(EventKind::Modify(ModifyKind::Data( + DataChange::Content, + ))) + .add_path(media_path), + ), + ) + .await?; + + let commands = wait_for_commands(&executor, 2).await; + assert_eq!(commands.len(), 2, "the retained batch should be retried"); + assert_eq!(wait_for_acked_events(&bus, 1).await.len(), 1); + service.unregister_library(library_id).await; + Ok(()) + } + #[tokio::test] async fn fs_watch_service_skips_duplicate_idempotency_keys() -> Result<()> { let tmp = tempdir().unwrap(); diff --git a/crates/ferrex-core/src/domain/scan/orchestration/dispatcher.rs b/crates/ferrex-core/src/domain/scan/orchestration/dispatcher.rs index 7d4b65d5..67767734 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/dispatcher.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/dispatcher.rs @@ -34,9 +34,10 @@ use crate::domain::scan::orchestration::{ events::{ScanEvent, ScanEventBus}, job::{ AnalyzeScanHierarchy, DependencyKey, EnqueueRequest, EpisodeMatchJob, - FolderScanJob, ImageFetchJob, IndexUpsertJob, JobPayload, JobPriority, - MediaAnalyzeJob, MediaFingerprint, MetadataEnrichJob, ScanReason, - SeriesResolveJob, TranscriptExtractJob, TranscriptExtractTrigger, + FolderScanJob, ImageFetchJob, IndexUpsertJob, JobHandle, JobPayload, + JobPriority, MediaAnalyzeJob, MediaFingerprint, MetadataEnrichJob, + ScanReason, SeriesResolveJob, TranscriptExtractJob, + TranscriptExtractTrigger, }, lease::JobLease, queue::QueueService, @@ -365,7 +366,12 @@ where Self { enqueuer } } - async fn enqueue(&self, request: EnqueueRequest) -> DispatchStatus { + async fn enqueue( + &self, + parent_correlation: Option, + mut request: EnqueueRequest, + ) -> DispatchStatus { + request.correlation_id = parent_correlation; match self.enqueuer.enqueue(request).await { Ok(_) => DispatchStatus::Success, Err(err) => classify_media_error(err), @@ -374,14 +380,29 @@ where async fn enqueue_many( &self, + parent_correlation: Option, requests: Vec, ) -> DispatchStatus { - match self.enqueuer.enqueue_many(requests).await { + match self + .enqueue_many_with_handles(parent_correlation, requests) + .await + { Ok(_) => DispatchStatus::Success, Err(err) => classify_media_error(err), } } + async fn enqueue_many_with_handles( + &self, + parent_correlation: Option, + mut requests: Vec, + ) -> Result> { + for request in &mut requests { + request.correlation_id = parent_correlation; + } + self.enqueuer.enqueue_many(requests).await + } + async fn release_dependency( &self, library_id: crate::types::ids::LibraryId, @@ -462,6 +483,23 @@ impl FollowUpPlanner { EnqueueRequest::new(analyze_priority, JobPayload::MediaAnalyze(analyze)) } + fn folder_scan_for_discovery( + &self, + context: &FolderScanContext, + reason: ScanReason, + ) -> EnqueueRequest { + let job = FolderScanJob { + context: context.clone(), + scan_reason: reason, + enqueue_time: Utc::now(), + device_id: None, + }; + EnqueueRequest::new( + priority_for_reason(&reason), + JobPayload::FolderScan(job), + ) + } + fn metadata_after_analysis( &self, job: &MediaAnalyzeJob, @@ -1128,7 +1166,11 @@ where folder_name, job.scan_reason, ); - match self.follow_ups.enqueue(req).await { + match self + .follow_ups + .enqueue(lease.job.correlation_id, req) + .await + { DispatchStatus::Success => {} status => return status, } @@ -1162,7 +1204,11 @@ where discovered_events.push(media.clone()); let req = self.planner.media_analyze_for_discovery(media); - match self.follow_ups.enqueue(req).await { + match self + .follow_ups + .enqueue(lease.job.correlation_id, req) + .await + { DispatchStatus::Success => {} DispatchStatus::Retry { error } => { tracing::warn!( @@ -1206,13 +1252,37 @@ where return classify_media_error(err); } - // Emit FolderDiscovered for each child; orchestrator enqueues from events. - for child in &children { + // Make every child durable before publishing the observational + // discovery events. The scan-event broadcast channel is bounded, + // so it cannot be the ownership boundary for executable work. + let child_requests = children + .iter() + .map(|child| { + self.planner + .folder_scan_for_discovery(child, job.scan_reason) + }) + .collect(); + let child_handles = match self + .follow_ups + .enqueue_many_with_handles( + lease.job.correlation_id, + child_requests, + ) + .await + { + Ok(handles) => handles, + Err(err) => return classify_media_error(err), + }; + + for (child, handle) in children.iter().zip(child_handles) { + let durable_job_id = handle.merged_into.unwrap_or(handle.job_id); if let Err(err) = self .events .publish_scan_event(ScanEvent::FolderDiscovered { context: Box::new(child.clone()), reason: job.scan_reason, + correlation_id: lease.job.correlation_id, + durable_job_id: Some(durable_job_id), }) .await { @@ -1409,20 +1479,24 @@ where error: "folder scan payload routed to media pipeline".into(), }, JobPayload::SeriesResolve(job) => { - self.handle_series_resolve(job).await + self.handle_series_resolve(lease.job.correlation_id, job) + .await } JobPayload::MediaAnalyze(job) => { - self.handle_media_analyze(job).await + self.handle_media_analyze(lease.job.correlation_id, job) + .await } JobPayload::MetadataEnrich(job) => { - self.handle_metadata_enrich(job).await + self.handle_metadata_enrich(lease.job.correlation_id, job) + .await } JobPayload::IndexUpsert(job) => { self.handle_index_upsert(lease, job).await } JobPayload::ImageFetch(job) => self.handle_image_fetch(job).await, JobPayload::EpisodeMatch(job) => { - self.handle_episode_match(job).await + self.handle_episode_match(lease.job.correlation_id, job) + .await } JobPayload::TranscriptExtract(job) => { self.handle_transcript_extract(lease, job).await @@ -1432,6 +1506,7 @@ where async fn handle_media_analyze( &self, + parent_correlation: Option, job: &MediaAnalyzeJob, ) -> DispatchStatus { // TODO: Refactor clone @@ -1481,7 +1556,10 @@ where .metadata_after_resolved_episode_analysis( job, &analyzed, hierarchy, ); - return self.follow_ups.enqueue(req).await; + return self + .follow_ups + .enqueue(parent_correlation, req) + .await; } EpisodeDependencyDecision::Deferred { dependency_key } => { let req = self.planner.episode_match_after_analysis( @@ -1490,18 +1568,22 @@ where episode_hierarchy, dependency_key, ); - return self.follow_ups.enqueue(req).await; + return self + .follow_ups + .enqueue(parent_correlation, req) + .await; } } } } let req = self.planner.metadata_after_analysis(job, &analyzed); - self.follow_ups.enqueue(req).await + self.follow_ups.enqueue(parent_correlation, req).await } async fn handle_series_resolve( &self, + parent_correlation: Option, job: &SeriesResolveJob, ) -> DispatchStatus { let resolution = match self.series_coordinator.resolve_series(job).await @@ -1559,11 +1641,12 @@ where } let req = self.planner.index_for_ready(job.library_id, &ready); - self.follow_ups.enqueue(req).await + self.follow_ups.enqueue(parent_correlation, req).await } async fn handle_metadata_enrich( &self, + parent_correlation: Option, job: &MetadataEnrichJob, ) -> DispatchStatus { let analyzed = MediaAnalyzed { @@ -1609,14 +1692,18 @@ where if !ready.image_jobs.is_empty() { let image_requests = self.planner.image_fetches_for_ready(&ready); - match self.follow_ups.enqueue_many(image_requests).await { + match self + .follow_ups + .enqueue_many(parent_correlation, image_requests) + .await + { DispatchStatus::Success => {} status => return status, } } let req = self.planner.index_for_ready(job.library_id, &ready); - self.follow_ups.enqueue(req).await + self.follow_ups.enqueue(parent_correlation, req).await } async fn handle_index_upsert( @@ -1743,7 +1830,9 @@ where TranscriptExtractTrigger::IndexUpsert, lease.job.correlation_id, ); - self.follow_ups.enqueue(request).await + self.follow_ups + .enqueue(lease.job.correlation_id, request) + .await } fn transcript_attempt(lease: &JobLease) -> i32 { @@ -2072,6 +2161,7 @@ where async fn handle_episode_match( &self, + parent_correlation: Option, job: &EpisodeMatchJob, ) -> DispatchStatus { let hierarchy = match self @@ -2084,7 +2174,7 @@ where }; let req = self.planner.metadata_after_episode_match(job, hierarchy); - self.follow_ups.enqueue(req).await + self.follow_ups.enqueue(parent_correlation, req).await } } @@ -2155,7 +2245,7 @@ mod tests { use crate::types::ids::{LibraryId, MovieID, SeriesID}; use crate::types::library::LibraryType; use ferrex_model::details::ExternalIds; - use ferrex_model::image::MediaImages; + use ferrex_model::image::{ImageSize, MediaImages}; use ferrex_model::titles::MovieTitle; use ferrex_model::urls::{MovieURL, UrlLike}; use ferrex_model::{ @@ -2163,7 +2253,7 @@ mod tests { MovieReference, MovieReferenceBatchSize, VideoMediaType, }; use sqlx::PgPool; - use std::collections::HashMap; + use std::collections::{HashMap, HashSet}; use tokio::sync::Mutex; use tokio::time::Duration; use uuid::Uuid; @@ -3031,6 +3121,31 @@ mod tests { ) } + fn lease_for_payload_with_correlation( + payload: JobPayload, + correlation_id: Uuid, + ) -> JobLease { + let mut record = JobRecord::new(payload, JobPriority::P1); + record.correlation_id = Some(correlation_id); + JobLease::new( + record, + "test-worker".into(), + chrono::Duration::seconds(30), + ) + } + + fn lease_for_enqueue_request(request: &EnqueueRequest) -> JobLease { + let mut record = + JobRecord::new(request.payload.clone(), request.priority); + record.correlation_id = request.correlation_id; + record.dependency_key = request.dependency_key.clone(); + JobLease::new( + record, + "test-worker".into(), + chrono::Duration::seconds(30), + ) + } + fn series_hint(title: &str) -> SeriesHint { SeriesHint { title: title.to_string(), @@ -3317,9 +3432,11 @@ mod tests { std::fs::set_permissions(&ffprobe_path, perms).unwrap(); } - let mut timed_text_config = TimedTextExtractionConfig::default(); - timed_text_config.ffprobe_path = ffprobe_path; - timed_text_config.ffmpeg_path = library_root.join("missing-ffmpeg"); + let timed_text_config = TimedTextExtractionConfig { + ffprobe_path, + ffmpeg_path: library_root.join("missing-ffmpeg"), + ..TimedTextExtractionConfig::default() + }; let dispatcher = DefaultJobDispatcher::new( Arc::clone(&queue), @@ -3354,10 +3471,11 @@ mod tests { path_norm: media_path.to_string_lossy().to_string(), idempotency_key: "timed-text-runtime-test".to_string(), }; - let lease = JobLease::new( - JobRecord::new(JobPayload::IndexUpsert(job), JobPriority::P1), - "timed-text-test".to_string(), - chrono::Duration::seconds(30), + let parent_correlation = + Uuid::from_u128(0x6290000000000000000000000000c001); + let lease = lease_for_payload_with_correlation( + JobPayload::IndexUpsert(job), + parent_correlation, ); let status = dispatcher.dispatch(&lease).await; @@ -3366,6 +3484,7 @@ mod tests { assert!(transcripts.recorded().await.is_empty()); let enqueued = queue.enqueued().await; assert_eq!(enqueued.len(), 1); + assert_eq!(enqueued[0].correlation_id, Some(parent_correlation)); let JobPayload::TranscriptExtract(transcript_job) = enqueued[0].payload.clone() else { @@ -3379,14 +3498,7 @@ mod tests { TranscriptExtractTrigger::IndexUpsert ); - let transcript_lease = JobLease::new( - JobRecord::new( - JobPayload::TranscriptExtract(transcript_job), - JobPriority::P2, - ), - "timed-text-test".to_string(), - chrono::Duration::seconds(30), - ); + let transcript_lease = lease_for_enqueue_request(&enqueued[0]); let status = dispatcher.dispatch(&transcript_lease).await; assert_eq!(status, DispatchStatus::Success); @@ -3586,8 +3698,145 @@ mod tests { assert_eq!(folder_discovered_count, 0); } + #[tokio::test] + async fn child_burst_is_durable_before_bounded_discovery_events_lag() { + const CHILD_COUNT: usize = 300; + + let parent_correlation = + Uuid::from_u128(0x5760000000000000000000000000c022); + let library_id = + LibraryId(Uuid::from_u128(0x57600000000000000000000000000022)); + let parent_context = FolderScanContext::Movie(MovieFolderScanContext { + library_id, + movie_root_path: MovieRootPath::try_new_under_library_root( + "/library", + "/library/Burst Parent", + ) + .unwrap(), + }); + let children: Vec<_> = (0..CHILD_COUNT) + .map(|index| { + FolderScanContext::Movie(MovieFolderScanContext { + library_id, + movie_root_path: MovieRootPath::try_new_under_library_root( + "/library", + format!("/library/Child {index:03}"), + ) + .unwrap(), + }) + }) + .collect(); + let expected_paths: HashSet<_> = children + .iter() + .map(|child| child.folder_path_norm().to_string()) + .collect(); + + let queue = Arc::new(RecordingQueue::default()); + let events = Arc::new(InProcJobEventBus::new(256)); + let mut scan_rx = events.subscribe_scan(); + let cursors = Arc::new(MemoryCursorRepository::default()); + let series_states: Arc> = + Arc::new(Box::new(InMemorySeriesScanStateRepository::default())); + let series_resolver = + Arc::new(StubSeriesResolver::new(Arc::clone(&series_states))); + let folder_actor = Arc::new(StubFolderActor { + plan: FolderListingPlan { + directories: children + .iter() + .map(|child| PathBuf::from(child.folder_path_norm())) + .collect(), + media_files: vec![], + ancillary_files: vec![], + generated_listing_hash: "child-burst-listing".into(), + total_entries: CHILD_COUNT, + folder_missing: false, + }, + discovered: vec![], + children: children.clone(), + summary: FolderScanSummary { + context: parent_context.clone(), + discovered_files: 0, + enqueued_subfolders: CHILD_COUNT, + listing_hash: "child-burst-listing".into(), + outcome: FolderScanOutcome::Changed, + completed_at: Utc::now(), + }, + }) as Arc; + let actors = DispatcherActors::new( + folder_actor, + Arc::new(StubAnalyzeActor), + Arc::new(StubMetadataActor), + Arc::new(StubIndexActor), + Arc::new(StubImageActor), + ); + let dispatcher = DefaultJobDispatcher::new( + Arc::clone(&queue), + Arc::clone(&events), + cursors, + Arc::clone(&series_states), + series_resolver, + actors, + CorrelationCache::default(), + ); + let lease = lease_for_payload_with_correlation( + JobPayload::FolderScan(FolderScanJob { + context: parent_context, + scan_reason: ScanReason::BulkSeed, + enqueue_time: Utc::now(), + device_id: None, + }), + parent_correlation, + ); + + assert_eq!(dispatcher.dispatch(&lease).await, DispatchStatus::Success); + + let enqueued = queue.enqueued().await; + assert_eq!(enqueued.len(), CHILD_COUNT); + assert!(enqueued.iter().all(|request| { + matches!(request.payload, JobPayload::FolderScan(_)) + && request.correlation_id == Some(parent_correlation) + })); + let durable_paths: HashSet<_> = enqueued + .iter() + .filter_map(|request| match &request.payload { + JobPayload::FolderScan(job) => { + Some(job.context.folder_path_norm().to_string()) + } + _ => None, + }) + .collect(); + assert_eq!(durable_paths, expected_paths); + + let skipped = match scan_rx.recv().await { + Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { + skipped + } + other => { + panic!("expected >256 discovery burst to lag, got {other:?}") + } + }; + assert!(skipped >= 45); + + let mut observed_marked_discoveries = 0; + while let Ok(event) = scan_rx.try_recv() { + if let ScanEvent::FolderDiscovered { + durable_job_id, + correlation_id, + .. + } = event + { + assert!(durable_job_id.is_some()); + assert_eq!(correlation_id, Some(parent_correlation)); + observed_marked_discoveries += 1; + } + } + assert!(observed_marked_discoveries > 0); + } + #[tokio::test] async fn series_root_scan_enqueues_resolve_and_season_discovery() { + let parent_correlation = + Uuid::from_u128(0x5760000000000000000000000000c020); let library_id = LibraryId(Uuid::from_u128(0x57600000000000000000000000000001)); let library_root = "/library"; @@ -3666,12 +3915,15 @@ mod tests { correlations.clone(), ); - let lease = lease_for_payload(JobPayload::FolderScan(FolderScanJob { - context: series_context.clone(), - scan_reason: ScanReason::BulkSeed, - enqueue_time: Utc::now(), - device_id: None, - })); + let lease = lease_for_payload_with_correlation( + JobPayload::FolderScan(FolderScanJob { + context: series_context.clone(), + scan_reason: ScanReason::BulkSeed, + enqueue_time: Utc::now(), + device_id: None, + }), + parent_correlation, + ); let status = dispatcher.dispatch(&lease).await; assert!(matches!(status, DispatchStatus::Success)); @@ -3684,7 +3936,11 @@ mod tests { assert!(matches!(state.status, SeriesScanStatus::Discovered)); let enqueued = queue.enqueued().await; - assert_eq!(enqueued.len(), 1, "series root scan enqueues one job"); + assert_eq!( + enqueued.len(), + 2, + "series root scan durably enqueues resolve and season scan" + ); let JobPayload::SeriesResolve(series_job) = &enqueued[0].payload else { panic!("expected SeriesResolve follow-up"); }; @@ -3692,7 +3948,9 @@ mod tests { assert_eq!(series_job.series_root_path, series_root_path); assert_eq!(series_job.folder_name, "Deterministic Show"); assert!(matches!(series_job.scan_reason, ScanReason::BulkSeed)); - assert!(enqueued[0].correlation_id.is_none()); + assert_eq!(enqueued[0].correlation_id, Some(parent_correlation)); + assert!(matches!(enqueued[1].payload, JobPayload::FolderScan(_))); + assert_eq!(enqueued[1].correlation_id, Some(parent_correlation)); let mut saw_series_resolve_enqueue = false; while let Ok(event) = job_rx.try_recv() { @@ -3719,8 +3977,15 @@ mod tests { let mut saw_completion = false; while let Ok(event) = scan_rx.try_recv() { match event { - ScanEvent::FolderDiscovered { context, reason } => { + ScanEvent::FolderDiscovered { + context, + reason, + correlation_id, + durable_job_id, + } => { assert!(matches!(reason, ScanReason::BulkSeed)); + assert_eq!(correlation_id, Some(parent_correlation)); + assert!(durable_job_id.is_some()); let FolderScanContext::Season(ctx) = *context else { panic!("expected season FolderDiscovered context"); }; @@ -3748,6 +4013,203 @@ mod tests { saw_season_discovered, "season folder discovery should be emitted deterministically" ); + + let episode_path = "/library/Deterministic Show/Season 1/S01E01.mkv"; + let analyze_lease = lease_for_payload_with_correlation( + JobPayload::MediaAnalyze(media_analyze_episode_job( + library_id, + series_root_path.clone(), + "Deterministic Show", + episode_path, + 1, + )), + parent_correlation, + ); + assert_eq!( + dispatcher.dispatch(&analyze_lease).await, + DispatchStatus::Success + ); + let enqueued = queue.enqueued().await; + assert_eq!(enqueued.len(), 3); + assert!(matches!(enqueued[2].payload, JobPayload::EpisodeMatch(_))); + assert!(enqueued[2].dependency_key.is_some()); + assert_eq!(enqueued[2].correlation_id, Some(parent_correlation)); + + let resolve_lease = lease_for_enqueue_request(&enqueued[0]); + assert_eq!( + dispatcher.dispatch(&resolve_lease).await, + DispatchStatus::Success + ); + let enqueued = queue.enqueued().await; + assert_eq!(enqueued.len(), 4); + assert!(matches!(enqueued[3].payload, JobPayload::IndexUpsert(_))); + assert_eq!(enqueued[3].correlation_id, Some(parent_correlation)); + + let episode_lease = lease_for_enqueue_request(&enqueued[2]); + assert_eq!( + dispatcher.dispatch(&episode_lease).await, + DispatchStatus::Success + ); + let enqueued = queue.enqueued().await; + assert_eq!(enqueued.len(), 5); + assert!(matches!(enqueued[4].payload, JobPayload::MetadataEnrich(_))); + assert_eq!(enqueued[4].correlation_id, Some(parent_correlation)); + } + + #[tokio::test] + async fn movie_pipeline_followups_inherit_parent_correlation() { + struct MetadataWithImageActor; + + #[async_trait] + impl MetadataActor for MetadataWithImageActor { + async fn enrich( + &self, + command: MetadataCommand, + ) -> Result { + Ok(MediaReadyForIndex { + library_id: command.job.library_id, + media_id: command.job.media_id, + variant: command.job.variant, + hierarchy: command.job.hierarchy.clone(), + node: command.job.node.clone(), + normalized_title: None, + analyzed: command.analyzed, + prepared_at: Utc::now(), + image_jobs: vec![ImageFetchJob { + library_id: command.job.library_id, + iid: Uuid::from_u128( + 0x5760000000000000000000000000a021, + ), + imz: ImageSize::poster(), + priority_hint: ImageFetchPriority::Poster, + }], + }) + } + } + + let parent_correlation = + Uuid::from_u128(0x5760000000000000000000000000c021); + let library_id = + LibraryId(Uuid::from_u128(0x57600000000000000000000000000021)); + let library_root = "/library"; + let movie_root_path = MovieRootPath::try_new_under_library_root( + library_root, + "/library/Correlated Movie", + ) + .unwrap(); + let context = FolderScanContext::Movie(MovieFolderScanContext { + library_id, + movie_root_path: movie_root_path.clone(), + }); + let media_path = "/library/Correlated Movie/movie.mkv"; + let media_id = MediaID::new(VideoMediaType::Movie); + let hierarchy = AnalyzeScanHierarchy::Movie(MovieScanHierarchy { + movie_root_path, + movie_id: None, + extra_tag: None, + }); + + let queue = Arc::new(RecordingQueue::default()); + let events = Arc::new(InProcJobEventBus::new(32)); + let cursors = Arc::new(MemoryCursorRepository::default()); + let series_states: Arc> = + Arc::new(Box::new(InMemorySeriesScanStateRepository::default())); + let series_resolver = + Arc::new(StubSeriesResolver::new(Arc::clone(&series_states))); + let folder_actor = Arc::new(StubFolderActor { + plan: FolderListingPlan { + directories: vec![], + media_files: vec![PathBuf::from(media_path)], + ancillary_files: vec![], + generated_listing_hash: "correlated-movie-listing".into(), + total_entries: 1, + folder_missing: false, + }, + discovered: vec![MediaFileDiscovered { + library_id, + path_norm: media_path.into(), + fingerprint: MediaFingerprint::default(), + classified_as: MediaKindHint::Movie, + media_id, + variant: VideoMediaType::Movie, + node: ScanNodeKind::MovieFolder, + hierarchy, + context: context.clone(), + scan_reason: ScanReason::BulkSeed, + }], + children: vec![], + summary: FolderScanSummary { + context: context.clone(), + discovered_files: 1, + enqueued_subfolders: 0, + listing_hash: "correlated-movie-listing".into(), + outcome: FolderScanOutcome::Changed, + completed_at: Utc::now(), + }, + }) as Arc; + let actors = DispatcherActors::new( + folder_actor, + Arc::new(StubAnalyzeActor), + Arc::new(MetadataWithImageActor), + Arc::new(StubIndexActor), + Arc::new(StubImageActor), + ); + let dispatcher = DefaultJobDispatcher::new( + Arc::clone(&queue), + events, + cursors, + series_states, + series_resolver, + actors, + CorrelationCache::default(), + ); + + let folder_lease = lease_for_payload_with_correlation( + JobPayload::FolderScan(FolderScanJob { + context, + scan_reason: ScanReason::BulkSeed, + enqueue_time: Utc::now(), + device_id: None, + }), + parent_correlation, + ); + assert_eq!( + dispatcher.dispatch(&folder_lease).await, + DispatchStatus::Success + ); + + let enqueued = queue.enqueued().await; + assert_eq!(enqueued.len(), 1); + assert!(matches!(enqueued[0].payload, JobPayload::MediaAnalyze(_))); + assert_eq!(enqueued[0].correlation_id, Some(parent_correlation)); + + let analyze_lease = lease_for_enqueue_request(&enqueued[0]); + assert_eq!( + dispatcher.dispatch(&analyze_lease).await, + DispatchStatus::Success + ); + + let enqueued = queue.enqueued().await; + assert_eq!(enqueued.len(), 2); + assert!(matches!(enqueued[1].payload, JobPayload::MetadataEnrich(_))); + assert_eq!(enqueued[1].correlation_id, Some(parent_correlation)); + + let metadata_lease = lease_for_enqueue_request(&enqueued[1]); + assert_eq!( + dispatcher.dispatch(&metadata_lease).await, + DispatchStatus::Success + ); + + let enqueued = queue.enqueued().await; + assert_eq!(enqueued.len(), 4); + assert!(matches!(enqueued[2].payload, JobPayload::ImageFetch(_))); + assert!(matches!(enqueued[3].payload, JobPayload::IndexUpsert(_))); + assert!( + enqueued[2..] + .iter() + .all(|request| request.correlation_id + == Some(parent_correlation)) + ); } #[tokio::test] diff --git a/crates/ferrex-core/src/domain/scan/orchestration/enqueuer.rs b/crates/ferrex-core/src/domain/scan/orchestration/enqueuer.rs index 6d0fdd5d..e3cf94a7 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/enqueuer.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/enqueuer.rs @@ -58,6 +58,12 @@ where ) -> Option { if handle.accepted { request.correlation_id + } else if request.correlation_id.is_some() { + // Attribute the merge notification to the requesting run. The + // existing job can be shared by more than one active run, so its + // durable/cached owner remains unchanged and each requester + // persists the shared job ID in its own scan-run metadata. + request.correlation_id } else if let Some(existing) = handle.merged_into { self.correlations .fetch(&existing) @@ -595,6 +601,46 @@ mod tests { ); } + #[tokio::test] + async fn correlated_merge_is_routed_to_requester_without_stealing_owner() { + let queue = Arc::new(RecordingQueue::default()); + let publisher = Arc::new(RecordingPublisher::default()); + let correlations = CorrelationCache::default(); + let existing_job = + JobId(Uuid::from_u128(0x88888888888888888888888888888888)); + let old_correlation = + Uuid::from_u128(0x99999999999999999999999999999999); + let requested_correlation = + Uuid::from_u128(0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa); + correlations.remember(existing_job, old_correlation).await; + let pipeline = enqueuer( + Arc::clone(&queue), + Arc::clone(&publisher), + correlations.clone(), + ); + let mut request = image_request( + JobPriority::P1, + Uuid::from_u128(0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb), + ); + request.correlation_id = Some(requested_correlation); + queue + .push_enqueue_result(JobHandle::merged( + existing_job, + &request.payload, + request.priority, + )) + .await; + + pipeline.enqueue(request).await.expect("enqueue"); + + let events = publisher.events().await; + assert_eq!(events[0].meta.correlation_id, requested_correlation); + assert_eq!( + correlations.fetch(&existing_job).await, + Some(old_correlation) + ); + } + #[tokio::test] async fn enqueue_preserves_dependency_key_for_episode_match() { let queue = Arc::new(RecordingQueue::default()); diff --git a/crates/ferrex-core/src/domain/scan/orchestration/events.rs b/crates/ferrex-core/src/domain/scan/orchestration/events.rs index 377d2298..922e9678 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/events.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/events.rs @@ -154,6 +154,11 @@ pub struct ScanSeedSummary { pub correlation_id: Option, pub mode: ScanSeedMode, pub queued_folders: usize, + /// Durable queue IDs enrolled by the seed batch. For merged work this is + /// the existing shared job ID, allowing recovery even if enqueue events + /// were dropped from the broadcast stream. + #[serde(default)] + pub enrolled_job_ids: Vec, pub completed_at: DateTime, } @@ -164,6 +169,14 @@ pub enum ScanEvent { context: Box, /// Why this folder should be scanned; used to determine priority reason: ScanReason, + /// Scan-run lineage inherited by the child folder job. + #[serde(default)] + correlation_id: Option, + /// Durable child job created before this observational event was + /// published. Older producers omit this field and are still routed + /// through the compatibility enqueue path. + #[serde(default, skip_serializing_if = "Option::is_none")] + durable_job_id: Option, }, MediaFileDiscovered(Box), FolderScanCompleted(FolderScanSummary), diff --git a/crates/ferrex-core/src/domain/scan/orchestration/maintenance.rs b/crates/ferrex-core/src/domain/scan/orchestration/maintenance.rs index 28d8267f..fd33b030 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/maintenance.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/maintenance.rs @@ -112,6 +112,11 @@ pub struct MaintenancePlan { pub stale_cursor_count: usize, pub new_root_folder_count: usize, pub skipped_root_entries: usize, + /// Whether root discovery deferred otherwise eligible folders because the + /// configured per-pass discovery bound was reached. + pub root_discovery_truncated: bool, + /// Number of eligible root folders deferred to a later planning pass. + pub deferred_root_entries: usize, pub errors: Vec, } @@ -124,6 +129,8 @@ impl MaintenancePlan { stale_cursor_count: 0, new_root_folder_count: 0, skipped_root_entries: 0, + root_discovery_truncated: false, + deferred_root_entries: 0, errors: Vec::new(), } } @@ -199,10 +206,16 @@ pub async fn plan_maintenance_sweep_from_summaries( let root_discovery = discover_new_root_folders( input.library, &known_cursor_paths, - input.limits.max_root_entries_per_library, + input + .limits + .max_root_entries_per_library + .min(input.limits.max_jobs_per_library), + input.now, ) .await; plan.skipped_root_entries += root_discovery.skipped_entries; + plan.root_discovery_truncated = root_discovery.truncated; + plan.deferred_root_entries = root_discovery.deferred_entries; plan.errors.extend(root_discovery.errors); for folder_path_norm in root_discovery.folders { @@ -384,6 +397,8 @@ fn matching_root( struct RootDiscovery { folders: Vec, skipped_entries: usize, + truncated: bool, + deferred_entries: usize, errors: Vec, } @@ -391,11 +406,11 @@ async fn discover_new_root_folders( library: &MaintenanceLibrary, known_cursor_paths: &HashSet, max_root_entries: usize, + now: DateTime, ) -> RootDiscovery { let mut discovery = RootDiscovery::default(); - let mut visited_entries = 0usize; - 'roots: for root in &library.paths { + for root in &library.paths { let root_norm = match normalize_path(root) { Ok(path) => path, Err(err) => { @@ -418,10 +433,6 @@ async fn discover_new_root_folders( }; loop { - if visited_entries >= max_root_entries { - break 'roots; - } - let entry = match rd.next_entry().await { Ok(Some(entry)) => entry, Ok(None) => break, @@ -433,7 +444,6 @@ async fn discover_new_root_folders( continue; } }; - visited_entries += 1; let name = entry.file_name(); let name = name.to_string_lossy(); @@ -474,13 +484,53 @@ async fn discover_new_root_folders( } }; - if !known_cursor_paths.contains(&path_norm) { - discovery.folders.push(path_norm); - } + discovery.folders.push(path_norm); } } + // Directory iteration order is filesystem-specific. Sort the eligible + // folders into deterministic partitions, then rotate the active partition + // by maintenance epoch. The library UUID supplies a stable offset so large + // libraries do not all enumerate the same partition at once. Rotation is + // independent of job completion and survives restarts without another + // persistence table. discovery + .folders + .retain(|path| !known_cursor_paths.contains(path)); + discovery.folders.sort_unstable(); + discovery.folders.dedup(); + if discovery.folders.len() > max_root_entries { + discovery.truncated = true; + let total_entries = discovery.folders.len(); + let page_count = total_entries.div_ceil(max_root_entries); + let page_index = + root_discovery_partition_index(library, now, page_count); + let page_start = page_index.saturating_mul(max_root_entries); + let page_end = page_start + .saturating_add(max_root_entries) + .min(total_entries); + discovery.deferred_entries = + total_entries.saturating_sub(page_end - page_start); + discovery.folders = discovery.folders[page_start..page_end].to_vec(); + } + discovery +} + +fn root_discovery_partition_index( + library: &MaintenanceLibrary, + now: DateTime, + page_count: usize, +) -> usize { + if page_count <= 1 { + return 0; + } + + let interval_seconds = library.scan_interval().num_seconds().max(1); + let epoch = now.timestamp().div_euclid(interval_seconds); + let page_count_i64 = i64::try_from(page_count).unwrap_or(i64::MAX); + let epoch_index = epoch.rem_euclid(page_count_i64) as usize; + let stable_offset = (library.id.0.as_u128() % page_count as u128) as usize; + (stable_offset + epoch_index) % page_count } #[cfg(test)] @@ -627,6 +677,55 @@ mod tests { assert!(job.context.folder_path_norm().ends_with("New Movie")); } + #[tokio::test] + async fn root_discovery_rotates_beyond_entry_512_without_completed_jobs() { + let temp = tempfile::tempdir().expect("tempdir"); + let root = temp.path().to_path_buf(); + for index in 0..513 { + std::fs::create_dir(root.join(format!("Movie {index:04}"))) + .expect("movie dir"); + } + + let library = test_library(root, None, false); + let repo = FakeCursorRepository::default(); + let mut first_epoch = + DateTime::::from_timestamp(0, 0).expect("unix epoch"); + while root_discovery_partition_index(&library, first_epoch, 2) != 0 { + first_epoch += library.scan_interval(); + } + let first = plan_maintenance_sweep( + &library, + &repo, + MaintenancePlanningLimits::new(512, 512), + first_epoch, + ) + .await + .expect("first plan"); + + assert_eq!(first.requests.len(), 512); + assert!(first.root_discovery_truncated); + assert_eq!(first.deferred_root_entries, 1); + + // Do not write cursors for the first page. The next maintenance epoch + // must rotate independently of whether those jobs succeeded. + let second = plan_maintenance_sweep( + &library, + &repo, + MaintenancePlanningLimits::new(512, 512), + first_epoch + library.scan_interval(), + ) + .await + .expect("second plan"); + + assert_eq!(second.requests.len(), 1); + assert!(second.root_discovery_truncated); + assert_eq!(second.deferred_root_entries, 512); + let JobPayload::FolderScan(job) = &second.requests[0].payload else { + panic!("expected folder scan") + }; + assert!(job.context.folder_path_norm().ends_with("Movie 0512")); + } + #[tokio::test] async fn plan_ignores_root_media_and_manifest_files() { let temp = tempfile::tempdir().expect("tempdir"); diff --git a/crates/ferrex-core/src/domain/scan/orchestration/mod.rs b/crates/ferrex-core/src/domain/scan/orchestration/mod.rs index fdc0a3cd..d149f48f 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/mod.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/mod.rs @@ -108,8 +108,9 @@ pub use maintenance::{ }; pub use persistence::{PostgresCursorRepository, PostgresQueueService}; pub use queue::{ - LeaseExpiryScanner, QueueInstrumentation, QueueService, QueueSnapshot, - QueueSnapshotEntry, ReadyQueueCount, + DurableJobState, FailOutcome, LeaseExpiryScanner, QueueInstrumentation, + QueueService, QueueSnapshot, QueueSnapshotEntry, QueueTransitionOutcome, + ReadyQueueCount, }; pub use runtime::{ InProcJobEventBus, JobEventStream, LibraryActorHandle, diff --git a/crates/ferrex-core/src/domain/scan/orchestration/persistence.rs b/crates/ferrex-core/src/domain/scan/orchestration/persistence.rs index 2b030604..c61cbe6d 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/persistence.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/persistence.rs @@ -12,14 +12,15 @@ use tracing::{debug, info, trace, warn}; use crate::domain::scan::orchestration::{ config::RetryConfig, + events::stable_path_key, job::{ DependencyKey, EnqueueRequest, JobHandle, JobId, JobKind, JobPayload, - JobPriority, ScanReason, TranscriptExtractJob, + JobPriority, JobState, ScanReason, TranscriptExtractJob, }, lease::{DequeueRequest, JobLease, LeaseId, LeaseRenewal}, queue::{ - LeaseExpiryScanner, QueueInstrumentation, QueueService, QueueSnapshot, - ReadyQueueCount, + DurableJobState, FailOutcome, LeaseExpiryScanner, QueueInstrumentation, + QueueService, QueueSnapshot, QueueTransitionOutcome, ReadyQueueCount, }, scan_cursor::{ScanCursor, ScanCursorId, ScanCursorRepository}, }; @@ -183,6 +184,84 @@ where Ok(()) } +async fn enroll_job<'e, E>( + executor: E, + correlation_id: Option, + job_id: JobId, +) -> Result<()> +where + E: Executor<'e, Database = sqlx::Postgres>, +{ + let Some(correlation_id) = correlation_id else { + return Ok(()); + }; + + sqlx::query!( + r#" + INSERT INTO orchestrator_job_enrollments (correlation_id, job_id) + VALUES ($1, $2) + ON CONFLICT (correlation_id, job_id) DO NOTHING + "#, + correlation_id, + job_id.0, + ) + .execute(executor) + .await + .map_err(|err| { + MediaError::Internal(format!( + "orchestrator job enrollment failed: {err}" + )) + })?; + + Ok(()) +} + +/// Atomically linearize a merge against an active durable job and enroll the +/// requesting correlation. A terminal transition racing the earlier lookup +/// wins this check and forces the caller to enqueue a new job generation. +async fn enroll_active_job<'e, E>( + executor: E, + correlation_id: Option, + job_id: JobId, +) -> Result +where + E: Executor<'e, Database = sqlx::Postgres>, +{ + let row = sqlx::query!( + r#" + WITH mergeable_job AS MATERIALIZED ( + SELECT id + FROM orchestrator_jobs + WHERE id = $2 + AND state IN ('ready','deferred','leased') + FOR UPDATE + ), enrollment AS ( + INSERT INTO orchestrator_job_enrollments (correlation_id, job_id) + SELECT $1::uuid, id + FROM mergeable_job + WHERE $1::uuid IS NOT NULL + ON CONFLICT (correlation_id, job_id) DO NOTHING + RETURNING job_id + ) + SELECT + EXISTS(SELECT 1 FROM mergeable_job) AS "mergeable!", + (SELECT COUNT(*)::bigint FROM enrollment) AS "enrollment_count!" + "#, + correlation_id, + job_id.0, + ) + .fetch_one(executor) + .await + .map_err(|err| { + MediaError::Internal(format!( + "active orchestrator job enrollment failed: {err}" + )) + })?; + + let _ = row.enrollment_count; + Ok(row.mergeable) +} + impl fmt::Debug for PostgresQueueService { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("PostgresQueueService") @@ -491,14 +570,38 @@ impl PostgresQueueService { } } - /// Fetch grouped ready counts directly from persistence. Used to prime the - /// in-memory scheduler after a cold start. + fn parse_state(state: &str) -> Result { + match state { + "ready" => Ok(JobState::Ready), + "deferred" => Ok(JobState::Deferred), + "leased" => Ok(JobState::Leased), + "completed" => Ok(JobState::Completed), + "failed" => Ok(JobState::Failed), + "dead_letter" => Ok(JobState::DeadLetter), + other => Err(MediaError::Internal(format!( + "queue returned unknown job state {other}" + ))), + } + } + + /// Fetch grouped schedulable counts directly from persistence. Used to + /// prime and repair the in-memory scheduler. pub async fn ready_counts_grouped(&self) -> Result> { let rows = sqlx::query!( r#" - SELECT kind, library_id, priority, COUNT(*)::bigint AS ready + SELECT + kind, + library_id, + priority, + COUNT(*) FILTER ( + WHERE state = 'ready' AND available_at <= NOW() + )::bigint AS "ready!", + COUNT(*) FILTER ( + WHERE state = 'leased' AND lease_expires_at > NOW() + )::bigint AS "leased!" FROM orchestrator_jobs - WHERE state = 'ready' + WHERE (state = 'ready' AND available_at <= NOW()) + OR (state = 'leased' AND lease_expires_at > NOW()) GROUP BY kind, library_id, priority "# ) @@ -514,12 +617,14 @@ impl PostgresQueueService { let kind = JobKind::from_i16(row.kind)?; let priority = Self::parse_priority(row.priority)?; - let ready = row.ready.unwrap_or(0).max(0i64) as usize; + let ready = row.ready.max(0i64) as usize; + let leased = row.leased.max(0i64) as usize; counts.push(ReadyQueueCount { kind, library_id: LibraryId(row.library_id), priority, ready, + leased, }); } @@ -605,10 +710,14 @@ impl QueueService for PostgresQueueService { "ready" }; - // Fast path: if an active job with the same dedupe_key exists, merge without - // causing a unique violation. This avoids noisy ERROR logs in Postgres. - if let Some(existing) = sqlx::query!( - r#" + // A merge target may become terminal at any point between discovery + // and enrollment. Retry the complete enqueue decision so that race + // creates a new generation rather than returning a terminal handle. + for _enqueue_attempt in 0..4 { + // Fast path: if an active job with the same dedupe_key exists, merge without + // causing a unique violation. This avoids noisy ERROR logs in Postgres. + if let Some(existing) = sqlx::query!( + r#" SELECT id, priority, state, attempts FROM orchestrator_jobs WHERE dedupe_key = $1 @@ -616,100 +725,115 @@ impl QueueService for PostgresQueueService { ORDER BY created_at ASC LIMIT 1 "#, - &dedupe_key, - ) - .fetch_optional(&self.pool) - .await - .map_err(|e| { - MediaError::Internal(format!("enqueue precheck failed: {e}")) - })? { - let existing_uuid = existing.id; - let existing_id = - crate::domain::scan::orchestration::job::JobId(existing_uuid); - let existing_priority = existing.priority; - let existing_state = existing.state; - let existing_attempts = existing.attempts; - // Try to elevate priority if incoming is higher and the job is not leased - if priority_val < existing_priority { - let _ = sqlx::query!( - r#" + &dedupe_key, + ) + .fetch_optional(&self.pool) + .await + .map_err(|e| { + MediaError::Internal(format!("enqueue precheck failed: {e}")) + })? { + let existing_uuid = existing.id; + let existing_id = + crate::domain::scan::orchestration::job::JobId( + existing_uuid, + ); + let existing_priority = existing.priority; + let existing_state = existing.state; + let existing_attempts = existing.attempts; + if !enroll_active_job(&self.pool, correlation_id, existing_id) + .await? + { + continue; + } + // Try to elevate priority if incoming is higher and the job is not leased + if priority_val < existing_priority { + let _ = sqlx::query!( + r#" UPDATE orchestrator_jobs SET priority = $1, available_at = LEAST(available_at, NOW()), updated_at = NOW() WHERE id = $2 AND state IN ('ready','deferred') "#, - priority_val, - existing_uuid - ) - .execute(&self.pool) - .await; - } - if correlation_id.is_some() { - let _ = sqlx::query!( - r#" + priority_val, + existing_uuid + ) + .execute(&self.pool) + .await; + } + if correlation_id.is_some() { + let _ = sqlx::query!( + r#" UPDATE orchestrator_jobs SET correlation_id = COALESCE(correlation_id, $1), updated_at = NOW() WHERE id = $2 AND correlation_id IS NULL "#, - correlation_id, - existing_uuid - ) - .execute(&self.pool) - .await; - } - if existing_state.as_str() != "leased" - && transcript_payload(&request.payload).is_some() - { - replace_pending_transcript_payload( - &self.pool, - existing_uuid, - &payload_json, - ) - .await - .map_err(|err| { - MediaError::Internal(format!( - "transcript merge payload update failed: {err}" - )) - })?; - } - if existing_state.as_str() != "leased" - && let Some(job) = transcript_payload(&request.payload) - && let Err(err) = mark_transcript_queue_status( - &self.pool, - &job, - TranscriptProcessingState::Queued, - existing_attempts, - self.retry_config.max_attempts, - None, - None, - correlation_id, - ) - .await - { - warn!(error = %err, job = %existing_uuid, "failed to mark transcript job queued"); + correlation_id, + existing_uuid + ) + .execute(&self.pool) + .await; + } + if existing_state.as_str() != "leased" + && transcript_payload(&request.payload).is_some() + { + replace_pending_transcript_payload( + &self.pool, + existing_uuid, + &payload_json, + ) + .await + .map_err(|err| { + MediaError::Internal(format!( + "transcript merge payload update failed: {err}" + )) + })?; + } + if existing_state.as_str() != "leased" + && let Some(job) = transcript_payload(&request.payload) + && let Err(err) = mark_transcript_queue_status( + &self.pool, + &job, + TranscriptProcessingState::Queued, + existing_attempts, + self.retry_config.max_attempts, + None, + None, + correlation_id, + ) + .await + { + warn!(error = %err, job = %existing_uuid, "failed to mark transcript job queued"); + } + return Ok(JobHandle::merged( + existing_id, + &request.payload, + request.priority, + )); } - return Ok(JobHandle::merged( - existing_id, - &request.payload, - request.priority, - )); - } - // Attempt insert; rely on partial unique index uq_jobs_dedupe_active. - // We cannot reference a partial unique index in ON CONFLICT directly, so we - // perform a plain INSERT and treat unique violations as merge events. - let insert_res = sqlx::query!( + // Attempt insert; rely on partial unique index uq_jobs_dedupe_active. + // We cannot reference a partial unique index in ON CONFLICT directly, so we + // perform a plain INSERT and treat unique violations as merge events. + let insert_res = sqlx::query!( r#" - INSERT INTO orchestrator_jobs ( - id, library_id, kind, payload, priority, state, - attempts, available_at, lease_owner, lease_id, lease_expires_at, - dedupe_key, dependency_key, correlation_id, last_error, - created_at, updated_at + WITH inserted_job AS ( + INSERT INTO orchestrator_jobs ( + id, library_id, kind, payload, priority, state, + attempts, available_at, lease_owner, lease_id, lease_expires_at, + dedupe_key, dependency_key, correlation_id, last_error, + created_at, updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, 0, NOW(), NULL, NULL, NULL, $7, $8, $9, NULL, NOW(), NOW()) + RETURNING id ) - VALUES ($1, $2, $3, $4, $5, $6, 0, NOW(), NULL, NULL, NULL, $7, $8, $9, NULL, NOW(), NOW()) + INSERT INTO orchestrator_job_enrollments (correlation_id, job_id) + SELECT $9, id + FROM inserted_job + WHERE $9 IS NOT NULL + ON CONFLICT (correlation_id, job_id) DO NOTHING "#, job_id.0, library_id, @@ -724,36 +848,36 @@ impl QueueService for PostgresQueueService { .execute(&self.pool) .await; - match insert_res { - Ok(_) => { - trace!("enqueue accepted new job {}", job_id.0); - if let Some(job) = transcript_payload(&request.payload) - && let Err(err) = mark_transcript_queue_status( - &self.pool, - &job, - TranscriptProcessingState::Queued, - 0, - self.retry_config.max_attempts, - None, - None, - correlation_id, - ) - .await - { - warn!(error = %err, job = %job_id.0, "failed to mark transcript job queued"); + match insert_res { + Ok(_) => { + trace!("enqueue accepted new job {}", job_id.0); + if let Some(job) = transcript_payload(&request.payload) + && let Err(err) = mark_transcript_queue_status( + &self.pool, + &job, + TranscriptProcessingState::Queued, + 0, + self.retry_config.max_attempts, + None, + None, + correlation_id, + ) + .await + { + warn!(error = %err, job = %job_id.0, "failed to mark transcript job queued"); + } + return Ok(JobHandle::accepted( + job_id, + &request.payload, + request.priority, + )); } - return Ok(JobHandle::accepted( - job_id, - &request.payload, - request.priority, - )); - } - Err(sqlx::Error::Database(db_err)) => { - // Unique violation => merge - let code = db_err.code().map(|c| c.to_string()); - if code.as_deref() == Some("23505") { - let existing = sqlx::query!( - r#" + Err(sqlx::Error::Database(db_err)) => { + // Unique violation => merge + let code = db_err.code().map(|c| c.to_string()); + if code.as_deref() == Some("23505") { + let existing = sqlx::query!( + r#" SELECT id, priority, available_at, state, attempts FROM orchestrator_jobs WHERE dedupe_key = $1 @@ -761,25 +885,32 @@ impl QueueService for PostgresQueueService { ORDER BY created_at ASC LIMIT 1 "#, - &dedupe_key, - ) - .fetch_optional(&self.pool) - .await - .map_err(|e| { - MediaError::Internal(format!( - "enqueue conflict lookup failed: {e}" - )) - })?; + &dedupe_key, + ) + .fetch_optional(&self.pool) + .await + .map_err(|e| { + MediaError::Internal(format!( + "enqueue conflict lookup failed: {e}" + )) + })?; - if let Some(row) = existing { - let row_id = row.id; - let row_state = row.state; - let row_attempts = row.attempts; - // Elevate priority if incoming is higher (lower numeric value) - let existing_pri = row.priority; - if priority_val < existing_pri { - let update = sqlx::query!( - r#" + if let Some(row) = existing + && enroll_active_job( + &self.pool, + correlation_id, + JobId(row.id), + ) + .await? + { + let row_id = row.id; + let row_state = row.state; + let row_attempts = row.attempts; + // Elevate priority if incoming is higher (lower numeric value) + let existing_pri = row.priority; + if priority_val < existing_pri { + let update = sqlx::query!( + r#" UPDATE orchestrator_jobs SET priority = $1, available_at = LEAST(available_at, NOW()), @@ -787,37 +918,37 @@ impl QueueService for PostgresQueueService { WHERE id = $2 AND state IN ('ready','deferred') "#, - priority_val, - row_id - ) - .execute(&self.pool) - .await - .map_err(|e| { - MediaError::Internal(format!( - "enqueue merge elevation failed: {e}" - )) - })?; + priority_val, + row_id + ) + .execute(&self.pool) + .await + .map_err(|e| { + MediaError::Internal(format!( + "enqueue merge elevation failed: {e}" + )) + })?; - if update.rows_affected() > 0 { - info!( - "enqueue merged and elevated priority for job {} to {}", - row_id, priority_val - ); + if update.rows_affected() > 0 { + info!( + "enqueue merged and elevated priority for job {} to {}", + row_id, priority_val + ); + } else { + // Likely leased or moved terminal concurrently; best-effort merge only + info!( + "enqueue merge: elevation skipped due to state transition for job {}", + row_id + ); + } } else { - // Likely leased or moved terminal concurrently; best-effort merge only info!( - "enqueue merge: elevation skipped due to state transition for job {}", + "enqueue merged into existing job {} without priority change", row_id ); } - } else { - info!( - "enqueue merged into existing job {} without priority change", - row_id - ); - } - if correlation_id.is_some() { - let _ = sqlx::query!( + if correlation_id.is_some() { + let _ = sqlx::query!( r#" UPDATE orchestrator_jobs SET correlation_id = COALESCE(correlation_id, $1), @@ -830,11 +961,12 @@ impl QueueService for PostgresQueueService { ) .execute(&self.pool) .await; - } - if row_state.as_str() != "leased" - && transcript_payload(&request.payload).is_some() - { - replace_pending_transcript_payload( + } + if row_state.as_str() != "leased" + && transcript_payload(&request.payload) + .is_some() + { + replace_pending_transcript_payload( &self.pool, row_id, &payload_json, @@ -845,45 +977,53 @@ impl QueueService for PostgresQueueService { "transcript merge payload update failed: {err}" )) })?; - } - if row_state.as_str() != "leased" - && let Some(job) = - transcript_payload(&request.payload) - && let Err(err) = mark_transcript_queue_status( - &self.pool, - &job, - TranscriptProcessingState::Queued, - row_attempts, - self.retry_config.max_attempts, - None, - None, - correlation_id, - ) - .await - { - warn!(error = %err, job = %row_id, "failed to mark transcript job queued"); - } - return Ok(JobHandle::merged( - crate::domain::scan::orchestration::job::JobId( - row_id, - ), - &request.payload, - request.priority, - )); - } else { - // No active row found; try a fresh insert once and, on conflict again, return the found ID - let job_id2 = + } + if row_state.as_str() != "leased" + && let Some(job) = + transcript_payload(&request.payload) + && let Err(err) = mark_transcript_queue_status( + &self.pool, + &job, + TranscriptProcessingState::Queued, + row_attempts, + self.retry_config.max_attempts, + None, + None, + correlation_id, + ) + .await + { + warn!(error = %err, job = %row_id, "failed to mark transcript job queued"); + } + return Ok(JobHandle::merged( + crate::domain::scan::orchestration::job::JobId( + row_id, + ), + &request.payload, + request.priority, + )); + } else { + // No active row found; try a fresh insert once and, on conflict again, return the found ID + let job_id2 = crate::domain::scan::orchestration::job::JobId::new( ); - let retry = sqlx::query!( + let retry = sqlx::query!( r#" - INSERT INTO orchestrator_jobs ( - id, library_id, kind, payload, priority, state, - attempts, available_at, lease_owner, lease_id, lease_expires_at, - dedupe_key, dependency_key, correlation_id, - last_error, created_at, updated_at + WITH inserted_job AS ( + INSERT INTO orchestrator_jobs ( + id, library_id, kind, payload, priority, state, + attempts, available_at, lease_owner, lease_id, lease_expires_at, + dedupe_key, dependency_key, correlation_id, + last_error, created_at, updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, 0, NOW(), NULL, NULL, NULL, $7, $8, $9, NULL, NOW(), NOW()) + RETURNING id ) - VALUES ($1, $2, $3, $4, $5, $6, 0, NOW(), NULL, NULL, NULL, $7, $8, $9, NULL, NOW(), NOW()) + INSERT INTO orchestrator_job_enrollments (correlation_id, job_id) + SELECT $9, id + FROM inserted_job + WHERE $9 IS NOT NULL + ON CONFLICT (correlation_id, job_id) DO NOTHING "#, job_id2.0, library_id, @@ -898,13 +1038,13 @@ impl QueueService for PostgresQueueService { .execute(&self.pool) .await; - match retry { - Ok(_) => { - info!( - "enqueue accepted new job {} on retry", - job_id2.0 - ); - if let Some(job) = + match retry { + Ok(_) => { + info!( + "enqueue accepted new job {} on retry", + job_id2.0 + ); + if let Some(job) = transcript_payload(&request.payload) && let Err(err) = mark_transcript_queue_status( @@ -921,21 +1061,21 @@ impl QueueService for PostgresQueueService { { warn!(error = %err, job = %job_id2.0, "failed to mark transcript job queued"); } - return Ok(JobHandle::accepted( - job_id2, - &request.payload, - request.priority, - )); - } - Err(sqlx::Error::Database(db_err2)) - if db_err2 - .code() - .map(|c| c.to_string()) - .as_deref() - == Some("23505") => - { - // Another concurrent inserter won; fetch and return the winner - let winner = sqlx::query!( + return Ok(JobHandle::accepted( + job_id2, + &request.payload, + request.priority, + )); + } + Err(sqlx::Error::Database(db_err2)) + if db_err2 + .code() + .map(|c| c.to_string()) + .as_deref() + == Some("23505") => + { + // Another concurrent inserter won; fetch and return the winner + let winner = sqlx::query!( r#" SELECT id, state, attempts, correlation_id FROM orchestrator_jobs @@ -954,17 +1094,26 @@ impl QueueService for PostgresQueueService { )) })?; - if let Some(w) = winner { - let winner_id = w.id; - let winner_state = w.state; - let winner_attempts = w.attempts; - let winner_correlation_id = - w.correlation_id; - if winner_state.as_str() != "leased" - && transcript_payload(&request.payload) - .is_some() + if let Some(w) = winner + && enroll_active_job( + &self.pool, + correlation_id, + JobId(w.id), + ) + .await? { - replace_pending_transcript_payload( + let winner_id = w.id; + let winner_state = w.state; + let winner_attempts = w.attempts; + let winner_correlation_id = + w.correlation_id; + if winner_state.as_str() != "leased" + && transcript_payload( + &request.payload, + ) + .is_some() + { + replace_pending_transcript_payload( &self.pool, winner_id, &payload_json, @@ -975,8 +1124,8 @@ impl QueueService for PostgresQueueService { "transcript merge payload update failed: {err}" )) })?; - } - if winner_state.as_str() != "leased" + } + if winner_state.as_str() != "leased" && let Some(job) = transcript_payload(&request.payload) && let Err(err) = mark_transcript_queue_status( @@ -993,39 +1142,43 @@ impl QueueService for PostgresQueueService { { warn!(error = %err, job = %winner_id, "failed to mark transcript job queued"); } - return Ok(JobHandle::merged( + return Ok(JobHandle::merged( crate::domain::scan::orchestration::job::JobId( winner_id, ), &request.payload, request.priority, )); - } + } - return Err(MediaError::Internal( - "enqueue conflict retry: could not resolve existing row".into(), - )); - } - Err(e) => { - return Err(MediaError::Internal(format!( - "enqueue retry insert failed: {e}" - ))); + continue; + } + Err(e) => { + return Err(MediaError::Internal(format!( + "enqueue retry insert failed: {e}" + ))); + } } } + } else { + return Err(MediaError::Internal(format!( + "enqueue insert failed: {}", + db_err + ))); } - } else { + } + Err(e) => { return Err(MediaError::Internal(format!( - "enqueue insert failed: {}", - db_err + "enqueue insert failed: {e}" ))); } } - Err(e) => { - return Err(MediaError::Internal(format!( - "enqueue insert failed: {e}" - ))); - } } + + Err(MediaError::Internal( + "enqueue could not linearize after repeated concurrent terminal transitions" + .into(), + )) } async fn enqueue_many( @@ -1047,92 +1200,100 @@ impl QueueService for PostgresQueueService { Option, )> = Vec::new(); - for request in requests { - request.validate()?; - let job_id = crate::domain::scan::orchestration::job::JobId::new(); - let payload_json = - serde_json::to_value(&request.payload).map_err(|e| { - MediaError::Internal(format!( - "failed to serialize job payload: {e}" - )) - })?; - let library_id = request.payload.library_id().to_uuid(); - let kind = request.payload.kind(); - let dedupe_key = request.dedupe_key().to_string(); - let priority_val: i16 = request.priority as u8 as i16; - let correlation_id = request.correlation_id; - let dependency_key = request - .dependency_key - .as_ref() - .map(|key| key.as_str().to_string()); - let state = if dependency_key.is_some() { - "deferred" - } else { - "ready" - }; + 'requests: for request in requests { + for _enqueue_attempt in 0..4 { + request.validate()?; + let job_id = + crate::domain::scan::orchestration::job::JobId::new(); + let payload_json = serde_json::to_value(&request.payload) + .map_err(|e| { + MediaError::Internal(format!( + "failed to serialize job payload: {e}" + )) + })?; + let library_id = request.payload.library_id().to_uuid(); + let kind = request.payload.kind(); + let dedupe_key = request.dedupe_key().to_string(); + let priority_val: i16 = request.priority as u8 as i16; + let correlation_id = request.correlation_id; + let dependency_key = request + .dependency_key + .as_ref() + .map(|key| key.as_str().to_string()); + let state = if dependency_key.is_some() { + "deferred" + } else { + "ready" + }; - // Fast-path merge check inside transaction - if let Some(existing) = sqlx::query!( - r#" + // Fast-path merge check inside transaction + if let Some(existing) = sqlx::query!( + r#" SELECT id, priority, state, attempts, correlation_id FROM orchestrator_jobs WHERE dedupe_key = $1 AND state IN ('ready','deferred','leased') ORDER BY created_at ASC LIMIT 1 + FOR UPDATE "#, - &dedupe_key, - ) - .fetch_optional(&mut *tx) - .await - .map_err(|e| { - MediaError::Internal(format!( - "enqueue_many precheck failed: {e}" - )) - })? { - let existing_uuid = existing.id; - let existing_id = - crate::domain::scan::orchestration::job::JobId( - existing_uuid, - ); - let existing_priority = existing.priority; - let existing_state = existing.state; - let existing_attempts = existing.attempts; - let existing_correlation_id = existing.correlation_id; - if priority_val < existing_priority { - let _ = sqlx::query!( - r#" + &dedupe_key, + ) + .fetch_optional(&mut *tx) + .await + .map_err(|e| { + MediaError::Internal(format!( + "enqueue_many precheck failed: {e}" + )) + })? { + let existing_uuid = existing.id; + let existing_id = + crate::domain::scan::orchestration::job::JobId( + existing_uuid, + ); + let existing_priority = existing.priority; + let existing_state = existing.state; + let existing_attempts = existing.attempts; + let existing_correlation_id = existing.correlation_id; + if !enroll_active_job(&mut *tx, correlation_id, existing_id) + .await? + { + continue; + } + if priority_val < existing_priority { + let _ = sqlx::query!( + r#" UPDATE orchestrator_jobs SET priority = $1, available_at = LEAST(available_at, NOW()), updated_at = NOW() WHERE id = $2 AND state IN ('ready','deferred') "#, - priority_val, - existing_uuid - ) - .execute(&mut *tx) - .await; - } - if correlation_id.is_some() { - let _ = sqlx::query!( - r#" + priority_val, + existing_uuid + ) + .execute(&mut *tx) + .await; + } + if correlation_id.is_some() { + let _ = sqlx::query!( + r#" UPDATE orchestrator_jobs SET correlation_id = COALESCE(correlation_id, $1), updated_at = NOW() WHERE id = $2 AND correlation_id IS NULL "#, - correlation_id, - existing_uuid - ) - .execute(&mut *tx) - .await; - } - if existing_state.as_str() != "leased" - && transcript_payload(&request.payload).is_some() - { - replace_pending_transcript_payload( + correlation_id, + existing_uuid + ) + .execute(&mut *tx) + .await; + } + if existing_state.as_str() != "leased" + && transcript_payload(&request.payload).is_some() + { + replace_pending_transcript_payload( &mut *tx, existing_uuid, &payload_json, @@ -1143,26 +1304,26 @@ impl QueueService for PostgresQueueService { "enqueue_many transcript merge payload update failed: {err}" )) })?; - } - if existing_state.as_str() != "leased" - && let Some(job) = transcript_payload(&request.payload) - { - transcript_status_updates.push(( - job, - existing_attempts, - correlation_id.or(existing_correlation_id), + } + if existing_state.as_str() != "leased" + && let Some(job) = transcript_payload(&request.payload) + { + transcript_status_updates.push(( + job, + existing_attempts, + correlation_id.or(existing_correlation_id), + )); + } + out.push(JobHandle::merged( + existing_id, + &request.payload, + request.priority, )); + continue 'requests; } - out.push(JobHandle::merged( - existing_id, - &request.payload, - request.priority, - )); - continue; - } - // Try insert; merge on unique violation - let insert_res = sqlx::query!( + // Try insert; merge on unique violation + let insert_res = sqlx::query!( r#" INSERT INTO orchestrator_jobs ( id, library_id, kind, payload, priority, state, @@ -1171,6 +1332,7 @@ impl QueueService for PostgresQueueService { created_at, updated_at ) VALUES ($1, $2, $3, $4, $5, $6, 0, NOW(), NULL, NULL, NULL, $7, $8, $9, NULL, NOW(), NOW()) + ON CONFLICT DO NOTHING "#, job_id.0, library_id, @@ -1185,25 +1347,26 @@ impl QueueService for PostgresQueueService { .execute(&mut *tx) .await; - match insert_res { - Ok(_) => { - info!("enqueue_many accepted new job {}", job_id.0); - if let Some(job) = transcript_payload(&request.payload) { - transcript_status_updates.push(( - job, - 0, - correlation_id, + match insert_res { + Ok(result) if result.rows_affected() > 0 => { + info!("enqueue_many accepted new job {}", job_id.0); + enroll_job(&mut *tx, correlation_id, job_id).await?; + if let Some(job) = transcript_payload(&request.payload) + { + transcript_status_updates.push(( + job, + 0, + correlation_id, + )); + } + out.push(JobHandle::accepted( + job_id, + &request.payload, + request.priority, )); + continue 'requests; } - out.push(JobHandle::accepted( - job_id, - &request.payload, - request.priority, - )); - } - Err(sqlx::Error::Database(db_err)) => { - let code = db_err.code().map(|c| c.to_string()); - if code.as_deref() == Some("23505") { + Ok(_) => { let existing = sqlx::query!( r#" SELECT id, priority, available_at, state, attempts, correlation_id @@ -1212,6 +1375,7 @@ impl QueueService for PostgresQueueService { AND state IN ('ready','deferred','leased') ORDER BY created_at ASC LIMIT 1 + FOR UPDATE "#, request.dedupe_key().to_string(), ) @@ -1223,7 +1387,14 @@ impl QueueService for PostgresQueueService { )) })?; - if let Some(row) = existing { + if let Some(row) = existing + && enroll_active_job( + &mut *tx, + correlation_id, + JobId(row.id), + ) + .await? + { let row_id = row.id; let existing_pri = row.priority; let row_state = row.state; @@ -1297,26 +1468,28 @@ impl QueueService for PostgresQueueService { &request.payload, request.priority, )); + continue 'requests; } else { - return Err(MediaError::Internal( - "enqueue_many conflict but no existing job found".into(), - )); + // The conflict winner became terminal before it + // could be enrolled. Retry this request so the next + // iteration inserts a new active generation. + continue; } - } else { - // Unexpected DB error => abort whole batch + } + Err(e) => { drop(tx.rollback().await); return Err(MediaError::Internal(format!( - "enqueue_many insert failed: {db_err}" + "enqueue_many insert failed: {e}" ))); } } - Err(e) => { - drop(tx.rollback().await); - return Err(MediaError::Internal(format!( - "enqueue_many insert failed: {e}" - ))); - } } + + drop(tx.rollback().await); + return Err(MediaError::Internal( + "enqueue_many could not linearize after repeated concurrent terminal transitions" + .into(), + )); } tx.commit().await.map_err(|e| { @@ -1387,27 +1560,12 @@ impl QueueService for PostgresQueueService { ORDER BY available_at, attempts, created_at FOR UPDATE SKIP LOCKED LIMIT 1 - ), fallback AS ( - SELECT id - FROM orchestrator_jobs - WHERE state = 'ready' - AND kind = $1 - AND available_at <= NOW() - AND NOT EXISTS (SELECT 1 FROM next) - ORDER BY available_at, attempts, created_at - FOR UPDATE SKIP LOCKED - LIMIT 1 ) SELECT j.id, j.payload, j.priority, j.attempts, j.available_at, j.dedupe_key, j.dependency_key, j.correlation_id, j.created_at FROM orchestrator_jobs j - JOIN ( - SELECT id FROM next - UNION ALL - SELECT id FROM fallback - LIMIT 1 - ) pick ON pick.id = j.id + JOIN next ON next.id = j.id "#, kind, selector.library_id.as_uuid(), @@ -1649,6 +1807,14 @@ impl QueueService for PostgresQueueService { } async fn complete(&self, lease_id: LeaseId) -> Result<()> { + let _ = self.complete_with_outcome(lease_id).await?; + Ok(()) + } + + async fn complete_with_outcome( + &self, + lease_id: LeaseId, + ) -> Result { let res = sqlx::query!( r#" UPDATE orchestrator_jobs @@ -1668,8 +1834,10 @@ impl QueueService for PostgresQueueService { if res.rows_affected() > 0 { debug!("completed job with lease {:?}", lease_id.0); + Ok(QueueTransitionOutcome::Applied) + } else { + Ok(QueueTransitionOutcome::Missing) } - Ok(()) } async fn fail( @@ -1678,6 +1846,16 @@ impl QueueService for PostgresQueueService { retryable: bool, error: Option, ) -> Result<()> { + let _ = self.fail_with_outcome(lease_id, retryable, error).await?; + Ok(()) + } + + async fn fail_with_outcome( + &self, + lease_id: LeaseId, + retryable: bool, + error: Option, + ) -> Result { let mut tx = self.pool.begin().await.map_err(|e| { MediaError::Internal(format!("begin fail tx failed: {e}")) })?; @@ -1700,7 +1878,7 @@ impl QueueService for PostgresQueueService { let Some(row) = row else { drop(tx); - return Ok(()); + return Ok(FailOutcome::Missing); }; let row_id = row.id; @@ -1814,7 +1992,7 @@ impl QueueService for PostgresQueueService { delay_ms, library_under_pressure ); - Ok(()) + Ok(FailOutcome::RetryScheduled) } else { // Terminal: dead-letter or failed let new_state = if retryable { "dead_letter" } else { "failed" }; @@ -1845,8 +2023,8 @@ impl QueueService for PostgresQueueService { MediaError::Internal(format!("fail tx commit failed: {e}")) })?; - if let Some(job) = transcript_job { - if let Err(err) = mark_transcript_queue_status( + if let Some(job) = transcript_job + && let Err(err) = mark_transcript_queue_status( &self.pool, &job, TranscriptProcessingState::Failed, @@ -1857,16 +2035,21 @@ impl QueueService for PostgresQueueService { row_correlation_id, ) .await - { - warn!(error = %err, job = %row_id, "failed to mark transcript terminal failure status"); - } + { + warn!(error = %err, job = %row_id, "failed to mark transcript terminal failure status"); } warn!( "job {} moved to {} after attempts {}", row_id, new_state, attempts_before ); - Ok(()) + Ok(FailOutcome::Terminal { + state: if retryable { + JobState::DeadLetter + } else { + JobState::Failed + }, + }) } } @@ -1875,6 +2058,15 @@ impl QueueService for PostgresQueueService { lease_id: LeaseId, error: Option, ) -> Result<()> { + let _ = self.dead_letter_with_outcome(lease_id, error).await?; + Ok(()) + } + + async fn dead_letter_with_outcome( + &self, + lease_id: LeaseId, + error: Option, + ) -> Result { let row = sqlx::query!( r#" SELECT id, attempts, payload, correlation_id @@ -1934,8 +2126,10 @@ impl QueueService for PostgresQueueService { } } warn!("job with lease {:?} moved to dead_letter", lease_id.0); + Ok(QueueTransitionOutcome::Applied) + } else { + Ok(QueueTransitionOutcome::Missing) } - Ok(()) } async fn cancel_job(&self, job_id: JobId) -> Result<()> { @@ -2012,6 +2206,66 @@ impl QueueService for PostgresQueueService { Ok(row.count as usize) } + async fn durable_job_states( + &self, + correlation_id: uuid::Uuid, + job_ids: &[JobId], + ) -> Result> { + let job_ids: Vec = + job_ids.iter().map(|job_id| job_id.0).collect(); + let rows = sqlx::query!( + r#" + SELECT job.id, job.kind, job.state, job.attempts, job.payload, + job.dedupe_key, job.correlation_id, job.last_error, + job.created_at, job.updated_at + FROM orchestrator_jobs job + WHERE job.id = ANY($2::uuid[]) + OR EXISTS ( + SELECT 1 + FROM orchestrator_job_enrollments enrollment + WHERE enrollment.correlation_id = $1 + AND enrollment.job_id = job.id + ) + ORDER BY job.created_at ASC, job.id ASC + "#, + correlation_id, + &job_ids, + ) + .fetch_all(&self.pool) + .await + .map_err(|err| { + MediaError::Internal(format!( + "durable job reconciliation query failed: {err}" + )) + })?; + + let mut states = Vec::with_capacity(rows.len()); + for row in rows { + let payload: JobPayload = serde_json::from_value(row.payload) + .map_err(|err| { + MediaError::Internal(format!( + "durable job payload decode failed: {err}" + )) + })?; + let kind = JobKind::from_i16(row.kind)?; + + states.push(DurableJobState { + job_id: JobId(row.id), + kind, + state: Self::parse_state(&row.state)?, + attempts: row.attempts.max(0) as u16, + dedupe_key: row.dedupe_key, + correlation_id: row.correlation_id, + path_key: stable_path_key(&payload), + last_error: row.last_error, + created_at: row.created_at, + updated_at: row.updated_at, + }); + } + + Ok(states) + } + async fn release_dependency( &self, library_id: LibraryId, @@ -2173,6 +2427,35 @@ mod tests { let queue = PostgresQueueService::new(pool.clone()) .await .expect("queue service initializes"); + let missing_lease = LeaseId::new(); + assert_eq!( + queue + .complete_with_outcome(missing_lease) + .await + .expect("missing completion is reported"), + QueueTransitionOutcome::Missing + ); + assert_eq!( + queue + .fail_with_outcome( + missing_lease, + true, + Some("missing retry".into()), + ) + .await + .expect("missing failure is reported"), + FailOutcome::Missing + ); + assert_eq!( + queue + .dead_letter_with_outcome( + missing_lease, + Some("missing dead letter".into()), + ) + .await + .expect("missing dead letter is reported"), + QueueTransitionOutcome::Missing + ); let library_id = LibraryId::new(); let library_root = format!("/queue-dedupe/{}", library_id.as_uuid()); seed_library(&pool, library_id, &library_root) @@ -2231,10 +2514,20 @@ mod tests { .expect("ready dequeue succeeds") .expect("ready job leased"); assert_eq!(ready_lease.job.id, ready.job_id); - queue - .complete(ready_lease.lease_id) - .await - .expect("ready job completed"); + assert_eq!( + queue + .complete_with_outcome(ready_lease.lease_id) + .await + .expect("ready job completed"), + QueueTransitionOutcome::Applied + ); + assert_eq!( + queue + .complete_with_outcome(ready_lease.lease_id) + .await + .expect("repeated completion is reported"), + QueueTransitionOutcome::Missing + ); assert_eq!( active_dedupe_count(&pool, &ready.dedupe_key) .await @@ -2332,23 +2625,52 @@ mod tests { Some(1) ); - queue - .complete(lease.lease_id) + assert_eq!( + queue + .dead_letter_with_outcome( + lease.lease_id, + Some("terminal test".into()), + ) + .await + .expect("leased job dead-lettered"), + QueueTransitionOutcome::Applied + ); + let terminal_race_correlation = uuid::Uuid::now_v7(); + assert!( + !enroll_active_job( + &pool, + Some(terminal_race_correlation), + leased.job_id, + ) .await - .expect("leased job completed"); + .expect("terminal merge enrollment is checked"), + "a terminal lookup target must not satisfy a new enqueue" + ); + let mut terminal_reenqueue_request = EnqueueRequest::new( + JobPriority::P1, + folder_scan_payload( + library_id, + &library_root, + &format!("{library_root}/leased-movie"), + ), + ); + terminal_reenqueue_request.correlation_id = + Some(terminal_race_correlation); let reenqueue_after_terminal = queue - .enqueue(EnqueueRequest::new( - JobPriority::P1, - folder_scan_payload( - library_id, - &library_root, - &format!("{library_root}/leased-movie"), - ), - )) + .enqueue(terminal_reenqueue_request) .await .expect("terminal re-enqueue succeeds"); assert!(reenqueue_after_terminal.accepted); assert_ne!(reenqueue_after_terminal.job_id, leased.job_id); + let terminal_race_jobs = queue + .durable_job_states(terminal_race_correlation, &[]) + .await + .expect("terminal race enrollment is queryable"); + assert_eq!(terminal_race_jobs.len(), 1); + assert_eq!( + terminal_race_jobs[0].job_id, reenqueue_after_terminal.job_id, + "the new correlation must enroll only the runnable generation" + ); assert_eq!( active_dedupe_count(&pool, &leased.dedupe_key) .await @@ -2356,6 +2678,211 @@ mod tests { 1 ); + let retry_payload = folder_scan_payload( + library_id, + &library_root, + &format!("{library_root}/retry-outcome"), + ); + queue + .enqueue(EnqueueRequest::new(JobPriority::P3, retry_payload)) + .await + .expect("retry outcome job enqueue succeeds"); + let retry_lease = queue + .dequeue(DequeueRequest { + kind: JobKind::FolderScan, + worker_id: "retry-outcome-worker".into(), + lease_ttl: chrono::Duration::seconds(30), + selector: Some(QueueSelector { + library_id, + priority: JobPriority::P3, + }), + }) + .await + .expect("retry outcome dequeue succeeds") + .expect("retry outcome job is leased"); + assert_eq!( + queue + .fail_with_outcome( + retry_lease.lease_id, + true, + Some("retryable".into()), + ) + .await + .expect("retry outcome persists"), + FailOutcome::RetryScheduled + ); + + let failed_payload = folder_scan_payload( + library_id, + &library_root, + &format!("{library_root}/failed-outcome"), + ); + queue + .enqueue(EnqueueRequest::new(JobPriority::P2, failed_payload)) + .await + .expect("failed outcome job enqueue succeeds"); + let failed_lease = queue + .dequeue(DequeueRequest { + kind: JobKind::FolderScan, + worker_id: "failed-outcome-worker".into(), + lease_ttl: chrono::Duration::seconds(30), + selector: Some(QueueSelector { + library_id, + priority: JobPriority::P2, + }), + }) + .await + .expect("failed outcome dequeue succeeds") + .expect("failed outcome job is leased"); + assert_eq!( + queue + .fail_with_outcome( + failed_lease.lease_id, + false, + Some("non-retryable".into()), + ) + .await + .expect("failed outcome persists"), + FailOutcome::Terminal { + state: JobState::Failed + } + ); + + let exhausted_payload = folder_scan_payload( + library_id, + &library_root, + &format!("{library_root}/exhausted-outcome"), + ); + queue + .enqueue(EnqueueRequest::new(JobPriority::P3, exhausted_payload)) + .await + .expect("exhausted outcome job enqueue succeeds"); + let exhausted_lease = queue + .dequeue(DequeueRequest { + kind: JobKind::FolderScan, + worker_id: "exhausted-outcome-worker".into(), + lease_ttl: chrono::Duration::seconds(30), + selector: Some(QueueSelector { + library_id, + priority: JobPriority::P3, + }), + }) + .await + .expect("exhausted outcome dequeue succeeds") + .expect("exhausted outcome job is leased"); + sqlx::query("UPDATE orchestrator_jobs SET attempts = $1 WHERE id = $2") + .bind(i32::from(queue.retry_config.max_attempts)) + .bind(exhausted_lease.job.id.0) + .execute(&pool) + .await + .expect("exhausted attempt count seeded"); + assert_eq!( + queue + .fail_with_outcome( + exhausted_lease.lease_id, + true, + Some("retry limit reached".into()), + ) + .await + .expect("exhausted outcome persists"), + FailOutcome::Terminal { + state: JobState::DeadLetter + } + ); + + // A scan that merges onto active work durably enrolls each resolved + // job under the new run without changing or broadly expanding the + // older correlation. + let old_correlation = uuid::Uuid::now_v7(); + let new_correlation = uuid::Uuid::now_v7(); + let lineage_root_payload = folder_scan_payload( + library_id, + &library_root, + &format!("{library_root}/lineage-root"), + ); + let mut old_root_request = + EnqueueRequest::new(JobPriority::P1, lineage_root_payload.clone()); + old_root_request.correlation_id = Some(old_correlation); + let old_root = queue + .enqueue(old_root_request) + .await + .expect("old correlated root enqueue succeeds"); + + let mut merged_root_request = + EnqueueRequest::new(JobPriority::P1, lineage_root_payload); + merged_root_request.correlation_id = Some(new_correlation); + let merged_root = queue + .enqueue(merged_root_request) + .await + .expect("new run merges onto old root"); + assert_eq!(merged_root.merged_into, Some(old_root.job_id)); + + let shared_child_payload = folder_scan_payload( + library_id, + &library_root, + &format!("{library_root}/shared-lineage-child"), + ); + let mut old_shared_child_request = + EnqueueRequest::new(JobPriority::P1, shared_child_payload.clone()); + old_shared_child_request.correlation_id = Some(old_correlation); + let old_shared_child = queue + .enqueue(old_shared_child_request) + .await + .expect("old shared child enqueue succeeds"); + + let mut unrelated_sibling_request = EnqueueRequest::new( + JobPriority::P1, + folder_scan_payload( + library_id, + &library_root, + &format!("{library_root}/unrelated-old-sibling"), + ), + ); + unrelated_sibling_request.correlation_id = Some(old_correlation); + let unrelated_sibling = queue + .enqueue(unrelated_sibling_request) + .await + .expect("unrelated old-correlation sibling enqueue succeeds"); + + let mut accepted_child_request = EnqueueRequest::new( + JobPriority::P1, + folder_scan_payload( + library_id, + &library_root, + &format!("{library_root}/new-lineage-child"), + ), + ); + accepted_child_request.correlation_id = Some(new_correlation); + let mut merged_child_request = + EnqueueRequest::new(JobPriority::P1, shared_child_payload); + merged_child_request.correlation_id = Some(new_correlation); + let descendants = queue + .enqueue_many(vec![accepted_child_request, merged_child_request]) + .await + .expect("new run descendant batch enqueue succeeds"); + assert!(descendants[0].accepted); + assert_eq!(descendants[1].merged_into, Some(old_shared_child.job_id)); + + let reconciled = queue + .durable_job_states(new_correlation, &[]) + .await + .expect("merged run durable reconciliation succeeds"); + let reconciled_ids: std::collections::HashSet<_> = + reconciled.iter().map(|job| job.job_id).collect(); + assert!(reconciled_ids.contains(&old_root.job_id)); + assert!( + reconciled_ids.contains(&descendants[0].job_id), + "accepted descendants must be durably enrolled" + ); + assert!( + reconciled_ids.contains(&old_shared_child.job_id), + "merged descendants must be durably enrolled" + ); + assert!( + !reconciled_ids.contains(&unrelated_sibling.job_id), + "tracked merged root must not enroll unrelated old-correlation siblings" + ); + sqlx::query("DELETE FROM libraries WHERE id = $1") .bind(library_id.to_uuid()) .execute(&pool) diff --git a/crates/ferrex-core/src/domain/scan/orchestration/queue.rs b/crates/ferrex-core/src/domain/scan/orchestration/queue.rs index 964c2beb..a4950fdf 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/queue.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/queue.rs @@ -2,23 +2,87 @@ use std::collections::HashMap; use async_trait::async_trait; use chrono::{DateTime, Utc}; +use ferrex_model::SubjectKey; use serde::{Deserialize, Serialize}; +use uuid::Uuid; use crate::error::Result; use super::{ - job::{EnqueueRequest, JobHandle, JobKind, JobPriority}, + job::{EnqueueRequest, JobHandle, JobId, JobKind, JobPriority, JobState}, lease::{DequeueRequest, JobLease, LeaseRenewal}, }; use crate::types::ids::LibraryId; -/// Aggregated ready-state counts grouped by queue dimensions. +/// Aggregated schedulable state grouped by queue dimensions. +/// +/// `ready` contains only jobs whose backoff has elapsed. `leased` contains +/// leases that have not expired. Together they let the in-memory scheduler +/// rebuild both advertised work and per-library capacity from PostgreSQL. #[derive(Clone, Debug)] pub struct ReadyQueueCount { pub kind: JobKind, pub library_id: LibraryId, pub priority: JobPriority, pub ready: usize, + pub leased: usize, +} + +/// Compact durable job view used to repair in-memory progress after event loss. +/// +/// PostgreSQL remains the authority for lifecycle state; broadcast events are +/// only a low-latency notification path. Consumers select the exact durable +/// enrollments recorded for a correlation and can also provide already-tracked +/// job IDs for legacy runs. This keeps unrelated work that merely shares an old +/// correlation out of the enrolling run. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DurableJobState { + pub job_id: JobId, + pub kind: JobKind, + pub state: JobState, + pub attempts: u16, + pub dedupe_key: String, + pub correlation_id: Option, + pub path_key: Option, + pub last_error: Option, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl DurableJobState { + pub fn is_terminal(&self) -> bool { + matches!( + self.state, + JobState::Completed | JobState::Failed | JobState::DeadLetter + ) + } + + pub fn is_terminal_failure(&self) -> bool { + matches!(self.state, JobState::Failed | JobState::DeadLetter) + } +} + +/// Whether a requested durable queue transition changed the leased row. +/// +/// A missing lease is distinct from an applied transition: callers must not +/// publish a terminal event or terminal actor notification unless the durable +/// transition was applied. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum QueueTransitionOutcome { + Applied, + Missing, +} + +/// Durable result of failing a leased job. +/// +/// Retryable failures may either schedule another attempt or cross the retry +/// limit and become terminal. `Missing` means no leased row was changed and +/// must not be treated as either outcome by runtime accounting. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FailOutcome { + RetryScheduled, + Terminal { state: JobState }, + Missing, } /// Abstracts the queue backend (persistence + scheduling) consumed by the orchestrator service. @@ -35,6 +99,18 @@ pub trait QueueService: Send + Sync { async fn complete(&self, lease_id: super::lease::LeaseId) -> Result<()>; + /// Complete a lease and report whether the durable transition was applied. + /// + /// The default preserves compatibility with queue implementations whose + /// legacy `complete` method already returns an error for a missing lease. + async fn complete_with_outcome( + &self, + lease_id: super::lease::LeaseId, + ) -> Result { + self.complete(lease_id).await?; + Ok(QueueTransitionOutcome::Applied) + } + async fn fail( &self, lease_id: super::lease::LeaseId, @@ -42,23 +118,69 @@ pub trait QueueService: Send + Sync { error: Option, ) -> Result<()>; + /// Fail a lease and distinguish a scheduled retry from a terminal state. + /// + /// Durable implementations that enforce an attempt limit must override + /// this method so a retry request that reaches the limit is reported as a + /// terminal outcome. + async fn fail_with_outcome( + &self, + lease_id: super::lease::LeaseId, + retryable: bool, + error: Option, + ) -> Result { + self.fail(lease_id, retryable, error).await?; + if retryable { + Ok(FailOutcome::RetryScheduled) + } else { + Ok(FailOutcome::Terminal { + state: JobState::Failed, + }) + } + } + async fn dead_letter( &self, lease_id: super::lease::LeaseId, error: Option, ) -> Result<()>; + /// Dead-letter a lease and report whether the durable transition applied. + async fn dead_letter_with_outcome( + &self, + lease_id: super::lease::LeaseId, + error: Option, + ) -> Result { + self.dead_letter(lease_id, error).await?; + Ok(QueueTransitionOutcome::Applied) + } + async fn cancel_job(&self, job_id: super::job::JobId) -> Result<()>; async fn queue_depth(&self, kind: JobKind) -> Result; + /// Load durable job lifecycle state for progress reconciliation. + /// + /// The correlation selects exact durable enrollment records owned by the + /// current run, while `job_ids` preserves compatibility with legacy runs + /// that tracked shared jobs only in memory. Implementations must not broaden + /// the selection to every job that shares an older correlation. + /// Non-durable implementations may retain the empty default. + async fn durable_job_states( + &self, + _correlation_id: Uuid, + _job_ids: &[JobId], + ) -> Result> { + Ok(Vec::new()) + } + async fn release_dependency( &self, library_id: crate::types::LibraryId, dependency_key: &super::job::DependencyKey, ) -> Result; - /// Fetch grouped ready counts for runtime scheduler priming. + /// Fetch grouped ready and active-lease counts for scheduler reconciliation. /// /// Queue implementations without durable startup state can keep the default /// empty snapshot. diff --git a/crates/ferrex-core/src/domain/scan/orchestration/runtime/README.md b/crates/ferrex-core/src/domain/scan/orchestration/runtime/README.md index af9fee28..01a1b458 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/runtime/README.md +++ b/crates/ferrex-core/src/domain/scan/orchestration/runtime/README.md @@ -6,6 +6,7 @@ Key points: - Worker pools per JobKind (FolderScan, MediaAnalyze, MetadataEnrich, IndexUpsert) - Leases: jobs are leased with a TTL and renewed before expiry. - Renewals: default TTL is 30s; renew when half elapsed or when <2s remain (renew_min_margin_ms). +- Execution deadline: each dispatcher attempt is bounded to 30 minutes by default (dispatch_timeout_ms); a timeout stops renewal and becomes a retryable failure. - Housekeeping periodically rescans for expired leases and resurrects eligible jobs. - Actors (library, folder, etc.) are kept resident to accept commands; they are not removed after Start. diff --git a/crates/ferrex-core/src/domain/scan/orchestration/runtime/event_bus.rs b/crates/ferrex-core/src/domain/scan/orchestration/runtime/event_bus.rs index a8d28ea5..dbede859 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/runtime/event_bus.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/runtime/event_bus.rs @@ -86,3 +86,44 @@ impl ScanEventStream for InProcJobEventBus { self.subscribe_scan() } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + domain::scan::orchestration::events::{EventMeta, JobEventPayload}, + types::LibraryId, + }; + use chrono::Utc; + + #[tokio::test] + async fn job_burst_above_256_reports_lag_to_reconciling_consumers() { + let bus = InProcJobEventBus::new(256); + let mut receiver = bus.subscribe(); + let library_id = LibraryId::new(); + + for sequence in 0..300 { + bus.publish(JobEvent { + meta: EventMeta::new( + None, + library_id, + format!("burst:{sequence}"), + None, + ), + payload: JobEventPayload::ThroughputTick { + queue_depths: Vec::new(), + sampled_at: Utc::now(), + }, + }) + .await + .expect("event publish succeeds"); + } + + match receiver.recv().await { + Err(broadcast::error::RecvError::Lagged(skipped)) => { + assert!(skipped >= 44); + } + other => panic!("expected lag after 300 events, got {other:?}"), + } + } +} diff --git a/crates/ferrex-core/src/domain/scan/orchestration/runtime/supervisor.rs b/crates/ferrex-core/src/domain/scan/orchestration/runtime/supervisor.rs index f07749d4..7db8672f 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/runtime/supervisor.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/runtime/supervisor.rs @@ -179,6 +179,12 @@ where self.scheduler.clone() } + /// Remove scheduler counts and reservations after a library's durable jobs + /// have been deleted. + pub async fn forget_library_scheduler_state(&self, library_id: LibraryId) { + self.scheduler.forget_library(library_id).await; + } + pub async fn register_library_actor( &self, library_id: LibraryId, @@ -189,6 +195,33 @@ where Ok(()) } + /// Remove and shut down a library actor so no new work can be accepted for + /// a library that is about to be deleted. + pub async fn unregister_library_actor( + &self, + library_id: LibraryId, + ) -> Result { + let actor = { + let mut guard = self.library_actors.write().await; + guard.remove(&library_id) + }; + + let Some(actor) = actor else { + return Ok(false); + }; + + let shutdown_result = actor + .lock() + .await + .handle_command(LibraryActorCommand::Shutdown) + .await; + if let Err(error) = shutdown_result { + self.library_actors.write().await.insert(library_id, actor); + return Err(error); + } + Ok(true) + } + pub async fn library_actor( &self, library_id: LibraryId, @@ -203,12 +236,20 @@ where } pub async fn start(&self) -> Result<()> { + let recovered_expired = self.queue.scan_expired_leases().await?; + if recovered_expired > 0 { + tracing::info!( + recovered_expired, + "recovered expired leases before scheduler priming" + ); + } self.task_graph .prime_scheduler_from_persistence(self.queue(), self.scheduler()) .await?; self.task_graph .spawn_scheduler_observer( + self.queue(), self.events(), self.scheduler(), self.correlations.clone(), @@ -263,6 +304,7 @@ where self.task_graph .spawn_housekeeper( self.queue(), + self.scheduler(), std::time::Duration::from_millis( self.config.lease.housekeeper_interval_ms, ), @@ -549,14 +591,19 @@ mod tests { DequeueRequest, JobLease, LeaseId, LeaseRenewal, QueueSelector, }; use crate::domain::scan::orchestration::persistence::PostgresQueueService; - use crate::domain::scan::orchestration::queue::ReadyQueueCount; + use crate::domain::scan::orchestration::queue::{ + FailOutcome, QueueTransitionOutcome, ReadyQueueCount, + }; use crate::domain::scan::orchestration::runtime::InProcJobEventBus; use crate::types::ids::LibraryId; use async_trait::async_trait; use sqlx::PgPool; use std::collections::{HashMap, VecDeque}; use std::fmt; - use std::sync::Arc; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; use std::time::Duration; use tokio::sync::Mutex as TokioMutex; use tokio::time; @@ -645,6 +692,28 @@ mod tests { } } + #[derive(Default)] + struct FirstCallNeverReturnsDispatcher { + calls: AtomicUsize, + } + + impl FirstCallNeverReturnsDispatcher { + fn call_count(&self) -> usize { + self.calls.load(Ordering::SeqCst) + } + } + + #[async_trait] + impl JobDispatcher for FirstCallNeverReturnsDispatcher { + async fn dispatch(&self, _lease: &JobLease) -> DispatchStatus { + if self.calls.fetch_add(1, Ordering::SeqCst) == 0 { + std::future::pending::().await + } else { + DispatchStatus::Success + } + } + } + async fn record_dispatch_start( state: &TokioMutex, lease: &JobLease, @@ -691,10 +760,27 @@ mod tests { dead_letters: usize, renewals: usize, expired_scans: usize, + complete_outcomes: VecDeque, + fail_outcomes: VecDeque, + dead_letter_outcomes: VecDeque, + } + + #[derive(Clone, Copy, Debug)] + enum ScriptedTransitionOutcome { + Applied, + Error, + } + + #[derive(Clone, Copy, Debug)] + enum ScriptedFailOutcome { + RetryScheduled, + Terminal(JobState), } #[derive(Clone, Copy, Debug, Default)] struct RecordingQueueStats { + ready: usize, + leased: usize, completed: usize, failures: usize, dead_letters: usize, @@ -718,12 +804,30 @@ mod tests { async fn stats(&self) -> RecordingQueueStats { let state = self.state.lock().await; RecordingQueueStats { + ready: state.ready.len(), + leased: state.leased.len(), completed: state.completed, failures: state.failures, dead_letters: state.dead_letters, renewals: state.renewals, } } + + async fn script_complete(&self, outcome: ScriptedTransitionOutcome) { + self.state.lock().await.complete_outcomes.push_back(outcome); + } + + async fn script_fail(&self, outcome: ScriptedFailOutcome) { + self.state.lock().await.fail_outcomes.push_back(outcome); + } + + async fn script_dead_letter(&self, outcome: ScriptedTransitionOutcome) { + self.state + .lock() + .await + .dead_letter_outcomes + .push_back(outcome); + } } #[async_trait] @@ -803,6 +907,23 @@ mod tests { Ok(()) } + async fn complete_with_outcome( + &self, + lease_id: LeaseId, + ) -> Result { + let scripted = + self.state.lock().await.complete_outcomes.pop_front(); + match scripted.unwrap_or(ScriptedTransitionOutcome::Applied) { + ScriptedTransitionOutcome::Applied => { + self.complete(lease_id).await?; + Ok(QueueTransitionOutcome::Applied) + } + ScriptedTransitionOutcome::Error => Err(MediaError::Internal( + "scripted complete persistence failure".into(), + )), + } + } + async fn fail( &self, lease_id: LeaseId, @@ -827,6 +948,43 @@ mod tests { Ok(()) } + async fn fail_with_outcome( + &self, + lease_id: LeaseId, + retryable: bool, + error: Option, + ) -> Result { + let scripted = self.state.lock().await.fail_outcomes.pop_front(); + let outcome = scripted.unwrap_or(if retryable { + ScriptedFailOutcome::RetryScheduled + } else { + ScriptedFailOutcome::Terminal(JobState::Failed) + }); + + match outcome { + ScriptedFailOutcome::RetryScheduled => { + self.fail(lease_id, true, error).await?; + Ok(FailOutcome::RetryScheduled) + } + ScriptedFailOutcome::Terminal(state) => { + let mut queue_state = self.state.lock().await; + queue_state.leased.remove(&lease_id).ok_or_else(|| { + MediaError::NotFound(format!( + "lease {} not found", + lease_id.0 + )) + })?; + queue_state.failures += 1; + match state { + JobState::Completed => queue_state.completed += 1, + JobState::DeadLetter => queue_state.dead_letters += 1, + _ => {} + } + Ok(FailOutcome::Terminal { state }) + } + } + } + async fn dead_letter( &self, lease_id: LeaseId, @@ -840,6 +998,24 @@ mod tests { Ok(()) } + async fn dead_letter_with_outcome( + &self, + lease_id: LeaseId, + error: Option, + ) -> Result { + let scripted = + self.state.lock().await.dead_letter_outcomes.pop_front(); + match scripted.unwrap_or(ScriptedTransitionOutcome::Applied) { + ScriptedTransitionOutcome::Applied => { + self.dead_letter(lease_id, error).await?; + Ok(QueueTransitionOutcome::Applied) + } + ScriptedTransitionOutcome::Error => Err(MediaError::Internal( + "scripted dead-letter persistence failure".into(), + )), + } + } + async fn cancel_job(&self, _job_id: JobId) -> Result<()> { Ok(()) } @@ -865,25 +1041,41 @@ mod tests { async fn ready_counts_grouped(&self) -> Result> { let state = self.state.lock().await; - let mut grouped: HashMap<(JobKind, LibraryId, JobPriority), usize> = - HashMap::new(); + let mut grouped: HashMap< + (JobKind, LibraryId, JobPriority), + (usize, usize), + > = HashMap::new(); for job in &state.ready { - *grouped + grouped .entry(( job.payload.kind(), job.payload.library_id(), job.priority, )) - .or_default() += 1; + .or_default() + .0 += 1; + } + for lease in state.leased.values() { + grouped + .entry(( + lease.job.payload.kind(), + lease.job.payload.library_id(), + lease.job.priority, + )) + .or_default() + .1 += 1; } Ok(grouped .into_iter() - .map(|((kind, library_id, priority), ready)| ReadyQueueCount { - kind, - library_id, - priority, - ready, + .map(|((kind, library_id, priority), (ready, leased))| { + ReadyQueueCount { + kind, + library_id, + priority, + ready, + leased, + } }) .collect()) } @@ -916,6 +1108,54 @@ mod tests { JobRecord::new(payload, priority) } + fn single_scan_worker_config() -> OrchestratorConfig { + let mut config = OrchestratorConfig::default(); + config.queue.max_parallel_scans = 1; + config.queue.max_parallel_series_resolve = 0; + config.queue.max_parallel_analyses = 0; + config.queue.max_parallel_metadata = 0; + config.queue.max_parallel_index = 0; + config.queue.max_parallel_image_fetch = 0; + config.queue.max_parallel_transcript_extract = 0; + config.queue.default_library_cap = 1; + config.budget.library_scan_limit = 1; + config.lease.housekeeper_interval_ms = 10_000; + config + } + + async fn wait_for_worker_accounting_to_release( + runtime: &OrchestratorRuntime< + RecordingQueue, + InProcJobEventBus, + InMemoryBudget, + >, + budget: &InMemoryBudget, + library_id: LibraryId, + ) { + time::timeout(Duration::from_secs(2), async { + loop { + let scheduler_idle = runtime + .scheduler() + .snapshot() + .await + .get(&library_id) + .copied() + .unwrap_or_default() + == (0, 0); + let (in_use, _) = budget + .utilization(WorkloadType::LibraryScan) + .await + .expect("budget utilization"); + if scheduler_idle && in_use == 0 { + break; + } + time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("worker releases scheduler and budget accounting"); + } + #[tokio::test] async fn task_graph_tracks_startup_tasks_and_shutdown_drains_them() { let mut config = OrchestratorConfig::default(); @@ -974,7 +1214,7 @@ mod tests { config.lease.lease_ttl_secs = 1; config.lease.renew_at_fraction = 0.0; config.lease.renew_min_margin_ms = 950; - config.lease.housekeeper_interval_ms = 10_000; + config.lease.housekeeper_interval_ms = 10; let queue = Arc::new(RecordingQueue::with_ready(vec![ folder_scan_record(library_id, "/library/movie-a", JobPriority::P1), @@ -1118,6 +1358,327 @@ mod tests { runtime.shutdown().await.expect("runtime shutdown succeeds"); } + #[tokio::test] + async fn dispatch_timeout_retries_never_returning_job_and_releases_accounting() + { + let library_id = LibraryId::new(); + let mut config = OrchestratorConfig::default(); + config.queue.max_parallel_scans = 1; + config.queue.max_parallel_series_resolve = 0; + config.queue.max_parallel_analyses = 0; + config.queue.max_parallel_metadata = 0; + config.queue.max_parallel_index = 0; + config.queue.max_parallel_image_fetch = 0; + config.queue.max_parallel_transcript_extract = 0; + config.queue.default_library_cap = 1; + config.budget.library_scan_limit = 1; + config.lease.dispatch_timeout_ms = 25; + config.lease.housekeeper_interval_ms = 10; + + let queue = + Arc::new(RecordingQueue::with_ready(vec![folder_scan_record( + library_id, + "/library/timeout-movie", + JobPriority::P1, + )])); + let events = Arc::new(InProcJobEventBus::new(64)); + let mut job_events = events.subscribe(); + let budget = Arc::new(InMemoryBudget::new(config.budget.clone())); + let dispatcher = Arc::new(FirstCallNeverReturnsDispatcher::default()); + + let runtime = OrchestratorRuntimeBuilder::new(config) + .with_queue(queue.clone()) + .with_events(events) + .with_budget(budget.clone()) + .with_dispatcher(dispatcher.clone()) + .build() + .expect("runtime build"); + + runtime.start().await.expect("runtime start"); + + time::timeout(Duration::from_secs(2), async { + loop { + let stats = queue.stats().await; + let scheduler_idle = runtime + .scheduler() + .snapshot() + .await + .get(&library_id) + .copied() + .unwrap_or_default() + == (0, 0); + if stats.failures == 1 && stats.completed == 1 && scheduler_idle + { + break; + } + time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("timed-out dispatch is retried and then completes"); + + assert_eq!(dispatcher.call_count(), 2); + let (in_use, _) = budget + .utilization(WorkloadType::LibraryScan) + .await + .expect("budget utilization"); + assert_eq!(in_use, 0, "timeout releases the workload budget"); + + let mut saw_retryable_timeout = false; + while let Ok(event) = job_events.try_recv() { + if matches!( + event.payload, + JobEventPayload::Failed { + retryable: true, + .. + } + ) { + saw_retryable_timeout = true; + } + } + assert!( + saw_retryable_timeout, + "timeout publishes a retryable failure event" + ); + + runtime.shutdown().await.expect("runtime shutdown succeeds"); + } + + #[tokio::test] + async fn complete_persistence_failure_does_not_publish_terminal_success() { + let library_id = LibraryId::new(); + let config = single_scan_worker_config(); + let queue = + Arc::new(RecordingQueue::with_ready(vec![folder_scan_record( + library_id, + "/library/complete-persistence-failure", + JobPriority::P1, + )])); + queue + .script_complete(ScriptedTransitionOutcome::Error) + .await; + let events = Arc::new(InProcJobEventBus::new(64)); + let mut job_events = events.subscribe(); + let budget = Arc::new(InMemoryBudget::new(config.budget.clone())); + let dispatcher = Arc::new(TestDispatcher::new(Duration::ZERO)); + + let runtime = OrchestratorRuntimeBuilder::new(config) + .with_queue(queue.clone()) + .with_events(events) + .with_budget(budget.clone()) + .with_dispatcher(dispatcher) + .build() + .expect("runtime build"); + runtime.start().await.expect("runtime start"); + + let mut payloads = Vec::new(); + time::timeout(Duration::from_secs(2), async { + loop { + let event = job_events + .recv() + .await + .expect("job event bus remains open"); + let done = matches!( + &event.payload, + JobEventPayload::Failed { + retryable: true, + .. + } + ); + payloads.push(event.payload); + if done { + break; + } + } + }) + .await + .expect("persistence uncertainty is published"); + + wait_for_worker_accounting_to_release( + &runtime, + budget.as_ref(), + library_id, + ) + .await; + time::sleep(Duration::from_millis(25)).await; + while let Ok(event) = job_events.try_recv() { + payloads.push(event.payload); + } + + let stats = queue.stats().await; + assert_eq!(stats.ready, 0); + assert_eq!( + stats.leased, 1, + "failed persistence keeps the lease durable" + ); + assert_eq!(stats.completed, 0); + assert!(!payloads.iter().any(|payload| matches!( + payload, + JobEventPayload::Completed { .. } + ))); + + runtime.shutdown().await.expect("runtime shutdown succeeds"); + } + + #[tokio::test] + async fn dead_letter_persistence_failure_does_not_publish_terminal_event() { + let library_id = LibraryId::new(); + let config = single_scan_worker_config(); + let queue = + Arc::new(RecordingQueue::with_ready(vec![folder_scan_record( + library_id, + "/library/dead-letter-persistence-failure", + JobPriority::P1, + )])); + queue + .script_dead_letter(ScriptedTransitionOutcome::Error) + .await; + let events = Arc::new(InProcJobEventBus::new(64)); + let mut job_events = events.subscribe(); + let budget = Arc::new(InMemoryBudget::new(config.budget.clone())); + let dispatcher = Arc::new(ScriptedDispatcher::new( + Duration::ZERO, + vec![DispatchStatus::DeadLetter { + error: "terminal dispatch failure".into(), + }], + )); + + let runtime = OrchestratorRuntimeBuilder::new(config) + .with_queue(queue.clone()) + .with_events(events) + .with_budget(budget.clone()) + .with_dispatcher(dispatcher) + .build() + .expect("runtime build"); + runtime.start().await.expect("runtime start"); + + let mut payloads = Vec::new(); + time::timeout(Duration::from_secs(2), async { + loop { + let event = job_events + .recv() + .await + .expect("job event bus remains open"); + let done = matches!( + &event.payload, + JobEventPayload::Failed { + retryable: true, + .. + } + ); + payloads.push(event.payload); + if done { + break; + } + } + }) + .await + .expect("persistence uncertainty is published"); + + wait_for_worker_accounting_to_release( + &runtime, + budget.as_ref(), + library_id, + ) + .await; + time::sleep(Duration::from_millis(25)).await; + while let Ok(event) = job_events.try_recv() { + payloads.push(event.payload); + } + + let stats = queue.stats().await; + assert_eq!(stats.ready, 0); + assert_eq!( + stats.leased, 1, + "failed persistence keeps the lease durable" + ); + assert_eq!(stats.dead_letters, 0); + assert!(!payloads.iter().any(|payload| matches!( + payload, + JobEventPayload::DeadLettered { .. } + ))); + + runtime.shutdown().await.expect("runtime shutdown succeeds"); + } + + #[tokio::test] + async fn retry_at_attempt_limit_publishes_durable_dead_letter_outcome() { + let library_id = LibraryId::new(); + let config = single_scan_worker_config(); + let queue = + Arc::new(RecordingQueue::with_ready(vec![folder_scan_record( + library_id, + "/library/retry-attempt-limit", + JobPriority::P1, + )])); + queue + .script_fail(ScriptedFailOutcome::Terminal(JobState::DeadLetter)) + .await; + let events = Arc::new(InProcJobEventBus::new(64)); + let mut job_events = events.subscribe(); + let budget = Arc::new(InMemoryBudget::new(config.budget.clone())); + let dispatcher = Arc::new(ScriptedDispatcher::new( + Duration::ZERO, + vec![DispatchStatus::Retry { + error: "last retry failed".into(), + }], + )); + + let runtime = OrchestratorRuntimeBuilder::new(config) + .with_queue(queue.clone()) + .with_events(events) + .with_budget(budget.clone()) + .with_dispatcher(dispatcher) + .build() + .expect("runtime build"); + runtime.start().await.expect("runtime start"); + + let mut payloads = Vec::new(); + time::timeout(Duration::from_secs(2), async { + loop { + let event = job_events + .recv() + .await + .expect("job event bus remains open"); + let done = matches!( + &event.payload, + JobEventPayload::DeadLettered { .. } + ); + payloads.push(event.payload); + if done { + break; + } + } + }) + .await + .expect("max-attempt retry publishes dead-letter event"); + + wait_for_worker_accounting_to_release( + &runtime, + budget.as_ref(), + library_id, + ) + .await; + while let Ok(event) = job_events.try_recv() { + payloads.push(event.payload); + } + + let stats = queue.stats().await; + assert_eq!(stats.ready, 0, "terminal retry is not re-advertised"); + assert_eq!(stats.leased, 0); + assert_eq!(stats.failures, 1); + assert_eq!(stats.dead_letters, 1); + assert!(!payloads.iter().any(|payload| matches!( + payload, + JobEventPayload::Failed { + retryable: true, + .. + } + ))); + + runtime.shutdown().await.expect("runtime shutdown succeeds"); + } + #[tokio::test] async fn scheduler_observer_skips_ready_for_merged_events() { let database_url = match std::env::var("DATABASE_URL") { @@ -1219,7 +1780,8 @@ mod tests { "enqueued event should seed correlation cache", ); - if let Some(reservation) = scheduler.reserve().await { + if let Some(reservation) = scheduler.reserve(JobKind::FolderScan).await + { scheduler.confirm(reservation.id).await; scheduler.record_completed(library_id).await; } diff --git a/crates/ferrex-core/src/domain/scan/orchestration/runtime/task_graph.rs b/crates/ferrex-core/src/domain/scan/orchestration/runtime/task_graph.rs index dc9e39b1..2ea9c38a 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/runtime/task_graph.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/runtime/task_graph.rs @@ -7,6 +7,7 @@ use super::{JobEventStream, ScanEventStream}; use crate::domain::scan::actors::{ LibraryActor, LibraryActorCommand, StartMode, }; +use crate::domain::scan::orchestration::context::FolderScanContext; use crate::domain::scan::orchestration::{ budget::{WorkloadBudget, WorkloadType}, config::LeaseConfig, @@ -18,11 +19,13 @@ use crate::domain::scan::orchestration::{ ScanSeedSummary, stable_path_key, }, job::{ - DedupeKey, EnqueueRequest, FolderScanJob, JobKind, JobPayload, - JobPriority, ScanReason, + DedupeKey, EnqueueRequest, FolderScanJob, JobId, JobKind, JobPayload, + JobPriority, JobState, ScanReason, + }, + lease::{DequeueRequest, JobLease, LeaseRenewal, QueueSelector}, + queue::{ + FailOutcome, LeaseExpiryScanner, QueueService, QueueTransitionOutcome, }, - lease::{DequeueRequest, LeaseRenewal, QueueSelector}, - queue::{LeaseExpiryScanner, QueueService}, scheduler::{ReadyCountEntry, WeightedFairScheduler}, }; use crate::{ @@ -55,6 +58,291 @@ struct RuntimeTaskHandle { handle: tokio::task::JoinHandle<()>, } +#[derive(Debug)] +enum WorkerTransition { + Completed, + RetryScheduled { + error: String, + }, + TerminalFailed { + error: String, + }, + DeadLettered { + error: String, + }, + Missing { + operation: &'static str, + error: String, + }, + PersistenceError { + operation: &'static str, + error: String, + }, +} + +impl WorkerTransition { + fn is_terminal(&self) -> bool { + matches!( + self, + Self::Completed + | Self::TerminalFailed { .. } + | Self::DeadLettered { .. } + ) + } +} + +fn worker_transition_from_complete( + outcome: Result, +) -> WorkerTransition { + match outcome { + Ok(QueueTransitionOutcome::Applied) => WorkerTransition::Completed, + Ok(QueueTransitionOutcome::Missing) => WorkerTransition::Missing { + operation: "complete", + error: "durable completion was not applied because the lease was missing" + .into(), + }, + Err(err) => WorkerTransition::PersistenceError { + operation: "complete", + error: format!("failed to persist job completion: {err}"), + }, + } +} + +fn worker_transition_from_fail( + outcome: Result, + execution_error: String, + operation: &'static str, +) -> WorkerTransition { + match outcome { + Ok(FailOutcome::RetryScheduled) => WorkerTransition::RetryScheduled { + error: execution_error, + }, + Ok(FailOutcome::Terminal { + state: JobState::DeadLetter, + }) => WorkerTransition::DeadLettered { + error: execution_error, + }, + Ok(FailOutcome::Terminal { + state: JobState::Failed, + }) => WorkerTransition::TerminalFailed { + error: execution_error, + }, + Ok(FailOutcome::Terminal { + state: JobState::Completed, + }) => WorkerTransition::Completed, + Ok(FailOutcome::Terminal { state }) => { + WorkerTransition::PersistenceError { + operation, + error: format!( + "queue reported non-terminal state {state:?} as terminal; original error: {execution_error}" + ), + } + } + Ok(FailOutcome::Missing) => WorkerTransition::Missing { + operation, + error: format!( + "durable failure transition was not applied because the lease was missing; original error: {execution_error}" + ), + }, + Err(err) => WorkerTransition::PersistenceError { + operation, + error: format!( + "failed to persist job failure: {err}; original error: {execution_error}" + ), + }, + } +} + +fn worker_transition_from_dead_letter( + outcome: Result, + execution_error: String, +) -> WorkerTransition { + match outcome { + Ok(QueueTransitionOutcome::Applied) => WorkerTransition::DeadLettered { + error: execution_error, + }, + Ok(QueueTransitionOutcome::Missing) => WorkerTransition::Missing { + operation: "dead_letter", + error: format!( + "durable dead-letter transition was not applied because the lease was missing; original error: {execution_error}" + ), + }, + Err(err) => WorkerTransition::PersistenceError { + operation: "dead_letter", + error: format!( + "failed to persist job dead-letter: {err}; original error: {execution_error}" + ), + }, + } +} + +#[allow(clippy::too_many_arguments)] +async fn finalize_worker_transition( + transition: WorkerTransition, + lease: &JobLease, + reserved_library_id: LibraryId, + scheduler: &WeightedFairScheduler, + events: &E, + correlations: &CorrelationCache, + mailbox_tx: &Arc< + Mutex>>, + >, +) where + E: ScanEventBus + ?Sized, +{ + let job_id = lease.job.id; + let job_kind = lease.job.payload.kind(); + let job_priority = lease.job.priority; + let library_id = lease.job.payload.library_id(); + let dedupe_key: DedupeKey = lease.job.payload.dedupe_key(); + let terminal = transition.is_terminal(); + + let (payload, command) = match transition { + WorkerTransition::Completed => ( + JobEventPayload::Completed { + job_id, + kind: job_kind, + priority: job_priority, + }, + LibraryActorCommand::JobCompleted { + job_id, + dedupe_key: dedupe_key.clone(), + }, + ), + WorkerTransition::RetryScheduled { error } => ( + JobEventPayload::Failed { + job_id, + kind: job_kind, + priority: job_priority, + retryable: true, + }, + LibraryActorCommand::JobFailed { + job_id, + dedupe_key: dedupe_key.clone(), + retryable: true, + error: Some(error), + }, + ), + WorkerTransition::TerminalFailed { error } => ( + JobEventPayload::Failed { + job_id, + kind: job_kind, + priority: job_priority, + retryable: false, + }, + LibraryActorCommand::JobFailed { + job_id, + dedupe_key: dedupe_key.clone(), + retryable: false, + error: Some(error), + }, + ), + WorkerTransition::DeadLettered { error } => ( + JobEventPayload::DeadLettered { + job_id, + kind: job_kind, + priority: job_priority, + }, + LibraryActorCommand::JobFailed { + job_id, + dedupe_key: dedupe_key.clone(), + retryable: false, + error: Some(error), + }, + ), + WorkerTransition::Missing { operation, error } => { + tracing::warn!( + job = %job_id.0, + lease = %lease.lease_id.0, + operation, + "durable queue transition was not applied because the lease was missing" + ); + ( + JobEventPayload::Failed { + job_id, + kind: job_kind, + priority: job_priority, + retryable: true, + }, + LibraryActorCommand::JobFailed { + job_id, + dedupe_key: dedupe_key.clone(), + retryable: true, + error: Some(error), + }, + ) + } + WorkerTransition::PersistenceError { operation, error } => { + tracing::error!( + job = %job_id.0, + lease = %lease.lease_id.0, + operation, + error = %error, + "durable queue transition failed" + ); + ( + JobEventPayload::Failed { + job_id, + kind: job_kind, + priority: job_priority, + retryable: true, + }, + LibraryActorCommand::JobFailed { + job_id, + dedupe_key: dedupe_key.clone(), + retryable: true, + error: Some(error), + }, + ) + } + }; + + let correlation_id = if terminal { + correlations + .take_persisted_or_generate(job_id, lease.job.correlation_id) + .await + } else { + correlations + .fetch_persisted_or_generate(job_id, lease.job.correlation_id) + .await + }; + let event = JobEvent::from_job( + Some(correlation_id), + library_id, + lease.job.dedupe_key.clone(), + stable_path_key(&lease.job.payload), + payload, + ); + if let Err(err) = events.publish(event).await { + tracing::error!(job = %job_id.0, "publish worker transition event failed: {err}"); + } + + if terminal { + scheduler.record_completed(reserved_library_id).await; + } else { + scheduler.release(reserved_library_id).await; + } + + let sender_opt = { + let guard = mailbox_tx.lock().await; + guard.clone() + }; + if let Some(sender) = sender_opt + && let Err(err) = sender + .send(OrchestratorCommand::Library { + library_id, + command, + completion: None, + }) + .await + { + tracing::warn!( + job = %job_id.0, + "failed to send library actor notification: {err}" + ); + } +} + impl fmt::Debug for RuntimeTaskHandle { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RuntimeTaskHandle") @@ -76,6 +364,135 @@ pub(super) struct RuntimeTaskGraph { tasks: Mutex>, } +fn folder_scan_request_from_discovery( + context: &FolderScanContext, + reason: ScanReason, + correlation_id: Option, + durable_job_id: Option, +) -> Option { + if durable_job_id.is_some() { + return None; + } + let job = FolderScanJob { + context: context.clone(), + scan_reason: reason, + enqueue_time: chrono::Utc::now(), + device_id: None, + }; + let priority = match reason { + ScanReason::HotChange | ScanReason::WatcherOverflow => JobPriority::P0, + ScanReason::UserRequested | ScanReason::BulkSeed => JobPriority::P1, + ScanReason::MaintenanceSweep => JobPriority::P2, + }; + let mut request = + EnqueueRequest::new(priority, JobPayload::FolderScan(job)); + request.correlation_id = correlation_id; + Some(request) +} + +async fn reconcile_scheduler_ready( + queue: &Q, + scheduler: &WeightedFairScheduler, +) -> Result +where + Q: QueueService + ?Sized, +{ + let counts = queue.ready_counts_grouped().await?; + let ready_total = counts.iter().map(|count| count.ready).sum(); + scheduler + .reconcile_ready_absolute(counts.into_iter().map(|count| { + ReadyCountEntry { + kind: count.kind, + library_id: count.library_id, + priority: count.priority, + count: count.ready, + leased: count.leased, + } + })) + .await; + Ok(ready_total) +} + +async fn observe_scheduler_events( + queue: Arc, + mut job_rx: tokio::sync::broadcast::Receiver, + scheduler: WeightedFairScheduler, + correlations: CorrelationCache, + shutdown: CancellationToken, +) where + Q: QueueService + ?Sized, +{ + loop { + tokio::select! { + _ = shutdown.cancelled() => { + tracing::info!("Scheduler observer shutting down"); + break; + } + event = job_rx.recv() => match event { + Ok(event) => { + match event.payload { + JobEventPayload::Enqueued { + job_id, + kind, + priority, + } => { + correlations + .remember(job_id, event.meta.correlation_id) + .await; + scheduler + .record_enqueued( + kind, + event.meta.library_id, + priority, + ) + .await; + } + JobEventPayload::Merged { + existing_job_id, + merged_job_id, + .. + } => { + correlations + .remember_if_absent( + existing_job_id, + event.meta.correlation_id, + ) + .await; + if merged_job_id != existing_job_id { + correlations + .remember_if_absent( + merged_job_id, + event.meta.correlation_id, + ) + .await; + } + } + _ => {} + } + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => break, + Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { + tracing::warn!( + "scheduler observer lagged, skipped {skipped} events; reconciling durable ready counts" + ); + match reconcile_scheduler_ready(queue.as_ref(), &scheduler) + .await + { + Ok(ready_total) => tracing::info!( + ready_total, + "scheduler ready counts reconciled after event lag" + ), + Err(err) => tracing::error!( + error = %err, + "scheduler ready reconciliation failed after event lag" + ), + } + } + } + } + } +} + impl fmt::Debug for RuntimeTaskGraph { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let task_count = self.try_task_count(); @@ -127,16 +544,7 @@ impl RuntimeTaskGraph { where Q: QueueService + ?Sized, { - let counts = queue.ready_counts_grouped().await?; - scheduler - .record_ready_bulk(counts.into_iter().map(|count| { - ReadyCountEntry { - library_id: count.library_id, - priority: count.priority, - count: count.ready, - } - })) - .await; + reconcile_scheduler_ready(queue.as_ref(), &scheduler).await?; Ok(()) } @@ -151,19 +559,6 @@ impl RuntimeTaskGraph { let mut domain_rx = events.subscribe_scan(); let shutdown = self.shutdown_token.clone(); - // Helper mirrors dispatcher priority mapping. - fn priority_for_reason(reason: &ScanReason) -> JobPriority { - match reason { - ScanReason::HotChange | ScanReason::WatcherOverflow => { - JobPriority::P0 - } - ScanReason::UserRequested | ScanReason::BulkSeed => { - JobPriority::P1 - } - ScanReason::MaintenanceSweep => JobPriority::P2, - } - } - self.spawn_task(RuntimeTaskKind::DomainEventRouter, async move { loop { tokio::select! { @@ -172,19 +567,23 @@ impl RuntimeTaskGraph { break; } evt = domain_rx.recv() => match evt { - Ok(ScanEvent::FolderDiscovered { context, reason }) => { - let job = FolderScanJob { - context: *context.clone(), - scan_reason: reason, - enqueue_time: chrono::Utc::now(), - device_id: None, - }; - let payload = JobPayload::FolderScan(job); - let priority = priority_for_reason(&reason); - let request = EnqueueRequest::new(priority, payload); + Ok(ScanEvent::FolderDiscovered { + context, + reason, + correlation_id, + durable_job_id, + }) => { + let request = folder_scan_request_from_discovery( + &context, + reason, + correlation_id, + durable_job_id, + ); - if let Err(err) = enqueuer.enqueue(request).await { - tracing::warn!(target: "scan::router", error = %err, folder = %context.folder_path_norm(), "failed to enqueue FolderScan from FolderDiscovered"); + if let Some(request) = request + && let Err(err) = enqueuer.enqueue(request).await + { + tracing::warn!(target: "scan::router", error = %err, folder = %context.folder_path_norm(), "failed to enqueue legacy FolderScan from FolderDiscovered"); } } Ok(_) => { /* ignore other domain events */ } @@ -286,14 +685,56 @@ impl RuntimeTaskGraph { }) .count(); - if !batch.is_empty() { - enqueuer.enqueue_many(batch).await.map_err( - |err| { - tracing::warn!(target: "scan::mailbox", error = %err, "failed to enqueue scan batch from actor request"); - err - }, - )?; - } + // LibraryActor admission marks folder paths + // active before persistence is attempted. If + // enqueue fails, release those provisional + // records so the retained watcher batch (or a + // retried seed command) can be admitted again. + let provisional_folder_jobs: Vec<_> = batch + .iter() + .filter(|request| { + matches!(request.payload, JobPayload::FolderScan(_)) + }) + .map(|request| request.dedupe_key()) + .collect(); + + let handles = if batch.is_empty() { + Vec::new() + } else { + match enqueuer.enqueue_many(batch).await { + Ok(handles) => handles, + Err(err) => { + tracing::warn!(target: "scan::mailbox", error = %err, "failed to enqueue scan batch from actor request; rolling back provisional actor state"); + let mut actor = actor_handle.lock().await; + for dedupe_key in provisional_folder_jobs { + if let Err(rollback_err) = actor + .handle_command(LibraryActorCommand::JobFailed { + job_id: JobId::new(), + dedupe_key, + retryable: true, + error: Some("durable enqueue failed".to_string()), + }) + .await + { + tracing::warn!(target: "scan::mailbox", error = %rollback_err, "failed to roll back provisional library actor state"); + } + } + return Err(err); + } + } + }; + let mut enrolled_job_ids: Vec<_> = handles + .iter() + .map(|handle| { + handle + .merged_into + .unwrap_or(handle.job_id) + }) + .collect(); + enrolled_job_ids.sort_unstable_by_key( + |job_id| job_id.0, + ); + enrolled_job_ids.dedup(); if let LibraryActorCommand::Start { mode, @@ -312,16 +753,18 @@ impl RuntimeTaskGraph { correlation_id: *correlation_id, mode, queued_folders, + enrolled_job_ids, completed_at: chrono::Utc::now(), }; - if let Err(err) = events - .publish_scan_event(ScanEvent::SeedCompleted( - summary, - )) + events + .publish_scan_event( + ScanEvent::SeedCompleted(summary), + ) .await - { - tracing::warn!(target: "scan::mailbox", error = %err, "failed to publish scan seed completion"); - } + .map_err(|err| { + tracing::warn!(target: "scan::mailbox", error = %err, "failed to publish scan seed completion"); + err + })?; } Ok(()) @@ -344,63 +787,30 @@ impl RuntimeTaskGraph { Ok(()) } - pub(super) async fn spawn_scheduler_observer( + pub(super) async fn spawn_scheduler_observer( &self, + queue: Arc, events: Arc, scheduler: WeightedFairScheduler, correlations: CorrelationCache, ) where + Q: QueueService + 'static, E: ScanEventBus + JobEventStream + 'static, { - let mut job_rx = events.subscribe_jobs(); + let job_rx = events.subscribe_jobs(); let shutdown = self.shutdown_token.clone(); - self.spawn_task(RuntimeTaskKind::SchedulerObserver, async move { - loop { - tokio::select! { - _ = shutdown.cancelled() => { - tracing::info!("Scheduler observer shutting down"); - break; - } - event = job_rx.recv() => match event { - Ok(event) => { - match event.payload { - JobEventPayload::Enqueued { job_id, priority, .. } => { - correlations.remember(job_id, event.meta.correlation_id).await; - scheduler - .record_enqueued(event.meta.library_id, priority) - .await; - } - JobEventPayload::Merged { - existing_job_id, - merged_job_id, - .. - } => { - correlations - .remember_if_absent(existing_job_id, event.meta.correlation_id) - .await; - if merged_job_id != existing_job_id { - correlations - .remember_if_absent( - merged_job_id, - event.meta.correlation_id, - ) - .await; - } - } - _ => {} - } - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => break, - Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { - tracing::warn!( - "scheduler observer lagged, skipped {skipped} events" - ); - } - } - } - } - }).await; + self.spawn_task( + RuntimeTaskKind::SchedulerObserver, + observe_scheduler_events( + queue, + job_rx, + scheduler, + correlations, + shutdown, + ), + ) + .await; } #[allow(clippy::too_many_arguments)] @@ -455,7 +865,7 @@ impl RuntimeTaskGraph { continue; } - let reservation = match scheduler.reserve().await { + let reservation = match scheduler.reserve(worker_kind).await { Some(reservation) => reservation, None => { tokio::time::sleep( @@ -538,22 +948,29 @@ impl RuntimeTaskGraph { tracing::error!( "budget acquire error: {err}" ); - let _ = q - .fail( + let execution_error = format!( + "budget acquire failed: {err}" + ); + let transition = worker_transition_from_fail( + q.fail_with_outcome( lease_id, true, - Some( - "budget acquire failed".into(), - ), - ) - .await; - scheduler.release(library_id).await; - scheduler - .record_enqueued( - library_id, - job_priority, + Some(execution_error.clone()), ) - .await; + .await, + execution_error, + "fail_after_budget_acquire", + ); + finalize_worker_transition( + transition, + &lease, + reservation.library_id, + &scheduler, + e.as_ref(), + &correlation_cache, + &mailbox_tx, + ) + .await; continue; } }; @@ -576,162 +993,75 @@ impl RuntimeTaskGraph { }, ); - let dispatch_status = d.dispatch(&lease).await; + let dispatch_timeout = + std::time::Duration::from_millis( + lease_cfg.dispatch_timeout_ms.max(1), + ); + let dispatch_status = match tokio::time::timeout( + dispatch_timeout, + d.dispatch(&lease), + ) + .await + { + Ok(status) => status, + Err(_) => { + let error = format!( + "job dispatch timed out after {} ms", + dispatch_timeout.as_millis() + ); + tracing::error!( + job = %job_id.0, + kind = ?job_kind, + library = %library_id, + timeout_ms = dispatch_timeout.as_millis(), + "{error}" + ); + DispatchStatus::Retry { error } + } + }; renew_task.stop().await; - let dedupe_key: DedupeKey = - lease.job.payload.dedupe_key(); - let library_id = lease.job.payload.library_id(); - let notify_command = match dispatch_status { + let transition = match dispatch_status { DispatchStatus::Success => { - if let Err(err) = q.complete(lease_id).await - { - tracing::error!( - "queue complete error: {err}" - ); - } - let correlation_id = correlation_cache - .take_persisted_or_generate( - job_id, - lease.job.correlation_id, - ) - .await; - let event = JobEvent::from_job( - Some(correlation_id), - library_id, - lease.job.dedupe_key.clone(), - stable_path_key(&lease.job.payload), - JobEventPayload::Completed { - job_id, - kind: job_kind, - priority: job_priority, - }, - ); - if let Err(err) = e.publish(event).await { - tracing::error!( - "publish complete event failed: {err}" - ); - } - scheduler.record_completed(library_id).await; - Some(LibraryActorCommand::JobCompleted { - job_id, - dedupe_key: dedupe_key.clone(), - }) + worker_transition_from_complete( + q.complete_with_outcome(lease_id).await, + ) } DispatchStatus::Retry { error } => { - if let Err(err) = q - .fail( + worker_transition_from_fail( + q.fail_with_outcome( lease_id, true, Some(error.clone()), ) - .await - { - tracing::error!( - "queue fail error: {err}" - ); - } - let correlation_id = correlation_cache - .fetch_persisted_or_generate( - job_id, - lease.job.correlation_id, - ) - .await; - let event = JobEvent::from_job( - Some(correlation_id), - library_id, - lease.job.dedupe_key.clone(), - stable_path_key(&lease.job.payload), - JobEventPayload::Failed { - job_id, - kind: job_kind, - priority: job_priority, - retryable: true, - }, - ); - if let Err(err) = e.publish(event).await { - tracing::error!( - "publish retry event failed: {err}" - ); - } - scheduler.release(library_id).await; - scheduler - .record_enqueued( - library_id, - job_priority, - ) - .await; - Some(LibraryActorCommand::JobFailed { - job_id, - dedupe_key: dedupe_key.clone(), - retryable: true, - error: Some(error), - }) + .await, + error, + "fail", + ) } DispatchStatus::DeadLetter { error } => { - if let Err(err) = q - .dead_letter( + worker_transition_from_dead_letter( + q.dead_letter_with_outcome( lease_id, Some(error.clone()), ) - .await - { - tracing::error!( - "queue dead-letter error: {err}" - ); - } - let correlation_id = correlation_cache - .take_persisted_or_generate( - job_id, - lease.job.correlation_id, - ) - .await; - let event = JobEvent::from_job( - Some(correlation_id), - library_id, - lease.job.dedupe_key.clone(), - stable_path_key(&lease.job.payload), - JobEventPayload::DeadLettered { - job_id, - kind: job_kind, - priority: job_priority, - }, - ); - if let Err(err) = e.publish(event).await { - tracing::error!( - "publish dead-letter event failed: {err}" - ); - } - scheduler.record_completed(library_id).await; - Some(LibraryActorCommand::JobFailed { - job_id, - dedupe_key: dedupe_key.clone(), - retryable: false, - error: Some(error), - }) + .await, + error, + ) } }; - if let Some(command) = notify_command { - let sender_opt = { - let guard = mailbox_tx.lock().await; - guard.clone() - }; - if let Some(sender) = sender_opt - && let Err(err) = sender - .send(OrchestratorCommand::Library { - library_id, - command, - completion: None, - }) - .await - { - tracing::warn!( - "failed to send library actor notification: {err}" - ); - } - } + finalize_worker_transition( + transition, + &lease, + reservation.library_id, + &scheduler, + e.as_ref(), + &correlation_cache, + &mailbox_tx, + ) + .await; let _ = b.release(token).await; } @@ -778,9 +1108,10 @@ impl RuntimeTaskGraph { pub(super) async fn spawn_housekeeper( &self, queue: Arc, + scheduler: WeightedFairScheduler, interval: std::time::Duration, ) where - Q: LeaseExpiryScanner + 'static, + Q: QueueService + LeaseExpiryScanner + 'static, { let shutdown = self.shutdown_token.clone(); self.spawn_task(RuntimeTaskKind::Housekeeper, async move { @@ -794,6 +1125,21 @@ impl RuntimeTaskGraph { if let Err(err) = queue.scan_expired_leases().await { tracing::warn!("housekeeper scan_expired_leases error: {err}"); } + match reconcile_scheduler_ready( + queue.as_ref(), + &scheduler, + ) + .await + { + Ok(ready_total) => tracing::trace!( + ready_total, + "scheduler ready counts reconciled" + ), + Err(err) => tracing::warn!( + error = %err, + "periodic scheduler ready reconciliation failed" + ), + } } } } @@ -973,10 +1319,24 @@ impl LeaseRenewalTask { Self { cancel_tx, handle } } - async fn stop(self) { + async fn stop(mut self) { let _ = self.cancel_tx.try_send(()); - if let Err(err) = self.handle.await { - tracing::warn!("lease renewal task failed: {err}"); + match tokio::time::timeout( + std::time::Duration::from_secs(1), + &mut self.handle, + ) + .await + { + Ok(Ok(())) => {} + Ok(Err(err)) => { + tracing::warn!("lease renewal task failed: {err}"); + } + Err(_) => { + tracing::warn!( + "lease renewal task did not stop within one second; aborting" + ); + self.handle.abort(); + } } } } @@ -1012,3 +1372,202 @@ fn workload_for(kind: JobKind) -> WorkloadType { JobKind::TranscriptExtract => WorkloadType::TranscriptExtraction, } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::domain::scan::context::{MovieFolderScanContext, MovieRootPath}; + use crate::domain::scan::orchestration::events::{ + EventMeta, JobEventPublisher, + }; + use crate::domain::scan::orchestration::queue::ReadyQueueCount; + use crate::domain::scan::orchestration::runtime::InProcJobEventBus; + use async_trait::async_trait; + use std::time::Duration; + use uuid::Uuid; + + struct ReadyCountQueue { + counts: Vec, + } + + #[async_trait] + impl QueueService for ReadyCountQueue { + async fn enqueue( + &self, + _request: EnqueueRequest, + ) -> Result + { + panic!("enqueue is not used by scheduler reconciliation tests") + } + + async fn dequeue( + &self, + _request: DequeueRequest, + ) -> Result> + { + panic!("dequeue is not used by scheduler reconciliation tests") + } + + async fn renew( + &self, + _renewal: LeaseRenewal, + ) -> Result + { + panic!("renew is not used by scheduler reconciliation tests") + } + + async fn complete( + &self, + _lease_id: crate::domain::scan::orchestration::lease::LeaseId, + ) -> Result<()> { + panic!("complete is not used by scheduler reconciliation tests") + } + + async fn fail( + &self, + _lease_id: crate::domain::scan::orchestration::lease::LeaseId, + _retryable: bool, + _error: Option, + ) -> Result<()> { + panic!("fail is not used by scheduler reconciliation tests") + } + + async fn dead_letter( + &self, + _lease_id: crate::domain::scan::orchestration::lease::LeaseId, + _error: Option, + ) -> Result<()> { + panic!("dead_letter is not used by reconciliation tests") + } + + async fn cancel_job(&self, _job_id: JobId) -> Result<()> { + panic!("cancel_job is not used by reconciliation tests") + } + + async fn queue_depth(&self, _kind: JobKind) -> Result { + panic!("queue_depth is not used by reconciliation tests") + } + + async fn release_dependency( + &self, + _library_id: LibraryId, + _dependency_key: &crate::domain::scan::orchestration::job::DependencyKey, + ) -> Result { + panic!("release_dependency is not used by reconciliation tests") + } + + async fn ready_counts_grouped(&self) -> Result> { + Ok(self.counts.clone()) + } + } + + #[test] + fn folder_discovery_request_preserves_parent_correlation() { + let library_id = + LibraryId(Uuid::from_u128(0x61100000000000000000000000000001)); + let correlation_id = + Uuid::from_u128(0x6110000000000000000000000000c001); + let context = FolderScanContext::Movie(MovieFolderScanContext { + library_id, + movie_root_path: MovieRootPath::try_new_under_library_root( + "/library", + "/library/Child Movie", + ) + .unwrap(), + }); + + let request = folder_scan_request_from_discovery( + &context, + ScanReason::BulkSeed, + Some(correlation_id), + None, + ) + .expect("legacy discovery requires enqueue"); + + assert_eq!(request.correlation_id, Some(correlation_id)); + assert_eq!(request.priority, JobPriority::P1); + let JobPayload::FolderScan(job) = request.payload else { + panic!("expected child FolderScan request"); + }; + assert_eq!(job.context.library_id(), context.library_id()); + assert_eq!(job.context.folder_path_norm(), context.folder_path_norm()); + assert_eq!(job.scan_reason, ScanReason::BulkSeed); + + assert!( + folder_scan_request_from_discovery( + &context, + ScanReason::BulkSeed, + Some(correlation_id), + Some(JobId::new()), + ) + .is_none(), + "a discovery event with a durable child must remain observational" + ); + } + + #[tokio::test] + async fn event_lag_above_256_recovers_ready_counts_from_durable_queue() { + let library_id = LibraryId::new(); + let queue = Arc::new(ReadyCountQueue { + counts: vec![ReadyQueueCount { + kind: JobKind::FolderScan, + library_id, + priority: JobPriority::P1, + ready: 1, + leased: 0, + }], + }); + let scheduler = WeightedFairScheduler::new( + &crate::domain::scan::orchestration::config::QueueConfig::default(), + crate::domain::scan::orchestration::config::PriorityWeights::default(), + ); + let events = Arc::new(InProcJobEventBus::new(256)); + let receiver = events.subscribe_jobs(); + + for sequence in 0..300 { + events + .publish(JobEvent { + meta: EventMeta::new( + None, + library_id, + format!("lag-regression:{sequence}"), + None, + ), + payload: JobEventPayload::ThroughputTick { + queue_depths: Vec::new(), + sampled_at: chrono::Utc::now(), + }, + }) + .await + .expect("event publish succeeds"); + } + + let shutdown = CancellationToken::new(); + let observer = tokio::spawn(observe_scheduler_events( + queue, + receiver, + scheduler.clone(), + CorrelationCache::default(), + shutdown.clone(), + )); + + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if scheduler + .snapshot() + .await + .get(&library_id) + .is_some_and(|(_, ready)| *ready == 1) + { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("lag reconciliation restores the durable ready count"); + + shutdown.cancel(); + observer.await.expect("observer exits cleanly"); + } +} diff --git a/crates/ferrex-core/src/domain/scan/orchestration/scan_run.rs b/crates/ferrex-core/src/domain/scan/orchestration/scan_run.rs index 95ed2912..dafd8a99 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/scan_run.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/scan_run.rs @@ -1,5 +1,8 @@ +use std::collections::HashSet; + use async_trait::async_trait; use chrono::{DateTime, Utc}; +use serde_json::{Value, json}; use sqlx::PgPool; use tracing::warn; use uuid::Uuid; @@ -10,6 +13,11 @@ use crate::{ types::LibraryId, }; +use super::job::JobId; + +const TRACKED_JOB_IDS_METADATA_KEY: &str = "tracked_job_ids"; +const SEED_COMPLETED_METADATA_KEY: &str = "seed_completed"; + /// Active durable scan statuses for partial-index conflict handling. const ACTIVE_STATUS_SQL: &str = "'pending','running','paused'"; @@ -28,6 +36,10 @@ pub struct LibraryScanRun { pub dead_lettered_items: u64, pub current_path: Option, pub last_error: Option, + /// Queue jobs enrolled by this run, including jobs shared by dedupe merge. + pub tracked_job_ids: Vec, + /// Whether the library actor has durably completed the initial seed command. + pub seed_completed: bool, pub sequence: u64, pub started_at: DateTime, pub terminal_at: Option>, @@ -95,6 +107,10 @@ pub struct LibraryScanRunProgressUpdate { pub retrying_items: u64, pub dead_lettered_items: u64, pub current_path: Option, + /// Complete set of queue jobs enrolled by this run. + pub tracked_job_ids: Vec, + /// Monotonic durable marker for completion of the initial seed command. + pub seed_completed: bool, pub sequence: u64, } @@ -114,6 +130,12 @@ pub trait ScanRunRepository: Send + Sync { async fn list_active(&self) -> Result>; + /// Load a run regardless of lifecycle status for idempotent finalization. + async fn load_by_scan_id( + &self, + scan_id: Uuid, + ) -> Result>; + async fn update_progress( &self, update: LibraryScanRunProgressUpdate, @@ -150,6 +172,74 @@ impl PostgresScanRunRepository { pub fn pool(&self) -> &PgPool { &self.pool } + + async fn load_metadata(&self, scan_id: Uuid) -> Result { + let metadata = sqlx::query_scalar!( + r#"SELECT metadata as "metadata!" FROM library_scan_runs WHERE scan_id = $1"#, + scan_id, + ) + .fetch_optional(&self.pool) + .await + .map_err(|err| { + MediaError::Internal(format!( + "load library scan run metadata failed: {err}" + )) + })? + .unwrap_or_else(|| json!({})); + + Ok(metadata) + } + + async fn hydrate_metadata( + &self, + mut run: LibraryScanRun, + ) -> Result { + let metadata = self.load_metadata(run.scan_id).await?; + run.tracked_job_ids = tracked_job_ids_from_metadata(&metadata)?; + run.seed_completed = seed_completed_from_metadata(&metadata)?; + Ok(run) + } +} + +fn tracked_job_ids_from_metadata(metadata: &Value) -> Result> { + let Some(raw_ids) = metadata.get(TRACKED_JOB_IDS_METADATA_KEY) else { + return Ok(Vec::new()); + }; + let values = raw_ids.as_array().ok_or_else(|| { + MediaError::Internal(format!( + "library_scan_runs.metadata.{TRACKED_JOB_IDS_METADATA_KEY} was not an array" + )) + })?; + + let mut ids = HashSet::with_capacity(values.len()); + for value in values { + let raw = value.as_str().ok_or_else(|| { + MediaError::Internal(format!( + "library_scan_runs.metadata.{TRACKED_JOB_IDS_METADATA_KEY} contained a non-string value" + )) + })?; + let id = Uuid::parse_str(raw).map_err(|err| { + MediaError::Internal(format!( + "library_scan_runs.metadata.{TRACKED_JOB_IDS_METADATA_KEY} contained invalid job ID '{raw}': {err}" + )) + })?; + ids.insert(JobId(id)); + } + + let mut ids: Vec<_> = ids.into_iter().collect(); + ids.sort_unstable_by_key(|job_id| job_id.0); + Ok(ids) +} + +fn seed_completed_from_metadata(metadata: &Value) -> Result { + let Some(value) = metadata.get(SEED_COMPLETED_METADATA_KEY) else { + return Ok(false); + }; + value.as_bool().ok_or_else(|| { + MediaError::Internal(format!( + "library_scan_runs.metadata.{SEED_COMPLETED_METADATA_KEY} was not a boolean" + )) + }) } #[derive(Debug, sqlx::FromRow)] @@ -210,6 +300,8 @@ impl TryFrom for LibraryScanRun { )?, current_path: row.current_path, last_error: row.last_error, + tracked_job_ids: Vec::new(), + seed_completed: false, sequence: i64_to_u64("sequence", row.sequence)?, started_at: row.started_at, terminal_at: row.terminal_at, @@ -309,7 +401,7 @@ impl ScanRunRepository for PostgresScanRunRepository { )) })?; return Ok(LibraryScanRunGetOrCreate { - run: row_to_run(row)?, + run: self.hydrate_metadata(row_to_run(row)?).await?, disposition: ScanStartDisposition::Created, }); } @@ -361,7 +453,7 @@ impl ScanRunRepository for PostgresScanRunRepository { )) })?; return Ok(LibraryScanRunGetOrCreate { - run: row_to_run(row)?, + run: self.hydrate_metadata(row_to_run(row)?).await?, disposition: ScanStartDisposition::Reused, }); } @@ -430,7 +522,10 @@ impl ScanRunRepository for PostgresScanRunRepository { )) })?; - row.map(row_to_run).transpose() + match row.map(row_to_run).transpose()? { + Some(run) => Ok(Some(self.hydrate_metadata(run).await?)), + None => Ok(None), + } } async fn list_active(&self) -> Result> { @@ -468,7 +563,55 @@ impl ScanRunRepository for PostgresScanRunRepository { )) })?; - rows.into_iter().map(row_to_run).collect() + let mut runs = Vec::with_capacity(rows.len()); + for row in rows { + runs.push(self.hydrate_metadata(row_to_run(row)?).await?); + } + Ok(runs) + } + + async fn load_by_scan_id( + &self, + scan_id: Uuid, + ) -> Result> { + let row = sqlx::query_as!( + LibraryScanRunRow, + r#" + SELECT + scan_id as "scan_id!", + library_id as "library_id!", + mode as "mode!", + run_key as "run_key!", + correlation_id, + status as "status!", + completed_items as "completed_items!", + total_items as "total_items!", + retrying_items as "retrying_items!", + dead_lettered_items as "dead_lettered_items!", + current_path, + last_error, + sequence as "sequence!", + started_at, + terminal_at, + created_at, + updated_at + FROM library_scan_runs + WHERE scan_id = $1 + "#, + scan_id, + ) + .fetch_optional(&self.pool) + .await + .map_err(|err| { + MediaError::Internal(format!( + "load library scan run by scan ID failed: {err}" + )) + })?; + + match row.map(row_to_run).transpose()? { + Some(run) => Ok(Some(self.hydrate_metadata(run).await?)), + None => Ok(None), + } } async fn update_progress( @@ -493,6 +636,14 @@ impl ScanRunRepository for PostgresScanRunRepository { let dead_lettered_items = u64_to_i64("dead_lettered_items", update.dead_lettered_items)?; let sequence = u64_to_i64("sequence", update.sequence)?; + let seed_completed = update.seed_completed; + let tracked_job_ids = json!( + update + .tracked_job_ids + .iter() + .map(|job_id| job_id.0) + .collect::>() + ); let row = sqlx::query_as!( LibraryScanRunRow, @@ -505,12 +656,45 @@ impl ScanRunRepository for PostgresScanRunRepository { dead_lettered_items = $6, current_path = $7, sequence = GREATEST(sequence, $8), + metadata = jsonb_set( + jsonb_set( + metadata, + '{tracked_job_ids}', + ( + SELECT COALESCE( + jsonb_agg(value ORDER BY value), + '[]'::jsonb + ) + FROM ( + SELECT DISTINCT value + FROM jsonb_array_elements( + COALESCE( + metadata -> 'tracked_job_ids', + '[]'::jsonb + ) || $9::jsonb + ) AS ids(value) + ) AS unique_ids + ), + true + ), + '{seed_completed}', + to_jsonb( + ( + CASE + WHEN jsonb_typeof(metadata -> 'seed_completed') = 'boolean' + THEN (metadata ->> 'seed_completed')::boolean + ELSE false + END + ) OR $10::boolean + ), + true + ), updated_at = NOW() WHERE scan_id = $1 AND status::text IN ('pending','running','paused') RETURNING - scan_id, - library_id, + scan_id as "scan_id!", + library_id as "library_id!", mode as "mode!", run_key as "run_key!", correlation_id, @@ -534,7 +718,9 @@ impl ScanRunRepository for PostgresScanRunRepository { retrying_items, dead_lettered_items, update.current_path, - sequence + sequence, + tracked_job_ids, + seed_completed, ) .fetch_optional(&self.pool) .await @@ -544,7 +730,10 @@ impl ScanRunRepository for PostgresScanRunRepository { )) })?; - row.map(row_to_run).transpose() + let Some(row) = row else { + return Ok(None); + }; + Ok(Some(self.hydrate_metadata(row_to_run(row)?).await?)) } async fn mark_terminal( @@ -603,7 +792,10 @@ impl ScanRunRepository for PostgresScanRunRepository { )) })?; - row.map(row_to_run).transpose() + match row.map(row_to_run).transpose()? { + Some(run) => Ok(Some(self.hydrate_metadata(run).await?)), + None => Ok(None), + } } } @@ -655,6 +847,8 @@ mod tests { dead_lettered_items: 0, current_path: None, last_error: None, + tracked_job_ids: Vec::new(), + seed_completed: false, sequence: 0, started_at: now, terminal_at: None, @@ -693,6 +887,13 @@ mod tests { .collect()) } + async fn load_by_scan_id( + &self, + scan_id: Uuid, + ) -> Result> { + Ok(self.runs.lock().await.get(&scan_id).cloned()) + } + async fn update_progress( &self, update: LibraryScanRunProgressUpdate, @@ -721,6 +922,10 @@ mod tests { run.retrying_items = update.retrying_items; run.dead_lettered_items = update.dead_lettered_items; run.current_path = update.current_path; + run.tracked_job_ids.extend(update.tracked_job_ids); + run.tracked_job_ids.sort_unstable_by_key(|job_id| job_id.0); + run.tracked_job_ids.dedup(); + run.seed_completed |= update.seed_completed; run.sequence = run.sequence.max(update.sequence); run.updated_at = Utc::now(); Ok(Some(run.clone())) @@ -1034,6 +1239,163 @@ mod tests { .expect("cleanup library succeeds"); } + #[tokio::test] + async fn postgres_progress_metadata_unions_merged_job_ids_for_restart() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => { + eprintln!("skipping: DATABASE_URL not set"); + return; + } + }; + let pool = match PgPool::connect(&database_url).await { + Ok(pool) => pool, + Err(err) => { + eprintln!( + "skipping: failed to connect to DATABASE_URL ({err})" + ); + return; + } + }; + if let Err(err) = crate::MIGRATOR.run(&pool).await { + eprintln!("skipping: migrations failed ({err})"); + return; + } + + let repo = PostgresScanRunRepository::new(pool.clone()); + let library_id = LibraryId(Uuid::now_v7()); + seed_library_for_postgres_test(&pool, library_id) + .await + .expect("library seeded"); + let active = repo + .get_or_create_active( + NewLibraryScanRun::new(library_id, ScanRunMode::Manual) + .running(), + ) + .await + .expect("active run created") + .run; + let original_job_id = JobId::new(); + let merged_job_id = JobId::new(); + + for (sequence, tracked_job_ids) in [ + (1, vec![original_job_id]), + // Deliberately omit the first ID to model an interleaved stale + // frame. PostgreSQL must union rather than overwrite metadata. + (2, vec![merged_job_id]), + // A later stale frame must not clear the monotonic seed marker. + (3, Vec::new()), + ] { + repo.update_progress(LibraryScanRunProgressUpdate { + scan_id: active.scan_id, + status: Some(ScanLifecycleStatus::Running), + completed_items: 0, + total_items: 1, + retrying_items: 0, + dead_lettered_items: 0, + current_path: Some("/library/shared".into()), + tracked_job_ids, + seed_completed: sequence == 2, + sequence, + }) + .await + .expect("tracked job metadata persists"); + } + + let restored = repo + .load_active(library_id, ScanRunMode::Manual) + .await + .expect("active run reload succeeds") + .expect("active run remains present"); + assert_eq!(restored.tracked_job_ids, { + let mut ids = vec![original_job_id, merged_job_id]; + ids.sort_unstable_by_key(|job_id| job_id.0); + ids + }); + assert!(restored.seed_completed); + + sqlx::query("DELETE FROM libraries WHERE id = $1") + .bind(library_id.to_uuid()) + .execute(&pool) + .await + .expect("cleanup library succeeds"); + } + + #[test] + fn tracked_job_metadata_rejects_malformed_values() { + let err = tracked_job_ids_from_metadata(&json!({ + (TRACKED_JOB_IDS_METADATA_KEY): ["not-a-uuid"] + })) + .expect_err("malformed durable tracking metadata is rejected"); + + assert!(err.to_string().contains("invalid job ID")); + } + + #[test] + fn seed_completion_metadata_defaults_false_and_rejects_non_boolean() { + assert!( + !seed_completed_from_metadata(&json!({})) + .expect("missing seed marker defaults to false") + ); + assert!( + seed_completed_from_metadata(&json!({ + (SEED_COMPLETED_METADATA_KEY): true + })) + .expect("boolean seed marker parses") + ); + + let err = seed_completed_from_metadata(&json!({ + (SEED_COMPLETED_METADATA_KEY): "true" + })) + .expect_err("non-boolean seed marker is rejected"); + assert!(err.to_string().contains("was not a boolean")); + } + + #[tokio::test] + async fn progress_metadata_is_monotonic() { + let repo = InMemoryScanRunRepository::default(); + let library_id = LibraryId::new(); + let run = repo + .get_or_create_active( + NewLibraryScanRun::new(library_id, ScanRunMode::Manual) + .running(), + ) + .await + .expect("run created") + .run; + let first_job = JobId::new(); + let second_job = JobId::new(); + + for (sequence, tracked_job_ids, seed_completed) in + [(1, vec![first_job], true), (2, vec![second_job], false)] + { + repo.update_progress(LibraryScanRunProgressUpdate { + scan_id: run.scan_id, + status: Some(ScanLifecycleStatus::Running), + completed_items: 0, + total_items: 2, + retrying_items: 0, + dead_lettered_items: 0, + current_path: None, + tracked_job_ids, + seed_completed, + sequence, + }) + .await + .expect("progress update succeeds"); + } + + let restored = repo + .load_by_scan_id(run.scan_id) + .await + .expect("run lookup succeeds") + .expect("run remains present"); + assert!(restored.seed_completed); + assert_eq!(restored.tracked_job_ids.len(), 2); + assert!(restored.tracked_job_ids.contains(&first_job)); + assert!(restored.tracked_job_ids.contains(&second_job)); + } + #[tokio::test] async fn progress_update_rejects_terminal_status() { let repo = InMemoryScanRunRepository::default(); @@ -1055,6 +1417,8 @@ mod tests { retrying_items: 0, dead_lettered_items: 0, current_path: None, + tracked_job_ids: Vec::new(), + seed_completed: false, sequence: 1, }) .await @@ -1138,6 +1502,14 @@ mod tests { assert_eq!(active_runs.len(), 1); assert_eq!(active_runs[0].scan_id, active.run.scan_id); + assert_eq!( + repo.load_by_scan_id(terminal.run.scan_id) + .await + .expect("terminal lookup succeeds") + .expect("terminal row remains durable") + .status, + ScanLifecycleStatus::Completed + ); } #[tokio::test] diff --git a/crates/ferrex-core/src/domain/scan/orchestration/scheduler.rs b/crates/ferrex-core/src/domain/scan/orchestration/scheduler.rs index 9cc05718..834a8674 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/scheduler.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/scheduler.rs @@ -9,13 +9,14 @@ use uuid::Uuid; use crate::types::ids::LibraryId; use super::config::{LibraryQueuePolicy, PriorityWeights, QueueConfig}; -use super::job::JobPriority; +use super::job::{JobKind, JobPriority}; /// Reservation handle returned by the scheduler when a worker is allowed to -/// attempt leasing work for a (library, priority) pair. +/// attempt leasing work for a (kind, library, priority) tuple. #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] pub struct SchedulingReservation { pub id: Uuid, + pub kind: JobKind, pub library_id: LibraryId, pub priority: JobPriority, } @@ -31,7 +32,7 @@ struct LibraryState { weight: u32, inflight: usize, pending: usize, - priorities: HashMap, + queues: HashMap<(JobKind, JobPriority), PriorityLibraryState>, } impl LibraryState { @@ -44,22 +45,24 @@ impl LibraryState { weight, inflight: 0, pending: 0, - priorities: HashMap::new(), + queues: HashMap::new(), } } - fn ensure_priority( + fn ensure_queue( &mut self, + kind: JobKind, priority: JobPriority, ) -> &mut PriorityLibraryState { - self.priorities.entry(priority).or_default() + self.queues.entry((kind, priority)).or_default() } - fn priority_state( + fn queue_state( &mut self, + kind: JobKind, priority: JobPriority, ) -> Option<&mut PriorityLibraryState> { - self.priorities.get_mut(&priority) + self.queues.get_mut(&(kind, priority)) } } @@ -70,7 +73,7 @@ impl fmt::Debug for LibraryState { .field("weight", &self.weight) .field("inflight", &self.inflight) .field("pending", &self.pending) - .field("priority_count", &self.priorities.len()) + .field("queue_count", &self.queues.len()) .finish() } } @@ -106,6 +109,7 @@ impl fmt::Debug for QueueDefaults { } struct ReservationState { + kind: JobKind, library_id: LibraryId, priority: JobPriority, weight_debt: i32, @@ -114,6 +118,7 @@ struct ReservationState { impl fmt::Debug for ReservationState { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ReservationState") + .field("kind", &self.kind) .field("library_id", &self.library_id) .field("priority", &self.priority) .field("weight_debt", &self.weight_debt) @@ -148,6 +153,7 @@ impl SchedulerState { fn select_for_priority( &mut self, + kind: JobKind, priority: JobPriority, ) -> Option<(LibraryId, i32)> { let mut selected: Option<(LibraryId, i32)> = None; @@ -158,7 +164,7 @@ impl SchedulerState { continue; } let weight = state.weight as i32; - if let Some(priority_state) = state.priority_state(priority) { + if let Some(priority_state) = state.queue_state(kind, priority) { if priority_state.ready == 0 { continue; } @@ -180,7 +186,7 @@ impl SchedulerState { return None; } if let Some(state) = self.libraries.get_mut(&library_id) - && let Some(priority_state) = state.priority_state(priority) + && let Some(priority_state) = state.queue_state(kind, priority) { priority_state.current_weight -= total_weight; priority_state.ready = priority_state.ready.saturating_sub(1); @@ -221,8 +227,8 @@ fn build_priority_ring(weights: PriorityWeights) -> Vec { } /// Weighted-fair scheduler shared by worker pools. The scheduler keeps a -/// minimal in-memory view of ready counts per (library, priority) and enforces -/// per-library in-flight caps when allocating leases. +/// minimal in-memory view of ready counts per (kind, library, priority) and +/// enforces per-library in-flight caps when allocating leases. #[derive(Clone)] pub struct WeightedFairScheduler { defaults: Arc, @@ -258,9 +264,11 @@ impl fmt::Debug for WeightedFairScheduler { /// Bulk ready counts used to prime the scheduler without emitting one event per job. #[derive(Clone, Debug)] pub struct ReadyCountEntry { + pub kind: JobKind, pub library_id: LibraryId, pub priority: JobPriority, pub count: usize, + pub leased: usize, } impl WeightedFairScheduler { @@ -279,41 +287,93 @@ impl WeightedFairScheduler { pub async fn record_ready( &self, + kind: JobKind, library_id: LibraryId, priority: JobPriority, ) { let mut state = self.state.lock().await; let library = state.ensure_library(library_id, &self.defaults); - let priority_state = library.ensure_priority(priority); + let priority_state = library.ensure_queue(kind, priority); priority_state.ready += 1; } - pub async fn record_ready_bulk(&self, entries: I) + /// Replace the scheduler's ready and in-flight views with a durable queue snapshot. + /// + /// Reservations are removed from ready accounting before a worker leases + /// the corresponding PostgreSQL row. A durable snapshot can therefore + /// still include those rows; subtracting pending reservations prevents the + /// same ready row from being advertised twice during reconciliation. The + /// durable leased total also repairs capacity leaked by an interrupted + /// worker or an earlier accounting mismatch. + pub async fn reconcile_ready_absolute(&self, entries: I) where I: IntoIterator, { let mut state = self.state.lock().await; - for entry in entries.into_iter() { - if entry.count == 0 { - continue; + let mut durable = HashMap::new(); + let mut durable_inflight = HashMap::new(); + for entry in entries { + let count = durable + .entry((entry.library_id, entry.kind, entry.priority)) + .or_insert(0usize); + *count = count.saturating_add(entry.count); + let inflight = + durable_inflight.entry(entry.library_id).or_insert(0usize); + *inflight = inflight.saturating_add(entry.leased); + } + + let mut pending = HashMap::new(); + for reservation in state.reservations.values() { + *pending + .entry(( + reservation.library_id, + reservation.kind, + reservation.priority, + )) + .or_insert(0usize) += 1; + } + + for (library_id, library) in state.libraries.iter_mut() { + for queue in library.queues.values_mut() { + queue.ready = 0; } - let library = - state.ensure_library(entry.library_id, &self.defaults); - let priority_state = library.ensure_priority(entry.priority); - priority_state.ready = - priority_state.ready.saturating_add(entry.count); + library.inflight = durable_inflight + .get(library_id) + .copied() + .unwrap_or_default(); + } + + for ((library_id, kind, priority), durable_count) in durable { + let library = state.ensure_library(library_id, &self.defaults); + let queue_state = library.ensure_queue(kind, priority); + let pending_count = pending + .get(&(library_id, kind, priority)) + .copied() + .unwrap_or_default(); + queue_state.ready = durable_count.saturating_sub(pending_count); + } + + // A library can have only durable leases and no eligible ready rows. + // Ensure it is represented so its authoritative in-flight count is not + // lost merely because every queued item is currently leased. + for (library_id, leased) in durable_inflight { + state.ensure_library(library_id, &self.defaults).inflight = leased; } } pub async fn record_enqueued( &self, + kind: JobKind, library_id: LibraryId, priority: JobPriority, ) { - self.record_ready(library_id, priority).await; + self.record_ready(kind, library_id, priority).await; } - pub async fn reserve(&self) -> Option { + pub async fn reserve( + &self, + kind: JobKind, + ) -> Option { if self.priority_ring.is_empty() { return None; } @@ -325,12 +385,13 @@ impl WeightedFairScheduler { (state.next_priority_index + 1) % self.priority_ring.len(); if let Some((library_id, weight_debt)) = - state.select_for_priority(priority) + state.select_for_priority(kind, priority) { let reservation_id = Uuid::now_v7(); state.reservations.insert( reservation_id, ReservationState { + kind, library_id, priority, weight_debt, @@ -338,6 +399,7 @@ impl WeightedFairScheduler { ); return Some(SchedulingReservation { id: reservation_id, + kind, library_id, priority, }); @@ -359,6 +421,7 @@ impl WeightedFairScheduler { } Some(SchedulingReservation { id: reservation_id, + kind: reservation.kind, library_id: reservation.library_id, priority: reservation.priority, }) @@ -372,7 +435,7 @@ impl WeightedFairScheduler { { library.pending = library.pending.saturating_sub(1); if let Some(priority_state) = - library.priority_state(reservation.priority) + library.queue_state(reservation.kind, reservation.priority) { priority_state.ready += 1; priority_state.current_weight += reservation.weight_debt; @@ -407,6 +470,19 @@ impl WeightedFairScheduler { self.release(library_id).await; } + /// Forget all in-memory scheduling state for a deleted library. + /// + /// Durable queue rows are removed by the library foreign-key cascade. This + /// companion cleanup prevents the scheduler's ready counts and outstanding + /// reservations from continuing to advertise that deleted work. + pub async fn forget_library(&self, library_id: LibraryId) { + let mut state = self.state.lock().await; + state.libraries.remove(&library_id); + state + .reservations + .retain(|_, reservation| reservation.library_id != library_id); + } + #[cfg(test)] pub async fn snapshot(&self) -> HashMap { let state = self.state.lock().await; @@ -418,7 +494,7 @@ impl WeightedFairScheduler { *id, ( lib.inflight, - lib.priorities.values().map(|p| p.ready).sum::(), + lib.queues.values().map(|p| p.ready).sum::(), ), ) }) @@ -439,14 +515,17 @@ mod tests { ); let library_id = LibraryId::new(); - scheduler.record_enqueued(library_id, JobPriority::P1).await; + scheduler + .record_enqueued(JobKind::FolderScan, library_id, JobPriority::P1) + .await; assert_eq!(scheduler.snapshot().await[&library_id], (0, 1)); let canceled = scheduler - .reserve() + .reserve(JobKind::FolderScan) .await .expect("ready job reserves capacity"); assert_eq!(canceled.library_id, library_id); + assert_eq!(canceled.kind, JobKind::FolderScan); assert_eq!(canceled.priority, JobPriority::P1); assert_eq!(scheduler.snapshot().await[&library_id], (0, 0)); @@ -454,7 +533,7 @@ mod tests { assert_eq!(scheduler.snapshot().await[&library_id], (0, 1)); let confirmed = scheduler - .reserve() + .reserve(JobKind::FolderScan) .await .expect("restored ready job reserves again"); scheduler @@ -466,13 +545,179 @@ mod tests { scheduler.record_completed(library_id).await; assert_eq!(scheduler.snapshot().await[&library_id], (0, 0)); - scheduler.record_enqueued(library_id, JobPriority::P1).await; + scheduler + .record_enqueued(JobKind::FolderScan, library_id, JobPriority::P1) + .await; let stale = scheduler - .reserve() + .reserve(JobKind::FolderScan) .await .expect("stale ready count reserves capacity"); scheduler.discard_stale(stale.id).await; assert_eq!(scheduler.snapshot().await[&library_id], (0, 0)); - assert!(scheduler.reserve().await.is_none()); + assert!(scheduler.reserve(JobKind::FolderScan).await.is_none()); + } + + #[tokio::test] + async fn forget_library_drops_counts_and_pending_reservations_idempotently() + { + let scheduler = WeightedFairScheduler::new( + &QueueConfig::default(), + PriorityWeights::default(), + ); + let deleted_library = LibraryId::new(); + let retained_library = LibraryId::new(); + + scheduler + .reconcile_ready_absolute([ReadyCountEntry { + kind: JobKind::FolderScan, + library_id: deleted_library, + priority: JobPriority::P1, + count: 3, + leased: 0, + }]) + .await; + + let inflight = scheduler + .reserve(JobKind::FolderScan) + .await + .expect("deleted library has ready work"); + assert_eq!(inflight.library_id, deleted_library); + scheduler + .confirm(inflight.id) + .await + .expect("first reservation confirms"); + + let pending = scheduler + .reserve(JobKind::FolderScan) + .await + .expect("deleted library has another ready job"); + assert_eq!(pending.library_id, deleted_library); + scheduler + .record_enqueued( + JobKind::FolderScan, + retained_library, + JobPriority::P2, + ) + .await; + + scheduler.forget_library(deleted_library).await; + scheduler.forget_library(deleted_library).await; + + let snapshot = scheduler.snapshot().await; + assert!(!snapshot.contains_key(&deleted_library)); + assert_eq!(snapshot[&retained_library], (0, 1)); + assert!(scheduler.confirm(pending.id).await.is_none()); + + let retained = scheduler + .reserve(JobKind::FolderScan) + .await + .expect("retained library remains schedulable"); + assert_eq!(retained.library_id, retained_library); + } + + #[tokio::test] + async fn workers_cannot_reserve_ready_work_for_another_kind() { + let scheduler = WeightedFairScheduler::new( + &QueueConfig::default(), + PriorityWeights::default(), + ); + let library_id = LibraryId::new(); + + scheduler + .record_enqueued(JobKind::FolderScan, library_id, JobPriority::P1) + .await; + + assert!(scheduler.reserve(JobKind::ImageFetch).await.is_none()); + let reservation = scheduler + .reserve(JobKind::FolderScan) + .await + .expect("the matching worker kind reserves the ready job"); + assert_eq!(reservation.kind, JobKind::FolderScan); + assert_eq!(reservation.library_id, library_id); + } + + #[tokio::test] + async fn absolute_reconciliation_replaces_stale_counts_and_preserves_pending() + { + let scheduler = WeightedFairScheduler::new( + &QueueConfig::default(), + PriorityWeights::default(), + ); + let library_id = LibraryId::new(); + + scheduler + .record_enqueued(JobKind::ImageFetch, library_id, JobPriority::P1) + .await; + scheduler + .reconcile_ready_absolute([ReadyCountEntry { + kind: JobKind::FolderScan, + library_id, + priority: JobPriority::P1, + count: 3, + leased: 0, + }]) + .await; + + assert!(scheduler.reserve(JobKind::ImageFetch).await.is_none()); + let pending = scheduler + .reserve(JobKind::FolderScan) + .await + .expect("durable folder work is schedulable"); + + scheduler + .reconcile_ready_absolute([ReadyCountEntry { + kind: JobKind::FolderScan, + library_id, + priority: JobPriority::P1, + count: 3, + leased: 0, + }]) + .await; + assert_eq!(scheduler.snapshot().await[&library_id], (0, 2)); + + scheduler.cancel(pending.id).await; + assert_eq!(scheduler.snapshot().await[&library_id], (0, 3)); + + scheduler.reconcile_ready_absolute(std::iter::empty()).await; + assert_eq!(scheduler.snapshot().await[&library_id], (0, 0)); + } + + #[tokio::test] + async fn absolute_reconciliation_repairs_stale_inflight_at_library_cap() { + let mut config = QueueConfig::default(); + config.default_library_cap = 1; + let scheduler = + WeightedFairScheduler::new(&config, PriorityWeights::default()); + let library_id = LibraryId::new(); + + scheduler + .record_enqueued(JobKind::FolderScan, library_id, JobPriority::P1) + .await; + let leaked = scheduler + .reserve(JobKind::FolderScan) + .await + .expect("ready job reserves capacity"); + scheduler + .confirm(leaked.id) + .await + .expect("reservation confirms"); + assert_eq!(scheduler.snapshot().await[&library_id], (1, 0)); + assert!(scheduler.reserve(JobKind::FolderScan).await.is_none()); + + scheduler + .reconcile_ready_absolute([ReadyCountEntry { + kind: JobKind::FolderScan, + library_id, + priority: JobPriority::P1, + count: 1, + leased: 0, + }]) + .await; + + assert_eq!(scheduler.snapshot().await[&library_id], (0, 1)); + assert!( + scheduler.reserve(JobKind::FolderScan).await.is_some(), + "PostgreSQL snapshot clears leaked in-flight capacity" + ); } } diff --git a/crates/ferrex-core/tests/support/scanner_runtime.rs b/crates/ferrex-core/tests/support/scanner_runtime.rs index 14b005c8..fde7a53a 100644 --- a/crates/ferrex-core/tests/support/scanner_runtime.rs +++ b/crates/ferrex-core/tests/support/scanner_runtime.rs @@ -601,6 +601,8 @@ pub fn scan_event_folder_discovered( ScanEvent::FolderDiscovered { context: Box::new(context), reason, + correlation_id: None, + durable_job_id: None, } } @@ -624,6 +626,7 @@ pub fn scan_event_seed_completed( correlation_id: None, mode, queued_folders, + enrolled_job_ids: Vec::new(), completed_at: Utc::now(), }) } diff --git a/crates/ferrex-model/src/scan.rs b/crates/ferrex-model/src/scan.rs index 74f9deb1..f094369e 100644 --- a/crates/ferrex-model/src/scan.rs +++ b/crates/ferrex-model/src/scan.rs @@ -183,7 +183,8 @@ pub mod orchestration { /// Cursor-based incremental maintenance scheduler configuration. #[cfg_attr(feature = "serde", serde(default))] pub maintenance: MaintenanceConfig, - /// Lease defaults (TTL, renewal thresholds, housekeeping cadence). + /// Lease defaults (TTL, renewal thresholds, execution deadline, + /// and housekeeping cadence). pub lease: LeaseConfig, /// Global concurrency budget configuration for actor workloads. pub budget: super::budget::BudgetConfig, @@ -343,6 +344,9 @@ pub mod orchestration { pub struct LeaseConfig { /// Default TTL for job leases (seconds). pub lease_ttl_secs: i64, + /// Maximum time one dispatcher invocation may run before the job + /// is released for a retry (milliseconds). + pub dispatch_timeout_ms: u64, /// Renew when remaining TTL drops below this fraction of the original TTL (e.g. 0.5). pub renew_at_fraction: f32, /// Minimum margin before expiry to trigger a renewal regardless of fraction (ms). @@ -355,6 +359,7 @@ pub mod orchestration { fn default() -> Self { Self { lease_ttl_secs: 30, + dispatch_timeout_ms: 30 * 60 * 1_000, renew_at_fraction: 0.5, renew_min_margin_ms: 2_000, housekeeper_interval_ms: 15_000, diff --git a/crates/ferrex-server/src/handlers/admin/dev_handlers.rs b/crates/ferrex-server/src/handlers/admin/dev_handlers.rs index 7757f2ae..cbd5a74d 100644 --- a/crates/ferrex-server/src/handlers/admin/dev_handlers.rs +++ b/crates/ferrex-server/src/handlers/admin/dev_handlers.rs @@ -7,12 +7,16 @@ use ferrex_model::MovieReferenceBatchSize; use ferrex_core::{ - api::types::ApiResponse, + api::types::{ + ApiResponse, + admin::{ResetDatabaseRequest, ResetDatabaseResult}, + }, domain::users::user::User, types::{LibraryId, LibraryType, library::Library}, }; use crate::{ + handlers::media::handle_library::delete_library_with_runtime_cleanup, handlers::users::{UserService, user_service::CreateUserParams}, infra::{ app_state::AppState, @@ -23,8 +27,10 @@ use crate::{ use axum::{Extension, Json, extract::State}; use serde::{Deserialize, Serialize}; +use sqlx::PgPool; use std::path::PathBuf; use tracing::{info, warn}; +use uuid::Uuid; /// Response for reset check endpoint #[derive(Debug, Serialize)] @@ -41,34 +47,299 @@ pub struct ResetCheckResponse { pub media_count: usize, } -/// Request to reset the database -#[derive(Debug, Deserialize)] -pub struct ResetDatabaseRequest { - /// Reset all users and authentication data - pub reset_users: bool, - /// Reset all libraries - pub reset_libraries: bool, - /// Reset all media entries - pub reset_media: bool, - /// Confirmation string - must equal "RESET_DATABASE" - pub confirmation: String, +#[derive(Debug, Default)] +struct UserCleanupCounts { + sessions: usize, + watch_status: usize, +} + +#[derive(Debug, Default)] +struct FullClearCounts { + intelligence_root_rows: u64, + collection_root_rows: u64, + maintenance_rows: u64, + media_cache_root_rows: u64, +} + +fn is_full_clear(request: &ResetDatabaseRequest) -> bool { + request.reset_users && request.reset_libraries && request.reset_media +} + +fn invoking_user_last( + mut users: Vec, + invoking_user_id: Uuid, +) -> Vec { + if let Some(index) = + users.iter().position(|user| user.id == invoking_user_id) + { + let invoking_user = users.remove(index); + users.push(invoking_user); + } + users +} + +fn checked_i64_count(value: i64, label: &str) -> AppResult { + usize::try_from(value).map_err(|_| { + AppError::internal(format!( + "Invalid {label} row count returned by database" + )) + }) +} + +fn checked_u64_count(value: u64, label: &str) -> AppResult { + usize::try_from(value).map_err(|_| { + AppError::internal(format!( + "{label} row count exceeds this platform's limit" + )) + }) +} + +async fn count_media_items( + pool: &PgPool, + library_ids: &[Uuid], +) -> AppResult { + if library_ids.is_empty() { + return Ok(0); + } + + let count = sqlx::query_scalar!( + r#" + SELECT ( + (SELECT COUNT(*) FROM movie_references WHERE library_id = ANY($1)) + + (SELECT COUNT(*) FROM series WHERE library_id = ANY($1)) + + (SELECT COUNT(*) FROM season_references WHERE library_id = ANY($1)) + + ( + SELECT COUNT(*) + FROM episode_references er + JOIN series s ON s.id = er.series_id + WHERE s.library_id = ANY($1) + ) + )::bigint AS "count!" + "#, + library_ids, + ) + .fetch_one(pool) + .await + .map_err(AppError::from)?; + + checked_i64_count(count, "media") +} + +async fn clear_full_wipe_roots(pool: &PgPool) -> AppResult { + let mut tx = pool.begin().await.map_err(AppError::from)?; + let mut counts = FullClearCounts::default(); + + // Provenance edges have kind-specific CHECK constraints. Delete them + // before any referenced intelligence row can be nulled by an FK action. + counts.intelligence_root_rows += + sqlx::query!("DELETE FROM intelligence_artifact_sources") + .execute(&mut *tx) + .await + .map_err(AppError::from)? + .rows_affected(); + counts.intelligence_root_rows += + sqlx::query!("DELETE FROM intelligence_artifacts") + .execute(&mut *tx) + .await + .map_err(AppError::from)? + .rows_affected(); + counts.intelligence_root_rows += + sqlx::query!("DELETE FROM intelligence_runs") + .execute(&mut *tx) + .await + .map_err(AppError::from)? + .rows_affected(); + counts.intelligence_root_rows += + sqlx::query!("DELETE FROM intelligence_search_documents") + .execute(&mut *tx) + .await + .map_err(AppError::from)? + .rows_affected(); + counts.intelligence_root_rows += + sqlx::query!("DELETE FROM intelligence_media_context") + .execute(&mut *tx) + .await + .map_err(AppError::from)? + .rows_affected(); + + // Shelf placements and imported sources intentionally outlive collection + // definitions, so they are roots rather than collection-owned children. + counts.collection_root_rows += + sqlx::query!("DELETE FROM collection_shelf_placements") + .execute(&mut *tx) + .await + .map_err(AppError::from)? + .rows_affected(); + counts.collection_root_rows += + sqlx::query!("DELETE FROM collection_sources") + .execute(&mut *tx) + .await + .map_err(AppError::from)? + .rows_affected(); + counts.collection_root_rows += + sqlx::query!("DELETE FROM collection_definitions") + .execute(&mut *tx) + .await + .map_err(AppError::from)? + .rows_affected(); + + counts.maintenance_rows += + sqlx::query!("DELETE FROM library_maintenance_operations") + .execute(&mut *tx) + .await + .map_err(AppError::from)? + .rows_affected(); + + tx.commit().await.map_err(AppError::from)?; + Ok(counts) +} + +async fn clear_global_media_caches( + pool: &PgPool, + counts: &mut FullClearCounts, +) -> AppResult<()> { + let mut tx = pool.begin().await.map_err(AppError::from)?; + + counts.media_cache_root_rows += sqlx::query!("DELETE FROM cached_images") + .execute(&mut *tx) + .await + .map_err(AppError::from)? + .rows_affected(); + counts.media_cache_root_rows += + sqlx::query!("DELETE FROM tmdb_image_variants") + .execute(&mut *tx) + .await + .map_err(AppError::from)? + .rows_affected(); + counts.media_cache_root_rows += sqlx::query!("DELETE FROM persons") + .execute(&mut *tx) + .await + .map_err(AppError::from)? + .rows_affected(); + + tx.commit().await.map_err(AppError::from)?; + Ok(()) } -/// Result of database reset operation -#[derive(Debug, Serialize, Default)] -pub struct ResetResult { - /// Number of users deleted - pub users_deleted: usize, - /// Number of sessions deleted - pub sessions_deleted: usize, - /// Number of roles reset - pub roles_reset: usize, - /// Number of libraries deleted - pub libraries_deleted: usize, - /// Number of media items deleted - pub media_deleted: usize, - /// Number of watch status entries deleted - pub watch_status_deleted: usize, +async fn prepare_user_reset( + pool: &PgPool, + invoking_user_id: Uuid, +) -> AppResult { + let mut tx = pool.begin().await.map_err(AppError::from)?; + + // These sync FKs use restrictive defaults instead of cascading from users. + sqlx::query!("DELETE FROM sync_session_history") + .execute(&mut *tx) + .await + .map_err(AppError::from)?; + sqlx::query!("DELETE FROM sync_sessions") + .execute(&mut *tx) + .await + .map_err(AppError::from)?; + + // A grantor can otherwise be kept alive by grants owned by another user. + // Preserve the grants themselves so the invoking administrator retains + // recovery authority until their user is deliberately deleted last. + sqlx::query!("UPDATE user_permissions SET granted_by = NULL") + .execute(&mut *tx) + .await + .map_err(AppError::from)?; + sqlx::query!("UPDATE user_roles SET granted_by = NULL") + .execute(&mut *tx) + .await + .map_err(AppError::from)?; + + // A device session can also cascade through `first_authenticated_by`, even + // when it belongs to the invoking user. Anchor their devices to their owner + // and detach their bearer sessions from the secondary device FK so deleting + // another user cannot invalidate the administrator's recovery session. + sqlx::query!( + r#" + UPDATE auth_device_sessions + SET first_authenticated_by = user_id + WHERE user_id = $1 + AND first_authenticated_by <> user_id + "#, + invoking_user_id, + ) + .execute(&mut *tx) + .await + .map_err(AppError::from)?; + sqlx::query!( + r#" + UPDATE auth_sessions + SET device_session_id = NULL + WHERE user_id = $1 + "#, + invoking_user_id, + ) + .execute(&mut *tx) + .await + .map_err(AppError::from)?; + + // Count sessions now but leave them user-owned. Their FK cascade runs with + // each user deletion, which keeps the invoking administrator's sessions + // usable if an earlier user cannot be deleted. + let sessions = sqlx::query_scalar!( + r#"SELECT COUNT(*)::bigint AS "count!" FROM auth_sessions"# + ) + .fetch_one(&mut *tx) + .await + .map_err(AppError::from)?; + let watch_status = sqlx::query!("DELETE FROM user_watch_progress") + .execute(&mut *tx) + .await + .map_err(AppError::from)? + .rows_affected() + + sqlx::query!("DELETE FROM user_completed_media") + .execute(&mut *tx) + .await + .map_err(AppError::from)? + .rows_affected() + + sqlx::query!("DELETE FROM user_episode_state") + .execute(&mut *tx) + .await + .map_err(AppError::from)? + .rows_affected() + + sqlx::query!("DELETE FROM user_view_history") + .execute(&mut *tx) + .await + .map_err(AppError::from)? + .rows_affected(); + + // These operational/authentication records either have SET NULL ownership + // or no owner FK and would otherwise survive a first-run reset. + sqlx::query!("DELETE FROM auth_events") + .execute(&mut *tx) + .await + .map_err(AppError::from)?; + sqlx::query!("DELETE FROM security_audit_log") + .execute(&mut *tx) + .await + .map_err(AppError::from)?; + sqlx::query!("DELETE FROM login_attempts") + .execute(&mut *tx) + .await + .map_err(AppError::from)?; + sqlx::query!("DELETE FROM rate_limit_state") + .execute(&mut *tx) + .await + .map_err(AppError::from)?; + sqlx::query!("DELETE FROM setup_claims") + .execute(&mut *tx) + .await + .map_err(AppError::from)?; + sqlx::query!("DELETE FROM auth_security_settings") + .execute(&mut *tx) + .await + .map_err(AppError::from)?; + + tx.commit().await.map_err(AppError::from)?; + + Ok(UserCleanupCounts { + sessions: checked_i64_count(sessions, "session")?, + watch_status: checked_u64_count(watch_status, "watch status")?, + }) } /// Check if database reset is available @@ -112,9 +383,12 @@ pub async fn check_reset_status( AppError::internal(format!("Failed to get libraries: {}", e)) })?; let libraries = demo_mode::filter_libraries(&state, libraries); - - // Get media count (this is a simplified count - you might want to add a dedicated method) - let media_count = 0; // TODO: Implement actual media count + let library_ids = libraries + .iter() + .map(|library| *library.id.as_uuid()) + .collect::>(); + let postgres = state.postgres(); + let media_count = count_media_items(postgres.pool(), &library_ids).await?; let response = ResetCheckResponse { is_development: cfg!(debug_assertions), @@ -135,7 +409,7 @@ pub async fn reset_database( State(state): State, Extension(user): Extension, Json(request): Json, -) -> AppResult>> { +) -> AppResult>> { // Check permissions let perms = state .unit_of_work() @@ -161,96 +435,144 @@ pub async fn reset_database( )); } + if request.reset_media && !request.reset_libraries { + return Err(AppError::bad_request( + "Media reset requires library reset so runtime state is cleaned safely", + )); + } + warn!( "Database reset requested with options: users={}, libraries={}, media={}", request.reset_users, request.reset_libraries, request.reset_media ); - let mut result = ResetResult::default(); + let full_clear = is_full_clear(&request); + let invoking_user_id = user.id; + let postgres = state.postgres(); + let pool = postgres.pool(); + let mut result = ResetDatabaseResult::default(); - if request.reset_users { - info!("Resetting user data..."); + // Resolve and count every target before mutating the database. This keeps + // the response tied to the exact library/user set selected for this run. + let libraries = if request.reset_libraries { + let libraries = state + .unit_of_work() + .libraries + .list_libraries() + .await + .map_err(|e| { + AppError::internal(format!("Failed to get libraries: {e}")) + })?; + demo_mode::filter_libraries(&state, libraries) + } else { + Vec::new() + }; + let library_ids = libraries + .iter() + .map(|library| *library.id.as_uuid()) + .collect::>(); + let media_count = count_media_items(pool, &library_ids).await?; - // Get all users before deletion for count - let users = - state - .unit_of_work() - .users - .get_all_users() - .await - .map_err(|e| { - AppError::internal(format!("Failed to get users: {}", e)) - })?; - result.users_deleted = users.len(); + let users = if request.reset_users { + state + .unit_of_work() + .users + .get_all_users() + .await + .map_err(|e| { + AppError::internal(format!("Failed to get users: {e}")) + })? + } else { + Vec::new() + }; - // Delete all users (this should cascade to related tables) - for user in users { - state - .unit_of_work() - .users - .delete_user(user.id) + // Full clear includes detached/global roots which do not cascade from a + // library or user. Removing provenance/collection roots first also avoids + // their restrictive CHECK/FK combinations blocking the owner deletions. + let mut full_clear_counts = if full_clear { + Some(clear_full_wipe_roots(pool).await?) + } else { + None + }; + + // Delete library-owned data before users. A failed library cascade must not + // leave the installation without the administrator who can repair it. + if request.reset_libraries { + info!("Resetting library data..."); + + for library in libraries { + delete_library_with_runtime_cleanup(&state, library.id) .await .map_err(|e| { AppError::internal(format!( - "Failed to delete user {}: {}", - user.id, e + "Failed to delete library {}: {}", + library.id, e )) })?; + result.libraries_deleted += 1; } - - // Roles are system data and shouldn't be deleted, just reset to defaults - // The migration scripts should handle ensuring default roles exist - result.roles_reset = 3; // admin, user, guest + result.media_deleted = media_count; info!( - "User data reset complete. {} users deleted", - result.users_deleted + "Library data reset complete. {} libraries and {} media items deleted", + result.libraries_deleted, result.media_deleted ); } - if request.reset_libraries { - info!("Resetting library data..."); + if let Some(counts) = full_clear_counts.as_mut() { + // Global image/person caches are no longer referenced after all library + // rows have been removed, so they can now be deleted transactionally. + clear_global_media_caches(pool, counts).await?; + } - // Get all libraries - let libraries = state - .unit_of_work() - .libraries - .list_libraries() - .await - .map_err(|e| { - AppError::internal(format!("Failed to get libraries: {}", e)) - })?; - let libraries = demo_mode::filter_libraries(&state, libraries); - result.libraries_deleted = libraries.len(); + if request.reset_users { + info!("Resetting user data..."); - // Delete all libraries - for library in libraries { + let cleanup_counts = prepare_user_reset(pool, invoking_user_id).await?; + + // Preserve the invoking administrator and their role until every other + // user deletion succeeds. If an earlier delete fails, they can log in + // again and retry or repair the remaining state. + for user in invoking_user_last(users, invoking_user_id) { state .unit_of_work() - .libraries - .delete_library(library.id) + .users + .delete_user(user.id) .await .map_err(|e| { AppError::internal(format!( - "Failed to delete library {}: {}", - library.id, e + "Failed to delete user {}: {}", + user.id, e )) })?; + result.users_deleted += 1; } + result.sessions_deleted = cleanup_counts.sessions; + result.watch_status_deleted = cleanup_counts.watch_status; + info!( - "Library data reset complete. {} libraries deleted", - result.libraries_deleted + "User data reset complete. {} users, {} sessions, and {} watch rows deleted", + result.users_deleted, + result.sessions_deleted, + result.watch_status_deleted ); } if request.reset_media { - info!("Resetting media data..."); + info!( + "Media reset completed through library-owned cascades: {} logical media items deleted", + result.media_deleted + ); + } - // This would require implementing media deletion methods - // For now, deleting libraries should cascade to media entries - warn!( - "Direct media deletion not yet implemented. Media will be deleted when libraries are deleted." + if let Some(counts) = full_clear_counts { + info!( + intelligence_root_rows = counts.intelligence_root_rows, + collection_root_rows = counts.collection_root_rows, + maintenance_rows = counts.maintenance_rows, + media_cache_root_rows = counts.media_cache_root_rows, + "Detached/global clear-all data deleted" ); } @@ -404,16 +726,30 @@ pub async fn seed_database( mod tests { use super::*; + fn test_user(id: Uuid, username: &str) -> User { + let now = chrono::Utc::now(); + User { + id, + username: username.to_string(), + display_name: username.to_string(), + avatar_url: None, + created_at: now, + updated_at: now, + last_login: None, + is_active: true, + email: None, + preferences: Default::default(), + } + } + #[test] fn test_reset_request_validation() { - let valid = ResetDatabaseRequest { - reset_users: true, - reset_libraries: true, - reset_media: false, - confirmation: "RESET_DATABASE".to_string(), - }; + let valid = ResetDatabaseRequest::clear_all_data(); - // Should be valid + assert!(is_full_clear(&valid)); + assert!(valid.reset_users); + assert!(valid.reset_libraries); + assert!(valid.reset_media); assert_eq!(valid.confirmation, "RESET_DATABASE"); let invalid = ResetDatabaseRequest { @@ -425,5 +761,38 @@ mod tests { // Should be invalid assert_ne!(invalid.confirmation, "RESET_DATABASE"); + assert!(!is_full_clear(&invalid)); + } + + #[test] + fn invoking_administrator_is_always_deleted_last() { + let first_id = Uuid::new_v4(); + let invoking_id = Uuid::new_v4(); + let third_id = Uuid::new_v4(); + let users = vec![ + test_user(invoking_id, "invoking"), + test_user(first_id, "first"), + test_user(third_id, "third"), + ]; + + let ordered = invoking_user_last(users, invoking_id); + let ordered_ids = + ordered.iter().map(|user| user.id).collect::>(); + + assert_eq!(ordered_ids, vec![first_id, third_id, invoking_id]); + } + + #[test] + fn missing_invoking_user_does_not_reorder_targets() { + let first_id = Uuid::new_v4(); + let second_id = Uuid::new_v4(); + let users = + vec![test_user(first_id, "first"), test_user(second_id, "second")]; + + let ordered = invoking_user_last(users, Uuid::new_v4()); + let ordered_ids = + ordered.iter().map(|user| user.id).collect::>(); + + assert_eq!(ordered_ids, vec![first_id, second_id]); } } diff --git a/crates/ferrex-server/src/handlers/media/handle_library.rs b/crates/ferrex-server/src/handlers/media/handle_library.rs index 9bf25b26..a2303fb5 100644 --- a/crates/ferrex-server/src/handlers/media/handle_library.rs +++ b/crates/ferrex-server/src/handlers/media/handle_library.rs @@ -5,7 +5,7 @@ use axum::{ http::header, response::{IntoResponse, Json, Response}, }; -use ferrex_core::domain::users::user::User; +use ferrex_core::domain::users::{rbac, user::User}; use ferrex_core::error::MediaError; use ferrex_core::query::{ filtering::hash_filter_spec, @@ -18,6 +18,7 @@ use ferrex_core::{ api::types::{ ApiResponse, CreateLibraryRequest, FetchMediaRequest, FilterIndicesRequest, IndicesResponse, LibraryMediaResponse, + ResetLibraryRequest, ResetLibraryResult, ScanCommandAcceptedResponse, ScanRunMode, UpdateLibraryRequest, }, types::LibraryType, @@ -44,12 +45,351 @@ use ferrex_core::domain::scan::orchestration::LibraryActorConfig; use futures::{StreamExt, TryStreamExt, stream}; use once_cell::sync::Lazy; use parking_lot::RwLock; +use sqlx::PgPool; const FILTER_CACHE_TTL: Duration = Duration::from_secs(30); const MIN_SCAN_INTERVAL_MINUTES: u32 = 1; static FILTER_CACHE: Lazy>> = Lazy::new(|| RwLock::new(HashMap::new())); +static LIBRARY_MAINTENANCE_LOCK: Lazy> = + Lazy::new(|| tokio::sync::Mutex::new(())); + +async fn require_library_permission( + state: &AppState, + user: &User, + permission: &str, +) -> Result<(), StatusCode> { + let permissions = state + .unit_of_work() + .rbac + .get_user_permissions(user.id) + .await + .map_err(|err| { + error!(user_id = %user.id, error = %err, "failed to load library permissions"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + if permissions.has_permission(permission) || permissions.has_role("admin") { + Ok(()) + } else { + Err(StatusCode::FORBIDDEN) + } +} + +fn actor_config_for_library( + state: &AppState, + library: &Library, +) -> LibraryActorConfig { + LibraryActorConfig { + library: LibraryReference { + id: library.id, + name: library.name.clone(), + library_type: library.library_type, + paths: library.paths.clone(), + }, + root_paths: library.paths.clone(), + max_outstanding_jobs: state + .config() + .scanner + .library_actor_max_outstanding_jobs, + } +} + +/// Stop runtime producers before deleting library-owned data. If persistence +/// rejects the delete, restore the actor/watch configuration so the library is +/// not left present-but-inert. +pub(crate) async fn delete_library_with_runtime_cleanup( + state: &AppState, + library_id: LibraryId, +) -> Result<(), String> { + let _maintenance_guard = LIBRARY_MAINTENANCE_LOCK.lock().await; + let libraries = state.unit_of_work().libraries.clone(); + let library = libraries + .get_library(library_id) + .await + .map_err(|err| { + format!("Failed to load library before deletion: {err}") + })? + .ok_or_else(|| "Library not found".to_string())?; + let orchestrator = state.scan_control().orchestrator(); + let actor_config = actor_config_for_library(state, &library); + + orchestrator + .unregister_library(&actor_config, library.watch_for_changes) + .await + .map_err(|err| format!("Failed to stop library scan runtime: {err}"))?; + + if let Err(delete_error) = libraries.delete_library(library_id).await { + let restore_result = orchestrator + .register_library(actor_config, library.watch_for_changes) + .await; + + return match restore_result { + Ok(()) => Err(format!("Delete failed: {delete_error}")), + Err(restore_error) => Err(format!( + "Delete failed: {delete_error}; scan runtime restoration also failed: {restore_error}" + )), + }; + } + + state.scan_control().forget_library(library_id).await; + invalidate_filter_cache_for(*library_id.as_uuid()); + Ok(()) +} + +async fn completed_reset_library( + pool: &PgPool, + operation_id: Uuid, +) -> Result, String> { + sqlx::query_scalar!( + r#" + SELECT library_id + FROM library_maintenance_operations + WHERE operation_id = $1 + AND operation = 'reset' + "#, + operation_id, + ) + .fetch_optional(pool) + .await + .map(|library_id| library_id.map(LibraryId)) + .map_err(|err| format!("Failed to load reset operation: {err}")) +} + +async fn reset_library_data( + pool: &PgPool, + library_id: LibraryId, + operation_id: Uuid, +) -> Result { + let mut tx = pool + .begin() + .await + .map_err(|err| format!("Failed to begin library reset: {err}"))?; + + sqlx::query!( + r#"SELECT pg_advisory_xact_lock(hashtextextended($1::uuid::text, 0))"#, + library_id.0, + ) + .execute(&mut *tx) + .await + .map_err(|err| format!("Failed to lock library reset operation: {err}"))?; + + if let Some(existing_library_id) = sqlx::query_scalar!( + r#" + SELECT library_id + FROM library_maintenance_operations + WHERE operation_id = $1 + AND operation = 'reset' + "#, + operation_id, + ) + .fetch_optional(&mut *tx) + .await + .map_err(|err| format!("Failed to replay library reset: {err}"))? + { + if existing_library_id != library_id.0 { + return Err(format!( + "Reset operation {operation_id} belongs to another library" + )); + } + tx.commit() + .await + .map_err(|err| format!("Failed to finish reset replay: {err}"))?; + return Ok(false); + } + + let restored_library_id = sqlx::query_scalar!( + r#" + WITH deleted AS ( + DELETE FROM libraries + WHERE id = $1 + RETURNING + id, + name, + library_type, + paths, + scan_interval_minutes, + enabled, + auto_scan, + watch_for_changes, + analyze_on_scan, + max_retry_attempts, + movie_ref_batch_size, + created_at + ) + INSERT INTO libraries ( + id, + name, + library_type, + paths, + scan_interval_minutes, + enabled, + auto_scan, + watch_for_changes, + analyze_on_scan, + max_retry_attempts, + movie_ref_batch_size, + created_at, + updated_at + ) + SELECT + id, + name, + library_type, + paths, + scan_interval_minutes, + enabled, + auto_scan, + watch_for_changes, + analyze_on_scan, + max_retry_attempts, + movie_ref_batch_size, + created_at, + NOW() + FROM deleted + RETURNING id + "#, + library_id.0, + ) + .fetch_optional(&mut *tx) + .await + .map_err(|err| format!("Failed to reset library-owned data: {err}"))? + .ok_or_else(|| "Library not found".to_string())?; + + sqlx::query!( + r#" + INSERT INTO library_maintenance_operations ( + operation_id, + library_id, + operation + ) + VALUES ($1, $2, 'reset') + "#, + operation_id, + restored_library_id, + ) + .execute(&mut *tx) + .await + .map_err(|err| format!("Failed to record library reset: {err}"))?; + + tx.commit() + .await + .map_err(|err| format!("Failed to commit library reset: {err}"))?; + Ok(true) +} + +async fn reset_library_with_runtime_cleanup( + state: &AppState, + library_id: LibraryId, + operation_id: Uuid, +) -> Result { + let _maintenance_guard = LIBRARY_MAINTENANCE_LOCK.lock().await; + let postgres = state.postgres(); + let pool = postgres.pool(); + let libraries = state.unit_of_work().libraries.clone(); + + if let Some(existing_library_id) = + completed_reset_library(pool, operation_id).await? + && existing_library_id != library_id + { + return Err(format!( + "Reset operation {operation_id} belongs to another library" + )); + } + + let library = libraries + .get_library(library_id) + .await + .map_err(|err| format!("Failed to load library before reset: {err}"))? + .ok_or_else(|| "Library not found".to_string())?; + let actor_config = actor_config_for_library(state, &library); + let reset_already_completed = + completed_reset_library(pool, operation_id).await?.is_some(); + + if !reset_already_completed { + state + .scan_control() + .orchestrator() + .unregister_library(&actor_config, library.watch_for_changes) + .await + .map_err(|err| { + format!("Failed to stop library scan runtime: {err}") + })?; + } + + let applied = match reset_library_data(pool, library_id, operation_id).await + { + Ok(applied) => applied, + Err(reset_error) => { + if !reset_already_completed { + let restore_result = state + .scan_control() + .orchestrator() + .register_library(actor_config, library.watch_for_changes) + .await; + return match restore_result { + Ok(()) => Err(reset_error), + Err(restore_error) => Err(format!( + "{reset_error}; scan runtime restoration also failed: {restore_error}" + )), + }; + } + return Err(reset_error); + } + }; + + if applied { + state.scan_control().forget_library(library_id).await; + invalidate_filter_cache_for(*library_id.as_uuid()); + } + + let restored_library = libraries + .get_library(library_id) + .await + .map_err(|err| format!("Failed to load reset library: {err}"))? + .ok_or_else(|| "Reset library was not restored".to_string())?; + state + .scan_control() + .orchestrator() + .register_library( + actor_config_for_library(state, &restored_library), + restored_library.watch_for_changes, + ) + .await + .map_err(|err| { + format!("Failed to restore library scan runtime: {err}") + })?; + + let scan = if restored_library.enabled { + let accepted = state + .scan_control() + .start_library_scan( + library_id, + Some(operation_id), + ScanRunMode::Manual, + ) + .await + .map_err(|err| { + format!( + "Library data reset completed, but the fresh scan could not be started: {err}" + ) + })?; + Some(ScanCommandAcceptedResponse { + scan_id: accepted.scan_id, + correlation_id: accepted.correlation_id, + status: accepted.status.into(), + mode: accepted.mode, + idempotency_key: accepted.idempotency_key, + run_key: accepted.run_key, + disposition: accepted.disposition, + }) + } else { + None + }; + + Ok(ResetLibraryResult { library_id, scan }) +} #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] struct FilterCacheKey { @@ -721,8 +1061,15 @@ pub async fn get_library_handler( /// Create a new library pub async fn create_library_handler( State(state): State, + Extension(user): Extension, Json(request): Json, ) -> Result>, StatusCode> { + require_library_permission( + &state, + &user, + rbac::permissions::LIBRARIES_CREATE, + ) + .await?; if demo_mode::is_demo_mode(&state) { return Ok(Json(ApiResponse::error( "Library creation is disabled in demo mode".to_string(), @@ -765,8 +1112,8 @@ pub async fn create_library_handler( media: None, auto_scan: request.auto_scan, watch_for_changes: request.watch_for_changes, - analyze_on_scan: false, - max_retry_attempts: 3, + analyze_on_scan: request.analyze_on_scan, + max_retry_attempts: request.max_retry_attempts, movie_ref_batch_size, }; @@ -857,9 +1204,17 @@ pub async fn create_library_handler( /// Update an existing library pub async fn update_library_handler( State(state): State, + Extension(user): Extension, Path(id): Path, // TODO: Use LibraryID directly Json(request): Json, ) -> Result>, StatusCode> { + require_library_permission( + &state, + &user, + rbac::permissions::LIBRARIES_UPDATE, + ) + .await?; + info!("Updating library: {}", id); // Get the existing library @@ -910,6 +1265,12 @@ pub async fn update_library_handler( if let Some(watch_for_changes) = request.watch_for_changes { library.watch_for_changes = watch_for_changes; } + if let Some(analyze_on_scan) = request.analyze_on_scan { + library.analyze_on_scan = analyze_on_scan; + } + if let Some(max_retry_attempts) = request.max_retry_attempts { + library.max_retry_attempts = max_retry_attempts; + } if let Some(size) = request.movie_ref_batch_size { match ferrex_core::types::ids::MovieReferenceBatchSize::new(size) { Ok(value) => { @@ -983,8 +1344,16 @@ pub async fn update_library_handler( /// Delete a library pub async fn delete_library_handler( State(state): State, + Extension(user): Extension, Path(id): Path, ) -> Result>, StatusCode> { + require_library_permission( + &state, + &user, + rbac::permissions::LIBRARIES_DELETE, + ) + .await?; + info!("Deleting library: {}", id); let library_uuid = @@ -996,10 +1365,7 @@ pub async fn delete_library_handler( return Ok(Json(ApiResponse::error("Library not found".to_string()))); } - match state - .unit_of_work() - .libraries - .delete_library(LibraryId(library_uuid)) + match delete_library_with_runtime_cleanup(&state, LibraryId(library_uuid)) .await { Ok(_) => { @@ -1012,3 +1378,147 @@ pub async fn delete_library_handler( } } } + +/// Atomically clear library-owned data while preserving library identity and +/// configuration, then start an idempotent fresh scan. +pub async fn reset_library_handler( + State(state): State, + Extension(user): Extension, + Path(id): Path, + Json(request): Json, +) -> Result>, StatusCode> { + require_library_permission( + &state, + &user, + rbac::permissions::LIBRARIES_DELETE, + ) + .await?; + require_library_permission( + &state, + &user, + rbac::permissions::LIBRARIES_CREATE, + ) + .await?; + require_library_permission( + &state, + &user, + rbac::permissions::LIBRARIES_SCAN, + ) + .await?; + + let library_id = + LibraryId(Uuid::parse_str(&id).map_err(|_| StatusCode::BAD_REQUEST)?); + if demo_mode::is_demo_mode(&state) + && !demo_mode::is_demo_library(&library_id) + { + return Ok(Json(ApiResponse::error("Library not found".to_string()))); + } + + match reset_library_with_runtime_cleanup( + &state, + library_id, + request.operation_id, + ) + .await + { + Ok(result) => Ok(Json(ApiResponse::success(result))), + Err(err) => { + error!(library_id = %library_id, error = %err, "failed to reset library"); + Ok(Json(ApiResponse::error(err))) + } + } +} + +#[cfg(test)] +mod reset_tests { + use super::*; + use sqlx::Row; + + #[sqlx::test(migrator = "ferrex_core::MIGRATOR")] + async fn reset_is_idempotent_and_preserves_library_configuration( + pool: PgPool, + ) { + let library_id = Uuid::now_v7(); + let operation_id = Uuid::now_v7(); + sqlx::query( + r#" + INSERT INTO libraries ( + id, name, library_type, paths, scan_interval_minutes, enabled, + auto_scan, watch_for_changes, analyze_on_scan, + max_retry_attempts, movie_ref_batch_size + ) + VALUES ($1, 'Reset Me', 'movies', ARRAY['/media/reset']::varchar[], + 137, true, false, true, true, 9, 333) + "#, + ) + .bind(library_id) + .execute(&pool) + .await + .expect("seed library"); + sqlx::query( + r#" + INSERT INTO library_scan_runs ( + scan_id, library_id, mode, correlation_id, status + ) + VALUES ($1, $2, 'manual', $3, 'running') + "#, + ) + .bind(Uuid::now_v7()) + .bind(library_id) + .bind(Uuid::now_v7()) + .execute(&pool) + .await + .expect("seed owned scan data"); + + assert!( + reset_library_data(&pool, LibraryId(library_id), operation_id,) + .await + .expect("apply reset") + ); + assert!( + !reset_library_data(&pool, LibraryId(library_id), operation_id,) + .await + .expect("replay reset") + ); + + let library = sqlx::query( + r#" + SELECT name, paths, scan_interval_minutes, enabled, auto_scan, + watch_for_changes, analyze_on_scan, max_retry_attempts, + movie_ref_batch_size, last_scan + FROM libraries + WHERE id = $1 + "#, + ) + .bind(library_id) + .fetch_one(&pool) + .await + .expect("load restored library"); + assert_eq!(library.get::("name"), "Reset Me"); + assert_eq!( + library.get::, _>("paths"), + vec!["/media/reset".to_string()] + ); + assert_eq!(library.get::("scan_interval_minutes"), 137); + assert!(library.get::("enabled")); + assert!(!library.get::("auto_scan")); + assert!(library.get::("watch_for_changes")); + assert!(library.get::("analyze_on_scan")); + assert_eq!(library.get::("max_retry_attempts"), 9); + assert_eq!(library.get::("movie_ref_batch_size"), 333); + assert!( + library + .get::>, _>("last_scan") + .is_none() + ); + + let owned_scan_count = sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM library_scan_runs WHERE library_id = $1", + ) + .bind(library_id) + .fetch_one(&pool) + .await + .expect("count reset scan rows"); + assert_eq!(owned_scan_count, 0); + } +} diff --git a/crates/ferrex-server/src/handlers/scan/handle_scan.rs b/crates/ferrex-server/src/handlers/scan/handle_scan.rs index 415d8b94..af16cbf7 100644 --- a/crates/ferrex-server/src/handlers/scan/handle_scan.rs +++ b/crates/ferrex-server/src/handlers/scan/handle_scan.rs @@ -667,6 +667,7 @@ pub async fn scan_config_handler( }, lease: LeaseConfigView { lease_ttl_secs: cfg.lease.lease_ttl_secs, + dispatch_timeout_ms: cfg.lease.dispatch_timeout_ms, }, watch: WatchConfigView { debounce_window_ms: cfg.watch.debounce_window_ms, diff --git a/crates/ferrex-server/src/infra/orchestration/maintenance.rs b/crates/ferrex-server/src/infra/orchestration/maintenance.rs index 3cc850d6..4a6054df 100644 --- a/crates/ferrex-server/src/infra/orchestration/maintenance.rs +++ b/crates/ferrex-server/src/infra/orchestration/maintenance.rs @@ -4,8 +4,8 @@ use std::sync::Arc; use chrono::{DateTime, Duration as ChronoDuration, Utc}; use ferrex_core::api::types::ScanRunMode; use ferrex_core::domain::scan::orchestration::{ - JobEvent, JobEventPayload, JobId, JobKind, MaintenanceLibrary, - MaintenancePlanningLimits, config::MaintenanceConfig, + DurableJobState, JobEvent, JobEventPayload, JobId, JobKind, + MaintenanceLibrary, MaintenancePlanningLimits, config::MaintenanceConfig, plan_maintenance_sweep, }; use ferrex_core::types::LibraryId; @@ -67,9 +67,106 @@ struct InFlightRun { correlation_id: Uuid, pending_jobs: HashSet, failed: bool, + enrolling: bool, + reconciliation_required: bool, started_at: DateTime, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RunOutcome { + Success, + Failed, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ReconcileStatus { + Missing, + Active, + Finished, +} + +impl InFlightRun { + fn new( + correlation_id: Uuid, + failed: bool, + started_at: DateTime, + ) -> Self { + Self { + correlation_id, + pending_jobs: HashSet::new(), + failed, + enrolling: true, + reconciliation_required: true, + started_at, + } + } + + fn track_job(&mut self, job_id: JobId) { + self.pending_jobs.insert(job_id); + } + + fn mark_failed(&mut self) { + self.failed = true; + } + + fn observe_terminal( + &mut self, + job_id: JobId, + terminal_failure: bool, + ) -> Option { + if !self.pending_jobs.remove(&job_id) { + return None; + } + self.failed |= terminal_failure; + self.outcome_if_finished() + } + + fn finish_enrollment(&mut self) { + self.enrolling = false; + } + + fn require_reconciliation(&mut self) { + self.reconciliation_required = true; + } + + fn reconcile(&mut self, jobs: &[DurableJobState]) -> Option { + // Discover downstream folder jobs carrying this run correlation, then + // reconcile all known IDs. Explicitly tracked merged jobs are retained + // even when PostgreSQL kept their older correlation. + for job in jobs { + if job.kind == JobKind::FolderScan + && job.correlation_id == Some(self.correlation_id) + { + self.pending_jobs.insert(job.job_id); + } + } + + for job in jobs { + if self.pending_jobs.contains(&job.job_id) && job.is_terminal() { + self.pending_jobs.remove(&job.job_id); + self.failed |= job.is_terminal_failure(); + } + } + + self.reconciliation_required = false; + self.outcome_if_finished() + } + + fn outcome_if_finished(&self) -> Option { + if self.enrolling + || self.reconciliation_required + || !self.pending_jobs.is_empty() + { + return None; + } + Some(if self.failed { + RunOutcome::Failed + } else { + RunOutcome::Success + }) + } +} + impl MaintenanceScheduler { async fn run(self: Arc) { let mut ticker = interval(Duration::from_millis( @@ -85,6 +182,7 @@ impl MaintenanceScheduler { break; } _ = ticker.tick() => { + self.reconcile_required_runs("scheduled_retry").await; self.expire_stalled_runs().await; self.schedule_due_libraries().await; } @@ -93,6 +191,13 @@ impl MaintenanceScheduler { Ok(event) => self.observe_job_event(event).await, Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { warn!(skipped, "maintenance scheduler lagged job events"); + { + let mut guard = self.in_flight.lock().await; + for run in guard.values_mut() { + run.require_reconciliation(); + } + } + self.reconcile_all_runs("job_event_lag").await; } Err(tokio::sync::broadcast::error::RecvError::Closed) => break, } @@ -160,6 +265,12 @@ impl MaintenanceScheduler { continue; } + if plan.root_discovery_truncated { + self.orchestrator.record_root_discovery_truncation( + plan.deferred_root_entries, + ); + } + if plan.requests.is_empty() { if plan.has_errors() { warn!( @@ -192,11 +303,23 @@ impl MaintenanceScheduler { now: DateTime, ) { let correlation_id = Uuid::now_v7(); - let mut pending_jobs = HashSet::new(); - let mut failed = plan.has_errors(); + let initially_failed = plan.has_errors(); let mut accepted = 0usize; let mut merged = 0usize; + { + let mut guard = self.in_flight.lock().await; + if guard.contains_key(&library_id) { + // Preserve the existing run; queue dedupe is only a secondary + // guard and must not replace its enrollment state. + return; + } + guard.insert( + library_id, + InFlightRun::new(correlation_id, initially_failed, now), + ); + } + for mut request in plan.requests { request.correlation_id = Some(correlation_id); match self.orchestrator.enqueue(request).await { @@ -206,10 +329,20 @@ impl MaintenanceScheduler { } else { merged += 1; } - pending_jobs.insert(handle.job_id); + let mut guard = self.in_flight.lock().await; + if let Some(run) = guard.get_mut(&library_id) + && run.correlation_id == correlation_id + { + run.track_job(handle.job_id); + } } Err(err) => { - failed = true; + let mut guard = self.in_flight.lock().await; + if let Some(run) = guard.get_mut(&library_id) + && run.correlation_id == correlation_id + { + run.mark_failed(); + } warn!( library = %library_id, error = %err, @@ -219,54 +352,70 @@ impl MaintenanceScheduler { } } - if pending_jobs.is_empty() { - if failed { - self.backoff(library_id, now).await; - } else if let Err(err) = self.mark_library_scanned(library_id).await - { - warn!( - library = %library_id, - error = %err, - "failed to update maintenance last_scan after empty enqueue" - ); - self.backoff(library_id, now).await; + let pending = { + let mut guard = self.in_flight.lock().await; + let Some(run) = guard.get_mut(&library_id) else { + return; + }; + if run.correlation_id != correlation_id { + return; } - return; - } - - let mut guard = self.in_flight.lock().await; - if guard.contains_key(&library_id) { - // A concurrent terminal event may have scheduled a new run; avoid - // replacing it. The queue dedupe still prevents duplicate floods. - return; - } + run.finish_enrollment(); + run.pending_jobs.len() + }; info!( library = %library_id, correlation = %correlation_id, accepted, merged, - pending = pending_jobs.len(), + pending, stale_cursors = plan.stale_cursor_count, new_root_folders = plan.new_root_folder_count, skipped_root_entries = plan.skipped_root_entries, + root_discovery_truncated = plan.root_discovery_truncated, + deferred_root_entries = plan.deferred_root_entries, plan_errors = plan.errors.len(), "scheduled incremental maintenance sweep" ); - guard.insert( - library_id, - InFlightRun { - correlation_id, - pending_jobs, - failed, - started_at: now, - }, - ); + // Jobs may have reached a terminal PostgreSQL state while enqueue was + // still returning (or their terminal events may already have fallen + // out of the bounded broadcast channel). Reconcile before waiting. + if let Err(err) = + self.reconcile_library(library_id, "post_enqueue").await + { + warn!( + library = %library_id, + correlation = %correlation_id, + error = %err, + "failed to reconcile maintenance jobs after enqueue" + ); + } } async fn observe_job_event(&self, event: JobEvent) { + let library_id = event.meta.library_id; + let event_correlation = event.meta.correlation_id; let (job_id, terminal_failure) = match event.payload { + JobEventPayload::Enqueued { + kind: JobKind::FolderScan, + job_id, + .. + } + | JobEventPayload::Merged { + kind: JobKind::FolderScan, + existing_job_id: job_id, + .. + } => { + let mut guard = self.in_flight.lock().await; + if let Some(run) = guard.get_mut(&library_id) + && run.correlation_id == event_correlation + { + run.track_job(job_id); + } + return; + } JobEventPayload::Completed { kind: JobKind::FolderScan, job_id, @@ -290,31 +439,181 @@ impl MaintenanceScheduler { _ => return, }; - let library_id = event.meta.library_id; let finished = { let mut guard = self.in_flight.lock().await; let Some(run) = guard.get_mut(&library_id) else { return; }; - if !run.pending_jobs.remove(&job_id) { - return; + let outcome = run.observe_terminal(job_id, terminal_failure); + let correlation_id = run.correlation_id; + if outcome.is_some() { + guard.remove(&library_id); + } + outcome.map(|outcome| (correlation_id, outcome)) + }; + + if let Some((correlation_id, outcome)) = finished { + self.finish_run(library_id, correlation_id, outcome).await; + } + } + + async fn expire_stalled_runs(&self) { + let now = Utc::now(); + let stall = ChronoDuration::milliseconds( + self.config.run_stall_timeout_ms.max(1) as i64, + ); + let expired = { + let guard = self.in_flight.lock().await; + guard + .iter() + .filter_map(|(library_id, run)| { + (now.signed_duration_since(run.started_at) >= stall) + .then_some((*library_id, run.correlation_id)) + }) + .collect::>() + }; + + for (library_id, correlation_id) in expired { + match self.reconcile_library(library_id, "pre_stall").await { + Ok(ReconcileStatus::Missing | ReconcileStatus::Finished) => { + continue; + } + Ok(ReconcileStatus::Active) => {} + Err(err) => { + warn!( + library = %library_id, + correlation = %correlation_id, + error = %err, + "durable maintenance reconciliation failed; deferring stall decision" + ); + continue; + } + } + + let removed = { + let mut guard = self.in_flight.lock().await; + let still_expired = guard + .get(&library_id) + .map(|run| { + run.correlation_id == correlation_id + && now.signed_duration_since(run.started_at) + >= stall + }) + .unwrap_or(false); + still_expired.then(|| guard.remove(&library_id)).flatten() + }; + if removed.is_none() { + continue; } - if terminal_failure { - run.failed = true; + warn!( + library = %library_id, + correlation = %correlation_id, + "maintenance sweep stalled before terminal state; last_scan not updated" + ); + self.backoff(library_id, now).await; + } + } + + async fn reconcile_all_runs(&self, reason: &'static str) { + let libraries: Vec = + self.in_flight.lock().await.keys().copied().collect(); + for library_id in libraries { + if let Err(err) = self.reconcile_library(library_id, reason).await { + warn!( + library = %library_id, + error = %err, + reason, + "failed to reconcile maintenance run from durable job state" + ); + } + } + } + + async fn reconcile_required_runs(&self, reason: &'static str) { + let libraries: Vec = self + .in_flight + .lock() + .await + .iter() + .filter_map(|(library_id, run)| { + run.reconciliation_required.then_some(*library_id) + }) + .collect(); + for library_id in libraries { + if let Err(err) = self.reconcile_library(library_id, reason).await { + warn!( + library = %library_id, + error = %err, + reason, + "required durable maintenance reconciliation still pending" + ); } + } + } + + async fn reconcile_library( + &self, + library_id: LibraryId, + reason: &'static str, + ) -> ferrex_core::error::Result { + let Some((correlation_id, pending_jobs)) = ({ + let guard = self.in_flight.lock().await; + guard.get(&library_id).map(|run| { + ( + run.correlation_id, + run.pending_jobs.iter().copied().collect::>(), + ) + }) + }) else { + return Ok(ReconcileStatus::Missing); + }; - if run.pending_jobs.is_empty() { - let run = guard.remove(&library_id).expect("run exists"); - Some((run.correlation_id, !run.failed)) - } else { - None + let durable = self + .orchestrator + .durable_job_states(correlation_id, &pending_jobs) + .await?; + tracing::debug!( + library = %library_id, + correlation = %correlation_id, + tracked_jobs = pending_jobs.len(), + durable_jobs = durable.len(), + reason, + "reconciling maintenance run from durable job state" + ); + + let finished = { + let mut guard = self.in_flight.lock().await; + let Some(run) = guard.get_mut(&library_id) else { + return Ok(ReconcileStatus::Missing); + }; + if run.correlation_id != correlation_id { + return Ok(ReconcileStatus::Missing); } + let outcome = run.reconcile(&durable); + if outcome.is_some() { + guard.remove(&library_id); + } + outcome }; - if let Some((correlation_id, success)) = finished { - if success { + if let Some(outcome) = finished { + self.finish_run(library_id, correlation_id, outcome).await; + Ok(ReconcileStatus::Finished) + } else { + Ok(ReconcileStatus::Active) + } + } + + async fn finish_run( + &self, + library_id: LibraryId, + correlation_id: Uuid, + outcome: RunOutcome, + ) { + match outcome { + RunOutcome::Success => { match self.mark_library_scanned(library_id).await { Ok(()) => info!( library = %library_id, @@ -331,7 +630,8 @@ impl MaintenanceScheduler { self.backoff(library_id, Utc::now()).await; } } - } else { + } + RunOutcome::Failed => { warn!( library = %library_id, correlation = %correlation_id, @@ -342,36 +642,6 @@ impl MaintenanceScheduler { } } - async fn expire_stalled_runs(&self) { - let now = Utc::now(); - let stall = ChronoDuration::milliseconds( - self.config.run_stall_timeout_ms.max(1) as i64, - ); - let expired = { - let mut guard = self.in_flight.lock().await; - let expired: Vec<_> = guard - .iter() - .filter_map(|(library_id, run)| { - (now.signed_duration_since(run.started_at) >= stall) - .then_some((*library_id, run.correlation_id)) - }) - .collect(); - for (library_id, _) in &expired { - guard.remove(library_id); - } - expired - }; - - for (library_id, correlation_id) in expired { - warn!( - library = %library_id, - correlation = %correlation_id, - "maintenance sweep stalled before terminal state; last_scan not updated" - ); - self.backoff(library_id, now).await; - } - } - async fn is_in_flight(&self, library_id: LibraryId) -> bool { self.in_flight.lock().await.contains_key(&library_id) } @@ -433,3 +703,102 @@ impl MaintenanceScheduler { .await } } + +#[cfg(test)] +mod tests { + use super::*; + use ferrex_core::domain::scan::orchestration::JobState; + + fn durable_folder_job( + correlation_id: Uuid, + job_id: JobId, + state: JobState, + ) -> DurableJobState { + DurableJobState { + job_id, + kind: JobKind::FolderScan, + state, + attempts: 0, + dedupe_key: format!("scan:test:{job_id}"), + correlation_id: Some(correlation_id), + path_key: None, + last_error: None, + created_at: Utc::now(), + updated_at: Utc::now(), + } + } + + #[test] + fn immediate_completion_during_enqueue_requires_post_enqueue_reconcile() { + let correlation_id = Uuid::now_v7(); + let job_id = JobId::new(); + let mut run = InFlightRun::new(correlation_id, false, Utc::now()); + + // The terminal event can beat the enqueue handle, so it cannot remove + // an ID that has not been enrolled yet. + assert_eq!(run.observe_terminal(job_id, false), None); + run.track_job(job_id); + run.finish_enrollment(); + assert!(run.reconciliation_required); + + assert_eq!( + run.reconcile(&[durable_folder_job( + correlation_id, + job_id, + JobState::Completed, + )]), + Some(RunOutcome::Success) + ); + } + + #[test] + fn lag_gate_blocks_event_only_completion_until_durable_read_succeeds() { + let correlation_id = Uuid::now_v7(); + let job_id = JobId::new(); + let mut run = InFlightRun::new(correlation_id, false, Utc::now()); + run.track_job(job_id); + run.finish_enrollment(); + + assert_eq!( + run.reconcile(&[durable_folder_job( + correlation_id, + job_id, + JobState::Leased, + )]), + None + ); + assert!(!run.reconciliation_required); + + run.require_reconciliation(); + assert_eq!(run.observe_terminal(job_id, false), None); + assert!(run.pending_jobs.is_empty()); + assert!(run.reconciliation_required); + + assert_eq!( + run.reconcile(&[durable_folder_job( + correlation_id, + job_id, + JobState::Completed, + )]), + Some(RunOutcome::Success) + ); + } + + #[test] + fn durable_terminal_failure_prevents_last_scan_success() { + let correlation_id = Uuid::now_v7(); + let job_id = JobId::new(); + let mut run = InFlightRun::new(correlation_id, false, Utc::now()); + run.track_job(job_id); + run.finish_enrollment(); + + assert_eq!( + run.reconcile(&[durable_folder_job( + correlation_id, + job_id, + JobState::DeadLetter, + )]), + Some(RunOutcome::Failed) + ); + } +} diff --git a/crates/ferrex-server/src/infra/orchestration/mod.rs b/crates/ferrex-server/src/infra/orchestration/mod.rs index 57a1e2b3..a8e0843b 100644 --- a/crates/ferrex-server/src/infra/orchestration/mod.rs +++ b/crates/ferrex-server/src/infra/orchestration/mod.rs @@ -47,7 +47,7 @@ use ferrex_core::domain::scan::orchestration::{ TranscriptExtractJob, TranscriptExtractTrigger, }, lease::{DequeueRequest, JobLease}, - queue::QueueService, + queue::{DurableJobState, QueueService}, runtime::{ InProcJobEventBus, LibraryActorHandle, LibraryCommandExecutor, OrchestratorRuntime, OrchestratorRuntimeBuilder, @@ -130,6 +130,8 @@ pub struct ScanOrchestrator { correlations: CorrelationCache, unit_of_work: Arc, maintenance: Mutex>, + root_discovery_truncated_sweeps: AtomicU64, + root_discovery_deferred_entries: AtomicU64, } impl fmt::Debug for ScanOrchestrator { @@ -250,6 +252,8 @@ impl ScanOrchestrator { correlations, unit_of_work, maintenance: Mutex::new(None), + root_discovery_truncated_sweeps: AtomicU64::new(0), + root_discovery_deferred_entries: AtomicU64::new(0), }) } @@ -281,6 +285,17 @@ impl ScanOrchestrator { self.events.subscribe_scan() } + pub(crate) async fn durable_job_states( + &self, + correlation_id: Uuid, + job_ids: &[ferrex_core::domain::scan::orchestration::JobId], + ) -> Result> { + self.runtime + .queue() + .durable_job_states(correlation_id, job_ids) + .await + } + pub fn config(&self) -> OrchestratorConfig { self.runtime.config().clone() } @@ -291,7 +306,7 @@ impl ScanOrchestrator { command: LibraryActorCommand, ) -> Result<()> { self.runtime - .submit_library_command(library_id, command) + .submit_library_command_and_wait(library_id, command) .await } @@ -336,6 +351,49 @@ impl ScanOrchestrator { self.watchers.unregister_library(library_id).await; } + /// Stop both filesystem watches and the actor mailbox for a library. + pub async fn unregister_library( + &self, + config: &LibraryActorConfig, + watch_for_changes: bool, + ) -> Result<()> { + let library_id = config.library.id; + self.watchers.unregister_library(library_id).await; + + if let Err(shutdown_error) = + self.runtime.unregister_library_actor(library_id).await + { + if watch_for_changes { + let roots = config + .root_paths + .iter() + .enumerate() + .map(|(idx, path)| { + (LibraryRootsId(idx as u16), path.clone()) + }) + .collect(); + if let Err(restore_error) = + self.watchers.register_library(library_id, roots).await + { + return Err(MediaError::Internal(format!( + "actor shutdown failed: {shutdown_error}; watcher restoration failed: {restore_error}" + ))); + } + } + return Err(shutdown_error); + } + + Ok(()) + } + + /// Drop scheduler state left behind after PostgreSQL cascades a library's + /// durable queue rows. + pub async fn forget_library_scheduler_state(&self, library_id: LibraryId) { + self.runtime + .forget_library_scheduler_state(library_id) + .await; + } + pub async fn incremental_status( &self, ) -> Result { @@ -374,9 +432,24 @@ impl ScanOrchestrator { stale_cursor_libraries: cursor.stale_cursor_libraries, stale_cursors: cursor.stale_cursors, oldest_cursor_staleness_ms: cursor.oldest_cursor_staleness_ms, + root_discovery_truncated_sweeps: self + .root_discovery_truncated_sweeps + .load(Ordering::Relaxed), + root_discovery_deferred_entries: self + .root_discovery_deferred_entries + .load(Ordering::Relaxed), }) } + fn record_root_discovery_truncation(&self, deferred_entries: usize) { + self.root_discovery_truncated_sweeps + .fetch_add(1, Ordering::Relaxed); + self.root_discovery_deferred_entries.fetch_add( + u64::try_from(deferred_entries).unwrap_or(u64::MAX), + Ordering::Relaxed, + ); + } + async fn file_watch_health(&self) -> Result { let queue = self.runtime.queue(); let row = sqlx::query!( diff --git a/crates/ferrex-server/src/infra/scan/movie_batch_notifier.rs b/crates/ferrex-server/src/infra/scan/movie_batch_notifier.rs index 1509008b..98a22df8 100644 --- a/crates/ferrex-server/src/infra/scan/movie_batch_notifier.rs +++ b/crates/ferrex-server/src/infra/scan/movie_batch_notifier.rs @@ -87,6 +87,15 @@ impl MovieBatchFinalizationNotifiers { let _ = notifier.stop_tx.send(true); notifier.task.abort(); } + + /// Stop notifier work immediately when its library has been deleted. + pub async fn forget_library(&self, library_id: LibraryId) { + let notifier = self.libraries.lock().await.remove(&library_id); + if let Some(notifier) = notifier { + let _ = notifier.stop_tx.send(true); + notifier.task.abort(); + } + } } async fn fetch_last_finalized_batch_id( diff --git a/crates/ferrex-server/src/infra/scan/scan_manager.rs b/crates/ferrex-server/src/infra/scan/scan_manager.rs index ea8a1e58..9859bc5f 100644 --- a/crates/ferrex-server/src/infra/scan/scan_manager.rs +++ b/crates/ferrex-server/src/infra/scan/scan_manager.rs @@ -12,10 +12,10 @@ use ferrex_core::{ LibraryRootsId, index::IndexingOutcome, }, orchestration::{ - JobEvent, LibraryActorCommand, LibraryScanRun, + DurableJobState, JobEvent, LibraryActorCommand, LibraryScanRun, LibraryScanRunProgressUpdate, NewLibraryScanRun, StartMode, events::{JobEventPayload, ScanEvent, ScanSeedSummary}, - job::{JobId, JobKind}, + job::{JobId, JobKind, JobState}, scan_cursor::{ScanCursor, ScanCursorRepository, normalize_path}, series::{SeriesBundleFinalization, SeriesBundleTracker}, }, @@ -223,6 +223,20 @@ impl ScanControlPlane { Arc::clone(&self.inner.orchestrator) } + /// Forget process-local state after a library and its durable scan rows + /// have been deleted successfully. + /// + /// PostgreSQL owns deletion of the durable queue and scan-run rows. This + /// removes their in-memory projections so they cannot remain visible as + /// active scans or consume scheduler reservations. + pub async fn forget_library(&self, library_id: LibraryId) { + self.inner.progress.forget_library(library_id).await; + self.inner + .orchestrator + .forget_library_scheduler_state(library_id) + .await; + } + pub fn subscribe_media_events( &self, ) -> broadcast::Receiver { @@ -298,22 +312,29 @@ impl ScanControlPlane { if disposition == ScanStartDisposition::Created { run.begin().await; + } - if let Err(err) = self - .inner - .orchestrator - .command_library( - durable.library_id, - LibraryActorCommand::Start { - mode: start_mode_from_scan_run_mode(durable.mode), - correlation_id: Some(durable.correlation_id), - }, - ) - .await - { - run.fail_with_reason("start_command_failed").await; - return Err(ScanControlError::internal(err.to_string())); + // Seed ownership is serialized per run and acknowledged only after the + // actor has made the complete seed batch durable. Reused concurrent + // starts therefore cannot dispatch duplicate Start commands. + if let Err(err) = + ensure_run_seeded(&run, &self.inner.orchestrator).await + { + let finalized = run.fail_with_reason("start_command_failed").await; + if finalized && run.start_mode() == StartMode::Bulk { + let _ = self + .inner + .orchestrator + .command_library( + durable.library_id, + LibraryActorCommand::Start { + mode: StartMode::Maintenance, + correlation_id: Some(run.correlation_id()), + }, + ) + .await; } + return Err(ScanControlError::internal(err.to_string())); } let snapshot = run.snapshot().await?; @@ -436,8 +457,21 @@ impl ScanControlPlane { .progress .lookup_for_library(scan_id, library_id) .await?; - let requested_correlation_id = Uuid::now_v7(); - run.pause(requested_correlation_id).await?; + let _transition = run.seed_transition.lock().await; + run.validate_pause().await?; + self.inner + .orchestrator + .command_library(library_id, LibraryActorCommand::Pause) + .await + .map_err(|err| ScanControlError::internal(err.to_string()))?; + if let Err(err) = run.pause().await { + let _ = self + .inner + .orchestrator + .command_library(library_id, LibraryActorCommand::Resume) + .await; + return Err(err); + } let snapshot = run.snapshot().await?; let mode = scan_run_mode_from_start_mode(run.start_mode()); let run_key = mode.run_key(run.library_id()); @@ -462,8 +496,40 @@ impl ScanControlPlane { .progress .lookup_for_library(scan_id, library_id) .await?; - let requested_correlation_id = Uuid::now_v7(); - run.resume(requested_correlation_id).await?; + let _transition = run.seed_transition.lock().await; + run.validate_resume().await?; + self.inner + .orchestrator + .command_library(library_id, LibraryActorCommand::Resume) + .await + .map_err(|err| ScanControlError::internal(err.to_string()))?; + if let Err(err) = run.resume().await { + let _ = self + .inner + .orchestrator + .command_library(library_id, LibraryActorCommand::Pause) + .await; + return Err(err); + } + if let Err(err) = seed_run_locked(&run, &self.inner.orchestrator).await + { + let finalized = + run.fail_with_reason("resume_seed_command_failed").await; + if finalized && run.start_mode() == StartMode::Bulk { + let _ = self + .inner + .orchestrator + .command_library( + library_id, + LibraryActorCommand::Start { + mode: StartMode::Maintenance, + correlation_id: Some(run.correlation_id()), + }, + ) + .await; + } + return Err(ScanControlError::internal(err.to_string())); + } let snapshot = run.snapshot().await?; let mode = scan_run_mode_from_start_mode(run.start_mode()); let run_key = mode.run_key(run.library_id()); @@ -488,8 +554,29 @@ impl ScanControlPlane { .progress .lookup_for_library(scan_id, library_id) .await?; - let requested_correlation_id = Uuid::now_v7(); - run.cancel(requested_correlation_id).await?; + let _transition = run.seed_transition.lock().await; + let finalized = run.cancel().await?; + if finalized + && run.start_mode() == StartMode::Bulk + && let Err(err) = self + .inner + .orchestrator + .command_library( + library_id, + LibraryActorCommand::Start { + mode: StartMode::Maintenance, + correlation_id: Some(run.correlation_id()), + }, + ) + .await + { + warn!( + library = %library_id, + scan = %scan_id, + error = %err, + "scan canceled but library actor did not return to maintenance" + ); + } let snapshot = run.snapshot().await?; let mode = scan_run_mode_from_start_mode(run.start_mode()); let run_key = mode.run_key(run.library_id()); @@ -599,6 +686,50 @@ impl ScanRunProgressArchive { events.remove(&evicted_scan_id); } } + + async fn forget_library(&self, library_id: LibraryId) { + let forgotten_scan_ids: HashSet = { + let mut history = self.history.write().await; + let scan_ids = history + .iter() + .filter(|entry| entry.library_id == library_id) + .map(|entry| entry.scan_id) + .collect(); + history.retain(|entry| entry.library_id != library_id); + scan_ids + }; + + if !forgotten_scan_ids.is_empty() { + self.final_events + .write() + .await + .retain(|scan_id, _| !forgotten_scan_ids.contains(scan_id)); + } + } +} + +async fn forget_active_library_runs( + active_by_scan_id: &RwLock>>, + active_by_run_key: &RwLock>>, + library_id: LibraryId, +) -> HashSet { + let removed_correlations = { + let mut by_scan_id = active_by_scan_id.write().await; + let correlations = by_scan_id + .values() + .filter(|run| run.library_id() == library_id) + .map(|run| run.correlation_id()) + .collect(); + by_scan_id.retain(|_, run| run.library_id() != library_id); + correlations + }; + + active_by_run_key + .write() + .await + .retain(|_, run| run.library_id() != library_id); + + removed_correlations } impl ScanRunProgressTracker { @@ -681,6 +812,52 @@ impl ScanRunProgressTracker { let run = self.register_durable_run(durable).await; run.seed_rehydrated_progress().await; + + // A paused row remains paused across restart. If it was paused + // before its seed completed, Resume owns the eventual seed claim. + if run.lifecycle_status().await == ScanLifecycleStatus::Paused { + if let Err(err) = self + .inner + .orchestrator + .command_library( + run.library_id(), + LibraryActorCommand::Pause, + ) + .await + { + warn!( + library = %run.library_id(), + scan = %run.scan_id(), + error = %err, + "failed to restore paused library actor during rehydration" + ); + } + } else if let Err(err) = + ensure_run_seeded(&run, &self.inner.orchestrator).await + { + warn!( + library = %run.library_id(), + scan = %run.scan_id(), + error = %err, + "failed to replay interrupted scan seed during rehydration" + ); + let finalized = run + .fail_with_reason("rehydrated_start_command_failed") + .await; + if finalized && run.start_mode() == StartMode::Bulk { + let _ = self + .inner + .orchestrator + .command_library( + run.library_id(), + LibraryActorCommand::Start { + mode: StartMode::Maintenance, + correlation_id: Some(run.correlation_id()), + }, + ) + .await; + } + } restored += 1; } @@ -701,6 +878,28 @@ impl ScanRunProgressTracker { snapshots } + async fn forget_library(&self, library_id: LibraryId) { + let removed_correlations = forget_active_library_runs( + &self.inner.active_by_scan_id, + &self.inner.active_by_run_key, + library_id, + ) + .await; + + self.inner.aggregator.forget_library(library_id).await; + self.inner + .movie_batch_notifiers + .forget_library(library_id) + .await; + self.inner.archive.forget_library(library_id).await; + + info!( + library = %library_id, + active_runs_removed = removed_correlations.len(), + "forgot deleted library scan progress" + ); + } + async fn history(&self, limit: usize) -> Vec { self.inner.archive.history(limit).await } @@ -873,6 +1072,13 @@ impl ScanLifecycleStatus { } } +fn lifecycle_allows_seed(status: &ScanLifecycleStatus) -> bool { + matches!( + status, + ScanLifecycleStatus::Pending | ScanLifecycleStatus::Running + ) +} + fn start_mode_from_scan_run_mode(mode: ScanRunMode) -> StartMode { match mode { ScanRunMode::Manual => StartMode::Bulk, @@ -938,6 +1144,8 @@ struct ScanRun { events: Mutex>, start_mode: StartMode, log: Mutex, + seed_transition: Mutex<()>, + finalization: Mutex, } #[derive(Debug)] @@ -961,10 +1169,12 @@ struct ScanRunState { last_activity_at: Option>, quiescence_started_at: Option>, last_error: Option, + tracked_job_ids: HashSet, item_states: HashMap, folder_outcomes_by_path: HashMap, historical_cursor_count: u64, seed_completed: bool, + durable_rebuild_pending: bool, } #[derive(Debug, Clone, Default)] @@ -1133,16 +1343,20 @@ impl ScanRun { last_activity_at: Some(durable.updated_at), quiescence_started_at: None, last_error: durable.last_error, + tracked_job_ids: durable.tracked_job_ids.into_iter().collect(), item_states: HashMap::new(), folder_outcomes_by_path: HashMap::new(), historical_cursor_count: 0, - seed_completed: false, + seed_completed: durable.seed_completed, + durable_rebuild_pending: true, }), tx, inner: Arc::downgrade(&inner), events: Mutex::new(VecDeque::with_capacity(EVENT_HISTORY_CAPACITY)), start_mode: start_mode_from_scan_run_mode(durable.mode), log: Mutex::new(ScanLogWatermark::default()), + seed_transition: Mutex::new(()), + finalization: Mutex::new(false), }) } @@ -1162,6 +1376,42 @@ impl ScanRun { self.start_mode } + async fn lifecycle_status(&self) -> ScanLifecycleStatus { + self.state.lock().await.status.clone() + } + + async fn validate_pause(&self) -> Result<(), ScanControlError> { + match self.lifecycle_status().await { + ScanLifecycleStatus::Running | ScanLifecycleStatus::Paused => { + Ok(()) + } + ScanLifecycleStatus::Completed + | ScanLifecycleStatus::Failed + | ScanLifecycleStatus::Canceled => { + Err(ScanControlError::ScanTerminal) + } + ScanLifecycleStatus::Pending => { + Err(ScanControlError::ScanNotRunning) + } + } + } + + async fn validate_resume(&self) -> Result<(), ScanControlError> { + match self.lifecycle_status().await { + ScanLifecycleStatus::Paused | ScanLifecycleStatus::Running => { + Ok(()) + } + ScanLifecycleStatus::Completed + | ScanLifecycleStatus::Failed + | ScanLifecycleStatus::Canceled => { + Err(ScanControlError::ScanTerminal) + } + ScanLifecycleStatus::Pending => { + Err(ScanControlError::ScanNotRunning) + } + } + } + fn run_key(&self) -> String { scan_run_mode_from_start_mode(self.start_mode).run_key(self.library_id) } @@ -1181,6 +1431,23 @@ impl ScanRun { self.emit_frame(ScanEventKind::Started, emitted).await; } + async fn seed_completed(&self) -> bool { + self.state.lock().await.seed_completed + } + + async fn mark_seed_command_completed(&self) { + let payload = { + let mut state = self.state.lock().await; + if state.is_terminal() { + return; + } + state.seed_completed = true; + state.last_activity_at = Some(Utc::now()); + state.build_current_payload() + }; + self.persist_progress_payload(&payload).await; + } + async fn seed_rehydrated_progress(&self) { let payload = { let state = self.state.lock().await; @@ -1348,70 +1615,86 @@ impl ScanRun { ); } - async fn pause( - &self, - correlation_id: Uuid, - ) -> Result<(), ScanControlError> { - let payload = { - let mut state = self.state.lock().await; - match state.status { - ScanLifecycleStatus::Running => { - state.status = ScanLifecycleStatus::Paused; - state.correlation_id = correlation_id; - state.build_payload() - } - ScanLifecycleStatus::Paused => return Ok(()), - ScanLifecycleStatus::Completed - | ScanLifecycleStatus::Failed - | ScanLifecycleStatus::Canceled => { - return Err(ScanControlError::ScanTerminal); - } - ScanLifecycleStatus::Pending => { - return Err(ScanControlError::ScanNotRunning); - } - } - }; - self.emit_frame(ScanEventKind::Progress, payload).await; + async fn pause(&self) -> Result<(), ScanControlError> { + if let Some(payload) = self + .persist_control_status(ScanLifecycleStatus::Paused) + .await? + { + self.publish_frame(ScanEventKind::Progress, payload).await; + } + Ok(()) + } + + async fn resume(&self) -> Result<(), ScanControlError> { + if let Some(payload) = self + .persist_control_status(ScanLifecycleStatus::Running) + .await? + { + self.publish_frame(ScanEventKind::Progress, payload).await; + } Ok(()) } - async fn resume( + async fn persist_control_status( &self, - correlation_id: Uuid, - ) -> Result<(), ScanControlError> { - let payload = { - let mut state = self.state.lock().await; - match state.status { - ScanLifecycleStatus::Paused => { - state.status = ScanLifecycleStatus::Running; - state.correlation_id = correlation_id; - state.build_payload() - } - ScanLifecycleStatus::Running => return Ok(()), + target: ScanLifecycleStatus, + ) -> Result, ScanControlError> { + let mut state = self.state.lock().await; + match (&state.status, &target) { + (ScanLifecycleStatus::Running, ScanLifecycleStatus::Paused) + | (ScanLifecycleStatus::Paused, ScanLifecycleStatus::Running) => {} + (current, requested) if current == requested => return Ok(None), + ( ScanLifecycleStatus::Completed | ScanLifecycleStatus::Failed - | ScanLifecycleStatus::Canceled => { - return Err(ScanControlError::ScanTerminal); - } - ScanLifecycleStatus::Pending => { - return Err(ScanControlError::ScanNotRunning); - } + | ScanLifecycleStatus::Canceled, + _, + ) => return Err(ScanControlError::ScanTerminal), + _ => return Err(ScanControlError::ScanNotRunning), + } + + if let Some(inner) = self.inner.upgrade() { + let counters = state.counter_snapshot(); + let update = LibraryScanRunProgressUpdate { + scan_id: self.scan_id, + status: Some(target.clone().into()), + completed_items: counters.completed_items, + total_items: state.total_items, + retrying_items: counters.retrying_items, + dead_lettered_items: counters.failed_items, + current_path: state.current_path.clone(), + tracked_job_ids: state.sorted_tracked_job_ids(), + seed_completed: state.seed_completed, + sequence: state.event_sequence.saturating_add(1), + }; + let persisted = inner + .unit_of_work + .scan_runs + .update_progress(update) + .await + .map_err(|err| { + ScanControlError::internal(format!( + "failed to persist scan lifecycle transition: {err}" + )) + })?; + if persisted.is_none() { + return Err(ScanControlError::internal( + "scan lifecycle transition lost its active durable row" + .to_string(), + )); } - }; - self.emit_frame(ScanEventKind::Progress, payload).await; - Ok(()) + } + + state.status = target; + Ok(Some(state.build_payload())) } - async fn cancel( - &self, - correlation_id: Uuid, - ) -> Result<(), ScanControlError> { + async fn cancel(&self) -> Result { let frame = { let mut state = self.state.lock().await; if state.is_terminal() { return Err(ScanControlError::ScanTerminal); } - state.correlation_id = correlation_id; state.last_error = Some("scan_cancelled".to_string()); state .transition(ScanPhase::Canceled, Utc::now()) @@ -1421,8 +1704,7 @@ impl ScanRun { }) }; self.emit_frame(frame.event, frame.payload).await; - self.finalize_history(ScanLifecycleStatus::Canceled).await; - Ok(()) + Ok(self.finalize_history(ScanLifecycleStatus::Canceled).await) } async fn snapshot(&self) -> Result { @@ -1459,6 +1741,51 @@ impl ScanRun { guard.iter().cloned().collect() } + async fn tracked_job_ids(&self) -> Vec { + let state = self.state.lock().await; + state.sorted_tracked_job_ids() + } + + async fn tracks_job_id(&self, job_id: JobId) -> bool { + self.state.lock().await.tracked_job_ids.contains(&job_id) + } + + async fn needs_pre_stall_reconciliation( + &self, + stall_timeout: ChronoDuration, + ) -> bool { + let state = self.state.lock().await; + !state.is_terminal() + && matches!( + state.phase, + ScanPhase::Processing | ScanPhase::Discovering + ) + && state.outstanding_items_stalled(stall_timeout, Utc::now()) + } + + async fn reconcile_durable_job_states(&self, jobs: &[DurableJobState]) { + let frames = { + let mut state = self.state.lock().await; + state.reconcile_durable_job_states(jobs) + }; + self.emit_frames(frames).await; + } + + async fn has_unmaterialized_durable_progress(&self) -> bool { + let state = self.state.lock().await; + state.durable_rebuild_pending + && (state.total_items > 0 + || state.completed_items > 0 + || state.dead_lettered_items > 0 + || state.retrying_items > 0) + } + + async fn has_active_items(&self) -> bool { + let state = self.state.lock().await; + !state.is_terminal() + && state.item_states.values().any(ScanItemState::is_active) + } + async fn record_job_enqueued( &self, idempotency_key: &str, @@ -1473,10 +1800,13 @@ impl ScanRun { if state.is_terminal() { Vec::new() } else { + let newly_tracked = state.tracked_job_ids.insert(job_id); let stale_terminal = state .item_states .get(idempotency_key) - .map(|item| item.is_terminal()) + .map(|item| { + item.is_terminal() && item.last_job_id == Some(job_id) + }) .unwrap_or(false); tracing::debug!( @@ -1503,7 +1833,14 @@ impl ScanRun { } // Do not bump run-level last_activity for stale retrograde events; avoid // keeping quiescence open due to out-of-order noise. - Vec::new() + if newly_tracked { + vec![QueuedFrame { + event: ScanEventKind::Progress, + payload: state.build_payload(), + }] + } else { + Vec::new() + } } else { let previous_phase = state.phase; state.status = ScanLifecycleStatus::Running; @@ -1535,8 +1872,8 @@ impl ScanRun { matches!(previous_phase, ScanPhase::Quiescing) && matches!(state.phase, ScanPhase::Processing); - if let Some(payload) = - state.build_payload_if(changed || reopened) + if let Some(payload) = state + .build_payload_if(changed || reopened || newly_tracked) { frames.push(QueuedFrame { event: ScanEventKind::Progress, @@ -1593,16 +1930,39 @@ impl ScanRun { self.emit_frames(frames).await; } - async fn record_seed_completed(&self, summary: &ScanSeedSummary) -> bool { + async fn record_seed_completed( + &self, + summary: &ScanSeedSummary, + terminalization_allowed: bool, + ) -> bool { let frame = { let mut state = self.state.lock().await; if state.is_terminal() { None } else { - state.mark_seed_completed( + let has_enrollment = !summary.enrolled_job_ids.is_empty(); + summary.enrolled_job_ids.iter().copied().for_each(|job_id| { + state.tracked_job_ids.insert(job_id); + }); + + if !terminalization_allowed { + state.seed_completed = true; + state.last_activity_at = Some(summary.completed_at); + has_enrollment.then(|| QueuedFrame { + event: ScanEventKind::Progress, + payload: state.build_payload(), + }) + } else if let Some(frame) = state.mark_seed_completed( summary.queued_folders, summary.completed_at, - ) + ) { + Some(frame) + } else { + has_enrollment.then(|| QueuedFrame { + event: ScanEventKind::Progress, + payload: state.build_payload(), + }) + } } }; @@ -1610,8 +1970,9 @@ impl ScanRun { let event = frame.event.clone(); self.emit_frame(frame.event, frame.payload).await; if matches!(event, ScanEventKind::Completed) { - self.finalize_history(ScanLifecycleStatus::Completed).await; - return true; + return self + .finalize_history(ScanLifecycleStatus::Completed) + .await; } } @@ -1707,6 +2068,7 @@ impl ScanRun { if state.is_terminal() { return; } + state.tracked_job_ids.insert(job_id); if let Some(item) = state.item_states.get_mut(idempotency_key) { if item.is_terminal() && item.last_job_id == Some(job_id) { return; @@ -1876,7 +2238,7 @@ impl ScanRun { self.emit_frames(frames).await; } - async fn fail_with_reason(&self, reason: &str) { + async fn fail_with_reason(&self, reason: &str) -> bool { let outcome = { let mut state = self.state.lock().await; if state.is_terminal() { @@ -1889,8 +2251,9 @@ impl ScanRun { if let Some(frame) = outcome { self.emit_frame(frame.event, frame.payload).await; - self.finalize_history(ScanLifecycleStatus::Failed).await; + return self.finalize_history(ScanLifecycleStatus::Failed).await; } + false } async fn try_complete( @@ -1901,7 +2264,11 @@ impl ScanRun { let (maybe_frame, finalize_status) = { let mut state = self.state.lock().await; if state.is_terminal() { - (None, None) + // Terminal state is applied in memory before its final + // PostgreSQL writes. If either write failed transiently, the + // run remains registered and this tick must retry durable + // finalization instead of no-oping forever. + (None, Some(state.status.clone())) } else { let now = Utc::now(); let mut frame: Option = None; @@ -1977,12 +2344,13 @@ impl ScanRun { }; if let Some(frame) = maybe_frame { - let event = frame.event.clone(); self.emit_frame(frame.event, frame.payload).await; if let Some(status) = finalize_status { - self.finalize_history(status).await; + return self.finalize_history(status).await; } - matches!(event, ScanEventKind::Completed) + false + } else if let Some(status) = finalize_status { + self.finalize_history(status).await } else { false } @@ -1992,6 +2360,15 @@ impl ScanRun { &self, event: ScanEventKind, payload: ScanProgressEvent, + ) { + self.persist_progress_payload(&payload).await; + self.publish_frame(event, payload).await; + } + + async fn publish_frame( + &self, + event: ScanEventKind, + payload: ScanProgressEvent, ) { let frame = ScanBroadcastFrame { event: event.clone(), @@ -2006,8 +2383,6 @@ impl ScanRun { history.push_back(frame.clone()); } - self.persist_progress_payload(&payload).await; - let _ = self.tx.send(frame.clone()); let error = if matches!(event, ScanEventKind::Failed) { self.failure_reason().await @@ -2132,6 +2507,10 @@ impl ScanRun { let Some(inner) = self.inner.upgrade() else { return; }; + let (tracked_job_ids, seed_completed) = { + let state = self.state.lock().await; + (state.sorted_tracked_job_ids(), state.seed_completed) + }; if let Err(err) = inner .unit_of_work @@ -2144,6 +2523,8 @@ impl ScanRun { retrying_items: payload.retrying_items, dead_lettered_items: payload.failed_items, current_path: payload.current_path.clone(), + tracked_job_ids, + seed_completed, sequence: payload.sequence, }) .await @@ -2162,64 +2543,168 @@ impl ScanRun { state.last_error.clone() } - async fn finalize_history(&self, terminal: ScanLifecycleStatus) { + async fn finalize_history(&self, terminal: ScanLifecycleStatus) -> bool { + let mut finalized = self.finalization.lock().await; + if *finalized { + return false; + } + let repository_terminal: ApiScanLifecycleStatus = + terminal.clone().into(); + let artifacts = { let state = self.state.lock().await; state.terminal_artifacts(terminal.clone()) }; let final_events = self.event_log().await; - if let Some(inner) = self.inner.upgrade() { - if let Err(err) = inner - .unit_of_work - .scan_runs - .update_progress(artifacts.progress) - .await - { + let Some(inner) = self.inner.upgrade() else { + return false; + }; + + let progress_updated = match inner + .unit_of_work + .scan_runs + .update_progress(artifacts.progress) + .await + { + Ok(Some(_)) => true, + Ok(None) => false, + Err(err) => { warn!( scan = %self.scan_id, status = ?terminal, error = %err, "failed to persist final scan progress; keeping run active" ); - return; + return false; } + }; - if let Err(err) = inner + let authoritative = if progress_updated { + match inner .unit_of_work .scan_runs .mark_terminal( self.scan_id, - terminal.clone().into(), + repository_terminal.clone(), artifacts.terminal_at, artifacts.last_error, ) .await { - warn!( - scan = %self.scan_id, - status = ?terminal, - error = %err, - "failed to persist terminal scan state; keeping run active" - ); - return; + Ok(Some(run)) => run.status == repository_terminal, + Ok(None) => false, + Err(err) => { + warn!( + scan = %self.scan_id, + status = ?terminal, + error = %err, + "failed to persist terminal scan state; keeping run active" + ); + return false; + } } + } else { + false + }; - inner - .finalize_run( - self.scan_id, - self.run_key(), - self.correlation_id, - artifacts.snapshot, - final_events, - ) - .await; + // `None` is an expected idempotency result when another finalizer won + // the compare-and-set. It is success only if PostgreSQL confirms this + // exact terminal state; a missing row or competing status remains + // authoritative and must not be replaced by local history. + let authoritative = if authoritative { + true + } else { + match inner + .unit_of_work + .scan_runs + .load_by_scan_id(self.scan_id) + .await + { + Ok(Some(run)) if run.status == repository_terminal => true, + Ok(Some(run)) => { + warn!( + scan = %self.scan_id, + requested_status = ?terminal, + durable_status = ?run.status, + "durable scan terminal state differs; keeping local run active" + ); + false + } + Ok(None) => { + warn!( + scan = %self.scan_id, + status = ?terminal, + "durable scan row disappeared during finalization; keeping local run active" + ); + false + } + Err(err) => { + warn!( + scan = %self.scan_id, + status = ?terminal, + error = %err, + "failed to verify terminal scan state; keeping local run active" + ); + false + } + } + }; + + if !authoritative { + return false; } + inner + .finalize_run( + self.scan_id, + self.run_key(), + self.correlation_id, + artifacts.snapshot, + final_events, + ) + .await; + *finalized = true; warn!(scan = %self.scan_id, status = ?terminal, "finalized scan run"); + true } } +async fn ensure_run_seeded( + run: &Arc, + orchestrator: &ScanOrchestrator, +) -> Result { + let _seed_guard = run.seed_transition.lock().await; + seed_run_locked(run, orchestrator).await +} + +async fn seed_run_locked( + run: &Arc, + orchestrator: &ScanOrchestrator, +) -> Result { + let status = run.lifecycle_status().await; + if run.seed_completed().await || !lifecycle_allows_seed(&status) { + return Ok(false); + } + + orchestrator + .command_library( + run.library_id(), + LibraryActorCommand::Start { + mode: run.start_mode(), + correlation_id: Some(run.correlation_id()), + }, + ) + .await + .map_err(|err| ScanControlError::internal(err.to_string()))?; + + // Mailbox acknowledgement occurs only after the complete seed batch has + // been enqueued durably. Persist that fact independently of the bounded + // SeedCompleted broadcast so restart recovery does not rescan it. + run.mark_seed_command_completed().await; + Ok(true) +} + #[derive(Clone, Debug)] struct ScanLogWatermark { last_log_at: Instant, @@ -2256,6 +2741,155 @@ impl ScanRunState { ) } + fn reconcile_durable_job_states( + &mut self, + jobs: &[DurableJobState], + ) -> Vec { + if self.is_terminal() || jobs.is_empty() { + return Vec::new(); + } + + if self.durable_rebuild_pending { + // Rehydrated runs carry aggregate counters but not per-job state. + // The first successful PostgreSQL read is a complete correlation + // snapshot, so rebuild from it instead of adding to the persisted + // aggregate and double-counting every recovered job. + self.completed_items = 0; + self.total_items = 0; + self.dead_lettered_items = 0; + self.retrying_items = 0; + self.item_states.clear(); + self.durable_rebuild_pending = false; + } + + // A dedupe key can have an old terminal row and a newer active row. + // PostgreSQL allows a new generation after the terminal transition; + // only the newest generation describes the current logical item. + let mut latest_by_dedupe: HashMap<&str, &DurableJobState> = + HashMap::with_capacity(jobs.len()); + for job in jobs { + self.tracked_job_ids.insert(job.job_id); + let replace = latest_by_dedupe + .get(job.dedupe_key.as_str()) + .map(|current| { + job.created_at > current.created_at + || (job.created_at == current.created_at + && (job.updated_at > current.updated_at + || (job.updated_at == current.updated_at + && job.job_id.0 > current.job_id.0))) + }) + .unwrap_or(true); + if replace { + latest_by_dedupe.insert(job.dedupe_key.as_str(), job); + } + } + let mut latest_jobs: Vec<_> = latest_by_dedupe.into_values().collect(); + latest_jobs.sort_unstable_by(|left, right| { + left.created_at + .cmp(&right.created_at) + .then_with(|| left.job_id.0.cmp(&right.job_id.0)) + }); + + let mut changed = false; + let mut saw_active = false; + let mut newest_activity: Option<(DateTime, Option)> = + None; + + for job in latest_jobs { + let target_status = match job.state { + JobState::Completed => { + if job.kind == JobKind::FolderScan { + job.path_key + .as_ref() + .and_then(subject_key_path) + .and_then(|path| { + self.folder_outcomes_by_path.get(path) + }) + .copied() + .map(Self::status_for_folder_outcome) + .unwrap_or(ScanItemStatus::Completed) + } else { + ScanItemStatus::Completed + } + } + JobState::Failed | JobState::DeadLetter => { + ScanItemStatus::DeadLettered + } + JobState::Ready if job.attempts > 0 => ScanItemStatus::Retrying, + JobState::Ready | JobState::Deferred | JobState::Leased => { + ScanItemStatus::InProgress + } + }; + saw_active |= target_status.is_active(); + + changed |= self.update_item_status( + &job.dedupe_key, + Some(job.job_id), + target_status, + job.updated_at, + job.path_key.clone(), + job.last_error.clone(), + ); + + if newest_activity + .as_ref() + .map(|(updated_at, _)| job.updated_at > *updated_at) + .unwrap_or(true) + { + newest_activity = Some((job.updated_at, job.path_key.clone())); + } + } + + if saw_active + && matches!( + self.phase, + ScanPhase::Initializing + | ScanPhase::Discovering + | ScanPhase::Quiescing + ) + { + self.handle_state_event(ScanStateEvent::NewItemFound, Utc::now()); + } else if matches!(self.phase, ScanPhase::Initializing) { + // A complete burst can disappear before any enqueue event is + // observed. Walk through the active phase so the recovered + // terminal rows can enter quiescence normally. + self.handle_state_event(ScanStateEvent::NewItemFound, Utc::now()); + } + + if let Some((updated_at, path_key)) = newest_activity { + let advances_activity = self + .last_activity_at + .map(|current| updated_at > current) + .unwrap_or(true); + if advances_activity { + self.last_activity_at = Some(updated_at); + if let Some(path_key) = path_key { + self.current_path = subject_key_path_owned(&path_key); + self.path_key = Some(path_key); + } + } + } + + let mut frames = Vec::new(); + if changed { + frames.push(QueuedFrame { + event: ScanEventKind::Progress, + payload: self.build_payload(), + }); + } + + if self.can_enter_quiescing() + && let Some(frame) = self.handle_state_event( + ScanStateEvent::AllItemsProcessed, + Utc::now(), + ) + { + frames.push(frame); + } + + frames + } + fn terminal_artifacts( &self, terminal: ScanLifecycleStatus, @@ -2291,6 +2925,8 @@ impl ScanRunState { retrying_items, dead_lettered_items: failed_items, current_path: self.current_path.clone(), + tracked_job_ids: self.sorted_tracked_job_ids(), + seed_completed: self.seed_completed, sequence: self.event_sequence, }, terminal_at, @@ -2705,6 +3341,9 @@ impl ScanRunState { path_key: Option, error: Option, ) -> bool { + if let Some(job_id) = job_id { + self.tracked_job_ids.insert(job_id); + } match self.item_states.entry(idempotency_key.to_string()) { Entry::Vacant(slot) => { self.total_items += 1; @@ -2731,9 +3370,16 @@ impl ScanRunState { Entry::Occupied(mut slot) => { let item = slot.get_mut(); let old_status = item.status; - - // Refuse retrograde transitions: once terminal, never go back to active. - if old_status.is_terminal() && !status.is_terminal() { + let same_generation = + job_id.is_none() || item.last_job_id == job_id; + + // Refuse retrograde transitions within one durable job. A new + // job ID for the same dedupe key is a newer generation and may + // legitimately move the logical item back to active. + if old_status.is_terminal() + && !status.is_terminal() + && same_generation + { tracing::debug!( target: "scan::state", scan = %self.scan_id, @@ -2758,6 +3404,7 @@ impl ScanRunState { } if matches!(old_status, ScanItemStatus::DeadLettered) && !matches!(status, ScanItemStatus::DeadLettered) + && same_generation { return false; } @@ -2837,6 +3484,12 @@ impl ScanRunState { == self.total_items } + fn sorted_tracked_job_ids(&self) -> Vec { + let mut ids: Vec<_> = self.tracked_job_ids.iter().copied().collect(); + ids.sort_unstable_by_key(|job_id| job_id.0); + ids + } + fn outstanding_items_stalled( &self, stall_timeout: ChronoDuration, @@ -2846,6 +3499,18 @@ impl ScanRunState { return false; } + // A large seed batch gives every ready item roughly the same old + // enqueue timestamp. Those untouched backlog entries are not evidence + // of a stalled run while other items are still completing. Require the + // run itself to have been inactive for the full timeout before using + // per-item timestamps to identify genuinely abandoned work. + if self + .last_activity_at + .is_some_and(|last_activity| now - last_activity <= stall_timeout) + { + return false; + } + let mut saw_active = false; for item in self.item_states.values() { if !item.is_active() { @@ -2893,11 +3558,665 @@ mod tests { last_activity_at: None, quiescence_started_at: None, last_error: None, + tracked_job_ids: HashSet::new(), item_states: HashMap::new(), folder_outcomes_by_path: HashMap::new(), historical_cursor_count: 0, seed_completed: false, + durable_rebuild_pending: false, + } + } + + #[test] + fn paused_and_terminal_runs_cannot_claim_a_seed() { + assert!(lifecycle_allows_seed(&ScanLifecycleStatus::Pending)); + assert!(lifecycle_allows_seed(&ScanLifecycleStatus::Running)); + assert!(!lifecycle_allows_seed(&ScanLifecycleStatus::Paused)); + assert!(!lifecycle_allows_seed(&ScanLifecycleStatus::Completed)); + assert!(!lifecycle_allows_seed(&ScanLifecycleStatus::Failed)); + assert!(!lifecycle_allows_seed(&ScanLifecycleStatus::Canceled)); + } + + fn durable_job( + correlation_id: Uuid, + job_id: JobId, + dedupe_key: impl Into, + state: JobState, + attempts: u16, + path: impl Into, + updated_at: DateTime, + ) -> DurableJobState { + DurableJobState { + job_id, + kind: JobKind::FolderScan, + state, + attempts, + dedupe_key: dedupe_key.into(), + correlation_id: Some(correlation_id), + path_key: SubjectKey::path(path.into()).ok(), + last_error: None, + created_at: updated_at, + updated_at, + } + } + + fn test_run(state: ScanRunState) -> Arc { + let (tx, _rx) = broadcast::channel(16); + Arc::new(ScanRun { + scan_id: state.scan_id, + library_id: state.library_id, + correlation_id: state.correlation_id, + state: Mutex::new(state), + tx, + inner: Weak::new(), + events: Mutex::new(VecDeque::new()), + start_mode: StartMode::Bulk, + log: Mutex::new(ScanLogWatermark::default()), + seed_transition: Mutex::new(()), + finalization: Mutex::new(false), + }) + } + + #[tokio::test] + async fn lifecycle_commands_keep_the_run_correlation_immutable() { + let mut state = test_state(); + state.phase = ScanPhase::Discovering; + state.status = ScanLifecycleStatus::Running; + let correlation_id = state.correlation_id; + let run = test_run(state); + + run.pause().await.expect("running scan pauses"); + assert_eq!( + run.snapshot().await.unwrap().correlation_id, + correlation_id + ); + + run.resume().await.expect("paused scan resumes"); + assert_eq!( + run.snapshot().await.unwrap().correlation_id, + correlation_id + ); + + assert!( + !run.cancel().await.expect("running scan cancels"), + "a local-only test run cannot claim durable finalization" + ); + assert_eq!( + run.snapshot().await.unwrap().correlation_id, + correlation_id + ); + } + + #[tokio::test] + async fn forgetting_active_library_runs_removes_both_registry_keys() { + let deleted_run = test_run(test_state()); + let retained_run = test_run(test_state()); + let deleted_scan_id = deleted_run.scan_id(); + let retained_scan_id = retained_run.scan_id(); + let deleted_correlation = deleted_run.correlation_id(); + let deleted_library = deleted_run.library_id(); + + let active_by_scan_id = RwLock::new(HashMap::from([ + (deleted_scan_id, Arc::clone(&deleted_run)), + (retained_scan_id, Arc::clone(&retained_run)), + ])); + let active_by_run_key = RwLock::new(HashMap::from([ + (deleted_run.run_key(), Arc::clone(&deleted_run)), + (retained_run.run_key(), Arc::clone(&retained_run)), + ])); + + let removed = forget_active_library_runs( + &active_by_scan_id, + &active_by_run_key, + deleted_library, + ) + .await; + let removed_again = forget_active_library_runs( + &active_by_scan_id, + &active_by_run_key, + deleted_library, + ) + .await; + + assert_eq!(removed, HashSet::from([deleted_correlation])); + assert!(removed_again.is_empty()); + assert!( + !active_by_scan_id + .read() + .await + .contains_key(&deleted_scan_id) + ); + assert!( + active_by_scan_id + .read() + .await + .contains_key(&retained_scan_id) + ); + assert_eq!(active_by_run_key.read().await.len(), 1); + assert!( + active_by_run_key + .read() + .await + .contains_key(&retained_run.run_key()) + ); + } + + #[tokio::test] + async fn durable_reconciliation_recovers_burst_larger_than_broadcast_capacity() + { + let mut state = test_state(); + let now = Utc::now(); + state.phase = ScanPhase::Discovering; + state.status = ScanLifecycleStatus::Running; + let (tx, mut receiver) = tokio::sync::broadcast::channel(256); + let jobs = (0..300) + .map(|index| { + tx.send(index).expect("lag test receiver remains open"); + durable_job( + state.correlation_id, + JobId::new(), + format!("scan:{}:{index}", state.library_id), + JobState::Completed, + 0, + format!("/library/movie-{index}"), + now + ChronoDuration::milliseconds(index), + ) + }) + .collect::>(); + + let skipped = match receiver.recv().await { + Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { + skipped + } + other => panic!("expected >256 burst to lag, got {other:?}"), + }; + assert!(skipped >= 44); + + let frames = state.reconcile_durable_job_states(&jobs); + + assert_eq!(state.total_items, 300); + assert_eq!(state.completed_items, 300); + assert_eq!(state.dead_lettered_items, 0); + assert_eq!(state.phase, ScanPhase::Quiescing); + assert!(frames.iter().any(|frame| { + matches!(frame.event, ScanEventKind::Progress) + && frame.payload.completed_items == 300 + })); + assert!( + frames + .iter() + .any(|frame| matches!(frame.event, ScanEventKind::Quiescing)) + ); + } + + #[tokio::test] + async fn lag_gate_blocks_terminalization_until_reconciliation_succeeds() { + let correlation_id = Uuid::now_v7(); + let gate = DurableReconciliationGate::default(); + let (tx, mut receiver) = tokio::sync::broadcast::channel(256); + for sequence in 0..300 { + tx.send(sequence).expect("lag test receiver remains open"); + } + + match receiver.recv().await { + Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => { + gate.require_all([correlation_id]).await; + } + other => panic!("expected lagged receiver, got {other:?}"), + } + + assert!(!gate.allows_terminal(correlation_id).await); + // A failed durable read does not call `reconciled`, so later events and + // quiescence ticks remain unable to terminalize the partial state. + assert!(gate.is_required(correlation_id).await); + assert!(!gate.allows_terminal(correlation_id).await); + + gate.reconciled(correlation_id).await; + assert!(gate.allows_terminal(correlation_id).await); + } + + #[tokio::test] + async fn retryable_failure_gate_waits_for_authoritative_queue_outcome() { + let correlation_id = Uuid::now_v7(); + let job_id = JobId::new(); + let now = Utc::now(); + let gate = DurableReconciliationGate::default(); + gate.require_retryable_failure(correlation_id, job_id).await; + + let leased = durable_job( + correlation_id, + job_id, + "scan:retry-uncertain", + JobState::Leased, + 0, + "/library/retry-uncertain", + now, + ); + assert_eq!( + gate.unresolved_retryable_failures(correlation_id, &[leased]) + .await, + vec![(job_id, Some(JobState::Leased))] + ); + assert!(!gate.allows_terminal(correlation_id).await); + + let ready = durable_job( + correlation_id, + job_id, + "scan:retry-uncertain", + JobState::Ready, + 1, + "/library/retry-uncertain", + now + ChronoDuration::milliseconds(1), + ); + assert!( + gate.unresolved_retryable_failures(correlation_id, &[ready]) + .await + .is_empty() + ); + gate.reconciled(correlation_id).await; + assert!(gate.allows_terminal(correlation_id).await); + } + + #[test] + fn durable_reconciliation_uses_newest_generation_per_dedupe_key() { + let mut state = test_state(); + let correlation_id = state.correlation_id; + let now = Utc::now(); + let old_job_id = JobId::new(); + let new_job_id = JobId::new(); + state.phase = ScanPhase::Processing; + state.status = ScanLifecycleStatus::Running; + + let old_terminal = durable_job( + correlation_id, + old_job_id, + "scan:shared-path", + JobState::Completed, + 0, + "/library/shared", + now - ChronoDuration::seconds(2), + ); + let new_active = durable_job( + correlation_id, + new_job_id, + "scan:shared-path", + JobState::Leased, + 0, + "/library/shared", + now, + ); + + state.reconcile_durable_job_states(&[old_terminal, new_active]); + + assert_eq!(state.total_items, 1); + assert_eq!(state.completed_items, 0); + assert_eq!(state.phase, ScanPhase::Processing); + assert_eq!( + state.item_states["scan:shared-path"].last_job_id, + Some(new_job_id) + ); + assert!(state.item_states["scan:shared-path"].is_active()); + } + + #[tokio::test] + async fn merged_job_id_forces_progress_frame_when_status_is_unchanged() { + let mut state = test_state(); + let old_job_id = JobId::new(); + let merged_job_id = JobId::new(); + state.phase = ScanPhase::Processing; + state.status = ScanLifecycleStatus::Running; + state.update_item_status( + "scan:shared-path", + Some(old_job_id), + ScanItemStatus::InProgress, + Utc::now(), + SubjectKey::path("/library/shared".to_string()).ok(), + None, + ); + let run = test_run(state); + let mut receiver = run.subscribe(); + + run.record_job_enqueued( + "scan:shared-path", + merged_job_id, + JobKind::FolderScan, + SubjectKey::path("/library/shared".to_string()).ok(), + ) + .await; + + let frame = receiver + .try_recv() + .expect("newly tracked merge emits persistence-bearing progress"); + assert!(matches!(frame.event, ScanEventKind::Progress)); + let tracked = run.tracked_job_ids().await; + assert!(tracked.contains(&old_job_id)); + assert!(tracked.contains(&merged_job_id)); + } + + #[tokio::test] + async fn pre_seed_empty_reconciliation_cannot_clear_enrollment_gate() { + let mut state = test_state(); + state.phase = ScanPhase::Discovering; + state.status = ScanLifecycleStatus::Running; + let correlation_id = state.correlation_id; + let library_id = state.library_id; + let shared_job_id = JobId::new(); + let run = test_run(state); + let mut receiver = run.subscribe(); + let gate = DurableReconciliationGate::default(); + gate.require_all([correlation_id]).await; + + // The first ticker can race Start and observe an empty database before + // its enqueue batch is complete. Production reconcile_run must not + // clear registration's gate from this pre-seed snapshot. + run.reconcile_durable_job_states(&[]).await; + assert!( + !gate + .clear_after_seeded_snapshot( + correlation_id, + run.seed_completed().await, + ) + .await + ); + assert!(gate.is_required(correlation_id).await); + + run.mark_seed_command_completed().await; + let summary = ScanSeedSummary { + library_id, + correlation_id: Some(correlation_id), + mode: ferrex_core::domain::scan::orchestration::ScanSeedMode::Bulk, + queued_folders: 1, + enrolled_job_ids: vec![shared_job_id], + completed_at: Utc::now(), + }; + + let terminalization_allowed = + gate.allows_terminal(correlation_id).await; + assert!(!terminalization_allowed); + assert!( + !run.record_seed_completed(&summary, terminalization_allowed) + .await + ); + + let frame = receiver.try_recv().expect( + "seed enrollment emits progress while durable reconciliation is gated", + ); + assert!(matches!(frame.event, ScanEventKind::Progress)); + assert!(run.tracks_job_id(shared_job_id).await); + assert!(run.state.lock().await.seed_completed); + // Seed completion alone cannot reinterpret the earlier empty snapshot + // as a no-work scan. A post-seed read is still mandatory. + assert!(gate.is_required(correlation_id).await); + assert!(!gate.allows_terminal(correlation_id).await); + let early_completion = if gate.allows_terminal(correlation_id).await { + run.try_complete( + ChronoDuration::zero(), + ChronoDuration::seconds(30), + ) + .await + } else { + false + }; + assert!(!early_completion); + assert_eq!(run.lifecycle_status().await, ScanLifecycleStatus::Running); + + run.reconcile_durable_job_states(&[durable_job( + correlation_id, + shared_job_id, + "scan:seeded-shared", + JobState::Leased, + 0, + "/library/shared", + Utc::now(), + )]) + .await; + assert!( + gate.clear_after_seeded_snapshot( + correlation_id, + run.seed_completed().await, + ) + .await + ); + assert!(gate.allows_terminal(correlation_id).await); + assert!(run.has_active_items().await); + assert!( + !run.try_complete( + ChronoDuration::milliseconds(1), + ChronoDuration::seconds(30), + ) + .await + ); + } + + #[test] + fn durable_reconciliation_repairs_terminal_job_before_stall() { + let mut state = test_state(); + let now = Utc::now(); + let stale_at = now - ChronoDuration::seconds(30); + let job_id = JobId::new(); + state.phase = ScanPhase::Processing; + state.status = ScanLifecycleStatus::Running; + state.update_item_status( + "scan:stalled", + Some(job_id), + ScanItemStatus::InProgress, + stale_at, + SubjectKey::path("/library/stalled".to_string()).ok(), + None, + ); + assert!( + state.outstanding_items_stalled(ChronoDuration::seconds(5), now) + ); + + state.reconcile_durable_job_states(&[durable_job( + state.correlation_id, + job_id, + "scan:stalled", + JobState::Completed, + 0, + "/library/stalled", + now, + )]); + + assert_eq!(state.completed_items, 1); + assert_eq!(state.total_items, 1); + assert_eq!(state.phase, ScanPhase::Quiescing); + assert!(!state.outstanding_items_stalled( + ChronoDuration::seconds(5), + now + ChronoDuration::seconds(30) + )); + } + + #[tokio::test] + async fn active_durable_snapshot_defers_same_tick_stall_terminalization() { + let mut state = test_state(); + let now = Utc::now(); + let stale_at = now - ChronoDuration::seconds(30); + let stall_timeout = ChronoDuration::seconds(5); + let job_id = JobId::new(); + state.phase = ScanPhase::Processing; + state.status = ScanLifecycleStatus::Running; + state.last_activity_at = Some(stale_at); + state.update_item_status( + "scan:racing-completion", + Some(job_id), + ScanItemStatus::InProgress, + stale_at, + SubjectKey::path("/library/racing-completion".to_string()).ok(), + None, + ); + let correlation_id = state.correlation_id; + let run = test_run(state); + + // PostgreSQL still reported the job active when the pre-stall query + // completed. The timestamps are old enough that an immediate + // try_complete would otherwise fail the run as stalled. + run.reconcile_durable_job_states(&[durable_job( + correlation_id, + job_id, + "scan:racing-completion", + JobState::Leased, + 0, + "/library/racing-completion", + stale_at, + )]) + .await; + assert!(run.has_active_items().await); + assert!( + run.state + .lock() + .await + .outstanding_items_stalled(stall_timeout, now) + ); + + // This is the check_quiescence gate: an active durable snapshot owns + // this tick's decision, even if the job completes just after it. + let terminalized = if run.has_active_items().await { + false + } else { + run.try_complete(ChronoDuration::zero(), stall_timeout) + .await + }; + assert!(!terminalized); + assert_eq!(run.state.lock().await.phase, ScanPhase::Processing); + + // The next durable observation sees the racing completion. Terminal + // durable states do not retain the gate and can quiesce normally. + run.reconcile_durable_job_states(&[durable_job( + correlation_id, + job_id, + "scan:racing-completion", + JobState::Completed, + 0, + "/library/racing-completion", + now, + )]) + .await; + assert!(!run.has_active_items().await); + assert_eq!(run.state.lock().await.phase, ScanPhase::Quiescing); + assert!( + !run.try_complete(ChronoDuration::zero(), stall_timeout) + .await, + "in-memory completion is not reported before durable finalization" + ); + assert_eq!(run.state.lock().await.phase, ScanPhase::Completed); + } + + #[tokio::test] + async fn terminal_run_does_not_report_completion_without_durable_confirmation() + { + let mut state = test_state(); + let now = Utc::now(); + state.phase = ScanPhase::Failed; + state.status = ScanLifecycleStatus::Failed; + state.update_item_status( + "scan:terminal-but-not-persisted", + Some(JobId::new()), + ScanItemStatus::InProgress, + now, + SubjectKey::path("/library/terminal".to_string()).ok(), + None, + ); + let run = test_run(state); + + assert!( + !run.has_active_items().await, + "terminal memory state must not block retrying its durable terminal write" + ); + assert!( + !run.try_complete( + ChronoDuration::milliseconds(1), + ChronoDuration::seconds(30), + ) + .await, + "maintenance side effects must wait for authoritative finalization" + ); + } + + #[test] + fn old_ready_backlog_does_not_stall_while_run_is_making_progress() { + let mut state = test_state(); + let now = Utc::now(); + let stall_timeout = ChronoDuration::seconds(25); + let seeded_at = now - ChronoDuration::seconds(26); + state.phase = ScanPhase::Processing; + state.status = ScanLifecycleStatus::Running; + + for index in 0..3_131 { + state.update_item_status( + &format!("folder:{index}"), + Some(JobId::new()), + ScanItemStatus::InProgress, + seeded_at, + SubjectKey::path(format!("/library/movie-{index}")).ok(), + None, + ); } + + let completed_at = now - ChronoDuration::seconds(1); + state.update_item_status( + "folder:0", + None, + ScanItemStatus::Completed, + completed_at, + SubjectKey::path("/library/movie-0".to_string()).ok(), + None, + ); + // Production completion handling records this run-level activity after + // applying the item transition. + state.last_activity_at = Some(completed_at); + + assert_eq!(state.completed_items, 1); + assert_eq!(state.total_items, 3_131); + assert!(state.item_states.values().any(ScanItemState::is_active)); + assert!( + !state.outstanding_items_stalled(stall_timeout, now), + "recent completion must keep an old ready backlog alive" + ); + + state.last_activity_at = Some(seeded_at); + assert!( + state.outstanding_items_stalled(stall_timeout, now), + "the same backlog is stalled once run-level progress also expires" + ); + } + + #[test] + fn rehydrated_run_rebuilds_without_double_counting_persisted_progress() { + let mut state = test_state(); + let now = Utc::now(); + state.phase = ScanPhase::Discovering; + state.status = ScanLifecycleStatus::Running; + state.total_items = 2; + state.completed_items = 2; + state.durable_rebuild_pending = true; + let jobs = [ + durable_job( + state.correlation_id, + JobId::new(), + "scan:rehydrated:1", + JobState::Completed, + 0, + "/library/one", + now, + ), + durable_job( + state.correlation_id, + JobId::new(), + "scan:rehydrated:2", + JobState::Completed, + 0, + "/library/two", + now + ChronoDuration::milliseconds(1), + ), + ]; + + state.reconcile_durable_job_states(&jobs); + + assert_eq!(state.total_items, 2); + assert_eq!(state.completed_items, 2); + assert_eq!(state.item_states.len(), 2); + assert!(!state.durable_rebuild_pending); } #[test] @@ -3337,11 +4656,12 @@ mod tests { let mut state = test_state(); let now = Utc::now(); let path = SubjectKey::path("/library/stable".to_string()).ok(); + let job_id = JobId::new(); state.handle_state_event(ScanStateEvent::RunStarted, now); state.update_item_status( "stable-job", - Some(JobId::new()), + Some(job_id), ScanItemStatus::Completed, now + ChronoDuration::milliseconds(1), path.clone(), @@ -3358,7 +4678,7 @@ mod tests { let demoted = state.update_item_status( "stable-job", - Some(JobId::new()), + Some(job_id), ScanItemStatus::InProgress, now + ChronoDuration::milliseconds(3), path, @@ -3564,6 +4884,37 @@ mod tests { assert_eq!(latest[0].scan_id, evicting_scan_id); assert_eq!(latest[0].completed_items, HISTORY_CAPACITY as u64 + 1); } + + #[tokio::test] + async fn progress_archive_forgets_only_deleted_library_entries() { + let archive = ScanRunProgressArchive::new(); + let deleted_library = LibraryId::new(); + let retained_library = LibraryId::new(); + let deleted_scan_id = Uuid::now_v7(); + let retained_scan_id = Uuid::now_v7(); + + archive + .record_terminal( + history_entry(deleted_scan_id, deleted_library, 1), + vec![replay_frame(deleted_scan_id, deleted_library, 1)], + ) + .await; + archive + .record_terminal( + history_entry(retained_scan_id, retained_library, 2), + vec![replay_frame(retained_scan_id, retained_library, 2)], + ) + .await; + + archive.forget_library(deleted_library).await; + archive.forget_library(deleted_library).await; + + assert!(archive.replay_events(&deleted_scan_id).await.is_none()); + assert!(archive.replay_events(&retained_scan_id).await.is_some()); + let history = archive.history(HISTORY_CAPACITY).await; + assert_eq!(history.len(), 1); + assert_eq!(history[0].library_id, retained_library); + } } #[derive(Clone)] @@ -3578,6 +4929,104 @@ struct ScanRunAggregatorInner { stall_timeout: ChronoDuration, catalog_events: CatalogEventProjection, series_bundles: Mutex>, + reconciliation_gate: DurableReconciliationGate, +} + +#[derive(Default)] +struct DurableReconciliationGate { + state: Mutex, +} + +#[derive(Default)] +struct DurableReconciliationRequirements { + required: HashSet, + retryable_failures: HashMap>, +} + +impl DurableReconciliationGate { + async fn require_all(&self, correlations: impl IntoIterator) { + self.state.lock().await.required.extend(correlations); + } + + async fn require_retryable_failure( + &self, + correlation_id: Uuid, + job_id: JobId, + ) { + let mut state = self.state.lock().await; + state.required.insert(correlation_id); + state + .retryable_failures + .entry(correlation_id) + .or_default() + .insert(job_id); + } + + async fn is_required(&self, correlation_id: Uuid) -> bool { + self.state.lock().await.required.contains(&correlation_id) + } + + async fn allows_terminal(&self, correlation_id: Uuid) -> bool { + !self.is_required(correlation_id).await + } + + async fn reconciled(&self, correlation_id: Uuid) { + let mut state = self.state.lock().await; + state.required.remove(&correlation_id); + state.retryable_failures.remove(&correlation_id); + } + + async fn clear_after_seeded_snapshot( + &self, + correlation_id: Uuid, + seed_completed: bool, + ) -> bool { + if !seed_completed { + return false; + } + self.reconciled(correlation_id).await; + true + } + + async fn unresolved_retryable_failures( + &self, + correlation_id: Uuid, + jobs: &[DurableJobState], + ) -> Vec<(JobId, Option)> { + let required = { + let state = self.state.lock().await; + state + .retryable_failures + .get(&correlation_id) + .cloned() + .unwrap_or_default() + }; + if required.is_empty() { + return Vec::new(); + } + + let durable_states: HashMap<_, _> = + jobs.iter().map(|job| (job.job_id, job.state)).collect(); + required + .into_iter() + .filter_map(|job_id| { + let state = durable_states.get(&job_id).copied(); + (!matches!( + state, + Some( + JobState::Ready + | JobState::DeadLetter + | JobState::Completed + ) + )) + .then_some((job_id, state)) + }) + .collect() + } + + async fn forget(&self, correlation_id: Uuid) { + self.reconciled(correlation_id).await; + } } #[derive(Debug)] @@ -3621,6 +5070,7 @@ impl ScanRunAggregator { stall_timeout: stall_window, catalog_events, series_bundles: Mutex::new(HashMap::new()), + reconciliation_gate: DurableReconciliationGate::default(), }); let aggregator = Self { @@ -3638,13 +5088,44 @@ impl ScanRunAggregator { } async fn register(&self, run: Arc) { + let correlation_id = run.correlation_id(); let mut guard = self.inner.runs.write().await; - guard.insert(run.correlation_id(), run); + guard.insert(correlation_id, run); + drop(guard); + // Registration covers both new and restart-rehydrated runs. Require a + // successful durable read before either path can terminalize; for a + // new run the initial snapshot is simply empty. + self.inner + .reconciliation_gate + .require_all([correlation_id]) + .await; } async fn drop(&self, correlation_id: &Uuid) { let mut guard = self.inner.runs.write().await; guard.remove(correlation_id); + drop(guard); + self.inner.reconciliation_gate.forget(*correlation_id).await; + } + + async fn forget_library(&self, library_id: LibraryId) { + let correlations = { + let mut runs = self.inner.runs.write().await; + let correlations: Vec<_> = runs + .iter() + .filter(|(_, run)| run.library_id() == library_id) + .map(|(correlation_id, _)| *correlation_id) + .collect(); + for correlation_id in &correlations { + runs.remove(correlation_id); + } + correlations + }; + + for correlation_id in correlations { + self.inner.reconciliation_gate.forget(correlation_id).await; + } + self.inner.series_bundles.lock().await.remove(&library_id); } } @@ -3664,6 +5145,8 @@ impl ScanRunAggregatorInner { Ok(event) => self.handle_job_event(event).await, Err(RecvError::Lagged(skipped)) => { warn!("scan aggregator lagged {skipped} events"); + self.mark_all_runs_reconciliation_required().await; + self.reconcile_all_runs("job_event_lag").await; } Err(RecvError::Closed) => break, } @@ -3673,6 +5156,8 @@ impl ScanRunAggregatorInner { Ok(event) => self.handle_scan_event(event).await, Err(RecvError::Lagged(skipped)) => { warn!("domain event stream lagged {skipped} events"); + self.mark_all_runs_reconciliation_required().await; + self.reconcile_all_runs("scan_event_lag").await; } Err(RecvError::Closed) => break, } @@ -3691,6 +5176,58 @@ impl ScanRunAggregatorInner { }; for run in runs { + let lag_reconciliation_required = self + .reconciliation_gate + .is_required(run.correlation_id()) + .await; + let pre_stall = + run.needs_pre_stall_reconciliation(self.stall_timeout).await; + if lag_reconciliation_required || pre_stall { + let reason = if lag_reconciliation_required { + "lag_retry" + } else { + "pre_stall" + }; + if let Err(err) = self.reconcile_run(&run, reason).await { + warn!( + scan = %run.scan_id(), + library = %run.library_id(), + error = %err, + reason, + "durable scan reconciliation failed; deferring terminal decision" + ); + continue; + } + + // PostgreSQL is authoritative for job lifecycle state. If a + // successful snapshot still contains active work, do not use + // the same stale timestamps to terminalize the run on this + // tick. A completion racing just after the snapshot will be + // observed by its event or by the next reconciliation. + if run.has_active_items().await { + tracing::debug!( + scan = %run.scan_id(), + library = %run.library_id(), + reason, + "durable jobs remain active; deferring terminal decision" + ); + continue; + } + } + + // A registration gate must survive an empty snapshot taken while + // Start is still enqueueing its seed batch. Seed completion is + // persisted separately; until a successful snapshot after that + // marker, total_items == 0 is not authoritative evidence of a + // no-work scan. + if !self + .reconciliation_gate + .allows_terminal(run.correlation_id()) + .await + { + continue; + } + if run .try_complete(self.quiescence_chrono, self.stall_timeout) .await @@ -3703,6 +5240,104 @@ impl ScanRunAggregatorInner { self.cleanup_series_bundle_trackers().await; } + async fn mark_all_runs_reconciliation_required(&self) { + let correlations: Vec = + self.runs.read().await.keys().copied().collect(); + self.reconciliation_gate.require_all(correlations).await; + } + + async fn reconcile_all_runs(&self, reason: &'static str) { + let runs: Vec> = { + let guard = self.runs.read().await; + guard.values().cloned().collect() + }; + + for run in runs { + if let Err(err) = self.reconcile_run(&run, reason).await { + warn!( + scan = %run.scan_id(), + library = %run.library_id(), + error = %err, + reason, + "failed to reconcile scan run from durable job state" + ); + } + } + } + + async fn reconcile_run( + &self, + run: &Arc, + reason: &'static str, + ) -> ferrex_core::error::Result<()> { + let tracked_job_ids = run.tracked_job_ids().await; + let jobs = self + .orchestrator + .durable_job_states(run.correlation_id(), &tracked_job_ids) + .await?; + if jobs.is_empty() + && (!tracked_job_ids.is_empty() + || run.has_unmaterialized_durable_progress().await) + { + return Err(ferrex_core::error::MediaError::Internal(format!( + "durable job snapshot for scan {} was empty despite tracked jobs or persisted progress", + run.scan_id() + ))); + } + let durable_job_ids: HashSet<_> = + jobs.iter().map(|job| job.job_id).collect(); + let missing_job_ids: Vec<_> = tracked_job_ids + .iter() + .copied() + .filter(|job_id| !durable_job_ids.contains(job_id)) + .collect(); + if !missing_job_ids.is_empty() { + return Err(ferrex_core::error::MediaError::Internal(format!( + "durable job snapshot for scan {} omitted {} tracked jobs", + run.scan_id(), + missing_job_ids.len() + ))); + } + let unresolved_retryable_failures = self + .reconciliation_gate + .unresolved_retryable_failures(run.correlation_id(), &jobs) + .await; + if !unresolved_retryable_failures.is_empty() { + return Err(ferrex_core::error::MediaError::Internal(format!( + "durable retry outcome for scan {} remains unresolved: {:?}", + run.scan_id(), + unresolved_retryable_failures + ))); + } + + tracing::debug!( + scan = %run.scan_id(), + library = %run.library_id(), + correlation = %run.correlation_id(), + tracked_jobs = tracked_job_ids.len(), + durable_jobs = jobs.len(), + reason, + "reconciling scan progress from durable job state" + ); + run.reconcile_durable_job_states(&jobs).await; + if !self + .reconciliation_gate + .clear_after_seeded_snapshot( + run.correlation_id(), + run.seed_completed().await, + ) + .await + { + tracing::debug!( + scan = %run.scan_id(), + library = %run.library_id(), + reason, + "durable snapshot preceded seed completion; retaining reconciliation gate" + ); + } + Ok(()) + } + async fn poll_series_bundle_finalizations(&self) { let now = Instant::now(); @@ -3750,21 +5385,66 @@ impl ScanRunAggregatorInner { } async fn handle_job_event(&self, event: JobEvent) { - let run = { + let candidates: Vec> = { let guard = self.runs.read().await; - guard.get(&event.meta.correlation_id).cloned() + guard.values().cloned().collect() + }; + let shared_job_id = match &event.payload { + JobEventPayload::Completed { job_id, .. } + | JobEventPayload::Failed { job_id, .. } + | JobEventPayload::DeadLettered { job_id, .. } => Some(*job_id), + _ => None, }; + let mut runs = Vec::new(); + for candidate in candidates { + let correlation_match = + candidate.correlation_id() == event.meta.correlation_id; + let tracked_job_match = if correlation_match { + false + } else if let Some(job_id) = shared_job_id { + candidate.library_id() == event.meta.library_id + && candidate.tracks_job_id(job_id).await + } else { + false + }; + if correlation_match || tracked_job_match { + runs.push(candidate); + } + } self.observe_series_bundle_job_event(&event).await; - if let Some(run) = run { - let completed = match event.payload { + if runs.is_empty() { + self.handle_orphan_event(&event).await; + return; + } + + for run in runs { + let terminalization_allowed = self + .reconciliation_gate + .allows_terminal(run.correlation_id()) + .await; + let completed = match &event.payload { JobEventPayload::Enqueued { kind, job_id, .. } | JobEventPayload::Dequeued { kind, job_id, .. } => { run.record_job_enqueued( &event.meta.idempotency_key, - job_id, - kind, + *job_id, + *kind, + event.meta.path_key.clone(), + ) + .await; + false + } + JobEventPayload::Merged { + existing_job_id, + kind, + .. + } => { + run.record_job_enqueued( + &event.meta.idempotency_key, + *existing_job_id, + *kind, event.meta.path_key.clone(), ) .await; @@ -3773,13 +5453,18 @@ impl ScanRunAggregatorInner { JobEventPayload::Completed { kind, job_id, .. } => { run.record_job_completed( &event.meta.idempotency_key, - job_id, - kind, + *job_id, + *kind, event.meta.path_key.clone(), ) .await; - run.try_complete(self.quiescence_chrono, self.stall_timeout) - .await + terminalization_allowed + && run + .try_complete( + self.quiescence_chrono, + self.stall_timeout, + ) + .await } JobEventPayload::Failed { kind, @@ -3787,17 +5472,45 @@ impl ScanRunAggregatorInner { job_id, .. } => { + if *retryable { + // The runtime can publish this event even when + // q.fail returned an error, so it is not proof that + // PostgreSQL actually moved the lease to Ready (or + // exhausted it to DeadLetter). Keep terminalization + // gated until an exact job-ID read observes the + // authoritative retry outcome. + self.reconciliation_gate + .require_retryable_failure( + run.correlation_id(), + *job_id, + ) + .await; + } run.record_job_failure( &event.meta.idempotency_key, - job_id, - kind, + *job_id, + *kind, None, event.meta.path_key.clone(), - retryable, + *retryable, ) .await; - if !retryable { + if *retryable { + if let Err(err) = self + .reconcile_run(&run, "retryable_job_failure") + .await + { + warn!( + scan = %run.scan_id(), + library = %run.library_id(), + job = %job_id, + error = %err, + "retryable job failure is not yet confirmed durably; terminalization remains gated" + ); + } + false + } else if terminalization_allowed { run.try_complete( self.quiescence_chrono, self.stall_timeout, @@ -3810,19 +5523,24 @@ impl ScanRunAggregatorInner { JobEventPayload::DeadLettered { kind, job_id, .. } => { run.record_job_dead_lettered( &event.meta.idempotency_key, - job_id, - kind, + *job_id, + *kind, None, event.meta.path_key.clone(), ) .await; - run.try_complete(self.quiescence_chrono, self.stall_timeout) - .await + terminalization_allowed + && run + .try_complete( + self.quiescence_chrono, + self.stall_timeout, + ) + .await } JobEventPayload::LeaseRenewed { job_id, .. } => { run.record_job_lease_renewed( &event.meta.idempotency_key, - job_id, + *job_id, event.meta.path_key.clone(), ) .await; @@ -3834,8 +5552,6 @@ impl ScanRunAggregatorInner { if completed { self.on_run_completed(run.clone()).await; } - } else { - self.handle_orphan_event(&event).await; } } @@ -3897,10 +5613,43 @@ impl ScanRunAggregatorInner { guard.get(&correlation_id).cloned() }; - if let Some(run) = run - && run.record_seed_completed(&summary).await - { - self.on_run_completed(run).await; + if let Some(run) = run { + let has_enrollment = + !summary.enrolled_job_ids.is_empty(); + if has_enrollment { + // Even if every enqueue notification was dropped, + // the seed mailbox has the exact durable job IDs. + // Force a PostgreSQL read before a zero-item run + // can make a terminal decision. + self.reconciliation_gate + .require_all([correlation_id]) + .await; + } + let terminalization_allowed = self + .reconciliation_gate + .allows_terminal(correlation_id) + .await; + let completed = run + .record_seed_completed( + &summary, + terminalization_allowed, + ) + .await; + if has_enrollment + && let Err(err) = self + .reconcile_run(&run, "post_seed_enrollment") + .await + { + warn!( + scan = %run.scan_id(), + library = %run.library_id(), + error = %err, + "post-seed durable reconciliation failed; terminalization remains gated" + ); + } + if completed { + self.on_run_completed(run).await; + } } } } @@ -4052,7 +5801,9 @@ impl ScanRunAggregatorInner { let library_id = run.library_id(); let command = LibraryActorCommand::Start { mode: StartMode::Maintenance, - correlation_id: Some(Uuid::now_v7()), + // The actor uses the ending run correlation as a generation guard: + // a delayed Maintenance(A) cannot replace a newer Bulk(B). + correlation_id: Some(run.correlation_id()), }; match self.orchestrator.command_library(library_id, command).await { diff --git a/crates/ferrex-server/src/routes/v1.rs b/crates/ferrex-server/src/routes/v1.rs index 7567cfea..358d1143 100644 --- a/crates/ferrex-server/src/routes/v1.rs +++ b/crates/ferrex-server/src/routes/v1.rs @@ -35,7 +35,8 @@ use crate::{ create_library_handler, delete_library_handler, get_libraries_with_media_handler, get_library_handler, get_library_media_handler, get_library_sorted_indices_handler, - post_library_filtered_indices_handler, update_library_handler, + post_library_filtered_indices_handler, reset_library_handler, + update_library_handler, }, handle_movie_batches::{ get_movie_reference_batch_bundle_handler, @@ -463,6 +464,7 @@ fn create_libraries_routes(state: AppState) -> Router { v1::libraries::ITEM, axum::routing::delete(delete_library_handler), ) + .route(v1::libraries::RESET, post(reset_library_handler)) .route( v1::libraries::DISCOVERY, get(discovery::get_library_discovery_handler), @@ -578,15 +580,6 @@ fn create_admin_routes(state: AppState) -> Router { axum::routing::delete(admin_handlers::revoke_user_session_admin), ) .route(v1::admin::STATS, get(admin_handlers::get_admin_stats)) - // Development/reset endpoints (admin only) - .route( - v1::admin::dev::RESET_CHECK, - get(dev_handlers::check_reset_status), - ) - .route( - v1::admin::dev::RESET_DATABASE, - post(dev_handlers::reset_database), - ) .route(MEDIA_ROOT_BROWSER, get(media_root::browse_media_root)) // Admin session management for PIN authentication .route( @@ -610,13 +603,32 @@ fn create_admin_routes(state: AppState) -> Router { .route(v1::admin::demo::RESET, post(demo_handlers::reset)) .route(v1::admin::demo::RESIZE, post(demo_handlers::resize)); - router + let admin_router = router // route_layer applies inside-out; add admin guard first so auth runs before it .route_layer(middleware::from_fn(auth::middleware::admin_middleware)) .route_layer(middleware::from_fn_with_state( state.clone(), auth::middleware::auth_middleware, - )) + )); + + // Reset handlers enforce the dedicated server:reset_database permission. + // Keeping them outside the broad user-admin guard allows a deliberately + // delegated reset operator to use the capability without users:* access. + let reset_router = Router::new() + .route( + v1::admin::dev::RESET_CHECK, + get(dev_handlers::check_reset_status), + ) + .route( + v1::admin::dev::RESET_DATABASE, + post(dev_handlers::reset_database), + ) + .route_layer(middleware::from_fn_with_state( + state, + auth::middleware::auth_middleware, + )); + + admin_router.merge(reset_router) } /// Create role management routes @@ -648,3 +660,14 @@ fn create_role_routes(state: AppState) -> Router { auth::middleware::auth_middleware, )) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn library_reset_route_is_valid_axum_syntax() { + let _: Router = Router::new() + .route(v1::libraries::RESET, post(reset_library_handler)); + } +} diff --git a/crates/ferrex-server/tests/mobile_media_flatbuffers.rs b/crates/ferrex-server/tests/mobile_media_flatbuffers.rs index 14a14c9b..6c0326ec 100644 --- a/crates/ferrex-server/tests/mobile_media_flatbuffers.rs +++ b/crates/ferrex-server/tests/mobile_media_flatbuffers.rs @@ -1537,6 +1537,8 @@ async fn series_bundle_finalization_emits_once_and_feeds_sync_fetch( series_context.clone(), )), reason: ScanReason::BulkSeed, + correlation_id: None, + durable_job_id: None, }, ) .await?; @@ -1555,6 +1557,8 @@ async fn series_bundle_finalization_emits_once_and_feeds_sync_fetch( season_context.clone(), )), reason: ScanReason::BulkSeed, + correlation_id: None, + durable_job_id: None, }, ) .await?; @@ -1688,6 +1692,8 @@ async fn series_bundle_finalization_emits_once_and_feeds_sync_fetch( series_context.clone(), )), reason: ScanReason::BulkSeed, + correlation_id: None, + durable_job_id: None, }, ) .await?; @@ -1698,6 +1704,8 @@ async fn series_bundle_finalization_emits_once_and_feeds_sync_fetch( season_context.clone(), )), reason: ScanReason::BulkSeed, + correlation_id: None, + durable_job_id: None, }, ) .await?; diff --git a/crates/ferrex-server/tests/scan_completion_regression.rs b/crates/ferrex-server/tests/scan_completion_regression.rs index 8341156b..ce437e28 100644 --- a/crates/ferrex-server/tests/scan_completion_regression.rs +++ b/crates/ferrex-server/tests/scan_completion_regression.rs @@ -32,7 +32,7 @@ use ferrex_core::{ context::{ FolderScanContext, MovieFolderScanContext, MovieRootPath, }, - job::{EnqueueRequest, JobPayload, JobPriority}, + job::{EnqueueRequest, JobKind, JobPayload, JobPriority}, persistence::{PostgresCursorRepository, PostgresQueueService}, scan_cursor::{ ScanCursor, ScanCursorId, ScanCursorRepository, normalize_path, @@ -296,7 +296,7 @@ async fn orchestrator_start_primes_persisted_ready_jobs_once() -> Result<()> { let scheduler = orchestrator.runtime().scheduler(); let reservation = scheduler - .reserve() + .reserve(JobKind::FolderScan) .await .expect("persisted ready job should be scheduled after startup"); assert_eq!(reservation.library_id, library_id); @@ -310,7 +310,7 @@ async fn orchestrator_start_primes_persisted_ready_jobs_once() -> Result<()> { scheduler.record_completed(library_id).await; assert!( - scheduler.reserve().await.is_none(), + scheduler.reserve(JobKind::FolderScan).await.is_none(), "startup must prime persisted ready jobs once, without phantom reservations" ); diff --git a/docs/src/content/docs/operator/configuration.md b/docs/src/content/docs/operator/configuration.md index aa7c3211..3485a8d5 100644 --- a/docs/src/content/docs/operator/configuration.md +++ b/docs/src/content/docs/operator/configuration.md @@ -114,7 +114,7 @@ just start --mode tailscale ## Scanner / Incremental Scans -Scanner settings can be supplied in `scanner.toml`, `scanner.json`, `config/scanner.toml`, `config/scanner.json`, `SCANNER_CONFIG_PATH`, or inline JSON via `SCANNER_CONFIG_JSON`. Existing installs that do not provide a scanner config keep safe defaults: libraries auto-scan every 60 minutes, filesystem watching is enabled per library, and the watcher uses `auto` strategy with native notifications falling back to polling. +Scanner settings can be supplied in `scanner.toml`, `scanner.json`, `config/scanner.toml`, `config/scanner.json`, `SCANNER_CONFIG_PATH`, or inline JSON via `SCANNER_CONFIG_JSON`. Existing installs that do not provide a scanner config keep safe defaults: libraries auto-scan every 60 minutes, filesystem watching is enabled per library, and the watcher uses `auto` strategy. On Linux, `auto` selects polling immediately for CIFS/SMB3 and NFS mounts; other filesystems use native notifications with polling fallback. Library create/update API payloads can override per-library policy: @@ -146,10 +146,19 @@ enabled = true tick_interval_ms = 60000 max_jobs_per_library = 128 max_root_entries_per_library = 512 + +[orchestrator.lease] +dispatch_timeout_ms = 1800000 # 30 minutes per job execution attempt ``` +When a root has more eligible folders than a maintenance pass can enqueue, Ferrex rotates deterministic partitions on successive scan intervals. This rotation does not depend on the prior partition completing successfully, so folders beyond the configured bound are still revisited. + +Job execution attempts are bounded to 30 minutes by default. When an actor or external dependency never returns, Ferrex stops renewing that attempt, releases its lease and workload capacity, and routes the job through the normal retry policy. Operators can lower `orchestrator.lease.dispatch_timeout_ms` for faster recovery, but it should remain above the longest expected scan, analysis, metadata, indexing, or image operation. + Network/container mount example (prefer bounded polling over unreliable native events): +Explicit `poll` remains useful for container mounts whose host filesystem type is hidden from the container. If `native` is forced on a detected CIFS/NFS mount, startup logs an operator warning because notifications may be unreliable. + ```toml video_extensions = ["mkv", "mp4", "mpeg", "ts"] ignored_extensions = ["tmp", "part", "download"] @@ -212,7 +221,7 @@ DATABASE_URL=postgres://... cargo test -p ferrex-core --test transcript_reposito DATABASE_URL=postgres://... cargo test -p ferrex-server --test intelligence_routes transcript_purge_and_rebuild_routes_remove_searchable_segments ``` -Invalid scanner config fails during startup with the field path in the error (for example, `scanner.orchestrator.watch.poll_interval_ms must be greater than 0`). Operators can inspect the effective policy and health counters via the scan config/metrics/status endpoints; these report watch strategy, poll/debounce/batch settings, transcript indexing controls, maintenance sweep policy, media/ignore filters, watcher registrations, replay lag, stale cursor counts, and overflow events. +Invalid scanner config fails during startup with the field path in the error (for example, `scanner.orchestrator.watch.poll_interval_ms must be greater than 0`). Operators can inspect the effective policy and health counters via the scan config/metrics/status endpoints; these report watch strategy, poll/debounce/batch settings, the job dispatch deadline, transcript indexing controls, maintenance sweep policy, media/ignore filters, watcher registrations, replay lag, stale cursor counts, overflow events, and root-discovery truncation/deferred-entry counters. ## Logging From 2140e8387e69d842e48c4153a05d96e275d9a79d Mon Sep 17 00:00:00 2001 From: Grayson Hieb Date: Sun, 9 Aug 2026 12:22:52 -0600 Subject: [PATCH 03/10] fix(player): wire destructive library controls --- .../src/adapters/api_client_adapter.rs | 34 +- crates/ferrex-player-api/src/api_client.rs | 49 +- crates/ferrex-player-api/src/services/api.rs | 25 +- .../src/testing/stubs/api.rs | 157 ++++++- .../ferrex-player-library/src/messages/mod.rs | 17 +- .../src/common/clear_database.rs | 52 ++- .../domains/auth/update_handlers/auth_flow.rs | 16 +- .../src/domains/library/update.rs | 74 ++- .../update_handlers/library_management.rs | 103 ++--- crates/ferrex-player-ui/src/domains/ui/mod.rs | 13 +- .../src/domains/ui/settings_ui/mod.rs | 37 +- .../src/domains/ui/settings_ui/update.rs | 69 ++- .../src/domains/ui/views/admin/mod.rs | 2 + .../src/domains/ui/views/admin/view_admin.rs | 96 +++- .../view_library_maintenance_confirmation.rs | 153 +++++++ .../ui/views/admin/view_library_management.rs | 421 ++++++++++++++++-- crates/ferrex-player-ui/src/state.rs | 3 +- crates/ferrex-player-ui/src/view.rs | 19 +- 18 files changed, 1117 insertions(+), 223 deletions(-) create mode 100644 crates/ferrex-player-ui/src/domains/ui/views/admin/view_library_maintenance_confirmation.rs diff --git a/crates/ferrex-player-api/src/adapters/api_client_adapter.rs b/crates/ferrex-player-api/src/adapters/api_client_adapter.rs index 2d1b0169..9f0653e7 100644 --- a/crates/ferrex-player-api/src/adapters/api_client_adapter.rs +++ b/crates/ferrex-player-api/src/adapters/api_client_adapter.rs @@ -11,6 +11,9 @@ use crate::{ }; use ferrex_player_foundation::repository::{RepositoryError, RepositoryResult}; +use ferrex_core::api::types::admin::{ + ResetDatabaseRequest, ResetDatabaseResult, +}; use ferrex_core::api::types::collections::*; use ferrex_core::player_prelude::{ ActiveScansResponse, AuthToken, AuthenticatedDevice, CreateLibraryRequest, @@ -19,8 +22,9 @@ use ferrex_core::player_prelude::{ LibraryMediaResponse, Media, MediaID, MediaQuery, MediaRootBrowseResponse, MediaWithStatus, MovieBatchFetchRequest, MovieBatchId, MovieBatchSyncRequest, MovieBatchSyncResponse, NextEpisode, - ScanCommandAcceptedResponse, ScanCommandRequest, ScanConfig, ScanMetrics, - SeasonWatchStatus, SeriesBundleFetchRequest, SeriesBundleSyncRequest, + ResetLibraryRequest, ResetLibraryResult, ScanCommandAcceptedResponse, + ScanCommandRequest, ScanConfig, ScanMetrics, SeasonWatchStatus, + SeriesBundleFetchRequest, SeriesBundleSyncRequest, SeriesBundleSyncResponse, SeriesID, SeriesWatchStatus, SortBy, SortOrder, StartScanRequest, UpdateLibraryRequest, UpdateProgressRequest, User, UserPermissions, UserWatchState, @@ -992,6 +996,32 @@ impl ApiService for ApiClientAdapter { Ok(()) } + async fn reset_library( + &self, + id: LibraryId, + request: ResetLibraryRequest, + ) -> RepositoryResult { + let path = replace_param( + v1::libraries::RESET, + "{id}", + id.as_uuid().to_string(), + ); + self.client + .post(&path, &request) + .await + .map_err(|e| RepositoryError::DeleteFailed(e.to_string())) + } + + async fn reset_database( + &self, + request: ResetDatabaseRequest, + ) -> RepositoryResult { + self.client + .post(v1::admin::dev::RESET_DATABASE, &request) + .await + .map_err(|e| RepositoryError::DeleteFailed(e.to_string())) + } + async fn start_library_scan( &self, library_id: LibraryId, diff --git a/crates/ferrex-player-api/src/api_client.rs b/crates/ferrex-player-api/src/api_client.rs index 779a5800..b3c26faa 100644 --- a/crates/ferrex-player-api/src/api_client.rs +++ b/crates/ferrex-player-api/src/api_client.rs @@ -47,6 +47,24 @@ pub type RefreshTokenCallback = Arc< >, >; +fn decode_api_response(response: ApiResponse) -> Result { + let ApiResponse { + status, + data, + error, + message, + } = response; + + if status != "success" || error.is_some() { + let detail = error.or(message).unwrap_or_else(|| { + format!("Server returned API status '{status}'") + }); + return Err(anyhow::anyhow!(detail)); + } + + data.ok_or_else(|| anyhow::anyhow!("Empty response from server")) +} + /// API client with authentication support #[derive(Clone)] pub struct ApiClient { @@ -264,10 +282,7 @@ impl ApiClient { )); } let api_response: ApiResponse = response.json().await?; - match api_response.data { - Some(data) => Ok(data), - None => Err(anyhow::anyhow!("Empty response from server")), - } + decode_api_response(api_response) } StatusCode::UNAUTHORIZED => { // Try to refresh token if we have a callback @@ -330,10 +345,7 @@ impl ApiClient { )); } let api_response: ApiResponse = response.json().await?; - match api_response.data { - Some(data) => Ok(data), - None => Err(anyhow::anyhow!("Empty response from server")), - } + decode_api_response(api_response) } StatusCode::UNAUTHORIZED => { // Don't retry, just clear token and return error @@ -1199,6 +1211,27 @@ mod tests { Uuid::parse_str(value).expect("valid uuid") } + #[test] + fn api_error_envelope_on_http_success_surfaces_server_detail() { + let error = decode_api_response::(ApiResponse::error( + "library deletion blocked by provenance".to_string(), + )) + .expect_err( + "error envelope must not be treated as a successful request", + ); + + assert_eq!(error.to_string(), "library deletion blocked by provenance"); + } + + #[test] + fn api_success_envelope_returns_payload() { + assert_eq!( + decode_api_response(ApiResponse::success("deleted".to_string())) + .expect("success envelope should return data"), + "deleted" + ); + } + #[test] fn collection_paths_use_shared_route_constants() { let collection_id = diff --git a/crates/ferrex-player-api/src/services/api.rs b/crates/ferrex-player-api/src/services/api.rs index 2f9f10db..72907cc1 100644 --- a/crates/ferrex-player-api/src/services/api.rs +++ b/crates/ferrex-player-api/src/services/api.rs @@ -8,6 +8,7 @@ use crate::api_types::{DemoResetRequest, DemoStatus}; use async_trait::async_trait; use ferrex_core::{ api::types::{ + admin::{ResetDatabaseRequest, ResetDatabaseResult}, collections::*, setup::{ConfirmClaimResponse, StartClaimResponse}, }, @@ -17,12 +18,12 @@ use ferrex_core::{ ImageManifestResponse, LatestProgressResponse, Library, LibraryId, Media, MediaQuery, MediaRootBrowseResponse, MediaWithStatus, MovieBatchFetchRequest, MovieBatchId, MovieBatchSyncRequest, - MovieBatchSyncResponse, NextEpisode, ScanCommandAcceptedResponse, - ScanCommandRequest, ScanConfig, ScanMetrics, SeasonWatchStatus, - SeriesBundleFetchRequest, SeriesBundleSyncRequest, - SeriesBundleSyncResponse, SeriesID, SeriesWatchStatus, - StartScanRequest, UpdateLibraryRequest, UpdateProgressRequest, User, - UserPermissions, UserWatchState, + MovieBatchSyncResponse, NextEpisode, ResetLibraryRequest, + ResetLibraryResult, ScanCommandAcceptedResponse, ScanCommandRequest, + ScanConfig, ScanMetrics, SeasonWatchStatus, SeriesBundleFetchRequest, + SeriesBundleSyncRequest, SeriesBundleSyncResponse, SeriesID, + SeriesWatchStatus, StartScanRequest, UpdateLibraryRequest, + UpdateProgressRequest, User, UserPermissions, UserWatchState, }, }; use ferrex_model::image::ImageQuery; @@ -154,6 +155,18 @@ pub trait ApiService: Send + Sync + Debug { ) -> RepositoryResult<()>; /// Delete a library on the server async fn delete_library(&self, id: LibraryId) -> RepositoryResult<()>; + /// Atomically clear a library while preserving its configuration and ID. + async fn reset_library( + &self, + id: LibraryId, + request: ResetLibraryRequest, + ) -> RepositoryResult; + + /// Perform an authenticated administrative database reset. + async fn reset_database( + &self, + request: ResetDatabaseRequest, + ) -> RepositoryResult; /// Start a library scan async fn start_library_scan( diff --git a/crates/ferrex-player-api/src/testing/stubs/api.rs b/crates/ferrex-player-api/src/testing/stubs/api.rs index 1a2cb09b..d27660bc 100644 --- a/crates/ferrex-player-api/src/testing/stubs/api.rs +++ b/crates/ferrex-player-api/src/testing/stubs/api.rs @@ -4,6 +4,9 @@ use std::sync::{Arc, RwLock}; use async_trait::async_trait; use chrono::{Duration, Utc}; +use ferrex_core::api::types::admin::{ + ResetDatabaseRequest, ResetDatabaseResult, +}; use ferrex_core::api::types::collections::*; use ferrex_core::domain::users::auth::{ device::AuthDeviceStatus, domain::value_objects::SessionScope, @@ -14,13 +17,13 @@ use ferrex_core::player_prelude::{ ImageManifestResponse, LatestProgressResponse, Library, LibraryId, LibraryType, Media, MediaQuery, MediaRootBrowseResponse, MediaWithStatus, MovieBatchFetchRequest, MovieBatchId, MovieBatchSyncRequest, - MovieBatchSyncResponse, Platform, Role, ScanCommandAcceptedResponse, - ScanCommandRequest, ScanConfig, ScanLifecycleStatus, ScanMetrics, - ScanRunMode, ScanStartDisposition, SeriesBundleFetchRequest, - SeriesBundleSyncRequest, SeriesBundleSyncResponse, SeriesID, - StartClaimResponse, StartScanRequest, UpdateLibraryRequest, - UpdateProgressRequest, User, UserPermissions, UserPreferences, - UserWatchState, + MovieBatchSyncResponse, Platform, ResetLibraryRequest, ResetLibraryResult, + Role, ScanCommandAcceptedResponse, ScanCommandRequest, ScanConfig, + ScanLifecycleStatus, ScanMetrics, ScanRunMode, ScanStartDisposition, + SeriesBundleFetchRequest, SeriesBundleSyncRequest, + SeriesBundleSyncResponse, SeriesID, StartClaimResponse, StartScanRequest, + UpdateLibraryRequest, UpdateProgressRequest, User, UserPermissions, + UserPreferences, UserWatchState, }; use ferrex_model::image::ImageQuery; use ferrex_model::{MediaID, MovieID, MovieReferenceBatchSize}; @@ -57,6 +60,7 @@ struct InnerApiState { current_user: Option, current_permissions: Option, playback_ticket_result: Option>, + reset_database_requests: Vec, } #[derive(Debug, Clone)] @@ -103,6 +107,7 @@ impl TestApiService { current_user: Some(sample_user), current_permissions: Some(sample_permissions), playback_ticket_result: None, + reset_database_requests: Vec::new(), })), base_url: Arc::from(base_url_string), } @@ -150,6 +155,14 @@ impl TestApiService { } } + /// Return the most recent administrative reset request observed by the stub. + pub fn last_reset_database_request(&self) -> Option { + self.inner + .read() + .ok() + .and_then(|guard| guard.reset_database_requests.last().cloned()) + } + /// Seed or replace a collection detail in the in-memory API stub. pub fn upsert_collection( &self, @@ -591,6 +604,8 @@ impl ApiService for TestApiService { enabled, auto_scan, watch_for_changes, + analyze_on_scan, + max_retry_attempts, movie_ref_batch_size, start_scan: _, } = request; @@ -606,8 +621,8 @@ impl ApiService for TestApiService { enabled, auto_scan, watch_for_changes, - analyze_on_scan: false, - max_retry_attempts: 3, + analyze_on_scan, + max_retry_attempts, movie_ref_batch_size: ferrex_model::MovieReferenceBatchSize::new( movie_ref_batch_size, ) @@ -648,6 +663,12 @@ impl ApiService for TestApiService { if let Some(watch_for_changes) = request.watch_for_changes { library.watch_for_changes = watch_for_changes; } + if let Some(analyze_on_scan) = request.analyze_on_scan { + library.analyze_on_scan = analyze_on_scan; + } + if let Some(max_retry_attempts) = request.max_retry_attempts { + library.max_retry_attempts = max_retry_attempts; + } if let Some(size) = request.movie_ref_batch_size { library.movie_ref_batch_size = ferrex_model::MovieReferenceBatchSize::new(size) @@ -669,6 +690,67 @@ impl ApiService for TestApiService { Ok(()) } + async fn reset_library( + &self, + id: LibraryId, + request: ResetLibraryRequest, + ) -> RepositoryResult { + let mut guard = self.inner.write().expect("lock poisoned"); + let enabled = guard + .libraries + .iter() + .find(|library| library.id == id) + .map(|library| library.enabled) + .ok_or_else(|| RepositoryError::NotFound { + entity_type: "Library".into(), + id: id.to_string(), + })?; + guard.library_media.remove(id.as_uuid()); + + let scan = enabled.then(|| { + let mode = ScanRunMode::Manual; + let run_key = mode.run_key(id); + ScanCommandAcceptedResponse { + scan_id: request.operation_id, + correlation_id: request.operation_id, + status: ScanLifecycleStatus::Running, + mode, + idempotency_key: run_key.clone(), + run_key, + disposition: ScanStartDisposition::Created, + } + }); + Ok(ResetLibraryResult { + library_id: id, + scan, + }) + } + + async fn reset_database( + &self, + request: ResetDatabaseRequest, + ) -> RepositoryResult { + let mut guard = self.inner.write().expect("lock poisoned"); + let mut result = ResetDatabaseResult::default(); + + if request.reset_libraries { + result.libraries_deleted = guard.libraries.len(); + guard.libraries.clear(); + guard.library_media.clear(); + } + if request.reset_users { + result.users_deleted = usize::from(guard.current_user.is_some()); + result.sessions_deleted = usize::from(guard.auth_token.is_some()); + result.roles_reset = 3; + guard.current_user = None; + guard.current_permissions = None; + guard.auth_token = None; + } + + guard.reset_database_requests.push(request); + Ok(result) + } + async fn start_library_scan( &self, _library_id: LibraryId, @@ -2078,6 +2160,63 @@ mod tests { } } + #[tokio::test] + async fn reset_library_preserves_identity_and_returns_deterministic_scan() { + let service = TestApiService::default(); + let original = service + .fetch_libraries() + .await + .expect("list libraries") + .into_iter() + .next() + .expect("sample library"); + let operation_id = Uuid::now_v7(); + + let result = service + .reset_library(original.id, ResetLibraryRequest { operation_id }) + .await + .expect("reset library"); + + assert_eq!(result.library_id, original.id); + assert_eq!( + result.scan.as_ref().map(|scan| scan.scan_id), + Some(operation_id) + ); + let preserved = service + .fetch_libraries() + .await + .expect("list reset libraries") + .into_iter() + .next() + .expect("preserved library"); + assert_eq!(preserved.id, original.id); + assert_eq!(preserved.analyze_on_scan, original.analyze_on_scan); + assert_eq!(preserved.max_retry_attempts, original.max_retry_attempts); + } + + #[tokio::test] + async fn clear_all_data_reset_records_contract_and_clears_state() { + let service = TestApiService::default(); + let request = ResetDatabaseRequest::clear_all_data(); + + let result = service + .reset_database(request.clone()) + .await + .expect("clear all data should succeed"); + + assert_eq!(service.last_reset_database_request(), Some(request)); + assert_eq!(result.libraries_deleted, 1); + assert_eq!(result.users_deleted, 1); + assert!( + service + .fetch_libraries() + .await + .expect("list libraries") + .is_empty() + ); + assert!(service.fetch_current_user().await.is_err()); + } + #[tokio::test] async fn collection_stub_lists_details_pages_and_reorders_items() { let service = TestApiService::default(); diff --git a/crates/ferrex-player-library/src/messages/mod.rs b/crates/ferrex-player-library/src/messages/mod.rs index 49501514..94d4d917 100644 --- a/crates/ferrex-player-library/src/messages/mod.rs +++ b/crates/ferrex-player-library/src/messages/mod.rs @@ -15,12 +15,15 @@ use crate::types::LibrariesBootstrapPayload; use ferrex_core::player_prelude::Library as CoreLibrary; use ferrex_core::player_prelude::{ LibraryId, LibraryMediaResponse, MediaFile, MovieBatchId, - ScanCommandAcceptedResponse, ScanConfig, ScanMetrics, ScanProgressEvent, - ScanSnapshotDto, SeriesID, + ResetLibraryResult, ScanCommandAcceptedResponse, ScanConfig, ScanMetrics, + ScanProgressEvent, ScanSnapshotDto, SeriesID, }; use ferrex_player_api::api_types::{Library as ApiLibrary, Media, MediaID}; use uuid::Uuid; +/// Server-side reset result; library identity is preserved. +pub type LibraryResetResult = ResetLibraryResult; + #[derive(Clone)] pub enum LibraryMessage { // Core library loading @@ -95,9 +98,9 @@ pub enum LibraryMessage { FetchScanConfig, ScanConfigLoaded(Result), - // Destructive: delete and recreate library with start_scan=true + // Destructive: atomically clear owned data and start a fresh scan ResetLibrary(LibraryId), - ResetLibraryDone(Result<(), String>), + ResetLibraryDone(Result), // Media references LibraryMediasLoaded(Result), @@ -545,7 +548,11 @@ impl std::fmt::Debug for LibraryMessage { write!(f, "Library::ResetLibrary({})", id) } Self::ResetLibraryDone(result) => match result { - Ok(()) => write!(f, "Library::ResetLibraryDone(Ok)"), + Ok(result) => write!( + f, + "Library::ResetLibraryDone(Ok: {})", + result.library_id + ), Err(e) => write!(f, "Library::ResetLibraryDone(Err: {})", e), }, } diff --git a/crates/ferrex-player-ui/src/common/clear_database.rs b/crates/ferrex-player-ui/src/common/clear_database.rs index e78ef142..6fd9ae2b 100644 --- a/crates/ferrex-player-ui/src/common/clear_database.rs +++ b/crates/ferrex-player-ui/src/common/clear_database.rs @@ -1,32 +1,25 @@ use iced::Task; use crate::{ - common::messages::{CrossDomainEvent, DomainMessage}, + common::messages::DomainMessage, + domains::auth::messages::AuthMessage, domains::ui::{ messages::UiMessage, settings_ui::SettingsUiMessage, types::ViewState, }, state::State, }; +use ferrex_core::api::types::admin::ResetDatabaseRequest; pub fn handle_clear_database(state: &mut State) -> Task { log::info!("Clearing all database contents"); - state.domains.ui.state.show_clear_database_confirm = false; // Hide confirmation dialog - let server_url = state.server_url.clone(); + state.domains.ui.state.library_maintenance_confirmation = None; + let api = state.api_service.clone(); Task::perform( async move { - let client = reqwest::Client::new(); - let url = format!("{}/maintenance/clear-database", server_url); - - match client.post(&url).send().await { - Ok(response) => { - if response.status().is_success() { - Ok(()) - } else { - Err(format!("Server error: {}", response.status())) - } - } - Err(e) => Err(format!("Request failed: {}", e)), - } + api.reset_database(ResetDatabaseRequest::clear_all_data()) + .await + .map(|_| ()) + .map_err(|error| error.to_string()) }, |result| { DomainMessage::Ui(UiMessage::Settings( @@ -40,6 +33,7 @@ pub fn handle_database_cleared( state: &mut State, result: Result<(), String>, ) -> Task { + state.domains.ui.state.library_maintenance_in_flight = None; match result { Ok(()) => { log::info!("Database cleared successfully"); @@ -57,7 +51,11 @@ pub fn handle_database_cleared( // Clear library state state.domains.library.state.library_form_data = None; state.domains.library.state.library_form_errors.clear(); + state.domains.library.state.library_form_success = None; state.domains.library.state.library_media_cache.clear(); + state.domains.library.state.libraries.clear(); + state.domains.library.state.load_state = + crate::domains::library::LibrariesLoadState::NotStarted; // Reset scan state state.loading = false; @@ -73,6 +71,9 @@ pub fn handle_database_cleared( // Clear UI state state.domains.ui.state.hovered_media_id = None; state.domains.ui.state.error_message = None; + state.domains.ui.state.current_library_id = None; + state.domains.ui.state.scope = + crate::domains::ui::shell_ui::Scope::Home; // Reset TabManager tabs //state.tab_manager.clear(); @@ -82,16 +83,27 @@ pub fn handle_database_cleared( // Reset view to library (in case user was in detail view) state.domains.ui.state.view = ViewState::Library; + state.is_authenticated = false; + state.domains.auth.state.is_authenticated = false; + state.domains.auth.state.user_permissions = None; log::info!("All local state cleared and reset"); - // Emit cross-domain event to trigger library refresh - Task::done(DomainMessage::Event(CrossDomainEvent::DatabaseCleared)) + // The full wipe includes users and sessions. Clear persisted local + // credentials as well so the next screen is a clean setup/login flow. + Task::done(DomainMessage::Auth(AuthMessage::ResetLocalAuthState)) } Err(e) => { log::error!("Failed to clear database: {}", e); - state.domains.ui.state.error_message = - Some(format!("Failed to clear database: {}", e)); + let message = format!("Failed to clear all data: {}", e); + state.domains.ui.state.error_message = Some(message.clone()); + state.domains.library.state.library_form_errors.clear(); + state + .domains + .library + .state + .library_form_errors + .push(message); Task::none() } } diff --git a/crates/ferrex-player-ui/src/domains/auth/update_handlers/auth_flow.rs b/crates/ferrex-player-ui/src/domains/auth/update_handlers/auth_flow.rs index 279da217..e67c63e3 100644 --- a/crates/ferrex-player-ui/src/domains/auth/update_handlers/auth_flow.rs +++ b/crates/ferrex-player-ui/src/domains/auth/update_handlers/auth_flow.rs @@ -1412,18 +1412,12 @@ pub fn handle_local_auth_state_reset( ) -> Task { match result { Ok(()) => { + // The server may have been reset at the same time as the local + // credentials (for example, Clear All Data). Reconcile setup + // status before choosing between login and first-run setup. state.domains.auth.state.auth_flow = - AuthenticationFlow::PreAuthLogin { - username: String::new(), - password: SecureCredential::new(String::new()), - show_password: false, - remember_device: false, - error: Some( - "Local auth and trusted-device state cleared. Sign in with your username and password to trust this device again." - .to_string(), - ), - loading: false, - }; + AuthenticationFlow::LoadingUsers; + return Task::done(AuthMessage::CheckSetupStatus); } Err(error) => { state.domains.auth.state.auth_flow = diff --git a/crates/ferrex-player-ui/src/domains/library/update.rs b/crates/ferrex-player-ui/src/domains/library/update.rs index ddc73cd2..9eb3bff2 100644 --- a/crates/ferrex-player-ui/src/domains/library/update.rs +++ b/crates/ferrex-player-ui/src/domains/library/update.rs @@ -508,26 +508,64 @@ pub fn update_library( } LibraryMessage::ResetLibraryDone(result) => { - if let Err(err) = result { - state.domains.ui.state.error_message = - Some(format!("Library reset failed: {}", err)); - } else { - // Refresh libraries and active scans after fresh rescan - let fetch = - super::update_handlers::library_loaded::fetch_libraries( - state.api_service.clone(), - state.disk_media_repo_cache.clone(), + let reset_library_id = + match state.domains.ui.state.library_maintenance_in_flight { + Some( + crate::domains::ui::LibraryMaintenanceAction::Reset( + library_id, + ), + ) => Some(library_id), + _ => None, + }; + state.domains.ui.state.library_maintenance_in_flight = None; + + match result { + Ok(reset) => { + debug_assert_eq!(reset_library_id, Some(reset.library_id)); + state.domains.library.state.library_form_errors.clear(); + state.domains.ui.state.error_message = None; + state.domains.library.state.library_form_success = Some( + if reset.scan.is_some() { + "Library reset successfully; a fresh scan has started" + .to_string() + } else { + "Library reset successfully; enable it to start a fresh scan" + .to_string() + }, ); - return DomainUpdateResult::task( - Task::perform(fetch, |res| { - LibraryMessage::LibrariesLoaded( - res.map_err(|e| format!("{:#}", e)), - ) - }) - .map(DomainMessage::Library), - ); + } + Err(err) => { + let message = format!("Library reset failed: {}", err); + state.domains.ui.state.error_message = + Some(message.clone()); + state.domains.library.state.library_form_success = None; + state.domains.library.state.library_form_errors.clear(); + state + .domains + .library + .state + .library_form_errors + .push(message); + } } - DomainUpdateResult::task(Task::none()) + + // Reconcile library contents and scan state after success or an + // error returned after the transactional reset boundary. + let fetch = super::update_handlers::library_loaded::fetch_libraries( + state.api_service.clone(), + state.disk_media_repo_cache.clone(), + ); + DomainUpdateResult::task(Task::batch([ + Task::perform(fetch, |res| { + LibraryMessage::LibrariesLoaded( + res.map_err(|e| format!("{:#}", e)), + ) + }) + .map(DomainMessage::Library), + Task::done(DomainMessage::Library( + LibraryMessage::FetchActiveScans, + )), + ])) } // Library form management - using actual handlers diff --git a/crates/ferrex-player-ui/src/domains/library/update_handlers/library_management.rs b/crates/ferrex-player-ui/src/domains/library/update_handlers/library_management.rs index 43fe3d9c..5b6fd9df 100644 --- a/crates/ferrex-player-ui/src/domains/library/update_handlers/library_management.rs +++ b/crates/ferrex-player-ui/src/domains/library/update_handlers/library_management.rs @@ -1,10 +1,11 @@ use crate::domains::library::messages::LibraryMessage; use crate::infra::api_types::Library; use crate::state::State; - use chrono::Utc; use ferrex_core::{ - api::types::{CreateLibraryRequest, UpdateLibraryRequest}, + api::types::{ + CreateLibraryRequest, ResetLibraryRequest, UpdateLibraryRequest, + }, types::{ids::LibraryId, library::LibraryType}, }; use ferrex_model::MovieReferenceBatchSize; @@ -28,6 +29,8 @@ fn create_library_request( "enabled": library.enabled, "auto_scan": library.auto_scan, "watch_for_changes": library.watch_for_changes, + "analyze_on_scan": library.analyze_on_scan, + "max_retry_attempts": library.max_retry_attempts, "movie_ref_batch_size": library.movie_ref_batch_size.get(), "start_scan": start_scan, })) @@ -46,6 +49,8 @@ fn update_library_request(library: &Library) -> UpdateLibraryRequest { "enabled": library.enabled, "auto_scan": library.auto_scan, "watch_for_changes": library.watch_for_changes, + "analyze_on_scan": library.analyze_on_scan, + "max_retry_attempts": library.max_retry_attempts, })) .expect("serialized library update request must match API schema") } @@ -170,6 +175,9 @@ pub fn handle_delete_library( library_id: LibraryId, _server_url: String, ) -> Task { + state.domains.ui.state.error_message = None; + state.domains.library.state.library_form_errors.clear(); + state.domains.library.state.library_form_success = None; let api = state.api_service.clone(); Task::perform( async move { @@ -188,6 +196,7 @@ pub fn handle_library_deleted( state: &mut State, result: Result, ) -> Task { + state.domains.ui.state.library_maintenance_in_flight = None; match result { Ok(library_id) => { log::info!( @@ -198,20 +207,37 @@ pub fn handle_library_deleted( state.domains.ui.state.scope = crate::domains::ui::shell_ui::Scope::Home; } - Task::perform( - super::library_loaded::fetch_libraries( - state.api_service.clone(), - state.disk_media_repo_cache.clone(), + state.domains.library.state.library_form_errors.clear(); + state.domains.ui.state.error_message = None; + state.domains.library.state.library_form_success = + Some("Library deleted successfully".to_string()); + Task::batch([ + Task::perform( + super::library_loaded::fetch_libraries( + state.api_service.clone(), + state.disk_media_repo_cache.clone(), + ), + |res| { + LibraryMessage::LibrariesLoaded( + res.map_err(|e| e.to_string()), + ) + }, ), - |res| { - LibraryMessage::LibrariesLoaded( - res.map_err(|e| e.to_string()), - ) - }, - ) + Task::done(LibraryMessage::FetchActiveScans), + ]) } Err(e) => { log::error!("Failed to delete library: {}", e); + let message = format!("Failed to delete library: {}", e); + state.domains.library.state.library_form_success = None; + state.domains.library.state.library_form_errors.clear(); + state + .domains + .library + .state + .library_form_errors + .push(message.clone()); + state.domains.ui.state.error_message = Some(message); Task::none() } } @@ -223,6 +249,7 @@ pub fn handle_show_library_form( ) -> Task { state.domains.library.state.library_form_errors.clear(); state.domains.library.state.library_form_success = None; + state.domains.ui.state.error_message = None; state.domains.library.state.library_form_data = Some(match library { Some(lib) => { // Editing existing library @@ -273,59 +300,25 @@ pub fn handle_hide_library_form(state: &mut State) -> Task { Task::none() } -/// Delete a library and recreate it with the same properties, triggering a fresh bulk scan. +/// Ask the server to atomically clear the library while preserving its identity +/// and settings, then start a fresh bulk scan. pub fn handle_reset_library( state: &mut State, library_id: LibraryId, ) -> Task { - use rkyv::{deserialize, rancor::Error}; - - // Read current library from repo (archived) - let archived = state - .domains - .library - .state - .repo_accessor - .get_archived_library_yoke(library_id.as_uuid()); - - let Ok(Some(yoke)) = archived else { - return Task::done(LibraryMessage::ResetLibraryDone(Err( - "library_not_found".to_string(), - ))); - }; - - let lib = match deserialize::( - *yoke.get(), - ) { - Ok(v) => v, - Err(_) => { - return Task::done(LibraryMessage::ResetLibraryDone(Err( - "deserialize_failed".to_string(), - ))); - } - }; - - let delete_api = state.api_service.clone(); - let create_api = state.api_service.clone(); + state.domains.library.state.library_form_errors.clear(); + state.domains.library.state.library_form_success = None; + state.domains.ui.state.error_message = None; + let api = state.api_service.clone(); Task::perform( async move { - // Delete existing library - delete_api - .delete_library(library_id) - .await - .map_err(|e| e.to_string())?; - - // Recreate with same properties and start_scan=true - let req = create_library_request(&lib, true); - let _id = create_api - .create_library(req) + api.reset_library(library_id, ResetLibraryRequest::default()) .await - .map_err(|e| e.to_string())?; - Ok::<(), String>(()) + .map_err(|e| e.to_string()) }, |result| match result { - Ok(()) => LibraryMessage::ResetLibraryDone(Ok(())), + Ok(reset) => LibraryMessage::ResetLibraryDone(Ok(reset)), Err(err) => LibraryMessage::ResetLibraryDone(Err(err)), }, ) diff --git a/crates/ferrex-player-ui/src/domains/ui/mod.rs b/crates/ferrex-player-ui/src/domains/ui/mod.rs index e8ac5923..cee99a35 100644 --- a/crates/ferrex-player-ui/src/domains/ui/mod.rs +++ b/crates/ferrex-player-ui/src/domains/ui/mod.rs @@ -71,6 +71,14 @@ use std::collections::{HashMap, HashSet}; use std::time::Instant; use uuid::Uuid; +/// Destructive library-dashboard action awaiting confirmation or completion. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LibraryMaintenanceAction { + Delete(LibraryId), + Reset(LibraryId), + ClearAllData, +} + /// UI domain state #[derive(Debug)] pub struct UIDomainState { @@ -135,8 +143,9 @@ pub struct UIDomainState { pub show_seasons_carousel: Option, pub season_episodes_carousel: Option, - // Dialog states - pub show_clear_database_confirm: bool, + // Dialog/destructive-operation states + pub library_maintenance_confirmation: Option, + pub library_maintenance_in_flight: Option, // Navigation history for back button functionality pub navigation_history: Vec, diff --git a/crates/ferrex-player-ui/src/domains/ui/settings_ui/mod.rs b/crates/ferrex-player-ui/src/domains/ui/settings_ui/mod.rs index 1660fa10..38939d68 100644 --- a/crates/ferrex-player-ui/src/domains/ui/settings_ui/mod.rs +++ b/crates/ferrex-player-ui/src/domains/ui/settings_ui/mod.rs @@ -16,7 +16,7 @@ use crate::{ }, state::SettingsSection, }, - ui::messages::UiMessage, + ui::{LibraryMaintenanceAction, messages::UiMessage}, }, infra::units::ByteSize, }; @@ -112,8 +112,8 @@ pub enum SettingsUiMessage { DemoRefreshStatus, // Database maintenance UI - ShowClearDatabaseConfirm, - HideClearDatabaseConfirm, + ShowLibraryMaintenanceConfirm(LibraryMaintenanceAction), + HideLibraryMaintenanceConfirm, ClearDatabase, DatabaseCleared(Result<(), String>), @@ -219,6 +219,21 @@ impl From for UiMessage { } } +impl LibraryMaintenanceAction { + /// Message dispatched only after the destructive confirmation is accepted. + pub fn confirmation_message(self) -> SettingsUiMessage { + match self { + Self::Delete(library_id) => { + SettingsUiMessage::DeleteLibrary(library_id) + } + Self::Reset(library_id) => { + SettingsUiMessage::ResetLibrary(library_id) + } + Self::ClearAllData => SettingsUiMessage::ClearDatabase, + } + } +} + impl SettingsUiMessage { pub fn name(&self) -> &'static str { match self { @@ -244,8 +259,12 @@ impl SettingsUiMessage { Self::DemoRefreshStatus => "UI::DemoRefreshStatus", // Database maintenance UI - Self::ShowClearDatabaseConfirm => "UI::ShowClearDatabaseConfirm", - Self::HideClearDatabaseConfirm => "UI::HideClearDatabaseConfirm", + Self::ShowLibraryMaintenanceConfirm(_) => { + "UI::ShowLibraryMaintenanceConfirm" + } + Self::HideLibraryMaintenanceConfirm => { + "UI::HideLibraryMaintenanceConfirm" + } Self::ClearDatabase => "UI::ClearDatabase", Self::DatabaseCleared(_) => "UI::DatabaseCleared", @@ -395,11 +414,11 @@ impl std::fmt::Debug for SettingsUiMessage { SettingsUiMessage::DemoRefreshStatus => { write!(f, "UI::DemoRefreshStatus") } - SettingsUiMessage::ShowClearDatabaseConfirm => { - write!(f, "UI::ShowClearDatabaseConfirm") + SettingsUiMessage::ShowLibraryMaintenanceConfirm(action) => { + write!(f, "UI::ShowLibraryMaintenanceConfirm({action:?})") } - SettingsUiMessage::HideClearDatabaseConfirm => { - write!(f, "UI::HideClearDatabaseConfirm") + SettingsUiMessage::HideLibraryMaintenanceConfirm => { + write!(f, "UI::HideLibraryMaintenanceConfirm") } SettingsUiMessage::ClearDatabase => write!(f, "UI::ClearDatabase"), SettingsUiMessage::DatabaseCleared(_) => { diff --git a/crates/ferrex-player-ui/src/domains/ui/settings_ui/update.rs b/crates/ferrex-player-ui/src/domains/ui/settings_ui/update.rs index 5bb3f092..05613490 100644 --- a/crates/ferrex-player-ui/src/domains/ui/settings_ui/update.rs +++ b/crates/ferrex-player-ui/src/domains/ui/settings_ui/update.rs @@ -16,6 +16,7 @@ use crate::{ }, }, ui::{ + LibraryMaintenanceAction, feedback_ui::{FeedbackMessage, ToastNotification}, settings_ui::{RuntimeConfigMessage, SettingsUiMessage}, }, @@ -178,18 +179,42 @@ pub fn update_settings_ui( SettingsUiMessage::DemoApplySizing => demo_controls::handle_apply_sizing(state), #[cfg(feature = "demo")] SettingsUiMessage::DemoRefreshStatus => demo_controls::handle_refresh_status(state), - SettingsUiMessage::ShowClearDatabaseConfirm => { - state.domains.ui.state.show_clear_database_confirm = true; - DomainUpdateResult::task(Task::none()) - } - SettingsUiMessage::HideClearDatabaseConfirm => { - state.domains.ui.state.show_clear_database_confirm = false; - DomainUpdateResult::task(Task::none()) + SettingsUiMessage::ShowLibraryMaintenanceConfirm(action) => { + if state + .domains + .ui + .state + .library_maintenance_in_flight + .is_none() + { + state.domains.ui.state.library_maintenance_confirmation = + Some(action); + state.domains.library.state.library_form_errors.clear(); + state.domains.library.state.library_form_success = None; + state.domains.ui.state.error_message = None; } + DomainUpdateResult::task(Task::none()) + } + SettingsUiMessage::HideLibraryMaintenanceConfirm => { + state.domains.ui.state.library_maintenance_confirmation = None; + DomainUpdateResult::task(Task::none()) + } SettingsUiMessage::ClearDatabase => { - let task = crate::common::clear_database::handle_clear_database(state); - DomainUpdateResult::task(task) - } + if state + .domains + .ui + .state + .library_maintenance_in_flight + .is_some() + { + return DomainUpdateResult::task(Task::none()); + } + state.domains.ui.state.library_maintenance_confirmation = None; + state.domains.ui.state.library_maintenance_in_flight = + Some(LibraryMaintenanceAction::ClearAllData); + let task = crate::common::clear_database::handle_clear_database(state); + DomainUpdateResult::task(task) + } SettingsUiMessage::DatabaseCleared(result) => { let task = crate::common::clear_database::handle_database_cleared(state, result); DomainUpdateResult::task(task) @@ -367,6 +392,18 @@ pub fn update_settings_ui( ))), ), SettingsUiMessage::DeleteLibrary(library_id) => { + if state + .domains + .ui + .state + .library_maintenance_in_flight + .is_some() + { + return DomainUpdateResult::task(Task::none()); + } + state.domains.ui.state.library_maintenance_confirmation = None; + state.domains.ui.state.library_maintenance_in_flight = + Some(LibraryMaintenanceAction::Delete(library_id)); DomainUpdateResult::task(Task::done(DomainMessage::Library( LibraryMessage::DeleteLibrary(library_id), ))) @@ -449,6 +486,18 @@ pub fn update_settings_ui( )), ), SettingsUiMessage::ResetLibrary(library_id) => { + if state + .domains + .ui + .state + .library_maintenance_in_flight + .is_some() + { + return DomainUpdateResult::task(Task::none()); + } + state.domains.ui.state.library_maintenance_confirmation = None; + state.domains.ui.state.library_maintenance_in_flight = + Some(LibraryMaintenanceAction::Reset(library_id)); DomainUpdateResult::task(Task::done(DomainMessage::Library( LibraryMessage::ResetLibrary(library_id), ))) diff --git a/crates/ferrex-player-ui/src/domains/ui/views/admin/mod.rs b/crates/ferrex-player-ui/src/domains/ui/views/admin/mod.rs index 8ed57af1..41f957ab 100644 --- a/crates/ferrex-player-ui/src/domains/ui/views/admin/mod.rs +++ b/crates/ferrex-player-ui/src/domains/ui/views/admin/mod.rs @@ -1,9 +1,11 @@ mod view_admin; mod view_admin_users; mod view_library_form; +mod view_library_maintenance_confirmation; mod view_library_management; pub use view_admin::view_admin_dashboard; pub use view_admin_users::view_admin_users; pub use view_library_form::view_library_form; +pub use view_library_maintenance_confirmation::view_library_maintenance_confirmation; pub use view_library_management::view_library_management; diff --git a/crates/ferrex-player-ui/src/domains/ui/views/admin/view_admin.rs b/crates/ferrex-player-ui/src/domains/ui/views/admin/view_admin.rs index 79f94364..4db688e8 100644 --- a/crates/ferrex-player-ui/src/domains/ui/views/admin/view_admin.rs +++ b/crates/ferrex-player-ui/src/domains/ui/views/admin/view_admin.rs @@ -7,7 +7,10 @@ use crate::{ common::ui_utils::icon_text, domains::{ auth::permissions::StatePermissionExt, - ui::{messages::UiMessage, settings_ui::SettingsUiMessage, theme}, + ui::{ + LibraryMaintenanceAction, messages::UiMessage, + settings_ui::SettingsUiMessage, theme, + }, }, state::State, }; @@ -27,6 +30,18 @@ use lucide_icons::Icon; )] pub fn view_admin_dashboard(state: &State) -> Element<'_, UiMessage> { let permissions = state.permission_checker(); + let maintenance_pending = state + .domains + .ui + .state + .library_maintenance_confirmation + .is_some() + || state + .domains + .ui + .state + .library_maintenance_in_flight + .is_some(); // Check if user has any admin permissions if !permissions.can_view_admin_dashboard() { @@ -77,6 +92,24 @@ pub fn view_admin_dashboard(state: &State) -> Element<'_, UiMessage> { .align_y(iced::Alignment::Center), ); + if let Some(error) = &state.domains.ui.state.error_message { + content = content.push( + container( + row![ + icon_text(Icon::OctagonAlert), + text(error) + .size(15) + .color(theme::MediaServerTheme::ERROR_COLOR), + ] + .spacing(12) + .align_y(iced::Alignment::Center), + ) + .padding([12, 16]) + .width(Length::Fill) + .style(theme::Container::ErrorBox.style()), + ); + } + // Build sections based on permissions let mut sections_row_1 = row![].spacing(20).align_y(iced::Alignment::Start); @@ -87,6 +120,16 @@ pub fn view_admin_dashboard(state: &State) -> Element<'_, UiMessage> { // Library Management section (only if user can manage libraries) if permissions.can_view_admin_dashboard() { + let manage_libraries = button("Manage Libraries") + .style(theme::Button::Primary.style()) + .padding([12, 20]) + .width(Length::Fill); + let manage_libraries = if maintenance_pending { + manage_libraries + } else { + manage_libraries + .on_press(SettingsUiMessage::ShowLibraryManagement.into()) + }; sections_row_1 = sections_row_1.push( container( column![ @@ -105,11 +148,7 @@ pub fn view_admin_dashboard(state: &State) -> Element<'_, UiMessage> { ] .align_y(iced::Alignment::Center), Space::new().height(20), - button("Manage Libraries") - .on_press(SettingsUiMessage::ShowLibraryManagement.into()) - .style(theme::Button::Primary.style()) - .padding([12, 20]) - .width(Length::Fill), + manage_libraries, ] .spacing(15) .padding(20), @@ -122,6 +161,15 @@ pub fn view_admin_dashboard(state: &State) -> Element<'_, UiMessage> { // User Management section (only if user can manage users) if permissions.can_view_users() { + let manage_users = button("Manage Users") + .style(theme::Button::Primary.style()) + .padding([12, 20]) + .width(Length::Fill); + let manage_users = if maintenance_pending { + manage_users + } else { + manage_users.on_press(SettingsUiMessage::ShowUserManagement.into()) + }; if has_row_1_content { sections_row_1 = sections_row_1.push(Space::new().width(20)); } @@ -143,11 +191,7 @@ pub fn view_admin_dashboard(state: &State) -> Element<'_, UiMessage> { ] .align_y(iced::Alignment::Center), Space::new().height(20), - button("Manage Users") - .on_press(SettingsUiMessage::ShowUserManagement.into()) - .style(theme::Button::Primary.style()) - .padding([12, 20]) - .width(Length::Fill), + manage_users, ] .spacing(15) .padding(20), @@ -195,6 +239,28 @@ pub fn view_admin_dashboard(state: &State) -> Element<'_, UiMessage> { // Dev Tools section (only if user can reset database or is admin) if permissions.can_reset_database() { + let label = if state.domains.ui.state.library_maintenance_in_flight + == Some(LibraryMaintenanceAction::ClearAllData) + { + "Clearing…" + } else { + "Clear All Data" + }; + let clear_button = button(label) + .style(theme::Button::Danger.style()) + .padding([12, 20]) + .width(Length::Fill); + let clear_button = if maintenance_pending { + clear_button + } else { + clear_button.on_press( + SettingsUiMessage::ShowLibraryMaintenanceConfirm( + LibraryMaintenanceAction::ClearAllData, + ) + .into(), + ) + }; + if has_row_2_content { sections_row_2 = sections_row_2.push(Space::new().width(20)); } @@ -216,13 +282,7 @@ pub fn view_admin_dashboard(state: &State) -> Element<'_, UiMessage> { ] .align_y(iced::Alignment::Center), Space::new().height(20), - button("Dev Tools") - .on_press( - SettingsUiMessage::ShowClearDatabaseConfirm.into() - ) - .style(theme::Button::Danger.style()) - .padding([12, 20]) - .width(Length::Fill), + clear_button, ] .spacing(15) .padding(20), diff --git a/crates/ferrex-player-ui/src/domains/ui/views/admin/view_library_maintenance_confirmation.rs b/crates/ferrex-player-ui/src/domains/ui/views/admin/view_library_maintenance_confirmation.rs new file mode 100644 index 00000000..3f0ec108 --- /dev/null +++ b/crates/ferrex-player-ui/src/domains/ui/views/admin/view_library_maintenance_confirmation.rs @@ -0,0 +1,153 @@ +use crate::{ + domains::ui::{ + LibraryMaintenanceAction, messages::UiMessage, + settings_ui::SettingsUiMessage, theme, + }, + state::State, +}; +use iced::{ + Element, Length, + widget::{Space, button, column, container, mouse_area, row, text}, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ConfirmationCopy { + title: String, + body: String, + confirm_label: &'static str, +} + +fn confirmation_copy( + action: LibraryMaintenanceAction, + library_name: Option<&str>, +) -> ConfirmationCopy { + let target = library_name.unwrap_or("this library"); + match action { + LibraryMaintenanceAction::Delete(_) => ConfirmationCopy { + title: format!("Delete {target}?"), + body: format!( + "This permanently deletes {target} and all indexed data owned by it. Media files on disk are not removed." + ), + confirm_label: "Delete Library", + }, + LibraryMaintenanceAction::Reset(_) => ConfirmationCopy { + title: format!("Reset {target}?"), + body: format!( + "This atomically clears {target}'s indexed data, preserves the library identity and settings, and starts a fresh scan." + ), + confirm_label: "Reset Library", + }, + LibraryMaintenanceAction::ClearAllData => ConfirmationCopy { + title: "Clear all server data?".to_string(), + body: "This permanently deletes every library and its indexed media, all users, and all sessions. You will be signed out and must run setup again. Media files on disk are not removed." + .to_string(), + confirm_label: "Clear All Data", + }, + } +} + +fn library_name( + state: &State, + action: LibraryMaintenanceAction, +) -> Option { + let library_id = match action { + LibraryMaintenanceAction::Delete(id) + | LibraryMaintenanceAction::Reset(id) => id, + LibraryMaintenanceAction::ClearAllData => return None, + }; + + state + .domains + .library + .state + .repo_accessor + .get_archived_library_yoke(library_id.as_uuid()) + .ok() + .flatten() + .map(|library| library.get().name.to_string()) +} + +pub fn view_library_maintenance_confirmation( + state: &State, +) -> Option> { + let action = state.domains.ui.state.library_maintenance_confirmation?; + let name = library_name(state, action); + let copy = confirmation_copy(action, name.as_deref()); + + let dialog = container( + column![ + text(copy.title) + .size(24) + .color(theme::MediaServerTheme::TEXT_PRIMARY), + text(copy.body) + .size(15) + .color(theme::MediaServerTheme::TEXT_SECONDARY), + Space::new().height(8), + row![ + Space::new().width(Length::Fill), + button("Cancel") + .on_press( + SettingsUiMessage::HideLibraryMaintenanceConfirm.into() + ) + .style(theme::Button::Secondary.style()), + button(copy.confirm_label) + .on_press(action.confirmation_message().into()) + .style(theme::Button::Destructive.style()), + ] + .spacing(12) + .align_y(iced::Alignment::Center), + ] + .spacing(16), + ) + .padding(28) + .width(Length::Fixed(560.0)) + .style(theme::Container::Modal.style()); + + let overlay = container(dialog) + .width(Length::Fill) + .height(Length::Fill) + .center_x(Length::Fill) + .center_y(Length::Fill) + .style(theme::Container::ModalOverlay.style()); + + Some(mouse_area(overlay).on_press(UiMessage::NoOp).into()) +} + +#[cfg(test)] +mod tests { + use super::*; + use ferrex_core::player_prelude::LibraryId; + use uuid::Uuid; + + #[test] + fn confirmations_name_the_target_and_dispatch_the_real_action() { + let library_id = LibraryId(Uuid::from_u128(42)); + + let delete = confirmation_copy( + LibraryMaintenanceAction::Delete(library_id), + Some("Family Movies"), + ); + assert!(delete.title.contains("Family Movies")); + assert!(matches!( + LibraryMaintenanceAction::Delete(library_id) + .confirmation_message(), + SettingsUiMessage::DeleteLibrary(id) if id == library_id + )); + + let reset = confirmation_copy( + LibraryMaintenanceAction::Reset(library_id), + Some("Family Movies"), + ); + assert!(reset.body.contains("Family Movies")); + assert!(matches!( + LibraryMaintenanceAction::Reset(library_id) + .confirmation_message(), + SettingsUiMessage::ResetLibrary(id) if id == library_id + )); + + assert!(matches!( + LibraryMaintenanceAction::ClearAllData.confirmation_message(), + SettingsUiMessage::ClearDatabase + )); + } +} diff --git a/crates/ferrex-player-ui/src/domains/ui/views/admin/view_library_management.rs b/crates/ferrex-player-ui/src/domains/ui/views/admin/view_library_management.rs index 771da544..3a5a0bfa 100644 --- a/crates/ferrex-player-ui/src/domains/ui/views/admin/view_library_management.rs +++ b/crates/ferrex-player-ui/src/domains/ui/views/admin/view_library_management.rs @@ -5,7 +5,8 @@ use crate::{ domains::{ auth::permissions::{self, StatePermissionExt}, ui::{ - messages::UiMessage, settings_ui::SettingsUiMessage, theme, + LibraryMaintenanceAction, messages::UiMessage, + settings_ui::SettingsUiMessage, theme, views::admin::view_library_form, }, }, @@ -36,6 +37,18 @@ use uuid::Uuid; )] pub fn view_library_management(state: &State) -> Element<'_, UiMessage> { let permissions = state.permission_checker(); + let destructive_action_pending = state + .domains + .ui + .state + .library_maintenance_confirmation + .is_some() + || state + .domains + .ui + .state + .library_maintenance_in_flight + .is_some(); // If form is open, show the form instead if let Some(form_data) = &state.domains.library.state.library_form_data { @@ -50,21 +63,40 @@ pub fn view_library_management(state: &State) -> Element<'_, UiMessage> { // Add Create Library button only if user has permission if permissions.has_permission("libraries:create") { - header_row = header_row.push( - button("Create Library") + let create_button = + button("Create Library").style(theme::Button::Primary.style()); + let create_button = if destructive_action_pending { + create_button + } else { + create_button .on_press(SettingsUiMessage::ShowLibraryForm(None).into()) - .style(theme::Button::Primary.style()), - ); + }; + header_row = header_row.push(create_button); header_row = header_row.push(Space::new().width(10)); } // Add Clear All Data button only if user can reset database if permissions.can_reset_database() { - header_row = header_row.push( - button("🗑 Clear All Data") - .on_press(SettingsUiMessage::ShowClearDatabaseConfirm.into()) - .style(theme::Button::Destructive.style()), - ); + let label = if state.domains.ui.state.library_maintenance_in_flight + == Some(LibraryMaintenanceAction::ClearAllData) + { + "Clearing…" + } else { + "🗑 Clear All Data" + }; + let clear_button = + button(label).style(theme::Button::Destructive.style()); + let clear_button = if destructive_action_pending { + clear_button + } else { + clear_button.on_press( + SettingsUiMessage::ShowLibraryMaintenanceConfirm( + LibraryMaintenanceAction::ClearAllData, + ) + .into(), + ) + }; + header_row = header_row.push(clear_button); } content = content.push(header_row); @@ -88,6 +120,27 @@ pub fn view_library_management(state: &State) -> Element<'_, UiMessage> { content = content.push(success_card); } + if !state.domains.library.state.library_form_errors.is_empty() { + let errors = state + .domains + .library + .state + .library_form_errors + .iter() + .map(|message| { + text(message) + .size(14) + .color(theme::MediaServerTheme::ERROR_COLOR) + .into() + }) + .collect::>>(); + content = content.push( + container(column(errors).spacing(6)) + .padding([12, 16]) + .style(theme::Container::ErrorBox.style()), + ); + } + content = content.push(scan_status_panel(state)); #[cfg(feature = "demo")] @@ -159,6 +212,18 @@ fn create_library_card<'a>( if let Some(library_yoke) = library_opt { let library = *library_yoke.get(); + let action_pending = state + .domains + .ui + .state + .library_maintenance_confirmation + .is_some() + || state + .domains + .ui + .state + .library_maintenance_in_flight + .is_some(); let library_type_icon = match library.library_type { ArchivedLibraryType::Movies => "🎬", @@ -205,6 +270,7 @@ fn create_library_card<'a>( .style(theme::Button::Secondary.style()); let scan_button = if active_manual_scan.is_some() || scan_start_pending + || action_pending { scan_button } else { @@ -213,46 +279,78 @@ fn create_library_card<'a>( ) }; action_buttons = action_buttons.push(scan_button); - // Reset: delete and recreate library with start_scan=true - action_buttons = action_buttons.push( - button("Reset Library") - .on_press( - SettingsUiMessage::ResetLibrary(LibraryId( - library.id.as_uuid(), - )) - .into(), + } + + // Reset crosses the same destructive and creation authority boundary as + // its atomic server-side delete/reinsert operation. + if permissions.can_scan_libraries() + && permissions.has_permission("libraries:delete") + && permissions.has_permission("libraries:create") + { + let library_id = LibraryId(library.id.as_uuid()); + let label = if state.domains.ui.state.library_maintenance_in_flight + == Some(LibraryMaintenanceAction::Reset(library_id)) + { + "Resetting…" + } else { + "Reset Library" + }; + let reset_button = + button(label).style(theme::Button::Secondary.style()); + let reset_button = if action_pending { + reset_button + } else { + reset_button.on_press( + SettingsUiMessage::ShowLibraryMaintenanceConfirm( + LibraryMaintenanceAction::Reset(library_id), ) - .style(theme::Button::Secondary.style()), - ); + .into(), + ) + }; + action_buttons = action_buttons.push(reset_button); } // Edit button (only if user has update permission) if permissions.has_permission("libraries:update") { - action_buttons = action_buttons.push( - button("Edit") - .on_press( - SettingsUiMessage::ShowLibraryForm(Some( - deserialize::(library) - .expect("Failed to deserialize library"), - )) - .into(), - ) - .style(theme::Button::Secondary.style()), - ); + let edit_button = + button("Edit").style(theme::Button::Secondary.style()); + let edit_button = if action_pending { + edit_button + } else { + edit_button.on_press( + SettingsUiMessage::ShowLibraryForm(Some( + deserialize::(library) + .expect("Failed to deserialize library"), + )) + .into(), + ) + }; + action_buttons = action_buttons.push(edit_button); } // Delete button (only if user has delete permission) if permissions.has_permission("libraries:delete") { - action_buttons = action_buttons.push( - button("Delete") - .on_press( - SettingsUiMessage::DeleteLibrary(LibraryId( - library.id.as_uuid(), - )) - .into(), + let library_id = LibraryId(library.id.as_uuid()); + let label = if state.domains.ui.state.library_maintenance_in_flight + == Some(LibraryMaintenanceAction::Delete(library_id)) + { + "Deleting…" + } else { + "Delete" + }; + let delete_button = + button(label).style(theme::Button::Destructive.style()); + let delete_button = if action_pending { + delete_button + } else { + delete_button.on_press( + SettingsUiMessage::ShowLibraryMaintenanceConfirm( + LibraryMaintenanceAction::Delete(library_id), ) - .style(theme::Button::Destructive.style()), - ); + .into(), + ) + }; + action_buttons = action_buttons.push(delete_button); } container( @@ -482,6 +580,10 @@ fn scan_status_panel(state: &State) -> Element<'_, UiMessage> { let status_label = scan_status_label( &snapshot, reason_details, + completed_items, + total_items, + validated_items, + known_unchanged_items, skipped_items, needs_attention_items, retrying_items, @@ -660,6 +762,10 @@ fn scan_status_panel(state: &State) -> Element<'_, UiMessage> { if let Some(copy) = scan_recovery_copy( &snapshot, reason_details, + completed_items, + total_items, + validated_items, + known_unchanged_items, skipped_items, needs_attention_items, ) { @@ -735,6 +841,10 @@ fn scan_panel_count_label(scans: &[ScanSnapshotDto]) -> String { fn scan_status_label( snapshot: &ScanSnapshotDto, reason_details: &[ScanPathReasonDetail], + completed_items: u64, + total_items: u64, + validated_items: u64, + known_unchanged_items: u64, skipped_items: u64, needs_attention_items: u64, retrying_items: u64, @@ -744,11 +854,26 @@ fn scan_status_label( { return ("Needs attention", theme::MediaServerTheme::ERROR); } - if scan_has_no_media_found(reason_details) { + if matches!(snapshot.status, ScanLifecycleStatus::Completed) + && scan_is_whole_library_no_media( + reason_details, + completed_items, + total_items, + validated_items, + known_unchanged_items, + skipped_items, + ) + { return ("No media found", theme::MediaServerTheme::WARNING); } if matches!(snapshot.status, ScanLifecycleStatus::Failed) - && skipped_items > 0 + && scan_is_whole_library_skipped( + completed_items, + total_items, + validated_items, + known_unchanged_items, + skipped_items, + ) { return ("Skipped", theme::MediaServerTheme::WARNING); } @@ -797,6 +922,10 @@ fn scan_has_rescan_recovery( fn scan_recovery_copy( snapshot: &ScanSnapshotDto, reason_details: &[ScanPathReasonDetail], + completed_items: u64, + total_items: u64, + validated_items: u64, + known_unchanged_items: u64, skipped_items: u64, needs_attention_items: u64, ) -> Option<&'static str> { @@ -813,7 +942,22 @@ fn scan_recovery_copy( Some( "Review the paths below, then rescan this library. Per-path retry is not available yet.", ) - } else if scan_has_no_media_found(reason_details) { + } else if matches!(snapshot.status, ScanLifecycleStatus::Failed) + && completed_items != total_items + { + Some( + "The scan stopped before all paths finished. Rescan this library to try again.", + ) + } else if matches!(snapshot.status, ScanLifecycleStatus::Completed) + && scan_is_whole_library_no_media( + reason_details, + completed_items, + total_items, + validated_items, + known_unchanged_items, + skipped_items, + ) + { Some( "Add supported media or update the folder, then rescan this library.", ) @@ -933,6 +1077,41 @@ fn scan_has_no_media_found(reason_details: &[ScanPathReasonDetail]) -> bool { .any(|detail| detail.reason_code == "no_supported_media_found") } +fn scan_is_whole_library_no_media( + reason_details: &[ScanPathReasonDetail], + completed_items: u64, + total_items: u64, + validated_items: u64, + known_unchanged_items: u64, + skipped_items: u64, +) -> bool { + scan_is_whole_library_skipped( + completed_items, + total_items, + validated_items, + known_unchanged_items, + skipped_items, + ) && !reason_details.is_empty() + && reason_details.iter().all(|detail| { + detail.category == ScanPathReasonCategory::Skipped + && detail.reason_code == "no_supported_media_found" + }) +} + +fn scan_is_whole_library_skipped( + completed_items: u64, + total_items: u64, + validated_items: u64, + known_unchanged_items: u64, + skipped_items: u64, +) -> bool { + total_items > 0 + && completed_items == total_items + && skipped_items == total_items + && validated_items == 0 + && known_unchanged_items == 0 +} + fn truncate_path(path: &str) -> String { const MAX_LEN: usize = 48; if path.len() <= MAX_LEN { @@ -1056,6 +1235,8 @@ mod tests { #[test] fn no_media_found_reason_gets_label_and_rescan_copy() { let mut scan = snapshot(ScanLifecycleStatus::Completed); + scan.completed_items = 1; + scan.total_items = 1; scan.skipped_items = 1; let details = vec![reason_detail( ScanPathReasonCategory::Skipped, @@ -1065,18 +1246,155 @@ mod tests { assert_eq!(reason_detail_label(&details[0]), "No media found"); assert_eq!( - scan_status_label(&scan, &details, 1, 0, 0).0, + scan_status_label(&scan, &details, 1, 1, 0, 0, 1, 0, 0).0, "No media found" ); assert!(scan_has_rescan_recovery(&scan, &details, 1, 0)); assert_eq!( - scan_recovery_copy(&scan, &details, 1, 0), + scan_recovery_copy(&scan, &details, 1, 1, 0, 0, 1, 0), Some( "Add supported media or update the folder, then rescan this library." ) ); } + #[test] + fn mixed_success_no_media_reason_preserves_terminal_status() { + let details = vec![reason_detail( + ScanPathReasonCategory::Skipped, + "no_supported_media_found", + None, + )]; + + for (validated_items, known_unchanged_items) in [(1, 0), (0, 1)] { + for (status, expected_label) in [ + (ScanLifecycleStatus::Completed, "Completed"), + (ScanLifecycleStatus::Failed, "Failed"), + ] { + let mut scan = snapshot(status); + scan.completed_items = + validated_items + known_unchanged_items + 1; + scan.total_items = scan.completed_items; + scan.validated_items = validated_items; + scan.known_unchanged_items = known_unchanged_items; + scan.skipped_items = 1; + + assert_eq!( + scan_status_label( + &scan, + &details, + scan.completed_items, + scan.total_items, + validated_items, + known_unchanged_items, + 1, + 0, + 0, + ) + .0, + expected_label + ); + assert!(scan_has_rescan_recovery(&scan, &details, 1, 0)); + assert_eq!( + scan_recovery_copy( + &scan, + &details, + scan.completed_items, + scan.total_items, + validated_items, + known_unchanged_items, + 1, + 0, + ), + Some( + "Skipped paths can be checked again with a library rescan." + ) + ); + } + } + } + + #[test] + fn incomplete_failed_scan_with_empty_path_preserves_failure() { + let mut scan = snapshot(ScanLifecycleStatus::Failed); + scan.completed_items = 1; + scan.total_items = 3_131; + scan.skipped_items = 1; + let details = vec![reason_detail( + ScanPathReasonCategory::Skipped, + "no_supported_media_found", + None, + )]; + + assert_eq!( + scan_status_label(&scan, &details, 1, 3_131, 0, 0, 1, 0, 0).0, + "Failed" + ); + assert_eq!( + scan_recovery_copy(&scan, &details, 1, 3_131, 0, 0, 1, 0), + Some( + "The scan stopped before all paths finished. Rescan this library to try again." + ) + ); + } + + #[test] + fn mixed_skip_reasons_are_not_whole_library_no_media() { + let mut scan = snapshot(ScanLifecycleStatus::Completed); + scan.completed_items = 2; + scan.total_items = 2; + scan.skipped_items = 2; + let details = vec![ + reason_detail( + ScanPathReasonCategory::Skipped, + "no_supported_media_found", + None, + ), + reason_detail( + ScanPathReasonCategory::Skipped, + "path_missing", + None, + ), + ]; + + assert_eq!( + scan_status_label(&scan, &details, 2, 2, 0, 0, 2, 0, 0).0, + "Completed" + ); + assert_eq!( + scan_recovery_copy(&scan, &details, 2, 2, 0, 0, 2, 0), + Some("Skipped paths can be checked again with a library rescan.") + ); + } + + #[test] + fn active_scan_with_early_empty_folder_preserves_active_status() { + let details = vec![reason_detail( + ScanPathReasonCategory::Skipped, + "no_supported_media_found", + None, + )]; + + for (status, expected_label) in [ + (ScanLifecycleStatus::Pending, "Pending"), + (ScanLifecycleStatus::Running, "Running"), + ] { + let mut scan = snapshot(status); + scan.completed_items = 1; + scan.total_items = 3_131; + scan.skipped_items = 1; + + assert_eq!( + scan_status_label(&scan, &details, 1, 3_131, 0, 0, 1, 0, 0,).0, + expected_label + ); + assert_eq!( + scan_recovery_copy(&scan, &details, 1, 3_131, 0, 0, 1, 0,), + None + ); + } + } + #[test] fn panel_count_reports_attention_instead_of_running_only() { let mut running = snapshot(ScanLifecycleStatus::Running); @@ -1091,7 +1409,18 @@ mod tests { running.retrying_items = 1; assert_eq!( - scan_status_label(&running, &[], 0, 0, running.retrying_items).0, + scan_status_label( + &running, + &[], + running.completed_items, + running.total_items, + running.validated_items, + running.known_unchanged_items, + running.skipped_items, + running.needs_attention_items, + running.retrying_items, + ) + .0, "Retrying" ); } diff --git a/crates/ferrex-player-ui/src/state.rs b/crates/ferrex-player-ui/src/state.rs index 3d9f1389..b295e00c 100644 --- a/crates/ferrex-player-ui/src/state.rs +++ b/crates/ferrex-player-ui/src/state.rs @@ -266,7 +266,8 @@ impl State { selected_watch_status: UiWatchStatus::Any, show_seasons_carousel: None, season_episodes_carousel: None, - show_clear_database_confirm: false, + library_maintenance_confirmation: None, + library_maintenance_in_flight: None, navigation_history: Vec::new(), poster_anim_active_until: None, motion_controller: MotionController::new(), diff --git a/crates/ferrex-player-ui/src/view.rs b/crates/ferrex-player-ui/src/view.rs index 7beab61d..95f10788 100644 --- a/crates/ferrex-player-ui/src/view.rs +++ b/crates/ferrex-player-ui/src/view.rs @@ -5,7 +5,8 @@ use crate::domains::ui::interaction_ui::InteractionMessage; use crate::domains::ui::theme; use crate::domains::ui::types::ViewState; use crate::domains::ui::views::admin::{ - view_admin_dashboard, view_admin_users, view_library_management, + view_admin_dashboard, view_admin_users, + view_library_maintenance_confirmation, view_library_management, }; use crate::domains::ui::views::auth::view_auth; use crate::domains::ui::views::collections::view_collection_detail; @@ -387,18 +388,30 @@ pub fn view( layered }; + let with_maintenance_confirmation = + if let Some(overlay) = view_library_maintenance_confirmation(state) { + Stack::new() + .push(with_search_overlay) + .push(overlay.map(DomainMessage::from)) + .width(Length::Fill) + .height(Length::Fill) + .into() + } else { + with_search_overlay + }; + // Overlay toast notifications if any are active if state.domains.ui.state.toast_manager.has_toasts() { let toast_overlay = crate::domains::ui::views::toast_overlay::view_toast_overlay(state); Stack::new() - .push(with_search_overlay) + .push(with_maintenance_confirmation) .push(toast_overlay.map(DomainMessage::from)) .width(Length::Fill) .height(Length::Fill) .into() } else { - with_search_overlay + with_maintenance_confirmation } } From 82d64a87272aa6afe6b43d5adcae3fe71edd30a6 Mon Sep 17 00:00:00 2001 From: Grayson Hieb Date: Sun, 9 Aug 2026 13:57:28 -0600 Subject: [PATCH 04/10] fix(scan): preserve incremental catalog projections --- .../src/domain/scan/actors/index.rs | 15 +- .../domain/scan/orchestration/dispatcher.rs | 4 +- .../src/domain/scan/orchestration/job.rs | 41 ++ .../domain/scan/orchestration/persistence.rs | 8 + .../src/domain/scan/orchestration/queue.rs | 9 +- .../scan/orchestration/runtime/event_bus.rs | 71 ++- .../tests/integration/orchestration.rs | 2 + .../tests/support/scanner_runtime.rs | 1 + .../src/infra/orchestration/maintenance.rs | 2 + .../src/infra/orchestration/mod.rs | 5 +- .../infra/scan/catalog_event_projection.rs | 189 ++++++-- .../src/infra/scan/movie_batch_notifier.rs | 187 ++++++-- .../src/infra/scan/scan_manager.rs | 403 ++++++++++++++++-- 13 files changed, 819 insertions(+), 118 deletions(-) diff --git a/crates/ferrex-core/src/domain/scan/actors/index.rs b/crates/ferrex-core/src/domain/scan/actors/index.rs index 44b3a32f..6aac9ed6 100644 --- a/crates/ferrex-core/src/domain/scan/actors/index.rs +++ b/crates/ferrex-core/src/domain/scan/actors/index.rs @@ -16,8 +16,11 @@ pub struct IndexCommand { pub ready: MediaReadyForIndex, } -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[derive( + Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq, +)] pub enum IndexingChange { + #[default] Created, Updated, } @@ -32,16 +35,10 @@ pub struct IndexingOutcome { pub indexed_at: DateTime, pub upserted: bool, pub media: Option, - #[serde(default = "IndexingOutcome::default_change")] + #[serde(default)] pub change: IndexingChange, } -impl IndexingOutcome { - fn default_change() -> IndexingChange { - IndexingChange::Created - } -} - #[async_trait] pub trait IndexerActor: Send + Sync { async fn index( @@ -88,7 +85,7 @@ impl IndexerActor for DefaultIndexerActor { indexed_at: Utc::now(), upserted: true, media: Some(media), - change: IndexingOutcome::default_change(), + change: job.change, }) } } diff --git a/crates/ferrex-core/src/domain/scan/orchestration/dispatcher.rs b/crates/ferrex-core/src/domain/scan/orchestration/dispatcher.rs index 67767734..1ede4c82 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/dispatcher.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/dispatcher.rs @@ -12,7 +12,7 @@ use crate::database::repository_ports::{ }; use crate::domain::scan::actors::image_fetch::ImageFetchActor; use crate::domain::scan::actors::index::{ - IndexCommand, IndexerActor, IndexingOutcome, + IndexCommand, IndexerActor, IndexingChange, IndexingOutcome, }; use crate::domain::scan::actors::metadata::{ MediaReadyForIndex, MetadataActor, MetadataCommand, @@ -584,6 +584,7 @@ impl FollowUpPlanner { "index:{}:{}", source_library_id, ready.analyzed.path_norm ), + change: IndexingChange::Created, }; // Bias index upserts to complete the item flow promptly. @@ -3470,6 +3471,7 @@ mod tests { node: ScanNodeKind::MovieFolder, path_norm: media_path.to_string_lossy().to_string(), idempotency_key: "timed-text-runtime-test".to_string(), + change: IndexingChange::Created, }; let parent_correlation = Uuid::from_u128(0x6290000000000000000000000000c001); diff --git a/crates/ferrex-core/src/domain/scan/orchestration/job.rs b/crates/ferrex-core/src/domain/scan/orchestration/job.rs index 81887f34..faea2ed5 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/job.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/job.rs @@ -2,6 +2,7 @@ use ferrex_model::{ImageSize, MediaID, VideoMediaType}; use tracing::error; use crate::{ + domain::scan::actors::index::IndexingChange, error::{MediaError, Result}, types::LibraryId, }; @@ -538,6 +539,11 @@ pub struct IndexUpsertJob { pub node: ScanNodeKind, pub path_norm: String, pub idempotency_key: String, + /// Catalog mutation semantics persisted with the durable job so a lost + /// live indexing outcome can be projected without changing Added into + /// Updated (or vice versa). + #[serde(default)] + pub change: IndexingChange, } /// Source that requested transcript extraction for a playable media file. @@ -737,6 +743,41 @@ mod tests { use super::*; use crate::domain::scan::orchestration::context::MovieRootPath; + #[test] + fn index_upsert_payload_persists_change_and_defaults_legacy_rows() { + let library_id = LibraryId::new(); + let movie_id = ferrex_model::MovieID::new(); + let job = IndexUpsertJob { + library_id, + media_id: MediaID::Movie(movie_id), + variant: VideoMediaType::Movie, + hierarchy: AnalyzeScanHierarchy::Movie(MovieScanHierarchy { + movie_root_path: MovieRootPath::try_new("/library/Movie") + .expect("valid movie root"), + movie_id: Some(movie_id), + extra_tag: None, + }), + node: ScanNodeKind::MovieFolder, + path_norm: "/library/Movie/Movie.mkv".into(), + idempotency_key: "index:movie".into(), + change: IndexingChange::Updated, + }; + + let encoded = serde_json::to_value(&job).expect("serialize index job"); + let round_trip: IndexUpsertJob = + serde_json::from_value(encoded.clone()).expect("decode index job"); + assert_eq!(round_trip.change, IndexingChange::Updated); + + let mut legacy = encoded; + legacy + .as_object_mut() + .expect("index job serializes as an object") + .remove("change"); + let legacy: IndexUpsertJob = + serde_json::from_value(legacy).expect("decode legacy index job"); + assert_eq!(legacy.change, IndexingChange::Created); + } + #[test] fn transcript_extract_dedupe_key_is_media_file_scoped() { let library_id = diff --git a/crates/ferrex-core/src/domain/scan/orchestration/persistence.rs b/crates/ferrex-core/src/domain/scan/orchestration/persistence.rs index c61cbe6d..076322ab 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/persistence.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/persistence.rs @@ -2248,10 +2248,18 @@ impl QueueService for PostgresQueueService { )) })?; let kind = JobKind::from_i16(row.kind)?; + let (media_id, indexing_change) = match &payload { + JobPayload::IndexUpsert(job) => { + (Some(job.media_id), Some(job.change)) + } + _ => (None, None), + }; states.push(DurableJobState { job_id: JobId(row.id), kind, + media_id, + indexing_change, state: Self::parse_state(&row.state)?, attempts: row.attempts.max(0) as u16, dedupe_key: row.dedupe_key, diff --git a/crates/ferrex-core/src/domain/scan/orchestration/queue.rs b/crates/ferrex-core/src/domain/scan/orchestration/queue.rs index a4950fdf..ab98017d 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/queue.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/queue.rs @@ -12,7 +12,8 @@ use super::{ job::{EnqueueRequest, JobHandle, JobId, JobKind, JobPriority, JobState}, lease::{DequeueRequest, JobLease, LeaseRenewal}, }; -use crate::types::ids::LibraryId; +use crate::domain::scan::actors::index::IndexingChange; +use crate::types::{LibraryId, MediaID}; /// Aggregated schedulable state grouped by queue dimensions. /// @@ -39,6 +40,12 @@ pub struct ReadyQueueCount { pub struct DurableJobState { pub job_id: JobId, pub kind: JobKind, + /// Stable catalog identity carried by durable media jobs. This lets + /// observers reconstruct a lost live projection without parsing dedupe + /// strings or relying on the bounded broadcast event. + pub media_id: Option, + /// Exact catalog mutation semantics captured in an index-upsert payload. + pub indexing_change: Option, pub state: JobState, pub attempts: u16, pub dedupe_key: String, diff --git a/crates/ferrex-core/src/domain/scan/orchestration/runtime/event_bus.rs b/crates/ferrex-core/src/domain/scan/orchestration/runtime/event_bus.rs index dbede859..4941172b 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/runtime/event_bus.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/runtime/event_bus.rs @@ -30,13 +30,21 @@ impl fmt::Debug for InProcJobEventBus { impl InProcJobEventBus { pub fn new(capacity: usize) -> Self { - let (sender, _) = broadcast::channel(capacity); - let (scan_sender, _) = broadcast::channel(capacity); + Self::with_capacities(capacity, capacity) + } + + /// Build the job lifecycle and scan-domain streams with independent + /// capacities. Job lifecycle notifications are recoverable from the + /// durable queue, while scan-domain events include live catalog + /// projections and therefore need enough headroom for normal scan bursts. + pub fn with_capacities(job_capacity: usize, scan_capacity: usize) -> Self { + let (sender, _) = broadcast::channel(job_capacity); + let (scan_sender, _) = broadcast::channel(scan_capacity); Self { sender, scan_sender, - job_channel_capacity: capacity, - scan_channel_capacity: capacity, + job_channel_capacity: job_capacity, + scan_channel_capacity: scan_capacity, } } @@ -126,4 +134,59 @@ mod tests { other => panic!("expected lag after 300 events, got {other:?}"), } } + + #[tokio::test] + async fn scan_stream_capacity_is_independent_from_job_reconciliation_stream() + { + let bus = InProcJobEventBus::with_capacities(8, 512); + let mut job_receiver = bus.subscribe(); + let mut scan_receiver = bus.subscribe_scan(); + let library_id = LibraryId::new(); + + for sequence in 0..300 { + bus.publish(JobEvent { + meta: EventMeta::new( + None, + library_id, + format!("burst:{sequence}"), + None, + ), + payload: JobEventPayload::ThroughputTick { + queue_depths: Vec::new(), + sampled_at: Utc::now(), + }, + }) + .await + .expect("job event publish succeeds"); + + bus.publish_scan_event(ScanEvent::SeedCompleted( + crate::domain::scan::orchestration::events::ScanSeedSummary { + library_id, + correlation_id: None, + mode: crate::domain::scan::orchestration::events::ScanSeedMode::Bulk, + queued_folders: sequence, + enrolled_job_ids: Vec::new(), + completed_at: Utc::now(), + }, + )) + .await + .expect("scan event publish succeeds"); + } + + assert!(matches!( + job_receiver.recv().await, + Err(broadcast::error::RecvError::Lagged(_)) + )); + + for expected in 0..300 { + let event = scan_receiver + .recv() + .await + .expect("scan-domain burst remains buffered"); + let ScanEvent::SeedCompleted(summary) = event else { + panic!("expected seed-completed scan event"); + }; + assert_eq!(summary.queued_folders, expected); + } + } } diff --git a/crates/ferrex-core/tests/integration/orchestration.rs b/crates/ferrex-core/tests/integration/orchestration.rs index c339836b..dc066a94 100644 --- a/crates/ferrex-core/tests/integration/orchestration.rs +++ b/crates/ferrex-core/tests/integration/orchestration.rs @@ -8,6 +8,7 @@ use tokio::task::JoinHandle; use ferrex_core::domain::scan::orchestration::context::{ ScanHierarchy, ScanNodeKind, }; +use ferrex_core::domain::scan::actors::index::IndexingChange; use ferrex_core::domain::scan::orchestration::job::{ EnqueueRequest, FolderScanJob, IndexUpsertJob, JobKind, JobPayload, JobPriority, MediaAnalyzeJob, MediaFingerprint, MetadataEnrichJob, @@ -466,6 +467,7 @@ async fn end_to_end_batch_mixed(pool: PgPool) { "index:{}:/e2e/mixed/index_{i}.json", lib_id ), + change: IndexingChange::Created, }; enqueues.push(EnqueueRequest::new( JobPriority::P3, diff --git a/crates/ferrex-core/tests/support/scanner_runtime.rs b/crates/ferrex-core/tests/support/scanner_runtime.rs index fde7a53a..8c49136b 100644 --- a/crates/ferrex-core/tests/support/scanner_runtime.rs +++ b/crates/ferrex-core/tests/support/scanner_runtime.rs @@ -576,6 +576,7 @@ pub fn index_upsert_job_from_ready( node: ready.node.clone(), idempotency_key: format!("index:{}:{path_norm}", ready.library_id), path_norm, + change: IndexingChange::Created, } } diff --git a/crates/ferrex-server/src/infra/orchestration/maintenance.rs b/crates/ferrex-server/src/infra/orchestration/maintenance.rs index 4a6054df..c316f210 100644 --- a/crates/ferrex-server/src/infra/orchestration/maintenance.rs +++ b/crates/ferrex-server/src/infra/orchestration/maintenance.rs @@ -717,6 +717,8 @@ mod tests { DurableJobState { job_id, kind: JobKind::FolderScan, + media_id: None, + indexing_change: None, state, attempts: 0, dedupe_key: format!("scan:test:{job_id}"), diff --git a/crates/ferrex-server/src/infra/orchestration/mod.rs b/crates/ferrex-server/src/infra/orchestration/mod.rs index a8e0843b..1474f7e0 100644 --- a/crates/ferrex-server/src/infra/orchestration/mod.rs +++ b/crates/ferrex-server/src/infra/orchestration/mod.rs @@ -162,7 +162,10 @@ impl ScanOrchestrator { config.budget.transcript_extraction_limit = transcript_concurrency; } - let events = Arc::new(InProcJobEventBus::new(256)); + // Job lifecycle state is recovered from PostgreSQL on lag. Scan-domain + // events also drive live per-item catalog projection, so isolate that + // stream and give it enough headroom for large discovery bursts. + let events = Arc::new(InProcJobEventBus::with_capacities(256, 8192)); let correlations = CorrelationCache::default(); let actors = Arc::new(ActorSystem::new( Arc::clone(&tmdb), diff --git a/crates/ferrex-server/src/infra/scan/catalog_event_projection.rs b/crates/ferrex-server/src/infra/scan/catalog_event_projection.rs index 5f4e163e..5e061236 100644 --- a/crates/ferrex-server/src/infra/scan/catalog_event_projection.rs +++ b/crates/ferrex-server/src/infra/scan/catalog_event_projection.rs @@ -7,8 +7,8 @@ use ferrex_core::{ error::MediaError, traits::prelude::MediaIDLike, types::{ - LibraryId, Media, MediaEvent, MediaID, MovieBatchId, ScanEventMetadata, - ScanProgressEvent, SeriesID, + LibraryId, Media, MediaEvent, MediaID, MovieBatchId, MovieReference, + ScanEventMetadata, ScanProgressEvent, SeriesID, }, }; use sha2::Digest; @@ -28,7 +28,12 @@ pub struct CatalogEventProjection { struct CatalogEventProjectionInner { unit_of_work: Arc, media_bus: Arc, - seen_media: Mutex>, + state: Mutex, +} + +#[derive(Debug, Default)] +struct CatalogEventProjectionState { + seen_media: HashSet, } impl fmt::Debug for CatalogEventProjection { @@ -72,7 +77,7 @@ impl CatalogEventProjection { inner: Arc::new(CatalogEventProjectionInner { unit_of_work, media_bus, - seen_media: Mutex::new(HashSet::new()), + state: Mutex::new(CatalogEventProjectionState::default()), }), } } @@ -112,21 +117,67 @@ impl CatalogEventProjection { path_norm: outcome.path_norm.clone(), })?; - let first_seen = { - let mut seen = self.inner.seen_media.lock().await; - seen.insert(outcome.media_id.to_uuid()) - }; + // Keep state mutation and publication under the same lock. The movie + // batch finalization path uses this lock as an ordering barrier, so a + // batch can never be published between marking a movie as added and + // publishing its `MovieAdded` event. + let mut state = self.inner.state.lock().await; + let first_seen = state.seen_media.insert(outcome.media_id.to_uuid()); + let event = + Self::indexed_media_event(media, outcome.change, first_seen); - Ok(Self::indexed_media_event(media, &outcome, first_seen) - .map(|event| self.publish_event(event))) + Ok(event.map(|event| self.publish_event(event))) + } + + /// Rebuild a missing live catalog projection from durable job identity. + /// + /// Completed index jobs can outlive the bounded scan-event broadcast that + /// normally carries their [`IndexingOutcome`]. Rehydrating the canonical + /// media reference makes PostgreSQL authoritative, while the shared state + /// lock preserves the same per-item-before-batch ordering as the live path. + pub async fn publish_reconciled_indexed_media( + &self, + library_id: LibraryId, + path_norm: &str, + media_id: MediaID, + change: IndexingChange, + ) -> Result, CatalogEventProjectionError> { + if self + .inner + .state + .lock() + .await + .seen_media + .contains(&media_id.to_uuid()) + { + return Ok(None); + } + + let media = self.load_media(media_id).await.ok_or_else(|| { + CatalogEventProjectionError::MissingMedia { + library_id, + path_norm: path_norm.to_string(), + } + })?; + + let mut state = self.inner.state.lock().await; + if state.seen_media.contains(&media_id.to_uuid()) { + return Ok(None); + } + + let first_seen = state.seen_media.insert(media_id.to_uuid()); + let event = Self::indexed_media_event(media, change, first_seen); + + Ok(event.map(|event| self.publish_event(event))) } pub async fn publish_movie_batch_finalized( &self, library_id: LibraryId, batch_id: MovieBatchId, - ) -> Result { - self.upsert_movie_batch_hash(&library_id, batch_id) + ) -> Result, CatalogEventProjectionError> { + let movies = self + .upsert_movie_batch_hash(&library_id, batch_id) .await .map_err(|source| { CatalogEventProjectionError::MovieBatchVersionHash { @@ -136,9 +187,20 @@ impl CatalogEventProjection { } })?; - Ok(self.publish_event(Self::movie_batch_finalized_event( - library_id, batch_id, - ))) + // Batch persistence happens before the scan-event consumer is + // guaranteed to have drained every queued `Indexed` outcome. Treat the + // per-item stream as the ordering authority: defer the marker until all + // members were projected by the live path or confirmed-lag recovery. + // This lock also prevents a marker from interleaving with a live + // per-item publication. + let state = self.inner.state.lock().await; + let Some(event) = Self::movie_batch_finalization_event_if_ready( + &state, &movies, library_id, batch_id, + ) else { + return Ok(None); + }; + + Ok(Some(self.publish_event(event))) } pub async fn publish_series_bundle_finalized( @@ -197,10 +259,10 @@ impl CatalogEventProjection { fn indexed_media_event( media: Media, - outcome: &IndexingOutcome, + requested_change: IndexingChange, first_seen: bool, ) -> Option { - let change = match outcome.change { + let change = match requested_change { IndexingChange::Created if first_seen => IndexingChange::Created, _ => IndexingChange::Updated, }; @@ -232,6 +294,18 @@ impl CatalogEventProjection { } } + fn movie_batch_finalization_event_if_ready( + state: &CatalogEventProjectionState, + movies: &[MovieReference], + library_id: LibraryId, + batch_id: MovieBatchId, + ) -> Option { + movies + .iter() + .all(|movie| state.seen_media.contains(&movie.id.to_uuid())) + .then(|| Self::movie_batch_finalized_event(library_id, batch_id)) + } + fn series_bundle_finalized_event( library_id: LibraryId, series_id: SeriesID, @@ -299,7 +373,7 @@ impl CatalogEventProjection { &self, library_id: &LibraryId, batch_id: MovieBatchId, - ) -> anyhow::Result<()> { + ) -> anyhow::Result> { let movies = self .inner .unit_of_work @@ -332,7 +406,7 @@ impl CatalogEventProjection { ) .await?; - Ok(()) + Ok(response.movies) } async fn upsert_series_bundle_hash( @@ -449,7 +523,7 @@ impl CatalogEventProjection { #[cfg(test)] mod tests { - use super::CatalogEventProjection; + use super::{CatalogEventProjection, CatalogEventProjectionState}; use crate::infra::scan::scan_manager::ScanEventKind; use chrono::Utc; use ferrex_core::{ @@ -590,6 +664,23 @@ mod tests { } } + fn movie_reference_in_batch( + library_id: LibraryId, + batch_id: MovieBatchId, + index: u64, + ) -> MovieReference { + let mut movie = movie_reference(); + let movie_id = MovieID::new(); + movie.id = movie_id; + movie.library_id = library_id; + movie.batch_id = Some(batch_id); + movie.tmdb_id = 1_000 + index; + movie.title = format!("Projected Movie {index}").into(); + movie.file.media_id = MediaID::Movie(movie_id); + movie.file.library_id = library_id; + movie + } + fn series_reference() -> Series { let now = Utc::now(); Series { @@ -674,7 +765,7 @@ mod tests { let first = CatalogEventProjection::indexed_media_event( Media::Movie(Box::new(movie.clone())), - &movie_outcome, + movie_outcome.change, true, ) .expect("movie event"); @@ -682,7 +773,7 @@ mod tests { let repeated = CatalogEventProjection::indexed_media_event( Media::Movie(Box::new(movie)), - &movie_outcome, + movie_outcome.change, false, ) .expect("movie event"); @@ -696,7 +787,7 @@ mod tests { ); let first_series = CatalogEventProjection::indexed_media_event( Media::Series(Box::new(series)), - &series_outcome, + series_outcome.change, true, ) .expect("series event"); @@ -714,7 +805,7 @@ mod tests { let event = CatalogEventProjection::indexed_media_event( Media::Movie(Box::new(movie)), - &movie_outcome, + movie_outcome.change, true, ) .expect("movie event"); @@ -728,7 +819,7 @@ mod tests { ); let event = CatalogEventProjection::indexed_media_event( Media::Series(Box::new(series)), - &series_outcome, + series_outcome.change, true, ) .expect("series event"); @@ -765,6 +856,54 @@ mod tests { ); } + #[test] + fn movie_batch_finalization_waits_without_synthesizing_additions() { + let library_id = LibraryId::new(); + let batch_id = MovieBatchId(3); + let movies = (0..100) + .map(|index| movie_reference_in_batch(library_id, batch_id, index)) + .collect::>(); + + let mut state = CatalogEventProjectionState::default(); + for movie in movies.iter().take(3) { + state.seen_media.insert(movie.id.to_uuid()); + } + + let pending = + CatalogEventProjection::movie_batch_finalization_event_if_ready( + &state, &movies, library_id, batch_id, + ); + + assert!(pending.is_none()); + assert_eq!(state.seen_media.len(), 3); + + state + .seen_media + .extend(movies.iter().map(|movie| movie.id.to_uuid())); + let finalized = + CatalogEventProjection::movie_batch_finalization_event_if_ready( + &state, &movies, library_id, batch_id, + ); + + assert!(matches!( + finalized, + Some(MediaEvent::MovieBatchFinalized { + library_id: event_library_id, + batch_id: event_batch_id, + }) if event_library_id == library_id && event_batch_id == batch_id + )); + assert_eq!(state.seen_media.len(), 100); + + let repeated = + CatalogEventProjection::movie_batch_finalization_event_if_ready( + &state, &movies, library_id, batch_id, + ); + assert!(matches!( + repeated, + Some(MediaEvent::MovieBatchFinalized { .. }) + )); + } + #[test] fn scan_progress_projection_preserves_wire_variants() { let started = CatalogEventProjection::scan_progress_media_event( diff --git a/crates/ferrex-server/src/infra/scan/movie_batch_notifier.rs b/crates/ferrex-server/src/infra/scan/movie_batch_notifier.rs index 98a22df8..3558d3ab 100644 --- a/crates/ferrex-server/src/infra/scan/movie_batch_notifier.rs +++ b/crates/ferrex-server/src/infra/scan/movie_batch_notifier.rs @@ -14,6 +14,14 @@ use tracing::{info, warn}; use super::catalog_event_projection::CatalogEventProjection; const MOVIE_BATCH_POLL_INTERVAL: Duration = Duration::from_millis(500); +const MOVIE_BATCH_FINAL_DRAIN_RETRY: Duration = Duration::from_millis(25); +const MOVIE_BATCH_FINAL_DRAIN_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum MovieBatchDrainStatus { + Complete, + Pending, +} #[derive(Debug)] struct LibraryNotifier { @@ -85,7 +93,14 @@ impl MovieBatchFinalizationNotifiers { } let _ = notifier.stop_tx.send(true); - notifier.task.abort(); + drop(guard); + if let Err(err) = notifier.task.await { + warn!( + library = %library_id, + error = %err, + "movie batch finalization task failed during final drain" + ); + } } /// Stop notifier work immediately when its library has been deleted. @@ -129,64 +144,144 @@ async fn movie_batch_notifier_loop( ticker.set_missed_tick_behavior(MissedTickBehavior::Delay); loop { - tokio::select! { - _ = stop_rx.changed() => { - if *stop_rx.borrow() { - break; - } + let stopping = tokio::select! { + result = stop_rx.changed() => { + result.is_err() || *stop_rx.borrow() } - _ = ticker.tick() => {} + _ = ticker.tick() => false, + }; + + let status = drain_ready_movie_batches( + library_id, + &unit_of_work, + &catalog_events, + &mut last_notified, + ) + .await; + + if !stopping { + continue; + } + + let deadline = time::Instant::now() + MOVIE_BATCH_FINAL_DRAIN_TIMEOUT; + let mut status = status; + while status == MovieBatchDrainStatus::Pending + && time::Instant::now() < deadline + { + time::sleep(MOVIE_BATCH_FINAL_DRAIN_RETRY).await; + status = drain_ready_movie_batches( + library_id, + &unit_of_work, + &catalog_events, + &mut last_notified, + ) + .await; + } + if status == MovieBatchDrainStatus::Pending { + warn!( + library = %library_id, + "movie batch finalization remained pending after final drain" + ); + } + break; + } +} + +async fn drain_ready_movie_batches( + library_id: LibraryId, + unit_of_work: &AppUnitOfWork, + catalog_events: &CatalogEventProjection, + last_notified: &mut Option, +) -> MovieBatchDrainStatus { + let finalized = match unit_of_work + .media_refs + .list_finalized_movie_reference_batches(&library_id) + .await + { + Ok(batch_ids) => batch_ids, + Err(err) => { + warn!( + "failed to list finalized movie batches for library {}: {}", + library_id, err + ); + return MovieBatchDrainStatus::Pending; + } + }; + + for batch_id in finalized { + if let Some(last) = *last_notified + && batch_id <= last + { + continue; } - let finalized = match unit_of_work - .media_refs - .list_finalized_movie_reference_batches(&library_id) + let receivers = catalog_events.receiver_count(); + let frame = match catalog_events + .publish_movie_batch_finalized(library_id, batch_id) .await { - Ok(batch_ids) => batch_ids, + Ok(Some(frame)) => frame, + Ok(None) => { + tracing::debug!( + library = %library_id, + batch_id = %batch_id, + "deferring movie batch finalization until per-item projections catch up" + ); + return MovieBatchDrainStatus::Pending; + } Err(err) => { warn!( - "failed to list finalized movie batches for library {}: {}", - library_id, err + "movie batch finalization projection failed (library {}, batch {}): {}", + library_id, batch_id, err ); - continue; + return MovieBatchDrainStatus::Pending; } }; + info!( + library = %library_id, + batch_id = %batch_id, + receivers = receivers, + sequence = frame.sequence, + "published movie batch finalization" + ); - if finalized.is_empty() { - continue; - } + *last_notified = Some(batch_id); + } - for batch_id in finalized { - if let Some(last) = last_notified - && batch_id <= last - { - continue; - } + MovieBatchDrainStatus::Complete +} - let receivers = catalog_events.receiver_count(); - let frame = match catalog_events - .publish_movie_batch_finalized(library_id, batch_id) - .await - { - Ok(frame) => frame, - Err(err) => { - warn!( - "movie batch finalization projection failed (library {}, batch {}): {}", - library_id, batch_id, err - ); - break; - } - }; - info!( - library = %library_id, - batch_id = %batch_id, - receivers = receivers, - sequence = frame.sequence, - "published movie batch finalization" - ); +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicBool, Ordering}; - last_notified = Some(batch_id); - } + #[tokio::test] + async fn run_finish_waits_for_notifier_final_drain() { + let library_id = LibraryId::new(); + let notifiers = MovieBatchFinalizationNotifiers::new(); + let drained = Arc::new(AtomicBool::new(false)); + let drained_by_task = Arc::clone(&drained); + let (stop_tx, mut stop_rx) = watch::channel(false); + let task = tokio::spawn(async move { + stop_rx.changed().await.expect("stop sender remains alive"); + assert!(*stop_rx.borrow()); + tokio::task::yield_now().await; + drained_by_task.store(true, Ordering::SeqCst); + }); + + notifiers.libraries.lock().await.insert( + library_id, + LibraryNotifier { + active_runs: 1, + stop_tx, + task, + }, + ); + + notifiers.on_run_finished(library_id).await; + + assert!(drained.load(Ordering::SeqCst)); + assert!(!notifiers.libraries.lock().await.contains_key(&library_id)); } } diff --git a/crates/ferrex-server/src/infra/scan/scan_manager.rs b/crates/ferrex-server/src/infra/scan/scan_manager.rs index 9859bc5f..af952ed3 100644 --- a/crates/ferrex-server/src/infra/scan/scan_manager.rs +++ b/crates/ferrex-server/src/infra/scan/scan_manager.rs @@ -14,6 +14,7 @@ use ferrex_core::{ orchestration::{ DurableJobState, JobEvent, LibraryActorCommand, LibraryScanRun, LibraryScanRunProgressUpdate, NewLibraryScanRun, StartMode, + context::SeriesRootPath, events::{JobEventPayload, ScanEvent, ScanSeedSummary}, job::{JobId, JobKind, JobState}, scan_cursor::{ScanCursor, ScanCursorRepository, normalize_path}, @@ -3589,6 +3590,8 @@ mod tests { DurableJobState { job_id, kind: JobKind::FolderScan, + media_id: None, + indexing_change: None, state, attempts, dedupe_key: dedupe_key.into(), @@ -3775,6 +3778,66 @@ mod tests { assert!(gate.allows_terminal(correlation_id).await); } + #[tokio::test] + async fn catalog_lag_gate_waits_for_seed_and_terminal_index_jobs() { + let correlation_id = Uuid::now_v7(); + let gate = DurableReconciliationGate::default(); + gate.require_catalog_projection_recovery([correlation_id]) + .await; + + assert!(gate.requires_catalog_projection(correlation_id).await); + assert!( + !gate + .clear_after_seeded_snapshot(correlation_id, false, true) + .await + ); + assert!( + !gate + .clear_after_seeded_snapshot(correlation_id, true, false) + .await + ); + assert!(gate.is_required(correlation_id).await); + assert!(gate.requires_catalog_projection(correlation_id).await); + + assert!( + gate.clear_after_seeded_snapshot(correlation_id, true, true) + .await + ); + assert!(gate.allows_terminal(correlation_id).await); + assert!(!gate.requires_catalog_projection(correlation_id).await); + } + + #[test] + fn catalog_lag_recovery_remains_pending_while_index_job_is_active() { + let correlation_id = Uuid::now_v7(); + let now = Utc::now(); + let mut active_index = durable_job( + correlation_id, + JobId::new(), + "index:active", + JobState::Leased, + 0, + "/library/active.mkv", + now, + ); + active_index.kind = JobKind::IndexUpsert; + active_index.media_id = Some(ferrex_core::types::MediaID::new( + ferrex_core::types::VideoMediaType::Movie, + )); + active_index.indexing_change = Some( + ferrex_core::domain::scan::actors::index::IndexingChange::Created, + ); + + assert!(!ScanRunAggregatorInner::catalog_index_jobs_terminal( + std::slice::from_ref(&active_index) + )); + + active_index.state = JobState::Completed; + assert!(ScanRunAggregatorInner::catalog_index_jobs_terminal(&[ + active_index + ])); + } + #[tokio::test] async fn retryable_failure_gate_waits_for_authoritative_queue_outcome() { let correlation_id = Uuid::now_v7(); @@ -3915,6 +3978,7 @@ mod tests { .clear_after_seeded_snapshot( correlation_id, run.seed_completed().await, + true, ) .await ); @@ -3974,6 +4038,7 @@ mod tests { gate.clear_after_seeded_snapshot( correlation_id, run.seed_completed().await, + true, ) .await ); @@ -4915,6 +4980,106 @@ mod tests { assert_eq!(history.len(), 1); assert_eq!(history[0].library_id, retained_library); } + + fn ready_series_bundle_entry( + library_id: LibraryId, + series_id: ferrex_core::types::SeriesID, + series_root_path: SeriesRootPath, + ) -> SeriesBundleTrackerEntry { + use ferrex_core::domain::scan::{ + actors::FolderScanSummary, + orchestration::{ + AnalyzeScanHierarchy, + context::{ + FolderScanContext, SeriesFolderScanContext, SeriesLink, + SeriesRef, SeriesScanHierarchy, + }, + }, + }; + + let context = FolderScanContext::Series(SeriesFolderScanContext { + library_id, + series_root_path: series_root_path.clone(), + }); + let mut entry = SeriesBundleTrackerEntry::new(Instant::now()); + entry.tracker.observe_folder_discovered(&context); + entry + .tracker + .observe_folder_scan_completed(&FolderScanSummary { + context, + discovered_files: 0, + enqueued_subfolders: 0, + listing_hash: "series-root".into(), + outcome: FolderScanOutcome::Changed, + completed_at: Utc::now(), + }); + entry.tracker.observe_indexed(&IndexingOutcome { + library_id, + path_norm: series_root_path.as_str().to_owned(), + media_id: ferrex_core::types::MediaID::Series(series_id), + hierarchy: AnalyzeScanHierarchy::Series(SeriesScanHierarchy { + series_root_path, + series: SeriesLink::Resolved(SeriesRef { + id: series_id, + slug: Some("claim-race".into()), + title: Some("Claim Race".into()), + }), + }), + indexed_at: Utc::now(), + upserted: true, + media: None, + change: ferrex_core::domain::scan::actors::index::IndexingChange::Created, + }); + entry + } + + #[tokio::test] + async fn concurrent_series_bundle_finalizers_claim_once() { + let library_id = LibraryId::new(); + let series_id = ferrex_core::types::SeriesID(Uuid::now_v7()); + let series_root_path = + SeriesRootPath::try_new("/library/Claim Race").unwrap(); + let entry = Arc::new(Mutex::new(ready_series_bundle_entry( + library_id, + series_id, + series_root_path.clone(), + ))); + let barrier = Arc::new(tokio::sync::Barrier::new(3)); + + let first = { + let entry = Arc::clone(&entry); + let barrier = Arc::clone(&barrier); + tokio::spawn(async move { + barrier.wait().await; + entry.lock().await.claim_next_finalization().is_some() + }) + }; + let second = { + let entry = Arc::clone(&entry); + let barrier = Arc::clone(&barrier); + tokio::spawn(async move { + barrier.wait().await; + entry.lock().await.claim_next_finalization().is_some() + }) + }; + + barrier.wait().await; + let (first, second) = tokio::join!(first, second); + let claims = usize::from(first.unwrap()) + usize::from(second.unwrap()); + assert_eq!(claims, 1, "only one worker may claim the same bundle"); + + let mut guard = entry.lock().await; + guard.settle_finalization(&series_root_path, false); + assert!( + guard.claim_next_finalization().is_some(), + "a failed publication must release its claim for polling retries" + ); + guard.settle_finalization(&series_root_path, true); + assert!( + guard.claim_next_finalization().is_none(), + "a published bundle must remain finalized" + ); + } } #[derive(Clone)] @@ -4940,6 +5105,7 @@ struct DurableReconciliationGate { #[derive(Default)] struct DurableReconciliationRequirements { required: HashSet, + catalog_projection_required: HashSet, retryable_failures: HashMap>, } @@ -4948,6 +5114,16 @@ impl DurableReconciliationGate { self.state.lock().await.required.extend(correlations); } + async fn require_catalog_projection_recovery( + &self, + correlations: impl IntoIterator, + ) { + let correlations = correlations.into_iter().collect::>(); + let mut state = self.state.lock().await; + state.required.extend(correlations.iter().copied()); + state.catalog_projection_required.extend(correlations); + } + async fn require_retryable_failure( &self, correlation_id: Uuid, @@ -4970,9 +5146,18 @@ impl DurableReconciliationGate { !self.is_required(correlation_id).await } + async fn requires_catalog_projection(&self, correlation_id: Uuid) -> bool { + self.state + .lock() + .await + .catalog_projection_required + .contains(&correlation_id) + } + async fn reconciled(&self, correlation_id: Uuid) { let mut state = self.state.lock().await; state.required.remove(&correlation_id); + state.catalog_projection_required.remove(&correlation_id); state.retryable_failures.remove(&correlation_id); } @@ -4980,8 +5165,9 @@ impl DurableReconciliationGate { &self, correlation_id: Uuid, seed_completed: bool, + catalog_projection_complete: bool, ) -> bool { - if !seed_completed { + if !seed_completed || !catalog_projection_complete { return false; } self.reconciled(correlation_id).await; @@ -5032,6 +5218,7 @@ impl DurableReconciliationGate { #[derive(Debug)] struct SeriesBundleTrackerEntry { tracker: SeriesBundleTracker, + finalizations_in_flight: HashSet, last_touched_at: Instant, last_polled_at: Instant, } @@ -5040,6 +5227,7 @@ impl SeriesBundleTrackerEntry { fn new(now: Instant) -> Self { Self { tracker: SeriesBundleTracker::default(), + finalizations_in_flight: HashSet::new(), last_touched_at: now, last_polled_at: now, } @@ -5048,6 +5236,30 @@ impl SeriesBundleTrackerEntry { fn touch(&mut self, now: Instant) { self.last_touched_at = now; } + + fn claim_next_finalization(&mut self) -> Option { + self.tracker + .finalization_candidates() + .into_iter() + .find(|candidate| { + self.finalizations_in_flight + .insert(candidate.series_root_path.clone()) + }) + } + + fn settle_finalization( + &mut self, + series_root_path: &SeriesRootPath, + published: bool, + ) { + if !self.finalizations_in_flight.remove(series_root_path) { + return; + } + + if published { + self.tracker.mark_finalized(series_root_path); + } + } } impl ScanRunAggregator { @@ -5081,9 +5293,14 @@ impl ScanRunAggregator { } fn spawn_worker(&self) { - let inner = Arc::clone(&self.inner); + let job_progress = Arc::clone(&self.inner); spawn(async move { - ScanRunAggregatorInner::run(inner).await; + ScanRunAggregatorInner::run_job_progress(job_progress).await; + }); + + let scan_events = Arc::clone(&self.inner); + spawn(async move { + ScanRunAggregatorInner::run_scan_events(scan_events).await; }); } @@ -5130,16 +5347,14 @@ impl ScanRunAggregator { } impl ScanRunAggregatorInner { - async fn run(self: Arc) { + async fn run_job_progress(self: Arc) { use tokio::sync::broadcast::error::RecvError; let mut receiver = self.orchestrator.subscribe_job_events(); - let mut domain_rx = self.orchestrator.subscribe_scan_events(); let mut ticker = interval(Duration::from_millis(500)); loop { tokio::select! { - biased; result = receiver.recv() => { match result { Ok(event) => self.handle_job_event(event).await, @@ -5151,17 +5366,6 @@ impl ScanRunAggregatorInner { Err(RecvError::Closed) => break, } } - result = domain_rx.recv() => { - match result { - Ok(event) => self.handle_scan_event(event).await, - Err(RecvError::Lagged(skipped)) => { - warn!("domain event stream lagged {skipped} events"); - self.mark_all_runs_reconciliation_required().await; - self.reconcile_all_runs("scan_event_lag").await; - } - Err(RecvError::Closed) => break, - } - } _ = ticker.tick() => { self.check_quiescence().await; } @@ -5169,6 +5373,26 @@ impl ScanRunAggregatorInner { } } + async fn run_scan_events(self: Arc) { + use tokio::sync::broadcast::error::RecvError; + + let mut receiver = self.orchestrator.subscribe_scan_events(); + loop { + match receiver.recv().await { + Ok(event) => self.handle_scan_event(event).await, + Err(RecvError::Lagged(skipped)) => { + warn!("domain event stream lagged {skipped} events"); + // Keep PostgreSQL authoritative for terminal progress, but + // never block catalog projection behind that durable read. + // The independent progress worker will reconcile this gate + // on its next quiescence tick. + self.mark_all_runs_catalog_reconciliation_required().await; + } + Err(RecvError::Closed) => break, + } + } + } + async fn check_quiescence(&self) { let runs: Vec> = { let guard = self.runs.read().await; @@ -5246,6 +5470,14 @@ impl ScanRunAggregatorInner { self.reconciliation_gate.require_all(correlations).await; } + async fn mark_all_runs_catalog_reconciliation_required(&self) { + let correlations: Vec = + self.runs.read().await.keys().copied().collect(); + self.reconciliation_gate + .require_catalog_projection_recovery(correlations) + .await; + } + async fn reconcile_all_runs(&self, reason: &'static str) { let runs: Vec> = { let guard = self.runs.read().await; @@ -5310,6 +5542,16 @@ impl ScanRunAggregatorInner { ))); } + let catalog_projection_complete = if self + .reconciliation_gate + .requires_catalog_projection(run.correlation_id()) + .await + { + self.reconcile_catalog_projections(run, &jobs).await + } else { + true + }; + tracing::debug!( scan = %run.scan_id(), library = %run.library_id(), @@ -5325,6 +5567,7 @@ impl ScanRunAggregatorInner { .clear_after_seeded_snapshot( run.correlation_id(), run.seed_completed().await, + catalog_projection_complete, ) .await { @@ -5338,6 +5581,90 @@ impl ScanRunAggregatorInner { Ok(()) } + async fn reconcile_catalog_projections( + &self, + run: &Arc, + jobs: &[DurableJobState], + ) -> bool { + let mut complete = Self::catalog_index_jobs_terminal(jobs); + for job in jobs.iter().filter(|job| { + job.kind == JobKind::IndexUpsert && job.state == JobState::Completed + }) { + let Some(media_id) = job.media_id else { + warn!( + scan = %run.scan_id(), + library = %run.library_id(), + job = %job.job_id.0, + "completed index job omitted durable media identity" + ); + complete = false; + continue; + }; + let Some(change) = job.indexing_change else { + warn!( + scan = %run.scan_id(), + library = %run.library_id(), + job = %job.job_id.0, + "completed index job omitted durable catalog change semantics" + ); + complete = false; + continue; + }; + let Some(path_norm) = + job.path_key.as_ref().and_then(subject_key_path) + else { + warn!( + scan = %run.scan_id(), + library = %run.library_id(), + job = %job.job_id.0, + "completed index job omitted durable path identity" + ); + complete = false; + continue; + }; + + match self + .catalog_events + .publish_reconciled_indexed_media( + run.library_id(), + path_norm, + media_id, + change, + ) + .await + { + Ok(Some(frame)) => { + tracing::debug!( + scan = %run.scan_id(), + library = %run.library_id(), + job = %job.job_id.0, + sequence = frame.sequence, + "recovered catalog projection from durable index job" + ); + } + Ok(None) => {} + Err(err) => { + complete = false; + warn!( + scan = %run.scan_id(), + library = %run.library_id(), + job = %job.job_id.0, + error = %err, + "failed to recover catalog projection from durable index job" + ); + } + } + } + + complete + } + + fn catalog_index_jobs_terminal(jobs: &[DurableJobState]) -> bool { + jobs.iter() + .filter(|job| job.kind == JobKind::IndexUpsert) + .all(DurableJobState::is_terminal) + } + async fn poll_series_bundle_finalizations(&self) { let now = Instant::now(); @@ -5379,6 +5706,7 @@ impl ScanRunAggregatorInner { let mut guard = self.series_bundles.lock().await; guard.retain(|library_id, entry| { active_libraries.contains(library_id) + || !entry.finalizations_in_flight.is_empty() || now.duration_since(entry.last_touched_at) < SERIES_BUNDLE_TRACKER_IDLE_TTL }); @@ -5756,31 +6084,44 @@ impl ScanRunAggregatorInner { } async fn try_emit_series_bundle_finalized(&self, library_id: LibraryId) { - let candidates: Vec = { - let guard = self.series_bundles.lock().await; - guard - .get(&library_id) - .map(|entry| entry.tracker.finalization_candidates()) - .unwrap_or_default() - }; + loop { + // The job-progress and scan-domain workers can both make the same + // bundle eligible at nearly the same time. Claim the candidate + // while holding the tracker mutex so only one worker can publish + // it across the asynchronous database/event projection below. + let finalization = { + let mut guard = self.series_bundles.lock().await; + guard + .get_mut(&library_id) + .and_then(SeriesBundleTrackerEntry::claim_next_finalization) + }; + let Some(finalization) = finalization else { + break; + }; - for finalization in candidates { let receivers = self.catalog_events.receiver_count(); - let Some(frame) = self + let frame = self .catalog_events .publish_series_bundle_finalized( finalization.library_id, finalization.series_id, ) - .await - else { - continue; - }; + .await; let mut guard = self.series_bundles.lock().await; if let Some(entry) = guard.get_mut(&library_id) { - entry.tracker.mark_finalized(&finalization.series_root_path); + entry.settle_finalization( + &finalization.series_root_path, + frame.is_some(), + ); } + drop(guard); + + let Some(frame) = frame else { + // Publication failures are retried by the periodic poll. Do + // not immediately reclaim the same candidate in a tight loop. + break; + }; info!( library = %finalization.library_id, From 7e993449953e464d7040518f427da1e36fb1e69d Mon Sep 17 00:00:00 2001 From: Grayson Hieb Date: Sun, 9 Aug 2026 13:57:45 -0600 Subject: [PATCH 05/10] fix(player): stream scan additions into home --- .../src/domains/library/update.rs | 275 +++++++++++++++++- .../src/domains/ui/tabs/state.rs | 5 + .../virtual_carousel_updates.rs | 2 +- 3 files changed, 272 insertions(+), 10 deletions(-) diff --git a/crates/ferrex-player-ui/src/domains/library/update.rs b/crates/ferrex-player-ui/src/domains/library/update.rs index 9eb3bff2..f661431d 100644 --- a/crates/ferrex-player-ui/src/domains/library/update.rs +++ b/crates/ferrex-player-ui/src/domains/library/update.rs @@ -15,7 +15,9 @@ use crate::{ tabs::{TabId, TabState}, update_handlers::{ emit_initial_all_tab_snapshots_combined, init_all_tab_view, + virtual_carousel_updates::maybe_send_snapshot_for_key, }, + views::virtual_carousel::types::{CarouselConfig, CarouselKey}, }, }, infra::api_types::Media, @@ -30,7 +32,7 @@ use std::{ sync::Arc, }; -use ferrex_core::player_prelude::{Library, LibraryId}; +use ferrex_core::player_prelude::{Library, LibraryId, MediaIDLike, MediaOps}; #[cfg(feature = "demo")] use ferrex_model::library::LibraryType; use ferrex_player_api::services::api::ApiService; @@ -1102,20 +1104,25 @@ fn apply_discovered_media_to_tabs( for (library_id, media_items) in additions { let tab_id = TabId::Library(*library_id); - if tab_id != active_tab { + let home_is_active = active_tab == TabId::Home; + if tab_id != active_tab && !home_is_active { continue; } - if let Some(TabState::Library(tab_state)) = - state.tab_manager.get_tab_mut(tab_id) + if let TabState::Library(tab_state) = + state.tab_manager.get_or_create_tab(tab_id) { let mut inserted_any = false; + let mut represented_on_home = false; for media in media_items { if tab_state.insert_media_reference(media) { inserted_any = true; } + represented_on_home |= home_is_active + && tab_state + .contains_cached_media_id(&media.media_id().to_uuid()); } - if inserted_any { + if inserted_any || represented_on_home { inline_updated.insert(*library_id); } } @@ -1138,8 +1145,8 @@ fn mark_tabs_after_media_changes( for library_id in libraries { let tab_id = TabId::Library(*library_id); - let skip_active_refresh = - inline_updated.contains(library_id) && active_tab == tab_id; + let skip_active_refresh = inline_updated.contains(library_id) + && (active_tab == tab_id || active_tab == TabId::Home); if skip_active_refresh { continue; @@ -1155,16 +1162,184 @@ fn mark_tabs_after_media_changes( state.tab_manager.refresh_active_tab(); } + if active_tab == TabId::Home && !inline_updated.is_empty() { + sync_home_library_carousels(state, inline_updated); + active_needs_refresh = true; + } + active_needs_refresh } +/// Keep the active Home view's per-library rails in sync with inline scan +/// additions without rebuilding every curated list for every discovered item. +/// Poster demand is limited by the carousel snapshot debounce, while the item +/// count itself changes immediately so each addition is visible to the view. +fn sync_home_library_carousels( + state: &mut State, + libraries: &HashSet, +) { + for library_id in libraries { + let Some(TabState::Library(tab_state)) = + state.tab_manager.get_tab(TabId::Library(*library_id)) + else { + continue; + }; + + let key = match tab_state.library_type { + crate::infra::api_types::LibraryType::Movies => { + CarouselKey::LibraryMovies(library_id.to_uuid()) + } + crate::infra::api_types::LibraryType::Series => { + CarouselKey::LibrarySeries(library_id.to_uuid()) + } + }; + let total = tab_state.cached_index_ids.len(); + + if let Some(carousel) = + state.domains.ui.state.carousel_registry.get_mut(&key) + { + if carousel.total_items != total { + carousel.set_total_items(total); + } + } else { + let width = state.window_size.width.max(1.0); + let scale = state.domains.ui.state.scaled_layout.scale; + state.domains.ui.state.carousel_registry.ensure_default( + key.clone(), + total, + width, + CarouselConfig::poster_defaults(), + scale, + ); + } + maybe_send_snapshot_for_key(state, &key, false); + } +} + #[cfg(test)] mod tests { use super::*; use ferrex_core::player_prelude::{ - ScanCommandAcceptedResponse, ScanLifecycleStatus, ScanRunMode, - ScanStartDisposition, + MediaID, MovieBatchId, MovieID, MovieReference, + MovieReferenceBatchResponse, ScanCommandAcceptedResponse, + ScanLifecycleStatus, ScanRunMode, ScanStartDisposition, + }; + use ferrex_model::{ + EnhancedMovieDetails, MediaFile, details::ExternalIds, + image::metadata::MediaImages, titles::MovieTitle, urls::MovieURL, }; + use uuid::Uuid; + + fn movie_details(id: u64, title: &str) -> EnhancedMovieDetails { + EnhancedMovieDetails { + id, + title: title.to_string(), + original_title: None, + overview: None, + release_date: None, + runtime: None, + vote_average: None, + vote_count: None, + popularity: None, + content_rating: None, + content_ratings: Vec::new(), + release_dates: Vec::new(), + genres: Vec::new(), + spoken_languages: Vec::new(), + production_companies: Vec::new(), + production_countries: Vec::new(), + homepage: None, + status: None, + tagline: None, + budget: None, + revenue: None, + poster_path: None, + backdrop_path: None, + logo_path: None, + primary_poster_iid: None, + primary_backdrop_iid: None, + images: MediaImages::default(), + cast: Vec::new(), + crew: Vec::new(), + videos: Vec::new(), + keywords: Vec::new(), + external_ids: ExternalIds::default(), + alternative_titles: Vec::new(), + translations: Vec::new(), + collection: None, + recommendations: Vec::new(), + similar: Vec::new(), + } + } + + fn movie_reference( + library_id: LibraryId, + batch_id: MovieBatchId, + index: u64, + ) -> MovieReference { + let movie_id = MovieID(Uuid::from_u128(10_000 + u128::from(index))); + let title = format!("Movie {index:03}"); + MovieReference { + id: movie_id, + library_id, + batch_id: Some(batch_id), + tmdb_id: index, + title: MovieTitle::from(title.clone()), + details: movie_details(index, &title), + endpoint: MovieURL::from(format!("/media/{index}")), + file: MediaFile { + id: Uuid::from_u128(20_000 + u128::from(index)), + media_id: MediaID::Movie(movie_id), + path: std::path::PathBuf::from(format!( + "/tmp/movie-{index}.mkv" + )), + filename: format!("movie-{index}.mkv"), + size: index + 1, + discovered_at: ferrex_model::chrono::Utc::now(), + created_at: ferrex_model::chrono::Utc::now(), + media_file_metadata: None, + library_id, + }, + theme_color: None, + } + } + + fn library_item_count(state: &State, library_id: LibraryId) -> usize { + match state + .tab_manager + .get_tab(TabId::Library(library_id)) + .expect("library tab exists") + { + TabState::Library(tab) => tab.grid_state.total_items, + TabState::Home(_) | TabState::Collections(_) => { + panic!("expected library tab") + } + } + } + + fn home_movie_item_count(state: &State, library_id: LibraryId) -> usize { + state + .domains + .ui + .state + .carousel_registry + .get(&CarouselKey::LibraryMovies(library_id.to_uuid())) + .expect("Home movie carousel exists") + .total_items + } + + fn home_recent_movie_count(state: &State) -> usize { + match state + .tab_manager + .get_tab(TabId::Home) + .expect("Home tab exists") + { + TabState::Home(tab) => tab.recent_movies.len(), + TabState::Library(_) | TabState::Collections(_) => { + panic!("expected Home tab") + } + } + } #[tokio::test(flavor = "current_thread")] async fn double_click_scan_started_reuses_one_active_scan_entry() { @@ -1205,4 +1380,86 @@ mod tests { assert_eq!(snapshot.mode, ScanRunMode::Manual); assert_eq!(snapshot.run_key, ScanRunMode::Manual.run_key(library_id)); } + + #[tokio::test(flavor = "current_thread")] + async fn home_movie_additions_stream_then_batch_replacement_deduplicates() { + use crate::infra::{ + api_types::LibraryType, repository::media_repo::MediaRepo, + }; + + let mut state = State::new("http://localhost:3000".to_string()); + *state.media_repo.write() = Some(MediaRepo::new_empty()); + + let library_id = LibraryId(Uuid::from_u128(1)); + let batch_id = MovieBatchId::new(1).expect("valid batch id"); + state + .tab_manager + .register_library(library_id, LibraryType::Movies); + init_all_tab_view(&mut state); + assert_eq!(state.tab_manager.active_tab_id(), TabId::Home); + assert_eq!(home_movie_item_count(&state, library_id), 0); + + let movies: Vec<_> = (0..100) + .map(|index| movie_reference(library_id, batch_id, index)) + .collect(); + + for (index, movie) in movies.iter().cloned().enumerate() { + let _ = update_library( + &mut state, + LibraryMessage::MediaDiscovered(vec![Media::Movie(Box::new( + movie, + ))]), + ); + assert_eq!( + home_movie_item_count(&state, library_id), + index + 1, + "Home must expose each MovieAdded event before batch finalization" + ); + assert_eq!( + home_recent_movie_count(&state), + 0, + "per-item additions must not rebuild every curated Home rail" + ); + } + + let batch = MovieReferenceBatchResponse { + library_id, + batch_id, + movies: movies.clone(), + }; + let batch_bytes = rkyv::to_bytes::(&batch) + .expect("serialize movie batch"); + let _ = update_library( + &mut state, + LibraryMessage::MovieBatchLoaded { + library_id, + batch_id, + result: Ok(batch_bytes), + }, + ); + + assert_eq!(home_movie_item_count(&state, library_id), 100); + assert_eq!(library_item_count(&state, library_id), 100); + assert!( + state + .domains + .library + .state + .repo_accessor + .is_movie_backed_by_batch(&movies[0].id) + .expect("batch lookup succeeds") + ); + + let _ = update_library( + &mut state, + LibraryMessage::MediaDiscovered(vec![Media::Movie(Box::new( + movies[0].clone(), + ))]), + ); + assert_eq!( + home_movie_item_count(&state, library_id), + 100, + "a late duplicate MovieAdded must not duplicate a batch-backed Home item" + ); + } } diff --git a/crates/ferrex-player-ui/src/domains/ui/tabs/state.rs b/crates/ferrex-player-ui/src/domains/ui/tabs/state.rs index c9ca70c9..4163c7e9 100644 --- a/crates/ferrex-player-ui/src/domains/ui/tabs/state.rs +++ b/crates/ferrex-player-ui/src/domains/ui/tabs/state.rs @@ -1555,6 +1555,11 @@ impl LibraryTabState { Some(left) } + /// Return whether a media ID is already represented in the cached ordering. + pub(crate) fn contains_cached_media_id(&self, id: &Uuid) -> bool { + self.cached_index_set.contains(id) + } + /// Insert a newly discovered media item into the cached ordering based on the /// current sort configuration. Returns true if the media was inserted. pub fn insert_media_reference(&mut self, media: &Media) -> bool { diff --git a/crates/ferrex-player-ui/src/domains/ui/update_handlers/virtual_carousel_updates.rs b/crates/ferrex-player-ui/src/domains/ui/update_handlers/virtual_carousel_updates.rs index 545819ae..357afe51 100644 --- a/crates/ferrex-player-ui/src/domains/ui/update_handlers/virtual_carousel_updates.rs +++ b/crates/ferrex-player-ui/src/domains/ui/update_handlers/virtual_carousel_updates.rs @@ -1619,7 +1619,7 @@ fn handle_release_snap_align( Task::none() } -pub(super) fn maybe_send_snapshot_for_key( +pub(crate) fn maybe_send_snapshot_for_key( state: &mut State, key: &CarouselKey, force: bool, From 7b4a2bcb239a84581b833297f9407010bce99dd5 Mon Sep 17 00:00:00 2001 From: Grayson Hieb Date: Sun, 9 Aug 2026 17:39:53 -0600 Subject: [PATCH 06/10] fix(scan): recover stalled series pipelines --- ...3a2a750b9329059ff23415bd7270babc53c3e.json | 15 - ...c2a154b89307d4cc2884c25b8b7882f538479.json | 25 - ...58b16cd9a16a857506efad6a208781495d20.json} | 4 +- ...6c6ee9f94f91916925f9209306ae197f0cd12.json | 40 - ...8409578e2fc77dba136a3f1e599d127d43a16.json | 142 -- .../domain/scan/orchestration/dispatcher.rs | 684 +++++- .../src/domain/scan/orchestration/job.rs | 1 + .../src/domain/scan/orchestration/mod.rs | 6 +- .../domain/scan/orchestration/persistence.rs | 1974 ++++++++++++++++- .../src/domain/scan/orchestration/queue.rs | 62 +- .../scan/orchestration/runtime/event_bus.rs | 32 + .../scan/orchestration/runtime/supervisor.rs | 210 ++ .../scan/orchestration/runtime/task_graph.rs | 153 +- .../orchestration/series/bundle_tracker.rs | 976 +++++++- .../scan/orchestration/series/coordinator.rs | 105 +- .../domain/scan/orchestration/series_state.rs | 416 +++- .../src/infra/orchestration/maintenance.rs | 1 + .../src/infra/orchestration/mod.rs | 9 +- .../src/infra/scan/scan_manager.rs | 688 ++++-- 19 files changed, 4902 insertions(+), 641 deletions(-) delete mode 100644 .sqlx/query-13a4bec38c568988dc6d070307f3a2a750b9329059ff23415bd7270babc53c3e.json delete mode 100644 .sqlx/query-442e8498eaaa251978202770355c2a154b89307d4cc2884c25b8b7882f538479.json rename .sqlx/{query-4ae8a4b8fa4dee0003e1154f2b8d76c719dad5e5185c52771451cf7b31cebf30.json => query-5777aecc6180c52bcac8b65c5c4358b16cd9a16a857506efad6a208781495d20.json} (80%) delete mode 100644 .sqlx/query-7ab16258ea647055c103d95b33f6c6ee9f94f91916925f9209306ae197f0cd12.json delete mode 100644 .sqlx/query-d68e1ff50b24fe00d12019d95668409578e2fc77dba136a3f1e599d127d43a16.json diff --git a/.sqlx/query-13a4bec38c568988dc6d070307f3a2a750b9329059ff23415bd7270babc53c3e.json b/.sqlx/query-13a4bec38c568988dc6d070307f3a2a750b9329059ff23415bd7270babc53c3e.json deleted file mode 100644 index 62e4a4e2..00000000 --- a/.sqlx/query-13a4bec38c568988dc6d070307f3a2a750b9329059ff23415bd7270babc53c3e.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE orchestrator_jobs\n SET state='dead_letter',\n lease_owner=NULL,\n lease_id=NULL,\n lease_expires_at=NULL,\n last_error=$2,\n updated_at=NOW()\n WHERE lease_id = $1::uuid AND state = 'leased'\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [] - }, - "hash": "13a4bec38c568988dc6d070307f3a2a750b9329059ff23415bd7270babc53c3e" -} diff --git a/.sqlx/query-442e8498eaaa251978202770355c2a154b89307d4cc2884c25b8b7882f538479.json b/.sqlx/query-442e8498eaaa251978202770355c2a154b89307d4cc2884c25b8b7882f538479.json deleted file mode 100644 index 573c50ac..00000000 --- a/.sqlx/query-442e8498eaaa251978202770355c2a154b89307d4cc2884c25b8b7882f538479.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE orchestrator_jobs\n SET state='leased',\n lease_owner=$1,\n lease_id=$2,\n lease_expires_at=$3,\n attempts = COALESCE(attempts, 0),\n updated_at=NOW()\n WHERE id = $4 AND state = 'ready'\n RETURNING lease_id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "lease_id", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Timestamptz", - "Uuid" - ] - }, - "nullable": [ - true - ] - }, - "hash": "442e8498eaaa251978202770355c2a154b89307d4cc2884c25b8b7882f538479" -} diff --git a/.sqlx/query-4ae8a4b8fa4dee0003e1154f2b8d76c719dad5e5185c52771451cf7b31cebf30.json b/.sqlx/query-5777aecc6180c52bcac8b65c5c4358b16cd9a16a857506efad6a208781495d20.json similarity index 80% rename from .sqlx/query-4ae8a4b8fa4dee0003e1154f2b8d76c719dad5e5185c52771451cf7b31cebf30.json rename to .sqlx/query-5777aecc6180c52bcac8b65c5c4358b16cd9a16a857506efad6a208781495d20.json index a94d7c5e..1328dc00 100644 --- a/.sqlx/query-4ae8a4b8fa4dee0003e1154f2b8d76c719dad5e5185c52771451cf7b31cebf30.json +++ b/.sqlx/query-5777aecc6180c52bcac8b65c5c4358b16cd9a16a857506efad6a208781495d20.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO series_scan_state (\n library_id, series_root_path, status, series_title, series_slug, series_year, series_region,\n seeded_at, last_attempt_at, attempts, created_at, updated_at\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), NOW(), 1, NOW(), NOW())\n ON CONFLICT (library_id, series_root_path)\n DO UPDATE SET\n status = CASE\n WHEN series_scan_state.status = 'resolved' THEN series_scan_state.status\n ELSE 'seeded'\n END,\n series_title = COALESCE(EXCLUDED.series_title, series_scan_state.series_title),\n series_slug = COALESCE(EXCLUDED.series_slug, series_scan_state.series_slug),\n series_year = COALESCE(EXCLUDED.series_year, series_scan_state.series_year),\n series_region = COALESCE(EXCLUDED.series_region, series_scan_state.series_region),\n seeded_at = COALESCE(series_scan_state.seeded_at, NOW()),\n last_attempt_at = NOW(),\n attempts = series_scan_state.attempts + 1,\n updated_at = NOW()\n RETURNING library_id, series_root_path, status as \"status: SeriesScanStatus\",\n series_id, series_title, series_slug, series_year, series_region,\n seeded_at, last_attempt_at, attempts,\n resolved_at, failed_at, failure_reason, created_at, updated_at\n ", + "query": "\n INSERT INTO series_scan_state (\n library_id, series_root_path, status, series_title, series_slug, series_year, series_region,\n seeded_at, last_attempt_at, attempts, created_at, updated_at\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), NOW(), 1, NOW(), NOW())\n ON CONFLICT (library_id, series_root_path)\n DO UPDATE SET\n status = CASE\n WHEN series_scan_state.status = 'resolved' THEN series_scan_state.status\n ELSE 'seeded'\n END,\n series_title = COALESCE(EXCLUDED.series_title, series_scan_state.series_title),\n series_slug = COALESCE(EXCLUDED.series_slug, series_scan_state.series_slug),\n series_year = COALESCE(EXCLUDED.series_year, series_scan_state.series_year),\n series_region = COALESCE(EXCLUDED.series_region, series_scan_state.series_region),\n seeded_at = COALESCE(series_scan_state.seeded_at, NOW()),\n last_attempt_at = NOW(),\n attempts = series_scan_state.attempts + 1,\n failed_at = CASE\n WHEN series_scan_state.status = 'resolved'\n THEN series_scan_state.failed_at\n ELSE NULL\n END,\n failure_reason = CASE\n WHEN series_scan_state.status = 'resolved'\n THEN series_scan_state.failure_reason\n ELSE NULL\n END,\n updated_at = NOW()\n RETURNING library_id, series_root_path, status as \"status: SeriesScanStatus\",\n series_id, series_title, series_slug, series_year, series_region,\n seeded_at, last_attempt_at, attempts,\n resolved_at, failed_at, failure_reason, created_at, updated_at\n ", "describe": { "columns": [ { @@ -138,5 +138,5 @@ false ] }, - "hash": "4ae8a4b8fa4dee0003e1154f2b8d76c719dad5e5185c52771451cf7b31cebf30" + "hash": "5777aecc6180c52bcac8b65c5c4358b16cd9a16a857506efad6a208781495d20" } diff --git a/.sqlx/query-7ab16258ea647055c103d95b33f6c6ee9f94f91916925f9209306ae197f0cd12.json b/.sqlx/query-7ab16258ea647055c103d95b33f6c6ee9f94f91916925f9209306ae197f0cd12.json deleted file mode 100644 index f677a6b8..00000000 --- a/.sqlx/query-7ab16258ea647055c103d95b33f6c6ee9f94f91916925f9209306ae197f0cd12.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT id, attempts, payload, correlation_id\n FROM orchestrator_jobs\n WHERE lease_id = $1::uuid AND state = 'leased'\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "attempts", - "type_info": "Int4" - }, - { - "ordinal": 2, - "name": "payload", - "type_info": "Jsonb" - }, - { - "ordinal": 3, - "name": "correlation_id", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - false, - false, - false, - true - ] - }, - "hash": "7ab16258ea647055c103d95b33f6c6ee9f94f91916925f9209306ae197f0cd12" -} diff --git a/.sqlx/query-d68e1ff50b24fe00d12019d95668409578e2fc77dba136a3f1e599d127d43a16.json b/.sqlx/query-d68e1ff50b24fe00d12019d95668409578e2fc77dba136a3f1e599d127d43a16.json deleted file mode 100644 index 86550a51..00000000 --- a/.sqlx/query-d68e1ff50b24fe00d12019d95668409578e2fc77dba136a3f1e599d127d43a16.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO series_scan_state (\n library_id, series_root_path, status,\n series_title, series_slug, series_year, series_region,\n attempts, created_at, updated_at\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, 0, NOW(), NOW())\n ON CONFLICT (library_id, series_root_path)\n DO UPDATE SET\n series_title = COALESCE(EXCLUDED.series_title, series_scan_state.series_title),\n series_slug = COALESCE(EXCLUDED.series_slug, series_scan_state.series_slug),\n series_year = COALESCE(EXCLUDED.series_year, series_scan_state.series_year),\n series_region = COALESCE(EXCLUDED.series_region, series_scan_state.series_region),\n status = CASE\n WHEN series_scan_state.status = 'resolved' THEN series_scan_state.status\n ELSE 'discovered'\n END,\n updated_at = NOW()\n RETURNING library_id, series_root_path, status as \"status: SeriesScanStatus\",\n series_id, series_title, series_slug, series_year, series_region,\n seeded_at, last_attempt_at, attempts,\n resolved_at, failed_at, failure_reason, created_at, updated_at\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "library_id", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "series_root_path", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "status: SeriesScanStatus", - "type_info": { - "Custom": { - "name": "series_scan_status", - "kind": { - "Enum": [ - "discovered", - "seeded", - "resolved", - "failed" - ] - } - } - } - }, - { - "ordinal": 3, - "name": "series_id", - "type_info": "Uuid" - }, - { - "ordinal": 4, - "name": "series_title", - "type_info": "Text" - }, - { - "ordinal": 5, - "name": "series_slug", - "type_info": "Text" - }, - { - "ordinal": 6, - "name": "series_year", - "type_info": "Int2" - }, - { - "ordinal": 7, - "name": "series_region", - "type_info": "Text" - }, - { - "ordinal": 8, - "name": "seeded_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 9, - "name": "last_attempt_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 10, - "name": "attempts", - "type_info": "Int4" - }, - { - "ordinal": 11, - "name": "resolved_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 12, - "name": "failed_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 13, - "name": "failure_reason", - "type_info": "Text" - }, - { - "ordinal": 14, - "name": "created_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 15, - "name": "updated_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - { - "Custom": { - "name": "series_scan_status", - "kind": { - "Enum": [ - "discovered", - "seeded", - "resolved", - "failed" - ] - } - } - }, - "Text", - "Text", - "Int2", - "Text" - ] - }, - "nullable": [ - false, - false, - false, - true, - true, - true, - true, - true, - true, - true, - false, - true, - true, - true, - false, - false - ] - }, - "hash": "d68e1ff50b24fe00d12019d95668409578e2fc77dba136a3f1e599d127d43a16" -} diff --git a/crates/ferrex-core/src/domain/scan/orchestration/dispatcher.rs b/crates/ferrex-core/src/domain/scan/orchestration/dispatcher.rs index 1ede4c82..c3927d0b 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/dispatcher.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/dispatcher.rs @@ -325,6 +325,7 @@ where E: ScanEventBus + Send + Sync + 'static, { enqueuer: PipelineEnqueuer, + queue: Arc, } impl Clone for FollowUpEnqueuer @@ -335,6 +336,7 @@ where fn clone(&self) -> Self { Self { enqueuer: self.enqueuer.clone(), + queue: Arc::clone(&self.queue), } } } @@ -361,9 +363,10 @@ where events: Arc, correlations: CorrelationCache, ) -> Self { - let enqueuer = PipelineEnqueuer::new(queue, events, correlations); + let enqueuer = + PipelineEnqueuer::new(Arc::clone(&queue), events, correlations); - Self { enqueuer } + Self { enqueuer, queue } } async fn enqueue( @@ -413,6 +416,16 @@ where .await .map(|_| ()) } + + async fn release_terminal_series_dependency( + &self, + library_id: crate::types::ids::LibraryId, + series_root_path: &crate::domain::scan::orchestration::context::SeriesRootPath, + ) -> Result { + self.queue + .release_terminal_series_dependency(library_id, series_root_path) + .await + } } #[async_trait] @@ -1569,10 +1582,32 @@ where episode_hierarchy, dependency_key, ); - return self + let status = self .follow_ups .enqueue(parent_correlation, req) .await; + if !matches!(status, DispatchStatus::Success) { + return status; + } + + // Resolution may have released this dependency after + // the state check above but before the deferred row was + // inserted. Let the queue atomically lock/check the + // authoritative series root and promote the matching + // deferred row. A separate state read here would race a + // fresh resolver enrollment. + if let Err(err) = self + .follow_ups + .release_terminal_series_dependency( + job.library_id, + &episode_hierarchy.series_root_path, + ) + .await + { + return classify_media_error(err); + } + + return DispatchStatus::Success; } } } @@ -1593,26 +1628,22 @@ where Err(err) => { let status = classify_media_error(err); if let DispatchStatus::DeadLetter { error } = &status { - let _ = self - .series_coordinator - .record_resolution_failure(job, error.clone()) - .await; if let Err(err) = self .series_coordinator - .release_blocked_episode_dependencies( - &self.follow_ups, - job.library_id, - &job.series_root_path, - ) + .record_resolution_failure(job, error.clone()) .await { - tracing::warn!( - target: "scan::dispatch", - error = %err, - series_root = %job.series_root_path.as_str(), - "failed to release dependency after series resolve dead-letter" - ); + // A resolver failure is not durably terminal until the + // authoritative series-root state records it. Retry a + // persistence failure instead of dead-lettering the job + // while episode work remains blocked behind a + // nonterminal root. + return classify_media_error(err); } + // The durable queue owns failed-series dependency release + // when it applies the lease's dead-letter transition. That + // shared transaction can distinguish this terminal + // generation from a root reopened after `mark_failed`. } return status; } @@ -2500,6 +2531,86 @@ mod tests { } } + #[derive(Clone)] + struct FailurePersistencePause { + failure_persisted: Arc, + resume: Arc, + } + + #[derive(Clone)] + struct FailingSeriesResolver { + states: Arc>, + fail_failure_persistence: bool, + pause_after_failure_persistence: Option, + } + + impl FailingSeriesResolver { + fn new( + states: Arc>, + fail_failure_persistence: bool, + ) -> Self { + Self { + states, + fail_failure_persistence, + pause_after_failure_persistence: None, + } + } + + fn paused_after_failure_persistence( + states: Arc>, + pause: FailurePersistencePause, + ) -> Self { + Self { + states, + fail_failure_persistence: false, + pause_after_failure_persistence: Some(pause), + } + } + } + + #[async_trait] + impl SeriesResolverPort for FailingSeriesResolver { + async fn resolve( + &self, + _job: &SeriesResolveJob, + ) -> Result { + Err(MediaError::InvalidMedia( + "series provider rejected resolver input".into(), + )) + } + + async fn mark_failed( + &self, + library_id: LibraryId, + series_root_path: SeriesRootPath, + reason: String, + ) -> Result<()> { + if self.fail_failure_persistence { + return Err(MediaError::Internal( + "series failure persistence unavailable".into(), + )); + } + + let _ = self + .states + .mark_failed(library_id, series_root_path, reason) + .await?; + if let Some(pause) = &self.pause_after_failure_persistence { + pause.failure_persisted.wait().await; + pause.resume.wait().await; + } + Ok(()) + } + + async fn get_state( + &self, + library_id: LibraryId, + series_root_path: &SeriesRootPath, + ) -> Result> { + self.states.get(library_id, series_root_path).await + } + } + struct StubIndexActor; #[async_trait] @@ -2717,28 +2828,100 @@ mod tests { } } + struct ResolveBeforeEpisodeInsert { + states: Arc>, + library_id: LibraryId, + series_root_path: SeriesRootPath, + series_ref: SeriesRef, + } + + #[derive(Clone)] + struct TerminalReleasePause { + terminal_observed: Arc, + resume: Arc, + } + #[derive(Default)] struct RecordingQueue { enqueued: Mutex>, released_dependencies: Mutex>, + resolve_before_episode_insert: + Mutex>, + series_states: Mutex>>>, + terminal_release_pause: Mutex>, + dependency_operation_order: Mutex>, } impl RecordingQueue { async fn enqueued(&self) -> Vec { self.enqueued.lock().await.clone() } + + async fn observe_series_states( + &self, + states: Arc>, + ) { + *self.series_states.lock().await = Some(states); + } + + async fn resolve_before_episode_insert( + &self, + resolution: ResolveBeforeEpisodeInsert, + ) { + self.observe_series_states(Arc::clone(&resolution.states)) + .await; + *self.resolve_before_episode_insert.lock().await = Some(resolution); + } + + async fn pause_terminal_release_after_observation( + &self, + states: Arc>, + pause: TerminalReleasePause, + ) { + self.observe_series_states(states).await; + *self.terminal_release_pause.lock().await = Some(pause); + } } #[async_trait] impl QueueService for RecordingQueue { async fn enqueue(&self, request: EnqueueRequest) -> Result { request.validate()?; + let resolution = + if matches!(request.payload, JobPayload::EpisodeMatch(_)) { + self.resolve_before_episode_insert.lock().await.take() + } else { + None + }; + let resolver_won = resolution.is_some(); + if let Some(resolution) = resolution { + resolution + .states + .mark_resolved( + resolution.library_id, + resolution.series_root_path, + resolution.series_ref, + ) + .await?; + // Model the resolver's dependency release winning immediately + // before this deferred row becomes durable. + self.dependency_operation_order + .lock() + .await + .push("resolver_release_before_insert"); + } let handle = JobHandle::accepted( JobId::new(), &request.payload, request.priority, ); self.enqueued.lock().await.push(request); + if resolver_won { + self.dependency_operation_order + .lock() + .await + .push("deferred_episode_insert"); + } Ok(handle) } @@ -2812,12 +2995,79 @@ mod tests { library_id: LibraryId, dependency_key: &DependencyKey, ) -> Result { + self.dependency_operation_order + .lock() + .await + .push("post_enqueue_release"); self.released_dependencies .lock() .await .push((library_id, dependency_key.clone())); Ok(0) } + + async fn release_terminal_series_dependency( + &self, + library_id: LibraryId, + series_root_path: &SeriesRootPath, + ) -> Result { + let Some(states) = self.series_states.lock().await.clone() else { + return Ok(0); + }; + let observed_terminal = states + .get(library_id, series_root_path) + .await? + .is_some_and(|state| { + matches!( + state.status, + SeriesScanStatus::Resolved | SeriesScanStatus::Failed + ) + }); + + if observed_terminal { + if let Some(pause) = + self.terminal_release_pause.lock().await.take() + { + self.dependency_operation_order + .lock() + .await + .push("stale_terminal_observed"); + pause.terminal_observed.wait().await; + pause.resume.wait().await; + } + } + + // Model the PostgreSQL operation's state-row lock/recheck after a + // concurrent enrollment wins. The production implementation does + // both this validation and promotion in one statement. + let terminal = states + .get(library_id, series_root_path) + .await? + .is_some_and(|state| { + matches!( + state.status, + SeriesScanStatus::Resolved | SeriesScanStatus::Failed + ) + }); + if !terminal { + self.dependency_operation_order + .lock() + .await + .push("atomic_release_rejected"); + return Ok(0); + } + + let dependency_key = DependencyKey::series_root(series_root_path); + self.dependency_operation_order + .lock() + .await + .push("atomic_post_enqueue_release"); + self.released_dependencies + .lock() + .await + .push((library_id, dependency_key)); + Ok(1) + } } #[derive(Default)] @@ -4322,6 +4572,404 @@ mod tests { assert!(correlations.fetch(&generated_handle.job_id).await.is_none()); } + #[tokio::test] + async fn post_enqueue_recheck_repairs_episode_dependency_lost_wakeup() { + let library_id = + LibraryId(Uuid::from_u128(0x57600000000000000000000000000023)); + let correlation_id = + Uuid::from_u128(0x5760000000000000000000000000c023); + let series_root_path = SeriesRootPath::try_new_under_library_root( + "/library", + "/library/Lost Wakeup Show", + ) + .unwrap(); + let dependency_key = DependencyKey::series_root(&series_root_path); + let series_ref = SeriesRef { + id: SeriesID(Uuid::from_u128(0x5760000000000000000000000000a023)), + slug: Some("lost-wakeup-show".into()), + title: Some("Lost Wakeup Show".into()), + }; + + let states: Arc> = + Arc::new(Box::new(InMemorySeriesScanStateRepository::default())); + let queue = Arc::new(RecordingQueue::default()); + queue + .resolve_before_episode_insert(ResolveBeforeEpisodeInsert { + states: Arc::clone(&states), + library_id, + series_root_path: series_root_path.clone(), + series_ref, + }) + .await; + let events = Arc::new(InProcJobEventBus::new(64)); + let cursors = Arc::new(MemoryCursorRepository::default()); + let series_resolver = + Arc::new(StubSeriesResolver::new(Arc::clone(&states))); + let dispatcher = DefaultJobDispatcher::new( + Arc::clone(&queue), + events, + cursors, + Arc::clone(&states), + series_resolver, + DispatcherActors::new( + noop_folder_actor(library_id), + Arc::new(StubAnalyzeActor), + Arc::new(StubMetadataActor), + Arc::new(StubIndexActor), + Arc::new(StubImageActor), + ), + CorrelationCache::default(), + ); + + let status = dispatcher + .dispatch(&lease_for_payload_with_correlation( + JobPayload::MediaAnalyze(media_analyze_episode_job( + library_id, + series_root_path.clone(), + "Lost Wakeup Show", + "/library/Lost Wakeup Show/Season 1/S01E01.mkv", + 1, + )), + correlation_id, + )) + .await; + assert_eq!(status, DispatchStatus::Success); + + let enqueued = queue.enqueued().await; + assert_eq!(enqueued.len(), 1); + assert!(matches!(enqueued[0].payload, JobPayload::EpisodeMatch(_))); + assert_eq!(enqueued[0].correlation_id, Some(correlation_id)); + assert_eq!(enqueued[0].dependency_key, Some(dependency_key.clone())); + assert_eq!( + queue.released_dependencies.lock().await.as_slice(), + &[(library_id, dependency_key)], + "the atomic post-enqueue repair must repeat the dependency release" + ); + assert_eq!( + queue.dependency_operation_order.lock().await.as_slice(), + &[ + "resolver_release_before_insert", + "deferred_episode_insert", + "atomic_post_enqueue_release", + ], + "the repair release must happen after the losing deferred insert" + ); + assert!( + states + .get(library_id, &series_root_path) + .await + .unwrap() + .is_some_and(|state| state.is_resolved()) + ); + } + + #[tokio::test] + async fn atomic_post_enqueue_repair_rejects_stale_failed_generation() { + let library_id = + LibraryId(Uuid::from_u128(0x57600000000000000000000000000024)); + let correlation_id = + Uuid::from_u128(0x5760000000000000000000000000c024); + let series_root_path = SeriesRootPath::try_new_under_library_root( + "/library", + "/library/Retry Enrollment Show", + ) + .unwrap(); + let states: Arc> = + Arc::new(Box::new(InMemorySeriesScanStateRepository::default())); + states + .mark_failed( + library_id, + series_root_path.clone(), + "historical failure".into(), + ) + .await + .expect("seed historical terminal state"); + + let terminal_observed = Arc::new(tokio::sync::Barrier::new(2)); + let resume = Arc::new(tokio::sync::Barrier::new(2)); + let queue = Arc::new(RecordingQueue::default()); + queue + .pause_terminal_release_after_observation( + Arc::clone(&states), + TerminalReleasePause { + terminal_observed: Arc::clone(&terminal_observed), + resume: Arc::clone(&resume), + }, + ) + .await; + + let events = Arc::new(InProcJobEventBus::new(64)); + let cursors = Arc::new(MemoryCursorRepository::default()); + let series_resolver = + Arc::new(StubSeriesResolver::new(Arc::clone(&states))); + let dispatcher = DefaultJobDispatcher::new( + Arc::clone(&queue), + events, + cursors, + Arc::clone(&states), + series_resolver, + DispatcherActors::new( + noop_folder_actor(library_id), + Arc::new(StubAnalyzeActor), + Arc::new(StubMetadataActor), + Arc::new(StubIndexActor), + Arc::new(StubImageActor), + ), + CorrelationCache::default(), + ); + let dispatch_root = series_root_path.clone(); + let dispatch = tokio::spawn(async move { + dispatcher + .dispatch(&lease_for_payload_with_correlation( + JobPayload::MediaAnalyze(media_analyze_episode_job( + library_id, + dispatch_root, + "Retry Enrollment Show", + "/library/Retry Enrollment Show/Season 1/S01E01.mkv", + 1, + )), + correlation_id, + )) + .await + }); + + // The queue authority has observed the old terminal state but has not + // committed a release. Enroll the retry before allowing its locked + // validation/promotion operation to continue. + terminal_observed.wait().await; + states + .enroll_resolution_generation( + library_id, + series_root_path.clone(), + Some(SeriesHint { + title: "Retry Enrollment Show".into(), + slug: Some("retry-enrollment-show".into()), + year: None, + region: None, + }), + ) + .await + .expect("enroll fresh resolver generation"); + resume.wait().await; + + assert_eq!(dispatch.await.unwrap(), DispatchStatus::Success); + assert!( + queue.released_dependencies.lock().await.is_empty(), + "stale failure must not release the freshly enrolled generation" + ); + assert_eq!( + queue.dependency_operation_order.lock().await.as_slice(), + &["stale_terminal_observed", "atomic_release_rejected"] + ); + assert_eq!( + states + .get(library_id, &series_root_path) + .await + .expect("state lookup") + .expect("retry enrollment exists") + .status, + SeriesScanStatus::Discovered + ); + } + + #[tokio::test] + async fn series_resolve_dead_letter_defers_release_to_queue_transition() { + let library_id = + LibraryId(Uuid::from_u128(0x57600000000000000000000000000025)); + let series_root_path = SeriesRootPath::try_new_under_library_root( + "/library", + "/library/Dead Letter Retry Show", + ) + .unwrap(); + let dependency_key = DependencyKey::series_root(&series_root_path); + let states: Arc> = + Arc::new(Box::new(InMemorySeriesScanStateRepository::default())); + states + .enroll_resolution_generation( + library_id, + series_root_path.clone(), + Some(series_hint("Dead Letter Retry Show")), + ) + .await + .expect("enroll initial resolver generation"); + + let queue = Arc::new(RecordingQueue::default()); + queue.observe_series_states(Arc::clone(&states)).await; + queue + .enqueue( + EnqueueRequest::new( + JobPriority::P0, + JobPayload::EpisodeMatch(episode_match_job( + library_id, + series_root_path.clone(), + "Dead Letter Retry Show", + "/library/Dead Letter Retry Show/Season 1/S01E01.mkv", + 1, + )), + ) + .with_dependency(dependency_key.clone()), + ) + .await + .expect("enqueue deferred episode match"); + + let failure_persisted = Arc::new(tokio::sync::Barrier::new(2)); + let resume = Arc::new(tokio::sync::Barrier::new(2)); + let series_resolver = + Arc::new(FailingSeriesResolver::paused_after_failure_persistence( + Arc::clone(&states), + FailurePersistencePause { + failure_persisted: Arc::clone(&failure_persisted), + resume: Arc::clone(&resume), + }, + )); + let dispatcher = DefaultJobDispatcher::new( + Arc::clone(&queue), + Arc::new(InProcJobEventBus::new(64)), + Arc::new(MemoryCursorRepository::default()), + Arc::clone(&states), + series_resolver, + DispatcherActors::new( + noop_folder_actor(library_id), + Arc::new(StubAnalyzeActor), + Arc::new(StubMetadataActor), + Arc::new(StubIndexActor), + Arc::new(StubImageActor), + ), + CorrelationCache::default(), + ); + let dispatch_root = series_root_path.clone(); + let dispatch = tokio::spawn(async move { + dispatcher + .dispatch(&lease_for_payload(JobPayload::SeriesResolve( + SeriesResolveJob { + library_id, + series_root_path: dispatch_root, + hint: Some(series_hint("Dead Letter Retry Show")), + folder_name: "Dead Letter Retry Show".into(), + scan_reason: ScanReason::BulkSeed, + }, + ))) + .await + }); + + // The old resolver has durably marked its generation Failed, but has + // not started dependency release. Enroll the retry in that window. + failure_persisted.wait().await; + assert_eq!( + states + .get(library_id, &series_root_path) + .await + .expect("state lookup") + .expect("failed state exists") + .status, + SeriesScanStatus::Failed + ); + states + .enroll_resolution_generation( + library_id, + series_root_path.clone(), + Some(series_hint("Dead Letter Retry Show")), + ) + .await + .expect("enroll retry generation"); + resume.wait().await; + + assert_eq!( + dispatch.await.expect("dispatcher task"), + DispatchStatus::DeadLetter { + error: "series provider rejected resolver input".into(), + } + ); + assert!( + queue.released_dependencies.lock().await.is_empty(), + "the failed generation must not release dependencies for its retry" + ); + assert!(queue.dependency_operation_order.lock().await.is_empty()); + let enqueued = queue.enqueued().await; + assert_eq!(enqueued.len(), 1); + assert_eq!(enqueued[0].dependency_key, Some(dependency_key)); + assert_eq!( + states + .get(library_id, &series_root_path) + .await + .expect("state lookup") + .expect("retry state exists") + .status, + SeriesScanStatus::Discovered + ); + } + + #[tokio::test] + async fn series_resolve_failure_persistence_error_retries_without_release() + { + let library_id = + LibraryId(Uuid::from_u128(0x57600000000000000000000000000026)); + let series_root_path = SeriesRootPath::try_new_under_library_root( + "/library", + "/library/Persistence Failure Show", + ) + .unwrap(); + let states: Arc> = + Arc::new(Box::new(InMemorySeriesScanStateRepository::default())); + states + .enroll_resolution_generation( + library_id, + series_root_path.clone(), + Some(series_hint("Persistence Failure Show")), + ) + .await + .expect("enroll resolver generation"); + + let queue = Arc::new(RecordingQueue::default()); + queue.observe_series_states(Arc::clone(&states)).await; + let dispatcher = DefaultJobDispatcher::new( + Arc::clone(&queue), + Arc::new(InProcJobEventBus::new(64)), + Arc::new(MemoryCursorRepository::default()), + Arc::clone(&states), + Arc::new(FailingSeriesResolver::new(Arc::clone(&states), true)), + DispatcherActors::new( + noop_folder_actor(library_id), + Arc::new(StubAnalyzeActor), + Arc::new(StubMetadataActor), + Arc::new(StubIndexActor), + Arc::new(StubImageActor), + ), + CorrelationCache::default(), + ); + + let status = dispatcher + .dispatch(&lease_for_payload(JobPayload::SeriesResolve( + SeriesResolveJob { + library_id, + series_root_path: series_root_path.clone(), + hint: Some(series_hint("Persistence Failure Show")), + folder_name: "Persistence Failure Show".into(), + scan_reason: ScanReason::BulkSeed, + }, + ))) + .await; + + assert_eq!( + status, + DispatchStatus::Retry { + error: "series failure persistence unavailable".into(), + }, + "the durable-state error must supersede the resolver dead-letter" + ); + assert!(queue.released_dependencies.lock().await.is_empty()); + assert!(queue.dependency_operation_order.lock().await.is_empty()); + assert_eq!( + states + .get(library_id, &series_root_path) + .await + .expect("state lookup") + .expect("nonterminal state remains") + .status, + SeriesScanStatus::Discovered + ); + } + #[tokio::test] async fn episode_match_dependency_key_defers_until_release() { let Some(pool) = postgres_pool_or_skip( diff --git a/crates/ferrex-core/src/domain/scan/orchestration/job.rs b/crates/ferrex-core/src/domain/scan/orchestration/job.rs index faea2ed5..7e58a70c 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/job.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/job.rs @@ -273,6 +273,7 @@ pub struct JobRecord { pub priority: JobPriority, pub state: JobState, pub attempts: u16, + /// Eligibility time while Ready; stable lease-start marker while Leased. pub available_at: DateTime, pub lease_owner: Option, pub lease_expires_at: Option>, diff --git a/crates/ferrex-core/src/domain/scan/orchestration/mod.rs b/crates/ferrex-core/src/domain/scan/orchestration/mod.rs index d149f48f..ac705927 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/mod.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/mod.rs @@ -108,9 +108,9 @@ pub use maintenance::{ }; pub use persistence::{PostgresCursorRepository, PostgresQueueService}; pub use queue::{ - DurableJobState, FailOutcome, LeaseExpiryScanner, QueueInstrumentation, - QueueService, QueueSnapshot, QueueSnapshotEntry, QueueTransitionOutcome, - ReadyQueueCount, + DurableJobState, DurableSeriesIdentity, FailOutcome, LeaseExpiryScanner, + QueueInstrumentation, QueueService, QueueSnapshot, QueueSnapshotEntry, + QueueTransitionOutcome, ReadyQueueCount, }; pub use runtime::{ InProcJobEventBus, JobEventStream, LibraryActorHandle, diff --git a/crates/ferrex-core/src/domain/scan/orchestration/persistence.rs b/crates/ferrex-core/src/domain/scan/orchestration/persistence.rs index 076322ab..f04eacc1 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/persistence.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/persistence.rs @@ -5,22 +5,25 @@ use crate::error::Result; use async_trait::async_trait; use chrono::Utc; use serde_json::from_value; -use sqlx::{Executor, PgPool}; +use sqlx::{Executor, PgPool, Row}; use std::fmt; use std::hash::{DefaultHasher, Hash, Hasher}; use tracing::{debug, info, trace, warn}; use crate::domain::scan::orchestration::{ config::RetryConfig, + context::{SeasonLink, SeriesLink, SeriesRootPath}, events::stable_path_key, job::{ - DependencyKey, EnqueueRequest, JobHandle, JobId, JobKind, JobPayload, - JobPriority, JobState, ScanReason, TranscriptExtractJob, + AnalyzeScanHierarchy, DependencyKey, EnqueueRequest, JobHandle, JobId, + JobKind, JobPayload, JobPriority, JobState, ScanReason, + TranscriptExtractJob, }, lease::{DequeueRequest, JobLease, LeaseId, LeaseRenewal}, queue::{ - DurableJobState, FailOutcome, LeaseExpiryScanner, QueueInstrumentation, - QueueService, QueueSnapshot, QueueTransitionOutcome, ReadyQueueCount, + DurableJobState, DurableSeriesIdentity, FailOutcome, + LeaseExpiryScanner, QueueInstrumentation, QueueService, QueueSnapshot, + QueueTransitionOutcome, ReadyQueueCount, }, scan_cursor::{ScanCursor, ScanCursorId, ScanCursorRepository}, }; @@ -47,6 +50,78 @@ fn bounded_queue_error(error: Option) -> Option { error.map(|message| message.chars().take(2048).collect()) } +fn resolved_series_id(series: &SeriesLink) -> Option { + match series { + SeriesLink::Resolved(reference) => Some(reference.id), + SeriesLink::Hint(_) => None, + } +} + +fn season_number(season: &SeasonLink) -> Option { + match season { + SeasonLink::Resolved(reference) => reference.number, + SeasonLink::Number(number) => Some(*number), + } +} + +fn hierarchy_series_identity( + hierarchy: &AnalyzeScanHierarchy, +) -> Option { + match hierarchy { + AnalyzeScanHierarchy::Movie(_) => None, + AnalyzeScanHierarchy::Series(hierarchy) => { + Some(DurableSeriesIdentity { + series_root_path: hierarchy.series_root_path.clone(), + series_id: resolved_series_id(&hierarchy.series), + season_number: None, + }) + } + AnalyzeScanHierarchy::Season(hierarchy) => { + Some(DurableSeriesIdentity { + series_root_path: hierarchy.series_root_path.clone(), + series_id: resolved_series_id(&hierarchy.series), + season_number: season_number(&hierarchy.season), + }) + } + AnalyzeScanHierarchy::Episode(hierarchy) => { + Some(DurableSeriesIdentity { + series_root_path: hierarchy.series_root_path.clone(), + series_id: resolved_series_id(&hierarchy.series), + season_number: season_number(&hierarchy.season), + }) + } + } +} + +fn durable_series_identity( + payload: &JobPayload, +) -> Option { + match payload { + JobPayload::SeriesResolve(job) => Some(DurableSeriesIdentity { + series_root_path: job.series_root_path.clone(), + series_id: None, + season_number: None, + }), + JobPayload::MediaAnalyze(job) => { + hierarchy_series_identity(&job.hierarchy) + } + JobPayload::MetadataEnrich(job) => { + hierarchy_series_identity(&job.hierarchy) + } + JobPayload::IndexUpsert(job) => { + hierarchy_series_identity(&job.hierarchy) + } + JobPayload::EpisodeMatch(job) => Some(DurableSeriesIdentity { + series_root_path: job.hierarchy.series_root_path.clone(), + series_id: resolved_series_id(&job.hierarchy.series), + season_number: season_number(&job.hierarchy.season), + }), + JobPayload::FolderScan(_) + | JobPayload::ImageFetch(_) + | JobPayload::TranscriptExtract(_) => None, + } +} + async fn mark_transcript_queue_status( pool: &PgPool, job: &TranscriptExtractJob, @@ -630,6 +705,108 @@ impl PostgresQueueService { Ok(counts) } + + /// Atomically release deferred EpisodeMatch rows for one terminal series + /// root. + /// + /// The row lock makes the terminal check linearizable with + /// `enroll_resolution_generation`: if a retry enrollment wins first this + /// observes nonterminal state, and if this wins first its release belongs to + /// the prior terminal generation. + pub async fn release_terminal_series_dependency( + &self, + library_id: LibraryId, + series_root_path: &SeriesRootPath, + ) -> Result { + let updated = sqlx::query( + r#" + WITH terminal_series AS MATERIALIZED ( + SELECT library_id, series_root_path + FROM series_scan_state + WHERE library_id = $2 + AND series_root_path = $3 + AND status IN ('resolved', 'failed') + FOR UPDATE + ) + UPDATE orchestrator_jobs AS job + SET state = 'ready', + dependency_key = NULL, + available_at = NOW(), + updated_at = NOW() + FROM terminal_series AS series + WHERE job.kind = $1 + AND job.state = 'deferred' + AND job.library_id = series.library_id + AND job.dependency_key = + 'series_root:' || series.series_root_path + "#, + ) + .bind(JobKind::EpisodeMatch as i16) + .bind(library_id.0) + .bind(series_root_path.as_str()) + .execute(&self.pool) + .await + .map_err(|error| { + MediaError::Internal(format!( + "terminal series dependency release failed: {error}" + )) + })?; + + Ok(updated.rows_affected()) + } + + /// Promote EpisodeMatch rows stranded behind a series dependency that has + /// already reached a terminal durable state. + /// + /// This closes the recovery side of the lost-wakeup window where + /// SeriesResolve releases a dependency immediately before the episode row + /// is inserted as deferred. Restricting the update to EpisodeMatch jobs and + /// an exact library/root dependency match avoids releasing unrelated work. + /// Locking terminal state rows serializes repair with fresh resolver + /// enrollment, so a historical failed snapshot cannot release the new + /// generation after enrollment has begun. + pub async fn repair_terminal_series_dependencies(&self) -> Result { + let updated = sqlx::query( + r#" + WITH terminal_series AS MATERIALIZED ( + SELECT series.library_id, series.series_root_path + FROM series_scan_state AS series + WHERE series.status IN ('resolved', 'failed') + AND EXISTS ( + SELECT 1 + FROM orchestrator_jobs AS blocked + WHERE blocked.kind = $1 + AND blocked.state = 'deferred' + AND blocked.library_id = series.library_id + AND blocked.dependency_key = + 'series_root:' || series.series_root_path + ) + FOR UPDATE + ) + UPDATE orchestrator_jobs AS job + SET state = 'ready', + dependency_key = NULL, + available_at = NOW(), + updated_at = NOW() + FROM terminal_series AS series + WHERE job.kind = $1 + AND job.state = 'deferred' + AND job.library_id = series.library_id + AND job.dependency_key = + 'series_root:' || series.series_root_path + "#, + ) + .bind(JobKind::EpisodeMatch as i16) + .execute(&self.pool) + .await + .map_err(|e| { + MediaError::Internal(format!( + "terminal series dependency repair failed: {e}" + )) + })?; + + Ok(updated.rows_affected()) + } } #[async_trait] @@ -680,8 +857,100 @@ impl QueueInstrumentation for PostgresQueueService { } } +/// Finalize authoritative series failure and promote only the matching +/// deferred episode work inside the caller's queue transition transaction. +/// +/// Callers must already hold the series-state row lock before locking the +/// SeriesResolve job row. `record_failure` is used by retry-budget exhaustion, +/// whose dispatcher never persisted `Failed`; direct dead-letter dispatch has +/// already done so and passes `false`. +async fn finalize_terminal_series_failure( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + library_id: LibraryId, + series_root_path: &SeriesRootPath, + failure_reason: Option<&str>, + record_failure: bool, +) -> Result { + if record_failure { + sqlx::query( + r#" + UPDATE series_scan_state + SET status = 'failed'::series_scan_status, + failed_at = NOW(), + failure_reason = $3, + updated_at = NOW() + WHERE library_id = $1 + AND series_root_path = $2 + AND status <> 'resolved'::series_scan_status + "#, + ) + .bind(library_id.0) + .bind(series_root_path.as_str()) + .bind(failure_reason.unwrap_or("series resolution retry limit reached")) + .execute(&mut **tx) + .await + .map_err(|e| { + MediaError::Internal(format!( + "terminal series-state failure update failed: {e}" + )) + })?; + } + + let dependency_key = DependencyKey::series_root(series_root_path); + let released = sqlx::query( + r#" + UPDATE orchestrator_jobs + SET state = 'ready', + dependency_key = NULL, + available_at = NOW(), + updated_at = NOW() + WHERE kind = $1 + AND state = 'deferred' + AND library_id = $2 + AND dependency_key = $3 + "#, + ) + .bind(JobKind::EpisodeMatch as i16) + .bind(library_id.0) + .bind(dependency_key.as_str()) + .execute(&mut **tx) + .await + .map_err(|e| { + MediaError::Internal(format!( + "terminal series dependency release failed: {e}" + )) + })?; + + Ok(released.rows_affected()) +} + +fn is_superseding_exhausted_series_generation( + status: &str, + series_updated_at: chrono::DateTime, + lease_started_at: chrono::DateTime, +) -> bool { + status == "discovered" && series_updated_at > lease_started_at +} + #[async_trait] impl QueueService for PostgresQueueService { + async fn release_terminal_series_dependency( + &self, + library_id: LibraryId, + series_root_path: &SeriesRootPath, + ) -> Result { + Self::release_terminal_series_dependency( + self, + library_id, + series_root_path, + ) + .await + } + + async fn repair_terminal_series_dependencies(&self) -> Result { + Self::repair_terminal_series_dependencies(self).await + } + async fn ready_counts_grouped(&self) -> Result> { Self::ready_counts_grouped(self).await } @@ -1537,7 +1806,7 @@ impl QueueService for PostgresQueueService { payload: serde_json::Value, priority: i16, attempts: i32, - available_at: chrono::DateTime, + _available_at: chrono::DateTime, dedupe_key: String, dependency_key: Option, correlation_id: Option, @@ -1581,7 +1850,7 @@ impl QueueService for PostgresQueueService { payload: row.payload, priority: row.priority, attempts: row.attempts, - available_at: row.available_at, + _available_at: row.available_at, dedupe_key: row.dedupe_key, dependency_key: row.dependency_key, correlation_id: row.correlation_id, @@ -1610,7 +1879,7 @@ impl QueueService for PostgresQueueService { payload: row.payload, priority: row.priority, attempts: row.attempts, - available_at: row.available_at, + _available_at: row.available_at, dedupe_key: row.dedupe_key, dependency_key: row.dependency_key, correlation_id: row.correlation_id, @@ -1628,8 +1897,10 @@ impl QueueService for PostgresQueueService { let lease_id = LeaseId::new(); let expires_at = chrono::Utc::now() + request.lease_ttl; - // Update to leased state - let updated = sqlx::query!( + // While a row is leased, `available_at` is its stable lease-start + // marker. Lease renewal deliberately leaves it unchanged; every path + // that returns a row to Ready assigns a new availability timestamp. + let updated = sqlx::query( r#" UPDATE orchestrator_jobs SET state='leased', @@ -1637,26 +1908,33 @@ impl QueueService for PostgresQueueService { lease_id=$2, lease_expires_at=$3, attempts = COALESCE(attempts, 0), + available_at=NOW(), updated_at=NOW() WHERE id = $4 AND state = 'ready' - RETURNING lease_id + RETURNING available_at "#, - request.worker_id, - lease_id.0, - expires_at, - row.id ) + .bind(&request.worker_id) + .bind(lease_id.0) + .bind(expires_at) + .bind(row.id) .fetch_optional(&mut *tx) .await .map_err(|e| { MediaError::Internal(format!("dequeue update->leased failed: {e}")) })?; - if updated.is_none() { + let Some(updated) = updated else { // Raced with state change; treat as empty drop(tx); return Ok(None); - } + }; + let lease_started_at: chrono::DateTime = + updated.try_get("available_at").map_err(|e| { + MediaError::Internal(format!( + "dequeue lease-start timestamp decode failed: {e}" + )) + })?; // Build JobRecord from the selected row and new lease fields let payload: JobPayload = @@ -1685,7 +1963,7 @@ impl QueueService for PostgresQueueService { priority, state: JobState::Leased, attempts: row.attempts.max(0) as u16, - available_at: row.available_at, + available_at: lease_started_at, lease_owner: Some(request.worker_id.clone()), lease_expires_at: Some(expires_at), backoff_until: None, @@ -1856,10 +2134,81 @@ impl QueueService for PostgresQueueService { retryable: bool, error: Option, ) -> Result { + // A retryable SeriesResolve can cross its attempt limit here without + // passing through the dispatcher's explicit dead-letter path. Read its + // identity first so a terminal transition can follow the same global + // series-state -> queue-row lock order as dependency release and the + // direct dead-letter transition below. + let terminal_series_identity = if let Some(candidate) = sqlx::query( + r#" + SELECT attempts, payload + FROM orchestrator_jobs + WHERE lease_id = $1::uuid AND state = 'leased' + "#, + ) + .bind(lease_id.0) + .fetch_optional(&self.pool) + .await + .map_err(|e| { + MediaError::Internal(format!("fail identity read failed: {e}")) + })? { + let attempts: i32 = candidate.try_get("attempts").map_err(|e| { + MediaError::Internal(format!( + "fail identity attempts decode failed: {e}" + )) + })?; + let payload: serde_json::Value = + candidate.try_get("payload").map_err(|e| { + MediaError::Internal(format!( + "fail identity payload decode failed: {e}" + )) + })?; + let payload: JobPayload = + serde_json::from_value(payload).map_err(|e| { + MediaError::Internal(format!( + "fail identity payload deserialize failed: {e}" + )) + })?; + let terminal = !retryable + || attempts >= i32::from(self.retry_config.max_attempts); + match payload { + JobPayload::SeriesResolve(job) if terminal => { + Some((job.library_id, job.series_root_path)) + } + _ => None, + } + } else { + return Ok(FailOutcome::Missing); + }; + let mut tx = self.pool.begin().await.map_err(|e| { MediaError::Internal(format!("begin fail tx failed: {e}")) })?; + let locked_series_state = + if let Some((library_id, root)) = &terminal_series_identity { + sqlx::query_as::<_, (String, chrono::DateTime)>( + r#" + SELECT status::text, updated_at + FROM series_scan_state + WHERE library_id = $1 + AND series_root_path = $2 + FOR UPDATE + "#, + ) + .bind(library_id.0) + .bind(root.as_str()) + .fetch_optional(&mut *tx) + .await + .map_err(|e| { + MediaError::Internal(format!( + "fail terminal series-state lock failed: {e}" + )) + })? + } else { + None + }; + // Lock the row and get current attempts let row = sqlx::query!( r#" @@ -1895,6 +2244,85 @@ impl QueueService for PostgresQueueService { row_id )) })?; + if let Some((library_id, root)) = &terminal_series_identity { + let identity_matches = matches!( + &payload, + JobPayload::SeriesResolve(job) + if job.library_id == *library_id + && job.series_root_path == *root + ); + if !identity_matches { + return Err(MediaError::Internal( + "terminal fail SeriesResolve identity changed while leased" + .into(), + )); + } + let Some((locked_status, series_updated_at)) = + locked_series_state.as_ref() + else { + return Err(MediaError::Internal(format!( + "terminal fail missing authoritative series state for {}", + root.as_str() + ))); + }; + let lease_started_at: chrono::DateTime = sqlx::query_scalar( + "SELECT available_at FROM orchestrator_jobs WHERE id = $1", + ) + .bind(row_id) + .fetch_one(&mut *tx) + .await + .map_err(|e| { + MediaError::Internal(format!( + "fail lease-start timestamp lookup failed: {e}" + )) + })?; + if is_superseding_exhausted_series_generation( + locked_status, + *series_updated_at, + lease_started_at, + ) { + let requeued = sqlx::query( + r#" + UPDATE orchestrator_jobs + SET state = 'ready', + attempts = 0, + available_at = NOW(), + lease_owner = NULL, + lease_id = NULL, + lease_expires_at = NULL, + dependency_key = NULL, + last_error = NULL, + updated_at = NOW() + WHERE id = $1 + AND lease_id = $2::uuid + AND state = 'leased' + "#, + ) + .bind(row_id) + .bind(lease_id.0) + .execute(&mut *tx) + .await + .map_err(|e| { + MediaError::Internal(format!( + "superseded exhausted SeriesResolve requeue failed: {e}" + )) + })?; + if requeued.rows_affected() != 1 { + return Ok(FailOutcome::Missing); + } + tx.commit().await.map_err(|e| { + MediaError::Internal(format!( + "superseded exhausted SeriesResolve commit failed: {e}" + )) + })?; + info!( + job = %row_id, + series_root = %root.as_str(), + "requeued exhausted SeriesResolve superseded by a fresh root enrollment" + ); + return Ok(FailOutcome::Requeued); + } + } let transcript_job = transcript_payload(&payload); let mut library_under_pressure = @@ -2019,6 +2447,17 @@ impl QueueService for PostgresQueueService { )) })?; + if let Some((library_id, root)) = &terminal_series_identity { + finalize_terminal_series_failure( + &mut tx, + *library_id, + root, + error.as_deref(), + true, + ) + .await?; + } + tx.commit().await.map_err(|e| { MediaError::Internal(format!("fail tx commit failed: {e}")) })?; @@ -2067,93 +2506,272 @@ impl QueueService for PostgresQueueService { lease_id: LeaseId, error: Option, ) -> Result { - let row = sqlx::query!( + // Read only enough identity to establish the global lock order for a + // SeriesResolve transition. The lease is revalidated under lock below; + // this first read never authorizes a state change by itself. + let candidate_payload = sqlx::query_scalar::<_, serde_json::Value>( r#" - SELECT id, attempts, payload, correlation_id + SELECT payload FROM orchestrator_jobs WHERE lease_id = $1::uuid AND state = 'leased' "#, - lease_id.0, ) + .bind(lease_id.0) .fetch_optional(&self.pool) .await .map_err(|e| { - MediaError::Internal(format!("dead_letter select failed: {e}")) + MediaError::Internal(format!( + "dead_letter identity read failed: {e}" + )) })?; - let res = sqlx::query!( - r#" - UPDATE orchestrator_jobs - SET state='dead_letter', - lease_owner=NULL, - lease_id=NULL, - lease_expires_at=NULL, - last_error=$2, - updated_at=NOW() - WHERE lease_id = $1::uuid AND state = 'leased' - "#, - lease_id.0, - error - ) - .execute(&self.pool) - .await - .map_err(|e| { - MediaError::Internal(format!("dead_letter update failed: {e}")) + let Some(candidate_payload) = candidate_payload else { + return Ok(QueueTransitionOutcome::Missing); + }; + let candidate_job: JobPayload = + serde_json::from_value(candidate_payload).map_err(|e| { + MediaError::Internal(format!( + "dead_letter payload decode failed: {e}" + )) + })?; + let series_identity = match &candidate_job { + JobPayload::SeriesResolve(job) => { + Some((job.library_id, job.series_root_path.clone())) + } + _ => None, + }; + + let mut tx = self.pool.begin().await.map_err(|e| { + MediaError::Internal(format!("begin dead_letter tx failed: {e}")) })?; - if res.rows_affected() > 0 { - if let Some(row) = row { - let row_id = row.id; - let attempts = row.attempts; - let row_correlation_id = row.correlation_id; - let row_payload = row.payload; - if let Ok(payload) = - serde_json::from_value::(row_payload) - && let Some(job) = transcript_payload(&payload) - && let Err(err) = mark_transcript_queue_status( - &self.pool, - &job, - TranscriptProcessingState::Failed, - attempts.saturating_add(1), - self.retry_config.max_attempts, - error.clone(), - None, - row_correlation_id, - ) - .await - { - warn!(error = %err, job = %row_id, "failed to mark transcript dead-letter status"); - } - } - warn!("job with lease {:?} moved to dead_letter", lease_id.0); - Ok(QueueTransitionOutcome::Applied) - } else { - Ok(QueueTransitionOutcome::Missing) - } - } + // Series-state operations consistently lock the authoritative root + // before queue rows. This matches enrollment and dependency repair and + // avoids a queue->series inversion with the housekeeper. + let series_status = + if let Some((library_id, series_root_path)) = &series_identity { + sqlx::query_scalar::<_, String>( + r#" + SELECT status::text + FROM series_scan_state + WHERE library_id = $1 + AND series_root_path = $2 + FOR UPDATE + "#, + ) + .bind(library_id.0) + .bind(series_root_path.as_str()) + .fetch_optional(&mut *tx) + .await + .map_err(|e| { + MediaError::Internal(format!( + "dead_letter series-state lock failed: {e}" + )) + })? + } else { + None + }; - async fn cancel_job(&self, job_id: JobId) -> Result<()> { - // Delete only non-leased jobs; leased jobs require different handling. - let row = sqlx::query!( + let row = sqlx::query( r#" - SELECT attempts, payload, correlation_id + SELECT id, attempts, payload, correlation_id FROM orchestrator_jobs - WHERE id = $1 AND state IN ('ready','deferred') + WHERE lease_id = $1::uuid AND state = 'leased' + FOR UPDATE "#, - job_id.0, ) - .fetch_optional(&self.pool) + .bind(lease_id.0) + .fetch_optional(&mut *tx) .await .map_err(|e| { - MediaError::Internal(format!("cancel_job select failed: {e}")) + MediaError::Internal(format!("dead_letter lease lock failed: {e}")) })?; - let res = sqlx::query!( - r#" - DELETE FROM orchestrator_jobs - WHERE id = $1 AND state IN ('ready','deferred') - "#, - job_id.0 + let Some(row) = row else { + return Ok(QueueTransitionOutcome::Missing); + }; + let row_id: uuid::Uuid = row.try_get("id").map_err(|e| { + MediaError::Internal(format!("dead_letter id decode failed: {e}")) + })?; + let attempts: i32 = row.try_get("attempts").map_err(|e| { + MediaError::Internal(format!( + "dead_letter attempts decode failed: {e}" + )) + })?; + let row_payload: serde_json::Value = + row.try_get("payload").map_err(|e| { + MediaError::Internal(format!( + "dead_letter payload row decode failed: {e}" + )) + })?; + let row_correlation_id: Option = + row.try_get("correlation_id").map_err(|e| { + MediaError::Internal(format!( + "dead_letter correlation decode failed: {e}" + )) + })?; + let locked_job: JobPayload = + serde_json::from_value(row_payload.clone()).map_err(|e| { + MediaError::Internal(format!( + "dead_letter locked payload decode failed: {e}" + )) + })?; + + if let Some((library_id, series_root_path)) = &series_identity { + let identity_matches = matches!( + &locked_job, + JobPayload::SeriesResolve(job) + if job.library_id == *library_id + && job.series_root_path == *series_root_path + ); + if !identity_matches { + return Err(MediaError::Internal( + "dead_letter leased SeriesResolve identity changed while leased" + .into(), + )); + } + if series_status.is_none() { + return Err(MediaError::Internal(format!( + "dead_letter missing authoritative series state for {}", + series_root_path.as_str() + ))); + } + + if matches!(series_status.as_deref(), Some("discovered" | "seeded")) + { + let requeued = sqlx::query( + r#" + UPDATE orchestrator_jobs + SET state = 'ready', + attempts = 0, + available_at = NOW(), + lease_owner = NULL, + lease_id = NULL, + lease_expires_at = NULL, + dependency_key = NULL, + last_error = NULL, + updated_at = NOW() + WHERE id = $1 + AND lease_id = $2::uuid + AND state = 'leased' + "#, + ) + .bind(row_id) + .bind(lease_id.0) + .execute(&mut *tx) + .await + .map_err(|e| { + MediaError::Internal(format!( + "superseded SeriesResolve requeue failed: {e}" + )) + })?; + if requeued.rows_affected() != 1 { + return Ok(QueueTransitionOutcome::Missing); + } + + tx.commit().await.map_err(|e| { + MediaError::Internal(format!( + "superseded SeriesResolve commit failed: {e}" + )) + })?; + info!( + job = %row_id, + series_root = %series_root_path.as_str(), + "requeued SeriesResolve superseded by a fresh root enrollment" + ); + return Ok(QueueTransitionOutcome::Requeued); + } + } + + let transitioned = sqlx::query( + r#" + UPDATE orchestrator_jobs + SET state = 'dead_letter', + lease_owner = NULL, + lease_id = NULL, + lease_expires_at = NULL, + last_error = $3, + updated_at = NOW() + WHERE id = $1 + AND lease_id = $2::uuid + AND state = 'leased' + "#, + ) + .bind(row_id) + .bind(lease_id.0) + .bind(error.clone()) + .execute(&mut *tx) + .await + .map_err(|e| { + MediaError::Internal(format!("dead_letter update failed: {e}")) + })?; + if transitioned.rows_affected() != 1 { + return Ok(QueueTransitionOutcome::Missing); + } + + // A failed SeriesResolve releases only its own deferred EpisodeMatch + // rows, and only while the same locked root remains terminal. Legacy + // rows missed before this transaction are still covered by the + // housekeeper's terminal dependency repair sweep. + if let Some((library_id, series_root_path)) = &series_identity + && matches!(series_status.as_deref(), Some("failed" | "resolved")) + { + finalize_terminal_series_failure( + &mut tx, + *library_id, + series_root_path, + error.as_deref(), + false, + ) + .await?; + } + + tx.commit().await.map_err(|e| { + MediaError::Internal(format!("dead_letter tx commit failed: {e}")) + })?; + + if let Some(job) = transcript_payload(&locked_job) + && let Err(err) = mark_transcript_queue_status( + &self.pool, + &job, + TranscriptProcessingState::Failed, + attempts.saturating_add(1), + self.retry_config.max_attempts, + error.clone(), + None, + row_correlation_id, + ) + .await + { + warn!(error = %err, job = %row_id, "failed to mark transcript dead-letter status"); + } + + warn!("job with lease {:?} moved to dead_letter", lease_id.0); + Ok(QueueTransitionOutcome::Applied) + } + + async fn cancel_job(&self, job_id: JobId) -> Result<()> { + // Delete only non-leased jobs; leased jobs require different handling. + let row = sqlx::query!( + r#" + SELECT attempts, payload, correlation_id + FROM orchestrator_jobs + WHERE id = $1 AND state IN ('ready','deferred') + "#, + job_id.0, + ) + .fetch_optional(&self.pool) + .await + .map_err(|e| { + MediaError::Internal(format!("cancel_job select failed: {e}")) + })?; + + let res = sqlx::query!( + r#" + DELETE FROM orchestrator_jobs + WHERE id = $1 AND state IN ('ready','deferred') + "#, + job_id.0 ) .execute(&self.pool) .await @@ -2260,6 +2878,7 @@ impl QueueService for PostgresQueueService { kind, media_id, indexing_change, + series_identity: durable_series_identity(&payload), state: Self::parse_state(&row.state)?, attempts: row.attempts.max(0) as u16, dedupe_key: row.dedupe_key, @@ -2309,9 +2928,16 @@ impl QueueService for PostgresQueueService { mod tests { use super::*; use crate::domain::scan::orchestration::{ - context::{FolderScanContext, MovieFolderScanContext, MovieRootPath}, - job::FolderScanJob, + context::{ + FolderScanContext, MovieFolderScanContext, MovieRootPath, + SeriesHint, SeriesRootPath, + }, + job::{FolderScanJob, SeriesResolveJob}, lease::QueueSelector, + series_state::{ + PostgresSeriesScanStateRepository, SeriesScanStateRepository, + SeriesScanStatus, + }, }; use sqlx::Row; use std::collections::HashMap; @@ -2336,6 +2962,100 @@ mod tests { }) } + fn series_resolve_payload( + library_id: LibraryId, + series_root_path: SeriesRootPath, + title: &str, + ) -> JobPayload { + JobPayload::SeriesResolve(SeriesResolveJob { + library_id, + series_root_path, + hint: None, + folder_name: title.into(), + scan_reason: ScanReason::UserRequested, + }) + } + + async fn seed_deferred_episode( + pool: &PgPool, + library_id: LibraryId, + series_root_path: &SeriesRootPath, + suffix: &str, + ) -> uuid::Uuid { + let job_id = uuid::Uuid::now_v7(); + let dependency_key = DependencyKey::series_root(series_root_path); + sqlx::query( + r#" + INSERT INTO orchestrator_jobs ( + id, library_id, kind, payload, priority, state, attempts, + available_at, dedupe_key, dependency_key, + created_at, updated_at + ) VALUES ( + $1, $2, $3, '{}'::jsonb, $4, 'deferred', 0, + NOW(), $5, $6, NOW(), NOW() + ) + "#, + ) + .bind(job_id) + .bind(library_id.0) + .bind(JobKind::EpisodeMatch as i16) + .bind(JobPriority::P1 as i16) + .bind(format!("series-terminal-race:{library_id}:{suffix}")) + .bind(dependency_key.as_str()) + .execute(pool) + .await + .expect("deferred episode row seeded"); + job_id + } + + async fn persisted_job_state(pool: &PgPool, job_id: uuid::Uuid) -> String { + sqlx::query_scalar::<_, String>( + "SELECT state FROM orchestrator_jobs WHERE id = $1", + ) + .bind(job_id) + .fetch_one(pool) + .await + .expect("job state loaded") + } + + #[test] + fn exhausted_series_supersession_requires_newer_discovered_generation() { + let lease_started_at = Utc::now(); + let older = lease_started_at - chrono::Duration::milliseconds(1); + let newer = lease_started_at + chrono::Duration::milliseconds(1); + + assert!(is_superseding_exhausted_series_generation( + "discovered", + newer, + lease_started_at, + )); + assert!(!is_superseding_exhausted_series_generation( + "discovered", + lease_started_at, + lease_started_at, + )); + assert!(!is_superseding_exhausted_series_generation( + "discovered", + older, + lease_started_at, + )); + assert!(!is_superseding_exhausted_series_generation( + "seeded", + newer, + lease_started_at, + )); + assert!(!is_superseding_exhausted_series_generation( + "failed", + newer, + lease_started_at, + )); + assert!(!is_superseding_exhausted_series_generation( + "resolved", + newer, + lease_started_at, + )); + } + async fn seed_library( pool: &PgPool, library_id: LibraryId, @@ -2359,6 +3079,30 @@ mod tests { Ok(()) } + async fn postgres_pool_or_skip(test_name: &str) -> Option { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => { + eprintln!("skipping {test_name}: DATABASE_URL not set"); + return None; + } + }; + let pool = match PgPool::connect(&database_url).await { + Ok(pool) => pool, + Err(err) => { + eprintln!( + "skipping {test_name}: failed to connect to DATABASE_URL ({err})" + ); + return None; + } + }; + if let Err(err) = crate::MIGRATOR.run(&pool).await { + eprintln!("skipping {test_name}: migrations failed ({err})"); + return None; + } + Some(pool) + } + async fn active_dedupe_count( pool: &PgPool, dedupe_key: &str, @@ -2407,6 +3151,1064 @@ mod tests { Ok(states) } + #[tokio::test] + async fn series_dead_letter_requeues_superseding_enrollment() { + let Some(pool) = postgres_pool_or_skip( + "series_dead_letter_requeues_superseding_enrollment", + ) + .await + else { + return; + }; + let queue = PostgresQueueService::new(pool.clone()) + .await + .expect("queue service initializes"); + let states = PostgresSeriesScanStateRepository::new(pool.clone()); + let library_id = LibraryId::new(); + let library_root = + format!("/queue-series-terminal-race/{}", library_id.as_uuid()); + seed_library(&pool, library_id, &library_root) + .await + .expect("library seeded"); + let series_root_path = + SeriesRootPath::try_new(format!("{library_root}/Superseding Show")) + .expect("series root"); + let other_root_path = + SeriesRootPath::try_new(format!("{library_root}/Other Show")) + .expect("other series root"); + states + .enroll_resolution_generation( + library_id, + series_root_path.clone(), + None, + ) + .await + .expect("initial generation enrolled"); + states + .mark_seeded(library_id, series_root_path.clone(), None) + .await + .expect("resolver attempt seeded"); + + let old_correlation = uuid::Uuid::now_v7(); + let mut old_request = EnqueueRequest::new( + JobPriority::P1, + series_resolve_payload( + library_id, + series_root_path.clone(), + "Superseding Show", + ), + ); + old_request.correlation_id = Some(old_correlation); + let old_job = queue + .enqueue(old_request) + .await + .expect("old resolver enqueued"); + let old_lease = queue + .dequeue(DequeueRequest { + kind: JobKind::SeriesResolve, + worker_id: "superseded-series-worker".into(), + lease_ttl: chrono::Duration::seconds(30), + selector: Some(QueueSelector { + library_id, + priority: JobPriority::P1, + }), + }) + .await + .expect("resolver dequeue") + .expect("resolver leased"); + assert_eq!(old_lease.job.id, old_job.job_id); + + states + .mark_failed( + library_id, + series_root_path.clone(), + "old resolver terminal failure".into(), + ) + .await + .expect("old failure persisted"); + let blocked_job = seed_deferred_episode( + &pool, + library_id, + &series_root_path, + "superseded-blocked", + ) + .await; + let unrelated_job = seed_deferred_episode( + &pool, + library_id, + &other_root_path, + "superseded-unrelated", + ) + .await; + + // Pause point: Failed is durable, but the old lease has not yet been + // dead-lettered. Root discovery reopens and merges its correlation onto + // that still-active row. + states + .enroll_resolution_generation( + library_id, + series_root_path.clone(), + None, + ) + .await + .expect("fresh generation enrolled"); + let new_correlation = uuid::Uuid::now_v7(); + let mut retry_request = EnqueueRequest::new( + JobPriority::P0, + series_resolve_payload( + library_id, + series_root_path.clone(), + "Superseding Show", + ), + ); + retry_request.correlation_id = Some(new_correlation); + let merged = queue + .enqueue(retry_request) + .await + .expect("retry enqueue merges while old lease is active"); + assert_eq!(merged.merged_into, Some(old_job.job_id)); + + assert_eq!( + queue + .dead_letter_with_outcome( + old_lease.lease_id, + Some("old resolver terminal failure".into()), + ) + .await + .expect("superseded dead-letter transition"), + QueueTransitionOutcome::Requeued + ); + let state = states + .get(library_id, &series_root_path) + .await + .expect("state lookup") + .expect("series state exists"); + assert_eq!(state.status, SeriesScanStatus::Discovered); + + let row = sqlx::query( + r#" + SELECT state, attempts, lease_id, last_error + FROM orchestrator_jobs + WHERE id = $1 + "#, + ) + .bind(old_job.job_id.0) + .fetch_one(&pool) + .await + .expect("requeued resolver loaded"); + assert_eq!(row.get::("state"), "ready"); + assert_eq!(row.get::("attempts"), 0); + assert!(row.get::, _>("lease_id").is_none()); + assert!(row.get::, _>("last_error").is_none()); + assert_eq!(persisted_job_state(&pool, blocked_job).await, "deferred"); + assert_eq!(persisted_job_state(&pool, unrelated_job).await, "deferred"); + + let retry_jobs = queue + .durable_job_states(new_correlation, &[]) + .await + .expect("retry enrollment reconciled"); + assert_eq!(retry_jobs.len(), 1); + assert_eq!(retry_jobs[0].job_id, old_job.job_id); + assert_eq!(retry_jobs[0].state, JobState::Ready); + + let retry_lease = queue + .dequeue(DequeueRequest { + kind: JobKind::SeriesResolve, + worker_id: "fresh-series-worker".into(), + lease_ttl: chrono::Duration::seconds(30), + selector: Some(QueueSelector { + library_id, + priority: JobPriority::P1, + }), + }) + .await + .expect("requeued resolver dequeue") + .expect("requeued resolver is immediately schedulable"); + assert_eq!(retry_lease.job.id, old_job.job_id); + states + .mark_seeded(library_id, series_root_path.clone(), None) + .await + .expect("fresh attempt seeded"); + states + .mark_failed( + library_id, + series_root_path, + "fresh resolver terminal failure".into(), + ) + .await + .expect("fresh terminal failure persisted"); + assert_eq!( + queue + .dead_letter_with_outcome( + retry_lease.lease_id, + Some("fresh resolver terminal failure".into()), + ) + .await + .expect("fresh terminal result applied"), + QueueTransitionOutcome::Applied + ); + assert_eq!(persisted_job_state(&pool, blocked_job).await, "ready"); + assert_eq!(persisted_job_state(&pool, unrelated_job).await, "deferred"); + } + + #[tokio::test] + async fn series_dead_letter_terminalizes_and_releases_exact_root() { + let Some(pool) = postgres_pool_or_skip( + "series_dead_letter_terminalizes_and_releases_exact_root", + ) + .await + else { + return; + }; + let queue = PostgresQueueService::new(pool.clone()) + .await + .expect("queue service initializes"); + let states = PostgresSeriesScanStateRepository::new(pool.clone()); + let library_id = LibraryId::new(); + let library_root = + format!("/queue-series-terminal-release/{}", library_id.as_uuid()); + seed_library(&pool, library_id, &library_root) + .await + .expect("library seeded"); + let series_root_path = + SeriesRootPath::try_new(format!("{library_root}/Terminal Show")) + .expect("series root"); + let other_root_path = SeriesRootPath::try_new(format!( + "{library_root}/Other Terminal Show" + )) + .expect("other series root"); + states + .enroll_resolution_generation( + library_id, + series_root_path.clone(), + None, + ) + .await + .expect("generation enrolled"); + states + .mark_seeded(library_id, series_root_path.clone(), None) + .await + .expect("resolver seeded"); + + let old_job = queue + .enqueue(EnqueueRequest::new( + JobPriority::P1, + series_resolve_payload( + library_id, + series_root_path.clone(), + "Terminal Show", + ), + )) + .await + .expect("resolver enqueued"); + let old_lease = queue + .dequeue(DequeueRequest { + kind: JobKind::SeriesResolve, + worker_id: "terminal-series-worker".into(), + lease_ttl: chrono::Duration::seconds(30), + selector: Some(QueueSelector { + library_id, + priority: JobPriority::P1, + }), + }) + .await + .expect("resolver dequeue") + .expect("resolver leased"); + states + .mark_failed( + library_id, + series_root_path.clone(), + "terminal provider response".into(), + ) + .await + .expect("terminal failure persisted"); + let blocked_job = seed_deferred_episode( + &pool, + library_id, + &series_root_path, + "terminal-blocked", + ) + .await; + let unrelated_job = seed_deferred_episode( + &pool, + library_id, + &other_root_path, + "terminal-unrelated", + ) + .await; + + assert_eq!( + queue + .dead_letter_with_outcome( + old_lease.lease_id, + Some("terminal provider response".into()), + ) + .await + .expect("terminal dead-letter transition"), + QueueTransitionOutcome::Applied + ); + assert_eq!( + persisted_job_state(&pool, old_job.job_id.0).await, + "dead_letter" + ); + assert_eq!(persisted_job_state(&pool, blocked_job).await, "ready"); + assert_eq!(persisted_job_state(&pool, unrelated_job).await, "deferred"); + + // If terminalization wins the race, the later claim observes no active + // resolver and creates a genuinely fresh durable job generation. + states + .enroll_resolution_generation( + library_id, + series_root_path.clone(), + None, + ) + .await + .expect("post-terminal generation enrolled"); + let new_correlation = uuid::Uuid::now_v7(); + let mut fresh_request = EnqueueRequest::new( + JobPriority::P1, + series_resolve_payload( + library_id, + series_root_path, + "Terminal Show", + ), + ); + fresh_request.correlation_id = Some(new_correlation); + let fresh_job = queue + .enqueue(fresh_request) + .await + .expect("fresh resolver enqueued"); + assert!(fresh_job.accepted); + assert_ne!(fresh_job.job_id, old_job.job_id); + let enrolled = queue + .durable_job_states(new_correlation, &[]) + .await + .expect("fresh enrollment reconciled"); + assert_eq!(enrolled.len(), 1); + assert_eq!(enrolled[0].job_id, fresh_job.job_id); + } + + #[tokio::test] + async fn exhausted_series_retry_preserves_fresh_enrollment_then_terminalizes() + { + let Some(pool) = postgres_pool_or_skip( + "exhausted_series_retry_preserves_fresh_enrollment_then_terminalizes", + ) + .await + else { + return; + }; + let retry_config = RetryConfig { + max_attempts: 1, + ..RetryConfig::default() + }; + let queue = + PostgresQueueService::new_with_retry(pool.clone(), retry_config) + .await + .expect("queue service initializes"); + let states = PostgresSeriesScanStateRepository::new(pool.clone()); + let library_id = LibraryId::new(); + let library_root = + format!("/queue-series-exhausted/{}", library_id.as_uuid()); + seed_library(&pool, library_id, &library_root) + .await + .expect("library seeded"); + let series_root_path = + SeriesRootPath::try_new(format!("{library_root}/Exhausted Show")) + .expect("series root"); + let other_root_path = SeriesRootPath::try_new(format!( + "{library_root}/Other Exhausted Show" + )) + .expect("other root"); + states + .enroll_resolution_generation( + library_id, + series_root_path.clone(), + None, + ) + .await + .expect("generation enrolled"); + states + .mark_seeded(library_id, series_root_path.clone(), None) + .await + .expect("attempt seeded"); + + let resolver = queue + .enqueue(EnqueueRequest::new( + JobPriority::P1, + series_resolve_payload( + library_id, + series_root_path.clone(), + "Exhausted Show", + ), + )) + .await + .expect("resolver enqueued"); + let lease = queue + .dequeue(DequeueRequest { + kind: JobKind::SeriesResolve, + worker_id: "exhausted-series-worker".into(), + lease_ttl: chrono::Duration::seconds(30), + selector: Some(QueueSelector { + library_id, + priority: JobPriority::P1, + }), + }) + .await + .expect("resolver dequeue") + .expect("resolver leased"); + let lease_started_at = lease.job.available_at; + sqlx::query("UPDATE orchestrator_jobs SET attempts = $1 WHERE id = $2") + .bind(i32::from(retry_config.max_attempts)) + .bind(resolver.job_id.0) + .execute(&pool) + .await + .expect("attempt limit seeded"); + let blocked_job = seed_deferred_episode( + &pool, + library_id, + &series_root_path, + "exhausted-blocked", + ) + .await; + let unrelated_job = seed_deferred_episode( + &pool, + library_id, + &other_root_path, + "exhausted-unrelated", + ) + .await; + + // Pause between fresh root enrollment and its queue enqueue. Enrollment + // must demote the running Seeded generation to Discovered so the old + // max-attempt transition can recognize that newer durable claim. + let enrolled = states + .enroll_resolution_generation( + library_id, + series_root_path.clone(), + None, + ) + .await + .expect("fresh root generation enrolled before enqueue"); + assert_eq!(enrolled.status, SeriesScanStatus::Discovered); + assert!(enrolled.updated_at > lease_started_at); + + // Renewal updates orchestrator_jobs.updated_at through the table + // trigger. The supersession signal must remain tied to the stable + // available_at lease-start marker instead. + sqlx::query("SELECT pg_sleep(0.005)") + .fetch_one(&pool) + .await + .expect("database clock advances before renewal"); + let renewed = queue + .renew(LeaseRenewal { + lease_id: lease.lease_id, + worker_id: "exhausted-series-worker".into(), + extend_by: chrono::Duration::seconds(10), + }) + .await + .expect("old resolver lease renews after enrollment"); + assert_eq!(renewed.job.available_at, lease_started_at); + assert!(renewed.job.updated_at > enrolled.updated_at); + assert_eq!( + queue + .fail_with_outcome( + renewed.lease_id, + true, + Some("provider unavailable through retry limit".into()), + ) + .await + .expect("superseded exhausted retry requeues"), + FailOutcome::Requeued + ); + let row = sqlx::query( + "SELECT state, attempts, lease_id, last_error FROM orchestrator_jobs WHERE id = $1", + ) + .bind(resolver.job_id.0) + .fetch_one(&pool) + .await + .expect("requeued resolver loaded"); + assert_eq!(row.get::("state"), "ready"); + assert_eq!(row.get::("attempts"), 0); + assert!(row.get::, _>("lease_id").is_none()); + assert!(row.get::, _>("last_error").is_none()); + assert_eq!(persisted_job_state(&pool, blocked_job).await, "deferred"); + assert_eq!(persisted_job_state(&pool, unrelated_job).await, "deferred"); + assert_eq!( + states + .get(library_id, &series_root_path) + .await + .expect("state lookup") + .expect("series state exists") + .status, + SeriesScanStatus::Discovered + ); + + let fresh_correlation = uuid::Uuid::now_v7(); + let mut fresh_request = EnqueueRequest::new( + JobPriority::P0, + series_resolve_payload( + library_id, + series_root_path.clone(), + "Exhausted Show", + ), + ); + fresh_request.correlation_id = Some(fresh_correlation); + let merged = queue + .enqueue(fresh_request) + .await + .expect("fresh correlation merges onto runnable resolver"); + assert_eq!(merged.merged_into, Some(resolver.job_id)); + let fresh_jobs = queue + .durable_job_states(fresh_correlation, &[]) + .await + .expect("fresh correlation enrollment loaded"); + assert_eq!(fresh_jobs.len(), 1); + assert_eq!(fresh_jobs[0].job_id, resolver.job_id); + assert_eq!(fresh_jobs[0].state, JobState::Ready); + + let fresh_lease = queue + .dequeue(DequeueRequest { + kind: JobKind::SeriesResolve, + worker_id: "fresh-exhausted-series-worker".into(), + lease_ttl: chrono::Duration::seconds(30), + selector: Some(QueueSelector { + library_id, + priority: JobPriority::P0, + }), + }) + .await + .expect("fresh resolver dequeue") + .expect("fresh resolver is immediately schedulable"); + assert_eq!(fresh_lease.job.id, resolver.job_id); + states + .mark_seeded(library_id, series_root_path.clone(), None) + .await + .expect("fresh attempt seeded"); + sqlx::query("UPDATE orchestrator_jobs SET attempts = $1 WHERE id = $2") + .bind(i32::from(retry_config.max_attempts)) + .bind(resolver.job_id.0) + .execute(&pool) + .await + .expect("fresh attempt limit seeded"); + assert_eq!( + queue + .fail_with_outcome( + fresh_lease.lease_id, + true, + Some("provider unavailable through retry limit".into()), + ) + .await + .expect("unsuperseded exhausted retry terminalizes"), + FailOutcome::Terminal { + state: JobState::DeadLetter + } + ); + let state = states + .get(library_id, &series_root_path) + .await + .expect("state lookup") + .expect("series state exists"); + assert_eq!(state.status, SeriesScanStatus::Failed); + assert_eq!( + state.failure_reason.as_deref(), + Some("provider unavailable through retry limit") + ); + assert_eq!( + persisted_job_state(&pool, resolver.job_id.0).await, + "dead_letter" + ); + assert_eq!(persisted_job_state(&pool, blocked_job).await, "ready"); + assert_eq!(persisted_job_state(&pool, unrelated_job).await, "deferred"); + + // A resolver can exhaust its budget before the handler ever calls + // mark_seeded. Its initial Discovered state predates the lease and must + // not be mistaken for a superseding generation. + let unseeded_root_path = SeriesRootPath::try_new(format!( + "{library_root}/Unseeded Exhausted Show" + )) + .expect("unseeded series root"); + let unseeded_state = states + .enroll_resolution_generation( + library_id, + unseeded_root_path.clone(), + None, + ) + .await + .expect("unseeded generation enrolled"); + let unseeded_resolver = queue + .enqueue(EnqueueRequest::new( + JobPriority::P1, + series_resolve_payload( + library_id, + unseeded_root_path.clone(), + "Unseeded Exhausted Show", + ), + )) + .await + .expect("unseeded resolver enqueued"); + let unseeded_lease = queue + .dequeue(DequeueRequest { + kind: JobKind::SeriesResolve, + worker_id: "unseeded-exhausted-series-worker".into(), + lease_ttl: chrono::Duration::seconds(30), + selector: Some(QueueSelector { + library_id, + priority: JobPriority::P1, + }), + }) + .await + .expect("unseeded resolver dequeue") + .expect("unseeded resolver leased"); + assert!(unseeded_state.updated_at <= unseeded_lease.job.available_at); + sqlx::query("UPDATE orchestrator_jobs SET attempts = $1 WHERE id = $2") + .bind(i32::from(retry_config.max_attempts)) + .bind(unseeded_resolver.job_id.0) + .execute(&pool) + .await + .expect("unseeded attempt limit seeded"); + let unseeded_blocked_job = seed_deferred_episode( + &pool, + library_id, + &unseeded_root_path, + "unseeded-exhausted-blocked", + ) + .await; + assert_eq!( + queue + .fail_with_outcome( + unseeded_lease.lease_id, + true, + Some("failed before mark_seeded".into()), + ) + .await + .expect("initial discovered generation terminalizes"), + FailOutcome::Terminal { + state: JobState::DeadLetter + } + ); + assert_eq!( + states + .get(library_id, &unseeded_root_path) + .await + .expect("unseeded state lookup") + .expect("unseeded state exists") + .status, + SeriesScanStatus::Failed + ); + assert_eq!( + persisted_job_state(&pool, unseeded_resolver.job_id.0).await, + "dead_letter" + ); + assert_eq!( + persisted_job_state(&pool, unseeded_blocked_job).await, + "ready" + ); + + // Episode observation after leasing is not a new root generation. It + // must leave the Seeded timestamp untouched so an exhausted resolver + // terminalizes instead of being spuriously requeued. + let observed_root_path = SeriesRootPath::try_new(format!( + "{library_root}/Observed Exhausted Show" + )) + .expect("observed series root"); + let original_hint = SeriesHint { + title: "Observed Exhausted Show".into(), + slug: Some("observed-exhausted-show".into()), + year: Some(2024), + region: Some("US".into()), + }; + let observed_seeded = states + .mark_seeded( + library_id, + observed_root_path.clone(), + Some(original_hint.clone()), + ) + .await + .expect("observed generation seeded"); + let observed_resolver = queue + .enqueue(EnqueueRequest::new( + JobPriority::P1, + series_resolve_payload( + library_id, + observed_root_path.clone(), + "Observed Exhausted Show", + ), + )) + .await + .expect("observed resolver enqueued"); + let observed_lease = queue + .dequeue(DequeueRequest { + kind: JobKind::SeriesResolve, + worker_id: "observed-exhausted-series-worker".into(), + lease_ttl: chrono::Duration::seconds(30), + selector: Some(QueueSelector { + library_id, + priority: JobPriority::P1, + }), + }) + .await + .expect("observed resolver dequeue") + .expect("observed resolver leased"); + assert!(observed_seeded.updated_at <= observed_lease.job.available_at); + sqlx::query("UPDATE orchestrator_jobs SET attempts = $1 WHERE id = $2") + .bind(i32::from(retry_config.max_attempts)) + .bind(observed_resolver.job_id.0) + .execute(&pool) + .await + .expect("observed attempt limit seeded"); + let observed = states + .mark_discovered( + library_id, + observed_root_path.clone(), + Some(SeriesHint { + title: "Episode Observation".into(), + slug: None, + year: Some(2026), + region: None, + }), + ) + .await + .expect("episode observes leased resolver"); + assert_eq!(observed.status, SeriesScanStatus::Seeded); + assert_eq!(observed.hint.as_ref(), Some(&original_hint)); + assert_eq!(observed.updated_at, observed_seeded.updated_at); + let observed_blocked_job = seed_deferred_episode( + &pool, + library_id, + &observed_root_path, + "observed-exhausted-blocked", + ) + .await; + assert_eq!( + queue + .fail_with_outcome( + observed_lease.lease_id, + true, + Some("episode observation is not enrollment".into()), + ) + .await + .expect("observed exhausted resolver terminalizes"), + FailOutcome::Terminal { + state: JobState::DeadLetter + } + ); + assert_eq!( + states + .get(library_id, &observed_root_path) + .await + .expect("observed state lookup") + .expect("observed state exists") + .status, + SeriesScanStatus::Failed + ); + assert_eq!( + persisted_job_state(&pool, observed_resolver.job_id.0).await, + "dead_letter" + ); + assert_eq!( + persisted_job_state(&pool, observed_blocked_job).await, + "ready" + ); + } + + #[tokio::test] + async fn series_dead_letter_without_authoritative_state_fails_closed() { + let Some(pool) = postgres_pool_or_skip( + "series_dead_letter_without_authoritative_state_fails_closed", + ) + .await + else { + return; + }; + let queue = PostgresQueueService::new(pool.clone()) + .await + .expect("queue service initializes"); + let library_id = LibraryId::new(); + let library_root = + format!("/queue-series-missing-state/{}", library_id.as_uuid()); + seed_library(&pool, library_id, &library_root) + .await + .expect("library seeded"); + let series_root_path = SeriesRootPath::try_new(format!( + "{library_root}/Missing State Show" + )) + .expect("series root"); + let resolver = queue + .enqueue(EnqueueRequest::new( + JobPriority::P1, + series_resolve_payload( + library_id, + series_root_path, + "Missing State Show", + ), + )) + .await + .expect("resolver enqueued"); + let lease = queue + .dequeue(DequeueRequest { + kind: JobKind::SeriesResolve, + worker_id: "missing-state-series-worker".into(), + lease_ttl: chrono::Duration::seconds(30), + selector: Some(QueueSelector { + library_id, + priority: JobPriority::P1, + }), + }) + .await + .expect("resolver dequeue") + .expect("resolver leased"); + + let err = queue + .dead_letter_with_outcome( + lease.lease_id, + Some("terminal provider response".into()), + ) + .await + .expect_err("missing authoritative state must fail closed"); + assert!( + err.to_string() + .contains("missing authoritative series state") + ); + assert_eq!( + persisted_job_state(&pool, resolver.job_id.0).await, + "leased", + "the failed transaction must preserve the durable lease for retry" + ); + } + + #[tokio::test] + async fn terminal_series_dependency_repair_is_scoped_and_idempotent() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => { + eprintln!("skipping: DATABASE_URL not set"); + return; + } + }; + + let pool = match PgPool::connect(&database_url).await { + Ok(pool) => pool, + Err(err) => { + eprintln!( + "skipping: failed to connect to DATABASE_URL ({err})" + ); + return; + } + }; + + if let Err(err) = crate::MIGRATOR.run(&pool).await { + eprintln!("skipping: migrations failed ({err})"); + return; + } + + let queue = PostgresQueueService::new(pool.clone()) + .await + .expect("queue service initializes"); + let library_id = LibraryId::new(); + let library_root = + format!("/queue-series-dependency-repair/{}", library_id.as_uuid()); + seed_library(&pool, library_id, &library_root) + .await + .expect("library seeded"); + + let resolved_root = format!("{library_root}/Resolved Series"); + let failed_root = format!("{library_root}/Failed Series"); + let pending_root = format!("{library_root}/Pending Series"); + for (root, status) in [ + (&resolved_root, "resolved"), + (&failed_root, "failed"), + (&pending_root, "seeded"), + ] { + sqlx::query( + r#" + INSERT INTO series_scan_state ( + library_id, series_root_path, status + ) VALUES ($1, $2, ($3::text)::series_scan_status) + "#, + ) + .bind(library_id.0) + .bind(root) + .bind(status) + .execute(&pool) + .await + .expect("series state seeded"); + } + + let resolved_key = format!("series_root:{resolved_root}"); + let failed_key = format!("series_root:{failed_root}"); + let pending_key = format!("series_root:{pending_root}"); + let missing_key = format!("series_root:{library_root}/Missing Series"); + let cases = [ + ( + "resolved-episode", + JobKind::EpisodeMatch as i16, + "deferred", + Some(resolved_key.as_str()), + ), + ( + "failed-episode", + JobKind::EpisodeMatch as i16, + "deferred", + Some(failed_key.as_str()), + ), + ( + "pending-episode", + JobKind::EpisodeMatch as i16, + "deferred", + Some(pending_key.as_str()), + ), + ( + "unmatched-episode", + JobKind::EpisodeMatch as i16, + "deferred", + Some(missing_key.as_str()), + ), + ( + "resolved-wrong-kind", + JobKind::MetadataEnrich as i16, + "deferred", + Some(resolved_key.as_str()), + ), + ( + "already-ready-episode", + JobKind::EpisodeMatch as i16, + "ready", + None, + ), + ]; + for (suffix, kind, state, dependency_key) in cases { + sqlx::query( + r#" + INSERT INTO orchestrator_jobs ( + id, library_id, kind, payload, priority, state, + dedupe_key, dependency_key, available_at, + created_at, updated_at + ) VALUES ( + $1, $2, $3, $4, $5, $6, + $7, $8, NOW() + INTERVAL '1 day', NOW(), NOW() + ) + "#, + ) + .bind(uuid::Uuid::now_v7()) + .bind(library_id.0) + .bind(kind) + .bind(serde_json::json!({})) + .bind(JobPriority::P1 as i16) + .bind(state) + .bind(format!("series-dependency-repair:{library_id}:{suffix}")) + .bind(dependency_key) + .execute(&pool) + .await + .expect("queue row seeded"); + } + + assert_eq!( + queue + .release_terminal_series_dependency( + library_id, + &SeriesRootPath::try_new(pending_root.clone()).unwrap(), + ) + .await + .expect("nonterminal scoped release succeeds"), + 0, + "a nonterminal root must not release deferred episode work" + ); + let resolved_root_path = + SeriesRootPath::try_new(resolved_root.clone()).unwrap(); + assert_eq!( + queue + .release_terminal_series_dependency( + library_id, + &resolved_root_path, + ) + .await + .expect("terminal scoped release succeeds"), + 1, + "the scoped operation releases only its exact terminal root" + ); + assert_eq!( + queue + .release_terminal_series_dependency( + library_id, + &resolved_root_path, + ) + .await + .expect("repeated terminal scoped release succeeds"), + 0, + "the scoped operation is idempotent" + ); + + assert_eq!( + queue + .repair_terminal_series_dependencies() + .await + .expect("dependency repair succeeds"), + 1, + "the sweep repairs the remaining failed root only" + ); + assert_eq!( + queue + .repair_terminal_series_dependencies() + .await + .expect("repeated dependency repair succeeds"), + 0, + "repeating the repair must be idempotent" + ); + + let rows = sqlx::query( + r#" + SELECT dedupe_key, state, dependency_key + FROM orchestrator_jobs + WHERE library_id = $1 + AND dedupe_key LIKE 'series-dependency-repair:%' + "#, + ) + .bind(library_id.0) + .fetch_all(&pool) + .await + .expect("repaired rows loaded"); + let states = rows + .into_iter() + .map(|row| { + let dedupe_key: String = row.get("dedupe_key"); + let suffix = dedupe_key + .rsplit(':') + .next() + .expect("dedupe suffix") + .to_string(); + let state: String = row.get("state"); + let dependency_key: Option = row.get("dependency_key"); + (suffix, (state, dependency_key)) + }) + .collect::>(); + + for suffix in ["resolved-episode", "failed-episode"] { + assert_eq!( + states.get(suffix), + Some(&("ready".to_string(), None)), + "{suffix} should be released" + ); + } + for suffix in [ + "pending-episode", + "unmatched-episode", + "resolved-wrong-kind", + ] { + assert_eq!( + states.get(suffix).map(|(state, _)| state.as_str()), + Some("deferred"), + "{suffix} must remain deferred" + ); + } + assert_eq!( + states + .get("already-ready-episode") + .map(|(state, _)| state.as_str()), + Some("ready") + ); + } + #[tokio::test] async fn enqueue_reuses_ready_deferred_and_leased_dedupe_rows() { let database_url = match std::env::var("DATABASE_URL") { diff --git a/crates/ferrex-core/src/domain/scan/orchestration/queue.rs b/crates/ferrex-core/src/domain/scan/orchestration/queue.rs index ab98017d..f4c29b78 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/queue.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/queue.rs @@ -9,11 +9,12 @@ use uuid::Uuid; use crate::error::Result; use super::{ + context::SeriesRootPath, job::{EnqueueRequest, JobHandle, JobId, JobKind, JobPriority, JobState}, lease::{DequeueRequest, JobLease, LeaseRenewal}, }; use crate::domain::scan::actors::index::IndexingChange; -use crate::types::{LibraryId, MediaID}; +use crate::types::{LibraryId, MediaID, SeriesID}; /// Aggregated schedulable state grouped by queue dimensions. /// @@ -46,6 +47,10 @@ pub struct DurableJobState { pub media_id: Option, /// Exact catalog mutation semantics captured in an index-upsert payload. pub indexing_change: Option, + /// Compact series hierarchy extracted while decoding the durable payload. + /// Keeping only the identity needed by projections avoids retaining the + /// full (and potentially large) job payload in reconciliation snapshots. + pub series_identity: Option, pub state: JobState, pub attempts: u16, pub dedupe_key: String, @@ -56,6 +61,17 @@ pub struct DurableJobState { pub updated_at: DateTime, } +/// Minimal durable series hierarchy needed to rebuild a dropped bundle event. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DurableSeriesIdentity { + /// Stable series folder represented by the durable payload hierarchy. + pub series_root_path: SeriesRootPath, + /// Catalog identity when the payload hierarchy had resolved the series. + pub series_id: Option, + /// Logical season number carried by season or episode hierarchy. + pub season_number: Option, +} + impl DurableJobState { pub fn is_terminal(&self) -> bool { matches!( @@ -77,6 +93,9 @@ impl DurableJobState { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum QueueTransitionOutcome { Applied, + /// The requested terminal transition was superseded by newer durable work + /// and the same job row was made runnable again instead. + Requeued, Missing, } @@ -88,7 +107,12 @@ pub enum QueueTransitionOutcome { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum FailOutcome { RetryScheduled, - Terminal { state: JobState }, + /// A newer durable generation superseded terminalization and the same job + /// row is immediately runnable again. + Requeued, + Terminal { + state: JobState, + }, Missing, } @@ -153,6 +177,14 @@ pub trait QueueService: Send + Sync { ) -> Result<()>; /// Dead-letter a lease and report whether the durable transition applied. + /// + /// Implementations that coordinate terminal job state with other durable + /// state may return [`QueueTransitionOutcome::Requeued`] when a newer + /// generation supersedes this terminal result. Callers must treat that as + /// nonterminal retry work and must not publish a dead-letter event. + /// PostgreSQL-backed SeriesResolve implementations also own the exact-root + /// dependency release in this transition so failed series state, queue + /// terminality, and episode promotion share one linearization boundary. async fn dead_letter_with_outcome( &self, lease_id: super::lease::LeaseId, @@ -187,6 +219,32 @@ pub trait QueueService: Send + Sync { dependency_key: &super::job::DependencyKey, ) -> Result; + /// Atomically promote deferred EpisodeMatch rows for one series root only + /// when that root is still terminal in authoritative series state. + /// + /// Durable implementations must serialize this check with fresh resolver + /// enrollment; callers must not perform a separate terminal-state read. + async fn release_terminal_series_dependency( + &self, + _library_id: crate::types::LibraryId, + _series_root_path: &super::context::SeriesRootPath, + ) -> Result { + Ok(0) + } + + /// Repair deferred episode jobs whose series dependency is already + /// terminal in durable state. + /// + /// Series resolution and episode enrollment are independent durable + /// transitions. If resolution wins immediately before the deferred + /// EpisodeMatch row is inserted, the one-shot dependency release cannot + /// observe that row. Durable queue implementations should atomically + /// promote those stranded rows back to ready. The default keeps + /// non-durable queue implementations compatible. + async fn repair_terminal_series_dependencies(&self) -> Result { + Ok(0) + } + /// Fetch grouped ready and active-lease counts for scheduler reconciliation. /// /// Queue implementations without durable startup state can keep the default diff --git a/crates/ferrex-core/src/domain/scan/orchestration/runtime/event_bus.rs b/crates/ferrex-core/src/domain/scan/orchestration/runtime/event_bus.rs index 4941172b..350cb71f 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/runtime/event_bus.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/runtime/event_bus.rs @@ -135,6 +135,38 @@ mod tests { } } + #[tokio::test] + async fn production_sized_job_stream_buffers_bursts_above_256() { + let bus = InProcJobEventBus::with_capacities(32_768, 8192); + let mut receiver = bus.subscribe(); + let library_id = LibraryId::new(); + + for sequence in 0..300 { + bus.publish(JobEvent { + meta: EventMeta::new( + None, + library_id, + format!("burst:{sequence}"), + None, + ), + payload: JobEventPayload::ThroughputTick { + queue_depths: Vec::new(), + sampled_at: Utc::now(), + }, + }) + .await + .expect("event publish succeeds"); + } + + for expected in 0..300 { + let event = receiver + .recv() + .await + .expect("production-sized job stream retains the burst"); + assert_eq!(event.meta.idempotency_key, format!("burst:{expected}")); + } + } + #[tokio::test] async fn scan_stream_capacity_is_independent_from_job_reconciliation_stream() { diff --git a/crates/ferrex-core/src/domain/scan/orchestration/runtime/supervisor.rs b/crates/ferrex-core/src/domain/scan/orchestration/runtime/supervisor.rs index 7db8672f..548d19c2 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/runtime/supervisor.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/runtime/supervisor.rs @@ -768,12 +768,14 @@ mod tests { #[derive(Clone, Copy, Debug)] enum ScriptedTransitionOutcome { Applied, + Requeued, Error, } #[derive(Clone, Copy, Debug)] enum ScriptedFailOutcome { RetryScheduled, + Requeued, Terminal(JobState), } @@ -918,6 +920,11 @@ mod tests { self.complete(lease_id).await?; Ok(QueueTransitionOutcome::Applied) } + ScriptedTransitionOutcome::Requeued => { + Err(MediaError::Internal( + "scripted completion cannot requeue".into(), + )) + } ScriptedTransitionOutcome::Error => Err(MediaError::Internal( "scripted complete persistence failure".into(), )), @@ -966,6 +973,23 @@ mod tests { self.fail(lease_id, true, error).await?; Ok(FailOutcome::RetryScheduled) } + ScriptedFailOutcome::Requeued => { + let mut state = self.state.lock().await; + let mut lease = + state.leased.remove(&lease_id).ok_or_else(|| { + MediaError::NotFound(format!( + "lease {} not found", + lease_id.0 + )) + })?; + lease.job.state = JobState::Ready; + lease.job.attempts = 0; + lease.job.lease_owner = None; + lease.job.lease_expires_at = None; + state.failures += 1; + state.ready.push_back(lease.job); + Ok(FailOutcome::Requeued) + } ScriptedFailOutcome::Terminal(state) => { let mut queue_state = self.state.lock().await; queue_state.leased.remove(&lease_id).ok_or_else(|| { @@ -1010,6 +1034,23 @@ mod tests { self.dead_letter(lease_id, error).await?; Ok(QueueTransitionOutcome::Applied) } + ScriptedTransitionOutcome::Requeued => { + let mut state = self.state.lock().await; + let mut lease = + state.leased.remove(&lease_id).ok_or_else(|| { + MediaError::NotFound(format!( + "lease {} not found", + lease_id.0 + )) + })?; + lease.job.state = JobState::Ready; + lease.job.attempts = 0; + lease.job.lease_owner = None; + lease.job.lease_expires_at = None; + state.failures += 1; + state.ready.push_back(lease.job); + Ok(QueueTransitionOutcome::Requeued) + } ScriptedTransitionOutcome::Error => Err(MediaError::Internal( "scripted dead-letter persistence failure".into(), )), @@ -1601,6 +1642,175 @@ mod tests { runtime.shutdown().await.expect("runtime shutdown succeeds"); } + #[tokio::test] + async fn superseded_dead_letter_requeues_without_terminal_event() { + let library_id = LibraryId::new(); + let config = single_scan_worker_config(); + let queue = + Arc::new(RecordingQueue::with_ready(vec![folder_scan_record( + library_id, + "/library/superseded-dead-letter", + JobPriority::P1, + )])); + queue + .script_dead_letter(ScriptedTransitionOutcome::Requeued) + .await; + let events = Arc::new(InProcJobEventBus::new(64)); + let mut job_events = events.subscribe(); + let budget = Arc::new(InMemoryBudget::new(config.budget.clone())); + let dispatcher = Arc::new(ScriptedDispatcher::new( + Duration::ZERO, + vec![ + DispatchStatus::DeadLetter { + error: "superseded terminal result".into(), + }, + DispatchStatus::Success, + ], + )); + + let runtime = OrchestratorRuntimeBuilder::new(config) + .with_queue(queue.clone()) + .with_events(events) + .with_budget(budget.clone()) + .with_dispatcher(dispatcher) + .build() + .expect("runtime build"); + runtime.start().await.expect("runtime start"); + + time::timeout(Duration::from_secs(2), async { + loop { + let stats = queue.stats().await; + if stats.failures == 1 && stats.completed == 1 { + break; + } + time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("requeued work is immediately scheduled and completed"); + + wait_for_worker_accounting_to_release( + &runtime, + budget.as_ref(), + library_id, + ) + .await; + let mut payloads = Vec::new(); + while let Ok(event) = job_events.try_recv() { + payloads.push(event.payload); + } + + let stats = queue.stats().await; + assert_eq!(stats.dead_letters, 0); + assert_eq!( + payloads + .iter() + .filter(|payload| matches!( + payload, + JobEventPayload::Dequeued { .. } + )) + .count(), + 2, + "the requeued row must be advertised without waiting for housekeeper" + ); + assert!(payloads.iter().any(|payload| matches!( + payload, + JobEventPayload::Failed { + retryable: true, + .. + } + ))); + assert!(!payloads.iter().any(|payload| matches!( + payload, + JobEventPayload::DeadLettered { .. } + ))); + + runtime.shutdown().await.expect("runtime shutdown succeeds"); + } + + #[tokio::test] + async fn superseded_exhausted_failure_requeues_without_housekeeper() { + let library_id = LibraryId::new(); + let config = single_scan_worker_config(); + let queue = + Arc::new(RecordingQueue::with_ready(vec![folder_scan_record( + library_id, + "/library/superseded-exhausted-failure", + JobPriority::P1, + )])); + queue.script_fail(ScriptedFailOutcome::Requeued).await; + let events = Arc::new(InProcJobEventBus::new(64)); + let mut job_events = events.subscribe(); + let budget = Arc::new(InMemoryBudget::new(config.budget.clone())); + let dispatcher = Arc::new(ScriptedDispatcher::new( + Duration::ZERO, + vec![ + DispatchStatus::Retry { + error: "superseded exhausted result".into(), + }, + DispatchStatus::Success, + ], + )); + + let runtime = OrchestratorRuntimeBuilder::new(config) + .with_queue(queue.clone()) + .with_events(events) + .with_budget(budget.clone()) + .with_dispatcher(dispatcher) + .build() + .expect("runtime build"); + runtime.start().await.expect("runtime start"); + + time::timeout(Duration::from_secs(2), async { + loop { + let stats = queue.stats().await; + if stats.failures == 1 && stats.completed == 1 { + break; + } + time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("requeued exhausted work is immediately completed"); + wait_for_worker_accounting_to_release( + &runtime, + budget.as_ref(), + library_id, + ) + .await; + + let mut payloads = Vec::new(); + while let Ok(event) = job_events.try_recv() { + payloads.push(event.payload); + } + let stats = queue.stats().await; + assert_eq!(stats.dead_letters, 0); + assert_eq!( + payloads + .iter() + .filter(|payload| matches!( + payload, + JobEventPayload::Dequeued { .. } + )) + .count(), + 2, + "FailOutcome::Requeued must advertise ready work before housekeeper" + ); + assert!(payloads.iter().any(|payload| matches!( + payload, + JobEventPayload::Failed { + retryable: true, + .. + } + ))); + assert!(!payloads.iter().any(|payload| matches!( + payload, + JobEventPayload::DeadLettered { .. } + ))); + + runtime.shutdown().await.expect("runtime shutdown succeeds"); + } + #[tokio::test] async fn retry_at_attempt_limit_publishes_durable_dead_letter_outcome() { let library_id = LibraryId::new(); diff --git a/crates/ferrex-core/src/domain/scan/orchestration/runtime/task_graph.rs b/crates/ferrex-core/src/domain/scan/orchestration/runtime/task_graph.rs index 2ea9c38a..41b53df9 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/runtime/task_graph.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/runtime/task_graph.rs @@ -64,6 +64,9 @@ enum WorkerTransition { RetryScheduled { error: String, }, + Requeued { + error: String, + }, TerminalFailed { error: String, }, @@ -96,6 +99,12 @@ fn worker_transition_from_complete( ) -> WorkerTransition { match outcome { Ok(QueueTransitionOutcome::Applied) => WorkerTransition::Completed, + Ok(QueueTransitionOutcome::Requeued) => { + WorkerTransition::PersistenceError { + operation: "complete", + error: "queue requeued a job during completion".into(), + } + } Ok(QueueTransitionOutcome::Missing) => WorkerTransition::Missing { operation: "complete", error: "durable completion was not applied because the lease was missing" @@ -117,6 +126,9 @@ fn worker_transition_from_fail( Ok(FailOutcome::RetryScheduled) => WorkerTransition::RetryScheduled { error: execution_error, }, + Ok(FailOutcome::Requeued) => WorkerTransition::Requeued { + error: execution_error, + }, Ok(FailOutcome::Terminal { state: JobState::DeadLetter, }) => WorkerTransition::DeadLettered { @@ -161,6 +173,9 @@ fn worker_transition_from_dead_letter( Ok(QueueTransitionOutcome::Applied) => WorkerTransition::DeadLettered { error: execution_error, }, + Ok(QueueTransitionOutcome::Requeued) => WorkerTransition::Requeued { + error: execution_error, + }, Ok(QueueTransitionOutcome::Missing) => WorkerTransition::Missing { operation: "dead_letter", error: format!( @@ -196,6 +211,7 @@ async fn finalize_worker_transition( let library_id = lease.job.payload.library_id(); let dedupe_key: DedupeKey = lease.job.payload.dedupe_key(); let terminal = transition.is_terminal(); + let requeued = matches!(&transition, WorkerTransition::Requeued { .. }); let (payload, command) = match transition { WorkerTransition::Completed => ( @@ -223,6 +239,20 @@ async fn finalize_worker_transition( error: Some(error), }, ), + WorkerTransition::Requeued { error } => ( + JobEventPayload::Failed { + job_id, + kind: job_kind, + priority: job_priority, + retryable: true, + }, + LibraryActorCommand::JobFailed { + job_id, + dedupe_key: dedupe_key.clone(), + retryable: true, + error: Some(error), + }, + ), WorkerTransition::TerminalFailed { error } => ( JobEventPayload::Failed { job_id, @@ -321,6 +351,11 @@ async fn finalize_worker_transition( scheduler.record_completed(reserved_library_id).await; } else { scheduler.release(reserved_library_id).await; + if requeued { + scheduler + .record_enqueued(job_kind, reserved_library_id, job_priority) + .await; + } } let sender_opt = { @@ -413,6 +448,37 @@ where Ok(ready_total) } +async fn run_housekeeper_cycle(queue: &Q, scheduler: &WeightedFairScheduler) +where + Q: QueueService + LeaseExpiryScanner + ?Sized, +{ + if let Err(err) = queue.scan_expired_leases().await { + tracing::warn!("housekeeper scan_expired_leases error: {err}"); + } + + match queue.repair_terminal_series_dependencies().await { + Ok(0) => {} + Ok(repaired) => tracing::info!( + repaired, + "repaired deferred episode jobs with terminal series dependencies" + ), + Err(err) => tracing::warn!( + error = %err, + "terminal series dependency repair failed" + ), + } + + match reconcile_scheduler_ready(queue, scheduler).await { + Ok(ready_total) => { + tracing::trace!(ready_total, "scheduler ready counts reconciled") + } + Err(err) => tracing::warn!( + error = %err, + "periodic scheduler ready reconciliation failed" + ), + } +} + async fn observe_scheduler_events( queue: Arc, mut job_rx: tokio::sync::broadcast::Receiver, @@ -1122,28 +1188,12 @@ impl RuntimeTaskGraph { break; } _ = tokio::time::sleep(interval) => { - if let Err(err) = queue.scan_expired_leases().await { - tracing::warn!("housekeeper scan_expired_leases error: {err}"); - } - match reconcile_scheduler_ready( - queue.as_ref(), - &scheduler, - ) - .await - { - Ok(ready_total) => tracing::trace!( - ready_total, - "scheduler ready counts reconciled" - ), - Err(err) => tracing::warn!( - error = %err, - "periodic scheduler ready reconciliation failed" - ), - } + run_housekeeper_cycle(queue.as_ref(), &scheduler).await; } } } - }).await; + }) + .await; } pub(super) async fn shutdown( @@ -1388,6 +1438,15 @@ mod tests { struct ReadyCountQueue { counts: Vec, + call_order: Option>>>, + } + + impl ReadyCountQueue { + fn record_call(&self, call: &'static str) { + if let Some(call_order) = &self.call_order { + call_order.lock().expect("call order lock").push(call); + } + } } #[async_trait] @@ -1456,11 +1515,25 @@ mod tests { panic!("release_dependency is not used by reconciliation tests") } + async fn repair_terminal_series_dependencies(&self) -> Result { + self.record_call("repair_terminal_series_dependencies"); + Ok(1) + } + async fn ready_counts_grouped(&self) -> Result> { + self.record_call("ready_counts_grouped"); Ok(self.counts.clone()) } } + #[async_trait] + impl LeaseExpiryScanner for ReadyCountQueue { + async fn scan_expired_leases(&self) -> Result { + self.record_call("scan_expired_leases"); + Ok(0) + } + } + #[test] fn folder_discovery_request_preserves_parent_correlation() { let library_id = @@ -1516,6 +1589,7 @@ mod tests { ready: 1, leased: 0, }], + call_order: None, }); let scheduler = WeightedFairScheduler::new( &crate::domain::scan::orchestration::config::QueueConfig::default(), @@ -1570,4 +1644,45 @@ mod tests { shutdown.cancel(); observer.await.expect("observer exits cleanly"); } + + #[tokio::test] + async fn housekeeper_repairs_terminal_series_dependencies_before_ready_counts() + { + let library_id = LibraryId::new(); + let call_order = Arc::new(std::sync::Mutex::new(Vec::new())); + let queue = ReadyCountQueue { + counts: vec![ReadyQueueCount { + kind: JobKind::EpisodeMatch, + library_id, + priority: JobPriority::P1, + ready: 1, + leased: 0, + }], + call_order: Some(call_order.clone()), + }; + let scheduler = WeightedFairScheduler::new( + &crate::domain::scan::orchestration::config::QueueConfig::default(), + crate::domain::scan::orchestration::config::PriorityWeights::default(), + ); + + run_housekeeper_cycle(&queue, &scheduler).await; + + assert_eq!( + *call_order.lock().expect("call order lock"), + vec![ + "scan_expired_leases", + "repair_terminal_series_dependencies", + "ready_counts_grouped", + ] + ); + assert_eq!( + scheduler + .snapshot() + .await + .get(&library_id) + .map(|(_, ready)| *ready), + Some(1), + "the scheduler must observe the repaired ready row in the same cycle" + ); + } } diff --git a/crates/ferrex-core/src/domain/scan/orchestration/series/bundle_tracker.rs b/crates/ferrex-core/src/domain/scan/orchestration/series/bundle_tracker.rs index 72a97236..9a850734 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/series/bundle_tracker.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/series/bundle_tracker.rs @@ -9,10 +9,14 @@ use crate::{ SeriesRootPath, }, orchestration::events::{JobEvent, JobEventPayload}, - orchestration::job::{AnalyzeScanHierarchy, JobKind}, + orchestration::{ + job::{AnalyzeScanHierarchy, JobKind, JobState}, + queue::DurableJobState, + }, }, types::{LibraryId, SeriesID}, }; +use chrono::{DateTime, Utc}; use ferrex_model::{EpisodeID, MediaID}; #[derive(Debug, Clone)] @@ -20,6 +24,7 @@ pub struct SeriesBundleFinalization { pub library_id: LibraryId, pub series_id: SeriesID, pub series_root_path: SeriesRootPath, + generation: u64, } #[derive(Debug, Default)] @@ -50,10 +55,12 @@ impl SeriesBundleTracker { ctx.series_root_path.clone(), ) }); - progress + let folder_added = progress .expected_season_folders .insert(ctx.season_folder_path.clone()); - progress.expected_season_numbers.insert(ctx.season_number); + let number_added = + progress.expected_season_numbers.insert(ctx.season_number); + progress.bump_generation_if(folder_added || number_added); } FolderScanContext::Movie(_) => {} } @@ -74,7 +81,9 @@ impl SeriesBundleTracker { ctx.series_root_path.clone(), ) }); + let changed = !progress.root_scan_completed; progress.root_scan_completed = true; + progress.bump_generation_if(changed); } FolderScanContext::Season(ctx) => { let progress = self @@ -86,9 +95,10 @@ impl SeriesBundleTracker { ctx.series_root_path.clone(), ) }); - progress + let changed = progress .completed_season_folders .insert(ctx.season_folder_path.clone()); + progress.bump_generation_if(changed); } FolderScanContext::Movie(_) => {} } @@ -117,16 +127,17 @@ impl SeriesBundleTracker { ) }); - progress.expected_episode_ids.insert(episode_id); + let mut changed = progress.expected_episode_ids.insert(episode_id); if let Some(episode_path) = EpisodeFilePathNorm::try_new(event.path_norm.clone()) { - progress.expected_episode_paths.insert(episode_path); + changed |= progress.expected_episode_paths.insert(episode_path); } if let Some(season_number) = SeasonNumber::from_link(&hierarchy.season) { - progress.expected_season_numbers.insert(season_number.0); + changed |= progress.expected_season_numbers.insert(season_number.0); } + progress.bump_generation_if(changed); } pub fn observe_indexed(&mut self, outcome: &IndexingOutcome) { @@ -142,11 +153,14 @@ impl SeriesBundleTracker { ) }); - if let Some(series_id) = + let changed = if let Some(series_id) = SeriesIdResolution::from_link(&hierarchy.series) { - progress.series_id = Some(series_id); - } + progress.update_series_id(series_id) + } else { + false + }; + progress.bump_generation_if(changed); } AnalyzeScanHierarchy::Season(hierarchy) => { let progress = self @@ -159,17 +173,21 @@ impl SeriesBundleTracker { ) }); - if let Some(series_id) = + let mut changed = if let Some(series_id) = SeriesIdResolution::from_link(&hierarchy.series) { - progress.series_id = Some(series_id); - } + progress.update_series_id(series_id) + } else { + false + }; if let Some(season_number) = SeasonNumber::from_link(&hierarchy.season) { - progress.indexed_season_numbers.insert(season_number.0); + changed |= + progress.indexed_season_numbers.insert(season_number.0); } + progress.bump_generation_if(changed); } AnalyzeScanHierarchy::Episode(hierarchy) => { let MediaID::Episode(episode_id) = outcome.media_id else { @@ -186,24 +204,32 @@ impl SeriesBundleTracker { ) }); - if let Some(series_id) = + let mut changed = if let Some(series_id) = SeriesIdResolution::from_link(&hierarchy.series) { - progress.series_id = Some(series_id); - } + progress.update_series_id(series_id) + } else { + false + }; - progress.indexed_episode_ids.insert(episode_id); + changed |= progress.indexed_episode_ids.insert(episode_id); if let Some(episode_path) = EpisodeFilePathNorm::try_new(outcome.path_norm.clone()) { - progress.completed_episode_paths.insert(episode_path); + changed |= progress + .episode_jobs_by_path + .entry(episode_path) + .or_default() + .observe_catalog_indexed(); } if let Some(season_number) = SeasonNumber::from_link(&hierarchy.season) { - progress.indexed_season_numbers.insert(season_number.0); + changed |= + progress.indexed_season_numbers.insert(season_number.0); } + progress.bump_generation_if(changed); } AnalyzeScanHierarchy::Movie(_) => {} } @@ -240,70 +266,234 @@ impl SeriesBundleTracker { ) }); - match &event.payload { - JobEventPayload::Enqueued { kind, .. } => { - if matches!( - kind, - JobKind::MediaAnalyze - | JobKind::EpisodeMatch - | JobKind::MetadataEnrich - | JobKind::IndexUpsert - ) { - progress.expected_episode_paths.insert(episode_path); - } + let changed = match &event.payload { + JobEventPayload::Enqueued { job_id, kind, .. } + if is_episode_pipeline_kind(*kind) => + { + let expected = progress + .expected_episode_paths + .insert(episode_path.clone()); + let enrolled = progress + .episode_jobs_by_path + .entry(episode_path) + .or_default() + .enroll(*job_id, *kind); + expected || enrolled } - JobEventPayload::Merged { kind, .. } => { - if matches!( - kind, - JobKind::MediaAnalyze - | JobKind::EpisodeMatch - | JobKind::MetadataEnrich - | JobKind::IndexUpsert - ) { - progress.expected_episode_paths.insert(episode_path); - } + JobEventPayload::Merged { + existing_job_id, + kind, + .. + } if is_episode_pipeline_kind(*kind) => { + let expected = progress + .expected_episode_paths + .insert(episode_path.clone()); + let enrolled = progress + .episode_jobs_by_path + .entry(episode_path) + .or_default() + .enroll(*existing_job_id, *kind); + expected || enrolled } - JobEventPayload::Completed { kind, .. } => { - // Index-upsert is the end of the episode pipeline; earlier stages - // must not count as episode completion. - if *kind == JobKind::IndexUpsert { - progress.completed_episode_paths.insert(episode_path); - } + JobEventPayload::Dequeued { job_id, kind, .. } + if is_episode_pipeline_kind(*kind) => + { + let expected = progress + .expected_episode_paths + .insert(episode_path.clone()); + let active = progress + .episode_jobs_by_path + .entry(episode_path) + .or_default() + .observe_transition( + *job_id, + *kind, + EpisodeJobStatus::Active, + ); + expected || active } - JobEventPayload::DeadLettered { .. } => { - // Dead-letter at any stage is terminal for this episode file. - progress.completed_episode_paths.insert(episode_path); + JobEventPayload::Completed { job_id, kind, .. } + if is_episode_pipeline_kind(*kind) => + { + let expected = progress + .expected_episode_paths + .insert(episode_path.clone()); + let status = if *kind == JobKind::IndexUpsert { + EpisodeJobStatus::Complete + } else { + EpisodeJobStatus::StageCompleted + }; + let terminal = progress + .episode_jobs_by_path + .entry(episode_path) + .or_default() + .observe_transition(*job_id, *kind, status); + expected || terminal } - JobEventPayload::Failed { retryable, .. } => { - // Non-retryable failures are terminal for this episode file. - if !retryable { - progress.completed_episode_paths.insert(episode_path); - } + JobEventPayload::DeadLettered { job_id, kind, .. } + if is_episode_pipeline_kind(*kind) => + { + let expected = progress + .expected_episode_paths + .insert(episode_path.clone()); + let terminal = progress + .episode_jobs_by_path + .entry(episode_path) + .or_default() + .observe_transition( + *job_id, + *kind, + EpisodeJobStatus::Complete, + ); + expected || terminal } - JobEventPayload::Dequeued { .. } - | JobEventPayload::LeaseRenewed { .. } - | JobEventPayload::LeaseExpired { .. } - | JobEventPayload::ThroughputTick { .. } => {} - } + JobEventPayload::Failed { + job_id, + kind, + retryable, + .. + } if is_episode_pipeline_kind(*kind) => { + let expected = progress + .expected_episode_paths + .insert(episode_path.clone()); + let status = if *retryable { + EpisodeJobStatus::Active + } else { + EpisodeJobStatus::Complete + }; + let transitioned = progress + .episode_jobs_by_path + .entry(episode_path) + .or_default() + .observe_transition(*job_id, *kind, status); + expected || transitioned + } + JobEventPayload::LeaseRenewed { job_id, .. } + | JobEventPayload::LeaseExpired { job_id, .. } => progress + .episode_jobs_by_path + .get_mut(&episode_path) + .is_some_and(|path| path.observe_active_existing(*job_id)), + JobEventPayload::Enqueued { .. } + | JobEventPayload::Merged { .. } + | JobEventPayload::Dequeued { .. } + | JobEventPayload::Completed { .. } + | JobEventPayload::DeadLettered { .. } + | JobEventPayload::Failed { .. } + | JobEventPayload::ThroughputTick { .. } => false, + }; + progress.bump_generation_if(changed); } - pub fn finalization_candidates(&self) -> Vec { - let mut out = Vec::new(); + /// Rebuild finalization-relevant episode pipeline state from PostgreSQL. + /// + /// Broadcast job events are only hints. When a consumer lags, the durable + /// snapshot restores episode enrollment, terminal paths, and the compact + /// identity emitted by completed index jobs without retaining job payloads + /// in the tracker. + pub fn reconcile_durable_job_states( + &mut self, + library_id: LibraryId, + jobs: &[DurableJobState], + ) { + let mut latest_by_path: HashMap< + (SeriesRootPath, EpisodeFilePathNorm), + &DurableJobState, + > = HashMap::new(); - for progress in self.by_root.values() { - if progress.finalized { + for job in jobs { + if !is_episode_pipeline_kind(job.kind) { continue; } - if !progress.discovery_complete() { + let Some(identity) = &job.series_identity else { + continue; + }; + let episode_path = + job.path_key.as_ref().and_then(|key| match key { + ferrex_model::SubjectKey::Path(path) => { + EpisodeFilePathNorm::try_new(path.to_string()) + } + ferrex_model::SubjectKey::Opaque(_) => None, + }); + + let progress = self + .by_root + .entry(identity.series_root_path.clone()) + .or_insert_with(|| { + SeriesBundleProgress::new( + library_id, + identity.series_root_path.clone(), + ) + }); + if progress.library_id != library_id { continue; } - if !progress.seasons_complete() { + let mut changed = false; + if let Some(episode_path) = &episode_path { + changed |= progress + .expected_episode_paths + .insert(episode_path.clone()); + let key = + (identity.series_root_path.clone(), episode_path.clone()); + let replace = latest_by_path + .get(&key) + .is_none_or(|current| durable_job_is_newer(job, current)); + if replace { + latest_by_path.insert(key, job); + } + } + + if job.kind == JobKind::IndexUpsert + && job.state == JobState::Completed + { + if let Some(series_id) = identity.series_id { + changed |= progress.update_series_id(series_id); + } + if let Some(season_number) = identity.season_number { + changed |= + progress.indexed_season_numbers.insert(season_number); + } + if let Some(MediaID::Episode(episode_id)) = job.media_id { + changed |= progress.indexed_episode_ids.insert(episode_id); + } + if let Some(episode_path) = episode_path { + changed |= progress + .episode_jobs_by_path + .entry(episode_path) + .or_default() + .observe_catalog_indexed(); + } + } + + progress.bump_generation_if(changed); + } + + // Only the newest durable job generation for an episode path may + // decide whether that path is complete. Historical terminal rows stay + // useful for catalog identity, but a later active generation reopens + // the path and invalidates a finalization claim. + for ((series_root_path, episode_path), job) in latest_by_path { + let Some(progress) = self.by_root.get_mut(&series_root_path) else { + continue; + }; + if progress.library_id != library_id { continue; } + let changed = progress + .episode_jobs_by_path + .entry(episode_path) + .or_default() + .reconcile_durable(job); + progress.bump_generation_if(changed); + } + } - if !progress.episodes_complete() { + pub fn finalization_candidates(&self) -> Vec { + let mut out = Vec::new(); + + for progress in self.by_root.values() { + if !progress.ready_for_finalization() { continue; } @@ -315,12 +505,40 @@ impl SeriesBundleTracker { library_id: progress.library_id, series_id, series_root_path: progress.series_root_path.clone(), + generation: progress.generation, }); } out } + /// Marks a claimed bundle finalized only if it is still the same eligible + /// bundle after the caller's asynchronous publication work completes. + /// + /// Folder discovery and episode enrollment can race publication. Rechecking + /// here prevents a stale claim from hiding work that arrived while the + /// publication was in flight. + pub fn mark_finalized_if_still_eligible( + &mut self, + candidate: &SeriesBundleFinalization, + ) -> bool { + let Some(progress) = self.by_root.get_mut(&candidate.series_root_path) + else { + return false; + }; + + if progress.library_id != candidate.library_id + || progress.series_id != Some(candidate.series_id) + || progress.generation != candidate.generation + || !progress.ready_for_finalization() + { + return false; + } + + progress.finalized = true; + true + } + pub fn mark_finalized(&mut self, series_root_path: &SeriesRootPath) { let Some(progress) = self.by_root.get_mut(series_root_path) else { return; @@ -346,8 +564,9 @@ struct SeriesBundleProgress { expected_episode_ids: HashSet, indexed_episode_ids: HashSet, expected_episode_paths: HashSet, - completed_episode_paths: HashSet, + episode_jobs_by_path: HashMap, finalized: bool, + generation: u64, } impl SeriesBundleProgress { @@ -364,8 +583,9 @@ impl SeriesBundleProgress { expected_episode_ids: HashSet::new(), indexed_episode_ids: HashSet::new(), expected_episode_paths: HashSet::new(), - completed_episode_paths: HashSet::new(), + episode_jobs_by_path: HashMap::new(), finalized: false, + generation: 0, } } @@ -387,13 +607,248 @@ impl SeriesBundleProgress { fn episodes_complete(&self) -> bool { if !self.expected_episode_paths.is_empty() { - return self - .expected_episode_paths - .is_subset(&self.completed_episode_paths); + return self.expected_episode_paths.iter().all(|path| { + self.episode_jobs_by_path + .get(path) + .is_some_and(EpisodePathProgress::is_complete) + }); } self.expected_episode_ids .is_subset(&self.indexed_episode_ids) } + + fn ready_for_finalization(&self) -> bool { + !self.finalized + && self.discovery_complete() + && self.seasons_complete() + && self.episodes_complete() + && self.series_id.is_some() + } + + fn update_series_id(&mut self, series_id: SeriesID) -> bool { + if self.series_id == Some(series_id) { + return false; + } + self.series_id = Some(series_id); + true + } + + fn bump_generation_if(&mut self, changed: bool) { + if changed { + self.generation = self.generation.saturating_add(1); + } + } +} + +#[derive(Debug, Default)] +struct EpisodePathProgress { + /// Catalog indexing outcomes do not carry a job ID. They are valid + /// completion evidence only until a concrete job generation is observed. + catalog_indexed: bool, + seen_job_ids: HashSet, + latest_job: Option, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +struct EpisodeJobGeneration { + job_id: crate::domain::scan::orchestration::job::JobId, + kind: JobKind, + status: EpisodeJobStatus, + durable_order: Option, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum EpisodeJobStatus { + Active, + StageCompleted, + Complete, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +struct DurableEpisodeJobOrder { + created_at: DateTime, + updated_at: DateTime, + job_id: crate::domain::scan::orchestration::job::JobId, +} + +impl DurableEpisodeJobOrder { + fn from_job(job: &DurableJobState) -> Self { + Self { + created_at: job.created_at, + updated_at: job.updated_at, + job_id: job.job_id, + } + } + + fn is_newer_than(&self, other: &Self) -> bool { + self.created_at > other.created_at + || (self.created_at == other.created_at + && (self.updated_at > other.updated_at + || (self.updated_at == other.updated_at + && self.job_id.0 > other.job_id.0))) + } +} + +impl EpisodePathProgress { + fn is_complete(&self) -> bool { + self.latest_job + .as_ref() + .map(|job| job.status == EpisodeJobStatus::Complete) + .unwrap_or(self.catalog_indexed) + } + + fn observe_catalog_indexed(&mut self) -> bool { + if self.catalog_indexed { + return false; + } + self.catalog_indexed = true; + true + } + + fn enroll( + &mut self, + job_id: crate::domain::scan::orchestration::job::JobId, + kind: JobKind, + ) -> bool { + if !self.seen_job_ids.insert(job_id) { + return false; + } + + self.latest_job = Some(EpisodeJobGeneration { + job_id, + kind, + status: EpisodeJobStatus::Active, + durable_order: None, + }); + true + } + + fn observe_transition( + &mut self, + job_id: crate::domain::scan::orchestration::job::JobId, + kind: JobKind, + status: EpisodeJobStatus, + ) -> bool { + if let Some(latest) = self.latest_job.as_mut() + && latest.job_id == job_id + { + let changed = latest.kind != kind || latest.status != status; + latest.kind = kind; + latest.status = status; + return changed; + } + + if !self.seen_job_ids.insert(job_id) { + return false; + } + + let replace = self + .latest_job + .as_ref() + .is_none_or(|latest| job_id.0 > latest.job_id.0); + if !replace { + return false; + } + + self.latest_job = Some(EpisodeJobGeneration { + job_id, + kind, + status, + durable_order: None, + }); + true + } + + fn observe_active_existing( + &mut self, + job_id: crate::domain::scan::orchestration::job::JobId, + ) -> bool { + let Some(latest) = self.latest_job.as_mut() else { + return false; + }; + if latest.job_id != job_id || latest.status == EpisodeJobStatus::Active + { + return false; + } + latest.status = EpisodeJobStatus::Active; + true + } + + fn reconcile_durable(&mut self, job: &DurableJobState) -> bool { + let durable_order = DurableEpisodeJobOrder::from_job(job); + let status = durable_episode_job_status(job); + + if let Some(latest) = self.latest_job.as_mut() + && latest.job_id == job.job_id + { + if latest + .durable_order + .as_ref() + .is_some_and(|current| !durable_order.is_newer_than(current)) + { + return false; + } + let changed = latest.kind != job.kind || latest.status != status; + latest.kind = job.kind; + latest.status = status; + latest.durable_order = Some(durable_order); + return changed; + } + + let replace = match self.latest_job.as_ref() { + None => true, + Some(latest) => match latest.durable_order.as_ref() { + Some(current) => durable_order.is_newer_than(current), + None => { + !self.seen_job_ids.contains(&job.job_id) + && job.job_id.0 > latest.job_id.0 + } + }, + }; + self.seen_job_ids.insert(job.job_id); + if !replace { + return false; + } + + self.latest_job = Some(EpisodeJobGeneration { + job_id: job.job_id, + kind: job.kind, + status, + durable_order: Some(durable_order), + }); + true + } +} + +fn durable_episode_job_status(job: &DurableJobState) -> EpisodeJobStatus { + if job.is_terminal_failure() + || (job.kind == JobKind::IndexUpsert + && job.state == JobState::Completed) + { + EpisodeJobStatus::Complete + } else if job.state == JobState::Completed { + EpisodeJobStatus::StageCompleted + } else { + EpisodeJobStatus::Active + } +} + +fn is_episode_pipeline_kind(kind: JobKind) -> bool { + matches!( + kind, + JobKind::MediaAnalyze + | JobKind::EpisodeMatch + | JobKind::MetadataEnrich + | JobKind::IndexUpsert + ) +} + +fn durable_job_is_newer( + candidate: &DurableJobState, + current: &DurableJobState, +) -> bool { + DurableEpisodeJobOrder::from_job(candidate) + .is_newer_than(&DurableEpisodeJobOrder::from_job(current)) } #[derive(Debug, Clone, Copy)] @@ -443,11 +898,139 @@ mod tests { EpisodeLink, EpisodeScanHierarchy, FolderScanContext, ScanNodeKind, SeasonFolderScanContext, SeasonScanHierarchy, SeriesFolderScanContext, }; - use crate::domain::scan::orchestration::job::MediaFingerprint; + use crate::domain::scan::orchestration::job::{JobId, MediaFingerprint}; + use crate::domain::scan::orchestration::queue::DurableSeriesIdentity; use chrono::Utc; use ferrex_model::{LibraryId as ModelLibraryId, VideoMediaType}; use uuid::Uuid; + fn pending_durable_episode_bundle() -> ( + SeriesBundleTracker, + LibraryId, + SeriesRootPath, + SeasonFolderScanContext, + EpisodeID, + SeriesID, + String, + ) { + let library_id = LibraryId(Uuid::from_u128(101)); + let series_id = SeriesID(Uuid::from_u128(102)); + let episode_id = EpisodeID(Uuid::from_u128(103)); + let series_root = + SeriesRootPath::try_new("/demo/Shows/Durable").unwrap(); + let (season_folder_path, season_number) = + SeasonFolderPath::try_new_under_series_root( + &series_root, + "/demo/Shows/Durable/Season 1", + ) + .unwrap(); + let series_context = SeriesFolderScanContext { + library_id, + series_root_path: series_root.clone(), + }; + let season_context = SeasonFolderScanContext { + library_id, + series_root_path: series_root.clone(), + season_folder_path, + season_number, + }; + let episode_path = + "/demo/Shows/Durable/Season 1/S01E01.mkv".to_string(); + let mut tracker = SeriesBundleTracker::default(); + tracker.observe_folder_discovered(&FolderScanContext::Series( + series_context.clone(), + )); + tracker.observe_folder_discovered(&FolderScanContext::Season( + season_context.clone(), + )); + tracker.observe_media_discovered(&MediaFileDiscovered { + library_id: ModelLibraryId(library_id.0), + path_norm: episode_path.clone(), + fingerprint: MediaFingerprint::default(), + classified_as: MediaKindHint::Episode, + media_id: MediaID::Episode(episode_id), + variant: VideoMediaType::Episode, + node: ScanNodeKind::EpisodeFile, + hierarchy: AnalyzeScanHierarchy::Episode(EpisodeScanHierarchy { + series_root_path: series_root.clone(), + series: SeriesLink::Hint( + crate::domain::scan::orchestration::context::SeriesHint { + title: "Durable".into(), + slug: None, + year: None, + region: None, + }, + ), + season: SeasonLink::Number(season_number), + episode: EpisodeLink::Hint( + crate::domain::scan::orchestration::context::EpisodeHint { + number: 1, + title: None, + }, + ), + }), + context: FolderScanContext::Season(season_context.clone()), + scan_reason: ScanReason::BulkSeed, + }); + tracker.observe_folder_scan_completed(&FolderScanSummary { + context: FolderScanContext::Season(season_context.clone()), + discovered_files: 1, + enqueued_subfolders: 0, + listing_hash: "durable-season".into(), + outcome: FolderScanOutcome::Changed, + completed_at: Utc::now(), + }); + tracker.observe_folder_scan_completed(&FolderScanSummary { + context: FolderScanContext::Series(series_context), + discovered_files: 0, + enqueued_subfolders: 1, + listing_hash: "durable-root".into(), + outcome: FolderScanOutcome::Changed, + completed_at: Utc::now(), + }); + + ( + tracker, + library_id, + series_root, + season_context, + episode_id, + series_id, + episode_path, + ) + } + + fn durable_series_job( + kind: JobKind, + state: JobState, + series_root_path: SeriesRootPath, + series_id: Option, + season_number: Option, + path: &str, + media_id: Option, + ) -> DurableJobState { + let now = Utc::now(); + DurableJobState { + job_id: JobId::new(), + kind, + media_id, + indexing_change: None, + series_identity: Some(DurableSeriesIdentity { + series_root_path, + series_id, + season_number, + }), + state, + attempts: 1, + dedupe_key: format!("durable:{kind:?}:{path}"), + correlation_id: Some(Uuid::now_v7()), + path_key: ferrex_model::SubjectKey::path(path).ok(), + last_error: None, + created_at: now, + updated_at: now, + } + } + #[test] fn drains_finalized_once_series_discovery_and_episodes_done() { let library_id = LibraryId(Uuid::from_u128(1)); @@ -589,7 +1172,10 @@ mod tests { assert_eq!(finalized[0].library_id, library_id); assert_eq!(finalized[0].series_id, series_id); - tracker.mark_finalized(&series_root); + // Replaying an already-observed durable outcome is idempotent and must + // not invalidate the generation captured by the candidate. + tracker.observe_indexed(&indexed); + assert!(tracker.mark_finalized_if_still_eligible(&finalized[0])); // Only yields once once marked. assert!(tracker.finalization_candidates().is_empty()); @@ -699,4 +1285,246 @@ mod tests { tracker.mark_finalized(&series_root); assert!(tracker.finalization_candidates().is_empty()); } + + #[test] + fn durable_completed_index_restores_dropped_bundle_terminal_event() { + let ( + mut tracker, + library_id, + series_root, + _season_context, + episode_id, + series_id, + episode_path, + ) = pending_durable_episode_bundle(); + assert!(tracker.finalization_candidates().is_empty()); + + let completed_index = durable_series_job( + JobKind::IndexUpsert, + JobState::Completed, + series_root, + Some(series_id), + Some(1), + &episode_path, + Some(MediaID::Episode(episode_id)), + ); + tracker.reconcile_durable_job_states( + library_id, + std::slice::from_ref(&completed_index), + ); + + let candidates = tracker.finalization_candidates(); + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].series_id, series_id); + let generation = candidates[0].generation; + + // Re-reading the same authoritative snapshot must not invalidate a + // claim by advancing the tracker generation. + tracker.reconcile_durable_job_states(library_id, &[completed_index]); + assert_eq!(tracker.finalization_candidates()[0].generation, generation); + } + + #[test] + fn durable_dead_letter_restores_dropped_bundle_terminal_event() { + let ( + mut tracker, + library_id, + series_root, + season_context, + _episode_id, + series_id, + episode_path, + ) = pending_durable_episode_bundle(); + + // The catalog already has enough series/season identity to publish the + // bundle. Only the episode pipeline terminal notification is dropped. + tracker.observe_indexed(&IndexingOutcome { + library_id: ModelLibraryId(library_id.0), + path_norm: season_context.season_folder_path.as_str().to_string(), + media_id: MediaID::Season(ferrex_model::SeasonID(Uuid::from_u128( + 104, + ))), + hierarchy: AnalyzeScanHierarchy::Season(SeasonScanHierarchy { + series_root_path: series_root.clone(), + series: SeriesLink::Resolved( + crate::domain::scan::orchestration::context::SeriesRef { + id: series_id, + slug: None, + title: Some("Durable".into()), + }, + ), + season: SeasonLink::Number(1), + }), + indexed_at: Utc::now(), + upserted: true, + media: None, + change: crate::domain::scan::actors::index::IndexingChange::Created, + }); + assert!(tracker.finalization_candidates().is_empty()); + + let dead_letter = durable_series_job( + JobKind::MetadataEnrich, + JobState::DeadLetter, + series_root, + Some(series_id), + Some(1), + &episode_path, + None, + ); + tracker.reconcile_durable_job_states(library_id, &[dead_letter]); + + let candidates = tracker.finalization_candidates(); + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].series_id, series_id); + } + + #[test] + fn same_path_reenrollment_during_publish_invalidates_stale_claim() { + let ( + mut tracker, + library_id, + series_root, + _season_context, + episode_id, + series_id, + episode_path, + ) = pending_durable_episode_bundle(); + let completed_index = durable_series_job( + JobKind::IndexUpsert, + JobState::Completed, + series_root.clone(), + Some(series_id), + Some(1), + &episode_path, + Some(MediaID::Episode(episode_id)), + ); + tracker.reconcile_durable_job_states(library_id, &[completed_index]); + let stale_claim = tracker + .finalization_candidates() + .into_iter() + .next() + .expect("the original path generation is complete"); + + let replacement_job_id = JobId::new(); + let reenqueued = JobEvent::from_job( + None, + library_id, + "episode:same-path:index:new".into(), + ferrex_model::SubjectKey::path(&episode_path).ok(), + JobEventPayload::Enqueued { + job_id: replacement_job_id, + kind: JobKind::IndexUpsert, + priority: + crate::domain::scan::orchestration::job::JobPriority::P0, + }, + ); + tracker.observe_job_event(&reenqueued); + + assert!( + tracker.finalization_candidates().is_empty(), + "a new job ID for the same episode path must reopen the bundle" + ); + assert!( + !tracker.mark_finalized_if_still_eligible(&stale_claim), + "the pre-reenrollment publication claim must be rejected" + ); + let generation_after_reenrollment = + tracker.by_root[&series_root].generation; + + tracker.observe_job_event(&reenqueued); + assert_eq!( + tracker.by_root[&series_root].generation, + generation_after_reenrollment, + "replaying the same job enrollment must be idempotent" + ); + + tracker.observe_job_event(&JobEvent::from_job( + None, + library_id, + "episode:same-path:index:new".into(), + ferrex_model::SubjectKey::path(&episode_path).ok(), + JobEventPayload::DeadLettered { + job_id: replacement_job_id, + kind: JobKind::IndexUpsert, + priority: + crate::domain::scan::orchestration::job::JobPriority::P0, + }, + )); + let replacement_claim = tracker + .finalization_candidates() + .into_iter() + .next() + .expect("the replacement terminal generation is publishable"); + assert!(replacement_claim.generation > stale_claim.generation); + } + + #[test] + fn durable_new_active_generation_reopens_old_completed_path() { + let ( + mut tracker, + library_id, + series_root, + _season_context, + episode_id, + series_id, + episode_path, + ) = pending_durable_episode_bundle(); + let now = Utc::now(); + let mut old_completed = durable_series_job( + JobKind::IndexUpsert, + JobState::Completed, + series_root.clone(), + Some(series_id), + Some(1), + &episode_path, + Some(MediaID::Episode(episode_id)), + ); + old_completed.created_at = now - chrono::Duration::seconds(2); + old_completed.updated_at = now - chrono::Duration::seconds(1); + + let mut new_active = durable_series_job( + JobKind::MediaAnalyze, + JobState::Leased, + series_root.clone(), + Some(series_id), + Some(1), + &episode_path, + None, + ); + new_active.created_at = now; + new_active.updated_at = now; + + // Reverse input order to prove reconciliation selects by durable job + // generation, not iteration order or set union. + tracker.reconcile_durable_job_states( + library_id, + &[new_active.clone(), old_completed.clone()], + ); + assert!( + tracker.finalization_candidates().is_empty(), + "the newer active row must override historical completion for the path" + ); + let active_generation = tracker.by_root[&series_root].generation; + + tracker.reconcile_durable_job_states( + library_id, + &[old_completed.clone(), new_active.clone()], + ); + assert_eq!( + tracker.by_root[&series_root].generation, active_generation, + "replaying the same durable snapshot must be idempotent" + ); + + new_active.state = JobState::DeadLetter; + new_active.updated_at = now + chrono::Duration::seconds(1); + tracker.reconcile_durable_job_states( + library_id, + &[old_completed, new_active], + ); + let candidate = + tracker.finalization_candidates().into_iter().next().expect( + "the newest terminal durable generation completes the path", + ); + assert_eq!(candidate.series_id, series_id); + } } diff --git a/crates/ferrex-core/src/domain/scan/orchestration/series/coordinator.rs b/crates/ferrex-core/src/domain/scan/orchestration/series/coordinator.rs index b7b448a1..739a3574 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/series/coordinator.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/series/coordinator.rs @@ -97,7 +97,11 @@ impl SeriesCoordinator { Self { states, resolver } } - /// Record that a series root was discovered by folder scanning. + /// Enroll a series root for resolution before its job can be enqueued. + /// + /// This explicitly marks every non-resolved root as a newly discovered + /// generation, so a terminalizing resolver can distinguish fresh work from + /// the attempt whose result it is trying to persist. pub async fn record_root_discovery( &self, library_id: LibraryId, @@ -106,7 +110,7 @@ impl SeriesCoordinator { ) -> Result { let state = self .states - .mark_discovered(library_id, series_root_path, hint) + .enroll_resolution_generation(library_id, series_root_path, hint) .await?; Ok(SeriesDiscoveryOutcome::new(state)) } @@ -512,6 +516,30 @@ mod tests { Some("provider returned 404") ); + let decision = coordinator + .prepare_episode_dependency( + library_id, + &episode_hierarchy(series_root.clone()), + ) + .await + .expect("late episode observes terminal series state"); + assert_eq!( + decision, + EpisodeDependencyDecision::Deferred { + dependency_key: DependencyKey::series_root(&series_root), + } + ); + assert_eq!( + states + .get(library_id, &series_root) + .await + .expect("state lookup") + .expect("failed state remains") + .status, + SeriesScanStatus::Failed, + "late episode discovery must not erase terminal failure" + ); + let releaser = RecordingReleaser::default(); coordinator .release_blocked_episode_dependencies( @@ -520,7 +548,7 @@ mod tests { &series_root, ) .await - .expect("release dependency"); + .expect("terminal resolver releases dependency"); assert_eq!( releaser.releases.lock().await.as_slice(), @@ -528,6 +556,77 @@ mod tests { ); } + #[tokio::test] + async fn fresh_root_enrollment_cannot_release_from_historical_failure() { + let (coordinator, states) = coordinator_with_states(); + let library_id = lib(70); + let series_root = root("/demo/Shows/Retry After Failure"); + let job = SeriesResolveJob { + library_id, + series_root_path: series_root.clone(), + hint: Some(hint("Retry After Failure")), + folder_name: "Retry After Failure".into(), + scan_reason: ScanReason::BulkSeed, + }; + + coordinator + .record_resolution_failure( + &job, + "historical provider failure".into(), + ) + .await + .expect("record historical failure"); + + // Root discovery enrolls the fresh generation before the caller can + // enqueue SeriesResolve. At this point the resolver intentionally has + // not called mark_seeded yet. + let enrollment = coordinator + .record_root_discovery( + library_id, + series_root.clone(), + job.hint.clone(), + ) + .await + .expect("enroll fresh resolver generation"); + assert!(enrollment.should_enqueue_resolution()); + assert_eq!(enrollment.state().status, SeriesScanStatus::Discovered); + assert!(enrollment.state().failure_reason.is_none()); + + let decision = coordinator + .prepare_episode_dependency( + library_id, + &episode_hierarchy(series_root.clone()), + ) + .await + .expect("episode waits for freshly enrolled resolver"); + assert_eq!( + decision, + EpisodeDependencyDecision::Deferred { + dependency_key: DependencyKey::series_root(&series_root), + } + ); + + let seeded = states + .mark_seeded(library_id, series_root.clone(), job.hint.clone()) + .await + .expect("resolver starts fresh generation"); + assert_eq!(seeded.status, SeriesScanStatus::Seeded); + + coordinator + .record_resolution_failure(&job, "fresh provider failure".into()) + .await + .expect("fresh generation reaches terminal failure"); + assert_eq!( + states + .get(library_id, &series_root) + .await + .expect("fresh terminal state lookup") + .expect("fresh terminal state exists") + .status, + SeriesScanStatus::Failed + ); + } + #[test] fn episode_metadata_enrich_validation_rejects_unresolved_series() { let library_id = lib(8); diff --git a/crates/ferrex-core/src/domain/scan/orchestration/series_state.rs b/crates/ferrex-core/src/domain/scan/orchestration/series_state.rs index ef3ef946..01fa6a14 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/series_state.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/series_state.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use tokio::sync::Mutex; use crate::{ - error::Result, + error::{MediaError, Result}, types::{LibraryId, ids::SeriesID}, }; @@ -57,6 +57,9 @@ pub trait SeriesScanStateRepository: Send + Sync { series_root_path: &SeriesRootPath, ) -> Result>; + /// Record an observational episode discovery. This inserts a missing root + /// but never mutates an existing row; only root enrollment may begin a + /// fresh resolution generation. async fn mark_discovered( &self, library_id: LibraryId, @@ -64,6 +67,17 @@ pub trait SeriesScanStateRepository: Send + Sync { hint: Option, ) -> Result; + /// Enroll a root for a fresh SeriesResolve generation before its job can be + /// enqueued. Unlike observational episode discovery, this explicitly marks + /// every non-resolved root `Discovered`. That state is the durable freshness + /// signal terminalizing workers use to preserve a newer generation. + async fn enroll_resolution_generation( + &self, + library_id: LibraryId, + series_root_path: SeriesRootPath, + hint: Option, + ) -> Result; + async fn mark_seeded( &self, library_id: LibraryId, @@ -102,7 +116,7 @@ impl SeriesScanStateRepository for InMemorySeriesScanStateRepository { Ok(guard.get(&(library_id, series_root_path.clone())).cloned()) } - async fn mark_discovered( + async fn enroll_resolution_generation( &self, library_id: LibraryId, series_root_path: SeriesRootPath, @@ -133,11 +147,48 @@ impl SeriesScanStateRepository for InMemorySeriesScanStateRepository { } if !matches!(entry.status, SeriesScanStatus::Resolved) { entry.status = SeriesScanStatus::Discovered; + entry.series_id = None; + entry.resolved_at = None; + entry.failed_at = None; + entry.failure_reason = None; } entry.updated_at = now; Ok(entry.clone()) } + async fn mark_discovered( + &self, + library_id: LibraryId, + series_root_path: SeriesRootPath, + hint: Option, + ) -> Result { + let mut guard = self.states.lock().await; + if let Some(existing) = + guard.get(&(library_id, series_root_path.clone())) + { + return Ok(existing.clone()); + } + + let now = Utc::now(); + let state = SeriesScanState { + library_id, + series_root_path: series_root_path.clone(), + status: SeriesScanStatus::Discovered, + series_id: None, + hint, + seeded_at: None, + last_attempt_at: None, + attempts: 0, + resolved_at: None, + failed_at: None, + failure_reason: None, + created_at: now, + updated_at: now, + }; + guard.insert((library_id, series_root_path), state.clone()); + Ok(state) + } + async fn mark_seeded( &self, library_id: LibraryId, @@ -169,6 +220,8 @@ impl SeriesScanStateRepository for InMemorySeriesScanStateRepository { } if !matches!(entry.status, SeriesScanStatus::Resolved) { entry.status = SeriesScanStatus::Seeded; + entry.failed_at = None; + entry.failure_reason = None; } entry.last_attempt_at = Some(now); entry.attempts = entry.attempts.saturating_add(1); @@ -273,7 +326,7 @@ mod tests { } #[tokio::test] - async fn discovered_hint_is_not_overwritten_by_none() { + async fn episode_observation_inserts_missing_root_then_preserves_it() { let repo = InMemorySeriesScanStateRepository::default(); let library_id = lib(1); let root = series_root("/demo/Shows/Example"); @@ -289,13 +342,61 @@ mod tests { .mark_discovered(library_id, root.clone(), Some(hint.clone())) .await .expect("mark discovered"); - assert_eq!(first.hint.as_ref().map(|h| &h.title), Some(&hint.title)); + assert_eq!(first.status, SeriesScanStatus::Discovered); + assert_eq!(first.hint.as_ref(), Some(&hint)); + let replacement_hint = SeriesHint { + title: "Replacement".into(), + slug: Some("replacement".into()), + year: Some(2025), + region: Some("CA".into()), + }; let second = repo - .mark_discovered(library_id, root.clone(), None) + .mark_discovered(library_id, root.clone(), Some(replacement_hint)) .await .expect("mark discovered again"); - assert_eq!(second.hint.as_ref().map(|h| &h.title), Some(&hint.title)); + assert_eq!(second.status, first.status); + assert_eq!(second.hint, first.hint); + assert_eq!(second.created_at, first.created_at); + assert_eq!(second.updated_at, first.updated_at); + } + + #[tokio::test] + async fn episode_observation_preserves_seeded_state() { + let repo = InMemorySeriesScanStateRepository::default(); + let library_id = lib(10); + let root = series_root("/demo/Shows/Seeded Observation"); + let original_hint = SeriesHint { + title: "Seeded Observation".into(), + slug: Some("seeded-observation".into()), + year: Some(2020), + region: Some("US".into()), + }; + let seeded = repo + .mark_seeded(library_id, root.clone(), Some(original_hint.clone())) + .await + .expect("series seeded"); + + let observed = repo + .mark_discovered( + library_id, + root, + Some(SeriesHint { + title: "Spoofed Enrollment".into(), + slug: None, + year: Some(2026), + region: None, + }), + ) + .await + .expect("episode observes seeded series"); + + assert_eq!(observed.status, SeriesScanStatus::Seeded); + assert_eq!(observed.hint.as_ref(), Some(&original_hint)); + assert_eq!(observed.updated_at, seeded.updated_at); + assert_eq!(observed.seeded_at, seeded.seeded_at); + assert_eq!(observed.last_attempt_at, seeded.last_attempt_at); + assert_eq!(observed.attempts, seeded.attempts); } #[tokio::test] @@ -334,17 +435,18 @@ mod tests { let root = series_root("/demo/Shows/Rediscovered"); let series_id = SeriesID(Uuid::from_u128(5)); - repo.mark_resolved( - library_id, - root.clone(), - SeriesRef { - id: series_id, - slug: Some("rediscovered".into()), - title: Some("Rediscovered".into()), - }, - ) - .await - .expect("mark resolved"); + let resolved = repo + .mark_resolved( + library_id, + root.clone(), + SeriesRef { + id: series_id, + slug: Some("rediscovered".into()), + title: Some("Rediscovered".into()), + }, + ) + .await + .expect("mark resolved"); let after = repo .mark_discovered( @@ -363,6 +465,8 @@ mod tests { assert_eq!(after.series_id, Some(series_id)); assert_eq!(after.status, SeriesScanStatus::Resolved); assert!(after.is_resolved()); + assert_eq!(after.hint, resolved.hint); + assert_eq!(after.updated_at, resolved.updated_at); assert_eq!( repo.get(library_id, &root) .await @@ -372,6 +476,115 @@ mod tests { SeriesScanStatus::Resolved ); } + + #[tokio::test] + async fn root_enrollment_reopens_failure_but_episode_discovery_does_not() { + let repo = InMemorySeriesScanStateRepository::default(); + let library_id = lib(6); + let root = series_root("/demo/Shows/Failed Then Rediscovered"); + + let failed = repo + .mark_failed( + library_id, + root.clone(), + "provider returned 404".into(), + ) + .await + .expect("mark failed"); + + let rediscovered = repo + .mark_discovered( + library_id, + root.clone(), + Some(SeriesHint { + title: "Failed Then Rediscovered".into(), + slug: None, + year: None, + region: None, + }), + ) + .await + .expect("mark discovered after failure"); + assert_eq!(rediscovered.status, SeriesScanStatus::Failed); + assert_eq!( + rediscovered.failure_reason.as_deref(), + Some("provider returned 404") + ); + assert_eq!(rediscovered.hint, failed.hint); + assert_eq!(rediscovered.updated_at, failed.updated_at); + + let enrolled = repo + .enroll_resolution_generation(library_id, root.clone(), None) + .await + .expect("root enrollment begins a fresh generation"); + assert_eq!(enrolled.status, SeriesScanStatus::Discovered); + assert!(enrolled.series_id.is_none()); + assert!(enrolled.resolved_at.is_none()); + assert!(enrolled.failed_at.is_none()); + assert!(enrolled.failure_reason.is_none()); + + let reseeded = repo + .mark_seeded(library_id, root, None) + .await + .expect("enrolled resolver marks its attempt seeded"); + assert_eq!(reseeded.status, SeriesScanStatus::Seeded); + assert!(reseeded.failed_at.is_none()); + assert!(reseeded.failure_reason.is_none()); + } + + #[tokio::test] + async fn root_enrollment_marks_seeded_generation_discovered() { + let repo = InMemorySeriesScanStateRepository::default(); + let library_id = lib(7); + let root = series_root("/demo/Shows/Seeded Then Reenrolled"); + + let seeded = repo + .mark_seeded(library_id, root.clone(), None) + .await + .expect("attempt seeded"); + assert_eq!(seeded.status, SeriesScanStatus::Seeded); + + let enrolled = repo + .enroll_resolution_generation(library_id, root, None) + .await + .expect("fresh generation enrolled"); + assert_eq!(enrolled.status, SeriesScanStatus::Discovered); + assert_eq!(enrolled.attempts, seeded.attempts); + assert_eq!(enrolled.seeded_at, seeded.seeded_at); + assert_eq!(enrolled.last_attempt_at, seeded.last_attempt_at); + assert!(enrolled.series_id.is_none()); + assert!(enrolled.resolved_at.is_none()); + assert!(enrolled.failed_at.is_none()); + assert!(enrolled.failure_reason.is_none()); + } + + #[tokio::test] + async fn root_enrollment_preserves_resolved_generation() { + let repo = InMemorySeriesScanStateRepository::default(); + let library_id = lib(8); + let root = series_root("/demo/Shows/Resolved Then Reenrolled"); + let series_id = SeriesID(Uuid::from_u128(9)); + repo.mark_resolved( + library_id, + root.clone(), + SeriesRef { + id: series_id, + slug: Some("resolved-then-reenrolled".into()), + title: Some("Resolved Then Reenrolled".into()), + }, + ) + .await + .expect("series resolved"); + + let enrolled = repo + .enroll_resolution_generation(library_id, root, None) + .await + .expect("resolved root observed by enrollment"); + assert_eq!(enrolled.status, SeriesScanStatus::Resolved); + assert_eq!(enrolled.series_id, Some(series_id)); + assert!(enrolled.resolved_at.is_some()); + assert!(enrolled.is_resolved()); + } } #[cfg(feature = "database")] @@ -456,7 +669,7 @@ impl SeriesScanStateRepository for PostgresSeriesScanStateRepository { })) } - async fn mark_discovered( + async fn enroll_resolution_generation( &self, library_id: LibraryId, series_root_path: SeriesRootPath, @@ -468,75 +681,134 @@ impl SeriesScanStateRepository for PostgresSeriesScanStateRepository { ( Some(hint.title.clone()), hint.slug.clone(), - hint.year.map(|v| v as i16), + hint.year.map(|value| value as i16), hint.region.clone(), ) }) .unwrap_or((None, None, None, None)); - let row = sqlx::query!( + sqlx::query( r#" INSERT INTO series_scan_state ( library_id, series_root_path, status, series_title, series_slug, series_year, series_region, attempts, created_at, updated_at ) - VALUES ($1, $2, $3, $4, $5, $6, $7, 0, NOW(), NOW()) + VALUES ( + $1, $2, 'discovered'::series_scan_status, + $3, $4, $5, $6, 0, NOW(), NOW() + ) ON CONFLICT (library_id, series_root_path) DO UPDATE SET - series_title = COALESCE(EXCLUDED.series_title, series_scan_state.series_title), - series_slug = COALESCE(EXCLUDED.series_slug, series_scan_state.series_slug), - series_year = COALESCE(EXCLUDED.series_year, series_scan_state.series_year), - series_region = COALESCE(EXCLUDED.series_region, series_scan_state.series_region), + series_title = COALESCE( + EXCLUDED.series_title, + series_scan_state.series_title + ), + series_slug = COALESCE( + EXCLUDED.series_slug, + series_scan_state.series_slug + ), + series_year = COALESCE( + EXCLUDED.series_year, + series_scan_state.series_year + ), + series_region = COALESCE( + EXCLUDED.series_region, + series_scan_state.series_region + ), status = CASE - WHEN series_scan_state.status = 'resolved' THEN series_scan_state.status - ELSE 'discovered' + WHEN series_scan_state.status <> 'resolved' + THEN 'discovered'::series_scan_status + ELSE series_scan_state.status + END, + series_id = CASE + WHEN series_scan_state.status <> 'resolved' THEN NULL + ELSE series_scan_state.series_id + END, + resolved_at = CASE + WHEN series_scan_state.status <> 'resolved' THEN NULL + ELSE series_scan_state.resolved_at + END, + failed_at = CASE + WHEN series_scan_state.status <> 'resolved' THEN NULL + ELSE series_scan_state.failed_at + END, + failure_reason = CASE + WHEN series_scan_state.status <> 'resolved' THEN NULL + ELSE series_scan_state.failure_reason END, updated_at = NOW() - RETURNING library_id, series_root_path, status as "status: SeriesScanStatus", - series_id, series_title, series_slug, series_year, series_region, - seeded_at, last_attempt_at, attempts, - resolved_at, failed_at, failure_reason, created_at, updated_at "#, - library_id.0, - series_root_path.as_str(), - SeriesScanStatus::Discovered as SeriesScanStatus, - title, - slug, - year, - region ) - .fetch_one(&self.pool) + .bind(library_id.0) + .bind(series_root_path.as_str()) + .bind(title) + .bind(slug) + .bind(year) + .bind(region) + .execute(&self.pool) .await?; - Ok(SeriesScanState { - library_id, - series_root_path: SeriesRootPath::try_new(row.series_root_path)?, - status: row.status, - series_id: row.series_id.map(SeriesID), - hint: if row.series_title.is_some() - || row.series_slug.is_some() - || row.series_year.is_some() - || row.series_region.is_some() - { - Some(SeriesHint { - title: row.series_title.unwrap_or_default(), - slug: row.series_slug, - year: row.series_year.map(|v| v as u16), - region: row.series_region, - }) - } else { - None - }, - seeded_at: row.seeded_at, - last_attempt_at: row.last_attempt_at, - attempts: row.attempts as u32, - resolved_at: row.resolved_at, - failed_at: row.failed_at, - failure_reason: row.failure_reason, - created_at: row.created_at, - updated_at: row.updated_at, - }) + self.get(library_id, &series_root_path) + .await? + .ok_or_else(|| { + MediaError::Internal(format!( + "series resolution enrollment disappeared for {}", + series_root_path.as_str() + )) + }) + } + + async fn mark_discovered( + &self, + library_id: LibraryId, + series_root_path: SeriesRootPath, + hint: Option, + ) -> Result { + let (title, slug, year, region) = hint + .as_ref() + .map(|hint| { + ( + Some(hint.title.clone()), + hint.slug.clone(), + hint.year.map(|v| v as i16), + hint.region.clone(), + ) + }) + .unwrap_or((None, None, None, None)); + + sqlx::query( + r#" + INSERT INTO series_scan_state ( + library_id, series_root_path, status, + series_title, series_slug, series_year, series_region, + attempts, created_at, updated_at + ) + VALUES ( + $1, $2, 'discovered'::series_scan_status, + $3, $4, $5, $6, 0, NOW(), NOW() + ) + ON CONFLICT (library_id, series_root_path) + DO NOTHING + "#, + ) + .bind(library_id.0) + .bind(series_root_path.as_str()) + .bind(title) + .bind(slug) + .bind(year) + .bind(region) + .execute(&self.pool) + .await?; + + self.get(library_id, &series_root_path) + .await? + .ok_or_else(|| { + MediaError::Internal(format!( + "observed series state disappeared for {}", + series_root_path.as_str() + )) + }) } async fn mark_seeded( @@ -577,6 +849,16 @@ impl SeriesScanStateRepository for PostgresSeriesScanStateRepository { seeded_at = COALESCE(series_scan_state.seeded_at, NOW()), last_attempt_at = NOW(), attempts = series_scan_state.attempts + 1, + failed_at = CASE + WHEN series_scan_state.status = 'resolved' + THEN series_scan_state.failed_at + ELSE NULL + END, + failure_reason = CASE + WHEN series_scan_state.status = 'resolved' + THEN series_scan_state.failure_reason + ELSE NULL + END, updated_at = NOW() RETURNING library_id, series_root_path, status as "status: SeriesScanStatus", series_id, series_title, series_slug, series_year, series_region, diff --git a/crates/ferrex-server/src/infra/orchestration/maintenance.rs b/crates/ferrex-server/src/infra/orchestration/maintenance.rs index c316f210..c0ad8a97 100644 --- a/crates/ferrex-server/src/infra/orchestration/maintenance.rs +++ b/crates/ferrex-server/src/infra/orchestration/maintenance.rs @@ -719,6 +719,7 @@ mod tests { kind: JobKind::FolderScan, media_id: None, indexing_change: None, + series_identity: None, state, attempts: 0, dedupe_key: format!("scan:test:{job_id}"), diff --git a/crates/ferrex-server/src/infra/orchestration/mod.rs b/crates/ferrex-server/src/infra/orchestration/mod.rs index 1474f7e0..af2f6269 100644 --- a/crates/ferrex-server/src/infra/orchestration/mod.rs +++ b/crates/ferrex-server/src/infra/orchestration/mod.rs @@ -162,10 +162,11 @@ impl ScanOrchestrator { config.budget.transcript_extraction_limit = transcript_concurrency; } - // Job lifecycle state is recovered from PostgreSQL on lag. Scan-domain - // events also drive live per-item catalog projection, so isolate that - // stream and give it enough headroom for large discovery bursts. - let events = Arc::new(InProcJobEventBus::with_capacities(256, 8192)); + // Give lifecycle events enough headroom for discovery and terminal + // bursts. PostgreSQL remains the recovery authority if this exceptional + // buffer is exceeded. Scan-domain events also drive live per-item + // catalog projection, so keep that stream independently buffered. + let events = Arc::new(InProcJobEventBus::with_capacities(32_768, 8192)); let correlations = CorrelationCache::default(); let actors = Arc::new(ActorSystem::new( Arc::clone(&tmdb), diff --git a/crates/ferrex-server/src/infra/scan/scan_manager.rs b/crates/ferrex-server/src/infra/scan/scan_manager.rs index af952ed3..1ff7f42e 100644 --- a/crates/ferrex-server/src/infra/scan/scan_manager.rs +++ b/crates/ferrex-server/src/infra/scan/scan_manager.rs @@ -49,7 +49,7 @@ use std::{ use tokio::{ spawn, sync::{Mutex, RwLock, broadcast}, - time::interval, + time::{MissedTickBehavior, interval}, }; use tracing::{info, instrument, warn}; @@ -69,6 +69,8 @@ const DEFAULT_QUIESCENCE: Duration = Duration::from_secs(3); const STALLED_SCAN_TIMEOUT_MULTIPLIER: u32 = 5; const SERIES_BUNDLE_TRACKER_IDLE_TTL: Duration = Duration::from_secs(10 * 60); const SERIES_BUNDLE_POLL_INTERVAL: Duration = Duration::from_secs(1); +const DURABLE_RECONCILIATION_INITIAL_BACKOFF: Duration = Duration::from_secs(5); +const DURABLE_RECONCILIATION_MAX_BACKOFF: Duration = Duration::from_secs(30); fn subject_key_path(key: &SubjectKey) -> Option<&str> { match key { @@ -3592,6 +3594,7 @@ mod tests { kind: JobKind::FolderScan, media_id: None, indexing_change: None, + series_identity: None, state, attempts, dedupe_key: dedupe_key.into(), @@ -3774,37 +3777,202 @@ mod tests { assert!(gate.is_required(correlation_id).await); assert!(!gate.allows_terminal(correlation_id).await); - gate.reconciled(correlation_id).await; + let ticket = gate.begin_reconciliation(correlation_id).await; + assert!(gate.reconciled(correlation_id, &ticket).await); assert!(gate.allows_terminal(correlation_id).await); } + #[tokio::test] + async fn reconciliation_pacing_backs_off_until_a_new_trigger_resets_it() { + let correlation_id = Uuid::now_v7(); + let pacing = DurableReconciliationPacing::default(); + let started_at = Instant::now(); + + assert!(pacing.allows_attempt(correlation_id, started_at).await); + pacing.defer_next_attempt(correlation_id, started_at).await; + assert!( + !pacing + .allows_attempt( + correlation_id, + started_at + DURABLE_RECONCILIATION_INITIAL_BACKOFF + - Duration::from_millis(1), + ) + .await + ); + + pacing.forget(correlation_id).await; + assert!( + pacing + .allows_attempt( + correlation_id, + started_at + Duration::from_millis(1), + ) + .await, + "a new lag or enrollment trigger must force one immediate attempt" + ); + pacing.defer_next_attempt(correlation_id, started_at).await; + assert!( + !pacing + .allows_attempt( + correlation_id, + started_at + Duration::from_millis(2), + ) + .await, + "the same still-required gate must not requery every tick" + ); + + let second_attempt = + started_at + DURABLE_RECONCILIATION_INITIAL_BACKOFF; + assert!(pacing.allows_attempt(correlation_id, second_attempt).await); + pacing + .defer_next_attempt(correlation_id, second_attempt) + .await; + assert!( + !pacing + .allows_attempt( + correlation_id, + second_attempt + DURABLE_RECONCILIATION_INITIAL_BACKOFF, + ) + .await, + "an unchanged active snapshot doubles the retry delay" + ); + assert!( + pacing + .allows_attempt( + correlation_id, + second_attempt + DURABLE_RECONCILIATION_INITIAL_BACKOFF * 2, + ) + .await + ); + + pacing.forget(correlation_id).await; + assert!( + pacing.allows_attempt(correlation_id, second_attempt).await, + "terminal or forgotten runs must discard their cooldown" + ); + } + #[tokio::test] async fn catalog_lag_gate_waits_for_seed_and_terminal_index_jobs() { let correlation_id = Uuid::now_v7(); let gate = DurableReconciliationGate::default(); gate.require_catalog_projection_recovery([correlation_id]) .await; + let ticket = gate.begin_reconciliation(correlation_id).await; - assert!(gate.requires_catalog_projection(correlation_id).await); + assert!(ticket.catalog_projection_required); assert!( !gate - .clear_after_seeded_snapshot(correlation_id, false, true) + .clear_after_seeded_snapshot( + correlation_id, + &ticket, + false, + true, + ) .await ); assert!( !gate - .clear_after_seeded_snapshot(correlation_id, true, false) + .clear_after_seeded_snapshot( + correlation_id, + &ticket, + true, + false, + ) .await ); assert!(gate.is_required(correlation_id).await); - assert!(gate.requires_catalog_projection(correlation_id).await); + assert!( + gate.begin_reconciliation(correlation_id) + .await + .catalog_projection_required + ); assert!( - gate.clear_after_seeded_snapshot(correlation_id, true, true) + gate.clear_after_seeded_snapshot( + correlation_id, + &ticket, + true, + true, + ) + .await + ); + assert!(gate.allows_terminal(correlation_id).await); + assert!( + !gate + .begin_reconciliation(correlation_id) + .await + .catalog_projection_required + ); + } + + #[tokio::test] + async fn reconciliation_generation_preserves_triggers_arriving_in_flight() { + let correlation_id = Uuid::now_v7(); + let retry_job_id = JobId::new(); + let gate = DurableReconciliationGate::default(); + + gate.require_all([correlation_id]).await; + let snapshot_ticket = gate.begin_reconciliation(correlation_id).await; + + // A second lag arrives while the durable snapshot is in flight. + gate.require_all([correlation_id]).await; + assert!( + !gate + .clear_after_seeded_snapshot( + correlation_id, + &snapshot_ticket, + true, + true, + ) .await ); + assert!(gate.is_required(correlation_id).await); + + let lag_ticket = gate.begin_reconciliation(correlation_id).await; + // Catalog recovery is requested while that snapshot is projecting. + gate.require_catalog_projection_recovery([correlation_id]) + .await; + assert!( + !gate + .clear_after_seeded_snapshot( + correlation_id, + &lag_ticket, + true, + true, + ) + .await + ); + + let catalog_ticket = gate.begin_reconciliation(correlation_id).await; + assert!(catalog_ticket.catalog_projection_required); + // A retry outcome becomes uncertain before the catalog pass settles. + gate.require_retryable_failure(correlation_id, retry_job_id) + .await; + assert!( + !gate + .clear_after_seeded_snapshot( + correlation_id, + &catalog_ticket, + true, + true, + ) + .await + ); + + let latest_ticket = gate.begin_reconciliation(correlation_id).await; + assert!(latest_ticket.catalog_projection_required); + assert!(latest_ticket.retryable_failures.contains(&retry_job_id)); + assert!( + gate.clear_after_seeded_snapshot( + correlation_id, + &latest_ticket, + true, + true, + ) + .await + ); assert!(gate.allows_terminal(correlation_id).await); - assert!(!gate.requires_catalog_projection(correlation_id).await); } #[test] @@ -3845,6 +4013,7 @@ mod tests { let now = Utc::now(); let gate = DurableReconciliationGate::default(); gate.require_retryable_failure(correlation_id, job_id).await; + let ticket = gate.begin_reconciliation(correlation_id).await; let leased = durable_job( correlation_id, @@ -3856,8 +4025,7 @@ mod tests { now, ); assert_eq!( - gate.unresolved_retryable_failures(correlation_id, &[leased]) - .await, + gate.unresolved_retryable_failures(&ticket, &[leased]).await, vec![(job_id, Some(JobState::Leased))] ); assert!(!gate.allows_terminal(correlation_id).await); @@ -3872,11 +4040,11 @@ mod tests { now + ChronoDuration::milliseconds(1), ); assert!( - gate.unresolved_retryable_failures(correlation_id, &[ready]) + gate.unresolved_retryable_failures(&ticket, &[ready]) .await .is_empty() ); - gate.reconciled(correlation_id).await; + assert!(gate.reconciled(correlation_id, &ticket).await); assert!(gate.allows_terminal(correlation_id).await); } @@ -3968,6 +4136,7 @@ mod tests { let mut receiver = run.subscribe(); let gate = DurableReconciliationGate::default(); gate.require_all([correlation_id]).await; + let pre_seed_ticket = gate.begin_reconciliation(correlation_id).await; // The first ticker can race Start and observe an empty database before // its enqueue batch is complete. Production reconcile_run must not @@ -3977,6 +4146,7 @@ mod tests { !gate .clear_after_seeded_snapshot( correlation_id, + &pre_seed_ticket, run.seed_completed().await, true, ) @@ -4024,6 +4194,7 @@ mod tests { assert!(!early_completion); assert_eq!(run.lifecycle_status().await, ScanLifecycleStatus::Running); + let post_seed_ticket = gate.begin_reconciliation(correlation_id).await; run.reconcile_durable_job_states(&[durable_job( correlation_id, shared_job_id, @@ -4037,6 +4208,7 @@ mod tests { assert!( gate.clear_after_seeded_snapshot( correlation_id, + &post_seed_ticket, run.seed_completed().await, true, ) @@ -5051,7 +5223,7 @@ mod tests { let barrier = Arc::clone(&barrier); tokio::spawn(async move { barrier.wait().await; - entry.lock().await.claim_next_finalization().is_some() + entry.lock().await.claim_next_finalization() }) }; let second = { @@ -5059,27 +5231,95 @@ mod tests { let barrier = Arc::clone(&barrier); tokio::spawn(async move { barrier.wait().await; - entry.lock().await.claim_next_finalization().is_some() + entry.lock().await.claim_next_finalization() }) }; barrier.wait().await; let (first, second) = tokio::join!(first, second); - let claims = usize::from(first.unwrap()) + usize::from(second.unwrap()); + let first = first.unwrap(); + let second = second.unwrap(); + let claims = + usize::from(first.is_some()) + usize::from(second.is_some()); assert_eq!(claims, 1, "only one worker may claim the same bundle"); + let claimed = first.or(second).expect("one worker claimed the bundle"); let mut guard = entry.lock().await; - guard.settle_finalization(&series_root_path, false); - assert!( - guard.claim_next_finalization().is_some(), - "a failed publication must release its claim for polling retries" + assert!(!guard.settle_finalization(&claimed, false)); + let retry = guard.claim_next_finalization().expect( + "a failed publication releases its claim for polling retries", ); - guard.settle_finalization(&series_root_path, true); + assert!(guard.settle_finalization(&retry, true)); assert!( guard.claim_next_finalization().is_none(), "a published bundle must remain finalized" ); } + + #[test] + fn series_bundle_claim_revalidates_work_enrolled_during_publish() { + use ferrex_core::domain::scan::orchestration::job::JobPriority; + + let library_id = LibraryId::new(); + let series_id = ferrex_core::types::SeriesID(Uuid::now_v7()); + let series_root_path = + SeriesRootPath::try_new("/library/Claim Race").unwrap(); + let episode_path = "/library/Claim Race/Season 1/S01E02.mkv"; + let mut entry = ready_series_bundle_entry( + library_id, + series_id, + series_root_path.clone(), + ); + + let claimed = entry + .claim_next_finalization() + .expect("the initial bundle is ready"); + let late_job_id = JobId::new(); + + // Simulate episode enrollment while catalog publication is awaiting its + // database projection. + entry.tracker.observe_job_event(&JobEvent::from_job( + None, + library_id, + "episode:late:index".into(), + SubjectKey::path(episode_path).ok(), + JobEventPayload::Enqueued { + job_id: late_job_id, + kind: JobKind::IndexUpsert, + priority: JobPriority::P0, + }, + )); + assert!( + entry.claim_next_finalization().is_none(), + "the original in-flight claim prevents a duplicate publisher" + ); + + // Even if the late episode terminalizes before publication returns, the + // original claim still describes the pre-enrollment generation. It must + // be rejected and published again from the updated generation. + entry.tracker.observe_job_event(&JobEvent::from_job( + None, + library_id, + "episode:late:index".into(), + SubjectKey::path(episode_path).ok(), + JobEventPayload::DeadLettered { + job_id: late_job_id, + kind: JobKind::IndexUpsert, + priority: JobPriority::P0, + }, + )); + assert!( + !entry.settle_finalization(&claimed, true), + "a successful publication from a stale generation must not finalize the root" + ); + + let retry = entry + .claim_next_finalization() + .expect("the updated terminal generation must be published again"); + assert_eq!(retry.series_root_path, series_root_path); + assert!(entry.settle_finalization(&retry, true)); + assert!(entry.claim_next_finalization().is_none()); + } } #[derive(Clone)] @@ -5095,6 +5335,53 @@ struct ScanRunAggregatorInner { catalog_events: CatalogEventProjection, series_bundles: Mutex>, reconciliation_gate: DurableReconciliationGate, + durable_reconciliation_pacing: DurableReconciliationPacing, +} + +#[derive(Default)] +struct DurableReconciliationPacing { + state: Mutex>, +} + +#[derive(Clone, Copy)] +struct DurableReconciliationPacingState { + not_before: Instant, + backoff: Duration, +} + +impl DurableReconciliationPacing { + async fn allows_attempt(&self, correlation_id: Uuid, now: Instant) -> bool { + self.state + .lock() + .await + .get(&correlation_id) + .is_none_or(|state| now >= state.not_before) + } + + async fn defer_next_attempt(&self, correlation_id: Uuid, now: Instant) { + let mut state = self.state.lock().await; + let backoff = state + .get(&correlation_id) + .map(|current| { + current + .backoff + .checked_mul(2) + .unwrap_or(DURABLE_RECONCILIATION_MAX_BACKOFF) + .min(DURABLE_RECONCILIATION_MAX_BACKOFF) + }) + .unwrap_or(DURABLE_RECONCILIATION_INITIAL_BACKOFF); + state.insert( + correlation_id, + DurableReconciliationPacingState { + not_before: now + backoff, + backoff, + }, + ); + } + + async fn forget(&self, correlation_id: Uuid) { + self.state.lock().await.remove(&correlation_id); + } } #[derive(Default)] @@ -5104,24 +5391,57 @@ struct DurableReconciliationGate { #[derive(Default)] struct DurableReconciliationRequirements { - required: HashSet, - catalog_projection_required: HashSet, - retryable_failures: HashMap>, + next_generation: u64, + by_correlation: HashMap, +} + +#[derive(Clone, Default)] +struct DurableReconciliationRequirement { + generation: u64, + catalog_projection_required: bool, + retryable_failures: HashSet, +} + +#[derive(Clone, Default)] +struct DurableReconciliationTicket { + generation: Option, + catalog_projection_required: bool, + retryable_failures: HashSet, +} + +impl DurableReconciliationRequirements { + fn require( + &mut self, + correlation_id: Uuid, + ) -> &mut DurableReconciliationRequirement { + self.next_generation = self + .next_generation + .checked_add(1) + .expect("durable reconciliation generation exhausted"); + let generation = self.next_generation; + let requirement = + self.by_correlation.entry(correlation_id).or_default(); + requirement.generation = generation; + requirement + } } impl DurableReconciliationGate { async fn require_all(&self, correlations: impl IntoIterator) { - self.state.lock().await.required.extend(correlations); + let mut state = self.state.lock().await; + for correlation_id in correlations { + state.require(correlation_id); + } } async fn require_catalog_projection_recovery( &self, correlations: impl IntoIterator, ) { - let correlations = correlations.into_iter().collect::>(); let mut state = self.state.lock().await; - state.required.extend(correlations.iter().copied()); - state.catalog_projection_required.extend(correlations); + for correlation_id in correlations { + state.require(correlation_id).catalog_projection_required = true; + } } async fn require_retryable_failure( @@ -5130,71 +5450,87 @@ impl DurableReconciliationGate { job_id: JobId, ) { let mut state = self.state.lock().await; - state.required.insert(correlation_id); state + .require(correlation_id) .retryable_failures - .entry(correlation_id) - .or_default() .insert(job_id); } async fn is_required(&self, correlation_id: Uuid) -> bool { - self.state.lock().await.required.contains(&correlation_id) + self.state + .lock() + .await + .by_correlation + .contains_key(&correlation_id) } async fn allows_terminal(&self, correlation_id: Uuid) -> bool { !self.is_required(correlation_id).await } - async fn requires_catalog_projection(&self, correlation_id: Uuid) -> bool { + async fn begin_reconciliation( + &self, + correlation_id: Uuid, + ) -> DurableReconciliationTicket { self.state .lock() .await - .catalog_projection_required - .contains(&correlation_id) + .by_correlation + .get(&correlation_id) + .map(|requirement| DurableReconciliationTicket { + generation: Some(requirement.generation), + catalog_projection_required: requirement + .catalog_projection_required, + retryable_failures: requirement.retryable_failures.clone(), + }) + .unwrap_or_default() } - async fn reconciled(&self, correlation_id: Uuid) { + async fn reconciled( + &self, + correlation_id: Uuid, + ticket: &DurableReconciliationTicket, + ) -> bool { let mut state = self.state.lock().await; - state.required.remove(&correlation_id); - state.catalog_projection_required.remove(&correlation_id); - state.retryable_failures.remove(&correlation_id); + let current_generation = state + .by_correlation + .get(&correlation_id) + .map(|requirement| requirement.generation); + if current_generation != ticket.generation { + return false; + } + state.by_correlation.remove(&correlation_id); + true } async fn clear_after_seeded_snapshot( &self, correlation_id: Uuid, + ticket: &DurableReconciliationTicket, seed_completed: bool, catalog_projection_complete: bool, ) -> bool { if !seed_completed || !catalog_projection_complete { return false; } - self.reconciled(correlation_id).await; - true + self.reconciled(correlation_id, ticket).await } async fn unresolved_retryable_failures( &self, - correlation_id: Uuid, + ticket: &DurableReconciliationTicket, jobs: &[DurableJobState], ) -> Vec<(JobId, Option)> { - let required = { - let state = self.state.lock().await; - state - .retryable_failures - .get(&correlation_id) - .cloned() - .unwrap_or_default() - }; - if required.is_empty() { + if ticket.retryable_failures.is_empty() { return Vec::new(); } let durable_states: HashMap<_, _> = jobs.iter().map(|job| (job.job_id, job.state)).collect(); - required - .into_iter() + ticket + .retryable_failures + .iter() + .copied() .filter_map(|job_id| { let state = durable_states.get(&job_id).copied(); (!matches!( @@ -5211,7 +5547,11 @@ impl DurableReconciliationGate { } async fn forget(&self, correlation_id: Uuid) { - self.reconciled(correlation_id).await; + self.state + .lock() + .await + .by_correlation + .remove(&correlation_id); } } @@ -5249,16 +5589,17 @@ impl SeriesBundleTrackerEntry { fn settle_finalization( &mut self, - series_root_path: &SeriesRootPath, + finalization: &SeriesBundleFinalization, published: bool, - ) { - if !self.finalizations_in_flight.remove(series_root_path) { - return; + ) -> bool { + if !self + .finalizations_in_flight + .remove(&finalization.series_root_path) + { + return false; } - if published { - self.tracker.mark_finalized(series_root_path); - } + published && self.tracker.mark_finalized_if_still_eligible(finalization) } } @@ -5275,6 +5616,8 @@ impl ScanRunAggregator { .unwrap_or(Duration::from_secs(60)); let stall_window = ChronoDuration::from_std(stall_std) .unwrap_or_else(|_| ChronoDuration::seconds(60)); + let durable_reconciliation_pacing = + DurableReconciliationPacing::default(); let inner = Arc::new(ScanRunAggregatorInner { orchestrator, runs: RwLock::new(HashMap::new()), @@ -5283,6 +5626,7 @@ impl ScanRunAggregator { catalog_events, series_bundles: Mutex::new(HashMap::new()), reconciliation_gate: DurableReconciliationGate::default(), + durable_reconciliation_pacing, }); let aggregator = Self { @@ -5293,9 +5637,17 @@ impl ScanRunAggregator { } fn spawn_worker(&self) { - let job_progress = Arc::clone(&self.inner); + let job_events = Arc::clone(&self.inner); spawn(async move { - ScanRunAggregatorInner::run_job_progress(job_progress).await; + ScanRunAggregatorInner::run_job_events(job_events).await; + }); + + let durable_reconciliation = Arc::clone(&self.inner); + spawn(async move { + ScanRunAggregatorInner::run_durable_reconciliation( + durable_reconciliation, + ) + .await; }); let scan_events = Arc::clone(&self.inner); @@ -5316,6 +5668,10 @@ impl ScanRunAggregator { .reconciliation_gate .require_all([correlation_id]) .await; + self.inner + .durable_reconciliation_pacing + .forget(correlation_id) + .await; } async fn drop(&self, correlation_id: &Uuid) { @@ -5323,6 +5679,10 @@ impl ScanRunAggregator { guard.remove(correlation_id); drop(guard); self.inner.reconciliation_gate.forget(*correlation_id).await; + self.inner + .durable_reconciliation_pacing + .forget(*correlation_id) + .await; } async fn forget_library(&self, library_id: LibraryId) { @@ -5341,38 +5701,47 @@ impl ScanRunAggregator { for correlation_id in correlations { self.inner.reconciliation_gate.forget(correlation_id).await; + self.inner + .durable_reconciliation_pacing + .forget(correlation_id) + .await; } self.inner.series_bundles.lock().await.remove(&library_id); } } impl ScanRunAggregatorInner { - async fn run_job_progress(self: Arc) { + async fn run_job_events(self: Arc) { use tokio::sync::broadcast::error::RecvError; let mut receiver = self.orchestrator.subscribe_job_events(); - let mut ticker = interval(Duration::from_millis(500)); loop { - tokio::select! { - result = receiver.recv() => { - match result { - Ok(event) => self.handle_job_event(event).await, - Err(RecvError::Lagged(skipped)) => { - warn!("scan aggregator lagged {skipped} events"); - self.mark_all_runs_reconciliation_required().await; - self.reconcile_all_runs("job_event_lag").await; - } - Err(RecvError::Closed) => break, - } - } - _ = ticker.tick() => { - self.check_quiescence().await; + match receiver.recv().await { + Ok(event) => self.handle_job_event(event).await, + Err(RecvError::Lagged(skipped)) => { + warn!("scan aggregator lagged {skipped} events"); + // Keep draining immediately. PostgreSQL reconciliation can + // require decoding every job in a large run; doing that in + // this receiver loop creates a feedback cycle where the + // durable read itself causes another broadcast lag. + self.mark_all_runs_reconciliation_required().await; } + Err(RecvError::Closed) => break, } } } + async fn run_durable_reconciliation(self: Arc) { + let mut ticker = interval(Duration::from_millis(500)); + ticker.set_missed_tick_behavior(MissedTickBehavior::Delay); + + loop { + ticker.tick().await; + self.check_quiescence().await; + } + } + async fn run_scan_events(self: Arc) { use tokio::sync::broadcast::error::RecvError; @@ -5400,14 +5769,30 @@ impl ScanRunAggregatorInner { }; for run in runs { - let lag_reconciliation_required = self + let explicit_reconciliation_required = self .reconciliation_gate .is_required(run.correlation_id()) .await; let pre_stall = run.needs_pre_stall_reconciliation(self.stall_timeout).await; - if lag_reconciliation_required || pre_stall { - let reason = if lag_reconciliation_required { + let reconciliation_requested = + explicit_reconciliation_required || pre_stall; + let reconciliation_due = reconciliation_requested + && self + .durable_reconciliation_pacing + .allows_attempt(run.correlation_id(), Instant::now()) + .await; + + if reconciliation_requested && !reconciliation_due { + // The last attempt still observed active jobs, an incomplete + // seed, or an unavailable durable store. Preserve that gate + // until its bounded retry is due instead of re-reading the + // entire run every 500 ms. + continue; + } + + if reconciliation_due { + let reason = if explicit_reconciliation_required { "lag_retry" } else { "pre_stall" @@ -5420,6 +5805,12 @@ impl ScanRunAggregatorInner { reason, "durable scan reconciliation failed; deferring terminal decision" ); + self.durable_reconciliation_pacing + .defer_next_attempt( + run.correlation_id(), + Instant::now(), + ) + .await; continue; } @@ -5467,33 +5858,26 @@ impl ScanRunAggregatorInner { async fn mark_all_runs_reconciliation_required(&self) { let correlations: Vec = self.runs.read().await.keys().copied().collect(); - self.reconciliation_gate.require_all(correlations).await; + self.reconciliation_gate + .require_all(correlations.iter().copied()) + .await; + for correlation_id in correlations { + self.durable_reconciliation_pacing + .forget(correlation_id) + .await; + } } async fn mark_all_runs_catalog_reconciliation_required(&self) { let correlations: Vec = self.runs.read().await.keys().copied().collect(); self.reconciliation_gate - .require_catalog_projection_recovery(correlations) + .require_catalog_projection_recovery(correlations.iter().copied()) .await; - } - - async fn reconcile_all_runs(&self, reason: &'static str) { - let runs: Vec> = { - let guard = self.runs.read().await; - guard.values().cloned().collect() - }; - - for run in runs { - if let Err(err) = self.reconcile_run(&run, reason).await { - warn!( - scan = %run.scan_id(), - library = %run.library_id(), - error = %err, - reason, - "failed to reconcile scan run from durable job state" - ); - } + for correlation_id in correlations { + self.durable_reconciliation_pacing + .forget(correlation_id) + .await; } } @@ -5502,6 +5886,14 @@ impl ScanRunAggregatorInner { run: &Arc, reason: &'static str, ) -> ferrex_core::error::Result<()> { + // This ticket fences every async read and projection below. A lag, + // retry, or catalog trigger that arrives while reconciliation is in + // flight advances the generation and cannot be cleared by this older + // snapshot. + let reconciliation_ticket = self + .reconciliation_gate + .begin_reconciliation(run.correlation_id()) + .await; let tracked_job_ids = run.tracked_job_ids().await; let jobs = self .orchestrator @@ -5532,7 +5924,7 @@ impl ScanRunAggregatorInner { } let unresolved_retryable_failures = self .reconciliation_gate - .unresolved_retryable_failures(run.correlation_id(), &jobs) + .unresolved_retryable_failures(&reconciliation_ticket, &jobs) .await; if !unresolved_retryable_failures.is_empty() { return Err(ferrex_core::error::MediaError::Internal(format!( @@ -5542,15 +5934,12 @@ impl ScanRunAggregatorInner { ))); } - let catalog_projection_complete = if self - .reconciliation_gate - .requires_catalog_projection(run.correlation_id()) - .await - { - self.reconcile_catalog_projections(run, &jobs).await - } else { - true - }; + let catalog_projection_complete = + if reconciliation_ticket.catalog_projection_required { + self.reconcile_catalog_projections(run, &jobs).await + } else { + true + }; tracing::debug!( scan = %run.scan_id(), @@ -5562,10 +5951,22 @@ impl ScanRunAggregatorInner { "reconciling scan progress from durable job state" ); run.reconcile_durable_job_states(&jobs).await; + if jobs.iter().any(|job| job.series_identity.is_some()) { + let now = Instant::now(); + let mut guard = self.series_bundles.lock().await; + let entry = guard + .entry(run.library_id()) + .or_insert_with(|| SeriesBundleTrackerEntry::new(now)); + entry.touch(now); + entry + .tracker + .reconcile_durable_job_states(run.library_id(), &jobs); + } if !self .reconciliation_gate .clear_after_seeded_snapshot( run.correlation_id(), + &reconciliation_ticket, run.seed_completed().await, catalog_projection_complete, ) @@ -5575,9 +5976,24 @@ impl ScanRunAggregatorInner { scan = %run.scan_id(), library = %run.library_id(), reason, - "durable snapshot preceded seed completion; retaining reconciliation gate" + "durable snapshot did not satisfy the current reconciliation generation; retaining gate" ); } + + if self + .reconciliation_gate + .is_required(run.correlation_id()) + .await + || run.has_active_items().await + { + self.durable_reconciliation_pacing + .defer_next_attempt(run.correlation_id(), Instant::now()) + .await; + } else { + self.durable_reconciliation_pacing + .forget(run.correlation_id()) + .await; + } Ok(()) } @@ -5813,6 +6229,9 @@ impl ScanRunAggregatorInner { *job_id, ) .await; + self.durable_reconciliation_pacing + .forget(run.correlation_id()) + .await; } run.record_job_failure( &event.meta.idempotency_key, @@ -5825,18 +6244,9 @@ impl ScanRunAggregatorInner { .await; if *retryable { - if let Err(err) = self - .reconcile_run(&run, "retryable_job_failure") - .await - { - warn!( - scan = %run.scan_id(), - library = %run.library_id(), - job = %job_id, - error = %err, - "retryable job failure is not yet confirmed durably; terminalization remains gated" - ); - } + // Keep the broadcast receiver draining. The dedicated + // durable worker observes the reset gate on its next + // tick and confirms the authoritative retry outcome. false } else if terminalization_allowed { run.try_complete( @@ -5893,19 +6303,6 @@ impl ScanRunAggregatorInner { .or_insert_with(|| SeriesBundleTrackerEntry::new(now)); entry.touch(now); entry.tracker.observe_job_event(event); - - drop(guard); - - match &event.payload { - JobEventPayload::Completed { .. } - | JobEventPayload::DeadLettered { .. } - | JobEventPayload::Failed { - retryable: false, .. - } => { - self.try_emit_series_bundle_finalized(library_id).await; - } - _ => {} - } } async fn handle_scan_event(&self, event: ScanEvent) { @@ -5944,6 +6341,12 @@ impl ScanRunAggregatorInner { if let Some(run) = run { let has_enrollment = !summary.enrolled_job_ids.is_empty(); + // Seed completion changes whether the registration + // snapshot is authoritative, even for an empty run. + // Let the dedicated worker retry immediately. + self.durable_reconciliation_pacing + .forget(correlation_id) + .await; if has_enrollment { // Even if every enqueue notification was dropped, // the seed mailbox has the exact durable job IDs. @@ -5968,6 +6371,12 @@ impl ScanRunAggregatorInner { .reconcile_run(&run, "post_seed_enrollment") .await { + self.durable_reconciliation_pacing + .defer_next_attempt( + correlation_id, + Instant::now(), + ) + .await; warn!( scan = %run.scan_id(), library = %run.library_id(), @@ -6110,10 +6519,7 @@ impl ScanRunAggregatorInner { let mut guard = self.series_bundles.lock().await; if let Some(entry) = guard.get_mut(&library_id) { - entry.settle_finalization( - &finalization.series_root_path, - frame.is_some(), - ); + entry.settle_finalization(&finalization, frame.is_some()); } drop(guard); From 680910cfc251abd068443cc7cd143c654897d1b6 Mon Sep 17 00:00:00 2001 From: Grayson Hieb Date: Tue, 11 Aug 2026 17:48:34 -0600 Subject: [PATCH 07/10] fix(scan): shorten default maintenance cadence --- crates/ferrex-core/src/api/scan.rs | 6 ++-- crates/ferrex-core/src/api/types/library.rs | 6 ++-- crates/ferrex-model/src/lib.rs | 5 +++- crates/ferrex-model/src/library.rs | 29 ++++++++++++++++++- .../update_handlers/library_management.rs | 7 +++-- .../ui/views/admin/view_library_form.rs | 2 +- .../src/handlers/scan/handle_scan.rs | 4 ++- .../content/docs/operator/configuration.md | 4 +-- 8 files changed, 49 insertions(+), 14 deletions(-) diff --git a/crates/ferrex-core/src/api/scan.rs b/crates/ferrex-core/src/api/scan.rs index e1e60aba..37758039 100644 --- a/crates/ferrex-core/src/api/scan.rs +++ b/crates/ferrex-core/src/api/scan.rs @@ -1,5 +1,7 @@ use chrono::{DateTime, Utc}; -use ferrex_model::{MediaID, scan::scanner::settings}; +use ferrex_model::{ + DEFAULT_SCAN_INTERVAL_MINUTES, MediaID, scan::scanner::settings, +}; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -84,7 +86,7 @@ impl Default for IncrementalScanPolicyView { Self { default_auto_scan: true, default_watch_for_changes: true, - default_scan_interval_minutes: 60, + default_scan_interval_minutes: DEFAULT_SCAN_INTERVAL_MINUTES, watch_strategy: "auto".to_string(), poll_interval_ms: 30_000, debounce_window_ms: 250, diff --git a/crates/ferrex-core/src/api/types/library.rs b/crates/ferrex-core/src/api/types/library.rs index 39ed3106..0e9fca7f 100644 --- a/crates/ferrex-core/src/api/types/library.rs +++ b/crates/ferrex-core/src/api/types/library.rs @@ -10,7 +10,7 @@ use uuid::Uuid; use crate::types::details::LibraryReference; use crate::types::ids::LibraryId; use crate::types::ids::{MovieBatchId, SeriesID}; -use crate::types::library::LibraryType; +use crate::types::library::{DEFAULT_SCAN_INTERVAL_MINUTES, LibraryType}; use crate::types::media::{ EpisodeReference, Media, MovieReference, SeasonReference, Series, }; @@ -222,7 +222,7 @@ pub struct ResetLibraryResult { } fn default_scan_interval() -> u32 { - 60 + DEFAULT_SCAN_INTERVAL_MINUTES } fn default_enabled() -> bool { @@ -258,7 +258,7 @@ mod tests { assert!(request.auto_scan); assert!(request.watch_for_changes); - assert_eq!(request.scan_interval_minutes, 60); + assert_eq!(request.scan_interval_minutes, 15); assert!(!request.analyze_on_scan); assert_eq!(request.max_retry_attempts, 3); } diff --git a/crates/ferrex-model/src/lib.rs b/crates/ferrex-model/src/lib.rs index 626b4def..7ab33300 100644 --- a/crates/ferrex-model/src/lib.rs +++ b/crates/ferrex-model/src/lib.rs @@ -94,7 +94,10 @@ pub use image::{ pub use image_events::ImageReadyEvent; #[cfg(feature = "rkyv")] pub use library::{ArchivedLibrary, ArchivedLibraryExt, ArchivedLibraryType}; -pub use library::{Library, LibraryLike, LibraryLikeMut, LibraryType}; +pub use library::{ + DEFAULT_SCAN_INTERVAL_MINUTES, Library, LibraryLike, LibraryLikeMut, + LibraryType, +}; #[cfg(feature = "rkyv")] pub use media::{ ArchivedEpisodeReference, ArchivedMedia, ArchivedMovieReference, diff --git a/crates/ferrex-model/src/library.rs b/crates/ferrex-model/src/library.rs index af2f9533..5036e5a2 100644 --- a/crates/ferrex-model/src/library.rs +++ b/crates/ferrex-model/src/library.rs @@ -11,6 +11,13 @@ use crate::media::Media; use super::ids::{LibraryId, MovieReferenceBatchSize}; +/// Default cadence for bounded automatic library maintenance. +/// +/// Filesystem watchers normally discover new media immediately. This interval +/// is the fallback for missed notifications and filesystems without reliable +/// native events. +pub const DEFAULT_SCAN_INTERVAL_MINUTES: u32 = 15; + /// Read-only operations for library-like types pub trait LibraryLike { fn needs_scan(&self) -> bool; @@ -189,7 +196,7 @@ impl LibraryLikeMut for Library { name, library_type, paths, - scan_interval_minutes: 60, + scan_interval_minutes: DEFAULT_SCAN_INTERVAL_MINUTES, last_scan: None, enabled: true, auto_scan: true, @@ -259,6 +266,26 @@ pub struct LibraryScanResult { pub duration_seconds: f64, } +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn new_library_uses_incremental_discovery_default() { + let library = ::new( + "Movies".to_string(), + LibraryType::Movies, + vec![PathBuf::from("/media/movies")], + ); + + assert_eq!( + library.scan_interval_minutes, + DEFAULT_SCAN_INTERVAL_MINUTES + ); + assert_eq!(DEFAULT_SCAN_INTERVAL_MINUTES, 15); + } +} + #[cfg(feature = "rkyv")] pub use archived::ArchivedLibraryExt; diff --git a/crates/ferrex-player-ui/src/domains/library/update_handlers/library_management.rs b/crates/ferrex-player-ui/src/domains/library/update_handlers/library_management.rs index 5b6fd9df..6e12231c 100644 --- a/crates/ferrex-player-ui/src/domains/library/update_handlers/library_management.rs +++ b/crates/ferrex-player-ui/src/domains/library/update_handlers/library_management.rs @@ -8,7 +8,7 @@ use ferrex_core::{ }, types::{ids::LibraryId, library::LibraryType}, }; -use ferrex_model::MovieReferenceBatchSize; +use ferrex_model::{DEFAULT_SCAN_INTERVAL_MINUTES, MovieReferenceBatchSize}; use iced::Task; use serde_json::json; use std::path::PathBuf; @@ -282,7 +282,8 @@ pub fn handle_show_library_form( name: String::new(), library_type: "Movies".to_string(), paths: String::new(), - scan_interval_minutes: "60".to_string(), + scan_interval_minutes: DEFAULT_SCAN_INTERVAL_MINUTES + .to_string(), enabled: true, auto_scan: true, watch_for_changes: true, @@ -494,7 +495,7 @@ pub fn handle_submit_library_form(state: &mut State) -> Task { scan_interval_minutes: form_data .scan_interval_minutes .parse() - .unwrap_or(60), + .unwrap_or(DEFAULT_SCAN_INTERVAL_MINUTES), last_scan: None, enabled: form_data.enabled, media: None, diff --git a/crates/ferrex-player-ui/src/domains/ui/views/admin/view_library_form.rs b/crates/ferrex-player-ui/src/domains/ui/views/admin/view_library_form.rs index 9e013372..7c91a74e 100644 --- a/crates/ferrex-player-ui/src/domains/ui/views/admin/view_library_form.rs +++ b/crates/ferrex-player-ui/src/domains/ui/views/admin/view_library_form.rs @@ -212,7 +212,7 @@ pub fn view_library_form<'a>( text("Minimum 1 minute; disable automatic sweeps with the checkbox below") .size(12) .color(theme::MediaServerTheme::TEXT_DIMMED), - text_input("60", &form_data.scan_interval_minutes) + text_input("15", &form_data.scan_interval_minutes) .on_input(|s| { SettingsUiMessage::UpdateLibraryFormScanInterval(s).into() }) diff --git a/crates/ferrex-server/src/handlers/scan/handle_scan.rs b/crates/ferrex-server/src/handlers/scan/handle_scan.rs index af16cbf7..da49e7ec 100644 --- a/crates/ferrex-server/src/handlers/scan/handle_scan.rs +++ b/crates/ferrex-server/src/handlers/scan/handle_scan.rs @@ -25,6 +25,8 @@ use tokio_stream::{StreamExt, wrappers::BroadcastStream}; use tracing::warn; use uuid::Uuid; +use ferrex_model::DEFAULT_SCAN_INTERVAL_MINUTES; + use crate::infra::app_state::AppState; use crate::infra::demo_mode; use crate::infra::scan::scan_manager::{ @@ -688,7 +690,7 @@ pub async fn scan_config_handler( let incremental_policy = IncrementalScanPolicyView { default_auto_scan: true, default_watch_for_changes: true, - default_scan_interval_minutes: 60, + default_scan_interval_minutes: DEFAULT_SCAN_INTERVAL_MINUTES, watch_strategy: watch_strategy_label(cfg.watch.strategy), poll_interval_ms: cfg.watch.poll_interval_ms, debounce_window_ms: cfg.watch.debounce_window_ms, diff --git a/docs/src/content/docs/operator/configuration.md b/docs/src/content/docs/operator/configuration.md index 3485a8d5..f563e637 100644 --- a/docs/src/content/docs/operator/configuration.md +++ b/docs/src/content/docs/operator/configuration.md @@ -114,7 +114,7 @@ just start --mode tailscale ## Scanner / Incremental Scans -Scanner settings can be supplied in `scanner.toml`, `scanner.json`, `config/scanner.toml`, `config/scanner.json`, `SCANNER_CONFIG_PATH`, or inline JSON via `SCANNER_CONFIG_JSON`. Existing installs that do not provide a scanner config keep safe defaults: libraries auto-scan every 60 minutes, filesystem watching is enabled per library, and the watcher uses `auto` strategy. On Linux, `auto` selects polling immediately for CIFS/SMB3 and NFS mounts; other filesystems use native notifications with polling fallback. +Scanner settings can be supplied in `scanner.toml`, `scanner.json`, `config/scanner.toml`, `config/scanner.json`, `SCANNER_CONFIG_PATH`, or inline JSON via `SCANNER_CONFIG_JSON`. New libraries default to bounded automatic maintenance every 15 minutes, filesystem watching is enabled per library, and the watcher uses `auto` strategy. Watch events remain the primary low-latency path; the shorter maintenance interval is the fallback for missed notifications and newly added top-level media folders. Existing libraries keep their persisted interval until an operator updates them. On Linux, `auto` selects polling immediately for CIFS/SMB3 and NFS mounts; other filesystems use native notifications with polling fallback. Library create/update API payloads can override per-library policy: @@ -123,7 +123,7 @@ Library create/update API payloads can override per-library policy: "name": "Movies", "library_type": "Movies", "paths": ["/media/movies"], - "scan_interval_minutes": 60, + "scan_interval_minutes": 15, "auto_scan": true, "watch_for_changes": true } From 36ee77246d32e80284905ef152c96222a24e1352 Mon Sep 17 00:00:00 2001 From: Grayson Hieb Date: Tue, 11 Aug 2026 17:48:49 -0600 Subject: [PATCH 08/10] fix(sqlx): check scan persistence queries --- ...2f05418f5cc393bbbf44fd3b0f2bd0dbc98b6.json | 16 ++ ...be65911f9f5b2be7cfce4a1df466a8a5f788e.json | 15 ++ ...dd065cb0379da976920072faccc4525f81c37.json | 22 +++ ...eb805509ba95e0648a6a8cc3dc0e5b3ebad1d.json | 23 +++ ...35a27747539c84e666ae2ffea288961ac4ca4.json | 40 +++++ ...d4f0fc0858580190ca2fc8ec7186e0acc0462.json | 16 ++ ...0ad1a19fa3c6f12740513a01837abb81a30d9.json | 25 ++++ ...9eef3f9644d2614e52267b5a2050c13431071.json | 22 +++ ...e85b5355836b87155a082d3019c89d1acaeb2.json | 16 ++ ...2d1b9df185d287387330da2da6b6e8ceea9c3.json | 14 ++ ...ac4fab7f27840ce4e2276affdad4e3bdd811c.json | 16 ++ ...19bb2e5f37d2792050e6a00b173bd6d976353.json | 19 +++ ...241cd7c4a0ea2353d7c3424518f45f1d4c9e7.json | 28 ++++ ...c59aa4fbba8e8dff0e682ce6a34b13601868e.json | 19 +++ ...eb5a14bd77694db6857c1cc8d2ed699705cb7.json | 29 ++++ .../domain/scan/orchestration/persistence.rs | 137 +++++++----------- .../domain/scan/orchestration/series_state.rs | 28 ++-- 17 files changed, 388 insertions(+), 97 deletions(-) create mode 100644 .sqlx/query-05a0badecd1a35f884a197b350b2f05418f5cc393bbbf44fd3b0f2bd0dbc98b6.json create mode 100644 .sqlx/query-05d5bfe6f102c8a466f1afe7b6bbe65911f9f5b2be7cfce4a1df466a8a5f788e.json create mode 100644 .sqlx/query-2d80917bbfced8688d306f8e876dd065cb0379da976920072faccc4525f81c37.json create mode 100644 .sqlx/query-31b975ca8bf361e885e8205954eeb805509ba95e0648a6a8cc3dc0e5b3ebad1d.json create mode 100644 .sqlx/query-386f193c8d84b877df2947af35a35a27747539c84e666ae2ffea288961ac4ca4.json create mode 100644 .sqlx/query-3c8620d54f3f7ca5ad6081ec722d4f0fc0858580190ca2fc8ec7186e0acc0462.json create mode 100644 .sqlx/query-52cd80ec4daa9afb1c62b58e6250ad1a19fa3c6f12740513a01837abb81a30d9.json create mode 100644 .sqlx/query-57c30af5ecc28468a2bc7fd6c029eef3f9644d2614e52267b5a2050c13431071.json create mode 100644 .sqlx/query-a770d0fefa58e5bd399442d61efe85b5355836b87155a082d3019c89d1acaeb2.json create mode 100644 .sqlx/query-ad60e15410f77140ce3d10be9422d1b9df185d287387330da2da6b6e8ceea9c3.json create mode 100644 .sqlx/query-bb9203b4be164755777322b495eac4fab7f27840ce4e2276affdad4e3bdd811c.json create mode 100644 .sqlx/query-c1e0c26a5bd13c4b288de62d41219bb2e5f37d2792050e6a00b173bd6d976353.json create mode 100644 .sqlx/query-da3b8505043ce39fae1c95f4b01241cd7c4a0ea2353d7c3424518f45f1d4c9e7.json create mode 100644 .sqlx/query-f82cd6017b89b65ae6f2291066dc59aa4fbba8e8dff0e682ce6a34b13601868e.json create mode 100644 .sqlx/query-fa2b4080075f79456850b174b14eb5a14bd77694db6857c1cc8d2ed699705cb7.json diff --git a/.sqlx/query-05a0badecd1a35f884a197b350b2f05418f5cc393bbbf44fd3b0f2bd0dbc98b6.json b/.sqlx/query-05a0badecd1a35f884a197b350b2f05418f5cc393bbbf44fd3b0f2bd0dbc98b6.json new file mode 100644 index 00000000..c228700d --- /dev/null +++ b/.sqlx/query-05a0badecd1a35f884a197b350b2f05418f5cc393bbbf44fd3b0f2bd0dbc98b6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE series_scan_state\n SET status = 'failed'::series_scan_status,\n failed_at = NOW(),\n failure_reason = $3,\n updated_at = NOW()\n WHERE library_id = $1\n AND series_root_path = $2\n AND status <> 'resolved'::series_scan_status\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "05a0badecd1a35f884a197b350b2f05418f5cc393bbbf44fd3b0f2bd0dbc98b6" +} diff --git a/.sqlx/query-05d5bfe6f102c8a466f1afe7b6bbe65911f9f5b2be7cfce4a1df466a8a5f788e.json b/.sqlx/query-05d5bfe6f102c8a466f1afe7b6bbe65911f9f5b2be7cfce4a1df466a8a5f788e.json new file mode 100644 index 00000000..c5073eac --- /dev/null +++ b/.sqlx/query-05d5bfe6f102c8a466f1afe7b6bbe65911f9f5b2be7cfce4a1df466a8a5f788e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE orchestrator_jobs\n SET state = 'ready',\n attempts = 0,\n available_at = NOW(),\n lease_owner = NULL,\n lease_id = NULL,\n lease_expires_at = NULL,\n dependency_key = NULL,\n last_error = NULL,\n updated_at = NOW()\n WHERE id = $1\n AND lease_id = $2::uuid\n AND state = 'leased'\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "05d5bfe6f102c8a466f1afe7b6bbe65911f9f5b2be7cfce4a1df466a8a5f788e" +} diff --git a/.sqlx/query-2d80917bbfced8688d306f8e876dd065cb0379da976920072faccc4525f81c37.json b/.sqlx/query-2d80917bbfced8688d306f8e876dd065cb0379da976920072faccc4525f81c37.json new file mode 100644 index 00000000..29063e48 --- /dev/null +++ b/.sqlx/query-2d80917bbfced8688d306f8e876dd065cb0379da976920072faccc4525f81c37.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT payload\n FROM orchestrator_jobs\n WHERE lease_id = $1::uuid AND state = 'leased'\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "payload", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "2d80917bbfced8688d306f8e876dd065cb0379da976920072faccc4525f81c37" +} diff --git a/.sqlx/query-31b975ca8bf361e885e8205954eeb805509ba95e0648a6a8cc3dc0e5b3ebad1d.json b/.sqlx/query-31b975ca8bf361e885e8205954eeb805509ba95e0648a6a8cc3dc0e5b3ebad1d.json new file mode 100644 index 00000000..faca9cdd --- /dev/null +++ b/.sqlx/query-31b975ca8bf361e885e8205954eeb805509ba95e0648a6a8cc3dc0e5b3ebad1d.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT status::text AS \"status!\"\n FROM series_scan_state\n WHERE library_id = $1\n AND series_root_path = $2\n FOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "status!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "31b975ca8bf361e885e8205954eeb805509ba95e0648a6a8cc3dc0e5b3ebad1d" +} diff --git a/.sqlx/query-386f193c8d84b877df2947af35a35a27747539c84e666ae2ffea288961ac4ca4.json b/.sqlx/query-386f193c8d84b877df2947af35a35a27747539c84e666ae2ffea288961ac4ca4.json new file mode 100644 index 00000000..7702bdfe --- /dev/null +++ b/.sqlx/query-386f193c8d84b877df2947af35a35a27747539c84e666ae2ffea288961ac4ca4.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, attempts, payload, correlation_id\n FROM orchestrator_jobs\n WHERE lease_id = $1::uuid AND state = 'leased'\n FOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "attempts", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "payload", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "correlation_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "386f193c8d84b877df2947af35a35a27747539c84e666ae2ffea288961ac4ca4" +} diff --git a/.sqlx/query-3c8620d54f3f7ca5ad6081ec722d4f0fc0858580190ca2fc8ec7186e0acc0462.json b/.sqlx/query-3c8620d54f3f7ca5ad6081ec722d4f0fc0858580190ca2fc8ec7186e0acc0462.json new file mode 100644 index 00000000..64ee0739 --- /dev/null +++ b/.sqlx/query-3c8620d54f3f7ca5ad6081ec722d4f0fc0858580190ca2fc8ec7186e0acc0462.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE orchestrator_jobs\n SET state = 'ready',\n dependency_key = NULL,\n available_at = NOW(),\n updated_at = NOW()\n WHERE kind = $1\n AND state = 'deferred'\n AND library_id = $2\n AND dependency_key = $3\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int2", + "Uuid", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3c8620d54f3f7ca5ad6081ec722d4f0fc0858580190ca2fc8ec7186e0acc0462" +} diff --git a/.sqlx/query-52cd80ec4daa9afb1c62b58e6250ad1a19fa3c6f12740513a01837abb81a30d9.json b/.sqlx/query-52cd80ec4daa9afb1c62b58e6250ad1a19fa3c6f12740513a01837abb81a30d9.json new file mode 100644 index 00000000..e024a4df --- /dev/null +++ b/.sqlx/query-52cd80ec4daa9afb1c62b58e6250ad1a19fa3c6f12740513a01837abb81a30d9.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE orchestrator_jobs\n SET state='leased',\n lease_owner=$1,\n lease_id=$2,\n lease_expires_at=$3,\n attempts = COALESCE(attempts, 0),\n available_at=NOW(),\n updated_at=NOW()\n WHERE id = $4 AND state = 'ready'\n RETURNING available_at\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "available_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Timestamptz", + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "52cd80ec4daa9afb1c62b58e6250ad1a19fa3c6f12740513a01837abb81a30d9" +} diff --git a/.sqlx/query-57c30af5ecc28468a2bc7fd6c029eef3f9644d2614e52267b5a2050c13431071.json b/.sqlx/query-57c30af5ecc28468a2bc7fd6c029eef3f9644d2614e52267b5a2050c13431071.json new file mode 100644 index 00000000..d47cd8c8 --- /dev/null +++ b/.sqlx/query-57c30af5ecc28468a2bc7fd6c029eef3f9644d2614e52267b5a2050c13431071.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT available_at FROM orchestrator_jobs WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "available_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "57c30af5ecc28468a2bc7fd6c029eef3f9644d2614e52267b5a2050c13431071" +} diff --git a/.sqlx/query-a770d0fefa58e5bd399442d61efe85b5355836b87155a082d3019c89d1acaeb2.json b/.sqlx/query-a770d0fefa58e5bd399442d61efe85b5355836b87155a082d3019c89d1acaeb2.json new file mode 100644 index 00000000..a3b69e8a --- /dev/null +++ b/.sqlx/query-a770d0fefa58e5bd399442d61efe85b5355836b87155a082d3019c89d1acaeb2.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE orchestrator_jobs\n SET state = 'dead_letter',\n lease_owner = NULL,\n lease_id = NULL,\n lease_expires_at = NULL,\n last_error = $3,\n updated_at = NOW()\n WHERE id = $1\n AND lease_id = $2::uuid\n AND state = 'leased'\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a770d0fefa58e5bd399442d61efe85b5355836b87155a082d3019c89d1acaeb2" +} diff --git a/.sqlx/query-ad60e15410f77140ce3d10be9422d1b9df185d287387330da2da6b6e8ceea9c3.json b/.sqlx/query-ad60e15410f77140ce3d10be9422d1b9df185d287387330da2da6b6e8ceea9c3.json new file mode 100644 index 00000000..09dccba6 --- /dev/null +++ b/.sqlx/query-ad60e15410f77140ce3d10be9422d1b9df185d287387330da2da6b6e8ceea9c3.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH terminal_series AS MATERIALIZED (\n SELECT series.library_id, series.series_root_path\n FROM series_scan_state AS series\n WHERE series.status IN ('resolved', 'failed')\n AND EXISTS (\n SELECT 1\n FROM orchestrator_jobs AS blocked\n WHERE blocked.kind = $1\n AND blocked.state = 'deferred'\n AND blocked.library_id = series.library_id\n AND blocked.dependency_key =\n 'series_root:' || series.series_root_path\n )\n FOR UPDATE\n )\n UPDATE orchestrator_jobs AS job\n SET state = 'ready',\n dependency_key = NULL,\n available_at = NOW(),\n updated_at = NOW()\n FROM terminal_series AS series\n WHERE job.kind = $1\n AND job.state = 'deferred'\n AND job.library_id = series.library_id\n AND job.dependency_key =\n 'series_root:' || series.series_root_path\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int2" + ] + }, + "nullable": [] + }, + "hash": "ad60e15410f77140ce3d10be9422d1b9df185d287387330da2da6b6e8ceea9c3" +} diff --git a/.sqlx/query-bb9203b4be164755777322b495eac4fab7f27840ce4e2276affdad4e3bdd811c.json b/.sqlx/query-bb9203b4be164755777322b495eac4fab7f27840ce4e2276affdad4e3bdd811c.json new file mode 100644 index 00000000..e3b87783 --- /dev/null +++ b/.sqlx/query-bb9203b4be164755777322b495eac4fab7f27840ce4e2276affdad4e3bdd811c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH terminal_series AS MATERIALIZED (\n SELECT library_id, series_root_path\n FROM series_scan_state\n WHERE library_id = $2\n AND series_root_path = $3\n AND status IN ('resolved', 'failed')\n FOR UPDATE\n )\n UPDATE orchestrator_jobs AS job\n SET state = 'ready',\n dependency_key = NULL,\n available_at = NOW(),\n updated_at = NOW()\n FROM terminal_series AS series\n WHERE job.kind = $1\n AND job.state = 'deferred'\n AND job.library_id = series.library_id\n AND job.dependency_key =\n 'series_root:' || series.series_root_path\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int2", + "Uuid", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bb9203b4be164755777322b495eac4fab7f27840ce4e2276affdad4e3bdd811c" +} diff --git a/.sqlx/query-c1e0c26a5bd13c4b288de62d41219bb2e5f37d2792050e6a00b173bd6d976353.json b/.sqlx/query-c1e0c26a5bd13c4b288de62d41219bb2e5f37d2792050e6a00b173bd6d976353.json new file mode 100644 index 00000000..70eaddad --- /dev/null +++ b/.sqlx/query-c1e0c26a5bd13c4b288de62d41219bb2e5f37d2792050e6a00b173bd6d976353.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO series_scan_state (\n library_id, series_root_path, status,\n series_title, series_slug, series_year, series_region,\n attempts, created_at, updated_at\n )\n VALUES (\n $1, $2, 'discovered'::series_scan_status,\n $3, $4, $5, $6, 0, NOW(), NOW()\n )\n ON CONFLICT (library_id, series_root_path)\n DO UPDATE SET\n series_title = COALESCE(\n EXCLUDED.series_title,\n series_scan_state.series_title\n ),\n series_slug = COALESCE(\n EXCLUDED.series_slug,\n series_scan_state.series_slug\n ),\n series_year = COALESCE(\n EXCLUDED.series_year,\n series_scan_state.series_year\n ),\n series_region = COALESCE(\n EXCLUDED.series_region,\n series_scan_state.series_region\n ),\n status = CASE\n WHEN series_scan_state.status <> 'resolved'\n THEN 'discovered'::series_scan_status\n ELSE series_scan_state.status\n END,\n series_id = CASE\n WHEN series_scan_state.status <> 'resolved' THEN NULL\n ELSE series_scan_state.series_id\n END,\n resolved_at = CASE\n WHEN series_scan_state.status <> 'resolved' THEN NULL\n ELSE series_scan_state.resolved_at\n END,\n failed_at = CASE\n WHEN series_scan_state.status <> 'resolved' THEN NULL\n ELSE series_scan_state.failed_at\n END,\n failure_reason = CASE\n WHEN series_scan_state.status <> 'resolved' THEN NULL\n ELSE series_scan_state.failure_reason\n END,\n updated_at = NOW()\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Text", + "Int2", + "Text" + ] + }, + "nullable": [] + }, + "hash": "c1e0c26a5bd13c4b288de62d41219bb2e5f37d2792050e6a00b173bd6d976353" +} diff --git a/.sqlx/query-da3b8505043ce39fae1c95f4b01241cd7c4a0ea2353d7c3424518f45f1d4c9e7.json b/.sqlx/query-da3b8505043ce39fae1c95f4b01241cd7c4a0ea2353d7c3424518f45f1d4c9e7.json new file mode 100644 index 00000000..e816837d --- /dev/null +++ b/.sqlx/query-da3b8505043ce39fae1c95f4b01241cd7c4a0ea2353d7c3424518f45f1d4c9e7.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT attempts, payload\n FROM orchestrator_jobs\n WHERE lease_id = $1::uuid AND state = 'leased'\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "attempts", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "payload", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "da3b8505043ce39fae1c95f4b01241cd7c4a0ea2353d7c3424518f45f1d4c9e7" +} diff --git a/.sqlx/query-f82cd6017b89b65ae6f2291066dc59aa4fbba8e8dff0e682ce6a34b13601868e.json b/.sqlx/query-f82cd6017b89b65ae6f2291066dc59aa4fbba8e8dff0e682ce6a34b13601868e.json new file mode 100644 index 00000000..69f62ac6 --- /dev/null +++ b/.sqlx/query-f82cd6017b89b65ae6f2291066dc59aa4fbba8e8dff0e682ce6a34b13601868e.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO series_scan_state (\n library_id, series_root_path, status,\n series_title, series_slug, series_year, series_region,\n attempts, created_at, updated_at\n )\n VALUES (\n $1, $2, 'discovered'::series_scan_status,\n $3, $4, $5, $6, 0, NOW(), NOW()\n )\n ON CONFLICT (library_id, series_root_path)\n DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Text", + "Int2", + "Text" + ] + }, + "nullable": [] + }, + "hash": "f82cd6017b89b65ae6f2291066dc59aa4fbba8e8dff0e682ce6a34b13601868e" +} diff --git a/.sqlx/query-fa2b4080075f79456850b174b14eb5a14bd77694db6857c1cc8d2ed699705cb7.json b/.sqlx/query-fa2b4080075f79456850b174b14eb5a14bd77694db6857c1cc8d2ed699705cb7.json new file mode 100644 index 00000000..069ff276 --- /dev/null +++ b/.sqlx/query-fa2b4080075f79456850b174b14eb5a14bd77694db6857c1cc8d2ed699705cb7.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT status::text AS \"status!\", updated_at\n FROM series_scan_state\n WHERE library_id = $1\n AND series_root_path = $2\n FOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "status!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "updated_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "fa2b4080075f79456850b174b14eb5a14bd77694db6857c1cc8d2ed699705cb7" +} diff --git a/crates/ferrex-core/src/domain/scan/orchestration/persistence.rs b/crates/ferrex-core/src/domain/scan/orchestration/persistence.rs index f04eacc1..19430150 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/persistence.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/persistence.rs @@ -5,7 +5,7 @@ use crate::error::Result; use async_trait::async_trait; use chrono::Utc; use serde_json::from_value; -use sqlx::{Executor, PgPool, Row}; +use sqlx::{Executor, PgPool}; use std::fmt; use std::hash::{DefaultHasher, Hash, Hasher}; use tracing::{debug, info, trace, warn}; @@ -718,7 +718,7 @@ impl PostgresQueueService { library_id: LibraryId, series_root_path: &SeriesRootPath, ) -> Result { - let updated = sqlx::query( + let updated = sqlx::query!( r#" WITH terminal_series AS MATERIALIZED ( SELECT library_id, series_root_path @@ -740,10 +740,10 @@ impl PostgresQueueService { AND job.dependency_key = 'series_root:' || series.series_root_path "#, + JobKind::EpisodeMatch as i16, + library_id.0, + series_root_path.as_str(), ) - .bind(JobKind::EpisodeMatch as i16) - .bind(library_id.0) - .bind(series_root_path.as_str()) .execute(&self.pool) .await .map_err(|error| { @@ -766,7 +766,7 @@ impl PostgresQueueService { /// enrollment, so a historical failed snapshot cannot release the new /// generation after enrollment has begun. pub async fn repair_terminal_series_dependencies(&self) -> Result { - let updated = sqlx::query( + let updated = sqlx::query!( r#" WITH terminal_series AS MATERIALIZED ( SELECT series.library_id, series.series_root_path @@ -795,8 +795,8 @@ impl PostgresQueueService { AND job.dependency_key = 'series_root:' || series.series_root_path "#, + JobKind::EpisodeMatch as i16, ) - .bind(JobKind::EpisodeMatch as i16) .execute(&self.pool) .await .map_err(|e| { @@ -872,7 +872,7 @@ async fn finalize_terminal_series_failure( record_failure: bool, ) -> Result { if record_failure { - sqlx::query( + sqlx::query!( r#" UPDATE series_scan_state SET status = 'failed'::series_scan_status, @@ -883,10 +883,10 @@ async fn finalize_terminal_series_failure( AND series_root_path = $2 AND status <> 'resolved'::series_scan_status "#, + library_id.0, + series_root_path.as_str(), + failure_reason.unwrap_or("series resolution retry limit reached"), ) - .bind(library_id.0) - .bind(series_root_path.as_str()) - .bind(failure_reason.unwrap_or("series resolution retry limit reached")) .execute(&mut **tx) .await .map_err(|e| { @@ -897,7 +897,7 @@ async fn finalize_terminal_series_failure( } let dependency_key = DependencyKey::series_root(series_root_path); - let released = sqlx::query( + let released = sqlx::query!( r#" UPDATE orchestrator_jobs SET state = 'ready', @@ -909,10 +909,10 @@ async fn finalize_terminal_series_failure( AND library_id = $2 AND dependency_key = $3 "#, + JobKind::EpisodeMatch as i16, + library_id.0, + dependency_key.as_str(), ) - .bind(JobKind::EpisodeMatch as i16) - .bind(library_id.0) - .bind(dependency_key.as_str()) .execute(&mut **tx) .await .map_err(|e| { @@ -1900,7 +1900,7 @@ impl QueueService for PostgresQueueService { // While a row is leased, `available_at` is its stable lease-start // marker. Lease renewal deliberately leaves it unchanged; every path // that returns a row to Ready assigns a new availability timestamp. - let updated = sqlx::query( + let updated = sqlx::query_scalar!( r#" UPDATE orchestrator_jobs SET state='leased', @@ -1913,11 +1913,11 @@ impl QueueService for PostgresQueueService { WHERE id = $4 AND state = 'ready' RETURNING available_at "#, + &request.worker_id, + lease_id.0, + expires_at, + row.id, ) - .bind(&request.worker_id) - .bind(lease_id.0) - .bind(expires_at) - .bind(row.id) .fetch_optional(&mut *tx) .await .map_err(|e| { @@ -1929,12 +1929,7 @@ impl QueueService for PostgresQueueService { drop(tx); return Ok(None); }; - let lease_started_at: chrono::DateTime = - updated.try_get("available_at").map_err(|e| { - MediaError::Internal(format!( - "dequeue lease-start timestamp decode failed: {e}" - )) - })?; + let lease_started_at: chrono::DateTime = updated; // Build JobRecord from the selected row and new lease fields let payload: JobPayload = @@ -2139,30 +2134,21 @@ impl QueueService for PostgresQueueService { // identity first so a terminal transition can follow the same global // series-state -> queue-row lock order as dependency release and the // direct dead-letter transition below. - let terminal_series_identity = if let Some(candidate) = sqlx::query( + let terminal_series_identity = if let Some(candidate) = sqlx::query!( r#" SELECT attempts, payload FROM orchestrator_jobs WHERE lease_id = $1::uuid AND state = 'leased' "#, + lease_id.0, ) - .bind(lease_id.0) .fetch_optional(&self.pool) .await .map_err(|e| { MediaError::Internal(format!("fail identity read failed: {e}")) })? { - let attempts: i32 = candidate.try_get("attempts").map_err(|e| { - MediaError::Internal(format!( - "fail identity attempts decode failed: {e}" - )) - })?; - let payload: serde_json::Value = - candidate.try_get("payload").map_err(|e| { - MediaError::Internal(format!( - "fail identity payload decode failed: {e}" - )) - })?; + let attempts = candidate.attempts; + let payload = candidate.payload; let payload: JobPayload = serde_json::from_value(payload).map_err(|e| { MediaError::Internal(format!( @@ -2187,17 +2173,17 @@ impl QueueService for PostgresQueueService { let locked_series_state = if let Some((library_id, root)) = &terminal_series_identity { - sqlx::query_as::<_, (String, chrono::DateTime)>( + sqlx::query!( r#" - SELECT status::text, updated_at + SELECT status::text AS "status!", updated_at FROM series_scan_state WHERE library_id = $1 AND series_root_path = $2 FOR UPDATE "#, + library_id.0, + root.as_str(), ) - .bind(library_id.0) - .bind(root.as_str()) .fetch_optional(&mut *tx) .await .map_err(|e| { @@ -2205,6 +2191,7 @@ impl QueueService for PostgresQueueService { "fail terminal series-state lock failed: {e}" )) })? + .map(|row| (row.status, row.updated_at)) } else { None }; @@ -2265,10 +2252,10 @@ impl QueueService for PostgresQueueService { root.as_str() ))); }; - let lease_started_at: chrono::DateTime = sqlx::query_scalar( + let lease_started_at: chrono::DateTime = sqlx::query_scalar!( "SELECT available_at FROM orchestrator_jobs WHERE id = $1", + row_id, ) - .bind(row_id) .fetch_one(&mut *tx) .await .map_err(|e| { @@ -2281,7 +2268,7 @@ impl QueueService for PostgresQueueService { *series_updated_at, lease_started_at, ) { - let requeued = sqlx::query( + let requeued = sqlx::query!( r#" UPDATE orchestrator_jobs SET state = 'ready', @@ -2297,9 +2284,9 @@ impl QueueService for PostgresQueueService { AND lease_id = $2::uuid AND state = 'leased' "#, + row_id, + lease_id.0, ) - .bind(row_id) - .bind(lease_id.0) .execute(&mut *tx) .await .map_err(|e| { @@ -2509,14 +2496,14 @@ impl QueueService for PostgresQueueService { // Read only enough identity to establish the global lock order for a // SeriesResolve transition. The lease is revalidated under lock below; // this first read never authorizes a state change by itself. - let candidate_payload = sqlx::query_scalar::<_, serde_json::Value>( + let candidate_payload = sqlx::query_scalar!( r#" SELECT payload FROM orchestrator_jobs WHERE lease_id = $1::uuid AND state = 'leased' "#, + lease_id.0, ) - .bind(lease_id.0) .fetch_optional(&self.pool) .await .map_err(|e| { @@ -2550,17 +2537,17 @@ impl QueueService for PostgresQueueService { // avoids a queue->series inversion with the housekeeper. let series_status = if let Some((library_id, series_root_path)) = &series_identity { - sqlx::query_scalar::<_, String>( + sqlx::query_scalar!( r#" - SELECT status::text + SELECT status::text AS "status!" FROM series_scan_state WHERE library_id = $1 AND series_root_path = $2 FOR UPDATE "#, + library_id.0, + series_root_path.as_str(), ) - .bind(library_id.0) - .bind(series_root_path.as_str()) .fetch_optional(&mut *tx) .await .map_err(|e| { @@ -2572,15 +2559,15 @@ impl QueueService for PostgresQueueService { None }; - let row = sqlx::query( + let row = sqlx::query!( r#" SELECT id, attempts, payload, correlation_id FROM orchestrator_jobs WHERE lease_id = $1::uuid AND state = 'leased' FOR UPDATE "#, + lease_id.0, ) - .bind(lease_id.0) .fetch_optional(&mut *tx) .await .map_err(|e| { @@ -2590,26 +2577,10 @@ impl QueueService for PostgresQueueService { let Some(row) = row else { return Ok(QueueTransitionOutcome::Missing); }; - let row_id: uuid::Uuid = row.try_get("id").map_err(|e| { - MediaError::Internal(format!("dead_letter id decode failed: {e}")) - })?; - let attempts: i32 = row.try_get("attempts").map_err(|e| { - MediaError::Internal(format!( - "dead_letter attempts decode failed: {e}" - )) - })?; - let row_payload: serde_json::Value = - row.try_get("payload").map_err(|e| { - MediaError::Internal(format!( - "dead_letter payload row decode failed: {e}" - )) - })?; - let row_correlation_id: Option = - row.try_get("correlation_id").map_err(|e| { - MediaError::Internal(format!( - "dead_letter correlation decode failed: {e}" - )) - })?; + let row_id = row.id; + let attempts = row.attempts; + let row_payload = row.payload; + let row_correlation_id = row.correlation_id; let locked_job: JobPayload = serde_json::from_value(row_payload.clone()).map_err(|e| { MediaError::Internal(format!( @@ -2639,7 +2610,7 @@ impl QueueService for PostgresQueueService { if matches!(series_status.as_deref(), Some("discovered" | "seeded")) { - let requeued = sqlx::query( + let requeued = sqlx::query!( r#" UPDATE orchestrator_jobs SET state = 'ready', @@ -2655,9 +2626,9 @@ impl QueueService for PostgresQueueService { AND lease_id = $2::uuid AND state = 'leased' "#, + row_id, + lease_id.0, ) - .bind(row_id) - .bind(lease_id.0) .execute(&mut *tx) .await .map_err(|e| { @@ -2683,7 +2654,7 @@ impl QueueService for PostgresQueueService { } } - let transitioned = sqlx::query( + let transitioned = sqlx::query!( r#" UPDATE orchestrator_jobs SET state = 'dead_letter', @@ -2696,10 +2667,10 @@ impl QueueService for PostgresQueueService { AND lease_id = $2::uuid AND state = 'leased' "#, + row_id, + lease_id.0, + error.clone(), ) - .bind(row_id) - .bind(lease_id.0) - .bind(error.clone()) .execute(&mut *tx) .await .map_err(|e| { diff --git a/crates/ferrex-core/src/domain/scan/orchestration/series_state.rs b/crates/ferrex-core/src/domain/scan/orchestration/series_state.rs index 01fa6a14..cd6c8cd4 100644 --- a/crates/ferrex-core/src/domain/scan/orchestration/series_state.rs +++ b/crates/ferrex-core/src/domain/scan/orchestration/series_state.rs @@ -687,7 +687,7 @@ impl SeriesScanStateRepository for PostgresSeriesScanStateRepository { }) .unwrap_or((None, None, None, None)); - sqlx::query( + sqlx::query!( r#" INSERT INTO series_scan_state ( library_id, series_root_path, status, @@ -739,13 +739,13 @@ impl SeriesScanStateRepository for PostgresSeriesScanStateRepository { END, updated_at = NOW() "#, + library_id.0, + series_root_path.as_str(), + title, + slug, + year, + region, ) - .bind(library_id.0) - .bind(series_root_path.as_str()) - .bind(title) - .bind(slug) - .bind(year) - .bind(region) .execute(&self.pool) .await?; @@ -777,7 +777,7 @@ impl SeriesScanStateRepository for PostgresSeriesScanStateRepository { }) .unwrap_or((None, None, None, None)); - sqlx::query( + sqlx::query!( r#" INSERT INTO series_scan_state ( library_id, series_root_path, status, @@ -791,13 +791,13 @@ impl SeriesScanStateRepository for PostgresSeriesScanStateRepository { ON CONFLICT (library_id, series_root_path) DO NOTHING "#, + library_id.0, + series_root_path.as_str(), + title, + slug, + year, + region, ) - .bind(library_id.0) - .bind(series_root_path.as_str()) - .bind(title) - .bind(slug) - .bind(year) - .bind(region) .execute(&self.pool) .await?; From f4264ff9a20c5735db4d2f776806c82394a3fe86 Mon Sep 17 00:00:00 2001 From: Grayson Hieb Date: Tue, 11 Aug 2026 17:49:02 -0600 Subject: [PATCH 09/10] fix(deps): resolve newly disclosed advisories --- Cargo.lock | 289 +++++++++-------------------------------------------- Cargo.toml | 3 +- deny.toml | 1 + 3 files changed, 52 insertions(+), 241 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8859745a..295a3839 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -59,17 +59,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "ahash" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" -dependencies = [ - "getrandom 0.2.17", - "once_cell", - "version_check", -] - [[package]] name = "ahash" version = "0.8.12" @@ -192,7 +181,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -203,7 +192,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -824,18 +813,6 @@ dependencies = [ "core2", ] -[[package]] -name = "bitvec" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] - [[package]] name = "blake2" version = "0.10.6" @@ -891,30 +868,6 @@ dependencies = [ "piper", ] -[[package]] -name = "borsh" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" -dependencies = [ - "borsh-derive", - "bytes", - "cfg_aliases", -] - -[[package]] -name = "borsh-derive" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfcfdc083699101d5a7965e49925975f2f55060f94f9a05e7187be95d530ca59" -dependencies = [ - "once_cell", - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "bstr" version = "1.12.1" @@ -938,41 +891,19 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" -[[package]] -name = "bytecheck" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" -dependencies = [ - "bytecheck_derive 0.6.12", - "ptr_meta 0.1.4", - "simdutf8", -] - [[package]] name = "bytecheck" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0caa33a2c0edca0419d15ac723dff03f1956f7978329b1e3b5fdaaaed9d3ca8b" dependencies = [ - "bytecheck_derive 0.8.2", - "ptr_meta 0.3.1", + "bytecheck_derive", + "ptr_meta", "rancor", "simdutf8", "uuid", ] -[[package]] -name = "bytecheck_derive" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "bytecheck_derive" version = "0.8.2" @@ -1541,9 +1472,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] @@ -1763,7 +1694,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1980,7 +1911,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2136,7 +2067,7 @@ version = "0.1.2-alpha" dependencies = [ "ferrex-model", "log", - "rkyv 0.8.16", + "rkyv", "uuid", ] @@ -2175,7 +2106,7 @@ dependencies = [ "redis", "regex", "reqwest", - "rkyv 0.8.16", + "rkyv", "serde", "serde_json", "serde_urlencoded", @@ -2208,7 +2139,7 @@ name = "ferrex-model" version = "0.1.2-alpha" dependencies = [ "chrono", - "rkyv 0.8.16", + "rkyv", "serde", "sqlx", "tmdb-api", @@ -2250,7 +2181,7 @@ dependencies = [ "log", "parking_lot", "reqwest", - "rkyv 0.8.16", + "rkyv", "serde", "serde_json", "tokio", @@ -2274,7 +2205,7 @@ dependencies = [ "log", "lucide-icons", "png", - "rkyv 0.8.16", + "rkyv", "serde", "serde_json", "tempfile", @@ -2343,7 +2274,7 @@ dependencies = [ "parking_lot", "reqwest", "reqwest-eventsource", - "rkyv 0.8.16", + "rkyv", "serde", "serde_json", "sha2", @@ -2404,7 +2335,7 @@ dependencies = [ "log", "parking_lot", "rancor", - "rkyv 0.8.16", + "rkyv", "serde", "serde_json", "sha2", @@ -2427,7 +2358,7 @@ dependencies = [ "log", "num_cpus", "profiling", - "rkyv 0.8.16", + "rkyv", "thiserror 2.0.18", "tokio", ] @@ -2514,7 +2445,7 @@ dependencies = [ "regex", "reqwest", "reqwest-eventsource", - "rkyv 0.8.16", + "rkyv", "serde", "serde_json", "sha2", @@ -2581,7 +2512,7 @@ dependencies = [ "redis", "regex", "reqwest", - "rkyv 0.8.16", + "rkyv", "rustls", "rustls-pki-types", "serde", @@ -2865,12 +2796,6 @@ dependencies = [ "libc", ] -[[package]] -name = "funty" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" - [[package]] name = "futures" version = "0.3.32" @@ -3102,7 +3027,7 @@ dependencies = [ "gobject-sys", "libc", "system-deps", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3419,15 +3344,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash 0.7.8", -] - [[package]] name = "hashbrown" version = "0.14.5" @@ -5078,7 +4994,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -5987,33 +5903,13 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "ptr_meta" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" -dependencies = [ - "ptr_meta_derive 0.1.4", -] - [[package]] name = "ptr_meta" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" dependencies = [ - "ptr_meta_derive 0.3.1", -] - -[[package]] -name = "ptr_meta_derive" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", + "ptr_meta_derive", ] [[package]] @@ -6080,9 +5976,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quick-xml" -version = "0.39.2" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", ] @@ -6163,19 +6059,13 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "radium" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" - [[package]] name = "rancor" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" dependencies = [ - "ptr_meta 0.3.1", + "ptr_meta", ] [[package]] @@ -6484,22 +6374,13 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" -[[package]] -name = "rend" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" -dependencies = [ - "bytecheck 0.6.12", -] - [[package]] name = "rend" version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" dependencies = [ - "bytecheck 0.8.2", + "bytecheck", ] [[package]] @@ -6623,61 +6504,32 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.7.46" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1" +checksum = "d9776093b7ca170454ab1406954f7b7d97a57c51dc6c0642957fb2ef25c2d399" dependencies = [ - "bitvec", - "bytecheck 0.6.12", - "bytes", - "hashbrown 0.12.3", - "ptr_meta 0.1.4", - "rend 0.4.2", - "rkyv_derive 0.7.46", - "seahash", - "tinyvec", - "uuid", -] - -[[package]] -name = "rkyv" -version = "0.8.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" -dependencies = [ - "bytecheck 0.8.2", + "bytecheck", "bytes", "hashbrown 0.17.1", "indexmap", "munge", - "ptr_meta 0.3.1", + "ptr_meta", "rancor", - "rend 0.5.3", - "rkyv_derive 0.8.16", + "rend", + "rkyv_derive", "tinyvec", "uuid", ] [[package]] name = "rkyv_derive" -version = "0.7.46" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" +checksum = "1c25ef604ac7dd839d44d64648952ea23c97866f124ff671b0ed2cf3ad9bb06e" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", -] - -[[package]] -name = "rkyv_derive" -version = "0.8.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", + "syn 3.0.3", ] [[package]] @@ -6721,23 +6573,6 @@ dependencies = [ "thiserror 2.0.18", ] -[[package]] -name = "rust_decimal" -version = "1.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ce901f9a19d251159075a4c37af514c3b8ef99c22e02dd8c19161cf397ee94a" -dependencies = [ - "arrayvec", - "borsh", - "bytes", - "num-traits", - "rand 0.8.6", - "rkyv 0.7.46", - "serde", - "serde_json", - "wasm-bindgen", -] - [[package]] name = "rustc-hash" version = "1.1.0" @@ -6769,7 +6604,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -6782,7 +6617,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6866,12 +6701,6 @@ dependencies = [ "tiny-skia", ] -[[package]] -name = "seahash" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" - [[package]] name = "self_cell" version = "1.2.2" @@ -7229,7 +7058,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -7327,7 +7156,6 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rust_decimal", "serde", "serde_json", "sha2", @@ -7411,7 +7239,6 @@ dependencies = [ "percent-encoding", "rand 0.8.6", "rsa", - "rust_decimal", "serde", "sha1", "sha2", @@ -7454,7 +7281,6 @@ dependencies = [ "num-bigint", "once_cell", "rand 0.8.6", - "rust_decimal", "serde", "serde_json", "sha2", @@ -7614,7 +7440,7 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "subwave_appsink" version = "0.1.0" -source = "git+https://github.com/Lowband21/subwave.git?branch=main#4de8fd485a8077d17fd0f25e7b426988ac0da116" +source = "git+https://github.com/Lowband21/subwave.git?branch=main#2e83aef5a85b3e328be296469548277bd34225be" dependencies = [ "gstreamer", "gstreamer-app", @@ -7628,7 +7454,7 @@ dependencies = [ [[package]] name = "subwave_core" version = "0.1.0" -source = "git+https://github.com/Lowband21/subwave.git?branch=main#4de8fd485a8077d17fd0f25e7b426988ac0da116" +source = "git+https://github.com/Lowband21/subwave.git?branch=main#2e83aef5a85b3e328be296469548277bd34225be" dependencies = [ "gstreamer", "thiserror 2.0.18", @@ -7638,7 +7464,7 @@ dependencies = [ [[package]] name = "subwave_unified" version = "0.1.0" -source = "git+https://github.com/Lowband21/subwave.git?branch=main#4de8fd485a8077d17fd0f25e7b426988ac0da116" +source = "git+https://github.com/Lowband21/subwave.git?branch=main#2e83aef5a85b3e328be296469548277bd34225be" dependencies = [ "gstreamer", "iced", @@ -7653,7 +7479,7 @@ dependencies = [ [[package]] name = "subwave_wayland" version = "0.1.0" -source = "git+https://github.com/Lowband21/subwave.git?branch=main#4de8fd485a8077d17fd0f25e7b426988ac0da116" +source = "git+https://github.com/Lowband21/subwave.git?branch=main#2e83aef5a85b3e328be296469548277bd34225be" dependencies = [ "ab_glyph", "gstreamer", @@ -7698,9 +7524,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.109" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -7709,9 +7535,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.117" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -7788,12 +7614,6 @@ dependencies = [ "version-compare", ] -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - [[package]] name = "target-lexicon" version = "0.13.3" @@ -7810,7 +7630,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -8341,7 +8161,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset", "tempfile", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -8825,9 +8645,9 @@ dependencies = [ [[package]] name = "wayland-scanner" -version = "0.31.9" +version = "0.31.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c86287151a309799b821ca709b7345a048a2956af05957c89cb824ab919fa4e3" +checksum = "338e30461b3a2b67d70eb30a6d89f8e0c93a833e07d2ae89085cd070c4a00ac0" dependencies = [ "proc-macro2", "quick-xml", @@ -9652,7 +9472,7 @@ version = "0.30.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6755fa58a9f8350bd1e472d4c3fcc25f824ec358933bba33306d0b63df5978d" dependencies = [ - "ahash 0.8.12", + "ahash", "android-activity", "atomic-waker", "bitflags 2.11.0", @@ -9816,15 +9636,6 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" -[[package]] -name = "wyz" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" -dependencies = [ - "tap", -] - [[package]] name = "x11-dl" version = "2.21.0" diff --git a/Cargo.toml b/Cargo.toml index c0a59e51..c418295b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -155,7 +155,7 @@ serde = { version = "1.0", default-features = false, features = [ serde_json = "1.0" serde_urlencoded = "0.7" flatbuffers = "25.12.19" -rkyv = { version = "0.8.13", default-features = false, features = [ +rkyv = { version = "0.8.17", default-features = false, features = [ "std", "alloc", "bytecheck", @@ -226,7 +226,6 @@ sqlx = { version = "0.8.6", features = [ "macros", "derive", "bigdecimal", - "rust_decimal", "ipnetwork", "json", ] } diff --git a/deny.toml b/deny.toml index e68767d3..0d0e6c36 100644 --- a/deny.toml +++ b/deny.toml @@ -78,6 +78,7 @@ ignore = [ { id = "RUSTSEC-2024-0384", reason = "`instant` is required by `notify` v7; upstream replacement not yet available" }, { id = "RUSTSEC-2024-0436", reason = "`paste` comes from `wgpu` transitive deps; awaiting ecosystem migration" }, { id = "RUSTSEC-2026-0105", reason = "`core2` is pulled in transitively through `image`/`ravif`; no safe upgrade path is available yet" }, + { id = "RUSTSEC-2026-0192", reason = "`ttf-parser` is required by the Iced and `fontdue` font stacks; RustSec lists no safe upgrade path, so replacement requires upstream ecosystem migration" }, ] # If this is true, then cargo deny will use the git executable to fetch advisory database. # If this is false, then it uses a built-in git library. From d01cd854f3d853f83200e906a1f2da8e3affb395 Mon Sep 17 00:00:00 2001 From: Grayson Hieb Date: Tue, 11 Aug 2026 17:56:05 -0600 Subject: [PATCH 10/10] fix(deps): update anyhow past unsound release --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 295a3839..0ac46be4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -197,9 +197,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "arbitrary" diff --git a/Cargo.toml b/Cargo.toml index c418295b..ace6ba44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -167,7 +167,7 @@ rkyv = { version = "0.8.17", default-features = false, features = [ rancor = { version = "0.1.1", default-features = false, features = ["alloc"] } # Error handling -anyhow = "1.0" +anyhow = "1.0.103" thiserror = "2.0" # Logging