diff --git a/README.md b/README.md index 7008f0658..5356d3bd9 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,11 @@ upgrade and is unsupported. The `/app/config` and `/app/cache` mounts must be wr support SQLite WAL locking, `fsync`, and atomic file replacement. This includes ordinary Docker bind mounts and named volumes, plus local Unraid shares and TrueNAS datasets with the usual container permissions. NFS, SMB, and other network mounts are safe only when -they provide those SQLite filesystem guarantees. +they provide those SQLite filesystem guarantees. On Docker Desktop for Windows, prefer +named Docker volumes over Windows-path bind mounts for `/app/config` and `/app/cache`; +the Windows mount translation does not reliably honor atomic file replacement. The +startup upgrade detects this and falls back to a verified direct copy, but named +volumes remain the recommended setup on Windows. ### 3. First-run setup diff --git a/backend/api/v1/routes/library_operations_target.py b/backend/api/v1/routes/library_operations_target.py index 52e7f84ba..63ae3b059 100644 --- a/backend/api/v1/routes/library_operations_target.py +++ b/backend/api/v1/routes/library_operations_target.py @@ -220,6 +220,16 @@ async def restore_review( return await _review_action(review_id, "restore", body, admin, service) +@router.post("/reviews/{review_id}/dismiss", response_model=ReviewActionResponse) +async def dismiss_review( + admin: CurrentAdminDep, + review_id: str, + service: LibraryReviewServiceDep, + body: ReviewActionRequest = MsgSpecBody(ReviewActionRequest), +) -> ReviewActionResponse: + return await _review_action(review_id, "dismiss", body, admin, service) + + @router.post("/reviews/{review_id}/candidate", response_model=ReviewActionResponse) async def accept_review_candidate( admin: CurrentAdminDep, diff --git a/backend/api/v1/routes/library_policies_target.py b/backend/api/v1/routes/library_policies_target.py index 3e3e659f2..58e31f4d1 100644 --- a/backend/api/v1/routes/library_policies_target.py +++ b/backend/api/v1/routes/library_policies_target.py @@ -13,7 +13,10 @@ LibrarySettingsResponse, LibrarySettingsUpdateRequest, ) -from core.dependencies import TargetLibraryPolicyServiceDep +from core.dependencies import ( + LegacyPendingMigrationServiceDep, + TargetLibraryPolicyServiceDep, +) from infrastructure.msgspec_fastapi import MsgSpecBody, MsgSpecRoute from middleware import CurrentAdminDep @@ -39,12 +42,16 @@ async def get_library_settings( @router.put("", response_model=LibrarySettingsResponse) async def update_library_settings( service: TargetLibraryPolicyServiceDep, + pending_migration: LegacyPendingMigrationServiceDep, request: LibrarySettingsUpdateRequest = MsgSpecBody(LibrarySettingsUpdateRequest), ) -> LibrarySettingsResponse: - return await service.save_settings( + response = await service.save_settings( request.settings, expected_policy_revision=request.expected_policy_revision, ) + if response.enabled: + await pending_migration.schedule() + return response @router.get("/policy-tree", response_model=LibraryPolicyTreeResponse) @@ -80,6 +87,10 @@ async def get_restorable_library_roots( @router.post("/restore-roots", response_model=LibrarySettingsResponse) async def restore_library_roots( service: TargetLibraryPolicyServiceDep, + pending_migration: LegacyPendingMigrationServiceDep, request: LibraryRestoreRootsRequest = MsgSpecBody(LibraryRestoreRootsRequest), ) -> LibrarySettingsResponse: - return await service.restore_roots(request) + response = await service.restore_roots(request) + if response.enabled: + await pending_migration.schedule() + return response diff --git a/backend/api/v1/routes/library_scan_target.py b/backend/api/v1/routes/library_scan_target.py index ed58aa819..effc71475 100644 --- a/backend/api/v1/routes/library_scan_target.py +++ b/backend/api/v1/routes/library_scan_target.py @@ -19,6 +19,8 @@ ScanEstimateResponse, ScanRunCurrentResponse, ScanRunDetailResponse, + ScanRunFailureItem, + ScanRunFailuresResponse, ScanRunHistoryResponse, ScanRunRequestBody, ScanRunRequestedResponse, @@ -27,6 +29,8 @@ from core.dependencies import ( LibraryAdministrativeWorkServiceDep, LibraryPolicyResolverDep, + MbProviderAvailabilityDep, + NativeLibraryStoreDep, TargetIdentificationQueueDep, TargetLibraryScanCoordinatorDep, ) @@ -116,6 +120,7 @@ async def library_activity( coordinator: TargetLibraryScanCoordinatorDep, identification: TargetIdentificationQueueDep, administrative_work: LibraryAdministrativeWorkServiceDep, + mb_provider_available: MbProviderAvailabilityDep, ) -> LibraryActivityResponse: revisions = await identification.stream_revisions() runs = await coordinator.current() @@ -270,7 +275,7 @@ async def library_activity( state = "pausing" elif control_state == "paused": state = "paused" - elif waiting: + elif counts.get("running", 0) or identification_snapshot["claimable_count"]: state = "running" elif identification_snapshot["failure_event_id"] is not None: state = "failed" @@ -298,13 +303,17 @@ async def library_activity( needs_review_count=counts.get("needs_review", 0), failed_count=counts.get("failed", 0), deferred_count=identification_snapshot["deferred_count"], + deferred_reason_counts=identification_snapshot[ + "deferred_reason_counts" + ], + attention_count=identification_snapshot["attention_count"], priority_band=( _IDENTIFICATION_PRIORITY_LABELS.get(active_priority, "Queued work") if active_priority is not None else None ), oldest_backlog_at=identification_snapshot["started_at"], - provider_unavailable=bool(identification_snapshot["deferred_count"]), + provider_unavailable=not mb_provider_available(), control_revision=identification_snapshot["control_revision"], failure_event_id=identification_snapshot["failure_event_id"], failure_at=identification_snapshot["failure_at"], @@ -532,6 +541,34 @@ async def scan_run_detail( return ScanRunDetailResponse(snapshot=await coordinator.snapshot(run_id)) +@router.get("/scan-runs/{run_id}/failures", response_model=ScanRunFailuresResponse) +async def scan_run_failures( + run_id: str, + _: CurrentAdminDep, + store: NativeLibraryStoreDep, + limit: int = Query(default=50, ge=1, le=200), + cursor: int | None = Query(default=None, ge=1), +) -> ScanRunFailuresResponse: + await store.get_scan_run(run_id) + items, next_cursor = await store.list_scan_run_failures( + run_id, limit=limit, cursor_rowid=cursor + ) + return ScanRunFailuresResponse( + items=[ + ScanRunFailureItem( + root_id=item.root_id, + relative_path=item.relative_path, + failure_code=item.failure_code, + failure_detail=item.failure_detail, + phase=item.phase, + recorded_at=item.recorded_at, + ) + for item in items + ], + next_cursor=next_cursor, + ) + + async def _control( run_id: str, action: str, diff --git a/backend/api/v1/routes/playlists.py b/backend/api/v1/routes/playlists.py index 6d64fe3f7..f157b60e2 100644 --- a/backend/api/v1/routes/playlists.py +++ b/backend/api/v1/routes/playlists.py @@ -99,6 +99,7 @@ def _track_to_response(t) -> PlaylistTrackResponse: duration=t.duration, created_at=t.created_at, plex_rating_key=getattr(t, "plex_rating_key", None), + library_file_id=getattr(t, "library_file_id", None), ) diff --git a/backend/api/v1/routes/stream.py b/backend/api/v1/routes/stream.py index c27712f01..b4f367a86 100644 --- a/backend/api/v1/routes/stream.py +++ b/backend/api/v1/routes/stream.py @@ -15,7 +15,12 @@ get_navidrome_playback_service, get_plex_playback_service, ) -from core.exceptions import ExternalServiceError, PlaybackNotAllowedError, ResourceNotFoundError +from core.exceptions import ( + ExternalServiceError, + JellyfinAuthError, + PlaybackNotAllowedError, + ResourceNotFoundError, +) from infrastructure.msgspec_fastapi import MsgSpecBody, MsgSpecRoute from middleware import CurrentUserDep from services.jellyfin_playback_service import JellyfinPlaybackService @@ -32,16 +37,22 @@ async def stream_jellyfin_audio( item_id: str, request: Request, + current_user: CurrentUserDep, playback_service: JellyfinPlaybackService = Depends(get_jellyfin_playback_service), ) -> StreamingResponse: try: range_header = request.headers.get("Range") - return await playback_service.proxy_stream(item_id, range_header=range_header) + return await playback_service.proxy_stream( + item_id, range_header=range_header, user_id=current_user.id + ) except ResourceNotFoundError: raise HTTPException(status_code=404, detail="Audio item not found") except PlaybackNotAllowedError as e: logger.warning("Playback not allowed for %s: %s", item_id, e) raise HTTPException(status_code=403, detail="Playback not allowed") + except JellyfinAuthError as e: + logger.warning("Jellyfin auth failure streaming %s: %s", item_id, e) + raise HTTPException(status_code=502, detail="Failed to stream from Jellyfin") except ExternalServiceError as e: if "416" in str(e): raise HTTPException(status_code=416, detail="Range not satisfiable") @@ -51,15 +62,19 @@ async def stream_jellyfin_audio( @router.head("/jellyfin/{item_id}") async def head_jellyfin_audio( item_id: str, + current_user: CurrentUserDep, playback_service: JellyfinPlaybackService = Depends(get_jellyfin_playback_service), ) -> Response: try: - return await playback_service.proxy_head(item_id) + return await playback_service.proxy_head(item_id, user_id=current_user.id) except ResourceNotFoundError: raise HTTPException(status_code=404, detail="Audio item not found") except PlaybackNotAllowedError as e: logger.warning("Playback not allowed for %s: %s", item_id, e) raise HTTPException(status_code=403, detail="Playback not allowed") + except JellyfinAuthError as e: + logger.warning("Jellyfin auth failure heading %s: %s", item_id, e) + raise HTTPException(status_code=502, detail="Failed to resolve Jellyfin stream") except ExternalServiceError as e: logger.error("Jellyfin head stream error for %s: %s", item_id, e) raise HTTPException(status_code=502, detail="Failed to resolve Jellyfin stream") @@ -194,10 +209,11 @@ async def stream_local_file( @router.head("/navidrome/{item_id}") async def head_navidrome_audio( item_id: str, + current_user: CurrentUserDep, playback_service: NavidromePlaybackService = Depends(get_navidrome_playback_service), ) -> Response: try: - return await playback_service.proxy_head(item_id) + return await playback_service.proxy_head(item_id, user_id=current_user.id) except ValueError: raise HTTPException(status_code=400, detail="Invalid stream request") except ExternalServiceError: @@ -208,10 +224,13 @@ async def head_navidrome_audio( async def stream_navidrome_audio( item_id: str, request: Request, + current_user: CurrentUserDep, playback_service: NavidromePlaybackService = Depends(get_navidrome_playback_service), ) -> StreamingResponse: try: - return await playback_service.proxy_stream(item_id, request.headers.get("Range")) + return await playback_service.proxy_stream( + item_id, request.headers.get("Range"), user_id=current_user.id + ) except ValueError: raise HTTPException(status_code=400, detail="Invalid stream request") except ExternalServiceError as e: @@ -254,10 +273,11 @@ async def navidrome_stopped( @router.head("/plex/{part_key:path}") async def head_plex_audio( part_key: str, + current_user: CurrentUserDep, playback_service: PlexPlaybackService = Depends(get_plex_playback_service), ) -> Response: try: - return await playback_service.proxy_head(part_key) + return await playback_service.proxy_head(part_key, user_id=current_user.id) except ValueError: raise HTTPException(status_code=400, detail="Invalid stream request") except ExternalServiceError: @@ -268,10 +288,13 @@ async def head_plex_audio( async def stream_plex_audio( part_key: str, request: Request, + current_user: CurrentUserDep, playback_service: PlexPlaybackService = Depends(get_plex_playback_service), ) -> StreamingResponse: try: - return await playback_service.proxy_stream(part_key, request.headers.get("Range")) + return await playback_service.proxy_stream( + part_key, request.headers.get("Range"), user_id=current_user.id + ) except ValueError: raise HTTPException(status_code=400, detail="Invalid stream request") except ExternalServiceError as e: diff --git a/backend/api/v1/schemas/advanced_settings.py b/backend/api/v1/schemas/advanced_settings.py index 5f2ae9c21..9e3d11253 100644 --- a/backend/api/v1/schemas/advanced_settings.py +++ b/backend/api/v1/schemas/advanced_settings.py @@ -101,6 +101,7 @@ class AdvancedSettings(AppStruct): audiodb_enabled: bool = True audiodb_name_search_fallback: bool = False direct_remote_images_enabled: bool = True + prefer_local_cover_art: bool = True audiodb_api_key: str = "123" cache_ttl_audiodb_found: int = 604800 cache_ttl_audiodb_not_found: int = 86400 @@ -286,6 +287,7 @@ class AdvancedSettingsFrontend(AppStruct): audiodb_enabled: bool = True audiodb_name_search_fallback: bool = False direct_remote_images_enabled: bool = True + prefer_local_cover_art: bool = True audiodb_api_key: str = "123" cache_ttl_audiodb_found: int = 168 cache_ttl_audiodb_not_found: int = 24 @@ -494,6 +496,7 @@ def from_backend(settings: AdvancedSettings) -> "AdvancedSettingsFrontend": audiodb_enabled=settings.audiodb_enabled, audiodb_name_search_fallback=settings.audiodb_name_search_fallback, direct_remote_images_enabled=settings.direct_remote_images_enabled, + prefer_local_cover_art=settings.prefer_local_cover_art, audiodb_api_key=_mask_api_key(settings.audiodb_api_key), cache_ttl_audiodb_found=settings.cache_ttl_audiodb_found // 3600, cache_ttl_audiodb_not_found=settings.cache_ttl_audiodb_not_found // 3600, @@ -581,6 +584,7 @@ def to_backend(self) -> AdvancedSettings: audiodb_enabled=self.audiodb_enabled, audiodb_name_search_fallback=self.audiodb_name_search_fallback, direct_remote_images_enabled=self.direct_remote_images_enabled, + prefer_local_cover_art=self.prefer_local_cover_art, audiodb_api_key=self.audiodb_api_key, cache_ttl_audiodb_found=self.cache_ttl_audiodb_found * 3600, cache_ttl_audiodb_not_found=self.cache_ttl_audiodb_not_found * 3600, diff --git a/backend/api/v1/schemas/library_operations.py b/backend/api/v1/schemas/library_operations.py index 8bc56151f..ec9d9e30f 100644 --- a/backend/api/v1/schemas/library_operations.py +++ b/backend/api/v1/schemas/library_operations.py @@ -347,6 +347,17 @@ class RepairApplyRequest(AppStruct): confirmation: bool +class SuggestedEditionSummary(AppStruct): + release_mbid: str + release_group_mbid: str + title: str + track_count: int + competing_count: int + date: str | None = None + country: str | None = None + status: str | None = None + + class RepairFindingResponse(AppStruct): id: str local_album_id: str @@ -362,6 +373,7 @@ class RepairFindingResponse(AppStruct): apply_eligible: bool state: str apply_result: str | None = None + suggested_edition: SuggestedEditionSummary | None = None updated_at: float = 0.0 row_revision: int = 1 diff --git a/backend/api/v1/schemas/library_policies.py b/backend/api/v1/schemas/library_policies.py index b011dfdc2..7828b3a58 100644 --- a/backend/api/v1/schemas/library_policies.py +++ b/backend/api/v1/schemas/library_policies.py @@ -31,6 +31,10 @@ class TypedLibrarySettings(AppStruct): staging_path: str = "" naming_template: str = DEFAULT_NAMING_TEMPLATE acoustid_api_key: str = "" + # Master switch (GH #276): when False the target application stops claiming + # new scan/identification/organization work. Deliberately excluded from the + # policy revision hash so toggling never churns boundary transitions. + enabled: bool = True class LibrarySettingsResponse(TypedLibrarySettings): diff --git a/backend/api/v1/schemas/library_scan_target.py b/backend/api/v1/schemas/library_scan_target.py index d640e3181..d1d2f6c5a 100644 --- a/backend/api/v1/schemas/library_scan_target.py +++ b/backend/api/v1/schemas/library_scan_target.py @@ -52,6 +52,20 @@ class ScanRunDetailResponse(AppStruct): snapshot: ScanRunSnapshot +class ScanRunFailureItem(AppStruct): + root_id: str + relative_path: str + failure_code: str + failure_detail: str + phase: Literal["discovering", "indexing", "reconciling"] + recorded_at: float + + +class ScanRunFailuresResponse(AppStruct): + items: list[ScanRunFailureItem] + next_cursor: int | None = None + + class LibraryActivityItem(AppStruct): kind: Literal["scan", "identification"] state: str @@ -67,6 +81,8 @@ class LibraryActivityItem(AppStruct): needs_review_count: int = 0 failed_count: int = 0 deferred_count: int = 0 + deferred_reason_counts: dict[str, int] = msgspec.field(default_factory=dict) + attention_count: int = 0 priority_band: str | None = None oldest_backlog_at: float | None = None provider_unavailable: bool = False diff --git a/backend/api/v1/schemas/playlists.py b/backend/api/v1/schemas/playlists.py index f38c9ba85..b0cfc5433 100644 --- a/backend/api/v1/schemas/playlists.py +++ b/backend/api/v1/schemas/playlists.py @@ -20,6 +20,7 @@ class PlaylistTrackResponse(AppStruct): duration: int | None = None created_at: str = "" plex_rating_key: str | None = None + library_file_id: str | None = None class PlaylistSummaryResponse(AppStruct): diff --git a/backend/core/dependencies/__init__.py b/backend/core/dependencies/__init__.py index e0cc16c0a..6768241c7 100644 --- a/backend/core/dependencies/__init__.py +++ b/backend/core/dependencies/__init__.py @@ -122,6 +122,7 @@ get_target_identification_queue, get_library_administrative_work_service, get_target_album_identification_service, + get_mb_provider_availability, get_target_album_coverage_service, get_target_reidentification_service, get_target_album_edition_finder_service, @@ -279,6 +280,7 @@ TargetIdentificationQueueDep, LibraryAdministrativeWorkServiceDep, TargetAlbumIdentificationServiceDep, + MbProviderAvailabilityDep, TargetAlbumCoverageServiceDep, TargetReidentificationServiceDep, TargetAlbumEditionFinderServiceDep, diff --git a/backend/core/dependencies/repo_providers.py b/backend/core/dependencies/repo_providers.py index 8429450e8..965745321 100644 --- a/backend/core/dependencies/repo_providers.py +++ b/backend/core/dependencies/repo_providers.py @@ -20,6 +20,7 @@ get_disk_cache, get_library_db, get_mbid_store, + get_native_library_store, get_preferences_service, ) @@ -500,7 +501,9 @@ def get_follow_store() -> "FollowStore": ) -def _build_coverart_repository(*, library_repo=None, library_db=None): +def _build_coverart_repository( + *, library_repo=None, library_db=None, native_library_store=None +): from repositories.coverart_repository import CoverArtRepository settings = get_settings() @@ -531,21 +534,28 @@ def _build_coverart_repository(*, library_repo=None, library_db=None): * 1024, cover_non_monitored_ttl_seconds=advanced.cache_ttl_recently_viewed_bytes, library_db=library_db, + local_cover_priority=lambda: get_preferences_service() + .get_advanced_settings() + .prefer_local_cover_art, + native_library_store=native_library_store, ) @singleton def get_coverart_repository() -> "CoverArtRepository": return _build_coverart_repository( - library_repo=get_library_repository(), library_db=get_library_db() + library_repo=get_library_repository(), + library_db=get_library_db(), + native_library_store=get_native_library_store(), ) @singleton def get_target_coverart_repository() -> "CoverArtRepository": - """Provider-only cover lookup with no retained legacy catalog authority.""" + """Provider cover lookup with native folder/embedded art, but no retained + legacy catalog authority.""" - return _build_coverart_repository() + return _build_coverart_repository(native_library_store=get_native_library_store()) @singleton diff --git a/backend/core/dependencies/service_providers.py b/backend/core/dependencies/service_providers.py index 097f40f85..eca672266 100644 --- a/backend/core/dependencies/service_providers.py +++ b/backend/core/dependencies/service_providers.py @@ -2,6 +2,7 @@ import asyncio import logging +from collections.abc import Callable from infrastructure.cache.cache_keys import ( library_raw_albums_key, @@ -443,6 +444,7 @@ def get_target_library_scan_coordinator() -> "LibraryScanCoordinator": from services.native.library_reconciler import LibraryReconciler from services.native.library_scan_coordinator import LibraryScanCoordinator from services.native.library_scan_events import LibraryScanEventPublisher + from services.native.local_album_grouping_service import LocalAlbumGroupingService from .cache_providers import get_native_library_store @@ -451,7 +453,12 @@ def get_target_library_scan_coordinator() -> "LibraryScanCoordinator": return LibraryScanCoordinator( store, LibraryInventoryScanner(store, filesystem_coordinator=filesystem), - LibraryIndexer(store, get_audio_tagger(), filesystem_coordinator=filesystem), + LibraryIndexer( + store, + get_audio_tagger(), + grouping=LocalAlbumGroupingService(store, get_target_identification_queue()), + filesystem_coordinator=filesystem, + ), LibraryReconciler(store, filesystem), get_library_policy_resolver, LibraryScanEventPublisher(store, get_sse_publisher()), @@ -474,7 +481,10 @@ def get_target_identification_queue() -> "IdentificationQueueService": from .cache_providers import get_native_library_store - return IdentificationQueueService(get_native_library_store()) + return IdentificationQueueService( + get_native_library_store(), + provider_available=get_mb_provider_availability(), + ) @singleton @@ -488,6 +498,13 @@ def get_library_administrative_work_service() -> "LibraryAdministrativeWorkServi return LibraryAdministrativeWorkService(get_native_library_store()) +def get_mb_provider_availability() -> Callable[[], bool]: + """Live MusicBrainz breaker read shared by identification and activity routes.""" + from repositories.musicbrainz_base import mb_circuit_breaker + + return lambda: not mb_circuit_breaker.is_open() + + @singleton def get_target_album_identification_service() -> "AlbumIdentificationService": from services.native.album_candidate_service import AlbumCandidateService @@ -496,7 +513,6 @@ def get_target_album_identification_service() -> "AlbumIdentificationService": from services.native.conditional_fingerprint_service import ( ConditionalFingerprintService, ) - from repositories.musicbrainz_base import mb_circuit_breaker from .cache_providers import get_native_library_store @@ -520,7 +536,7 @@ async def invalidate(_domains: set[str]) -> None: ConditionalFingerprintService(store, get_audio_fingerprinter()), invalidate, _schedule_identified_album_work, - provider_available=lambda: not mb_circuit_breaker.is_open(), + provider_available=get_mb_provider_availability(), ) @@ -634,6 +650,7 @@ def get_target_identity_repair_service() -> "IdentityRepairService": get_musicbrainz_identification_repository(), AlbumEvidenceEngine(), get_musicbrainz_repository(), + provider_available=get_mb_provider_availability(), ) @@ -2580,6 +2597,9 @@ def get_acquisition_cleanup_service() -> "AcquisitionCleanupService": lambda: Path( get_preferences_service().get_sabnzbd_connection_raw().downloads_mount ), + sab_category_getter=lambda: get_preferences_service() + .get_sabnzbd_connection_raw() + .category, ) diff --git a/backend/core/dependencies/type_aliases.py b/backend/core/dependencies/type_aliases.py index 40fe6ab8f..347bf6394 100644 --- a/backend/core/dependencies/type_aliases.py +++ b/backend/core/dependencies/type_aliases.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Callable from typing import Annotated from fastapi import Depends @@ -168,6 +169,7 @@ get_library_administrative_work_service, get_target_album_coverage_service, get_target_album_identification_service, + get_mb_provider_availability, get_target_reidentification_service, get_target_album_edition_finder_service, get_target_library_review_service, @@ -302,6 +304,9 @@ TargetAlbumIdentificationServiceDep = Annotated[ AlbumIdentificationService, Depends(get_target_album_identification_service) ] +MbProviderAvailabilityDep = Annotated[ + Callable[[], bool], Depends(get_mb_provider_availability) +] TargetAlbumCoverageServiceDep = Annotated[ AlbumCoverageService, Depends(get_target_album_coverage_service) ] diff --git a/backend/core/tasks.py b/backend/core/tasks.py index bfcf5c762..2efa6c068 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -424,9 +424,14 @@ async def warm_navidrome_mbid_cache(service_getter=None) -> None: try: service = service_getter() await service.warm_mbid_cache() + except asyncio.CancelledError: + break except Exception as e: logger.error("Navidrome MBID cache warming failed: %s", e, exc_info=True) - await asyncio.sleep(14400) + try: + await asyncio.sleep(14400) + except asyncio.CancelledError: + break async def warm_plex_mbid_cache(service_getter=None) -> None: @@ -441,9 +446,14 @@ async def warm_plex_mbid_cache(service_getter=None) -> None: service = service_getter() await service.warm_mbid_cache() await service.persist_if_dirty() + except asyncio.CancelledError: + break except Exception as e: logger.error("Plex MBID cache warming failed: %s", e, exc_info=True) - await asyncio.sleep(14400) + try: + await asyncio.sleep(14400) + except asyncio.CancelledError: + break async def warm_artist_discovery_cache_periodically( @@ -761,155 +771,160 @@ async def warm_audiodb_cache_periodically( await asyncio.sleep(_AUDIODB_SWEEP_INITIAL_DELAY) while True: + try: + await _run_audiodb_sweep_cycle( + audiodb_image_service, + library_db, + preferences_service, + precache_service, + workload_gate, + ) + except asyncio.CancelledError: + break + except Exception as e: + logger.error("AudioDB sweep cycle failed: %s", e, exc_info=True) try: await asyncio.sleep(_AUDIODB_SWEEP_INTERVAL) + except asyncio.CancelledError: + break - if workload_gate is not None: - await workload_gate.wait_until_available() - - settings = preferences_service.get_advanced_settings() - if not settings.audiodb_enabled: - continue - - cursor = preferences_service.get_setting("audiodb_sweep_cursor") - all_items = await library_db.get_enrichment_candidates( - after_mbid=cursor, - limit=_AUDIODB_SWEEP_MAX_ITEMS, - ) - if not all_items: - preferences_service.save_setting("audiodb_sweep_cursor", None) - preferences_service.save_setting("audiodb_sweep_last_completed", time()) - continue - items_needing_refresh: list[tuple[str, str, dict]] = [] - inspected_cursor = cursor - inspection_complete = True - for entity_type, mbid, data in all_items: - if workload_gate is not None and workload_gate.scan_active: - inspection_complete = False - break - inspected_cursor = f"{entity_type}:{mbid}" - if len(items_needing_refresh) >= _AUDIODB_SWEEP_MAX_ITEMS: - break - if entity_type == "artist": - cached = await audiodb_image_service.get_cached_artist_images(mbid) - else: - cached = await audiodb_image_service.get_cached_album_images(mbid) - if cached is None: - items_needing_refresh.append((entity_type, mbid, data)) - - if not items_needing_refresh: - page_complete = inspection_complete and ( - len(all_items) < _AUDIODB_SWEEP_MAX_ITEMS - ) - preferences_service.save_setting( - "audiodb_sweep_cursor", None if page_complete else inspected_cursor - ) - if page_complete: - preferences_service.save_setting( - "audiodb_sweep_last_completed", time() - ) - continue +async def _run_audiodb_sweep_cycle( + audiodb_image_service: "AudioDBImageService", + library_db: "LibraryDB", + preferences_service: "PreferencesService", + precache_service: "LibraryPrecacheService | None", + workload_gate: "BackgroundWorkloadGate | None", +) -> None: + if workload_gate is not None: + await workload_gate.wait_until_available() - processed = 0 - processed_cursor = cursor - bytes_ok = 0 - bytes_fail = 0 - for entity_type, mbid, data in items_needing_refresh: - if workload_gate is not None and workload_gate.scan_active: - break - if not preferences_service.get_advanced_settings().audiodb_enabled: - break + settings = preferences_service.get_advanced_settings() + if not settings.audiodb_enabled: + return - try: - if entity_type == "artist": - name = data.get("name") if isinstance(data, dict) else None - result = ( - await audiodb_image_service.fetch_and_cache_artist_images( - mbid, - name, - is_monitored=True, - ) - ) - if ( - result - and not result.is_negative - and result.thumb_url - and precache_service - ): - if await precache_service._download_audiodb_bytes( - result.thumb_url, "artist", mbid - ): - bytes_ok += 1 - else: - bytes_fail += 1 - else: - artist_name = ( - data.get("artist_name") - if isinstance(data, dict) - else getattr(data, "artist_name", None) - ) - album_name = ( - data.get("title") - if isinstance(data, dict) - else getattr(data, "title", None) - ) - result = ( - await audiodb_image_service.fetch_and_cache_album_images( - mbid, - artist_name=artist_name, - album_name=album_name, - is_monitored=True, - ) - ) - if ( - result - and not result.is_negative - and result.album_thumb_url - and precache_service - ): - if await precache_service._download_audiodb_bytes( - result.album_thumb_url, "album", mbid - ): - bytes_ok += 1 - else: - bytes_fail += 1 - except Exception as e: - logger.error( - "audiodb.sweep action=item_error entity_type=%s mbid=%s error=%s", - entity_type, - mbid[:8], - e, - exc_info=True, - ) + cursor = preferences_service.get_setting("audiodb_sweep_cursor") + all_items = await library_db.get_enrichment_candidates( + after_mbid=cursor, + limit=_AUDIODB_SWEEP_MAX_ITEMS, + ) + if not all_items: + preferences_service.save_setting("audiodb_sweep_cursor", None) + preferences_service.save_setting("audiodb_sweep_last_completed", time()) + return - processed += 1 - processed_cursor = f"{entity_type}:{mbid}" - if processed % _AUDIODB_SWEEP_CURSOR_PERSIST_INTERVAL == 0: - preferences_service.save_setting( - "audiodb_sweep_cursor", processed_cursor - ) + items_needing_refresh: list[tuple[str, str, dict]] = [] + inspected_cursor = cursor + inspection_complete = True + for entity_type, mbid, data in all_items: + if workload_gate is not None and workload_gate.scan_active: + inspection_complete = False + break + inspected_cursor = f"{entity_type}:{mbid}" + if len(items_needing_refresh) >= _AUDIODB_SWEEP_MAX_ITEMS: + break + if entity_type == "artist": + cached = await audiodb_image_service.get_cached_artist_images(mbid) + else: + cached = await audiodb_image_service.get_cached_album_images(mbid) + if cached is None: + items_needing_refresh.append((entity_type, mbid, data)) + + if not items_needing_refresh: + page_complete = inspection_complete and ( + len(all_items) < _AUDIODB_SWEEP_MAX_ITEMS + ) + preferences_service.save_setting( + "audiodb_sweep_cursor", None if page_complete else inspected_cursor + ) + if page_complete: + preferences_service.save_setting("audiodb_sweep_last_completed", time()) + return - await asyncio.sleep(_AUDIODB_SWEEP_INTER_ITEM_DELAY) + processed = 0 + processed_cursor = cursor + bytes_ok = 0 + bytes_fail = 0 + for entity_type, mbid, data in items_needing_refresh: + if workload_gate is not None and workload_gate.scan_active: + break + if not preferences_service.get_advanced_settings().audiodb_enabled: + break - if processed >= len(items_needing_refresh) and inspection_complete: - page_complete = len(all_items) < _AUDIODB_SWEEP_MAX_ITEMS - preferences_service.save_setting( - "audiodb_sweep_cursor", None if page_complete else inspected_cursor + try: + if entity_type == "artist": + name = data.get("name") if isinstance(data, dict) else None + result = await audiodb_image_service.fetch_and_cache_artist_images( + mbid, + name, + is_monitored=True, ) - if page_complete: - preferences_service.save_setting( - "audiodb_sweep_last_completed", time() - ) + if ( + result + and not result.is_negative + and result.thumb_url + and precache_service + ): + if await precache_service._download_audiodb_bytes( + result.thumb_url, "artist", mbid + ): + bytes_ok += 1 + else: + bytes_fail += 1 else: - preferences_service.save_setting( - "audiodb_sweep_cursor", processed_cursor + artist_name = ( + data.get("artist_name") + if isinstance(data, dict) + else getattr(data, "artist_name", None) ) - - except asyncio.CancelledError: - break + album_name = ( + data.get("title") + if isinstance(data, dict) + else getattr(data, "title", None) + ) + result = await audiodb_image_service.fetch_and_cache_album_images( + mbid, + artist_name=artist_name, + album_name=album_name, + is_monitored=True, + ) + if ( + result + and not result.is_negative + and result.album_thumb_url + and precache_service + ): + if await precache_service._download_audiodb_bytes( + result.album_thumb_url, "album", mbid + ): + bytes_ok += 1 + else: + bytes_fail += 1 except Exception as e: - logger.error("AudioDB sweep cycle failed: %s", e, exc_info=True) + logger.error( + "audiodb.sweep action=item_error entity_type=%s mbid=%s error=%s", + entity_type, + mbid[:8], + e, + exc_info=True, + ) + + processed += 1 + processed_cursor = f"{entity_type}:{mbid}" + if processed % _AUDIODB_SWEEP_CURSOR_PERSIST_INTERVAL == 0: + preferences_service.save_setting("audiodb_sweep_cursor", processed_cursor) + + await asyncio.sleep(_AUDIODB_SWEEP_INTER_ITEM_DELAY) + + if processed >= len(items_needing_refresh) and inspection_complete: + page_complete = len(all_items) < _AUDIODB_SWEEP_MAX_ITEMS + preferences_service.save_setting( + "audiodb_sweep_cursor", None if page_complete else inspected_cursor + ) + if page_complete: + preferences_service.save_setting("audiodb_sweep_last_completed", time()) + else: + preferences_service.save_setting("audiodb_sweep_cursor", processed_cursor) def start_audiodb_sweep_task( @@ -1342,18 +1357,19 @@ async def prune_recycle_bin_periodically( await asyncio.sleep(600) while True: try: - policy = preferences_service.get_download_policy() library = preferences_service.get_typed_library_settings() - bin_path = resolve_bin_path( - policy.recycle_bin_path, - [root.path for root in library.library_roots], - ) - if bin_path is not None: - removed = await asyncio.to_thread( - prune, bin_path, policy.recycle_retention_days + if library.enabled: + policy = preferences_service.get_download_policy() + bin_path = resolve_bin_path( + policy.recycle_bin_path, + [root.path for root in library.library_roots], ) - if removed: - logger.info("Recycle bin prune removed %d entries", removed) + if bin_path is not None: + removed = await asyncio.to_thread( + prune, bin_path, policy.recycle_retention_days + ) + if removed: + logger.info("Recycle bin prune removed %d entries", removed) except asyncio.CancelledError: break except Exception as e: diff --git a/backend/infrastructure/cache/cache_keys.py b/backend/infrastructure/cache/cache_keys.py index d3ef0829f..7414cb990 100644 --- a/backend/infrastructure/cache/cache_keys.py +++ b/backend/infrastructure/cache/cache_keys.py @@ -16,6 +16,7 @@ MB_RECORDING_SEARCH_PREFIX = "mb:recording:search:" MB_RECORDING_TO_RG_PREFIX = "mb:recording_to_rg:" MB_ARTIST_RELS_PREFIX = "mb:artist_rels:" +MB_ARTIST_RGS_PREFIX = "mb:artist_rgs:" MB_ARTISTS_BY_TAG_PREFIX = "mb_artists_by_tag:" MB_RG_BY_TAG_PREFIX = "mb_rg_by_tag:" MB_URL_RESOLUTION_PREFIX = "mb:url:resolution:" @@ -152,6 +153,7 @@ def musicbrainz_prefixes() -> list[str]: MB_RECORDING_SEARCH_PREFIX, MB_RECORDING_TO_RG_PREFIX, MB_ARTIST_RELS_PREFIX, + MB_ARTIST_RGS_PREFIX, MB_ARTISTS_BY_TAG_PREFIX, MB_RG_BY_TAG_PREFIX, MB_URL_RESOLUTION_PREFIX, @@ -288,6 +290,10 @@ def mb_artist_detail_key(mbid: str) -> str: return f"{MB_ARTIST_DETAIL_PREFIX}{mbid}" +def mb_artist_release_groups_key(artist_mbid: str) -> str: + return f"{MB_ARTIST_RGS_PREFIX}{artist_mbid.casefold()}" + + def mb_release_group_key(mbid: str, includes: Optional[list[str]] = None) -> str: includes_str = ",".join(sorted(includes)) if includes else "default" return f"{MB_RG_DETAIL_PREFIX}{mbid}:{includes_str}" diff --git a/backend/infrastructure/persistence/native_library_schema.py b/backend/infrastructure/persistence/native_library_schema.py index 32624b328..1f32125af 100644 --- a/backend/infrastructure/persistence/native_library_schema.py +++ b/backend/infrastructure/persistence/native_library_schema.py @@ -1288,6 +1288,17 @@ PRIMARY KEY(run_id, root_id, relative_path) ); +CREATE TABLE IF NOT EXISTS library_scan_failures ( + run_id TEXT NOT NULL REFERENCES library_scan_runs(id) ON DELETE CASCADE, + root_id TEXT NOT NULL, + relative_path TEXT NOT NULL, + failure_code TEXT NOT NULL, + failure_detail TEXT NOT NULL DEFAULT '', + phase TEXT NOT NULL CHECK(phase IN ('discovering','indexing','reconciling')), + recorded_at REAL NOT NULL, + PRIMARY KEY(run_id, root_id, relative_path, phase, failure_code) +); + CREATE TABLE IF NOT EXISTS library_scan_management_candidates ( run_id TEXT NOT NULL REFERENCES library_scan_runs(id) ON DELETE CASCADE, local_album_id TEXT NOT NULL REFERENCES local_albums(id) ON DELETE CASCADE, @@ -1710,6 +1721,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_scan_runs_single_queued ON library_scan_runs((1)) WHERE state = 'queued'; CREATE INDEX IF NOT EXISTS idx_scan_inventory_processing ON library_scan_inventory(run_id, processing_state, root_id, relative_path); +CREATE INDEX IF NOT EXISTS idx_scan_failures_run ON library_scan_failures(run_id); CREATE INDEX IF NOT EXISTS idx_scan_inventory_management_candidates ON library_scan_inventory(run_id, processing_state, comparison_result, local_track_id); CREATE INDEX IF NOT EXISTS idx_scan_management_candidates_due ON library_scan_management_candidates(state, next_attempt_at, run_id, local_album_id); CREATE INDEX IF NOT EXISTS idx_scan_grouping_pending ON library_scan_grouping_contexts(run_id, state, root_id, relative_directory); diff --git a/backend/infrastructure/persistence/native_library_store.py b/backend/infrastructure/persistence/native_library_store.py index 343540d1b..2132e3405 100644 --- a/backend/infrastructure/persistence/native_library_store.py +++ b/backend/infrastructure/persistence/native_library_store.py @@ -100,6 +100,7 @@ OperationWorkItem, RepairFinding, ReviewDecision, + ScanFailureRecord, ScanInventoryItem, ScanRequest, ScanRequestResult, @@ -131,6 +132,7 @@ AUTOMATIC_SAFE_EVIDENCE_REASONS = frozenset( {"SUPPORTED", "ACCEPTED", "SUPPORTED_EMBEDDED_IDS"} ) +ATTENTION_FAILURE_CODES = frozenset({"MAX_DEFERRALS_EXCEEDED", "SUBJECT_NOT_AVAILABLE"}) BULK_PREVIEW_BATCH_SIZE = 500 BULK_PREVIEW_CLEANUP_BATCH_SIZE = 5_000 MANAGEMENT_PERSISTENCE_BATCH_SIZE = 500 @@ -1185,6 +1187,9 @@ def _ensure_tables(self) -> None: "ALTER TABLE library_identity_repair_findings ADD COLUMN reason_code TEXT NOT NULL DEFAULT ''", "ALTER TABLE library_identity_repair_findings ADD COLUMN apply_eligible INTEGER NOT NULL DEFAULT 0 CHECK(apply_eligible IN (0,1))", "ALTER TABLE library_identity_repair_findings ADD COLUMN apply_result TEXT", + "ALTER TABLE library_identity_repair_findings ADD COLUMN suggested_release_mbid TEXT", + "ALTER TABLE library_identity_repair_findings ADD COLUMN suggested_release_group_mbid TEXT", + "ALTER TABLE library_identity_repair_findings ADD COLUMN suggested_edition_json TEXT NOT NULL DEFAULT '{}'", "ALTER TABLE library_catalog_actions ADD COLUMN local_artist_id TEXT REFERENCES local_artists(id) ON DELETE RESTRICT", "ALTER TABLE library_scan_run_scopes ADD COLUMN scope_id TEXT", "ALTER TABLE library_scan_run_scopes ADD COLUMN root_path TEXT", @@ -1249,6 +1254,7 @@ def _ensure_tables(self) -> None: "ALTER TABLE library_management_import_journal ADD COLUMN baseline_file_mtime_ns INTEGER", "ALTER TABLE library_management_import_journal ADD COLUMN baseline_file_mode INTEGER", "ALTER TABLE library_operation_jobs ADD COLUMN next_attempt_at REAL", + "ALTER TABLE library_identification_jobs ADD COLUMN attention_cause TEXT", ): try: connection.execute(statement) @@ -6527,10 +6533,16 @@ def _insert_evidence( ) async def enqueue_identification_job( - self, job: IdentificationJob, *, expected_policy_revision: str | None = None + self, + job: IdentificationJob, + *, + expected_policy_revision: str | None = None, + resurrect_attention: bool = True, ) -> str: job_id, _ = await self.enqueue_identification_job_result( - job, expected_policy_revision=expected_policy_revision + job, + expected_policy_revision=expected_policy_revision, + resurrect_attention=resurrect_attention, ) return job_id @@ -6539,6 +6551,7 @@ async def enqueue_identification_job_result( job: IdentificationJob, *, expected_policy_revision: str | None = None, + resurrect_attention: bool = True, ) -> tuple[str, bool]: def operation(connection: sqlite3.Connection) -> tuple[str, bool]: if expected_policy_revision is not None: @@ -6555,7 +6568,9 @@ def operation(connection: sqlite3.Connection) -> tuple[str, bool]: raise StaleRevisionError( "The library policy changed while identification was queued." ) - return self._enqueue_identification_job_result(connection, job) + return self._enqueue_identification_job_result( + connection, job, resurrect_attention=resurrect_attention + ) result = await self._write(operation) if result[0]: @@ -6574,12 +6589,16 @@ async def enqueue_identification_job_results( grouping_context: tuple[str, str] | None = None, queue_cursor: str | None = None, background: bool = False, + resurrect_attention: bool = True, ) -> list[tuple[str, bool]]: """Enqueue one bounded job batch in a single store-owned transaction.""" def operation(connection: sqlite3.Connection) -> list[tuple[str, bool]]: results = [ - self._enqueue_identification_job_result(connection, job) for job in jobs + self._enqueue_identification_job_result( + connection, job, resurrect_attention=resurrect_attention + ) + for job in jobs ] created = sum(created for _, created in results) if scan_run_id is not None and created: @@ -6629,7 +6648,11 @@ def operation(connection: sqlite3.Connection) -> list[tuple[str, bool]]: return result def _enqueue_identification_job_result( - self, connection: sqlite3.Connection, job: IdentificationJob + self, + connection: sqlite3.Connection, + job: IdentificationJob, + *, + resurrect_attention: bool = True, ) -> tuple[str, bool]: subject_column = ( "local_album_id" if job.local_album_id is not None else "local_track_id" @@ -6660,7 +6683,8 @@ def _enqueue_identification_job_result( (job.created_at, decision["id"]), ) existing = connection.execute( - "SELECT id FROM library_identification_jobs WHERE dedupe_key = ? " + "SELECT id, state, last_failure_code, attention_cause " + "FROM library_identification_jobs WHERE dedupe_key = ? " + ( "AND state IN ('queued','running','paused') " if job.kind == "review_retry" @@ -6670,6 +6694,27 @@ def _enqueue_identification_job_result( (job.dedupe_key,), ).fetchone() if existing is not None: + if ( + resurrect_attention + and str(existing["state"]) == "failed" + and str(existing["last_failure_code"] or "") in ATTENTION_FAILURE_CODES + and str(existing["attention_cause"] or "") + == "PROVIDER_TEMPORARILY_UNAVAILABLE" + ): + # Terminal-failed attention jobs must not block later enqueues of + # the same dedupe key. Only provider-caused caps resurrect; + # deterministic failures stay terminal until the album's input + # changes (a new dedupe key). + connection.execute( + "UPDATE library_identification_jobs SET state = 'queued', " + "attempt_count = 0, not_before = ?, last_failure_code = NULL, " + "attention_cause = NULL, terminal_at = NULL, updated_at = ?, " + "row_revision = row_revision + 1, event_revision = event_revision + 1 " + "WHERE id = ?", + (job.not_before, job.created_at, existing["id"]), + ) + self._bump_stream(connection, "identification") + return str(existing["id"]), True return str(existing["id"]), False queued_subject = connection.execute( f"SELECT id FROM library_identification_jobs WHERE {subject_column} = ? " @@ -6991,6 +7036,27 @@ def operation(connection: sqlite3.Connection) -> dict[str, Any] | None: return await self._read(operation) + async def get_indexed_track_paths_for_release_group( + self, release_group_mbid: str + ) -> list[str]: + """Absolute file paths of indexed tracks sealed to this release group.""" + + def operation(connection: sqlite3.Connection) -> list[str]: + rows = connection.execute( + "SELECT t.file_path FROM local_tracks t " + "JOIN local_album_external_identities i " + "ON i.local_album_id = t.local_album_id " + "JOIN local_albums a ON a.id = t.local_album_id " + "WHERE i.provider = 'musicbrainz' " + "AND lower(i.release_group_mbid) = lower(?) " + "AND a.retired_into_album_id IS NULL " + "AND t.availability = 'indexed' ORDER BY t.file_path", + (release_group_mbid,), + ).fetchall() + return [str(row["file_path"]) for row in rows if row["file_path"]] + + return await self._read(operation) + async def get_attempt_evidence( self, attempt_id: str ) -> list[IdentificationEvidenceRecord]: @@ -7442,6 +7508,132 @@ def operation(connection: sqlite3.Connection) -> int: self.work_wakeups.notify_after("identification", not_before - now) return result + async def terminal_fail_identification_job( + self, + job_id: str, + *, + worker_id: str, + expected_job_revision: int, + failure_code: str, + now: float, + attention_cause: str | None = None, + ) -> int: + """Fail a running job terminally, keeping the row for auditability. + + Album-scoped terminal failures surface a review row so the album stays + findable in the review queue and can be dismissed or retried there. + """ + + def operation(connection: sqlite3.Connection) -> int: + row = connection.execute( + "UPDATE library_identification_jobs SET state = 'failed', " + "last_failure_code = ?, attention_cause = ?, terminal_at = ?, " + "lease_owner = NULL, lease_expires_at = NULL, heartbeat_at = NULL, " + "updated_at = ?, row_revision = row_revision + 1, " + "event_revision = event_revision + 1 WHERE id = ? AND state = 'running' " + "AND lease_owner = ? AND row_revision = ? RETURNING row_revision", + ( + failure_code, + attention_cause, + now, + now, + job_id, + worker_id, + expected_job_revision, + ), + ).fetchone() + if row is None: + raise StaleRevisionError( + "The identification job changed before it could be failed." + ) + job = connection.execute( + "SELECT local_album_id, local_track_id, input_revision " + "FROM library_identification_jobs WHERE id = ?", + (job_id,), + ).fetchone() + if ( + job is not None + and job["local_album_id"] is not None + and job["local_track_id"] is None + ): + active_review = connection.execute( + "SELECT id FROM library_identification_reviews " + "WHERE local_album_id = ? AND input_revision = ? " + "AND state != 'resolved'", + (job["local_album_id"], job["input_revision"]), + ).fetchone() + if active_review is None: + connection.execute( + "INSERT INTO library_identification_reviews " + "(id, local_album_id, state, reason_code, attempt_id, " + "input_revision, created_at, updated_at) " + "VALUES (?, ?, 'needs_review', ?, NULL, ?, ?, ?)", + ( + str(uuid.uuid4()), + job["local_album_id"], + failure_code, + job["input_revision"], + now, + now, + ), + ) + self._bump_stream(connection, "identification") + return int(row["row_revision"]) + + return await self._write(operation) + + async def reset_provider_identification_deferrals(self, *, now: float) -> int: + """Clear backoff on provider-deferred queued jobs after recovery. + + Only rows deferred for PROVIDER_TEMPORARILY_UNAVAILABLE are reset; other + deferral reasons keep their backoff untouched. + """ + + def operation(connection: sqlite3.Connection) -> int: + cursor = connection.execute( + "UPDATE library_identification_jobs SET attempt_count = 0, " + "not_before = 0, last_failure_code = NULL, updated_at = ?, " + "row_revision = row_revision + 1, event_revision = event_revision + 1 " + "WHERE state = 'queued' " + "AND last_failure_code = 'PROVIDER_TEMPORARILY_UNAVAILABLE' " + "AND row_revision < ? AND event_revision < ?", + (now, MAX_REVISION, MAX_REVISION), + ) + if cursor.rowcount: + self._bump_stream(connection, "identification") + return cursor.rowcount + + result = await self._write(operation) + if result: + self.work_wakeups.notify("identification") + return result + + async def gc_stale_identification_jobs( + self, *, now: float, grace_seconds: float + ) -> int: + """Terminally fail queued SUBJECT_NOT_AVAILABLE jobs past the grace period. + + Tracks can legitimately reappear, so the worker keeps deferring until the + grace expires; this sweep is the backstop for rows that are never claimed + again (the deferral cap bounds actively-claimed rows first). + """ + + def operation(connection: sqlite3.Connection) -> int: + cursor = connection.execute( + "UPDATE library_identification_jobs SET state = 'failed', " + "attention_cause = 'SUBJECT_NOT_AVAILABLE', terminal_at = ?, " + "updated_at = ?, row_revision = row_revision + 1, " + "event_revision = event_revision + 1 WHERE state = 'queued' " + "AND last_failure_code = 'SUBJECT_NOT_AVAILABLE' " + "AND updated_at < ? AND row_revision < ? AND event_revision < ?", + (now, now, now - grace_seconds, MAX_REVISION, MAX_REVISION), + ) + if cursor.rowcount: + self._bump_stream(connection, "identification") + return cursor.rowcount + + return await self._write(operation) + async def pause_identification_queue( self, *, @@ -7566,7 +7758,9 @@ def operation(connection: sqlite3.Connection) -> dict[str, Any]: return await self._read(operation) - async def get_identification_activity_snapshot(self) -> dict[str, Any]: + async def get_identification_activity_snapshot( + self, *, now: float + ) -> dict[str, Any]: """Return redacted aggregate queue state for activity and admin progress UI.""" def operation(connection: sqlite3.Connection) -> dict[str, Any]: @@ -7585,8 +7779,11 @@ def operation(connection: sqlite3.Connection) -> dict[str, Any]: } active = connection.execute( "SELECT MIN(created_at) AS started_at, MAX(updated_at) AS updated_at, " - "SUM(CASE WHEN last_failure_code IS NOT NULL THEN 1 ELSE 0 END) AS deferred_count " - "FROM library_identification_jobs WHERE state IN ('queued','running','paused')" + "SUM(CASE WHEN last_failure_code IS NOT NULL THEN 1 ELSE 0 END) AS deferred_count, " + "SUM(CASE WHEN state IN ('queued','paused') " + "AND (not_before IS NULL OR not_before <= ?) THEN 1 ELSE 0 END) AS claimable_count " + "FROM library_identification_jobs WHERE state IN ('queued','running','paused')", + (now,), ).fetchone() active_priority = connection.execute( "SELECT priority FROM library_identification_jobs " @@ -7614,6 +7811,22 @@ def operation(connection: sqlite3.Connection) -> dict[str, Any]: "WHERE state IN ('queued','running','paused')" ).fetchone()[0] ) + attention_count = int( + connection.execute( + "SELECT COUNT(*) FROM library_identification_jobs " + "WHERE state = 'failed' AND last_failure_code IN " + "('MAX_DEFERRALS_EXCEEDED','SUBJECT_NOT_AVAILABLE')" + ).fetchone()[0] + ) + deferred_reason_counts = { + str(row["last_failure_code"]): int(row["count"]) + for row in connection.execute( + "SELECT last_failure_code, COUNT(*) AS count " + "FROM library_identification_jobs " + "WHERE state IN ('queued','running','paused') " + "AND last_failure_code IS NOT NULL GROUP BY last_failure_code" + ).fetchall() + } return { "control_state": str(control["state"]), "control_revision": int(control["row_revision"]), @@ -7621,6 +7834,9 @@ def operation(connection: sqlite3.Connection) -> dict[str, Any]: "started_at": active["started_at"], "updated_at": active["updated_at"], "deferred_count": int(active["deferred_count"] or 0), + "claimable_count": int(active["claimable_count"] or 0), + "deferred_reason_counts": deferred_reason_counts, + "attention_count": attention_count, "kept_local_count": kept_local_count, "active_priority": ( int(active_priority["priority"]) @@ -9053,6 +9269,13 @@ def operation( ) return None, 0, True run_id = str(row["id"]) + deleted_failures = connection.execute( + "DELETE FROM library_scan_failures WHERE rowid IN (" + "SELECT rowid FROM library_scan_failures WHERE run_id=? LIMIT ?)", + (run_id, max(1, limit)), + ).rowcount + if deleted_failures: + return run_id, deleted_failures, False for table in ( "library_scan_grouping_evidence", "library_scan_grouping_edges", @@ -9725,6 +9948,15 @@ def operation(connection: sqlite3.Connection) -> ScanRun: for root_id, path, failure_code in failures ], ) + connection.executemany( + "INSERT OR IGNORE INTO library_scan_failures " + "(run_id, root_id, relative_path, failure_code, failure_detail, " + "phase, recorded_at) VALUES (?, ?, ?, ?, '', 'indexing', ?)", + [ + (run_id, root_id, path, failure_code, updated_at) + for root_id, path, failure_code in failures + ], + ) if catalog_changed: self._bump_catalog(connection) allowed = { @@ -9765,6 +9997,69 @@ def operation(connection: sqlite3.Connection) -> ScanRun: return await self._write_scan(operation) + async def record_scan_failures( + self, run_id: str, failures: list[ScanFailureRecord] + ) -> None: + """Persist per-path scan failures; the PK dedupes repeated reports.""" + + if not failures: + return + + def operation(connection: sqlite3.Connection) -> None: + connection.executemany( + "INSERT OR IGNORE INTO library_scan_failures " + "(run_id, root_id, relative_path, failure_code, failure_detail, " + "phase, recorded_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + [ + ( + run_id, + failure.root_id, + failure.relative_path, + failure.failure_code, + failure.failure_detail, + failure.phase, + failure.recorded_at, + ) + for failure in failures + ], + ) + + return await super()._background_write(operation) + + async def list_scan_run_failures( + self, + run_id: str, + *, + limit: int = 50, + cursor_rowid: int | None = None, + ) -> tuple[list[ScanFailureRecord], int | None]: + """Read one rowid-keyset page of recorded scan failures for a run.""" + + def operation( + connection: sqlite3.Connection, + ) -> tuple[list[ScanFailureRecord], int | None]: + rows = connection.execute( + "SELECT rowid, root_id, relative_path, failure_code, failure_detail, " + "phase, recorded_at FROM library_scan_failures WHERE run_id = ? " + "AND (? IS NULL OR rowid > ?) ORDER BY rowid LIMIT ?", + (run_id, cursor_rowid, cursor_rowid, max(1, limit) + 1), + ).fetchall() + page = rows[: max(1, limit)] + next_cursor = int(page[-1]["rowid"]) if len(rows) > len(page) else None + return [ + ScanFailureRecord( + root_id=str(row["root_id"]), + relative_path=str(row["relative_path"]), + failure_code=str(row["failure_code"]), + recorded_at=float(row["recorded_at"]), + failure_detail=str(row["failure_detail"]), + phase=row["phase"], + ) + for row in page + ], next_cursor + + return await self._read(operation) + async def upsert_scanned_track( self, *, @@ -16061,6 +16356,7 @@ def operation(connection: sqlite3.Connection) -> dict[str, Any]: "detach_keep_tagged": "keep_tagged", "exclude": "excluded", "restore": "resolved", + "dismiss": "resolved", }.get(action, action) if action == "exclude": if album_id is not None: @@ -16096,6 +16392,18 @@ def operation(connection: sqlite3.Connection) -> dict[str, Any]: "((local_album_id = ? AND ? IS NOT NULL) OR (local_track_id = ? AND ? IS NOT NULL))", (now, now, album_id, album_id, track_id, track_id), ) + if action == "dismiss": + # Dismissed attention must not keep the failed job counting in + # attention_count: the review is resolved and nothing else would + # clear the markers for a never-resurrecting cap. + connection.execute( + "UPDATE library_identification_jobs SET last_failure_code = NULL, " + "attention_cause = NULL, updated_at = ?, " + "row_revision = row_revision + 1, event_revision = event_revision + 1 " + "WHERE state = 'failed' AND input_revision = ? AND " + "((local_album_id = ? AND ? IS NOT NULL) OR (local_track_id = ? AND ? IS NOT NULL))", + (now, review["input_revision"], album_id, album_id, track_id, track_id), + ) updated = connection.execute( "UPDATE library_identification_reviews SET state = ?, reason_code = ?, " "decided_by_user_id = ?, decided_at = ?, updated_at = ?, " @@ -25410,8 +25718,11 @@ def operation(connection: sqlite3.Connection) -> dict[str, Any]: raise ResourceNotFoundError("Re-identification job not found.") if ( job["state"] != "ready" - or int(job["row_revision"]) != expected_job_revision - ): + and not ( + job["state"] == "succeeded" + and decision_mode == "leave_unmanaged" + ) + ) or int(job["row_revision"]) != expected_job_revision: raise StaleRevisionError( "The re-identification candidates changed before selection." ) @@ -25463,18 +25774,20 @@ def operation(connection: sqlite3.Connection) -> dict[str, Any]: ) result = json.loads(str(snapshot["result_json"])) attempt_id = str(result["attempt_id"]) - evidence_row = connection.execute( - "SELECT * FROM library_identification_evidence WHERE attempt_id = ? " - "AND candidate_key = ?", - (attempt_id, candidate_key), - ).fetchone() - if evidence_row is None: - raise StaleRevisionError( - "The selected candidate is no longer available." + evidence = None + if candidate_key != "": + evidence_row = connection.execute( + "SELECT * FROM library_identification_evidence WHERE attempt_id = ? " + "AND candidate_key = ?", + (attempt_id, candidate_key), + ).fetchone() + if evidence_row is None: + raise StaleRevisionError( + "The selected candidate is no longer available." + ) + evidence = msgspec.json.decode( + bytes(evidence_row["evidence_json"]), type=CandidateEvidence ) - evidence = msgspec.json.decode( - bytes(evidence_row["evidence_json"]), type=CandidateEvidence - ) if decision_mode == "leave_unmanaged": before_identity = ( dict(current_album_identity) @@ -25490,7 +25803,8 @@ def operation(connection: sqlite3.Connection) -> dict[str, Any]: (actor_user_id, now, snapshot["local_album_id"]), ) elif ( - is_valid_mbid(evidence.release_group_mbid) + evidence is not None + and is_valid_mbid(evidence.release_group_mbid) and evidence.album_title_classification != "contradictory" and evidence.album_artist_classification != "contradictory" ): @@ -25562,6 +25876,10 @@ def operation(connection: sqlite3.Connection) -> dict[str, Any]: before=before_identity, now=now, ) + if evidence is None: + raise StaleRevisionError( + "The selected candidate is no longer available." + ) if decision_mode == "custom_edition": if not confirmation: raise CustomEditionNotSealableError( @@ -26051,7 +26369,7 @@ def _finish_reidentification_management_decision_tx( updated = connection.execute( "UPDATE library_operation_jobs SET state='succeeded',terminal_code=?," "terminal_at=?,updated_at=?,row_revision=row_revision+1," - "event_revision=event_revision+1 WHERE id=? AND state='ready' " + "event_revision=event_revision+1 WHERE id=? AND state IN ('ready','succeeded') " "AND row_revision=? RETURNING *", (terminal_code, now, now, job_id, expected_job_revision), ).fetchone() @@ -27166,7 +27484,9 @@ def operation(connection: sqlite3.Connection) -> None: "INSERT INTO library_identity_repair_findings " "(id, job_id, local_album_id, evidence_id, expected_album_revision, " "expected_identity_revision, finding_code, confidence, reason_code, " - "apply_eligible, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + "apply_eligible, suggested_release_mbid, suggested_release_group_mbid, " + "suggested_edition_json, created_at, updated_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", [ ( finding.id, @@ -27179,6 +27499,9 @@ def operation(connection: sqlite3.Connection) -> None: finding.confidence, finding.reason_code, int(finding.apply_eligible), + finding.suggested_release_mbid, + finding.suggested_release_group_mbid, + finding.suggested_edition_json, updated_at, updated_at, ) @@ -27188,6 +27511,33 @@ def operation(connection: sqlite3.Connection) -> None: await self._write(operation) + async def get_latest_album_identification_evidence( + self, local_album_id: str + ) -> tuple[dict[str, Any], list[dict[str, Any]]] | None: + """Latest attempt (any trigger) that still has uncompacted evidence.""" + + def operation( + connection: sqlite3.Connection, + ) -> tuple[dict[str, Any], list[dict[str, Any]]] | None: + attempt = connection.execute( + "SELECT a.* FROM library_identification_attempts a " + "WHERE a.local_album_id = ? AND EXISTS (" + "SELECT 1 FROM library_identification_evidence e " + "WHERE e.attempt_id = a.id AND e.compacted = 0) " + "ORDER BY a.completed_at DESC, a.id DESC LIMIT 1", + (local_album_id,), + ).fetchone() + if attempt is None: + return None + rows = connection.execute( + "SELECT * FROM library_identification_evidence " + "WHERE attempt_id = ? AND compacted = 0 ORDER BY candidate_key", + (attempt["id"],), + ).fetchall() + return dict(attempt), [dict(row) for row in rows] + + return await self._read(operation) + async def create_repair_operation( self, job: OperationJob, @@ -27469,6 +27819,57 @@ def operation(connection: sqlite3.Connection) -> dict[str, int]: return await self._read(operation) + async def defer_repair_audit_work( + self, + *, + job_id: str, + ordinal: int | None, + worker_id: str, + reason_code: str, + now: float, + retry_not_before: float | None = None, + ) -> dict[str, Any]: + """Release a running repair audit back to queued for a later retry. + + The running work item (when one is claimed) is returned to 'pending' + with the failure code; the whole job is queued with next_attempt_at so + the audit resumes exactly at the deferred item. Nothing is marked + succeeded and no finding row is written. + """ + + def operation(connection: sqlite3.Connection) -> dict[str, Any]: + if ordinal is not None: + work_update = connection.execute( + "UPDATE library_operation_work SET state = 'pending', failure_code = ?, " + "updated_at = ?, row_revision = row_revision + 1 " + "WHERE job_id = ? AND ordinal = ? AND state = 'running'", + (reason_code, now, job_id, ordinal), + ) + if work_update.rowcount != 1: + raise StaleRevisionError( + "The repair audit work lease changed while it was deferred." + ) + updated = connection.execute( + "UPDATE library_operation_jobs SET state = 'queued', lease_owner = NULL, " + "lease_expires_at = NULL, heartbeat_at = NULL, updated_at = ?, " + "next_attempt_at = ?, " + "row_revision = row_revision + 1, event_revision = event_revision + 1 " + "WHERE id = ? AND kind = 'repair' AND state = 'running' " + "AND lease_owner = ? RETURNING *", + (now, retry_not_before, job_id, worker_id), + ).fetchone() + if updated is None: + raise StaleRevisionError( + "The repair audit lease changed while it was deferred." + ) + self._bump_stream(connection, "operation") + return dict(updated) + + result = await self._write(operation) + if retry_not_before is not None: + self.work_wakeups.notify_after("operation", retry_not_before - now) + return result + async def save_repair_finding_for_work( self, job_id: str, @@ -27525,7 +27926,8 @@ def operation(connection: sqlite3.Connection) -> None: "INSERT INTO library_identity_repair_findings " "(id, job_id, local_album_id, evidence_id, expected_album_revision, " "expected_identity_revision, finding_code, confidence, reason_code, apply_eligible, " - "created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + "suggested_release_mbid, suggested_release_group_mbid, suggested_edition_json, " + "created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", ( finding.id, job_id, @@ -27537,6 +27939,9 @@ def operation(connection: sqlite3.Connection) -> None: finding.confidence, finding.reason_code, int(finding.apply_eligible), + finding.suggested_release_mbid, + finding.suggested_release_group_mbid, + finding.suggested_edition_json, now, now, ), @@ -27817,7 +28222,7 @@ def operation(connection: sqlite3.Connection) -> dict[str, Any]: if management_readiness: evidence_row = ( connection.execute( - "SELECT e.evidence_json, e.attempt_id, " + "SELECT e.evidence_json, e.attempt_id, e.compacted, " "a.input_tag_revision, a.input_file_revision, " "a.input_policy_revision FROM library_identification_evidence e " "JOIN library_identification_attempts a ON a.id = e.attempt_id " @@ -27845,164 +28250,181 @@ def operation(connection: sqlite3.Connection) -> dict[str, Any]: if evidence_row is not None else None ) - current_revisions = tuple(_album_input_revision(track_rows).split(":")) - proposed = { - item.local_track_id: item - for item in (evidence.track_evidence if evidence else []) - if item.classification == "supported" - } - stale = bool( - finding is None - or album is None - or identity is None - or evidence_row is None - or evidence is None - or int(album["row_revision"]) - != int(work["expected_subject_revision"]) - or int(identity["row_revision"]) - != int(finding["expected_identity_revision"]) - or str(identity["release_group_mbid"]) - != evidence.release_group_mbid - or str(identity["release_mbid"] or "") - != str(evidence.release_mbid or "") - or current_revisions - != ( - str(evidence_row["input_tag_revision"]), - str(evidence_row["input_file_revision"]), - str(evidence_row["input_policy_revision"]), - ) - or set(proposed) != {str(row["id"]) for row in track_rows} - ) - release_track_ids: set[str] = set() - if not stale: - for row in track_rows: - item = proposed[str(row["id"])] - if ( - not item.recording_mbid - or not item.release_track_mbid - or item.candidate_disc_number is None - or item.candidate_track_position is None - or item.release_track_mbid in release_track_ids - or not _recording_evidence_matches( - row["recording_mbid"], item - ) - or ( - row["identity_release_mbid"] - and row["identity_release_mbid"] - != evidence.release_mbid - ) - or ( - row["release_track_mbid"] - and row["release_track_mbid"] != item.release_track_mbid - ) - or ( - row["embedded_release_group_mbid"] - and row["embedded_release_group_mbid"] - != evidence.release_group_mbid - ) - or ( - row["embedded_release_mbid"] - and row["embedded_release_mbid"] - != evidence.release_mbid - ) - or not _recording_evidence_matches( - row["embedded_recording_mbid"], item + if finding is not None and finding["finding_code"] == ( + "exact_release_suggested" + ): + state, failure_code = self._apply_suggested_edition_tx( + connection, + work=work, + finding=finding, + album=album, + identity=identity, + evidence_row=evidence_row, + track_rows=track_rows, + evidence=evidence, + job_id=job_id, + actor_user_id=actor_user_id, + now=now, + ) + else: + current_revisions = tuple(_album_input_revision(track_rows).split(":")) + proposed = { + item.local_track_id: item + for item in (evidence.track_evidence if evidence else []) + if item.classification == "supported" + } + stale = bool( + finding is None + or album is None + or identity is None + or evidence_row is None + or evidence is None + or int(album["row_revision"]) + != int(work["expected_subject_revision"]) + or int(identity["row_revision"]) + != int(finding["expected_identity_revision"]) + or str(identity["release_group_mbid"]) + != evidence.release_group_mbid + or str(identity["release_mbid"] or "") + != str(evidence.release_mbid or "") + or current_revisions + != ( + str(evidence_row["input_tag_revision"]), + str(evidence_row["input_file_revision"]), + str(evidence_row["input_policy_revision"]), + ) + or set(proposed) != {str(row["id"]) for row in track_rows} + ) + release_track_ids: set[str] = set() + if not stale: + for row in track_rows: + item = proposed[str(row["id"])] + if ( + not item.recording_mbid + or not item.release_track_mbid + or item.candidate_disc_number is None + or item.candidate_track_position is None + or item.release_track_mbid in release_track_ids + or not _recording_evidence_matches( + row["recording_mbid"], item + ) + or ( + row["identity_release_mbid"] + and row["identity_release_mbid"] + != evidence.release_mbid + ) + or ( + row["release_track_mbid"] + and row["release_track_mbid"] != item.release_track_mbid + ) + or ( + row["embedded_release_group_mbid"] + and row["embedded_release_group_mbid"] + != evidence.release_group_mbid + ) + or ( + row["embedded_release_mbid"] + and row["embedded_release_mbid"] + != evidence.release_mbid + ) + or not _recording_evidence_matches( + row["embedded_recording_mbid"], item + ) + or ( + row["embedded_release_track_mbid"] + and row["embedded_release_track_mbid"] + != item.release_track_mbid + ) + ): + stale = True + break + release_track_ids.add(item.release_track_mbid) + if stale: + state = "skipped" + failure_code = "STALE_SUBJECT" + if finding is not None: + connection.execute( + "UPDATE library_identity_repair_findings SET finding_code = 'stale', " + "state = 'stale', " + "apply_result = 'STALE_SUBJECT', updated_at = ?, " + "row_revision = row_revision + 1 WHERE id = ?", + (now, finding["id"]), ) - or ( - row["embedded_release_track_mbid"] - and row["embedded_release_track_mbid"] - != item.release_track_mbid + else: + assert finding is not None + assert evidence is not None + assert evidence_row is not None + before = [ + { + "local_track_id": str(row["id"]), + "recording_mbid": row["recording_mbid"], + "release_mbid": row["identity_release_mbid"], + "release_track_mbid": row["release_track_mbid"], + } + for row in track_rows + ] + for local_track_id, item in proposed.items(): + connection.execute( + "INSERT INTO local_track_external_identities " + "(local_track_id, provider, recording_mbid, release_mbid, " + "release_track_mbid, medium_position, release_track_position, " + "decision_source, attempt_id, selected_at) " + "VALUES (?, 'musicbrainz', ?, ?, ?, ?, ?, 'manual', ?, ?) " + "ON CONFLICT(local_track_id, provider) DO UPDATE SET " + "recording_mbid = excluded.recording_mbid, " + "release_mbid = excluded.release_mbid, " + "release_track_mbid = excluded.release_track_mbid, " + "medium_position = excluded.medium_position, " + "release_track_position = excluded.release_track_position, " + "decision_source = 'manual', attempt_id = excluded.attempt_id, " + "selected_at = excluded.selected_at, " + "row_revision = row_revision + 1", + ( + local_track_id, + item.recording_mbid, + evidence.release_mbid, + item.release_track_mbid, + item.candidate_disc_number, + item.candidate_track_position, + evidence_row["attempt_id"], + now, + ), ) - ): - stale = True - break - release_track_ids.add(item.release_track_mbid) - if stale: - state = "skipped" - failure_code = "STALE_SUBJECT" - if finding is not None: - connection.execute( - "UPDATE library_identity_repair_findings SET finding_code = 'stale', " - "state = 'stale', " - "apply_result = 'STALE_SUBJECT', updated_at = ?, " - "row_revision = row_revision + 1 WHERE id = ?", - (now, finding["id"]), - ) - else: - assert finding is not None - assert evidence is not None - assert evidence_row is not None - before = [ - { - "local_track_id": str(row["id"]), - "recording_mbid": row["recording_mbid"], - "release_mbid": row["identity_release_mbid"], - "release_track_mbid": row["release_track_mbid"], - } - for row in track_rows - ] - for local_track_id, item in proposed.items(): + after = [ + { + "local_track_id": local_track_id, + "recording_mbid": item.recording_mbid, + "release_mbid": evidence.release_mbid, + "release_track_mbid": item.release_track_mbid, + "medium_position": item.candidate_disc_number, + "release_track_position": item.candidate_track_position, + } + for local_track_id, item in sorted(proposed.items()) + ] connection.execute( - "INSERT INTO local_track_external_identities " - "(local_track_id, provider, recording_mbid, release_mbid, " - "release_track_mbid, medium_position, release_track_position, " - "decision_source, attempt_id, selected_at) " - "VALUES (?, 'musicbrainz', ?, ?, ?, ?, ?, 'manual', ?, ?) " - "ON CONFLICT(local_track_id, provider) DO UPDATE SET " - "recording_mbid = excluded.recording_mbid, " - "release_mbid = excluded.release_mbid, " - "release_track_mbid = excluded.release_track_mbid, " - "medium_position = excluded.medium_position, " - "release_track_position = excluded.release_track_position, " - "decision_source = 'manual', attempt_id = excluded.attempt_id, " - "selected_at = excluded.selected_at, " - "row_revision = row_revision + 1", + "INSERT INTO library_catalog_actions " + "(id, actor_user_id, action_kind, local_album_id, " + "operation_job_id, before_json, after_json, reason_code, " + "created_at) VALUES (?,?,?,?,?,?,?,?,?)", ( - local_track_id, - item.recording_mbid, - evidence.release_mbid, - item.release_track_mbid, - item.candidate_disc_number, - item.candidate_track_position, - evidence_row["attempt_id"], + str(uuid.uuid4()), + actor_user_id, + "accept_management_track_mappings", + work["local_album_id"], + job_id, + json.dumps(before, sort_keys=True), + json.dumps(after, sort_keys=True), + "EXACT_RELEASE_MAPPINGS_ACCEPTED", now, ), ) - after = [ - { - "local_track_id": local_track_id, - "recording_mbid": item.recording_mbid, - "release_mbid": evidence.release_mbid, - "release_track_mbid": item.release_track_mbid, - "medium_position": item.candidate_disc_number, - "release_track_position": item.candidate_track_position, - } - for local_track_id, item in sorted(proposed.items()) - ] - connection.execute( - "INSERT INTO library_catalog_actions " - "(id, actor_user_id, action_kind, local_album_id, " - "operation_job_id, before_json, after_json, reason_code, " - "created_at) VALUES (?,?,?,?,?,?,?,?,?)", - ( - str(uuid.uuid4()), - actor_user_id, - "accept_management_track_mappings", - work["local_album_id"], - job_id, - json.dumps(before, sort_keys=True), - json.dumps(after, sort_keys=True), - "EXACT_RELEASE_MAPPINGS_ACCEPTED", - now, - ), - ) - connection.execute( - "UPDATE library_identity_repair_findings SET state = 'applied', " - "apply_result = 'MAPPINGS_ACCEPTED', updated_at = ?, " - "row_revision = row_revision + 1 WHERE id = ?", - (now, finding["id"]), - ) - self._bump_catalog(connection) + connection.execute( + "UPDATE library_identity_repair_findings SET state = 'applied', " + "apply_result = 'MAPPINGS_ACCEPTED', updated_at = ?, " + "row_revision = row_revision + 1 WHERE id = ?", + (now, finding["id"]), + ) + self._bump_catalog(connection) elif ( finding is None or album is None @@ -28139,6 +28561,207 @@ def operation(connection: sqlite3.Connection) -> dict[str, Any]: return await self._write(operation) + def _apply_suggested_edition_tx( + self, + connection: sqlite3.Connection, + *, + work: sqlite3.Row, + finding: sqlite3.Row, + album: sqlite3.Row | None, + identity: sqlite3.Row | None, + evidence_row: sqlite3.Row | None, + track_rows: list[sqlite3.Row], + evidence: CandidateEvidence | None, + job_id: str, + actor_user_id: str, + now: float, + ) -> tuple[str, str | None]: + """Seal one accepted suggested edition; mirrors manual candidate accept.""" + mapped = ( + _complete_track_identity_mapping(track_rows, evidence) + if evidence is not None + else None + ) + stale = bool( + album is None + or evidence_row is None + or evidence is None + or mapped is None + or int(evidence_row["compacted"]) + or int(album["row_revision"]) != int(work["expected_subject_revision"]) + or tuple(_album_input_revision(track_rows).split(":")) + != ( + str(evidence_row["input_tag_revision"]), + str(evidence_row["input_file_revision"]), + str(evidence_row["input_policy_revision"]), + ) + or (identity is None) != (finding["expected_identity_revision"] is None) + or ( + identity is not None + and ( + int(identity["row_revision"]) + != int(finding["expected_identity_revision"]) + or identity["release_mbid"] is not None + ) + ) + or evidence.reason_code not in AUTOMATIC_SAFE_EVIDENCE_REASONS + ) + if not stale: + assert evidence is not None + assert mapped is not None + for row, item in zip(track_rows, mapped, strict=True): + if ( + not _recording_evidence_matches(row["recording_mbid"], item) + or ( + row["identity_release_mbid"] + and row["identity_release_mbid"] != evidence.release_mbid + ) + or ( + row["release_track_mbid"] + and row["release_track_mbid"] != item.release_track_mbid + ) + or ( + row["embedded_release_group_mbid"] + and row["embedded_release_group_mbid"] + != evidence.release_group_mbid + ) + or ( + row["embedded_release_mbid"] + and row["embedded_release_mbid"] != evidence.release_mbid + ) + or not _recording_evidence_matches( + row["embedded_recording_mbid"], item + ) + or ( + row["embedded_release_track_mbid"] + and row["embedded_release_track_mbid"] + != item.release_track_mbid + ) + ): + stale = True + break + if stale: + connection.execute( + "UPDATE library_identity_repair_findings SET finding_code = 'stale', " + "state = 'stale', apply_result = 'STALE_SUBJECT', updated_at = ?, " + "row_revision = row_revision + 1 WHERE id = ?", + (now, finding["id"]), + ) + return "skipped", "STALE_SUBJECT" + assert evidence is not None + assert evidence_row is not None + assert mapped is not None + before = ( + { + "release_group_mbid": identity["release_group_mbid"], + "release_mbid": identity["release_mbid"], + } + if identity is not None + else {} + ) + connection.execute( + "INSERT INTO local_album_external_identities " + "(local_album_id, provider, release_group_mbid, release_mbid, " + "decision_source, matcher_version, attempt_id, selected_by_user_id, " + "selected_at) " + "VALUES (?, 'musicbrainz', ?, ?, 'manual', ?, ?, ?, ?) " + "ON CONFLICT(local_album_id, provider) DO UPDATE SET " + "release_group_mbid = excluded.release_group_mbid, " + "release_mbid = excluded.release_mbid, " + "decision_source = 'manual', matcher_version = excluded.matcher_version, " + "attempt_id = excluded.attempt_id, " + "selected_by_user_id = excluded.selected_by_user_id, " + "selected_at = excluded.selected_at, row_revision = row_revision + 1", + ( + work["local_album_id"], + evidence.release_group_mbid, + evidence.release_mbid, + evidence.matcher_version, + evidence_row["attempt_id"], + actor_user_id, + now, + ), + ) + connection.execute( + "DELETE FROM local_track_external_identities WHERE local_track_id IN " + "(SELECT id FROM local_tracks WHERE local_album_id = ? " + "AND availability = 'indexed')", + (work["local_album_id"],), + ) + for item in mapped: + connection.execute( + "INSERT INTO local_track_external_identities " + "(local_track_id, provider, recording_mbid, release_mbid, " + "release_track_mbid, medium_position, release_track_position, " + "decision_source, attempt_id, selected_at) " + "VALUES (?, 'musicbrainz', ?, ?, ?, ?, ?, 'manual', ?, ?)", + ( + item.local_track_id, + item.recording_mbid, + evidence.release_mbid, + item.release_track_mbid, + item.candidate_disc_number, + item.candidate_track_position, + evidence_row["attempt_id"], + now, + ), + ) + connection.execute( + "DELETE FROM library_custom_edition_active WHERE local_album_id = ?", + (work["local_album_id"],), + ) + connection.execute( + "DELETE FROM library_management_exclusions WHERE local_album_id = ?", + (work["local_album_id"],), + ) + connection.execute( + "UPDATE library_identification_reviews SET state = 'resolved', " + "reason_code = 'SUGGESTED_EDITION_ACCEPTED', decided_by_user_id = ?, " + "decided_at = ?, updated_at = ?, decision_revision = decision_revision + 1, " + "row_revision = row_revision + 1 WHERE local_album_id = ? " + "AND state != 'resolved'", + (actor_user_id, now, now, work["local_album_id"]), + ) + after = { + "release_group_mbid": evidence.release_group_mbid, + "release_mbid": evidence.release_mbid, + "tracks": [ + { + "local_track_id": item.local_track_id, + "recording_mbid": item.recording_mbid, + "release_track_mbid": item.release_track_mbid, + "medium_position": item.candidate_disc_number, + "release_track_position": item.candidate_track_position, + } + for item in mapped + ], + } + connection.execute( + "INSERT INTO library_catalog_actions " + "(id, actor_user_id, action_kind, local_album_id, operation_job_id, " + "before_json, after_json, reason_code, created_at) " + "VALUES (?,?,?,?,?,?,?,?,?)", + ( + str(uuid.uuid4()), + actor_user_id, + "accept_suggested_edition", + work["local_album_id"], + job_id, + json.dumps(before, sort_keys=True), + json.dumps(after, sort_keys=True), + "SUGGESTED_EDITION_ACCEPTED", + now, + ), + ) + connection.execute( + "UPDATE library_identity_repair_findings SET state = 'applied', " + "apply_result = 'EDITION_ACCEPTED', updated_at = ?, " + "row_revision = row_revision + 1 WHERE id = ?", + (now, finding["id"]), + ) + self._bump_catalog(connection) + return "succeeded", None + async def list_repair_findings( self, job_id: str, diff --git a/backend/maintenance/automatic_upgrade.py b/backend/maintenance/automatic_upgrade.py index 53c573710..5263920f9 100644 --- a/backend/maintenance/automatic_upgrade.py +++ b/backend/maintenance/automatic_upgrade.py @@ -30,6 +30,8 @@ UPGRADE_ID = "feedback-fixes-v1" MIGRATION_ID = "automatic-feedback-fixes-v1" +_PUBLISH_VERIFY_ATTEMPTS = 10 +_PUBLISH_VERIFY_INTERVAL_SECONDS = 0.25 _MARKER = "legacy_catalog_import_complete" _SOURCE_REVISION_PATH = Path("/app/.droppedneedle-source-revision") _ADMISSION_TOKEN_ENV = "DROPPEDNEEDLE_TARGET_ADMISSION_TOKEN" @@ -109,6 +111,15 @@ def _sha256(path: Path) -> str | None: return digest.hexdigest() +def _wait_for_content(path: Path, expected_sha256: str) -> bool: + for attempt in range(_PUBLISH_VERIFY_ATTEMPTS): + if _sha256(path) == expected_sha256: + return True + if attempt + 1 < _PUBLISH_VERIFY_ATTEMPTS: + time.sleep(_PUBLISH_VERIFY_INTERVAL_SECONDS) + return False + + def _database_has_marker(database: Path) -> bool: if not database.is_file(): return False @@ -145,10 +156,38 @@ def _sqlite_backup(source: Path, destination: Path) -> None: def _write_state(path: Path, payload: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) - atomic_write_json(path, payload) + try: + atomic_write_json(path, payload) + except OSError: + logger.warning("automatic_upgrade.state_rename_failed_using_direct_write") + _write_state_direct(path, payload) with path.open("rb") as handle: os.fsync(handle.fileno()) _fsync_directory(path.parent) + if _wait_for_state(path, payload): + return + logger.warning("automatic_upgrade.state_write_result_stale_using_direct_write") + _write_state_direct(path, payload) + with path.open("rb") as handle: + os.fsync(handle.fileno()) + _fsync_directory(path.parent) + if not _wait_for_state(path, payload): + raise OSError( + "The upgrade state file could not be verified after writing: " + str(path) + ) + + +def _write_state_direct(path: Path, payload: dict[str, Any]) -> None: + path.write_text(json.dumps(payload), encoding="utf-8") + + +def _wait_for_state(path: Path, payload: dict[str, Any]) -> bool: + for attempt in range(_PUBLISH_VERIFY_ATTEMPTS): + if _read_state(path) == payload: + return True + if attempt + 1 < _PUBLISH_VERIFY_ATTEMPTS: + time.sleep(_PUBLISH_VERIFY_INTERVAL_SECONDS) + return False def _read_state(path: Path) -> dict[str, Any] | None: @@ -301,6 +340,15 @@ def _fsync_directory(directory: Path) -> None: os.close(descriptor) +def _copy_file_in_place(source: Path, destination: Path) -> None: + with source.open("rb") as source_handle, destination.open("wb") as target_handle: + shutil.copyfileobj(source_handle, target_handle, length=1024 * 1024) + target_handle.flush() + os.fsync(target_handle.fileno()) + shutil.copystat(source, destination) + _fsync_directory(destination.parent) + + def _replace_file(source: Path, destination: Path) -> None: destination.parent.mkdir(parents=True, exist_ok=True) temporary = destination.with_name( @@ -312,8 +360,26 @@ def _replace_file(source: Path, destination: Path) -> None: target_handle.flush() os.fsync(target_handle.fileno()) shutil.copystat(source, temporary) - os.replace(temporary, destination) - _fsync_directory(destination.parent) + expected = _sha256(temporary) + try: + os.replace(temporary, destination) + except OSError: + logger.warning("automatic_upgrade.file_rename_failed_using_copy_fallback") + renamed = False + else: + _fsync_directory(destination.parent) + renamed = _wait_for_content(destination, expected) + if not renamed: + logger.warning( + "automatic_upgrade.file_rename_result_stale_using_copy_fallback" + ) + if not renamed: + _copy_file_in_place(source, destination) + if not _wait_for_content(destination, expected): + raise OSError( + "The upgraded file could not be verified after installation: " + + str(destination) + ) finally: temporary.unlink(missing_ok=True) @@ -327,10 +393,35 @@ def _replace_database(source: Path, destination: Path) -> None: _sqlite_backup(source, temporary) with temporary.open("rb") as handle: os.fsync(handle.fileno()) + expected = _sha256(temporary) for suffix in ("-wal", "-shm"): Path(f"{destination}{suffix}").unlink(missing_ok=True) - os.replace(temporary, destination) - _fsync_directory(destination.parent) + try: + os.replace(temporary, destination) + except OSError: + logger.warning("automatic_upgrade.file_rename_failed_using_copy_fallback") + renamed = False + else: + _fsync_directory(destination.parent) + renamed = _wait_for_content(destination, expected) + if not renamed: + logger.warning( + "automatic_upgrade.file_rename_result_stale_using_copy_fallback" + ) + if not renamed: + # truncate first: a backup onto an existing database rewrites header + # counters, so it would never hash-match the temporary + with destination.open("wb") as destination_handle: + destination_handle.truncate(0) + destination_handle.flush() + os.fsync(destination_handle.fileno()) + _sqlite_backup(source, destination) + _fsync_directory(destination.parent) + if not _wait_for_content(destination, expected): + raise OSError( + "The upgraded library database could not be verified " + "after installation." + ) finally: temporary.unlink(missing_ok=True) @@ -749,6 +840,7 @@ def run_automatic_copy_upgrade( "image_version": image_version, "backup_directory": str(backup.directory), "error_type": type(error).__name__, + "error_message": str(error), "restored_signature": _current_signature(database, config), } failure_evidence = getattr(error, "evidence", None) @@ -760,8 +852,9 @@ def run_automatic_copy_upgrade( except OSError: logger.error("automatic_upgrade.failure_state_write_failed") logger.error( - "automatic_upgrade.failed error_type=%s", + "automatic_upgrade.failed error_type=%s error_message=%s", type(error).__name__, + str(error), ) raise AutomaticUpgradeError( "The library upgrade could not be completed. Your previous database and " diff --git a/backend/models/library_work.py b/backend/models/library_work.py index 88c316a9e..aebb0c45c 100644 --- a/backend/models/library_work.py +++ b/backend/models/library_work.py @@ -207,6 +207,9 @@ class RepairFinding(AppStruct): expected_identity_revision: int | None = None reason_code: str = "" apply_eligible: bool = False + suggested_release_mbid: str | None = None + suggested_release_group_mbid: str | None = None + suggested_edition_json: str = "{}" class ScanInventoryItem(AppStruct): @@ -225,6 +228,15 @@ class ScanInventoryItem(AppStruct): scope_relative_path: str = "." +class ScanFailureRecord(AppStruct): + root_id: str + relative_path: str + failure_code: str + recorded_at: float + failure_detail: str = "" + phase: Literal["discovering", "indexing", "reconciling"] = "discovering" + + class ScannedTrackWrite(msgspec.Struct): artist: LocalArtist album: LocalAlbum diff --git a/backend/repositories/coverart_repository.py b/backend/repositories/coverart_repository.py index cec023304..d7c60e5e4 100644 --- a/backend/repositories/coverart_repository.py +++ b/backend/repositories/coverart_repository.py @@ -5,7 +5,7 @@ from datetime import datetime, timezone from email.utils import parsedate_to_datetime from pathlib import Path -from typing import Literal, Optional, TYPE_CHECKING +from typing import Callable, Literal, Optional, TYPE_CHECKING from urllib.parse import urlparse, urlunparse import aiofiles @@ -35,6 +35,7 @@ from repositories.coverart_disk_cache import CoverDiskCache from infrastructure.degradation import try_get_degradation_context from infrastructure.integration_result import IntegrationResult +from infrastructure.service_health import report_breaker_health from models.library_management_artwork import ArtworkCandidate, ArtworkImageType from repositories.coverart_management_models import CaaManagementResponse @@ -45,6 +46,7 @@ from services.audiodb_image_service import AudioDBImageService from services.audiodb_browse_queue import AudioDBBrowseQueue from infrastructure.persistence.library_db import LibraryDB + from infrastructure.persistence.native_library_store import NativeLibraryStore logger = logging.getLogger(__name__) @@ -78,8 +80,16 @@ def _sniff_image_content_type(data: bytes) -> Optional[str]: COVER_ART_ARCHIVE_BASE = "https://coverartarchive.org" COVER_NEGATIVE_TTL_SECONDS = 4 * 3600 +# Short marker for upstream-outage failures (breaker open / transient fetch error): +# repeats serve the placeholder immediately instead of re-paying the fetch chain, +# and recovered art reappears within minutes - unlike the 4h authoritative negative. +COVER_TRANSIENT_NEGATIVE_TTL_SECONDS = 900 COVER_MEMORY_MAX_ENTRIES = 128 COVER_MEMORY_MAX_BYTES = 16 * 1024 * 1024 +# Folder-art probe names, most deliberate first; matched case-insensitively. +_LOCAL_COVER_STEMS = ("cover", "folder", "front", "album", "artwork") +_LOCAL_COVER_EXTENSIONS = (".jpg", ".jpeg", ".png", ".webp") +_LOCAL_COVER_MAX_BYTES = 25 * 1024 * 1024 MANAGEMENT_ARTWORK_METADATA_MAX_BYTES = 5 * 1024 * 1024 MANAGEMENT_ARTWORK_CACHE_TTL_SECONDS = 3600 @@ -91,7 +101,15 @@ def _default_cache_dir() -> Path: _coverart_circuit_breaker = CircuitBreaker( - failure_threshold=5, success_threshold=2, timeout=60.0, name="coverart" + failure_threshold=5, + success_threshold=2, + timeout=60.0, + name="coverart", + on_state_change=report_breaker_health( + "coverartarchive", + "cover art", + message="Cover Art Archive is temporarily unavailable.", + ), ) _library_cover_circuit_breaker = CircuitBreaker( @@ -226,6 +244,8 @@ def __init__( cover_memory_cache_max_bytes: int = COVER_MEMORY_MAX_BYTES, cover_non_monitored_ttl_seconds: int = 604800, # 7 days; non-monitored covers change rarely library_db: Optional["LibraryDB"] = None, + local_cover_priority: Optional[Callable[[], bool]] = None, + native_library_store: Optional["NativeLibraryStore"] = None, ): self._client = http_client self._cache = cache @@ -233,6 +253,8 @@ def __init__( self._library_repo = library_repo self._jellyfin_repo = jellyfin_repo self._library_db = library_db + self._native_library_store = native_library_store + self._local_cover_priority = local_cover_priority self._tagger = AudioTagger() self.cache_dir = cache_dir self.cache_dir.mkdir(parents=True, exist_ok=True) @@ -863,8 +885,13 @@ async def get_artist_image( ExternalServiceError, RateLimitedError, ) as e: - # Transient failure: fail soft WITHOUT caching a negative (the artist may well have - # an image; it was just a blip) and without deferring - the next request retries. + # Transient failure: bank a SHORT negative (15 min) so a sustained upstream outage + # doesn't re-pay the full fetch chain on every request/poll; recovered art + # reappears on the first request after expiry. Authoritative no-art still uses + # the 4h negative below. + await self._disk_cache.write_negative( + file_path, ttl_seconds=COVER_TRANSIENT_NEGATIVE_TTL_SECONDS + ) _record_degradation(f"Artist image fetch failed for {artist_id[:8]}: {e}") return None @@ -923,9 +950,8 @@ async def get_release_group_cover( identifier = f"rg_{release_group_id}" suffix = size or "orig" file_path = self._disk_cache.get_file_path(identifier, suffix) - if cached_memory := await self._memory_get(identifier, suffix): - if cached_memory[2] in {"audiodb", "legacy-cache"}: + if self._is_terminal_cover_source(cached_memory[2]): return cached_memory preferred = await self._prefer_audiodb_album_cover( release_group_id, @@ -945,7 +971,7 @@ async def get_release_group_cover( source = cached[2].get("source") or source result = (cached[0], cached[1], source) await self._memory_set_from_result(identifier, suffix, result) - if source in {"audiodb", "legacy-cache"}: + if self._is_terminal_cover_source(source): return result return await self._prefer_audiodb_album_cover( release_group_id, @@ -956,6 +982,16 @@ async def get_release_group_cover( priority, ) + if self._local_cover_preferred(): + # The owner's own files beat every remote source: folder/embedded art is + # instant, follows the files, and stays up when the network sources don't. + # Tried before the negative check so a marker banked pre-toggle (or during + # an outage) can never hide art the files already have. + local = await self._local_album_cover(release_group_id, file_path) + if local is not None: + await self._memory_set_from_result(identifier, suffix, local) + return local + if await self._disk_cache.is_negative(file_path): return await self._prefer_audiodb_album_cover( release_group_id, @@ -966,6 +1002,24 @@ async def get_release_group_cover( priority, ) + if _coverart_circuit_breaker.is_open(): + # Sustained CAA outage: skip the inline 2-attempt fetch (~12-15s hang per + # request) and do NOT spawn the deferred best-release resolve - it would + # fail the same way and then write the 4h negative meant for authoritative + # no-art. Bank a short transient negative so repeats serve the placeholder + # immediately; the first request after TTL expiry re-probes the breaker. + await self._disk_cache.write_negative( + file_path, ttl_seconds=COVER_TRANSIENT_NEGATIVE_TTL_SECONDS + ) + return await self._prefer_audiodb_album_cover( + release_group_id, + identifier, + suffix, + file_path, + None, + priority, + ) + # Key encodes the defer mode so a fast (deferred) hot request never coalesces onto an # in-flight slow full best-release resolve and blocks on it. See get_artist_image. dedupe_key = f"cover:rg:{release_group_id}:{size}:{'deferred' if defer_best_release else 'full'}" @@ -991,11 +1045,12 @@ async def get_release_group_cover( # here or the background resolve would be short-circuited by it. self._spawn_deferred_rg_resolve(release_group_id, size) return None - if result is None: - # Last resort: every external source missed - serve art embedded in a - # local library file for this album, if any has some. Beats the - # turntable placeholder for albums the internet has no cover for. - result = await self._embedded_album_cover(release_group_id, file_path) + if result is None and not self._local_cover_preferred(): + # Last resort when the local-art preference is off: every external source + # missed - serve art from a local library file for this album, if any has + # some. Beats the turntable placeholder for albums the internet has no + # cover for. + result = await self._local_album_cover(release_group_id, file_path) if result is None: await self._disk_cache.write_negative( file_path, ttl_seconds=COVER_NEGATIVE_TTL_SECONDS @@ -1066,37 +1121,121 @@ def is_artist_cover_warming(self, artist_id: str, size: Optional[int]) -> bool: return False return f"{artist_id}:{size}" in self._deferred_artist_inflight - async def _embedded_album_cover( + def _local_cover_preferred(self) -> bool: + return bool(self._local_cover_priority and self._local_cover_priority()) + + def _is_terminal_cover_source(self, source: str) -> bool: + """Sources never displaced by a later AudioDB hit on the read path.""" + if source in {"audiodb", "legacy-cache"}: + return True + return self._local_cover_preferred() and source in {"folder", "embedded"} + + async def _local_album_cover( self, release_group_id: str, file_path: Path, ) -> Optional[tuple[bytes, str, str]]: - """Front cover embedded in a local file for this release group, cached to - disk so subsequent loads hit the normal cache path. ``None`` when the native - library isn't wired, the album has no local files, or none carry raster art.""" - if self._library_db is None: - return None - try: - rows = await self._library_db.get_library_files_for_album(release_group_id) - except Exception as e: # noqa: BLE001 - logger.debug( - f"Embedded-cover lookup failed for {release_group_id[:8]}: {e}" - ) + """Cover art beside or inside the owner's local files for this release group, + cached to disk so subsequent loads hit the normal cache path. ``None`` when + the native library isn't wired, the album has no local files, or none yield + raster art.""" + if self._library_db is None and self._native_library_store is None: return None - paths = [row["file_path"] for row in rows if row.get("file_path")] + paths: list[str] = [] + if self._native_library_store is not None: + try: + paths = await self._native_library_store.get_indexed_track_paths_for_release_group( + release_group_id + ) + except Exception as e: # noqa: BLE001 + logger.debug( + f"Native local-cover lookup failed for {release_group_id[:8]}: {e}" + ) + if not paths and self._library_db is not None: + # Legacy rows may point at pre-organization paths; only consulted when + # the native catalog has no sealed files for this release group. + try: + rows = await self._library_db.get_library_files_for_album( + release_group_id + ) + except Exception as e: # noqa: BLE001 + logger.debug( + f"Legacy local-cover lookup failed for {release_group_id[:8]}: {e}" + ) + rows = [] + paths = [row["file_path"] for row in rows if row.get("file_path")] if not paths: return None - extracted = await asyncio.to_thread(self._extract_first_embedded_cover, paths) + extracted = await asyncio.to_thread(self._extract_local_cover, paths) if extracted is None: return None - content, content_type = extracted + content, content_type, source = extracted await self._disk_cache.write( - file_path, content, content_type, {"source": "embedded"} + file_path, content, content_type, {"source": source} ) - return content, content_type, "embedded" + return content, content_type, source + + def _extract_local_cover( + self, paths: list[str] + ) -> Optional[tuple[bytes, str, str]]: + """Synchronous: an explicit folder image wins over embedded art. Runs in a + worker thread - directory scans and mutagen reads are blocking.""" + folder = self._folder_cover(paths) + if folder is not None: + data, content_type = folder + return data, content_type, "folder" + embedded = self._extract_first_embedded_cover(paths) + if embedded is not None: + data, content_type = embedded + return data, content_type, "embedded" + return None + + @staticmethod + def _folder_cover(paths: list[str]) -> Optional[tuple[bytes, str]]: + directories: list[Path] = [] + for raw_path in paths: + parent = Path(raw_path).parent + if parent not in directories: + directories.append(parent) + if len(directories) > 1: + # Disc subdirectories (Album/CD1/track.flac) keep their art at the root, + # so probe one level up when every track dir shares a single parent. + # Sibling releases (Original/ + Deluxe/) share such a parent too; their + # artist folder is only probed when neither release folder has its own + # art, and inherited artist-level art beats the placeholder. Tracks + # spanning library roots have no single parent and are never probed up. + parents = {d.parent for d in directories} + if len(parents) == 1: + ancestor = parents.pop() + if ancestor.name and ancestor not in directories: + directories.append(ancestor) + for directory in directories: + try: + entries = { + entry.name.casefold(): entry + for entry in directory.iterdir() + if entry.is_file() + } + except OSError: + continue + for stem in _LOCAL_COVER_STEMS: + for extension in _LOCAL_COVER_EXTENSIONS: + candidate = entries.get(stem + extension) + if candidate is None: + continue + try: + if not 0 < candidate.stat().st_size <= _LOCAL_COVER_MAX_BYTES: + continue + data = candidate.read_bytes() + except OSError: + continue + content_type = _sniff_image_content_type(data) + if content_type is not None: + return data, content_type + return None def _extract_first_embedded_cover( self, paths: list[str] @@ -1179,6 +1318,23 @@ async def get_release_cover( is_disconnected, ) + if _coverart_circuit_breaker.is_open(): + # Sustained CAA outage: skip the inline 2-attempt fetch (~12-15s hang per + # request). Bank a short transient negative so repeats serve the placeholder + # immediately; the first request after TTL expiry re-probes the breaker. + await self._disk_cache.write_negative( + file_path, ttl_seconds=COVER_TRANSIENT_NEGATIVE_TTL_SECONDS + ) + return await self._prefer_release_audiodb_cover( + release_id, + identifier, + suffix, + file_path, + None, + priority, + is_disconnected, + ) + dedupe_key = f"cover:rel:{release_id}:{size}" result = await _deduplicator.dedupe( dedupe_key, diff --git a/backend/services/artist_discovery_service.py b/backend/services/artist_discovery_service.py index 0824346e6..f12ea736e 100644 --- a/backend/services/artist_discovery_service.py +++ b/backend/services/artist_discovery_service.py @@ -37,10 +37,36 @@ DEFAULT_TOP_SONGS_COUNT = 10 DEFAULT_TOP_ALBUMS_COUNT = 10 _DISCOVERY_WORKER_TIMEOUT = 120 +_PRECACHE_MAX_CONSECUTIVE_FAILURES = 5 +_PRECACHE_PAUSE_SECONDS = 1800.0 # matches ListenBrainz _POPULARITY_DEGRADED_TTL # Module-level flag survives singleton cache invalidation / instance recreation _discovery_precache_running = False +# Module-level pause state survives singleton cache invalidation / instance +# recreation, same rationale as _discovery_precache_running above. +_precache_consecutive_failures = 0 +_precache_paused_until = 0.0 # time.monotonic deadline; 0 = not paused + + +def _record_precache_unit_failure() -> None: + global _precache_consecutive_failures, _precache_paused_until + _precache_consecutive_failures += 1 + if _precache_consecutive_failures >= _PRECACHE_MAX_CONSECUTIVE_FAILURES: + _precache_paused_until = monotonic() + _PRECACHE_PAUSE_SECONDS + _precache_consecutive_failures = 0 + logger.info( + "Discovery precache paused for %ds after %d consecutive unit " + "failures (upstream outage backoff); will probe again after the pause", + int(_PRECACHE_PAUSE_SECONDS), + _PRECACHE_MAX_CONSECUTIVE_FAILURES, + ) + + +def _record_precache_unit_success() -> None: + global _precache_consecutive_failures + _precache_consecutive_failures = 0 + def _dedupe_similar_artists(artists: list[SimilarArtist]) -> list[SimilarArtist]: """Drop entries with a missing or duplicate musicbrainz_id. @@ -624,6 +650,9 @@ async def precache_artist_discovery( global _discovery_precache_running if _discovery_precache_running: return 0 + if monotonic() < _precache_paused_until: + logger.debug("Discovery precache skipped: paused after repeated upstream failures") + return 0 _discovery_precache_running = True try: @@ -678,8 +707,14 @@ async def _do_precache_artist_discovery( async def process_artist(idx: int, mbid: str) -> bool: nonlocal cached_count, source_fetches, progress_counter + if monotonic() < _precache_paused_until: + return False try: async with sem: + if monotonic() < _precache_paused_until: + # Pause tripped while this unit queued on the semaphore: + # fast-complete without invoking sources. + return False for source in sources: if self._workload_gate is not None: await self._workload_gate.wait_until_available() @@ -745,8 +780,10 @@ async def process_artist(idx: int, mbid: str) -> bool: local_progress, current_item=artist_name, generation=generation ) + _record_precache_unit_success() return True except Exception as e: # noqa: BLE001 + _record_precache_unit_failure() logger.warning("Failed to precache discovery for %s: %s", mbid[:8], e) async with counter_lock: progress_counter += 1 @@ -766,6 +803,7 @@ async def process_artist_with_timeout(idx: int, mbid: str) -> bool: process_artist(idx, mbid), timeout=_DISCOVERY_WORKER_TIMEOUT ) except asyncio.TimeoutError: + _record_precache_unit_failure() logger.warning( "Discovery timed out for %s after %ds", mbid[:8], @@ -787,6 +825,8 @@ async def process_artist_with_timeout(idx: int, mbid: str) -> bool: chunk = max(discovery_concurrency * 4, 20) for i in range(0, len(artist_mbids), chunk): + if monotonic() < _precache_paused_until: + break if status_service and status_service.is_cancelled(): break batch = artist_mbids[i : i + chunk] diff --git a/backend/services/artist_service.py b/backend/services/artist_service.py index 623cb037a..23a5c5aa6 100644 --- a/backend/services/artist_service.py +++ b/backend/services/artist_service.py @@ -25,7 +25,10 @@ extract_wiki_info, build_base_artist_info, ) -from infrastructure.cache.cache_keys import ARTIST_INFO_PREFIX +from infrastructure.cache.cache_keys import ( + ARTIST_INFO_PREFIX, + mb_artist_release_groups_key, +) from infrastructure.cache.memory_cache import CacheInterface from infrastructure.cache.disk_cache import DiskMetadataCache from infrastructure.validators import validate_mbid @@ -34,7 +37,7 @@ from core.exceptions import ClientDisconnectedError, ResourceNotFoundError from services.audiodb_image_service import AudioDBImageService from repositories.audiodb_models import AudioDBArtistImages -from repositories.musicbrainz_base import extract_artist_name +from repositories.musicbrainz_base import extract_artist_name, mb_deduplicator if TYPE_CHECKING: from infrastructure.persistence import LibraryDB @@ -42,6 +45,10 @@ logger = logging.getLogger(__name__) +# MB is rate-limited to 1 req/s in-process; bound the cold browse to 10 pages +# (1000 release groups) so pathological artists don't hog the limiter. +_MAX_RG_PAGES = 10 + class ArtistService: def __init__( @@ -569,92 +576,108 @@ async def _filter_aware_release_page( source_total_count=None, ) - _SCAN_BATCH = 100 - _MAX_SCAN_BATCHES = 2 - seen_mbids: set[str] = set() - all_albums: list[ReleaseItem] = [] - all_singles: list[ReleaseItem] = [] - all_eps: list[ReleaseItem] = [] + full_list = await self._fetch_all_release_groups(artist_id, is_disconnected) + + if self._ownership is not None: + album_mbids, requested_mbids = await self._target_release_group_flags( + full_list, artist_name="" + ) + + albums, singles, eps = categorize_release_groups( + {"release-group-list": full_list}, + album_mbids, + included_primary_types, + included_secondary_types, + requested_mbids, + ) + # Stream order = UI section order; categorize_release_groups already + # sorts each bucket by year desc, so don't re-sort here. - raw_offset = offset - source_total: int | None = None - batches_scanned = 0 + tagged: list[tuple[str, ReleaseItem]] = ( + [("albums", item) for item in albums] + + [("eps", item) for item in eps] + + [("singles", item) for item in singles] + ) + + page = tagged[offset : offset + limit] + page_albums = [item for kind, item in page if kind == "albums"] + page_singles = [item for kind, item in page if kind == "singles"] + page_eps = [item for kind, item in page if kind == "eps"] + + next_offset = offset + limit if offset + limit < len(tagged) else None + return ArtistReleases( + albums=page_albums, + singles=page_singles, + eps=page_eps, + offset=offset, + limit=limit, + returned_count=len(page), + next_offset=next_offset, + has_more=next_offset is not None, + source_total_count=len(tagged), + ) + + async def _fetch_all_release_groups( + self, artist_id: str, is_disconnected: DisconnectCallable | None + ) -> list[dict[str, Any]]: + """Cached, request-coalesced full release-group browse. + + Stores raw MB dicts only; in_library/requested flags are recomputed + per request from library state, so library changes never invalidate it. + """ + cache_key = mb_artist_release_groups_key(artist_id) + cached = await self._cache.get(cache_key) + if cached is not None: + return cached + return await mb_deduplicator.dedupe( + cache_key, + lambda: self._fetch_all_release_groups_uncached(artist_id, is_disconnected), + ) - while batches_scanned < _MAX_SCAN_BATCHES: + async def _fetch_all_release_groups_uncached( + self, artist_id: str, is_disconnected: DisconnectCallable | None + ) -> list[dict[str, Any]]: + """Fetch all release-group pages (max _MAX_RG_PAGES), first-wins dedupe. + + MB re-sorts each browse page by GID against a different materialized + order, so pages can overlap or drift mid-fetch; dedupe by id survives + that. Only complete fetches are cached so an outage never poisons it. + """ + cache_key = mb_artist_release_groups_key(artist_id) + collected: dict[str, dict[str, Any]] = {} + raw_offset = 0 + total = 0 + pages = 0 + + while pages < _MAX_RG_PAGES: await check_disconnected(is_disconnected) release_groups, mb_total = await self._mb_repo.get_artist_release_groups( artist_id, raw_offset, - _SCAN_BATCH, + 100, priority=RequestPriority.USER_INITIATED, ) - await check_disconnected(is_disconnected) - if source_total is None: - source_total = mb_total - + total = mb_total or total if not release_groups: break - - batch_album_mbids = album_mbids - batch_requested_mbids = requested_mbids - if self._ownership is not None: - ( - batch_album_mbids, - batch_requested_mbids, - ) = await self._target_release_group_flags( - release_groups, artist_name="" - ) - - consumed = 0 - for release_group in release_groups: - consumed += 1 - page_albums, page_singles, page_eps = categorize_release_groups( - {"release-group-list": [release_group]}, - batch_album_mbids, - included_primary_types, - included_secondary_types, - batch_requested_mbids, - ) - for target, items in ( - (all_albums, page_albums), - (all_singles, page_singles), - (all_eps, page_eps), - ): - for item in items: - if item.id and item.id not in seen_mbids: - seen_mbids.add(item.id) - target.append(item) - if len(seen_mbids) >= limit: - break - - raw_offset += consumed - batches_scanned += 1 - - if len(seen_mbids) >= limit: - break - if raw_offset >= mb_total: + for group in release_groups: + group_id = group.get("id") + if not group_id: + continue + collected.setdefault(str(group_id).casefold(), group) + raw_offset += len(release_groups) + pages += 1 + if raw_offset >= total: break - for lst in (all_albums, all_singles, all_eps): - lst.sort(key=lambda x: (x.year is None, -(x.year or 0))) - - returned_count = len(all_albums) + len(all_singles) + len(all_eps) - - has_more = raw_offset < (source_total or 0) - - next_offset = raw_offset if has_more else None - - return ArtistReleases( - albums=all_albums, - singles=all_singles, - eps=all_eps, - offset=offset, - limit=limit, - returned_count=returned_count, - next_offset=next_offset, - has_more=has_more, - source_total_count=source_total, - ) + full_list = list(collected.values()) + if total > 0 and raw_offset >= total: + await self._cache.set( + cache_key, + full_list, + ttl_seconds=self._get_artist_ttl(in_library=False), + ) + return full_list async def _fetch_artist_data( self, diff --git a/backend/services/audiodb_browse_queue.py b/backend/services/audiodb_browse_queue.py index 7162aabc9..8887b8614 100644 --- a/backend/services/audiodb_browse_queue.py +++ b/backend/services/audiodb_browse_queue.py @@ -94,6 +94,7 @@ async def _process_queue( try: settings = preferences_service.get_advanced_settings() if not settings.audiodb_enabled: + await asyncio.sleep(_BROWSE_QUEUE_INTER_ITEM_DELAY) continue if item.entity_type == "artist": diff --git a/backend/services/jellyfin_library_service.py b/backend/services/jellyfin_library_service.py index 77b16dce7..5095efb54 100644 --- a/backend/services/jellyfin_library_service.py +++ b/backend/services/jellyfin_library_service.py @@ -239,6 +239,20 @@ async def match_album_by_mbid(self, musicbrainz_id: str) -> JellyfinAlbumMatch: tracks=tracks, ) + async def resolve_album_mbid(self, album_id: str) -> str | None: + """Map a Jellyfin album GUID to its MusicBrainz release-group MBID. + + Reads ProviderIds.MusicBrainzReleaseGroup, falling back to + MusicBrainzAlbum - the same precedence JellyfinAlbumDetail uses. + Returns None when the album or its provider ids are unavailable + (get_album_detail degrades to None on upstream failure). + """ + item = await self._jellyfin.get_album_detail(album_id) + if not item: + return None + pids = item.provider_ids or {} + return pids.get("MusicBrainzReleaseGroup") or pids.get("MusicBrainzAlbum") + async def get_artists( self, limit: int = 50, offset: int = 0 ) -> list[JellyfinArtistSummary]: @@ -584,6 +598,21 @@ async def import_playlist( return JellyfinImportResult(droppedneedle_playlist_id=re_check.id, already_imported=True) raise + # Map each distinct Jellyfin album GUID to its MusicBrainz MBID so the + # stored album_id can match the MBID-keyed local catalog (#150). One + # deduped fetch per album; failures keep the GUID (today's behavior). + album_mbids: dict[str, str] = {} + distinct_album_ids = sorted({t.album_id for t in detail.tracks if t.album_id}) + if distinct_album_ids: + resolved = await asyncio.gather( + *(self.resolve_album_mbid(guid) for guid in distinct_album_ids) + ) + album_mbids = { + guid: mbid + for guid, mbid in zip(distinct_album_ids, resolved) + if mbid + } + track_dicts = [] failed = 0 for t in detail.tracks: @@ -595,7 +624,7 @@ async def import_playlist( "duration": t.duration_seconds, "track_source_id": t.id, "source_type": "jellyfin", - "album_id": t.album_id, + "album_id": album_mbids.get(t.album_id) or t.album_id, "artist_id": t.artist_id, "track_number": t.track_number, "disc_number": t.disc_number, diff --git a/backend/services/jellyfin_playback_service.py b/backend/services/jellyfin_playback_service.py index 74644c07b..0948d720f 100644 --- a/backend/services/jellyfin_playback_service.py +++ b/backend/services/jellyfin_playback_service.py @@ -151,16 +151,44 @@ async def stop_playback( except (httpx.HTTPError, ExternalServiceError) as e: logger.warning("Stop report failed for %s: %s", item_id, e) - async def proxy_head(self, item_id: str) -> Response: - result: StreamProxyResult = await self._jellyfin.proxy_head_stream(item_id) + async def proxy_head(self, item_id: str, user_id: str | None = None) -> Response: + repo = await self._repo_for(user_id) + try: + result: StreamProxyResult = await repo.proxy_head_stream(item_id) + except JellyfinAuthError: + if repo is self._jellyfin: + raise + # linked account's token was revoked: the stream still rides the + # app-level account (attribution stays fail closed) + logger.warning( + "Per-user Jellyfin token rejected for user %s, " + "streaming %s via app account", + user_id, + item_id, + ) + result = await self._jellyfin.proxy_head_stream(item_id) return Response(status_code=200, headers=result.headers) async def proxy_stream( - self, item_id: str, range_header: str | None = None + self, item_id: str, range_header: str | None = None, user_id: str | None = None ) -> StreamingResponse: - result: StreamProxyResult = await self._jellyfin.proxy_get_stream( - item_id, range_header=range_header - ) + repo = await self._repo_for(user_id) + try: + result: StreamProxyResult = await repo.proxy_get_stream( + item_id, range_header=range_header + ) + except JellyfinAuthError: + if repo is self._jellyfin: + raise + logger.warning( + "Per-user Jellyfin token rejected for user %s, " + "streaming %s via app account", + user_id, + item_id, + ) + result = await self._jellyfin.proxy_get_stream( + item_id, range_header=range_header + ) return StreamingResponse( content=result.body_chunks, status_code=result.status_code, diff --git a/backend/services/native/acquisition_cleanup_service.py b/backend/services/native/acquisition_cleanup_service.py index 5da735321..b113bf93e 100644 --- a/backend/services/native/acquisition_cleanup_service.py +++ b/backend/services/native/acquisition_cleanup_service.py @@ -61,12 +61,14 @@ def __init__( client_getter: Callable[[str], DownloadClientProtocol], sab_mount_getter: Callable[[], Path], *, + sab_category_getter: Callable[[], str] = lambda: "*", clock: Callable[[], float] = time.time, ) -> None: self._store = download_store self._library_store = library_store self._client_getter = client_getter self._sab_mount_getter = sab_mount_getter + self._sab_category_getter = sab_category_getter self._clock = clock async def run_once(self, worker_id: str) -> int: @@ -591,6 +593,7 @@ async def reconcile_legacy_mount( return 0 processed = 0 + category = self._sab_category_getter() while processed < max(1, limit): if progress.current_directory is None: if not progress.pending_directories: @@ -625,7 +628,9 @@ async def reconcile_legacy_mount( match, is_symlink=is_symlink, ) - elif is_directory and not is_symlink: + elif is_directory and not is_symlink and _descend_allowed( + name, category + ): progress.pending_directories.append(relative.as_posix()) if processed >= max(1, limit): break @@ -994,14 +999,34 @@ def _remove_tree_contents(directory_fd: int) -> None: os.unlink(name, dir_fd=directory_fd) +def _descend_allowed(name: str, category: str) -> bool: + """Descend only into DN-owned trees: the job prefix, plus the configured SAB + category dir (case-insensitive) when it isn't ``*``. Sonarr/Radarr/Whisparr + trees sharing the mount are never visited.""" + if name.startswith("droppedneedle"): + return True + return category != "*" and name.lower() == category.lower() + + def _directory_entries(path: Path) -> list[tuple[str, bool, bool]]: - with os.scandir(path) as entries: - result = [ - ( - entry.name, - entry.is_dir(follow_symlinks=False), - entry.is_symlink(), - ) - for entry in entries - ] + try: + with os.scandir(path) as entries: + result = [] + for entry in entries: + try: + result.append( + ( + entry.name, + entry.is_dir(follow_symlinks=False), + entry.is_symlink(), + ) + ) + except (FileNotFoundError, NotADirectoryError): + logger.debug( + "Acquisition cleanup entry vanished mid-scan: %s", + path / entry.name, + ) + except (FileNotFoundError, NotADirectoryError): + logger.debug("Acquisition cleanup directory vanished: %s", path) + return [] return sorted(result, key=lambda value: value[0]) diff --git a/backend/services/native/album_identification_service.py b/backend/services/native/album_identification_service.py index c84abe913..b8ddd591c 100644 --- a/backend/services/native/album_identification_service.py +++ b/backend/services/native/album_identification_service.py @@ -374,10 +374,12 @@ async def run_claimed_job( str(job["local_album_id"]) ) if context is None: - await self._queue.defer( + # The album row is gone or retired: durable catalog state, so fail + # terminally (auditable) instead of deferring until the cap. + await self._queue.fail( job, worker_id, "SUBJECT_NOT_AVAILABLE", now=timestamp ) - return "provider_deferred" + return "attention" raw_tracks: list[dict] = [ row for row in context["tracks"] if row["availability"] == "indexed" ] diff --git a/backend/services/native/file_processor.py b/backend/services/native/file_processor.py index 9eee613b8..d260bc452 100644 --- a/backend/services/native/file_processor.py +++ b/backend/services/native/file_processor.py @@ -25,7 +25,7 @@ from rapidfuzz import fuzz -from core.exceptions import AutomaticManagementHoldError +from core.exceptions import AutomaticManagementHoldError, ConfigurationError from infrastructure.msgspec_fastapi import AppStruct from models.audio import AudioInfo, AudioTag from models.download_manifest import DownloadManifest, ExpectedFile, ExpectedTrack @@ -1472,6 +1472,18 @@ async def _hold_for_review( logger.warning("Could not hold %s for review: %s", source.name, exc) return False + def _root_library_path(self) -> Path: + """The first configured library root, checked at the moment of use. + + Held-retry paths run long after the original import - the roots may have + been cleared since. Unlike a missing held file (terminal), that is + recoverable, so it raises an actionable 400 instead of an IndexError 500.""" + if not self._library_paths: + raise ConfigurationError( + "No library root is configured - restore one in Settings → Library, then try again." + ) + return self._library_paths[0] + async def place_held_management_bundle( self, held_files: list["HeldImport"] ) -> list[Path]: @@ -1528,7 +1540,7 @@ async def place_held_management_bundle( held.artist_mbid or tag.musicbrainz_album_artist_id ), ) - target_path = self._library_paths[0] / self._naming.format_path( + target_path = self._root_library_path() / self._naming.format_path( held.naming_template or "", target_tag, info.file_format ) replacement: dict | None = None @@ -1612,7 +1624,7 @@ async def place_held_file(self, held: "HeldImport") -> Path: musicbrainz_album_artist_id=held.artist_mbid or tag.musicbrainz_album_artist_id, ) - target_path = self._library_paths[0] / self._naming.format_path( + target_path = self._root_library_path() / self._naming.format_path( held.naming_template or "", target_tag, info.file_format ) # D10 confirm-replace: an upgrade's held file (AcoustID disagreed, a human diff --git a/backend/services/native/identification_queue_service.py b/backend/services/native/identification_queue_service.py index 6cf092475..055545501 100644 --- a/backend/services/native/identification_queue_service.py +++ b/backend/services/native/identification_queue_service.py @@ -4,6 +4,7 @@ import time import uuid +from collections.abc import Callable from infrastructure.persistence.native_library_store import NativeLibraryStore from models.library_work import IdentificationJob @@ -14,11 +15,24 @@ PRIORITY_SUPPORTING_MAINTENANCE = 50 LEASE_SECONDS = 60.0 MAX_BACKOFF_SECONDS = 6 * 60 * 60 +MAX_DEFERRAL_ATTEMPTS = 10 +SUBJECT_NOT_AVAILABLE_GRACE_SECONDS = 24 * 60 * 60 class IdentificationQueueService: - def __init__(self, store: NativeLibraryStore) -> None: + def __init__( + self, + store: NativeLibraryStore, + *, + provider_available: Callable[[], bool] | None = None, + ) -> None: self._store = store + self._provider_available = provider_available + + def _resurrect_attention(self) -> bool: + if self._provider_available is None: + return True + return self._provider_available() async def enqueue_album( self, @@ -40,7 +54,9 @@ async def enqueue_album( now=now, ) return await self._store.enqueue_identification_job( - job, expected_policy_revision=expected_policy_revision + job, + expected_policy_revision=expected_policy_revision, + resurrect_attention=self._resurrect_attention(), ) async def enqueue_album_with_disposition( @@ -61,7 +77,9 @@ async def enqueue_album_with_disposition( requested_by_user_id=requested_by_user_id, now=now, ) - return await self._store.enqueue_identification_job_result(job) + return await self._store.enqueue_identification_job_result( + job, resurrect_attention=self._resurrect_attention() + ) async def enqueue_albums_with_disposition( self, @@ -90,6 +108,7 @@ async def enqueue_albums_with_disposition( grouping_context=grouping_context, queue_cursor=queue_cursor, background=background, + resurrect_attention=self._resurrect_attention(), ) @staticmethod @@ -140,6 +159,15 @@ async def defer( ) -> int: timestamp = time.time() if now is None else now attempts = max(1, int(job.get("attempt_count", 1))) + if attempts >= MAX_DEFERRAL_ATTEMPTS: + return await self._store.terminal_fail_identification_job( + str(job["id"]), + worker_id=worker_id, + expected_job_revision=int(job["row_revision"]), + failure_code="MAX_DEFERRALS_EXCEEDED", + attention_cause=failure_code, + now=timestamp, + ) backoff = min(MAX_BACKOFF_SECONDS, 30 * (2 ** min(attempts - 1, 10))) return await self._store.defer_identification_job( str(job["id"]), @@ -150,6 +178,29 @@ async def defer( now=timestamp, ) + async def fail( + self, + job: dict, + worker_id: str, + failure_code: str, + *, + now: float | None = None, + ) -> int: + """Terminally fail a running job, keeping the row for auditability.""" + return await self._store.terminal_fail_identification_job( + str(job["id"]), + worker_id=worker_id, + expected_job_revision=int(job["row_revision"]), + failure_code=failure_code, + attention_cause=failure_code, + now=time.time() if now is None else now, + ) + + async def reset_provider_deferrals(self, *, now: float | None = None) -> int: + return await self._store.reset_provider_identification_deferrals( + now=time.time() if now is None else now + ) + async def pause( self, requested_by_user_id: str | None, @@ -191,12 +242,19 @@ async def is_paused(self) -> bool: return (await self._store.get_identification_control())["state"] == "paused" async def recover(self, *, now: float | None = None) -> int: - return await self._store.recover_expired_identification_leases( - now=time.time() if now is None else now + timestamp = time.time() if now is None else now + recovered = await self._store.recover_expired_identification_leases( + now=timestamp + ) + await self._store.gc_stale_identification_jobs( + now=timestamp, grace_seconds=SUBJECT_NOT_AVAILABLE_GRACE_SECONDS ) + return recovered async def activity_snapshot(self) -> dict: - return await self._store.get_identification_activity_snapshot() + return await self._store.get_identification_activity_snapshot( + now=time.time() + ) async def stream_revisions(self) -> dict[str, int]: revisions = { diff --git a/backend/services/native/identity_repair_service.py b/backend/services/native/identity_repair_service.py index 53173c1ef..2ad8d6176 100644 --- a/backend/services/native/identity_repair_service.py +++ b/backend/services/native/identity_repair_service.py @@ -7,6 +7,8 @@ import uuid from collections.abc import Awaitable, Callable +import msgspec.json + from api.v1.schemas.library_operations import ( IdentityPreparationCreateRequest, IdentityPreparationEstimateResponse, @@ -16,10 +18,16 @@ RepairEstimateResponse, RepairFindingListResponse, RepairFindingResponse, + SuggestedEditionSummary, ) from core.exceptions import ExternalServiceError, ResourceNotFoundError, ValidationError from infrastructure.queue.priority_queue import RequestPriority -from infrastructure.persistence.native_library_store import NativeLibraryStore +from infrastructure.resilience.retry import CircuitOpenError +from infrastructure.persistence.native_library_store import ( + AUTOMATIC_SAFE_EVIDENCE_REASONS, + NativeLibraryStore, + _complete_track_identity_mapping, +) from models.identification import ( AlbumCandidate, CandidateEvidence, @@ -56,7 +64,16 @@ ) MANAGEMENT_READINESS_PURPOSE = "management_readiness" -MANAGEMENT_MAPPING_VERSION = "management-edition-readiness-v3" +MANAGEMENT_MAPPING_VERSION = "management-edition-readiness-v4" + +# MusicBrainz breaker timeout is 60 s; this 2x window (matching the artist +# reconciliation service) gives the breaker a recovery window between attempts. +_PROVIDER_DEFERRED_RETRY_SECONDS = 120.0 + + +class _ProviderUnavailable(Exception): + """Control-flow: the identity provider is unavailable, so the audit defers + the whole job instead of writing 'unverifiable' findings during the outage.""" class IdentityRepairService: @@ -66,11 +83,13 @@ def __init__( provider: IdentificationProviderProtocol | None = None, evidence: AlbumEvidenceEngine | None = None, canonical_provider: CanonicalMusicBrainzRepositoryProtocol | None = None, + provider_available: Callable[[], bool] | None = None, ) -> None: self._store = store self._provider = provider self._evidence = evidence or AlbumEvidenceEngine() self._canonical_provider = canonical_provider + self._provider_available = provider_available self._operations = LibraryOperationService(store) async def create( @@ -207,6 +226,7 @@ async def run_claimed_audit( *, now: float | None = None, checkpoint: Callable[[], Awaitable[None]] | None = None, + provider_available: Callable[[], bool] | None = None, ) -> OperationResponse: snapshot = await self._store.get_operation_snapshot(str(job["id"])) scope = ( @@ -219,8 +239,13 @@ async def run_claimed_audit( if scope else "existing_matches" ) + availability = ( + self._provider_available if provider_available is None else provider_available + ) while True: timestamp = time.time() if now is None else now + if availability is not None and not availability(): + return await self._defer_audit(str(job["id"]), worker_id, timestamp) controlled = await self._store.checkpoint_operation_control( str(job["id"]), worker_id, now=timestamp ) @@ -245,13 +270,21 @@ async def run_claimed_audit( ) if not renewed: raise ResourceNotFoundError("The identity check lease changed.") - if purpose == MANAGEMENT_READINESS_PURPOSE: - finding, attempt, evidence = await self._classify_management_readiness( - str(job["id"]), work, context, timestamp - ) - else: - finding, attempt, evidence = await self._classify( - str(job["id"]), work, context + try: + if purpose == MANAGEMENT_READINESS_PURPOSE: + finding, attempt, evidence = await self._classify_management_readiness( + str(job["id"]), work, context, timestamp + ) + else: + finding, attempt, evidence = await self._classify( + str(job["id"]), work, context + ) + except _ProviderUnavailable: + return await self._defer_audit( + str(job["id"]), + worker_id, + timestamp, + ordinal=int(work["ordinal"]), ) await self._store.save_repair_finding_for_work( str(job["id"]), @@ -266,6 +299,24 @@ async def run_claimed_audit( if checkpoint is not None: await checkpoint() + async def _defer_audit( + self, + job_id: str, + worker_id: str, + timestamp: float, + *, + ordinal: int | None = None, + ) -> OperationResponse: + deferred = await self._store.defer_repair_audit_work( + job_id=job_id, + ordinal=ordinal, + worker_id=worker_id, + reason_code="PROVIDER_DEFERRED", + now=timestamp, + retry_not_before=timestamp + _PROVIDER_DEFERRED_RETRY_SECONDS, + ) + return self._operations._response(deferred) + async def _classify_management_readiness( self, job_id: str, @@ -308,19 +359,8 @@ async def _classify_management_readiness( or not identity["release_group_mbid"] or not identity["release_mbid"] ): - return ( - self._finding( - job_id, - work, - "exact_release_required", - "EXACT_EDITION_NOT_ACCEPTED", - False, - identity_revision=( - int(identity["row_revision"]) if identity is not None else None - ), - ), - None, - [], + return await self._classify_exact_release_suggestion( + job_id, work, context, identity ) tracks = [row for row in context["tracks"] if row["availability"] == "indexed"] release_track_ids = [ @@ -359,19 +399,10 @@ async def _classify_management_readiness( includes=("artist-credits", "recordings", "release-groups"), priority=RequestPriority.BACKGROUND_SYNC, ) - except ExternalServiceError: - return ( - self._finding( - job_id, - work, - "unverifiable", - "PROVIDER_DEFERRED", - False, - identity_revision=int(identity["row_revision"]), - ), - None, - [], - ) + except (ExternalServiceError, CircuitOpenError) as error: + raise _ProviderUnavailable( + "MusicBrainz is unavailable; deferring the identity audit." + ) from error if release is None: return ( self._finding( @@ -415,19 +446,10 @@ async def _classify_management_readiness( tracks, candidate, ) - except ExternalServiceError: - return ( - self._finding( - job_id, - work, - "unverifiable", - "PROVIDER_DEFERRED", - False, - identity_revision=int(identity["row_revision"]), - ), - None, - [], - ) + except (ExternalServiceError, CircuitOpenError) as error: + raise _ProviderUnavailable( + "MusicBrainz is unavailable; deferring the identity audit." + ) from error evaluated = self._evidence.evaluate_candidate(local_tracks, candidate) self._disambiguate_duplicate_recordings( local_tracks, @@ -562,6 +584,141 @@ async def _classify_management_readiness( [record], ) + async def _classify_exact_release_suggestion( + self, + job_id: str, + work: dict, + context: dict, + identity: dict | None, + ) -> tuple[ + RepairFinding, + IdentificationAttempt | None, + list[IdentificationEvidenceRecord], + ]: + """Suggest one sealable exact edition from stored identification evidence.""" + album_id = str(work["local_album_id"]) + identity_revision = ( + int(identity["row_revision"]) if identity is not None else None + ) + + def bare() -> tuple[ + RepairFinding, + IdentificationAttempt | None, + list[IdentificationEvidenceRecord], + ]: + return ( + self._finding( + job_id, + work, + "exact_release_required", + "EXACT_EDITION_NOT_ACCEPTED", + False, + identity_revision=identity_revision, + ), + None, + [], + ) + + tracks = [row for row in context["tracks"] if row["availability"] == "indexed"] + if not tracks: + return bare() + stored = await self._store.get_latest_album_identification_evidence(album_id) + if stored is None: + return bare() + attempt, evidence_rows = stored + if album_input_revisions(tracks) != ( + str(attempt["input_tag_revision"]), + str(attempt["input_file_revision"]), + str(attempt["input_policy_revision"]), + ): + return bare() + suggestible: list[tuple[dict, CandidateEvidence]] = [] + for row in evidence_rows: + candidate_evidence = msgspec.json.decode( + bytes(row["evidence_json"]), type=CandidateEvidence + ) + if ( + candidate_evidence.reason_code in AUTOMATIC_SAFE_EVIDENCE_REASONS + and candidate_evidence.release_mbid + and _complete_track_identity_mapping(tracks, candidate_evidence) + is not None + ): + suggestible.append((row, candidate_evidence)) + if not suggestible: + return bare() + competing_count = len(suggestible) + if competing_count == 1: + winner_row, winner = suggestible[0] + summary: dict[str, object] = { + "title": winner.album_title, + "date": winner.release_date, + "country": None, + "status": None, + "track_count": len(winner.track_evidence) + + len(winner.unmatched_expected_tracks), + "competing_count": 1, + } + else: + ranked: list[ + tuple[tuple[int, str, int, str], dict, CandidateEvidence, dict] + ] = [] + for row, candidate_evidence in suggestible: + release: MbManagementRelease | None = None + if self._canonical_provider is not None: + try: + release = await self._canonical_provider.get_canonical_release( + str(candidate_evidence.release_mbid), + includes=("media",), + priority=RequestPriority.BACKGROUND_SYNC, + ) + except (ExternalServiceError, CircuitOpenError) as error: + raise _ProviderUnavailable( + "MusicBrainz is unavailable; deferring the identity audit." + ) from error + if release is None: + continue + summary = { + "title": ( + release.title + if release is not None + else candidate_evidence.album_title + ), + "date": (release.date if release is not None else None) + or candidate_evidence.release_date, + "country": release.country if release is not None else None, + "status": release.status if release is not None else None, + "track_count": ( + sum(medium.track_count for medium in release.media) + if release is not None + else len(candidate_evidence.track_evidence) + + len(candidate_evidence.unmatched_expected_tracks) + ), + "competing_count": competing_count, + } + key = ( + 0 if release is not None and release.status == "Official" else 1, + str(summary["date"] or "9999"), + 0 if release is not None and release.country == "XW" else 1, + str(candidate_evidence.release_mbid), + ) + ranked.append((key, row, candidate_evidence, summary)) + if not ranked: + return bare() + _, winner_row, winner, summary = min(ranked, key=lambda item: item[0]) + finding = self._finding( + job_id, + work, + "exact_release_suggested", + "EXACT_EDITION_SUGGESTED", + True, + evidence_id=str(winner_row["id"]), + identity_revision=identity_revision, + ) + finding.suggested_release_mbid = str(winner.release_mbid) + finding.suggested_release_group_mbid = winner.release_group_mbid + finding.suggested_edition_json = json.dumps(summary, sort_keys=True) + return finding, None, [] + async def _normalize_recording_redirects( self, local_tracks: list[GroupingTrack], @@ -907,7 +1064,10 @@ async def findings( categories = { "ready": ["ready"], "mapping_ready": ["mapping_ready"], - "exact_release_required": ["exact_release_required"], + "exact_release_required": [ + "exact_release_required", + "exact_release_suggested", + ], "needs_review": ["needs_review"], "unverifiable": ["unverifiable", "stale"], } @@ -943,6 +1103,22 @@ async def findings( next_cursor = None if result["has_more"] and rows: next_cursor = f"{rows[-1]['updated_at']}:{rows[-1]['id']}" + + def _suggested_edition(row: dict) -> SuggestedEditionSummary | None: + if not row["suggested_release_mbid"]: + return None + payload = json.loads(str(row["suggested_edition_json"])) + return SuggestedEditionSummary( + release_mbid=str(row["suggested_release_mbid"]), + release_group_mbid=str(row["suggested_release_group_mbid"]), + title=str(payload.get("title") or ""), + track_count=int(payload.get("track_count") or 0), + competing_count=int(payload.get("competing_count") or 1), + date=payload.get("date"), + country=payload.get("country"), + status=payload.get("status"), + ) + return RepairFindingListResponse( items=[ RepairFindingResponse( @@ -960,6 +1136,7 @@ async def findings( apply_eligible=bool(row["apply_eligible"]), state=str(row["state"]), apply_result=row["apply_result"], + suggested_edition=_suggested_edition(row), updated_at=float(row["updated_at"]), row_revision=int(row["row_revision"]), ) @@ -1011,7 +1188,6 @@ async def _classify( ) attempt: IdentificationAttempt | None = None records: list[IdentificationEvidenceRecord] = [] - provider_deferred = False stored: IdentificationEvidenceRecord | None = None candidate: AlbumCandidate | None = None fingerprint_filled = False @@ -1021,9 +1197,10 @@ async def _classify( str(identity["release_mbid"]), RequestPriority.BACKGROUND_SYNC, ) - except ExternalServiceError: - candidate = None - provider_deferred = True + except (ExternalServiceError, CircuitOpenError) as error: + raise _ProviderUnavailable( + "MusicBrainz is unavailable; deferring the identity audit." + ) from error if candidate is not None: grouping_tracks = [_to_grouping_track(row) for row in tracks] for track, row in zip(grouping_tracks, tracks, strict=True): @@ -1085,9 +1262,7 @@ async def _classify( job_id, work, "unverifiable", - "PROVIDER_DEFERRED" - if provider_deferred - else "EVIDENCE_UNAVAILABLE", + "EVIDENCE_UNAVAILABLE", False, identity_revision=int(identity["row_revision"]), ), diff --git a/backend/services/native/library_inventory_scanner.py b/backend/services/native/library_inventory_scanner.py index 7a732acf0..b4fc0f7d6 100644 --- a/backend/services/native/library_inventory_scanner.py +++ b/backend/services/native/library_inventory_scanner.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import errno import os import threading import time @@ -14,7 +15,12 @@ import msgspec from infrastructure.persistence.native_library_store import NativeLibraryStore -from models.library_work import ScanInventoryItem, ScanRun, ScanScope +from models.library_work import ( + ScanFailureRecord, + ScanInventoryItem, + ScanRun, + ScanScope, +) from services.local_files_service import AUDIO_EXTENSIONS from services.native.library_filesystem_coordinator import ( LibraryFilesystemCoordinator, @@ -28,6 +34,7 @@ Checkpoint = Callable[[str, str], Awaitable[bool]] DirectoryWalker = Callable[..., Iterator[tuple[str, list[str], list[str]]]] +DirectoryProbe = Callable[[Path], bool] logger = logging.getLogger(__name__) @@ -36,6 +43,25 @@ def _uncoordinated_read() -> Iterator[None]: yield +class _WalkHeartbeat: + """Thread-safe liveness signal written by the walk producer thread.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._touched_at = time.monotonic() + self.last_directory = "" + + def touch(self, directory: str = "") -> None: + with self._lock: + self._touched_at = time.monotonic() + if directory: + self.last_directory = directory + + def age(self) -> float: + with self._lock: + return time.monotonic() - self._touched_at + + class LibraryInventoryScanner: def __init__( self, @@ -43,10 +69,73 @@ def __init__( *, directory_walker: DirectoryWalker = os.walk, filesystem_coordinator: LibraryFilesystemCoordinator | None = None, + walk_deadline_seconds: float = 30.0, + directory_probe: DirectoryProbe = Path.is_dir, + max_detached_walkers: int = 4, ) -> None: self._store = store self._directory_walker = directory_walker self._filesystem = filesystem_coordinator + self._walk_deadline_seconds = walk_deadline_seconds + self._directory_probe = directory_probe + self._max_detached_walkers = max_detached_walkers + self._detached_walkers: set[asyncio.Task[None]] = set() + + def _finish_detached_walker(self, task: asyncio.Task[None]) -> None: + self._detached_walkers.discard(task) + if not task.cancelled(): + task.exception() + + def _detach_walker(self, task: asyncio.Task[None]) -> None: + if task.done(): + return + if len(self._detached_walkers) < self._max_detached_walkers: + self._detached_walkers.add(task) + task.add_done_callback(self._finish_detached_walker) + + async def _record_failure( + self, + run_id: str, + scope: ScanScope, + *, + relative_path: str, + failure_code: str, + failure_detail: str, + ) -> None: + await self._store.record_scan_failures( + run_id, + [ + ScanFailureRecord( + root_id=scope.root_id, + relative_path=relative_path, + failure_code=failure_code, + recorded_at=time.time(), + failure_detail=failure_detail, + phase="discovering", + ) + ], + ) + + @staticmethod + def _relativize(path: Path, root: Path) -> str: + try: + return PurePosixPath(*path.relative_to(root).parts).as_posix() + except ValueError: + return path.as_posix() + + @staticmethod + def _failure_relative_path( + exc: BaseException, root: Path, heartbeat: _WalkHeartbeat + ) -> str: + filename = getattr(exc, "filename", None) + candidate = ( + Path(str(filename)) + if filename + else (Path(heartbeat.last_directory) if heartbeat.last_directory else None) + ) + if candidate is None: + return "." + return LibraryInventoryScanner._relativize(candidate, root) async def discover( self, @@ -71,6 +160,13 @@ async def discover( if root is None and scope.root_path is not None: root = Path(scope.root_path) if root is None: + await self._record_failure( + run.id, + scope, + relative_path=scope.relative_path, + failure_code="ROOT_UNAVAILABLE", + failure_detail="The library root has no configured path.", + ) await self._store.complete_scan_scope_discovery( run.id, scope.root_id, @@ -89,8 +185,51 @@ async def discover( selected = ( root if scope.relative_path == "." else root / scope.relative_path ) - exists = await asyncio.to_thread(selected.is_dir) + try: + exists = await asyncio.wait_for( + asyncio.to_thread(self._directory_probe, selected), + timeout=self._walk_deadline_seconds, + ) + except TimeoutError: + logger.warning( + "library_scan event=walk_timeout run_id=%s root_id=%s path=%s", + run.id, + scope.root_id, + scope.relative_path, + ) + await self._record_failure( + run.id, + scope, + relative_path=scope.relative_path, + failure_code="WALK_TIMEOUT", + failure_detail=( + "The library root probe exceeded " + f"{self._walk_deadline_seconds:.1f}s." + ), + ) + await self._store.complete_scan_scope_discovery( + run.id, + scope.root_id, + scope.relative_path, + state="unavailable", + error_code="WALK_TIMEOUT", + ) + return await self._store.transition_scan_run( + run.id, + expected_state=current.state, + expected_revision=current.row_revision, + new_state="failed", + now=current.updated_at, + terminal_code="WALK_TIMEOUT", + ) if not exists: + await self._record_failure( + run.id, + scope, + relative_path=scope.relative_path, + failure_code="ROOT_UNAVAILABLE", + failure_detail=f"The library root path is missing: {selected}", + ) await self._store.complete_scan_scope_discovery( run.id, scope.root_id, @@ -117,7 +256,7 @@ async def discover( if self._filesystem is not None else None ) - current, completed = await self._walk_scope( + current, completed, walk_failure_code = await self._walk_scope( current, scope, root, @@ -146,7 +285,7 @@ async def discover( expected_revision=current.row_revision, new_state="failed", now=current.updated_at, - terminal_code="ROOT_PERMISSION_DENIED", + terminal_code=walk_failure_code or "ROOT_PERMISSION_DENIED", ) return current await self._store.complete_scan_scope_discovery( @@ -167,12 +306,13 @@ async def _walk_scope( resolver: LibraryPolicyResolver, checkpoint: Checkpoint, discovery_generation: int = 1, - ) -> tuple[ScanRun, bool]: + ) -> tuple[ScanRun, bool, str | None]: loop = asyncio.get_running_loop() queue: asyncio.Queue[tuple[Path, os.stat_result] | BaseException | None] = ( asyncio.Queue(maxsize=INVENTORY_QUEUE_SIZE) ) stopped = threading.Event() + heartbeat = _WalkHeartbeat() def producer() -> None: try: @@ -196,6 +336,7 @@ def onerror(error: OSError) -> None: directory, subdirectories, filenames = next(walker) except StopIteration: break + heartbeat.touch(directory) subdirectories[:] = [ name for name in subdirectories @@ -205,6 +346,7 @@ def onerror(error: OSError) -> None: tuple[Path, os.stat_result] | BaseException ] = [] for filename in filenames: + heartbeat.touch(directory) path = Path(directory) / filename if ( path.suffix.casefold() not in AUDIO_EXTENSIONS @@ -232,6 +374,8 @@ def onerror(error: OSError) -> None: current = run completed = True discard_remaining = False + detached = False + walk_failure_code: str | None = None discovered = 0 stale_cleanup_pending = True last_checkpoint = time.monotonic() @@ -241,6 +385,36 @@ def onerror(error: OSError) -> None: try: item = await asyncio.wait_for(queue.get(), timeout=0.25) except TimeoutError: + if heartbeat.age() > self._walk_deadline_seconds: + completed = False + stopped.set() + walk_failure_code = "WALK_TIMEOUT" + logger.warning( + "library_scan event=walk_timeout run_id=%s root_id=%s " + "path=%s", + run.id, + scope.root_id, + heartbeat.last_directory or scope.relative_path, + ) + await self._record_failure( + run.id, + scope, + relative_path=( + self._relativize(Path(heartbeat.last_directory), root) + if heartbeat.last_directory + else scope.relative_path + ), + failure_code="WALK_TIMEOUT", + failure_detail=( + "The directory walk made no progress for " + f"{self._walk_deadline_seconds:.1f}s." + ), + ) + # The producer thread is wedged in a syscall; awaiting it + # would wedge the scan worker, so it is detached instead. + self._detach_walker(producer_task) + detached = True + break if not await checkpoint(run.id, scope.policy_revision): completed = False stopped.set() @@ -255,6 +429,28 @@ def onerror(error: OSError) -> None: completed = False stopped.set() discard_remaining = True + walk_failure_code = "ROOT_PERMISSION_DENIED" + relative_path = self._failure_relative_path(item, root, heartbeat) + failure_code = ( + "WALK_" + errno.errorcode.get(item.errno, "EUNKNOWN") + if isinstance(item, OSError) and item.errno is not None + else "WALK_ERROR" + ) + logger.warning( + "library_scan event=walk_error run_id=%s root_id=%s path=%s " + "error=%s", + run.id, + scope.root_id, + relative_path, + item, + ) + await self._record_failure( + run.id, + scope, + relative_path=relative_path, + failure_code=failure_code, + failure_detail=str(item), + ) continue batch.append(item) if len(batch) >= INVENTORY_BATCH_SIZE: @@ -307,12 +503,17 @@ def onerror(error: OSError) -> None: try: await asyncio.wait_for(queue.get(), timeout=0.1) except TimeoutError: + if heartbeat.age() > self._walk_deadline_seconds: + self._detach_walker(producer_task) + detached = True + break continue - await asyncio.shield(producer_task) + if not detached: + await asyncio.shield(producer_task) raise finally: stopped.set() - if not producer_task.done(): + if not detached and not producer_task.done(): await producer_task if not completed: await self._store.complete_scan_scope_discovery( @@ -320,9 +521,9 @@ def onerror(error: OSError) -> None: scope.root_id, scope.relative_path, state="partially_read", - error_code="ROOT_PERMISSION_DENIED", + error_code=walk_failure_code or "ROOT_PERMISSION_DENIED", ) - return current, completed + return current, completed, walk_failure_code async def _persist_batch( self, diff --git a/backend/services/native/library_policy_resolver.py b/backend/services/native/library_policy_resolver.py index a5da9456f..4fe2afec9 100644 --- a/backend/services/native/library_policy_resolver.py +++ b/backend/services/native/library_policy_resolver.py @@ -181,6 +181,7 @@ def _normalise_and_validate( staging_path=str(staging) if staging is not None else "", naming_template=settings.naming_template, acoustid_api_key=settings.acoustid_api_key, + enabled=settings.enabled, ), warnings, ) diff --git a/backend/services/native/library_policy_service.py b/backend/services/native/library_policy_service.py index f267541b2..0164077a1 100644 --- a/backend/services/native/library_policy_service.py +++ b/backend/services/native/library_policy_service.py @@ -274,6 +274,7 @@ def _settings_response( staging_path=settings.staging_path, naming_template=settings.naming_template, acoustid_api_key=settings.acoustid_api_key, + enabled=settings.enabled, policy_revision=resolver.policy_revision, reconciliation_required=reconciliation_required, reconciliation_state=( diff --git a/backend/services/native/library_review_service.py b/backend/services/native/library_review_service.py index 6b9c3c14e..63317e377 100644 --- a/backend/services/native/library_review_service.py +++ b/backend/services/native/library_review_service.py @@ -640,8 +640,12 @@ def _available_actions( evidence: CandidateEvidence | None, ) -> list[str]: if review.state == "excluded" or review.effective_policy == "excluded": - return ["restore"] if review.exclusion_source == "item_decision" else [] - actions = ["exclude", "retry"] + return ( + ["restore", "dismiss"] + if review.exclusion_source == "item_decision" + else ["dismiss"] + ) + actions = ["exclude", "retry", "dismiss"] if identity is None: actions.append("keep_tagged") else: diff --git a/backend/services/native/library_scan_coordinator.py b/backend/services/native/library_scan_coordinator.py index 6377793da..198fcecbe 100644 --- a/backend/services/native/library_scan_coordinator.py +++ b/backend/services/native/library_scan_coordinator.py @@ -97,6 +97,11 @@ def _log_progress(self, run: ScanRun, event: str, *, force: bool = False) -> Non ) async def request_run(self, request: ScanRequest) -> ScanRequestResult: + if not self._resolver_getter().settings.enabled: + raise ValidationError( + "The local library is disabled. Enable it in Settings → Library " + "before starting a scan." + ) if not request.scopes: raise ValidationError("Select at least one library scope.") if any( @@ -179,6 +184,8 @@ async def control( ) async def recover(self) -> list[ScanRun]: + if not self._resolver_getter().settings.enabled: + return [] runs = await self._store.recover_scan_runs(now=self._clock()) self._pending_control_run_ids.clear() for run in runs: @@ -257,6 +264,8 @@ async def checkpoint(self, run_id: str, frozen_policy_revision: str) -> bool: return run.state in {"discovering", "indexing", "reconciling"} async def run_once(self, root_paths: dict[str, Path]) -> ScanRun | None: + if not self._resolver_getter().settings.enabled: + return None await self._store.cleanup_terminal_scan_inventory(limit=5_000) run = await self._store.get_resumable_scan_run() newly_claimed = run is None diff --git a/backend/services/native/library_scan_supervisor.py b/backend/services/native/library_scan_supervisor.py index a75bea9e8..3eb86a7b1 100644 --- a/backend/services/native/library_scan_supervisor.py +++ b/backend/services/native/library_scan_supervisor.py @@ -69,7 +69,10 @@ async def supervise_target_scans( ) -> None: wakeups = work_wakeups or DurableWorkWakeups() try: - await coordinator_getter().recover() + # None resolver getter means the scheduler is not wired up; the library + # is treated as enabled so plain supervisor deployments keep working. + if resolver_getter is None or resolver_getter().settings.enabled: + await coordinator_getter().recover() except asyncio.CancelledError: return except Exception: # noqa: BLE001 - startup recovery failure must not kill the supervisor @@ -80,21 +83,25 @@ async def supervise_target_scans( wait_seconds = EMPTY_RECOVERY_INTERVAL_SECONDS try: coordinator = coordinator_getter() + resolver = resolver_getter() if resolver_getter is not None else None + enabled = resolver is None or resolver.settings.enabled if ( - scheduler_getter is not None - and resolver_getter is not None + enabled + and scheduler_getter is not None + and resolver is not None and schedule_settings_getter is not None ): schedule = schedule_settings_getter() await scheduler_getter().tick( coordinator, - resolver_getter(), + resolver, frequency=schedule["frequency"], daily_time=schedule["daily_time"], timezone_name=schedule["timezone_name"], now=now_getter(), ) - processed = await coordinator.run_once(root_paths_getter()) is not None + if enabled: + processed = await coordinator.run_once(root_paths_getter()) is not None except asyncio.CancelledError: break except Exception: # noqa: BLE001 - the lifetime supervisor records and survives run failures diff --git a/backend/services/native/target_application_lifecycle.py b/backend/services/native/target_application_lifecycle.py index 44228c778..6e6d11540 100644 --- a/backend/services/native/target_application_lifecycle.py +++ b/backend/services/native/target_application_lifecycle.py @@ -127,7 +127,11 @@ async def _migrate_global_connections( async def run_target_one_time_migrations( - *, auth_store: Any, preferences: Any, cache_dir: Path + *, + auth_store: Any, + preferences: Any, + cache_dir: Path, + library_enabled: bool = True, ) -> None: """Run target data ratchets without touching the retained legacy catalog.""" @@ -161,23 +165,29 @@ async def run_target_one_time_migrations( ) if release_year_count: logger.info("Backfilled %d accepted release catalog years", release_year_count) - from core.dependencies.service_providers import ( - get_artist_identity_reconciliation_service, - get_catalog_identity_hygiene_service, - ) + if not library_enabled: + logger.info( + "Skipped catalog hygiene and artist reconciliation backfills: " + "the local library is disabled" + ) + else: + from core.dependencies.service_providers import ( + get_artist_identity_reconciliation_service, + get_catalog_identity_hygiene_service, + ) - hygiene_job = await get_catalog_identity_hygiene_service().enqueue_backfill() - logger.info( - "Queued bounded catalog identity hygiene backfill %s", - hygiene_job["id"], - ) - reconciliation_job = ( - await get_artist_identity_reconciliation_service().enqueue_backfill() - ) - logger.info( - "Queued bounded artist identity reconciliation backfill %s", - reconciliation_job["id"], - ) + hygiene_job = await get_catalog_identity_hygiene_service().enqueue_backfill() + logger.info( + "Queued bounded catalog identity hygiene backfill %s", + hygiene_job["id"], + ) + reconciliation_job = ( + await get_artist_identity_reconciliation_service().enqueue_backfill() + ) + logger.info( + "Queued bounded artist identity reconciliation backfill %s", + reconciliation_job["id"], + ) await auth_store.backfill_usernames() await auth_store.migrate_local_provider_to_username() await _migrate_shared_avatar(auth_store, cache_dir) diff --git a/backend/services/native/target_application_runtime.py b/backend/services/native/target_application_runtime.py index 1b2835931..86451f770 100644 --- a/backend/services/native/target_application_runtime.py +++ b/backend/services/native/target_application_runtime.py @@ -6,10 +6,12 @@ import logging import os import socket -from collections.abc import Callable +import time +from collections.abc import Awaitable, Callable from core.task_registry import TaskRegistry from infrastructure.queue.durable_work_wakeup import DurableWorkWakeups +from infrastructure.resilience.retry import CircuitState from services.native.album_identification_service import AlbumIdentificationService from services.native.identification_queue_service import IdentificationQueueService from services.native.library_operation_supervisor import LibraryOperationSupervisor @@ -28,6 +30,13 @@ OPERATION_RECOVERY_INTERVAL_SECONDS = 37.0 CONTRIBUTION_RECOVERY_INTERVAL_SECONDS = 43.0 ERROR_RETRY_INTERVAL_SECONDS = 1.0 +PROVIDER_HEALTH_SWEEP_INTERVAL_SECONDS = 60.0 +WATCHDOG_INTERVAL_SECONDS = 30.0 + +IDENTIFICATION_WORKER_TASK_NAME = "target-library-identification-worker" +OPERATION_WORKER_TASK_NAME = "target-library-operation-worker" +CONTRIBUTION_VERIFICATION_WORKER_TASK_NAME = "library-contribution-verification-worker" +TARGET_WORKER_WATCHDOG_TASK_NAME = "target-worker-watchdog" def _worker_id(kind: str) -> str: @@ -53,19 +62,25 @@ async def run_target_identification_worker( worker_id: str | None = None, workload_gate: BackgroundWorkloadGate | None = None, work_wakeups: DurableWorkWakeups | None = None, + provider_state_getter: Callable[[], CircuitState] | None = None, + probe_provider: Callable[[], Awaitable[None]] | None = None, + enabled_getter: Callable[[], bool] | None = None, ) -> None: owner = worker_id or _worker_id("identification") wakeups = work_wakeups or DurableWorkWakeups() + last_provider_sweep_at = 0.0 while True: revision = wakeups.revision("identification") processed = False wait_seconds = IDENTIFICATION_RECOVERY_INTERVAL_SECONDS + queue = queue_getter() + job: dict | None = None try: - if workload_gate is None or not workload_gate.scan_active: - queue = queue_getter() + if (enabled_getter is None or enabled_getter()) and ( + workload_gate is None or not workload_gate.scan_active + ): await queue.recover() if not await queue.is_paused(): - job = None if workload_gate is None or not workload_gate.scan_active: job = await queue.claim(owner) if job is not None: @@ -75,9 +90,32 @@ async def run_target_identification_worker( break except Exception: # noqa: BLE001 - a durable worker must survive one failed item logger.exception("Target identification worker iteration failed") + if job is not None: + try: + await queue.defer(job, owner, "UNEXPECTED_ERROR") + except Exception: # noqa: BLE001 - a crashed job must not kill the worker + logger.exception("Failed to defer crashed identification job") wait_seconds = ERROR_RETRY_INTERVAL_SECONDS if processed: continue + if provider_state_getter is not None: + now = time.time() + if now - last_provider_sweep_at >= PROVIDER_HEALTH_SWEEP_INTERVAL_SECONDS: + last_provider_sweep_at = now + try: + state = provider_state_getter() + if state is CircuitState.CLOSED: + # Provider recovered: immediately release provider-deferred + # jobs instead of waiting out up to 6h of stale backoff. + await queue_getter().reset_provider_deferrals(now=now) + elif state is CircuitState.HALF_OPEN and probe_provider is not None: + # Without traffic the breaker would sit HALF_OPEN forever; + # one bounded background probe resolves it either way. + await probe_provider() + except asyncio.CancelledError: + break + except Exception: # noqa: BLE001 - a failed probe must not kill the worker + logger.exception("Identification provider health sweep failed") try: await wakeups.wait( "identification", @@ -94,6 +132,7 @@ async def run_target_operation_worker( *, worker_id: str | None = None, work_wakeups: DurableWorkWakeups | None = None, + enabled_getter: Callable[[], bool] | None = None, ) -> None: owner = worker_id or _worker_id("operation") wakeups = work_wakeups or DurableWorkWakeups() @@ -102,11 +141,12 @@ async def run_target_operation_worker( processed = False wait_seconds = OPERATION_RECOVERY_INTERVAL_SECONDS try: - supervisor = supervisor_getter() - await supervisor.recover() - if recovery_getter is not None: - await recovery_getter().recover_once() - processed = await supervisor.run_once(owner) is not None + if enabled_getter is None or enabled_getter(): + supervisor = supervisor_getter() + await supervisor.recover() + if recovery_getter is not None: + await recovery_getter().recover_once() + processed = await supervisor.run_once(owner) is not None except asyncio.CancelledError: break except Exception: # noqa: BLE001 - a durable worker must survive one failed item @@ -159,19 +199,48 @@ async def run_library_contribution_verification_worker( break +async def run_target_worker_watchdog( + starters: dict[str, Callable[[], asyncio.Task[None]]], + *, + interval_seconds: float = WATCHDOG_INTERVAL_SECONDS, +) -> None: + """Restart target workers whose tasks died (crashes auto-unregister them).""" + registry = TaskRegistry.get_instance() + while True: + try: + for name, starter in starters.items(): + if not registry.is_running(name): + logger.warning("Restarting stopped target worker %s", name) + starter() + except asyncio.CancelledError: + break + except Exception: # noqa: BLE001 - the supervisor must survive a failed restart + logger.exception("Target worker watchdog iteration failed") + try: + await asyncio.sleep(interval_seconds) + except asyncio.CancelledError: + break + + def start_target_identification_worker( queue_getter: Callable[[], IdentificationQueueService], service_getter: Callable[[], AlbumIdentificationService], work_wakeups: DurableWorkWakeups, workload_gate: BackgroundWorkloadGate | None = None, + provider_state_getter: Callable[[], CircuitState] | None = None, + probe_provider: Callable[[], Awaitable[None]] | None = None, + enabled_getter: Callable[[], bool] | None = None, ) -> asyncio.Task[None]: - name = "target-library-identification-worker" + name = IDENTIFICATION_WORKER_TASK_NAME task = asyncio.create_task( run_target_identification_worker( queue_getter, service_getter, workload_gate=workload_gate, work_wakeups=work_wakeups, + provider_state_getter=provider_state_getter, + probe_provider=probe_provider, + enabled_getter=enabled_getter, ) ) TaskRegistry.get_instance().register(name, task) @@ -183,11 +252,15 @@ def start_target_operation_worker( supervisor_getter: Callable[[], LibraryOperationSupervisor], work_wakeups: DurableWorkWakeups, recovery_getter: Callable[[], LibraryManagementRecoveryService] | None = None, + enabled_getter: Callable[[], bool] | None = None, ) -> asyncio.Task[None]: - name = "target-library-operation-worker" + name = OPERATION_WORKER_TASK_NAME task = asyncio.create_task( run_target_operation_worker( - supervisor_getter, recovery_getter, work_wakeups=work_wakeups + supervisor_getter, + recovery_getter, + work_wakeups=work_wakeups, + enabled_getter=enabled_getter, ) ) TaskRegistry.get_instance().register(name, task) @@ -199,7 +272,7 @@ def start_library_contribution_verification_worker( worker_getter: Callable[[], LibraryContributionVerificationWorker], work_wakeups: DurableWorkWakeups, ) -> asyncio.Task[None]: - name = "library-contribution-verification-worker" + name = CONTRIBUTION_VERIFICATION_WORKER_TASK_NAME task = asyncio.create_task( run_library_contribution_verification_worker( worker_getter, work_wakeups=work_wakeups @@ -208,3 +281,13 @@ def start_library_contribution_verification_worker( TaskRegistry.get_instance().register(name, task) task.add_done_callback(lambda item: _log_worker_error(item, name=name)) return task + + +def start_target_worker_watchdog( + starters: dict[str, Callable[[], asyncio.Task[None]]], +) -> asyncio.Task[None]: + name = TARGET_WORKER_WATCHDOG_TASK_NAME + task = asyncio.create_task(run_target_worker_watchdog(starters)) + TaskRegistry.get_instance().register(name, task) + task.add_done_callback(lambda item: _log_worker_error(item, name=name)) + return task diff --git a/backend/services/native/target_library_policy_service.py b/backend/services/native/target_library_policy_service.py index 326937911..c6f87f8a4 100644 --- a/backend/services/native/target_library_policy_service.py +++ b/backend/services/native/target_library_policy_service.py @@ -301,6 +301,7 @@ async def restore_roots( staging_path=current.staging_path, naming_template=current.naming_template, acoustid_api_key=current.acoustid_api_key, + enabled=current.enabled, ), expected_policy_revision=request.expected_policy_revision, ) diff --git a/backend/services/navidrome_playback_service.py b/backend/services/navidrome_playback_service.py index 93e195736..f0af1796b 100644 --- a/backend/services/navidrome_playback_service.py +++ b/backend/services/navidrome_playback_service.py @@ -44,16 +44,18 @@ async def _repo_for(self, user_id: str | None) -> NavidromeRepositoryProtocol: def get_stream_url(self, song_id: str) -> str: return self._navidrome.build_stream_url(song_id) - async def proxy_head(self, item_id: str) -> Response: + async def proxy_head(self, item_id: str, user_id: str | None = None) -> Response: """Proxy a HEAD request to Navidrome and return a FastAPI Response.""" - result: StreamProxyResult = await self._navidrome.proxy_head_stream(item_id) + repo = await self._repo_for(user_id) + result: StreamProxyResult = await repo.proxy_head_stream(item_id) return Response(status_code=200, headers=result.headers) async def proxy_stream( - self, item_id: str, range_header: str | None = None + self, item_id: str, range_header: str | None = None, user_id: str | None = None ) -> StreamingResponse: """Proxy a GET stream from Navidrome and return a FastAPI StreamingResponse.""" - result: StreamProxyResult = await self._navidrome.proxy_get_stream( + repo = await self._repo_for(user_id) + result: StreamProxyResult = await repo.proxy_get_stream( item_id, range_header=range_header ) return StreamingResponse( diff --git a/backend/services/now_playing_poller.py b/backend/services/now_playing_poller.py index 5dc8fc099..88e794b58 100644 --- a/backend/services/now_playing_poller.py +++ b/backend/services/now_playing_poller.py @@ -147,7 +147,7 @@ async def run_now_playing_presence_loop( plex_service_getter(), ) except asyncio.CancelledError: - raise + break except Exception as e: # noqa: BLE001 logger.warning("now-playing presence loop cycle failed: %s", e) await asyncio.sleep(interval) diff --git a/backend/services/playlist_service.py b/backend/services/playlist_service.py index 77e594166..1c77ee0a5 100644 --- a/backend/services/playlist_service.py +++ b/backend/services/playlist_service.py @@ -801,9 +801,19 @@ async def _resolve_album_sources( nd_by_num: dict[tuple[int, int], tuple[str, str]] = {} plex_by_num: dict[tuple[int, int], tuple[str, str, str]] = {} + # Pre-fix Jellyfin imports stored the Jellyfin album GUID as album_id, + # which the MBID-keyed Jellyfin/local lookups can never match. Re-key it + # via the album's provider ids so legacy rows resolve without a + # migration. The cache key above stays the original album_id. + match_album_id = album_id if jf_service is not None: try: match = await jf_service.match_album_by_mbid(album_id) + if not match.found: + mbid = await jf_service.resolve_album_mbid(album_id) + if isinstance(mbid, str) and mbid and mbid != album_id: + match_album_id = mbid + match = await jf_service.match_album_by_mbid(mbid) if match.found: for t in match.tracks: key = _safe_track_number(t.track_number) @@ -821,7 +831,7 @@ async def _resolve_album_sources( if local_service is not None: try: - match = await local_service.match_album_by_mbid(album_id) + match = await local_service.match_album_by_mbid(match_album_id) if match.found: for t in match.tracks: key = _safe_track_number(t.track_number) diff --git a/backend/services/plex_playback_service.py b/backend/services/plex_playback_service.py index 32bdbf803..0ec2e6b3a 100644 --- a/backend/services/plex_playback_service.py +++ b/backend/services/plex_playback_service.py @@ -36,14 +36,16 @@ async def _repo_for(self, user_id: str | None) -> PlexRepositoryProtocol: return per_user return self._plex - async def proxy_head(self, part_key: str) -> Response: - result: StreamProxyResult = await self._plex.proxy_head_stream(part_key) + async def proxy_head(self, part_key: str, user_id: str | None = None) -> Response: + repo = await self._repo_for(user_id) + result: StreamProxyResult = await repo.proxy_head_stream(part_key) return Response(status_code=result.status_code, headers=result.headers) async def proxy_stream( - self, part_key: str, range_header: str | None = None + self, part_key: str, range_header: str | None = None, user_id: str | None = None ) -> StreamingResponse: - result: StreamProxyResult = await self._plex.proxy_get_stream( + repo = await self._repo_for(user_id) + result: StreamProxyResult = await repo.proxy_get_stream( part_key, range_header=range_header ) return StreamingResponse( diff --git a/backend/services/preferences_service.py b/backend/services/preferences_service.py index a8f186dbc..ee3f45efb 100644 --- a/backend/services/preferences_service.py +++ b/backend/services/preferences_service.py @@ -1237,6 +1237,7 @@ def get_typed_library_settings(self) -> TypedLibrarySettings: staging_path=settings.staging_path, naming_template=settings.naming_template, acoustid_api_key=ACOUSTID_KEY_MASK if settings.acoustid_api_key else "", + enabled=settings.enabled, ) def get_typed_library_settings_raw(self) -> TypedLibrarySettings: @@ -1249,6 +1250,7 @@ def get_typed_library_settings_raw(self) -> TypedLibrarySettings: staging_path=settings.staging_path, naming_template=settings.naming_template, acoustid_api_key=api_key, + enabled=settings.enabled, ) def get_library_settings(self) -> LibrarySettings: @@ -1340,6 +1342,7 @@ def retarget_library_roots_for_upgrade(self, replacements: dict[str, str]) -> No staging_path=current.staging_path, naming_template=current.naming_template, acoustid_api_key=current.acoustid_api_key, + enabled=current.enabled, ), allow_root_path_changes=True, ) @@ -1416,6 +1419,7 @@ def _save_typed_library_settings( naming_template=normalized.naming_template or DEFAULT_NAMING_TEMPLATE, acoustid_api_key=api_key, + enabled=normalized.enabled, ), ) except ConfigurationError: diff --git a/backend/services/settings_service.py b/backend/services/settings_service.py index 21559a53e..7b107066f 100644 --- a/backend/services/settings_service.py +++ b/backend/services/settings_service.py @@ -647,6 +647,7 @@ async def on_musicbrainz_settings_changed( self, settings: MusicBrainzConnectionSettings ) -> None: from repositories.musicbrainz_base import ( + get_mb_api_base, set_mb_api_base, mb_rate_limiter, mb_circuit_breaker, @@ -664,6 +665,19 @@ async def on_musicbrainz_settings_changed( settings.concurrent_searches, _OFFICIAL_MB_CONCURRENT_SEARCHES ) + # Compare against live module state, not stored settings: the route + # saves before calling this handler, so stored == incoming always. + if ( + get_mb_api_base() == settings.api_url + and mb_rate_limiter.rate == settings.rate_limit + and mb_rate_limiter.capacity == settings.concurrent_searches + ): + logger.info( + "MusicBrainz connection settings unchanged; " + "skipping circuit breaker reset and cache clear" + ) + return + set_mb_api_base(settings.api_url) mb_rate_limiter.update_rate(settings.rate_limit) mb_rate_limiter.update_capacity(settings.concurrent_searches) diff --git a/backend/target_application.py b/backend/target_application.py index 7f4b46581..aab226798 100644 --- a/backend/target_application.py +++ b/backend/target_application.py @@ -8,6 +8,7 @@ import logging import os +import asyncio from pathlib import Path from zoneinfo import ZoneInfo, ZoneInfoNotFoundError @@ -89,6 +90,7 @@ get_discovery_batch_service, get_home_service, get_home_charts_service, + get_legacy_pending_migration_service, get_local_files_service, get_navidrome_library_service, get_plex_library_service, @@ -187,7 +189,7 @@ StaleRevisionError, ValidationError, ) -from infrastructure.resilience.retry import CircuitOpenError +from infrastructure.resilience.retry import CircuitOpenError, CircuitState from core.task_registry import TaskRegistry from core.tasks import ( start_cache_cleanup_task, @@ -205,9 +207,14 @@ ) from services.native.library_scan_supervisor import start_target_scan_supervisor from services.native.target_application_runtime import ( + CONTRIBUTION_VERIFICATION_WORKER_TASK_NAME, + IDENTIFICATION_WORKER_TASK_NAME, + OPERATION_WORKER_TASK_NAME, + TARGET_WORKER_WATCHDOG_TASK_NAME, start_library_contribution_verification_worker, start_target_identification_worker, start_target_operation_worker, + start_target_worker_watchdog, ) from services.native.target_application_lifecycle import ( run_target_one_time_migrations, @@ -555,6 +562,7 @@ async def production_target_lifespan(app: FastAPI): auth_store=auth_store, preferences=preferences, cache_dir=settings.cache_dir, + library_enabled=preferences.get_typed_library_settings().enabled, ) logger.info("target_startup.data_ratchets_completed") async with target_startup_progress(settings, "management_recovery"): @@ -603,6 +611,10 @@ def schedule_settings() -> dict[str, str]: } work_wakeups = get_native_library_store().work_wakeups + + def library_enabled() -> bool: + return get_preferences_service().get_typed_library_settings().enabled + start_target_scan_supervisor( get_target_library_scan_coordinator, root_paths, @@ -611,35 +623,78 @@ def schedule_settings() -> dict[str, str]: resolver_getter=get_library_policy_resolver, schedule_settings_getter=schedule_settings, ) - start_target_identification_worker( - get_target_identification_queue, - get_target_album_identification_service, - work_wakeups, - workload_gate=get_background_workload_gate(), - ) - start_target_operation_worker( - get_target_library_operation_supervisor, - work_wakeups, - recovery_getter=get_library_management_recovery_service, - ) - start_library_contribution_verification_worker( - get_library_contribution_verification_worker, - work_wakeups, - ) + + def mb_provider_state() -> CircuitState: + from repositories.musicbrainz_base import mb_circuit_breaker + + return mb_circuit_breaker.state + + async def probe_mb_provider() -> None: + # Thin background-priority probe mirroring verify_musicbrainz; the + # breaker records the outcome, resolving HALF_OPEN without traffic. + from infrastructure.queue.priority_queue import RequestPriority + from repositories.musicbrainz_base import mb_api_get + + await mb_api_get( + "/artist", + params={"query": "test", "limit": 1}, + priority=RequestPriority.BACKGROUND_SYNC, + ) + + def start_identification_worker() -> asyncio.Task[None]: + return start_target_identification_worker( + get_target_identification_queue, + get_target_album_identification_service, + work_wakeups, + workload_gate=get_background_workload_gate(), + provider_state_getter=mb_provider_state, + probe_provider=probe_mb_provider, + enabled_getter=library_enabled, + ) + + def start_operation_worker() -> asyncio.Task[None]: + return start_target_operation_worker( + get_target_library_operation_supervisor, + work_wakeups, + recovery_getter=get_library_management_recovery_service, + enabled_getter=library_enabled, + ) + + def start_contribution_worker() -> asyncio.Task[None]: + return start_library_contribution_verification_worker( + get_library_contribution_verification_worker, + work_wakeups, + ) + + worker_starters = { + IDENTIFICATION_WORKER_TASK_NAME: start_identification_worker, + OPERATION_WORKER_TASK_NAME: start_operation_worker, + CONTRIBUTION_VERIFICATION_WORKER_TASK_NAME: start_contribution_worker, + } + for start_worker in worker_starters.values(): + start_worker() + start_target_worker_watchdog(worker_starters) await start_target_operational_runtime( settings=settings, preferences=preferences, auth_store=auth_store, ) + if library_enabled(): + try: + await get_legacy_pending_migration_service().schedule() + except Exception: # noqa: BLE001 + logger.exception("Legacy pending migration scheduling failed") logger.info("target_startup.operational_runtime_started") logger.info("DroppedNeedle target application started") try: yield finally: - await TaskRegistry.get_instance().cancel_all( - grace_period=settings.shutdown_grace_period - ) + registry = TaskRegistry.get_instance() + # Cancel the watchdog before the snapshot+cancel_all pass so it cannot + # restart a worker mid-shutdown and orphan the task. + await registry.cancel(TARGET_WORKER_WATCHDOG_TASK_NAME) + await registry.cancel_all(grace_period=settings.shutdown_grace_period) await cleanup_app_state( queue_manager_getter=get_target_discover_queue_manager, genre_prewarm_getter=get_target_genre_cover_prewarm_service, diff --git a/backend/tests/infrastructure/test_automatic_upgrade.py b/backend/tests/infrastructure/test_automatic_upgrade.py index 3546de127..fbb40cec8 100644 --- a/backend/tests/infrastructure/test_automatic_upgrade.py +++ b/backend/tests/infrastructure/test_automatic_upgrade.py @@ -188,6 +188,8 @@ def fail(_working: Path) -> dict[str, object]: ) ) assert state["failure_evidence"] == evidence + assert state["error_type"] == "_WorkingMigrationError" + assert state["error_message"] == "checked failure" assert "source_key" not in json.dumps(state["failure_evidence"]) @@ -1290,9 +1292,7 @@ def progress(_path: Path, _token: str) -> dict[str, object]: lambda *_args, **_kwargs: StalledProcess(), ) monkeypatch.setattr(automatic_upgrade, "_target_progress", progress) - monkeypatch.setattr( - automatic_upgrade, "_TARGET_STARTUP_HARD_TIMEOUT_SECONDS", 0.03 - ) + monkeypatch.setattr(automatic_upgrade, "_TARGET_STARTUP_HARD_TIMEOUT_SECONDS", 0.03) assert ( run_target_supervisor( @@ -1642,3 +1642,135 @@ def fail(_settings: Settings, **_kwargs: object) -> str: assert automatic_upgrade.main() == 1 assert not settings.config_file_path.exists() + + +def _failed_replace(_source: object, _destination: object) -> None: + raise OSError("rename unsupported") + + +def test_replace_file_falls_back_to_copy_when_rename_unavailable( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "source.txt" + destination = tmp_path / "nested" / "destination.txt" + source.write_bytes(b"published-content") + monkeypatch.setattr(automatic_upgrade.os, "replace", _failed_replace) + + automatic_upgrade._replace_file(source, destination) + + assert destination.read_bytes() == b"published-content" + + +def test_replace_database_falls_back_to_sqlite_copy_when_rename_unavailable( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "source.db" + destination = tmp_path / "destination.db" + _write_unmigrated_database(source) + _mark_migrated(source) + _write_unmigrated_database(destination, value="outdated") + monkeypatch.setattr(automatic_upgrade.os, "replace", _failed_replace) + + automatic_upgrade._replace_database(source, destination) + + assert automatic_upgrade._database_has_marker(destination) + assert _source_value(destination) == "original" + assert not Path(f"{destination}-wal").exists() + assert not Path(f"{destination}-shm").exists() + + +def test_replace_file_retries_transient_stale_read( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "source.txt" + destination = tmp_path / "destination.txt" + source.write_bytes(b"published-content") + real_sha256 = automatic_upgrade._sha256 + destination_reads = 0 + + def flaky_sha256(path: Path) -> str | None: + nonlocal destination_reads + if Path(path) == destination: + destination_reads += 1 + if destination_reads == 1: + return "stale" + return real_sha256(path) + + monkeypatch.setattr(automatic_upgrade, "_sha256", flaky_sha256) + monkeypatch.setattr(automatic_upgrade, "_PUBLISH_VERIFY_INTERVAL_SECONDS", 0) + + automatic_upgrade._replace_file(source, destination) + + assert destination.read_bytes() == b"published-content" + assert destination_reads > 1 + + +def test_replace_database_raises_when_content_never_verifies( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "source.db" + destination = tmp_path / "destination.db" + _write_unmigrated_database(source) + _mark_migrated(source) + _write_unmigrated_database(destination, value="outdated") + real_sha256 = automatic_upgrade._sha256 + + def wrong_destination_hash(path: Path) -> str | None: + if Path(path) == destination: + return "wrong" + return real_sha256(path) + + monkeypatch.setattr(automatic_upgrade, "_sha256", wrong_destination_hash) + monkeypatch.setattr(automatic_upgrade, "_PUBLISH_VERIFY_INTERVAL_SECONDS", 0) + + with pytest.raises(OSError, match="could not be verified"): + automatic_upgrade._replace_database(source, destination) + + assert automatic_upgrade._database_has_marker(destination) + assert _source_value(destination) == "original" + + +def test_upgrade_completes_when_atomic_rename_is_unavailable( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + settings = _settings(tmp_path) + _write_unmigrated_database(settings.library_db_path) + settings.config_file_path.parent.mkdir(parents=True) + settings.config_file_path.write_text('{"name":"before"}', encoding="utf-8") + monkeypatch.setenv("COMMIT_TAG", "test-version") + + def migrate(working: Path) -> dict[str, object]: + working_database = working / "cache" / "library.db" + with sqlite3.connect(working_database) as connection: + connection.execute("UPDATE source_value SET value = 'migrated'") + (working / "config" / "config.json").write_text( + '{"name":"after"}', encoding="utf-8" + ) + _mark_migrated(working_database) + return {"passed": True} + + monkeypatch.setattr(automatic_upgrade.os, "replace", _failed_replace) + result = run_automatic_copy_upgrade(settings, runner=migrate) + + assert result == "upgraded" + assert automatic_upgrade._database_has_marker(settings.library_db_path) + assert _source_value(settings.library_db_path) == "migrated" + state = json.loads( + (settings.cache_dir / f"automatic-upgrade-{UPGRADE_ID}.json").read_text( + encoding="utf-8" + ) + ) + assert state["stage"] == "completed" + assert settings.config_file_path.read_text(encoding="utf-8") == '{"name":"after"}' + + +def test_write_state_falls_back_when_rename_unavailable( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "state.json" + payload = {"stage": "completed", "attempt": 1} + monkeypatch.setattr(automatic_upgrade.os, "replace", _failed_replace) + + automatic_upgrade._write_state(path, payload) + + assert automatic_upgrade._read_state(path) == payload diff --git a/backend/tests/infrastructure/test_container_umask.py b/backend/tests/infrastructure/test_container_umask.py index 490fad1a4..c50c00731 100644 --- a/backend/tests/infrastructure/test_container_umask.py +++ b/backend/tests/infrastructure/test_container_umask.py @@ -39,6 +39,56 @@ def test_entrypoint_rejects_invalid_umask_before_startup(value: str) -> None: assert "must be three or four octal digits" in result.stdout +def test_entrypoint_fails_fast_when_app_is_shadowed(tmp_path: Path) -> None: + """A bind mount over /app (code files missing from an existing /app-like + directory) must fail with the actionable message before any other init + step, while a missing directory (outside the container) must not.""" + entrypoint = REPOSITORY_ROOT / "entrypoint.sh" + script = entrypoint.read_text() + + shadowed_root = tmp_path / "app" + shadowed_root.mkdir() + relocated = script.replace("/app", str(shadowed_root)) + script_file = tmp_path / "entrypoint.sh" + script_file.write_text(relocated) + + result = subprocess.run( + ["sh", str(script_file), "true"], + check=False, + capture_output=True, + text=True, + env={"UMASK": "888"}, + ) + + assert result.returncode == 1 + assert "does not contain the DroppedNeedle application code" in result.stdout + assert "Mount data subdirectories only" in result.stdout + + +def test_entrypoint_skips_shadow_check_when_app_missing(tmp_path: Path) -> None: + """Without an /app directory the shadow check is skipped (host/test + environment) and normal init validation runs instead.""" + entrypoint = REPOSITORY_ROOT / "entrypoint.sh" + script = entrypoint.read_text() + missing_root = tmp_path / "no-such-app" + assert not missing_root.exists() + relocated = script.replace("/app", str(missing_root)) + script_file = tmp_path / "entrypoint.sh" + script_file.write_text(relocated) + + result = subprocess.run( + ["sh", str(script_file), "true"], + check=False, + capture_output=True, + text=True, + env={"UMASK": "888"}, + ) + + assert result.returncode == 1 + assert "must be three or four octal digits" in result.stdout + assert "does not contain the DroppedNeedle application code" not in result.stdout + + def test_unraid_template_uses_the_secure_default() -> None: root = ElementTree.parse(REPOSITORY_ROOT / "templates/droppedneedle.xml").getroot() setting = next( diff --git a/backend/tests/infrastructure/test_native_library_store.py b/backend/tests/infrastructure/test_native_library_store.py index 50c639ad2..3686c8a67 100644 --- a/backend/tests/infrastructure/test_native_library_store.py +++ b/backend/tests/infrastructure/test_native_library_store.py @@ -33,6 +33,7 @@ OperationWorkItem, RepairFinding, ReviewDecision, + ScanFailureRecord, ScanInventoryItem, ScanRun, ScanScope, @@ -601,6 +602,7 @@ async def test_schema_is_idempotent_and_contains_complete_target_surface( "library_policy_transitions", "library_scan_runs", "library_scan_inventory", + "library_scan_failures", "library_scan_management_candidates", "library_scan_management_staging", "library_scan_grouping_contexts", @@ -657,6 +659,131 @@ def test_scan_management_candidate_schema_upgrade_is_idempotent( } <= indexes +@pytest.mark.asyncio +async def test_scan_failures_round_trip_pagination_and_dedupe( + store: NativeLibraryStore, +) -> None: + await store.create_scan_run( + ScanRun(id="scan-fail", kind="incremental", trigger="manual", queued_at=1) + ) + records = [ + ScanFailureRecord( + root_id="root-1", + relative_path=f"dir-{ordinal}", + failure_code="WALK_EACCES", + recorded_at=100 + ordinal, + failure_detail=f"detail {ordinal}", + phase="discovering", + ) + for ordinal in range(3) + ] + await store.record_scan_failures("scan-fail", records) + await store.record_scan_failures("scan-fail", [records[0]]) + + first_page, cursor = await store.list_scan_run_failures("scan-fail", limit=2) + + assert [item.relative_path for item in first_page] == ["dir-0", "dir-1"] + assert first_page[0].failure_code == "WALK_EACCES" + assert first_page[0].failure_detail == "detail 0" + assert first_page[0].phase == "discovering" + assert first_page[0].recorded_at == 100 + assert cursor is not None + + second_page, next_cursor = await store.list_scan_run_failures( + "scan-fail", limit=2, cursor_rowid=cursor + ) + assert [item.relative_path for item in second_page] == ["dir-2"] + assert next_cursor is None + + +@pytest.mark.asyncio +async def test_scan_failures_cascade_with_their_run( + store: NativeLibraryStore, db_path: Path +) -> None: + await store.create_scan_run( + ScanRun(id="scan-cascade", kind="incremental", trigger="manual", queued_at=1) + ) + await store.record_scan_failures( + "scan-cascade", + [ + ScanFailureRecord( + root_id="root-1", + relative_path="dir", + failure_code="WALK_TIMEOUT", + recorded_at=100, + ) + ], + ) + + with sqlite3.connect(db_path) as connection: + connection.execute("PRAGMA foreign_keys = ON") + connection.execute("DELETE FROM library_scan_runs WHERE id = 'scan-cascade'") + remaining = connection.execute( + "SELECT COUNT(*) FROM library_scan_failures" + ).fetchone()[0] + + assert remaining == 0 + + +@pytest.mark.asyncio +async def test_commit_scan_index_batch_records_failure_rows( + store: NativeLibraryStore, +) -> None: + await store.create_scan_run( + ScanRun(id="scan-index", kind="incremental", trigger="manual", queued_at=1) + ) + + await store.commit_scan_index_batch( + "scan-index", + writes=[], + states={}, + failures=[("root-1", "broken.flac", "TAG_READ_TIMEOUT")], + increments={}, + updated_at=2.0, + ) + + items, next_cursor = await store.list_scan_run_failures("scan-index") + assert next_cursor is None + assert [ + (item.root_id, item.relative_path, item.failure_code, item.phase) + for item in items + ] == [("root-1", "broken.flac", "TAG_READ_TIMEOUT", "indexing")] + assert items[0].recorded_at == 2.0 + + +@pytest.mark.asyncio +async def test_cleanup_terminal_scan_inventory_prunes_failure_rows( + store: NativeLibraryStore, db_path: Path +) -> None: + await store.create_scan_run( + ScanRun(id="scan-prune", kind="incremental", trigger="manual", queued_at=1) + ) + await store.record_scan_failures( + "scan-prune", + [ + ScanFailureRecord( + root_id="root-1", + relative_path="dir", + failure_code="WALK_EACCES", + recorded_at=100, + ) + ], + ) + with sqlite3.connect(db_path) as connection: + connection.execute( + "UPDATE library_scan_runs SET terminal_at = 10, " + "inventory_cleanup_pending = 1, state = 'failed' WHERE id = 'scan-prune'" + ) + + while True: + _run_id, _deleted, done = await store.cleanup_terminal_scan_inventory() + if done: + break + + items, _cursor = await store.list_scan_run_failures("scan-prune") + assert items == [] + + @pytest.mark.asyncio async def test_scan_management_candidates_follow_album_and_run_lifetimes( store: NativeLibraryStore, db_path: Path @@ -1354,6 +1481,66 @@ async def test_revision_failures_have_specific_safe_api_codes() -> None: assert b"secret path and counter" not in overflow.body +@pytest.mark.asyncio +async def test_identification_snapshot_counts_attention_and_deferral_reasons( + store: NativeLibraryStore, db_path: Path +) -> None: + subjects = { + "job-provider-queued": "album-1", + "job-provider-paused": "album-2", + "job-subject-queued": "album-3", + "job-attention-cap": "album-4", + "job-attention-subject": "album-5", + "job-failed-other": "album-6", + } + for suffix in ("1", "2", "3", "4", "5", "6"): + await store.create_catalog_membership(_membership(suffix)) + for job_id, album_id in subjects.items(): + await store.enqueue_identification_job( + IdentificationJob( + id=job_id, + dedupe_key=f"automatic:{album_id}:rev", + local_album_id=album_id, + created_at=10, + ) + ) + with sqlite3.connect(db_path) as connection: + connection.execute( + "UPDATE library_identification_jobs SET last_failure_code = " + "'PROVIDER_TEMPORARILY_UNAVAILABLE' WHERE id IN " + "('job-provider-queued', 'job-provider-paused')" + ) + connection.execute( + "UPDATE library_identification_jobs SET state = 'paused' " + "WHERE id = 'job-provider-paused'" + ) + connection.execute( + "UPDATE library_identification_jobs SET last_failure_code = " + "'SUBJECT_NOT_AVAILABLE' WHERE id = 'job-subject-queued'" + ) + connection.execute( + "UPDATE library_identification_jobs SET state = 'failed', terminal_at = 12, " + "last_failure_code = 'MAX_DEFERRALS_EXCEEDED' WHERE id = 'job-attention-cap'" + ) + connection.execute( + "UPDATE library_identification_jobs SET state = 'failed', terminal_at = 13, " + "last_failure_code = 'SUBJECT_NOT_AVAILABLE' " + "WHERE id = 'job-attention-subject'" + ) + connection.execute( + "UPDATE library_identification_jobs SET state = 'failed', terminal_at = 14, " + "last_failure_code = 'UNRELATED_CODE' WHERE id = 'job-failed-other'" + ) + + snapshot = await store.get_identification_activity_snapshot(now=10) + assert snapshot["attention_count"] == 2 + assert snapshot["deferred_reason_counts"] == { + "PROVIDER_TEMPORARILY_UNAVAILABLE": 2, + "SUBJECT_NOT_AVAILABLE": 1, + } + assert snapshot["deferred_count"] == 3 + + @pytest.mark.asyncio async def test_identification_activity_snapshot_and_revisioned_controls( store: NativeLibraryStore, db_path: Path @@ -1383,7 +1570,7 @@ async def test_identification_activity_snapshot_and_revisioned_controls( "WHERE id = 'job-failed'" ) - snapshot = await store.get_identification_activity_snapshot() + snapshot = await store.get_identification_activity_snapshot(now=12) assert snapshot["counts"] == {"failed": 1, "queued": 1} assert snapshot["started_at"] == 10 assert snapshot["failure_event_id"] == "job-failed" @@ -1405,7 +1592,7 @@ async def test_identification_activity_snapshot_and_revisioned_controls( ) ], ) - assert (await store.get_identification_activity_snapshot())[ + assert (await store.get_identification_activity_snapshot(now=13))[ "foreground_operation_count" ] == 1 @@ -1414,7 +1601,7 @@ async def test_identification_activity_snapshot_and_revisioned_controls( "UPDATE library_operation_jobs SET state = 'ready' " "WHERE id = 'foreground-operation'" ) - assert (await store.get_identification_activity_snapshot())[ + assert (await store.get_identification_activity_snapshot(now=13))[ "foreground_operation_count" ] == 0 @@ -1431,6 +1618,194 @@ async def test_identification_activity_snapshot_and_revisioned_controls( ) +def test_identification_jobs_attention_cause_ratchet_is_idempotent( + db_path: Path, +) -> None: + lock = threading.Lock() + NativeLibraryStore(db_path, lock) + NativeLibraryStore(db_path, lock) + NativeLibraryStore(db_path, lock) + + with sqlite3.connect(db_path) as connection: + columns = { + str(row[1]) + for row in connection.execute( + "PRAGMA table_info(library_identification_jobs)" + ) + } + assert "attention_cause" in columns + + +@pytest.mark.asyncio +async def test_terminal_fail_identification_job_surfaces_one_review_row( + store: NativeLibraryStore, db_path: Path +) -> None: + await store.create_catalog_membership(_membership()) + + async def fail_album_job( + job_id: str, kind: str, failure_code: str, now: float + ) -> None: + await store.enqueue_identification_job( + IdentificationJob( + id=job_id, + local_album_id="album-1", + kind=kind, + dedupe_key=f"{kind}:album-1:one", + input_revision="one", + priority=20, + created_at=now - 1, + ) + ) + claimed = await store.claim_identification_job( + "worker", now=now, lease_seconds=60 + ) + assert claimed is not None + await store.terminal_fail_identification_job( + str(claimed["id"]), + worker_id="worker", + expected_job_revision=int(claimed["row_revision"]), + failure_code=failure_code, + attention_cause=failure_code, + now=now, + ) + + await fail_album_job("job-auto", "automatic", "MAX_DEFERRALS_EXCEEDED", 3) + with sqlite3.connect(db_path) as connection: + reviews = connection.execute( + "SELECT state, reason_code, attempt_id, input_revision, local_track_id " + "FROM library_identification_reviews" + ).fetchall() + assert reviews == [("needs_review", "MAX_DEFERRALS_EXCEEDED", None, "one", None)] + + # A second terminal failure for the same album + input revision dedupes. + await fail_album_job("job-retry", "review_retry", "SUBJECT_NOT_AVAILABLE", 6) + with sqlite3.connect(db_path) as connection: + count = connection.execute( + "SELECT COUNT(*) FROM library_identification_reviews" + ).fetchone()[0] + assert count == 1 + + # Track-scoped terminal failures never create review rows. + await store.enqueue_identification_job( + IdentificationJob( + id="job-track", + local_track_id="track-1", + kind="automatic", + dedupe_key="automatic:track-1:one", + input_revision="one", + priority=20, + created_at=7, + ) + ) + claimed = await store.claim_identification_job("worker", now=8, lease_seconds=60) + assert claimed is not None + await store.terminal_fail_identification_job( + str(claimed["id"]), + worker_id="worker", + expected_job_revision=int(claimed["row_revision"]), + failure_code="MAX_DEFERRALS_EXCEEDED", + attention_cause="MAX_DEFERRALS_EXCEEDED", + now=9, + ) + with sqlite3.connect(db_path) as connection: + count = connection.execute( + "SELECT COUNT(*) FROM library_identification_reviews" + ).fetchone()[0] + assert count == 1 + + +@pytest.mark.asyncio +async def test_dismiss_review_resolves_without_touching_tracks_and_cancels_jobs( + store: NativeLibraryStore, db_path: Path +) -> None: + await store.create_catalog_membership(_membership()) + # A capped automatic job surfaces its own review row with attention markers. + await store.enqueue_identification_job( + IdentificationJob( + id="job-capped", + local_album_id="album-1", + kind="automatic", + dedupe_key="automatic:album-1:one", + input_revision="one", + priority=20, + created_at=1, + ) + ) + claimed = await store.claim_identification_job("worker", now=2, lease_seconds=60) + assert claimed is not None + await store.terminal_fail_identification_job( + str(claimed["id"]), + worker_id="worker", + expected_job_revision=int(claimed["row_revision"]), + failure_code="MAX_DEFERRALS_EXCEEDED", + attention_cause="UNEXPECTED_ERROR", + now=3, + ) + # An unrelated queued job for the same album is cancelled by the decision. + await store.enqueue_identification_job( + IdentificationJob( + id="job-auto", + local_album_id="album-1", + kind="automatic", + dedupe_key="automatic:album-1:two", + input_revision="two", + priority=20, + created_at=4, + ) + ) + with sqlite3.connect(db_path) as connection: + review_row = connection.execute( + "SELECT id FROM library_identification_reviews WHERE local_album_id = 'album-1'" + ).fetchone() + assert review_row is not None + review_id = str(review_row[0]) + catalog_revision = await store.get_catalog_revision() + assert (await store.get_identification_activity_snapshot(now=4))["attention_count"] == 1 + + result = await store.apply_review_decision( + review_id, + action="dismiss", + actor_user_id="admin", + expected_review_revision=1, + expected_catalog_revision=catalog_revision, + expected_identity_revision=None, + action_id="action-dismiss", + idempotency_key=None, + now=5, + ) + + assert result["review"]["state"] == "resolved" + assert result["review"]["reason_code"] == "DISMISS" + with sqlite3.connect(db_path) as connection: + review = connection.execute( + "SELECT state, reason_code, decided_by_user_id, decided_at " + "FROM library_identification_reviews WHERE id = ?", + (review_id,), + ).fetchone() + track = connection.execute( + "SELECT availability, manual_excluded FROM local_tracks WHERE id = 'track-1'" + ).fetchone() + queued = connection.execute( + "SELECT state, last_failure_code FROM library_identification_jobs " + "WHERE id = 'job-auto'" + ).fetchone() + capped = connection.execute( + "SELECT state, last_failure_code, attention_cause " + "FROM library_identification_jobs WHERE id = 'job-capped'" + ).fetchone() + audit = connection.execute( + "SELECT action_kind, reason_code FROM library_catalog_actions " + "WHERE id = 'action-dismiss'" + ).fetchone() + assert review == ("resolved", "DISMISS", "admin", 5) + assert track == ("indexed", 0) + assert queued == ("cancelled", "ADMIN_DECISION") + # The capped job stays failed for audit but stops counting as attention. + assert capped == ("failed", None, None) + assert (await store.get_identification_activity_snapshot(now=5))["attention_count"] == 0 + assert audit == ("dismiss", "DISMISS") + + @pytest.mark.asyncio async def test_attempts_and_evidence_are_immutable_and_corrections_use_new_ids( store: NativeLibraryStore, db_path: Path diff --git a/backend/tests/infrastructure/test_target_scan_lifecycle.py b/backend/tests/infrastructure/test_target_scan_lifecycle.py index 05f8f458c..a74adc44c 100644 --- a/backend/tests/infrastructure/test_target_scan_lifecycle.py +++ b/backend/tests/infrastructure/test_target_scan_lifecycle.py @@ -1,9 +1,11 @@ from __future__ import annotations import asyncio +import errno import os import sqlite3 import threading +import time from collections.abc import Awaitable, Callable from datetime import datetime from pathlib import Path @@ -110,14 +112,16 @@ def _coordinator( *, tag_read_timeout_seconds: float = 30.0, max_detached_tag_reads: int = 4, + walk_deadline_seconds: float = 30.0, on_indexed_album: Callable[[str], Awaitable[object]] | None = None, clock: Callable[[], float] = lambda: 1_800_000_000.0, ) -> LibraryScanCoordinator: reader = tag_reader or _TagReader() - scanner = ( - LibraryInventoryScanner(store) - if directory_walker is None - else LibraryInventoryScanner(store, directory_walker=directory_walker) + walker_kwargs: dict[str, DirectoryWalker] = {} + if directory_walker is not None: + walker_kwargs["directory_walker"] = directory_walker + scanner = LibraryInventoryScanner( + store, walk_deadline_seconds=walk_deadline_seconds, **walker_kwargs ) return LibraryScanCoordinator( store, @@ -701,6 +705,80 @@ def counted_walk(*args, **kwargs): ] +@pytest.mark.asyncio +async def test_walk_permission_error_fails_run_and_records_failed_path( + target_store: NativeLibraryStore, tmp_path: Path +) -> None: + root = tmp_path / "music" + root.mkdir() + resolver = _resolver(root) + + def denied_walk(*_args, **_kwargs): + raise PermissionError(errno.EACCES, "Permission denied", str(root / "secret")) + yield + + coordinator = _coordinator(target_store, resolver, directory_walker=denied_walk) + await coordinator.request_run(_request(resolver)) + failed = await coordinator.run_once({"root-a": root}) + + assert failed is not None and failed.state == "failed" + assert failed.terminal_code == "ROOT_PERMISSION_DENIED" + failures, next_cursor = await target_store.list_scan_run_failures(failed.id) + assert next_cursor is None + assert [ + (failure.failure_code, failure.relative_path, failure.phase) + for failure in failures + ] == [("WALK_EACCES", "secret", "discovering")] + + +@pytest.mark.asyncio +async def test_wedged_walk_fails_bounded_and_the_next_run_claims( + target_store: NativeLibraryStore, tmp_path: Path +) -> None: + root = tmp_path / "music" + root.mkdir() + (root / "track-1.flac").write_bytes(b"one") + resolver = _resolver(root) + wedged = threading.Event() + calls = 0 + + def walker(*_args, **_kwargs): + nonlocal calls + calls += 1 + if calls == 1: + yield (str(root), [], ["track-1.flac"]) + wedged.wait() + return + yield from os.walk(str(root), followlinks=False) + + coordinator = _coordinator( + target_store, + resolver, + directory_walker=walker, + walk_deadline_seconds=0.05, + ) + try: + await coordinator.request_run(_request(resolver)) + started = time.monotonic() + failed = await asyncio.wait_for( + coordinator.run_once({"root-a": root}), timeout=10 + ) + elapsed = time.monotonic() - started + finally: + wedged.set() + + assert elapsed < 5.0 + assert failed is not None and failed.state == "failed" + assert failed.terminal_code == "WALK_TIMEOUT" + failures, _cursor = await target_store.list_scan_run_failures(failed.id) + assert [failure.failure_code for failure in failures] == ["WALK_TIMEOUT"] + assert await coordinator.run_once({"root-a": root}) is None + + await coordinator.request_run(_request(resolver, trigger="automatic")) + completed = await coordinator.run_once({"root-a": root}) + assert completed is not None and completed.state == "completed" + + @pytest.mark.asyncio async def test_queued_scan_is_completed_before_staged_management_callbacks( target_store: NativeLibraryStore, tmp_path: Path diff --git a/backend/tests/repositories/test_coverart_embedded_fallback.py b/backend/tests/repositories/test_coverart_embedded_fallback.py index 6ddf11fb6..25c2efbcb 100644 --- a/backend/tests/repositories/test_coverart_embedded_fallback.py +++ b/backend/tests/repositories/test_coverart_embedded_fallback.py @@ -4,31 +4,36 @@ import pytest import repositories.coverart_repository as coverart_repository_module -from repositories.coverart_repository import CoverArtRepository, _sniff_image_content_type +from repositories.coverart_repository import ( + CoverArtRepository, + _sniff_image_content_type, +) -RELEASE_GROUP_MBID = '11111111-1111-1111-1111-111111111111' -_JPEG = b'\xff\xd8\xff\xe0' + b'\x00' * 16 -_PNG = b'\x89PNG\r\n\x1a\n' + b'\x00' * 8 +RELEASE_GROUP_MBID = "11111111-1111-1111-1111-111111111111" +_JPEG = b"\xff\xd8\xff\xe0" + b"\x00" * 16 +_PNG = b"\x89PNG\r\n\x1a\n" + b"\x00" * 8 def _miss_external(monkeypatch): async def dedupe_return_none(_key, _factory): return None - monkeypatch.setattr(coverart_repository_module._deduplicator, 'dedupe', dedupe_return_none) + monkeypatch.setattr( + coverart_repository_module._deduplicator, "dedupe", dedupe_return_none + ) @pytest.mark.parametrize( - 'data,expected', + "data,expected", [ - (_JPEG, 'image/jpeg'), - (_PNG, 'image/png'), - (b'GIF89a' + b'\x00' * 8, 'image/gif'), - (b'RIFF\x00\x00\x00\x00WEBP', 'image/webp'), + (_JPEG, "image/jpeg"), + (_PNG, "image/png"), + (b"GIF89a" + b"\x00" * 8, "image/gif"), + (b"RIFF\x00\x00\x00\x00WEBP", "image/webp"), (b'', None), - (b'not an image', None), - (b'\xff\xd8', None), # too short + (b"not an image", None), + (b"\xff\xd8", None), # too short ], ) def test_sniff_image_content_type(data, expected): @@ -36,16 +41,23 @@ def test_sniff_image_content_type(data, expected): @pytest.mark.asyncio -async def test_embedded_cover_served_when_every_external_source_misses(tmp_path, monkeypatch): - track = tmp_path / 'track.flac' - track.write_bytes(b'fake flac') +async def test_embedded_cover_served_when_every_external_source_misses( + tmp_path, monkeypatch +): + track = tmp_path / "track.flac" + track.write_bytes(b"fake flac") library_db = MagicMock() - library_db.get_library_files_for_album = AsyncMock(return_value=[{'file_path': str(track)}]) + library_db.get_library_files_for_album = AsyncMock( + return_value=[{"file_path": str(track)}] + ) async with httpx.AsyncClient() as http_client: repo = CoverArtRepository( - http_client=http_client, cache=MagicMock(), cache_dir=tmp_path, library_db=library_db + http_client=http_client, + cache=MagicMock(), + cache_dir=tmp_path, + library_db=library_db, ) repo._disk_cache.read = AsyncMock(return_value=None) repo._disk_cache.is_negative = AsyncMock(return_value=False) @@ -54,24 +66,26 @@ async def test_embedded_cover_served_when_every_external_source_misses(tmp_path, repo._tagger.read_cover_art = MagicMock(return_value=_JPEG) _miss_external(monkeypatch) - result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size='500') + result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size="500") - assert result == (_JPEG, 'image/jpeg', 'embedded') + assert result == (_JPEG, "image/jpeg", "embedded") repo._disk_cache.write_negative.assert_not_awaited() repo._disk_cache.write.assert_awaited_once() - assert repo._disk_cache.write.await_args.args[3] == {'source': 'embedded'} + assert repo._disk_cache.write.await_args.args[3] == {"source": "embedded"} @pytest.mark.asyncio async def test_no_library_db_falls_through_to_negative_cache(tmp_path, monkeypatch): async with httpx.AsyncClient() as http_client: - repo = CoverArtRepository(http_client=http_client, cache=MagicMock(), cache_dir=tmp_path) + repo = CoverArtRepository( + http_client=http_client, cache=MagicMock(), cache_dir=tmp_path + ) repo._disk_cache.read = AsyncMock(return_value=None) repo._disk_cache.is_negative = AsyncMock(return_value=False) repo._disk_cache.write_negative = AsyncMock() _miss_external(monkeypatch) - result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size='500') + result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size="500") assert result is None repo._disk_cache.write_negative.assert_awaited_once() @@ -79,23 +93,343 @@ async def test_no_library_db_falls_through_to_negative_cache(tmp_path, monkeypat @pytest.mark.asyncio async def test_non_raster_embedded_art_is_skipped(tmp_path, monkeypatch): - track = tmp_path / 'track.mp3' - track.write_bytes(b'fake mp3') + track = tmp_path / "track.mp3" + track.write_bytes(b"fake mp3") library_db = MagicMock() - library_db.get_library_files_for_album = AsyncMock(return_value=[{'file_path': str(track)}]) + library_db.get_library_files_for_album = AsyncMock( + return_value=[{"file_path": str(track)}] + ) async with httpx.AsyncClient() as http_client: repo = CoverArtRepository( - http_client=http_client, cache=MagicMock(), cache_dir=tmp_path, library_db=library_db + http_client=http_client, + cache=MagicMock(), + cache_dir=tmp_path, + library_db=library_db, ) repo._disk_cache.read = AsyncMock(return_value=None) repo._disk_cache.is_negative = AsyncMock(return_value=False) repo._disk_cache.write_negative = AsyncMock() - repo._tagger.read_cover_art = MagicMock(return_value=b'') + repo._tagger.read_cover_art = MagicMock(return_value=b"") _miss_external(monkeypatch) - result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size='500') + result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size="500") + + assert result is None + repo._disk_cache.write_negative.assert_awaited_once() + + +def _repo_with_library(tmp_path, http_client, library_db, *, prefer_local: bool): + repo = CoverArtRepository( + http_client=http_client, + cache=MagicMock(), + cache_dir=tmp_path, + library_db=library_db, + local_cover_priority=lambda: prefer_local, + ) + repo._disk_cache.read = AsyncMock(return_value=None) + repo._disk_cache.is_negative = AsyncMock(return_value=False) + repo._disk_cache.write_negative = AsyncMock() + repo._disk_cache.write = AsyncMock() + repo._album_fetcher.fetch_release_group_cover = AsyncMock(return_value=None) + repo._album_fetcher.fetch_cached_audiodb_cover = AsyncMock(return_value=None) + return repo + + +def _library_db_with(track): + library_db = MagicMock() + library_db.get_library_files_for_album = AsyncMock( + return_value=[{"file_path": str(track)}] + ) + return library_db + + +@pytest.mark.asyncio +async def test_folder_cover_served_before_network_when_preferred(tmp_path): + track = tmp_path / "track.flac" + track.write_bytes(b"fake flac") + (tmp_path / "cover.jpg").write_bytes(_JPEG) + + async with httpx.AsyncClient() as http_client: + repo = _repo_with_library( + tmp_path, http_client, _library_db_with(track), prefer_local=True + ) + + result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size="500") + + assert result == (_JPEG, "image/jpeg", "folder") + repo._album_fetcher.fetch_release_group_cover.assert_not_awaited() + repo._disk_cache.write_negative.assert_not_awaited() + assert repo._disk_cache.write.await_args.args[3] == {"source": "folder"} + + +@pytest.mark.asyncio +async def test_embedded_cover_served_before_network_when_preferred(tmp_path): + track = tmp_path / "track.flac" + track.write_bytes(b"fake flac") + + async with httpx.AsyncClient() as http_client: + repo = _repo_with_library( + tmp_path, http_client, _library_db_with(track), prefer_local=True + ) + repo._tagger.read_cover_art = MagicMock(return_value=_JPEG) + + result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size="500") + + assert result == (_JPEG, "image/jpeg", "embedded") + repo._album_fetcher.fetch_release_group_cover.assert_not_awaited() + repo._disk_cache.write_negative.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_folder_art_wins_over_embedded_art(tmp_path): + track = tmp_path / "track.flac" + track.write_bytes(b"fake flac") + (tmp_path / "Front.PNG").write_bytes(_PNG) + + async with httpx.AsyncClient() as http_client: + repo = _repo_with_library( + tmp_path, http_client, _library_db_with(track), prefer_local=True + ) + repo._tagger.read_cover_art = MagicMock(return_value=_JPEG) + + result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size="500") + + assert result == (_PNG, "image/png", "folder") + repo._tagger.read_cover_art.assert_not_called() + + +@pytest.mark.asyncio +async def test_network_sources_win_before_local_when_preference_off( + tmp_path, monkeypatch +): + track = tmp_path / "track.flac" + track.write_bytes(b"fake flac") + (tmp_path / "cover.jpg").write_bytes(_JPEG) + caa = (b"caa-bytes", "image/jpeg", "cover-art-archive") + + async def fake_dedupe(_key, _factory): + return caa + + monkeypatch.setattr(coverart_repository_module._deduplicator, "dedupe", fake_dedupe) + + async with httpx.AsyncClient() as http_client: + repo = _repo_with_library( + tmp_path, http_client, _library_db_with(track), prefer_local=False + ) + repo._tagger.read_cover_art = MagicMock(return_value=_JPEG) + + result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size="500") + + assert result == caa + repo._tagger.read_cover_art.assert_not_called() + + +@pytest.mark.asyncio +async def test_cached_local_cover_is_not_displaced_by_audiodb_when_preferred(tmp_path): + track = tmp_path / "track.flac" + track.write_bytes(b"fake flac") + + async with httpx.AsyncClient() as http_client: + repo = _repo_with_library( + tmp_path, http_client, _library_db_with(track), prefer_local=True + ) + repo._disk_cache.read = AsyncMock( + return_value=(_JPEG, "image/jpeg", {"source": "folder"}) + ) + repo._album_fetcher.fetch_cached_audiodb_cover = AsyncMock( + return_value=(b"audiodb-bytes", "image/jpeg", "audiodb") + ) + + result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size="500") + + assert result == (_JPEG, "image/jpeg", "folder") + repo._album_fetcher.fetch_cached_audiodb_cover.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_cached_local_cover_is_displaced_by_audiodb_when_preference_off(tmp_path): + track = tmp_path / "track.flac" + track.write_bytes(b"fake flac") + + async with httpx.AsyncClient() as http_client: + repo = _repo_with_library( + tmp_path, http_client, _library_db_with(track), prefer_local=False + ) + repo._disk_cache.read = AsyncMock( + return_value=(_JPEG, "image/jpeg", {"source": "folder"}) + ) + repo._album_fetcher.fetch_cached_audiodb_cover = AsyncMock( + return_value=(b"audiodb-bytes", "image/jpeg", "audiodb") + ) + + result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size="500") + + assert result == (b"audiodb-bytes", "image/jpeg", "audiodb") + + +@pytest.mark.asyncio +async def test_local_cover_beats_banked_negative_when_preferred(tmp_path): + track = tmp_path / "track.flac" + track.write_bytes(b"fake flac") + (tmp_path / "cover.jpg").write_bytes(_JPEG) + + async with httpx.AsyncClient() as http_client: + repo = _repo_with_library( + tmp_path, http_client, _library_db_with(track), prefer_local=True + ) + repo._disk_cache.is_negative = AsyncMock(return_value=True) + + result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size="500") + + assert result == (_JPEG, "image/jpeg", "folder") + repo._album_fetcher.fetch_release_group_cover.assert_not_awaited() + repo._disk_cache.write_negative.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_native_paths_win_over_stale_legacy_rows(tmp_path): + track = tmp_path / "track.flac" + track.write_bytes(b"fake flac") + (tmp_path / "cover.jpg").write_bytes(_JPEG) + + library_db = MagicMock() + library_db.get_library_files_for_album = AsyncMock( + return_value=[ + {"file_path": str(tmp_path / "gone-before-organize" / "old.flac")} + ] + ) + native_store = MagicMock() + native_store.get_indexed_track_paths_for_release_group = AsyncMock( + return_value=[str(track)] + ) + + async with httpx.AsyncClient() as http_client: + repo = CoverArtRepository( + http_client=http_client, + cache=MagicMock(), + cache_dir=tmp_path, + library_db=library_db, + native_library_store=native_store, + local_cover_priority=lambda: True, + ) + repo._disk_cache.read = AsyncMock(return_value=None) + repo._disk_cache.is_negative = AsyncMock(return_value=False) + repo._disk_cache.write_negative = AsyncMock() + repo._disk_cache.write = AsyncMock() + repo._album_fetcher.fetch_release_group_cover = AsyncMock(return_value=None) + repo._album_fetcher.fetch_cached_audiodb_cover = AsyncMock(return_value=None) + + result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size="500") + + assert result == (_JPEG, "image/jpeg", "folder") + library_db.get_library_files_for_album.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_multi_disc_cover_at_album_root(tmp_path): + (tmp_path / "CD1").mkdir() + (tmp_path / "CD2").mkdir() + (tmp_path / "CD1" / "track.flac").write_bytes(b"fake flac") + (tmp_path / "CD2" / "track.flac").write_bytes(b"fake flac") + (tmp_path / "cover.jpg").write_bytes(_JPEG) + + library_db = MagicMock() + library_db.get_library_files_for_album = AsyncMock( + return_value=[ + {"file_path": str(tmp_path / "CD1" / "track.flac")}, + {"file_path": str(tmp_path / "CD2" / "track.flac")}, + ] + ) + + async with httpx.AsyncClient() as http_client: + repo = _repo_with_library(tmp_path, http_client, library_db, prefer_local=True) + + result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size="500") + + assert result == (_JPEG, "image/jpeg", "folder") + + +@pytest.mark.asyncio +async def test_tracks_spanning_library_roots_never_probe_root(tmp_path): + root_a = tmp_path / "root_a" + root_b = tmp_path / "zone" / "root_b" + root_a.mkdir() + root_b.mkdir(parents=True) + (root_a / "track.flac").write_bytes(b"fake flac") + (root_b / "track.flac").write_bytes(b"fake flac") + (tmp_path / "cover.jpg").write_bytes(_JPEG) + + library_db = MagicMock() + library_db.get_library_files_for_album = AsyncMock( + return_value=[ + {"file_path": str(root_a / "track.flac")}, + {"file_path": str(root_b / "track.flac")}, + ] + ) + + async with httpx.AsyncClient() as http_client: + repo = _repo_with_library(tmp_path, http_client, library_db, prefer_local=True) + + result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size="500") + + # Distinct parents -> no ancestor probe: the cover.jpg above both roots is + # not art for this album. + assert result is None + repo._disk_cache.write_negative.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_non_raster_folder_art_is_skipped(tmp_path): + track = tmp_path / "track.flac" + track.write_bytes(b"fake flac") + (tmp_path / "cover.svg").write_bytes( + b'' + ) + + async with httpx.AsyncClient() as http_client: + repo = _repo_with_library( + tmp_path, http_client, _library_db_with(track), prefer_local=True + ) + + result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size="500") assert result is None repo._disk_cache.write_negative.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_oversized_folder_art_is_skipped(tmp_path): + track = tmp_path / "track.flac" + track.write_bytes(b"fake flac") + oversized = bytearray(_JPEG) + oversized.extend(b"\x00" * (25 * 1024 * 1024)) + (tmp_path / "cover.jpg").write_bytes(bytes(oversized)) + + async with httpx.AsyncClient() as http_client: + repo = _repo_with_library( + tmp_path, http_client, _library_db_with(track), prefer_local=True + ) + + result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size="500") + + assert result is None + repo._disk_cache.write_negative.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_local_art_still_last_resort_when_preference_off(tmp_path, monkeypatch): + track = tmp_path / "track.flac" + track.write_bytes(b"fake flac") + _miss_external(monkeypatch) + + async with httpx.AsyncClient() as http_client: + repo = _repo_with_library( + tmp_path, http_client, _library_db_with(track), prefer_local=False + ) + repo._tagger.read_cover_art = MagicMock(return_value=_JPEG) + + result = await repo.get_release_group_cover(RELEASE_GROUP_MBID, size="500") + + assert result == (_JPEG, "image/jpeg", "embedded") diff --git a/backend/tests/repositories/test_coverart_outage_failfast.py b/backend/tests/repositories/test_coverart_outage_failfast.py new file mode 100644 index 000000000..6f8ea4ab4 --- /dev/null +++ b/backend/tests/repositories/test_coverart_outage_failfast.py @@ -0,0 +1,166 @@ +"""Tests for cover-art fail-fast behavior during a sustained CAA outage.""" + +import time +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest + +import repositories.coverart_repository as coverart_module +from infrastructure.resilience.retry import CircuitState +from infrastructure.service_health import service_health +from repositories.coverart_artist import TransientImageFetchError +from repositories.coverart_repository import CoverArtRepository + +RG_MBID = "11111111-1111-1111-1111-111111111111" +REL_MBID = "22222222-2222-2222-2222-222222222222" +ARTIST_MBID = "33333333-3333-3333-3333-333333333333" + + +@pytest.fixture(autouse=True) +def _breaker_and_health_cleanup(): + yield + coverart_module._coverart_circuit_breaker.reset() + service_health.clear() + + +def _repo(tmp_path, http_client): + repo = CoverArtRepository(http_client=http_client, cache=MagicMock(), cache_dir=tmp_path) + repo._disk_cache.read = AsyncMock(return_value=None) + repo._disk_cache.is_negative = AsyncMock(return_value=False) + repo._disk_cache.write_negative = AsyncMock() + return repo + + +def _open_breaker(): + breaker = coverart_module._coverart_circuit_breaker + breaker.state = CircuitState.OPEN + breaker.last_failure_time = time.time() + return breaker + + +@pytest.mark.asyncio +async def test_release_group_cover_fails_fast_when_breaker_open(tmp_path): + """Breaker open: no inline fetch, no deferred resolve, transient negative banked.""" + _open_breaker() + async with httpx.AsyncClient() as http_client: + repo = _repo(tmp_path, http_client) + repo._album_fetcher.fetch_release_group_cover = AsyncMock() + repo._album_fetcher.fetch_cached_audiodb_cover = AsyncMock(return_value=None) + + result = await repo.get_release_group_cover( + RG_MBID, size="250", defer_best_release=True + ) + + assert result is None + repo._album_fetcher.fetch_release_group_cover.assert_not_awaited() + repo._disk_cache.write_negative.assert_awaited_once() + assert ( + repo._disk_cache.write_negative.await_args.kwargs["ttl_seconds"] + == coverart_module.COVER_TRANSIENT_NEGATIVE_TTL_SECONDS + ) + assert repo._deferred_rg_inflight == set() + + +@pytest.mark.asyncio +async def test_release_cover_fails_fast_when_breaker_open(tmp_path): + """Breaker open: release covers serve the placeholder instead of the inline fetch.""" + _open_breaker() + async with httpx.AsyncClient() as http_client: + repo = _repo(tmp_path, http_client) + repo._album_fetcher.fetch_release_cover = AsyncMock() + repo._album_fetcher.fetch_release_audiodb_cover = AsyncMock(return_value=None) + + result = await repo.get_release_cover(REL_MBID, size="500") + + assert result is None + repo._album_fetcher.fetch_release_cover.assert_not_awaited() + repo._disk_cache.write_negative.assert_awaited_once() + assert ( + repo._disk_cache.write_negative.await_args.kwargs["ttl_seconds"] + == coverart_module.COVER_TRANSIENT_NEGATIVE_TTL_SECONDS + ) + + +@pytest.mark.asyncio +async def test_breaker_open_marks_service_health_and_heals(): + """The CAA breaker mirrors into the service-health registry and heals on close.""" + breaker = coverart_module._coverart_circuit_breaker + breaker.reset() + service_health.clear() + + assert not service_health.is_degraded("coverartarchive") + + for _ in range(5): + breaker.record_failure() + + assert service_health.is_degraded("coverartarchive") + + breaker.state = CircuitState.HALF_OPEN + breaker.record_success() + breaker.record_success() + + assert not service_health.is_degraded("coverartarchive") + + +@pytest.mark.asyncio +async def test_artist_transient_negative_short_circuits_next_request(tmp_path): + """A transient artist failure banks a short negative; the next request skips the fetch.""" + async with httpx.AsyncClient() as http_client: + repo = _repo(tmp_path, http_client) + repo._disk_cache.is_negative = AsyncMock(side_effect=[False, True]) + repo._artist_fetcher.fetch_artist_image = AsyncMock( + side_effect=TransientImageFetchError("transient fetch failure") + ) + + first = await repo.get_artist_image(ARTIST_MBID, size=500) + + assert first is None + repo._disk_cache.write_negative.assert_awaited_once() + assert ( + repo._disk_cache.write_negative.await_args.kwargs["ttl_seconds"] + == coverart_module.COVER_TRANSIENT_NEGATIVE_TTL_SECONDS + ) + + second = await repo.get_artist_image(ARTIST_MBID, size=500) + + assert second is None + # The second request short-circuited at is_negative: no new fetch. + assert repo._artist_fetcher.fetch_artist_image.await_count == 1 + + +@pytest.mark.asyncio +async def test_local_cover_served_while_breaker_open(tmp_path): + """Local folder art never depended on CAA, so an open breaker must not hide it.""" + _open_breaker() + track = tmp_path / "track.flac" + track.write_bytes(b"fake flac") + cover = b"\xff\xd8\xff\xe0" + b"\x00" * 16 + (tmp_path / "cover.jpg").write_bytes(cover) + library_db = MagicMock() + library_db.get_library_files_for_album = AsyncMock( + return_value=[{"file_path": str(track)}] + ) + + async with httpx.AsyncClient() as http_client: + repo = CoverArtRepository( + http_client=http_client, + cache=MagicMock(), + cache_dir=tmp_path, + library_db=library_db, + local_cover_priority=lambda: True, + ) + repo._disk_cache.read = AsyncMock(return_value=None) + repo._disk_cache.is_negative = AsyncMock(return_value=False) + repo._disk_cache.write_negative = AsyncMock() + repo._disk_cache.write = AsyncMock() + repo._album_fetcher.fetch_release_group_cover = AsyncMock() + repo._album_fetcher.fetch_cached_audiodb_cover = AsyncMock(return_value=None) + + result = await repo.get_release_group_cover( + RG_MBID, size="250", defer_best_release=True + ) + + assert result == (cover, "image/jpeg", "folder") + repo._album_fetcher.fetch_release_group_cover.assert_not_awaited() + repo._disk_cache.write_negative.assert_not_awaited() diff --git a/backend/tests/repositories/test_coverart_repository_memory_cache.py b/backend/tests/repositories/test_coverart_repository_memory_cache.py index 9fcc3bf7c..bfb713542 100644 --- a/backend/tests/repositories/test_coverart_repository_memory_cache.py +++ b/backend/tests/repositories/test_coverart_repository_memory_cache.py @@ -99,7 +99,7 @@ async def test_non_image_payload_is_not_stored_in_memory_cache(tmp_path): @pytest.mark.asyncio -async def test_artist_transient_fetch_failure_does_not_write_negative_cache(tmp_path, monkeypatch): +async def test_artist_transient_fetch_failure_writes_transient_negative(tmp_path, monkeypatch): async with httpx.AsyncClient() as http_client: cache = MagicMock() repo = CoverArtRepository(http_client=http_client, cache=cache, cache_dir=tmp_path) @@ -116,7 +116,11 @@ async def dedupe_raise_transient(_key, _factory): result = await repo.get_artist_image(ARTIST_MBID, size=500) assert result is None - repo._disk_cache.write_negative.assert_not_awaited() + repo._disk_cache.write_negative.assert_awaited_once() + assert ( + repo._disk_cache.write_negative.await_args.kwargs["ttl_seconds"] + == coverart_repository_module.COVER_TRANSIENT_NEGATIVE_TTL_SECONDS + ) @pytest.mark.asyncio diff --git a/backend/tests/routes/test_downloads_routes.py b/backend/tests/routes/test_downloads_routes.py index 3028360d3..ef9e7b5ec 100644 --- a/backend/tests/routes/test_downloads_routes.py +++ b/backend/tests/routes/test_downloads_routes.py @@ -10,6 +10,7 @@ from core.dependencies import get_download_service from core.exceptions import ( ConflictError, + ConfigurationError, PermissionDeniedError, ResourceNotFoundError, ValidationError, @@ -538,6 +539,21 @@ def test_discard_management_hold_returns_album_level_result(): service.discard_management_hold.assert_awaited_once_with("t1", "admin-1", "admin") +def test_import_held_without_library_root_is_400_configuration_error(): + """Empty roots at 'Import anyway' time surfaces as an actionable 400, not a 500.""" + service = AsyncMock() + service.import_held.side_effect = ConfigurationError( + "No library root is configured - restore one in Settings → Library, then try again." + ) + + response = build_test_client(_app(service)).post("/downloads/held/1/import") + + assert response.status_code == 400 + body = response.json()["error"] + assert body["code"] == "CONFIGURATION_ERROR" + assert "library root" in body["message"] + + def test_held_audio_streams_file_and_supports_range(tmp_path): f = tmp_path / "held.flac" f.write_bytes(b"FLACDATA-0123456789") diff --git a/backend/tests/routes/test_library_operations_target_routes.py b/backend/tests/routes/test_library_operations_target_routes.py index c7959b668..763f008e0 100644 --- a/backend/tests/routes/test_library_operations_target_routes.py +++ b/backend/tests/routes/test_library_operations_target_routes.py @@ -11,9 +11,12 @@ OperationResponse, RepairFindingListResponse, RepairEstimateResponse, + RepairFindingResponse, + ReviewActionResponse, ReviewDetailResponse, ReviewListItem, ReviewListResponse, + SuggestedEditionSummary, ) from api.v1.schemas.artist_reconciliation import ( ArtistDuplicateGroupDetail, @@ -46,6 +49,14 @@ def services() -> dict[str, AsyncMock]: ), tracks=[], ) + review.act.return_value = ReviewActionResponse( + review_id="review-1", + state="resolved", + row_revision=2, + catalog_revision=1, + action_id="action-dismiss", + remaining_exclusion_source=None, + ) operation = AsyncMock() operation.get.return_value = OperationResponse( id="job-1", kind="repair", state="queued" @@ -152,6 +163,41 @@ def test_review_and_diagnostic_contracts( assert response.content == b"{}" +def test_review_dismiss_forwards_action_and_returns_resolved( + app: FastAPI, services: dict[str, AsyncMock] +) -> None: + override_admin_auth(app) + client = build_test_client(app) + response = client.post( + "/library/reviews/review-1/dismiss", + json={ + "expected_review_revision": 1, + "expected_catalog_revision": 1, + "idempotency_key": "dismiss-1", + }, + ) + assert response.status_code == 200 + assert response.json() == { + "review_id": "review-1", + "state": "resolved", + "row_revision": 2, + "catalog_revision": 1, + "action_id": "action-dismiss", + "operation_job_id": None, + "remaining_exclusion_source": None, + } + call = services["review"].act.await_args + assert call is not None + assert call.args[0] == "review-1" + assert call.args[1] == "dismiss" + assert call.args[2].expected_review_revision == 1 + assert call.args[2].expected_catalog_revision == 1 + assert call.args[2].expected_identity_revision is None + assert call.args[2].idempotency_key == "dismiss-1" + assert call.args[2].confirmation is False + assert call.args[3] == "test-admin-id" + + def test_target_operation_routes_are_admin_only(app: FastAPI) -> None: def reject_admin() -> None: raise HTTPException(status_code=403, detail="Admin access required") @@ -296,6 +342,72 @@ def test_management_identity_preparation_contracts( ) +def test_management_identity_preparation_findings_serialize_suggested_edition( + app: FastAPI, services: dict[str, AsyncMock] +) -> None: + override_admin_auth(app) + suggested = RepairFindingResponse( + id="finding-suggested", + local_album_id="album-1", + album_title="Album 1", + album_artist_name="Artist 1", + album_year=2020, + cover_available=False, + evidence_id="evidence-1", + review_id=None, + finding_code="exact_release_suggested", + reason_code="EXACT_EDITION_SUGGESTED", + confidence="complete", + apply_eligible=True, + state="open", + suggested_edition=SuggestedEditionSummary( + release_mbid="release-1", + release_group_mbid="rg-1", + title="Album 1", + track_count=11, + competing_count=3, + date="2019-03-01", + country="DE", + status="Official", + ), + ) + bare = RepairFindingResponse( + id="finding-bare", + local_album_id="album-2", + album_title="Album 2", + album_artist_name="Artist 2", + album_year=None, + cover_available=False, + evidence_id=None, + review_id=None, + finding_code="exact_release_required", + reason_code="EXACT_EDITION_NOT_ACCEPTED", + confidence="bounded", + apply_eligible=False, + state="open", + ) + services["repair"].findings.return_value = RepairFindingListResponse( + items=[suggested, bare] + ) + response = build_test_client(app).get( + "/library/management/identity-preparations/job-1/findings", + params={"finding_category": "exact_release_required"}, + ) + assert response.status_code == 200 + items = response.json()["items"] + assert items[0]["suggested_edition"] == { + "release_mbid": "release-1", + "release_group_mbid": "rg-1", + "title": "Album 1", + "track_count": 11, + "competing_count": 3, + "date": "2019-03-01", + "country": "DE", + "status": "Official", + } + assert items[1]["suggested_edition"] is None + + def test_route_errors_use_typed_envelopes( app: FastAPI, services: dict[str, AsyncMock] ) -> None: @@ -388,6 +500,7 @@ def test_target_operation_route_inventory_is_complete() -> None: ("POST", "/library/reviews/{review_id}/detach-and-keep-tagged"), ("POST", "/library/reviews/{review_id}/exclude"), ("POST", "/library/reviews/{review_id}/restore"), + ("POST", "/library/reviews/{review_id}/dismiss"), ("POST", "/library/reviews/{review_id}/candidate"), ("POST", "/library/reviews/bulk-preview"), ("POST", "/library/reviews/bulk-apply"), diff --git a/backend/tests/routes/test_library_policies_target.py b/backend/tests/routes/test_library_policies_target.py new file mode 100644 index 000000000..4ce416509 --- /dev/null +++ b/backend/tests/routes/test_library_policies_target.py @@ -0,0 +1,126 @@ +from unittest.mock import AsyncMock + +import pytest +from fastapi import FastAPI + +from api.v1.routes.library_policies_target import router +from api.v1.schemas.library_policies import LibrarySettingsResponse +from core.dependencies import get_legacy_pending_migration_service +from core.dependencies.service_providers import get_target_library_policy_service +from core.exceptions import StaleRevisionError +from tests.helpers import build_test_client, override_admin_auth + + +@pytest.fixture +def app() -> tuple[FastAPI, AsyncMock, AsyncMock]: + application = FastAPI() + application.include_router(router) + target = AsyncMock() + target.get_settings.return_value = LibrarySettingsResponse( + policy_revision="policy-2", + reconciliation_required=True, + reconciliation_state="awaiting_reconciliation", + pending_policy_revision="policy-2", + affected_scope_ids=["root"], + ) + target.save_settings.return_value = target.get_settings.return_value + target.restore_roots.return_value = target.get_settings.return_value + pending_migration = AsyncMock() + pending_migration.schedule.return_value = False + application.dependency_overrides[get_target_library_policy_service] = ( + lambda: target + ) + application.dependency_overrides[get_legacy_pending_migration_service] = ( + lambda: pending_migration + ) + override_admin_auth(application) + return application, target, pending_migration + + +def test_update_library_settings_schedules_pending_migration( + app: tuple[FastAPI, AsyncMock, AsyncMock], +) -> None: + application, _, pending_migration = app + client = build_test_client(application) + response = client.put( + "/settings/library", + json={ + "settings": {"library_roots": []}, + "expected_policy_revision": "policy-2", + }, + ) + assert response.status_code == 200 + pending_migration.schedule.assert_awaited_once() + + +def test_restore_roots_schedules_pending_migration( + app: tuple[FastAPI, AsyncMock, AsyncMock], +) -> None: + application, _, pending_migration = app + client = build_test_client(application) + response = client.post( + "/settings/library/restore-roots", + json={ + "expected_policy_revision": "policy-2", + "paths": {"root": "/music"}, + }, + ) + assert response.status_code == 200 + pending_migration.schedule.assert_awaited_once() + + +def test_update_library_settings_does_not_schedule_when_save_is_stale( + app: tuple[FastAPI, AsyncMock, AsyncMock], +) -> None: + application, target, pending_migration = app + target.save_settings.side_effect = StaleRevisionError("stale policy revision") + client = build_test_client(application) + response = client.put( + "/settings/library", + json={ + "settings": {"library_roots": []}, + "expected_policy_revision": "policy-1", + }, + ) + assert response.status_code == 409 + pending_migration.schedule.assert_not_awaited() + + +def test_update_library_settings_skips_schedule_when_library_disabled( + app: tuple[FastAPI, AsyncMock, AsyncMock], +) -> None: + application, target, pending_migration = app + target.save_settings.return_value = LibrarySettingsResponse( + policy_revision="policy-2", enabled=False + ) + client = build_test_client(application) + response = client.put( + "/settings/library", + json={ + "settings": {"library_roots": [], "enabled": False}, + "expected_policy_revision": "policy-2", + }, + ) + assert response.status_code == 200 + assert response.json()["enabled"] is False + pending_migration.schedule.assert_not_awaited() + + +def test_restore_roots_skips_schedule_when_library_disabled( + app: tuple[FastAPI, AsyncMock, AsyncMock], +) -> None: + application, target, pending_migration = app + target.restore_roots.return_value = LibrarySettingsResponse( + policy_revision="policy-2", enabled=False + ) + client = build_test_client(application) + response = client.post( + "/settings/library/restore-roots", + json={ + "expected_policy_revision": "policy-2", + "paths": {"root": "/music"}, + }, + ) + assert response.status_code == 200 + assert response.json()["enabled"] is False + pending_migration.schedule.assert_not_awaited() diff --git a/backend/tests/routes/test_stream_routes.py b/backend/tests/routes/test_stream_routes.py index dce8d27c5..d934863cd 100644 --- a/backend/tests/routes/test_stream_routes.py +++ b/backend/tests/routes/test_stream_routes.py @@ -7,7 +7,12 @@ from api.v1.routes.stream import router from core.dependencies import get_jellyfin_playback_service -from core.exceptions import ExternalServiceError, PlaybackNotAllowedError, ResourceNotFoundError +from core.exceptions import ( + ExternalServiceError, + JellyfinAuthError, + PlaybackNotAllowedError, + ResourceNotFoundError, +) from tests.helpers import override_user_auth @@ -99,11 +104,38 @@ def test_get_stream_returns_416_on_range_error(client, mock_playback_service): assert response.status_code == 416 +def test_get_stream_passes_current_user_id(client, mock_playback_service): + response = client.get("/stream/jellyfin/item-1") + + assert response.status_code == 200 + mock_playback_service.proxy_stream.assert_awaited_once_with( + "item-1", range_header=None, user_id="test-user-id" + ) + + +def test_get_stream_maps_upstream_auth_error_to_502(client, mock_playback_service): + mock_playback_service.proxy_stream.side_effect = JellyfinAuthError("token revoked") + + response = client.get("/stream/jellyfin/item-auth") + + assert response.status_code == 502 + + def test_head_stream_returns_proxied_headers(client, mock_playback_service): response = client.request("HEAD", "/stream/jellyfin/item-1") assert response.status_code == 200 - mock_playback_service.proxy_head.assert_awaited_once_with("item-1") + mock_playback_service.proxy_head.assert_awaited_once_with( + "item-1", user_id="test-user-id" + ) + + +def test_head_stream_maps_upstream_auth_error_to_502(client, mock_playback_service): + mock_playback_service.proxy_head.side_effect = JellyfinAuthError("token revoked") + + response = client.request("HEAD", "/stream/jellyfin/item-auth") + + assert response.status_code == 502 def test_head_stream_returns_404_when_missing(client, mock_playback_service): diff --git a/backend/tests/routes/test_target_application.py b/backend/tests/routes/test_target_application.py index bc428ae8d..2c629adc9 100644 --- a/backend/tests/routes/test_target_application.py +++ b/backend/tests/routes/test_target_application.py @@ -411,6 +411,7 @@ def starter_calls(path: Path, functions: set[str]) -> set[str]: "start_target_scan_supervisor", "start_target_identification_worker", "start_target_operation_worker", + "start_target_worker_watchdog", } <= target @@ -433,13 +434,18 @@ def test_target_lifecycle_events_sweep_uses_target_catalog_authority() -> None: @pytest.mark.parametrize( - ("admission_token", "expected_phase"), - [(None, "steady_state"), ("a" * 32, "admission")], + ("admission_token", "expected_phase", "library_enabled"), + [ + (None, "steady_state", True), + ("a" * 32, "admission", True), + (None, "steady_state", False), + ], ) def test_production_target_lifespan_selects_validation_phase_and_runs_runtime( monkeypatch: pytest.MonkeyPatch, admission_token: str | None, expected_phase: str, + library_enabled: bool, ) -> None: import target_application as target_module from core.dependencies import auth_providers @@ -463,7 +469,9 @@ def test_production_target_lifespan_selects_validation_phase_and_runs_runtime( memory_cache_cleanup_interval=60, disk_cache_cleanup_interval=60, ), - get_typed_library_settings=lambda: SimpleNamespace(library_roots=[]), + get_typed_library_settings=lambda: SimpleNamespace( + library_roots=[], enabled=library_enabled + ), get_library_scan_schedule=lambda: SimpleNamespace( scan_frequency="manual", daily_scan_time="03:00" ), @@ -539,21 +547,49 @@ def test_production_target_lifespan_selects_validation_phase_and_runs_runtime( "start_target_scan_supervisor", lambda *args, **kwargs: scan_supervisor_arguments.update(kwargs), ) + identification_worker_arguments: dict[str, object] = {} monkeypatch.setattr( - target_module, "start_target_identification_worker", lambda *a, **k: None + target_module, + "start_target_identification_worker", + lambda *a, **k: identification_worker_arguments.update(k), ) + operation_worker_arguments: dict[str, object] = {} monkeypatch.setattr( - target_module, "start_target_operation_worker", lambda *a, **k: None + target_module, + "start_target_operation_worker", + lambda *a, **k: operation_worker_arguments.update(k), ) monkeypatch.setattr( target_module, "start_library_contribution_verification_worker", lambda *a, **k: None, ) + watchdog_starters: dict[str, object] = {} + monkeypatch.setattr( + target_module, + "start_target_worker_watchdog", + lambda starters: watchdog_starters.update(starters), + ) + pending_migration = AsyncMock() + pending_migration.schedule.return_value = False + monkeypatch.setattr( + target_module, + "get_legacy_pending_migration_service", + lambda: pending_migration, + ) monkeypatch.setattr(auth_providers, "get_auth_service", lambda: auth) monkeypatch.setattr(auth_providers, "get_auth_store", lambda: auth_store) + registry = target_module.TaskRegistry.get_instance() + shutdown_order: list[str] = [] monkeypatch.setattr( - target_module.TaskRegistry.get_instance(), "cancel_all", AsyncMock() + registry, + "cancel", + AsyncMock(side_effect=lambda *a, **k: shutdown_order.append("cancel")), + ) + monkeypatch.setattr( + registry, + "cancel_all", + AsyncMock(side_effect=lambda *a, **k: shutdown_order.append("cancel_all")), ) monkeypatch.setenv("TZ", "Europe/London") if admission_token is None: @@ -567,7 +603,12 @@ def test_production_target_lifespan_selects_validation_phase_and_runs_runtime( validate.assert_awaited_once_with(expected_phase) admission.assert_awaited_once() - migrate.assert_awaited_once() + migrate.assert_awaited_once_with( + auth_store=auth_store, + preferences=preferences, + cache_dir=target_module.get_settings().cache_dir, + library_enabled=library_enabled, + ) operation_supervisor.recover.assert_awaited_once() recovery_service.recover_startup.assert_awaited_once() operational.assert_awaited_once_with( @@ -575,11 +616,29 @@ def test_production_target_lifespan_selects_validation_phase_and_runs_runtime( preferences=preferences, auth_store=auth_store, ) + identification_enabled_getter = identification_worker_arguments["enabled_getter"] + operation_enabled_getter = operation_worker_arguments["enabled_getter"] + assert callable(identification_enabled_getter) + assert callable(operation_enabled_getter) + assert identification_enabled_getter() is library_enabled + assert operation_enabled_getter() is library_enabled + if library_enabled: + pending_migration.schedule.assert_awaited_once() + else: + pending_migration.schedule.assert_not_awaited() schedule_settings_getter = scan_supervisor_arguments["schedule_settings_getter"] assert callable(schedule_settings_getter) assert schedule_settings_getter()["timezone_name"] == "Europe/London" assert schedule_settings_getter()["timezone_name"] == "Europe/London" timezone_name.assert_called_once_with() + assert set(watchdog_starters) == { + "target-library-identification-worker", + "target-library-operation-worker", + "library-contribution-verification-worker", + } + assert all(callable(starter) for starter in watchdog_starters.values()) + registry.cancel.assert_awaited_once_with("target-worker-watchdog") + assert shutdown_order == ["cancel", "cancel_all"] cleanup.assert_awaited_once() assert lifecycle_order == [ "validate", diff --git a/backend/tests/routes/test_target_library_scan_routes.py b/backend/tests/routes/test_target_library_scan_routes.py index 92549edb5..28b2964c2 100644 --- a/backend/tests/routes/test_target_library_scan_routes.py +++ b/backend/tests/routes/test_target_library_scan_routes.py @@ -1,7 +1,7 @@ from __future__ import annotations from types import SimpleNamespace -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import FastAPI, HTTPException @@ -15,12 +15,19 @@ from core.dependencies import ( get_library_administrative_work_service, get_library_policy_resolver, + get_mb_provider_availability, + get_native_library_store, get_target_identification_queue, get_target_library_scan_coordinator, ) -from core.exceptions import ResourceNotFoundError, StaleRevisionError +from core.exceptions import ResourceNotFoundError, StaleRevisionError, ValidationError from middleware import _get_current_admin -from models.library_work import LibraryWorkItem, ScanControlResult, ScanRequestResult +from models.library_work import ( + LibraryWorkItem, + ScanControlResult, + ScanFailureRecord, + ScanRequestResult, +) from services.native.library_policy_resolver import LibraryPolicyResolver from services.native.library_activity_events import activity_events from tests.helpers import build_test_client, override_admin_auth, override_user_auth @@ -86,6 +93,9 @@ def identification_queue() -> AsyncMock: "started_at": None, "updated_at": None, "deferred_count": 0, + "claimable_count": 0, + "deferred_reason_counts": {}, + "attention_count": 0, "failure_event_id": None, "failure_at": None, "foreground_operation_count": 0, @@ -107,11 +117,25 @@ def administrative_work() -> AsyncMock: return service +@pytest.fixture +def native_store() -> AsyncMock: + store = AsyncMock() + store.list_scan_run_failures.return_value = ([], None) + return store + + +@pytest.fixture +def mb_availability() -> MagicMock: + return MagicMock(return_value=True) + + @pytest.fixture def app( coordinator: AsyncMock, identification_queue: AsyncMock, administrative_work: AsyncMock, + native_store: AsyncMock, + mb_availability: MagicMock, resolver: LibraryPolicyResolver, ) -> FastAPI: application = FastAPI() @@ -126,6 +150,10 @@ def app( application.dependency_overrides[get_library_administrative_work_service] = ( lambda: administrative_work ) + application.dependency_overrides[get_native_library_store] = lambda: native_store + application.dependency_overrides[get_mb_provider_availability] = ( + lambda: mb_availability + ) return application @@ -278,6 +306,8 @@ def test_activity_is_authenticated_and_redacted( "started_at": 2.0, "updated_at": 11.0, "deferred_count": 1, + "deferred_reason_counts": {"PROVIDER_TEMPORARILY_UNAVAILABLE": 1}, + "attention_count": 2, "kept_local_count": 4, "active_priority": 30, "failure_event_id": "failure-opaque", @@ -301,9 +331,13 @@ def test_activity_is_authenticated_and_redacted( "needs_review_count": 2, "failed_count": 0, "deferred_count": 1, + "deferred_reason_counts": {"PROVIDER_TEMPORARILY_UNAVAILABLE": 1}, + "attention_count": 2, "priority_band": "Administrator retries", "oldest_backlog_at": 2.0, - "provider_unavailable": True, + # Regression: a live breaker read, not bool(deferred_count) - deferrals + # with a healthy provider must not raise a false provider alert. + "provider_unavailable": False, "control_revision": 8, "failure_event_id": "failure-opaque", "failure_at": 9.0, @@ -325,6 +359,69 @@ def test_activity_is_authenticated_and_redacted( assert failure["failure_event_id"] == "failure-opaque" +def test_activity_marks_provider_unavailable_from_live_breaker_read( + app: FastAPI, + identification_queue: AsyncMock, + mb_availability: MagicMock, +) -> None: + identification_queue.activity_snapshot.return_value = { + "control_state": "running", + "control_revision": 1, + "counts": {"queued": 2}, + "started_at": 2.0, + "updated_at": 11.0, + "deferred_count": 2, + "claimable_count": 2, + "deferred_reason_counts": {"PROVIDER_TEMPORARILY_UNAVAILABLE": 2}, + "attention_count": 0, + "kept_local_count": 0, + "active_priority": None, + "failure_event_id": None, + "failure_at": None, + "foreground_operation_count": 0, + } + mb_availability.return_value = False + override_user_auth(app, role="user") + + payload = build_test_client(app).get("/library/activity").json() + item = next(item for item in payload["items"] if item["kind"] == "identification") + assert item["provider_unavailable"] is True + assert item["deferred_count"] == 2 + assert item["deferred_reason_counts"] == {"PROVIDER_TEMPORARILY_UNAVAILABLE": 2} + assert item["attention_count"] == 0 + + +def test_activity_reports_identification_idle_until_work_is_claimable( + app: FastAPI, identification_queue: AsyncMock +) -> None: + # Deferred-not-due jobs are waiting but not claimable: the lane must not + # pretend the queue is running while nothing can be claimed. + identification_queue.activity_snapshot.return_value = { + **identification_queue.activity_snapshot.return_value, + "control_state": "running", + "counts": {"queued": 1}, + "claimable_count": 0, + "attention_count": 1, + "deferred_count": 1, + "deferred_reason_counts": {"PROVIDER_TEMPORARILY_UNAVAILABLE": 1}, + } + override_user_auth(app, role="user") + items = build_test_client(app).get("/library/activity").json()["items"] + identification = next(item for item in items if item["kind"] == "identification") + assert identification["state"] == "idle" + assert identification["waiting_count"] == 1 + + identification_queue.activity_snapshot.return_value = { + **identification_queue.activity_snapshot.return_value, + "counts": {"queued": 1}, + "claimable_count": 1, + } + items = build_test_client(app).get("/library/activity").json()["items"] + identification = next(item for item in items if item["kind"] == "identification") + assert identification["state"] == "running" + assert identification["attention_count"] == 1 + + def test_activity_projects_admin_work_and_scan_finalization_truthfully( admin_client, coordinator: AsyncMock, @@ -497,6 +594,98 @@ def test_missing_and_stale_run_use_typed_error_envelopes( assert response.json()["error"]["code"] == "STALE_REVISION" +def test_scan_runs_start_rejects_a_disabled_library_with_the_switch_message( + admin_client, coordinator: AsyncMock, resolver: LibraryPolicyResolver +) -> None: + coordinator.request_run.side_effect = ValidationError( + "The local library is disabled. Enable it in Settings → Library " + "before starting a scan." + ) + response = admin_client.post( + "/library/scan-runs", + json={ + "kind": "incremental", + "scope_ids": ["root-a"], + "expected_policy_revision": resolver.policy_revision, + }, + ) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "VALIDATION_ERROR" + assert "disabled" in response.json()["error"]["message"] + + +def test_scan_run_failures_returns_snake_case_items( + admin_client, native_store: AsyncMock +) -> None: + native_store.list_scan_run_failures.return_value = ( + [ + ScanFailureRecord( + root_id="root-a", + relative_path="Artist/Album", + failure_code="WALK_EACCES", + recorded_at=1_800_000_001.0, + failure_detail="[Errno 13] Permission denied", + phase="discovering", + ) + ], + 41, + ) + + response = admin_client.get("/library/scan-runs/run-1/failures?cursor=40") + + assert response.status_code == 200 + assert response.json() == { + "items": [ + { + "root_id": "root-a", + "relative_path": "Artist/Album", + "failure_code": "WALK_EACCES", + "failure_detail": "[Errno 13] Permission denied", + "phase": "discovering", + "recorded_at": 1_800_000_001.0, + } + ], + "next_cursor": 41, + } + native_store.list_scan_run_failures.assert_awaited_once_with( + "run-1", limit=50, cursor_rowid=40 + ) + + +def test_scan_run_failures_limit_bounds_are_validated(admin_client) -> None: + assert admin_client.get("/library/scan-runs/run-1/failures?limit=0").status_code == 422 + assert ( + admin_client.get("/library/scan-runs/run-1/failures?limit=201").status_code + == 422 + ) + + +def test_scan_run_failures_auth_matrix(app: FastAPI, native_store: AsyncMock) -> None: + assert ( + build_test_client(app).get("/library/scan-runs/run-1/failures").status_code + == 401 + ) + + def reject_admin(): + raise HTTPException(status_code=403, detail="Admin access required") + + app.dependency_overrides[_get_current_admin] = reject_admin + assert ( + build_test_client(app).get("/library/scan-runs/run-1/failures").status_code + == 403 + ) + + override_admin_auth(app) + override_user_auth(app, role="admin") + native_store.get_scan_run.side_effect = ResourceNotFoundError( + "Scan run not found: missing" + ) + missing = build_test_client(app).get("/library/scan-runs/missing/failures") + assert missing.status_code == 404 + assert missing.json()["error"]["code"] == "NOT_FOUND" + + def test_target_route_security_inventory_is_complete() -> None: paths = { route.path @@ -513,6 +702,7 @@ def test_target_route_security_inventory_is_complete() -> None: "/library/scan-runs/current", "/library/scan-runs/estimate", "/library/scan-runs/{run_id}", + "/library/scan-runs/{run_id}/failures", "/library/scan-runs/{run_id}/pause", "/library/scan-runs/{run_id}/resume", "/library/scan-runs/{run_id}/stop", diff --git a/backend/tests/security/test_auth_on_every_endpoint.py b/backend/tests/security/test_auth_on_every_endpoint.py index 4625f1bb5..dffb8b291 100644 --- a/backend/tests/security/test_auth_on_every_endpoint.py +++ b/backend/tests/security/test_auth_on_every_endpoint.py @@ -82,6 +82,7 @@ get_navidrome_folder_scope_service, get_now_playing_service, get_per_user_client_factory, + get_native_library_store, get_personal_mix_service, get_playlist_service, get_plex_playback_service, @@ -148,6 +149,7 @@ get_navidrome_folder_scope_service, get_now_playing_service, get_per_user_client_factory, + get_native_library_store, get_personal_mix_service, get_playlist_service, get_plex_playback_service, @@ -486,6 +488,11 @@ "/api/v1/library/reviews/review-1/restore", {"expected_review_revision": 1, "expected_catalog_revision": 1}, ), + ( + "POST", + "/api/v1/library/reviews/review-1/dismiss", + {"expected_review_revision": 1, "expected_catalog_revision": 1}, + ), ( "POST", "/api/v1/library/reviews/review-1/candidate", @@ -734,6 +741,7 @@ ("GET", "/api/v1/library/scan-runs", None), ("GET", "/api/v1/library/scan-runs/estimate", None), ("GET", "/api/v1/library/scan-runs/run-1", None), + ("GET", "/api/v1/library/scan-runs/run-1/failures", None), ( "POST", "/api/v1/library/scan-runs/run-1/pause", diff --git a/backend/tests/services/native/test_acquisition_cleanup_service.py b/backend/tests/services/native/test_acquisition_cleanup_service.py index 31a015f2a..6e515bd7f 100644 --- a/backend/tests/services/native/test_acquisition_cleanup_service.py +++ b/backend/tests/services/native/test_acquisition_cleanup_service.py @@ -840,7 +840,11 @@ async def test_legacy_reconciliation_cleans_only_unambiguous_terminal_tasks( ] library.bundles["attention-bundle"] = "needs_attention" service = AcquisitionCleanupService( - store, library, lambda source: client, lambda: root + store, + library, + lambda source: client, + lambda: root, + sab_category_getter=lambda: "audio", ) await service.recover_startup() @@ -921,3 +925,135 @@ async def test_reconciliation_refuses_unsafe_mounts(tmp_path: Path, mount: Path) ) assert await service.reconcile_legacy_mount() == 0 + + +@pytest.mark.asyncio +async def test_reconciliation_survives_directory_removed_between_passes( + tmp_path: Path, +): + root = tmp_path / "sab" + vanished = root / "droppedneedle-staging" + vanished.mkdir(parents=True) + store = _store(tmp_path) + client = _Client( + DownloadMaterialization( + state="missing", mount_root=str(root), mount_healthy=True + ) + ) + service = AcquisitionCleanupService( + store, _LibraryStore(), lambda source: client, lambda: root + ) + + assert await service.reconcile_legacy_mount(limit=1) == 1 + vanished.rmdir() + + await service.reconcile_legacy_mount() + + mount_key = hashlib.sha256(str(root.resolve()).encode()).hexdigest() + progress = await store.ensure_cleanup_reconciliation(mount_key, str(root)) + assert progress.completed is True + + +def test_directory_entries_skips_entry_removed_mid_scan(tmp_path: Path, monkeypatch): + class _VanishingEntry: + def __init__(self, name: str) -> None: + self.name = name + self.path = str(tmp_path / name) + + def is_dir(self, follow_symlinks: bool = False) -> bool: + raise FileNotFoundError(self.name) + + def is_symlink(self) -> bool: + return False + + class _FakeEntries: + def __enter__(self): + return iter([_VanishingEntry("gone")]) + + def __exit__(self, *args: object) -> bool: + return False + + monkeypatch.setattr(cleanup_module.os, "scandir", lambda path: _FakeEntries()) + assert cleanup_module._directory_entries(tmp_path) == [] + + +def test_directory_entries_returns_empty_for_vanished_directory(tmp_path: Path): + assert cleanup_module._directory_entries(tmp_path / "missing") == [] + + +@pytest.mark.asyncio +async def test_reconciliation_default_category_scopes_to_droppedneedle_prefix( + tmp_path: Path, monkeypatch +): + root = tmp_path / "sab" + (root / "droppedneedle-staging").mkdir(parents=True) + foreign = root / "sonarr" + foreign_job = foreign / f"droppedneedle-{'f' * 32}-0" + foreign_job.mkdir(parents=True) + store = _store(tmp_path) + client = _Client( + DownloadMaterialization( + state="missing", mount_root=str(root), mount_healthy=True + ) + ) + service = AcquisitionCleanupService( + store, _LibraryStore(), lambda source: client, lambda: root + ) + visited: list[str] = [] + original_entries = cleanup_module._directory_entries + + def recording_entries(path: Path): + visited.append(str(path)) + return original_entries(path) + + monkeypatch.setattr(cleanup_module, "_directory_entries", recording_entries) + + await service.reconcile_legacy_mount() + + assert str(root) in visited + assert str(root / "droppedneedle-staging") in visited + assert not any(str(foreign) in path for path in visited) + assert ( + await store.get_download_attempt_for_job( + "usenet", f"droppedneedle-{'f' * 32}-0" + ) + is None + ) + + +@pytest.mark.asyncio +async def test_reconciliation_configured_category_descends_only_there( + tmp_path: Path, +): + root = tmp_path / "sab" + category_job = root / "audio" / f"droppedneedle-{'f' * 32}-0" + category_job.mkdir(parents=True) + foreign_job = root / "movies" / f"droppedneedle-{'e' * 32}-0" + foreign_job.mkdir(parents=True) + store = _store(tmp_path) + client = _Client( + DownloadMaterialization( + state="missing", mount_root=str(root), mount_healthy=True + ) + ) + service = AcquisitionCleanupService( + store, + _LibraryStore(), + lambda source: client, + lambda: root, + sab_category_getter=lambda: "Audio", + ) + + await service.reconcile_legacy_mount() + + assert ( + await store.get_download_attempt_for_job( + "usenet", f"droppedneedle-{'f' * 32}-0" + ) + ).state == "needs_attention" + assert ( + await store.get_download_attempt_for_job( + "usenet", f"droppedneedle-{'e' * 32}-0" + ) + is None + ) diff --git a/backend/tests/services/native/test_identification_pipeline.py b/backend/tests/services/native/test_identification_pipeline.py index a1eb298c8..c9a259c9d 100644 --- a/backend/tests/services/native/test_identification_pipeline.py +++ b/backend/tests/services/native/test_identification_pipeline.py @@ -1,8 +1,10 @@ +import asyncio import json import sqlite3 import threading from collections.abc import Callable from pathlib import Path +from types import SimpleNamespace from unittest.mock import AsyncMock import msgspec @@ -48,7 +50,11 @@ from services.native.identification_evidence_projector import ( IdentificationEvidenceProjector, ) -from services.native.identification_queue_service import IdentificationQueueService +from services.native.identification_queue_service import ( + MAX_BACKOFF_SECONDS, + MAX_DEFERRAL_ATTEMPTS, + IdentificationQueueService, +) from services.native.identification_revisions import ( album_identity_revision, album_input_revisions, @@ -58,6 +64,9 @@ IdentificationWorkArbiter, ReidentificationService, ) +from services.native.target_application_runtime import ( + run_target_identification_worker, +) EMBEDDED_GROUP = "11111111-1111-4111-8111-111111111111" EMBEDDED_GROUP_OTHER = "22222222-2222-4222-8222-222222222222" @@ -1125,6 +1134,297 @@ async def test_transient_queue_backoff_is_typed_and_respects_not_before( assert await queue.claim("worker", now=40) is not None +@pytest.mark.asyncio +async def test_deferral_cap_terminates_job_in_attention_state( + store: NativeLibraryStore, db_path: Path +) -> None: + await _seed_album(store) + queue = IdentificationQueueService(store) + await queue.enqueue_album("album-1", input_revision="revision", now=1) + now = 2.0 + for attempt in range(1, MAX_DEFERRAL_ATTEMPTS + 1): + claimed = await queue.claim("worker", now=now) + assert claimed is not None + assert claimed["attempt_count"] == attempt + await queue.defer( + claimed, "worker", "PROVIDER_TEMPORARILY_UNAVAILABLE", now=now + ) + now += MAX_BACKOFF_SECONDS + 60 + with sqlite3.connect(db_path) as connection: + row = connection.execute( + "SELECT state, last_failure_code, terminal_at " + "FROM library_identification_jobs" + ).fetchone() + assert row[0] == "failed" + assert row[1] == "MAX_DEFERRALS_EXCEEDED" + assert row[2] == pytest.approx( + 2.0 + (MAX_DEFERRAL_ATTEMPTS - 1) * (MAX_BACKOFF_SECONDS + 60) + ) + assert await queue.claim("worker", now=now) is None + assert (await queue.activity_snapshot())["attention_count"] == 1 + + +@pytest.mark.asyncio +async def test_attention_failed_job_resurrects_only_for_provider_causes( + store: NativeLibraryStore, db_path: Path +) -> None: + await _seed_album(store) + await _seed_album(store, "2") + await _seed_album(store, "3") + await _seed_album(store, "4") + queue = IdentificationQueueService(store) + + # Deterministic cap (cause != PROVIDER_TEMPORARILY_UNAVAILABLE) stays + # terminal on a same-key enqueue: no resurrection. + job_id = await queue.enqueue_album("album-1", input_revision="revision", now=1) + claimed = await queue.claim("worker", now=2) + assert claimed is not None + await queue.fail(claimed, "worker", "MAX_DEFERRALS_EXCEEDED", now=3) + deduped_id, created = await queue.enqueue_album_with_disposition( + "album-1", input_revision="revision", now=4 + ) + assert deduped_id == job_id + assert created is False + with sqlite3.connect(db_path) as connection: + state = connection.execute( + "SELECT state FROM library_identification_jobs WHERE id = ?", + (job_id,), + ).fetchone()[0] + assert state == "failed" + + # Provider cap with the gate open: resurrects and clears the cause. + job_id = await queue.enqueue_album("album-2", input_revision="revision", now=5) + now = 6.0 + for _attempt in range(1, MAX_DEFERRAL_ATTEMPTS + 1): + claimed = await queue.claim("worker", now=now) + assert claimed is not None + await queue.defer( + claimed, "worker", "PROVIDER_TEMPORARILY_UNAVAILABLE", now=now + ) + now += MAX_BACKOFF_SECONDS + 60 + with sqlite3.connect(db_path) as connection: + cause = connection.execute( + "SELECT attention_cause FROM library_identification_jobs WHERE id = ?", + (job_id,), + ).fetchone()[0] + assert cause == "PROVIDER_TEMPORARILY_UNAVAILABLE" + + open_queue = IdentificationQueueService(store, provider_available=lambda: True) + resurrected_id, created = await open_queue.enqueue_album_with_disposition( + "album-2", input_revision="revision", now=now + 1 + ) + assert resurrected_id == job_id + assert created is True + with sqlite3.connect(db_path) as connection: + row = connection.execute( + "SELECT state, attempt_count, not_before, last_failure_code, " + "attention_cause, terminal_at FROM library_identification_jobs WHERE id = ?", + (job_id,), + ).fetchone() + assert row == ("queued", 0, 0, None, None, None) + reclaimed = await queue.claim("worker", now=now + 2) + assert reclaimed is not None + assert reclaimed["id"] == job_id + + # Provider cap with the gate closed: stays terminal (dedupes, no resurrection). + job_id = await queue.enqueue_album("album-3", input_revision="revision", now=20) + now = 21.0 + for _attempt in range(1, MAX_DEFERRAL_ATTEMPTS + 1): + claimed = await queue.claim("worker", now=now) + assert claimed is not None + await queue.defer( + claimed, "worker", "PROVIDER_TEMPORARILY_UNAVAILABLE", now=now + ) + now += MAX_BACKOFF_SECONDS + 60 + closed_queue = IdentificationQueueService(store, provider_available=lambda: False) + deduped_id, created = await closed_queue.enqueue_album_with_disposition( + "album-3", input_revision="revision", now=now + 1 + ) + assert deduped_id == job_id + assert created is False + with sqlite3.connect(db_path) as connection: + state = connection.execute( + "SELECT state FROM library_identification_jobs WHERE id = ?", + (job_id,), + ).fetchone()[0] + assert state == "failed" + + # A terminal failure without an attention code still dedupes (no resurrection). + await queue.enqueue_album("album-4", input_revision="revision", now=1) + other = await queue.claim("worker", now=now + 2) + assert other is not None + await queue.fail(other, "worker", "UNRELATED_TERMINAL_CODE", now=now + 3) + deduped_id, created = await queue.enqueue_album_with_disposition( + "album-4", input_revision="revision", now=now + 4 + ) + assert deduped_id == str(other["id"]) + assert created is False + with sqlite3.connect(db_path) as connection: + state = connection.execute( + "SELECT state FROM library_identification_jobs WHERE id = ?", + (other["id"],), + ).fetchone()[0] + assert state == "failed" + + +@pytest.mark.asyncio +async def test_identification_worker_crash_loop_hits_the_deferral_cap( + store: NativeLibraryStore, + db_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import services.native.identification_queue_service as queue_module + + await _seed_album(store) + queue = IdentificationQueueService(store) + await queue.enqueue_album("album-1", input_revision="revision", now=1) + + clock = {"now": 2.0} + + def advancing_time() -> float: + clock["now"] += MAX_BACKOFF_SECONDS + 60 + return clock["now"] + + monkeypatch.setattr(queue_module.time, "time", advancing_time) + service = AsyncMock() + service.run_claimed_job.side_effect = RuntimeError("boom") + wakeups = SimpleNamespace( + revision=lambda _kind: 0, + wait=AsyncMock( + side_effect=[None] * MAX_DEFERRAL_ATTEMPTS + [asyncio.CancelledError()] + ), + ) + await run_target_identification_worker( + lambda: queue, + lambda: service, + worker_id="test-worker", + work_wakeups=wakeups, + ) + + with sqlite3.connect(db_path) as connection: + row = connection.execute( + "SELECT state, last_failure_code, attention_cause, attempt_count " + "FROM library_identification_jobs WHERE local_album_id = 'album-1'" + ).fetchone() + review = connection.execute( + "SELECT state, reason_code, attempt_id " + "FROM library_identification_reviews" + ).fetchone() + # The crashed job deferred through the cap instead of looping forever, and + # the terminal failure surfaced a review row. + assert row == ("failed", "MAX_DEFERRALS_EXCEEDED", "UNEXPECTED_ERROR", 10) + assert review == ("needs_review", "MAX_DEFERRALS_EXCEEDED", None) + assert await queue.claim("worker", now=clock["now"] + 100) is None + + +@pytest.mark.asyncio +async def test_deleted_or_retired_subject_terminates_immediately( + store: NativeLibraryStore, db_path: Path +) -> None: + await _seed_album(store) + job = await _claimed_job(store) + provider = FakeProvider() + with sqlite3.connect(db_path) as connection: + connection.execute( + "UPDATE local_albums SET retired_into_album_id = 'album-merged' " + "WHERE id = 'album-1'" + ) + + outcome = await _service( + store, + provider, + FakeFingerprinter(FingerprintResult(status="disabled"), enabled=False), + ).run_claimed_job(job, "worker", now=3) + + assert outcome == "attention" + assert provider.calls == [] + with sqlite3.connect(db_path) as connection: + row = connection.execute( + "SELECT state, last_failure_code, terminal_at " + "FROM library_identification_jobs WHERE id = 'job-album-1'" + ).fetchone() + assert row[0] == "failed" + assert row[1] == "SUBJECT_NOT_AVAILABLE" + assert row[2] == 3 + queue = IdentificationQueueService(store) + assert await queue.claim("worker", now=4) is None + + +@pytest.mark.asyncio +async def test_empty_subject_defers_until_grace_sweep_terminates( + store: NativeLibraryStore, db_path: Path +) -> None: + await _seed_album(store) + job = await _claimed_job(store) + provider = FakeProvider() + with sqlite3.connect(db_path) as connection: + connection.execute( + "UPDATE local_tracks SET availability = 'missing' WHERE id = 'track-1'" + ) + + outcome = await _service( + store, + provider, + FakeFingerprinter(FingerprintResult(status="disabled"), enabled=False), + ).run_claimed_job(job, "worker", now=3) + + assert outcome == "provider_deferred" + queue = IdentificationQueueService(store) + await queue.recover(now=3 + 3600) + with sqlite3.connect(db_path) as connection: + state = connection.execute( + "SELECT state FROM library_identification_jobs WHERE id = 'job-album-1'" + ).fetchone()[0] + assert state == "queued" + + await queue.recover(now=3 + 25 * 3600) + with sqlite3.connect(db_path) as connection: + row = connection.execute( + "SELECT state, last_failure_code, terminal_at " + "FROM library_identification_jobs WHERE id = 'job-album-1'" + ).fetchone() + assert row[0] == "failed" + assert row[1] == "SUBJECT_NOT_AVAILABLE" + assert row[2] == 3 + 25 * 3600 + + +@pytest.mark.asyncio +async def test_reset_provider_deferrals_clears_backoff_and_leaves_other_reasons( + store: NativeLibraryStore, db_path: Path +) -> None: + await _seed_album(store) + await _seed_album(store, "2") + queue = IdentificationQueueService(store) + await queue.enqueue_album("album-1", input_revision="revision", now=1) + await queue.enqueue_album("album-2", input_revision="revision", now=1) + provider_job = await queue.claim("worker", now=2) + assert provider_job is not None + await queue.defer(provider_job, "worker", "PROVIDER_TEMPORARILY_UNAVAILABLE", now=2) + subject_job = await queue.claim("worker", now=3) + assert subject_job is not None + await queue.defer(subject_job, "worker", "SUBJECT_NOT_AVAILABLE", now=3) + assert await queue.claim("worker", now=4) is None + + assert await queue.reset_provider_deferrals(now=100) == 1 + + with sqlite3.connect(db_path) as connection: + provider_row = connection.execute( + "SELECT attempt_count, not_before, last_failure_code " + "FROM library_identification_jobs WHERE id = ?", + (provider_job["id"],), + ).fetchone() + subject_row = connection.execute( + "SELECT attempt_count, not_before, last_failure_code " + "FROM library_identification_jobs WHERE id = ?", + (subject_job["id"],), + ).fetchone() + assert provider_row == (0, 0, None) + assert subject_row == (1, 33, "SUBJECT_NOT_AVAILABLE") + reclaimed = await queue.claim("worker", now=100) + assert reclaimed is not None + assert reclaimed["id"] == provider_job["id"] + + @pytest.mark.asyncio async def test_pause_at_candidate_and_fingerprint_checkpoints_releases_lease_without_attempt_increment( store: NativeLibraryStore, db_path: Path diff --git a/backend/tests/services/native/test_library_policy_resolver.py b/backend/tests/services/native/test_library_policy_resolver.py index 91d8d8bfe..872477bbc 100644 --- a/backend/tests/services/native/test_library_policy_resolver.py +++ b/backend/tests/services/native/test_library_policy_resolver.py @@ -145,3 +145,23 @@ def test_revision_only_tracks_policy_input(tmp_path: Path) -> None: assert first.policy_revision == second.policy_revision assert first.policy_revision != excluded.policy_revision + + +def test_enabled_toggle_does_not_change_policy_revision(tmp_path: Path) -> None: + root = tmp_path / "Music" + root.mkdir() + enabled = LibraryPolicyResolver(_settings(root)) + disabled = _settings(root) + disabled.enabled = False + + assert LibraryPolicyResolver(disabled).policy_revision == enabled.policy_revision + + +def test_normalise_preserves_the_enabled_flag(tmp_path: Path) -> None: + root = tmp_path / "Music" + root.mkdir() + disabled = _settings(root) + disabled.enabled = False + + assert LibraryPolicyResolver(disabled).settings.enabled is False + assert LibraryPolicyResolver(_settings(root)).settings.enabled is True diff --git a/backend/tests/services/native/test_library_review_operations.py b/backend/tests/services/native/test_library_review_operations.py index bcedc0862..ff681e02e 100644 --- a/backend/tests/services/native/test_library_review_operations.py +++ b/backend/tests/services/native/test_library_review_operations.py @@ -6,6 +6,7 @@ from unittest.mock import AsyncMock import pytest +import msgspec.json from api.v1.schemas.library_operations import ( ArtistMergeApplyRequest, @@ -33,6 +34,7 @@ ValidationError, ) from infrastructure.persistence.native_library_store import NativeLibraryStore +from infrastructure.resilience.retry import CircuitOpenError from models.audio import FingerprintResult from models.identification import ( AlbumCandidate, @@ -180,6 +182,25 @@ async def get_exact_release_candidate(self, release_mbid, priority): raise ExternalServiceError("private provider failure") +class _OrderedRepairProvider(_RepairProvider): + def __init__(self) -> None: + self.exact_calls: list[str] = [] + + async def get_exact_release_candidate(self, release_mbid, priority): + self.exact_calls.append(release_mbid) + return await super().get_exact_release_candidate(release_mbid, priority) + + +class _CircuitOpenRepairProvider(_IdentificationProvider): + async def get_album_candidate( + self, release_group_mbid, target_track_count, priority + ): + raise CircuitOpenError("MusicBrainz breaker open") + + async def get_exact_release_candidate(self, release_mbid, priority): + raise CircuitOpenError("MusicBrainz breaker open") + + class _CanonicalReleaseProvider: def __init__( self, @@ -753,6 +774,23 @@ async def test_review_cursor_filters_and_detail_are_bounded( assert "keep_tagged" in detail.available_actions +@pytest.mark.asyncio +async def test_review_actions_include_dismiss_and_policy_excluded_only_dismiss( + store: NativeLibraryStore, +) -> None: + await _seed_album(store, "1") + await _seed_album(store, "2", policy="excluded") + service = LibraryReviewService(store) + + open_detail = await service.detail("review-1") + assert "dismiss" in open_detail.available_actions + assert "exclude" in open_detail.available_actions + assert "retry" in open_detail.available_actions + + policy_excluded = await service.detail("review-2") + assert policy_excluded.available_actions == ["dismiss"] + + @pytest.mark.asyncio async def test_review_supports_every_signed_filter_sort_and_typed_invalid_values( store: NativeLibraryStore, db_path: Path @@ -3755,6 +3793,79 @@ async def test_leave_unmanaged_retains_truthful_group_and_can_be_reenabled( assert await store.get_management_exclusion("album-1") is None +@pytest.mark.asyncio +async def test_leave_unmanaged_works_for_zero_candidate_albums( + store: NativeLibraryStore, +) -> None: + await _seed_album(store, "1") + + class _NoCandidateProvider: + async def search_album_candidate_ids( + self, artist, title, limit, priority + ) -> list[str]: + return [] + + async def search_recording_candidate_ids( + self, artist, title, limit, priority + ) -> list[str]: + return [] + + async def get_album_candidate( + self, release_group_mbid, target_track_count, priority + ) -> None: + return None + + worker = ExplicitReidentificationWorker( + store, + AlbumCandidateService(_NoCandidateProvider()), + AlbumEvidenceEngine(), + ) + + async def ready_job(idempotency_key: str, now: float) -> tuple[str, dict]: + created = await ReidentificationService(store).create_or_coalesce( + "album-1", + "admin", + idempotency_key=idempotency_key, + now=now, + ) + claimed = await store.claim_operation_job( + "worker", now=now + 1, lease_seconds=60, kind="explicit_reidentification" + ) + assert claimed is not None + ready = await worker.run_claimed(claimed, "worker", now=now + 2) + assert ready["state"] in ("ready", "succeeded") + return str(created["id"]), ready + + # Zero-candidate snapshot: leave_unmanaged needs no evidence and succeeds. + job_id, ready = await ready_job("zero-candidate", 10) + selected = await worker.select_candidate( + job_id, + expected_job_revision=int(ready["row_revision"]), + candidate_key="", + confirmation=True, + actor_user_id="admin", + decision_mode="leave_unmanaged", + now=13, + ) + assert selected["terminal_code"] == "LEFT_UNMANAGED" + exclusion = await store.get_management_exclusion("album-1") + assert exclusion is not None + assert exclusion.reason == "administrator_choice" + + # Exact release still requires an actual candidate: no evidence, no exit. + job_id, ready = await ready_job("exact-empty", 15) + with pytest.raises(StaleRevisionError): + await worker.select_candidate( + job_id, + expected_job_revision=int(ready["row_revision"]), + candidate_key="", + confirmation=True, + actor_user_id="admin", + decision_mode="exact_release", + now=18, + ) + + @pytest.mark.asyncio async def test_explicit_reidentification_accepts_provider_canonicalization_of_the_current_embedded_release( store: NativeLibraryStore, db_path: Path @@ -4800,7 +4911,7 @@ async def stop_after_first() -> None: @pytest.mark.asyncio -async def test_repair_audit_generates_missing_evidence_and_provider_failure_is_unverifiable( +async def test_repair_audit_generates_missing_evidence_and_defers_whole_job_when_provider_fails( store: NativeLibraryStore, db_path: Path ) -> None: await _seed_album(store, "1") @@ -4859,23 +4970,149 @@ async def test_repair_audit_generates_missing_evidence_and_provider_failure_is_u "worker", now=8, lease_seconds=60, kind="repair" ) assert claimed is not None - await unavailable.run_claimed_audit(claimed, "worker", now=9) - second_findings = await unavailable.findings(second.id) - by_album = {item.local_album_id: item for item in second_findings.items} - assert by_album["album-1"].finding_code == "unverifiable" - assert by_album["album-1"].reason_code == "PROVIDER_DEFERRED" - assert by_album["album-2"].finding_code == "unverifiable" - assert by_album["album-2"].reason_code == "PROVIDER_DEFERRED" - assert by_album["album-2"].apply_eligible is False - filtered = await unavailable.findings(second.id, finding_category="unverifiable") - assert {item.local_album_id for item in filtered.items} == { - "album-1", - "album-2", - } + deferred = await unavailable.run_claimed_audit(claimed, "worker", now=9) + assert deferred.state == "queued" + assert deferred.succeeded_count == 0 + job_row = await store.get_operation_job(second.id) + assert job_row is not None + assert job_row["state"] == "queued" + assert job_row["lease_owner"] is None + assert job_row["next_attempt_at"] == pytest.approx(9 + 120) + with sqlite3.connect(db_path) as connection: + work = connection.execute( + "SELECT local_album_id, state, failure_code FROM library_operation_work " + "WHERE job_id = ? ORDER BY ordinal", + (second.id,), + ).fetchall() + assert work == [ + ("album-1", "pending", "PROVIDER_DEFERRED"), + ("album-2", "pending", None), + ] + assert (await unavailable.findings(second.id)).items == [] + assert ( + await store.claim_operation_job( + "worker", now=100, lease_seconds=60, kind="repair" + ) + is None + ) + reclaimed = await store.claim_operation_job( + "worker", now=129, lease_seconds=60, kind="repair" + ) + assert reclaimed is not None + assert reclaimed["id"] == second.id + assert reclaimed["next_attempt_at"] is None with pytest.raises(ValidationError, match="category is invalid"): await unavailable.findings(second.id, finding_category="not-a-category") +@pytest.mark.asyncio +async def test_repair_audit_resumes_at_the_deferred_item_after_provider_recovery( + store: NativeLibraryStore, db_path: Path +) -> None: + await _seed_album(store, "1", identity_source="legacy_import") + await _seed_album(store, "2", identity_source="legacy_import") + repair = IdentityRepairService( + store, _UnavailableRepairProvider(), AlbumEvidenceEngine() + ) + created = await repair.create( + RepairCreateRequest(idempotency_key="repair-resume"), "admin", now=3 + ) + claimed = await store.claim_operation_job( + "worker", now=4, lease_seconds=60, kind="repair" + ) + assert claimed is not None + deferred = await repair.run_claimed_audit(claimed, "worker", now=5) + assert deferred.state == "queued" + with sqlite3.connect(db_path) as connection: + work = connection.execute( + "SELECT state, failure_code FROM library_operation_work WHERE job_id = ? " + "ORDER BY ordinal", + (created.id,), + ).fetchall() + assert work == [ + ("pending", "PROVIDER_DEFERRED"), + ("pending", None), + ] + + provider = _OrderedRepairProvider() + recovered = IdentityRepairService(store, provider, AlbumEvidenceEngine()) + reclaimed = await store.claim_operation_job( + "worker", now=5 + 120, lease_seconds=60, kind="repair" + ) + assert reclaimed is not None + assert reclaimed["id"] == created.id + ready = await recovered.run_claimed_audit(reclaimed, "worker", now=5 + 121) + assert ready.state == "ready" + assert ready.repair_summary is not None + assert ready.repair_summary.provider_deferred_count == 0 + assert ready.repair_summary.total_identities == 2 + # The deferred item is the first pending on resume: the audit continues + # exactly where the provider failure left it. + assert provider.exact_calls == ["release-1", "release-2"] + + +@pytest.mark.asyncio +async def test_repair_audit_defers_before_any_provider_call_when_breaker_open( + store: NativeLibraryStore, db_path: Path +) -> None: + await _seed_album(store, "1", identity_source="legacy_import") + provider = _OrderedRepairProvider() + repair = IdentityRepairService( + store, + provider, + AlbumEvidenceEngine(), + provider_available=lambda: False, + ) + created = await repair.create( + RepairCreateRequest(idempotency_key="repair-open-breaker"), "admin", now=3 + ) + claimed = await store.claim_operation_job( + "worker", now=4, lease_seconds=60, kind="repair" + ) + assert claimed is not None + deferred = await repair.run_claimed_audit(claimed, "worker", now=5) + assert deferred.state == "queued" + job_row = await store.get_operation_job(created.id) + assert job_row is not None + assert job_row["next_attempt_at"] == pytest.approx(5 + 120) + assert provider.exact_calls == [] + with sqlite3.connect(db_path) as connection: + work = connection.execute( + "SELECT state, failure_code FROM library_operation_work WHERE job_id = ?", + (created.id,), + ).fetchall() + assert work == [("pending", None)] + + +@pytest.mark.asyncio +async def test_repair_audit_defers_when_provider_raises_circuit_open_error( + store: NativeLibraryStore, db_path: Path +) -> None: + await _seed_album(store, "1", identity_source="legacy_import") + repair = IdentityRepairService( + store, _CircuitOpenRepairProvider(), AlbumEvidenceEngine() + ) + created = await repair.create( + RepairCreateRequest(idempotency_key="repair-circuit-open"), "admin", now=3 + ) + claimed = await store.claim_operation_job( + "worker", now=4, lease_seconds=60, kind="repair" + ) + assert claimed is not None + deferred = await repair.run_claimed_audit(claimed, "worker", now=5) + assert deferred.state == "queued" + job_row = await store.get_operation_job(created.id) + assert job_row is not None + assert job_row["next_attempt_at"] == pytest.approx(5 + 120) + with sqlite3.connect(db_path) as connection: + work = connection.execute( + "SELECT state, failure_code FROM library_operation_work WHERE job_id = ?", + (created.id,), + ).fetchall() + assert work == [("pending", "PROVIDER_DEFERRED")] + assert (await repair.findings(created.id)).items == [] + + @pytest.mark.asyncio async def test_repair_reuses_revision_keyed_fingerprint_as_shared_evidence( store: NativeLibraryStore, @@ -5467,17 +5704,21 @@ async def test_management_identity_preparation_defers_a_complete_mapping_when_pr ) assert claimed is not None - ready = await preparation.run_claimed_audit(claimed, "worker", now=5) - finding = ( - await preparation.findings(created.id, finding_category="unverifiable") - ).items[0] + deferred = await preparation.run_claimed_audit(claimed, "worker", now=5) assert provider.calls == ["release-1"] - assert ready.repair_summary is not None - assert ready.repair_summary.ready_album_count == 0 - assert ready.repair_summary.provider_deferred_count == 1 - assert finding.reason_code == "PROVIDER_DEFERRED" - assert finding.apply_eligible is False + assert deferred.state == "queued" + assert deferred.succeeded_count == 0 + job_row = await store.get_operation_job(created.id) + assert job_row is not None + assert job_row["next_attempt_at"] == pytest.approx(5 + 120) + with sqlite3.connect(db_path) as connection: + work = connection.execute( + "SELECT state, failure_code FROM library_operation_work WHERE job_id = ?", + (created.id,), + ).fetchall() + assert work == [("pending", "PROVIDER_DEFERRED")] + assert (await preparation.findings(created.id)).items == [] @pytest.mark.asyncio @@ -5891,10 +6132,8 @@ async def test_management_identity_preparation_defers_recording_redirect_failure "UPDATE local_track_external_identities SET recording_mbid = 'retired-recording' " "WHERE local_track_id = 'track-1-1'" ) - preparation = IdentityRepairService( - store, - canonical_provider=_CanonicalReleaseProvider(recording_unavailable=True), - ) + provider = _CanonicalReleaseProvider(recording_unavailable=True) + preparation = IdentityRepairService(store, canonical_provider=provider) created = await preparation.create_management_preparation( IdentityPreparationCreateRequest(idempotency_key="recording-provider-failure"), "admin", @@ -5904,14 +6143,20 @@ async def test_management_identity_preparation_defers_recording_redirect_failure "worker", now=4, lease_seconds=60, kind="repair" ) assert claimed is not None - await preparation.run_claimed_audit(claimed, "worker", now=5) + deferred = await preparation.run_claimed_audit(claimed, "worker", now=5) - finding = ( - await preparation.findings(created.id, finding_category="unverifiable") - ).items[0] - assert finding.reason_code == "PROVIDER_DEFERRED" - assert finding.evidence_id is None - assert finding.apply_eligible is False + assert provider.recording_calls == ["retired-recording"] + assert deferred.state == "queued" + job_row = await store.get_operation_job(created.id) + assert job_row is not None + assert job_row["next_attempt_at"] == pytest.approx(5 + 120) + with sqlite3.connect(db_path) as connection: + work = connection.execute( + "SELECT state, failure_code FROM library_operation_work WHERE job_id = ?", + (created.id,), + ).fetchall() + assert work == [("pending", "PROVIDER_DEFERRED")] + assert (await preparation.findings(created.id)).items == [] @pytest.mark.asyncio @@ -6125,6 +6370,7 @@ async def test_management_identity_preparation_blocks_missing_conflicting_and_st @pytest.mark.asyncio async def test_management_identity_preparation_defers_provider_failures( store: NativeLibraryStore, + db_path: Path, ) -> None: await _seed_album(store, "1", identity_source="legacy_import") preparation = IdentityRepairService( @@ -6139,14 +6385,19 @@ async def test_management_identity_preparation_defers_provider_failures( "worker", now=4, lease_seconds=60, kind="repair" ) assert claimed is not None - ready = await preparation.run_claimed_audit(claimed, "worker", now=5) - finding = ( - await preparation.findings(created.id, finding_category="unverifiable") - ).items[0] - assert ready.repair_summary is not None - assert ready.repair_summary.provider_deferred_count == 1 - assert finding.reason_code == "PROVIDER_DEFERRED" - assert finding.apply_eligible is False + deferred = await preparation.run_claimed_audit(claimed, "worker", now=5) + assert deferred.state == "queued" + assert deferred.succeeded_count == 0 + job_row = await store.get_operation_job(created.id) + assert job_row is not None + assert job_row["next_attempt_at"] == pytest.approx(5 + 120) + with sqlite3.connect(db_path) as connection: + work = connection.execute( + "SELECT state, failure_code FROM library_operation_work WHERE job_id = ?", + (created.id,), + ).fetchall() + assert work == [("pending", "PROVIDER_DEFERRED")] + assert (await preparation.findings(created.id)).items == [] @pytest.mark.asyncio @@ -6297,3 +6548,627 @@ async def test_diagnostic_export_is_bounded_redacted_and_ephemeral( "raw_provider_responses", "exception_text", ] + + +def _suggestion_evidence( + *, + suffix: str = "1", + release_mbid: str, + release_group_mbid: str = "rg-suggested", + reason_code: str = "SUPPORTED", + release_date: str | None = "2020-01-01", + album_title: str = "Album 1", + complete: bool = True, +) -> CandidateEvidence: + return CandidateEvidence( + release_group_mbid=release_group_mbid, + release_mbid=release_mbid, + album_title=album_title, + release_date=release_date, + track_evidence=[ + TrackEvidence( + local_track_id=f"track-{suffix}-1", + classification="supported", + candidate_track_title="Track 1", + candidate_disc_number=1, + candidate_track_position=1, + recording_mbid=f"recording-{release_mbid}", + release_track_mbid=( + f"release-track-{release_mbid}" if complete else None + ), + ) + ], + reason_code=reason_code, + matcher_version="identification-test", + ) + + +def _seed_stored_attempt( + db_path: Path, + *, + local_album_id: str, + attempt_id: str, + revisions: tuple[str, str, str], + evidence: list[tuple[str, CandidateEvidence]], + compacted: bool = False, + completed_at: float = 2, +) -> None: + with sqlite3.connect(db_path) as connection: + connection.execute( + "INSERT INTO library_identification_attempts " + "(id, local_album_id, trigger, input_tag_revision, input_policy_revision, " + "input_file_revision, matcher_version, state, terminal_reason_code, " + "started_at, completed_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)", + ( + attempt_id, + local_album_id, + "automatic", + revisions[0], + revisions[2], + revisions[1], + "identification-test", + "ambiguous", + "AMBIGUOUS", + completed_at - 1, + completed_at, + ), + ) + for evidence_id, candidate in evidence: + encoded = msgspec.json.encode(candidate) + connection.execute( + "INSERT INTO library_identification_evidence " + "(id, attempt_id, candidate_key, evidence_json, evidence_size_bytes, " + "compacted, created_at) VALUES (?,?,?,?,?,?,?)", + ( + evidence_id, + attempt_id, + f"candidate-{evidence_id}", + encoded, + len(encoded), + int(compacted), + completed_at, + ), + ) + + +def _tie_release( + release_mbid: str, + *, + status: str | None, + date: str | None, + country: str | None, + track_count: int = 1, +) -> MbManagementRelease: + return MbManagementRelease( + id=release_mbid, + title="Album 1", + status=status, + date=date, + country=country, + media=[MbManagementMedium(position=1, track_count=track_count)], + release_group=MbManagementReleaseGroup(id="rg-suggested", title="Album 1"), + ) + + +class _SuggestedEditionProvider: + def __init__( + self, + releases: dict[str, MbManagementRelease] | None = None, + *, + unavailable: bool = False, + ) -> None: + self.releases = releases or {} + self.unavailable = unavailable + self.calls: list[str] = [] + + async def get_canonical_release( + self, + release_mbid, + *, + includes, + preferred_locales=(), + artist_standardization="credited", + priority, + bypass_cache=False, + ): + self.calls.append(release_mbid) + if self.unavailable: + raise ExternalServiceError("private canonical provider failure") + return self.releases.get(release_mbid) + + async def resolve_recording_mbid(self, recording_mbid, *, priority): + return recording_mbid + + +async def _run_preparation( + store: NativeLibraryStore, + provider: object, + *, + idempotency_key: str, +): + preparation = IdentityRepairService(store, canonical_provider=provider) + created = await preparation.create_management_preparation( + IdentityPreparationCreateRequest(idempotency_key=idempotency_key), + "admin", + now=3, + ) + claimed = await store.claim_operation_job( + "worker", now=4, lease_seconds=60, kind="repair" + ) + assert claimed is not None + ready = await preparation.run_claimed_audit(claimed, "worker", now=5) + return preparation, created, ready + + +@pytest.mark.asyncio +async def test_management_identity_preparation_suggests_single_stored_candidate( + store: NativeLibraryStore, db_path: Path +) -> None: + await _seed_album(store, "1") + context = await store.get_album_identification_context("album-1") + assert context is not None + _seed_stored_attempt( + db_path, + local_album_id="album-1", + attempt_id="attempt-suggested", + revisions=album_input_revisions(context["tracks"]), + evidence=[ + ("evidence-suggested", _suggestion_evidence(release_mbid="release-one")) + ], + ) + provider = _SuggestedEditionProvider() + preparation, created, ready = await _run_preparation( + store, provider, idempotency_key="suggest-single" + ) + assert ready.repair_summary is not None + finding = ( + await preparation.findings( + created.id, finding_category="exact_release_required" + ) + ).items[0] + assert finding.finding_code == "exact_release_suggested" + assert finding.reason_code == "EXACT_EDITION_SUGGESTED" + assert finding.apply_eligible is True + assert finding.evidence_id == "evidence-suggested" + assert finding.suggested_edition is not None + assert finding.suggested_edition.release_mbid == "release-one" + assert finding.suggested_edition.release_group_mbid == "rg-suggested" + assert finding.suggested_edition.title == "Album 1" + assert finding.suggested_edition.date == "2020-01-01" + assert finding.suggested_edition.track_count == 1 + assert finding.suggested_edition.competing_count == 1 + assert provider.calls == [] + + +@pytest.mark.asyncio +async def test_management_identity_preparation_tie_breaks_official_first( + store: NativeLibraryStore, db_path: Path +) -> None: + await _seed_album(store, "1") + context = await store.get_album_identification_context("album-1") + assert context is not None + _seed_stored_attempt( + db_path, + local_album_id="album-1", + attempt_id="attempt-tie-official", + revisions=album_input_revisions(context["tracks"]), + evidence=[ + ("evidence-a", _suggestion_evidence(release_mbid="release-a")), + ("evidence-b", _suggestion_evidence(release_mbid="release-b")), + ], + ) + provider = _SuggestedEditionProvider( + { + "release-a": _tie_release( + "release-a", status="Promotion", date="2019-01-01", country="XW" + ), + "release-b": _tie_release( + "release-b", status="Official", date="2021-05-01", country="DE" + ), + } + ) + preparation, created, _ = await _run_preparation( + store, provider, idempotency_key="suggest-tie-official" + ) + finding = ( + await preparation.findings( + created.id, finding_category="exact_release_required" + ) + ).items[0] + assert finding.finding_code == "exact_release_suggested" + assert finding.evidence_id == "evidence-b" + assert finding.suggested_edition is not None + assert finding.suggested_edition.release_mbid == "release-b" + assert finding.suggested_edition.status == "Official" + assert finding.suggested_edition.competing_count == 2 + assert sorted(provider.calls) == ["release-a", "release-b"] + + +@pytest.mark.asyncio +async def test_management_identity_preparation_tie_breaks_earliest_then_worldwide( + store: NativeLibraryStore, db_path: Path +) -> None: + await _seed_album(store, "1") + context = await store.get_album_identification_context("album-1") + assert context is not None + _seed_stored_attempt( + db_path, + local_album_id="album-1", + attempt_id="attempt-tie-date", + revisions=album_input_revisions(context["tracks"]), + evidence=[ + ("evidence-a", _suggestion_evidence(release_mbid="release-a")), + ("evidence-b", _suggestion_evidence(release_mbid="release-b")), + ], + ) + provider = _SuggestedEditionProvider( + { + "release-a": _tie_release( + "release-a", status="Official", date="2020-01-01", country="XW" + ), + "release-b": _tie_release( + "release-b", status="Official", date="2019-03-01", country="DE" + ), + } + ) + preparation, created, _ = await _run_preparation( + store, provider, idempotency_key="suggest-tie-date" + ) + finding = ( + await preparation.findings( + created.id, finding_category="exact_release_required" + ) + ).items[0] + assert finding.suggested_edition is not None + assert finding.suggested_edition.release_mbid == "release-b" + assert finding.suggested_edition.date == "2019-03-01" + assert finding.suggested_edition.country == "DE" + assert finding.suggested_edition.competing_count == 2 + + +@pytest.mark.asyncio +async def test_management_identity_preparation_tie_breaks_worldwide_on_equal_dates( + store: NativeLibraryStore, db_path: Path +) -> None: + await _seed_album(store, "1") + context = await store.get_album_identification_context("album-1") + assert context is not None + _seed_stored_attempt( + db_path, + local_album_id="album-1", + attempt_id="attempt-tie-xw", + revisions=album_input_revisions(context["tracks"]), + evidence=[ + ("evidence-a", _suggestion_evidence(release_mbid="release-a")), + ("evidence-b", _suggestion_evidence(release_mbid="release-b")), + ], + ) + provider = _SuggestedEditionProvider( + { + "release-a": _tie_release( + "release-a", status="Official", date="2020-01-01", country="DE" + ), + "release-b": _tie_release( + "release-b", + status="Official", + date="2020-01-01", + country="XW", + track_count=11, + ), + } + ) + preparation, created, _ = await _run_preparation( + store, provider, idempotency_key="suggest-tie-xw" + ) + finding = ( + await preparation.findings( + created.id, finding_category="exact_release_required" + ) + ).items[0] + assert finding.suggested_edition is not None + assert finding.suggested_edition.release_mbid == "release-b" + assert finding.suggested_edition.country == "XW" + assert finding.suggested_edition.track_count == 11 + + +@pytest.mark.asyncio +async def test_management_identity_preparation_defers_suggestion_fetch_failures( + store: NativeLibraryStore, db_path: Path +) -> None: + await _seed_album(store, "1") + context = await store.get_album_identification_context("album-1") + assert context is not None + _seed_stored_attempt( + db_path, + local_album_id="album-1", + attempt_id="attempt-deferred", + revisions=album_input_revisions(context["tracks"]), + evidence=[ + ("evidence-a", _suggestion_evidence(release_mbid="release-a")), + ("evidence-b", _suggestion_evidence(release_mbid="release-b")), + ], + ) + preparation = IdentityRepairService( + store, canonical_provider=_SuggestedEditionProvider(unavailable=True) + ) + created = await preparation.create_management_preparation( + IdentityPreparationCreateRequest(idempotency_key="suggest-deferred"), + "admin", + now=3, + ) + claimed = await store.claim_operation_job( + "worker", now=4, lease_seconds=60, kind="repair" + ) + assert claimed is not None + deferred = await preparation.run_claimed_audit(claimed, "worker", now=5) + assert deferred.state == "queued" + job_row = await store.get_operation_job(created.id) + assert job_row is not None + assert job_row["next_attempt_at"] == pytest.approx(5 + 120) + with sqlite3.connect(db_path) as connection: + work = connection.execute( + "SELECT state, failure_code FROM library_operation_work WHERE job_id = ?", + (created.id,), + ).fetchall() + assert work == [("pending", "PROVIDER_DEFERRED")] + assert (await preparation.findings(created.id)).items == [] + + +@pytest.mark.asyncio +async def test_management_identity_preparation_bare_when_evidence_unusable( + store: NativeLibraryStore, db_path: Path +) -> None: + await _seed_album(store, "1") + await _seed_album(store, "2") + await _seed_album(store, "3") + await _seed_album(store, "4") + contexts = {} + for suffix in ("1", "2", "3", "4"): + context = await store.get_album_identification_context(f"album-{suffix}") + assert context is not None + contexts[suffix] = context + _seed_stored_attempt( + db_path, + local_album_id="album-1", + attempt_id="attempt-stale-revisions", + revisions=("stale-tag", "stale-file", "stale-policy"), + evidence=[("evidence-1", _suggestion_evidence(release_mbid="release-1"))], + ) + _seed_stored_attempt( + db_path, + local_album_id="album-2", + attempt_id="attempt-compacted", + revisions=album_input_revisions(contexts["2"]["tracks"]), + evidence=[("evidence-2", _suggestion_evidence(release_mbid="release-2"))], + compacted=True, + ) + _seed_stored_attempt( + db_path, + local_album_id="album-3", + attempt_id="attempt-incomplete", + revisions=album_input_revisions(contexts["3"]["tracks"]), + evidence=[ + ( + "evidence-3", + _suggestion_evidence(release_mbid="release-3", complete=False), + ) + ], + ) + preparation, created, _ = await _run_preparation( + store, _SuggestedEditionProvider(), idempotency_key="suggest-bare" + ) + findings = await preparation.findings( + created.id, finding_category="exact_release_required", limit=200 + ) + assert len(findings.items) == 4 + for item in findings.items: + assert item.finding_code == "exact_release_required" + assert item.reason_code == "EXACT_EDITION_NOT_ACCEPTED" + assert item.apply_eligible is False + assert item.suggested_edition is None + + +@pytest.mark.asyncio +async def test_management_identity_preparation_apply_seals_suggested_edition( + store: NativeLibraryStore, db_path: Path +) -> None: + await _seed_album(store, "1") + context = await store.get_album_identification_context("album-1") + assert context is not None + _seed_stored_attempt( + db_path, + local_album_id="album-1", + attempt_id="attempt-apply", + revisions=album_input_revisions(context["tracks"]), + evidence=[ + ("evidence-apply", _suggestion_evidence(release_mbid="release-apply")) + ], + ) + before_estimate = await store.estimate_management_identity_preparation([]) + assert before_estimate["exact_release_required_count"] == 1 + preparation, created, ready = await _run_preparation( + store, _SuggestedEditionProvider(), idempotency_key="suggest-apply" + ) + await preparation.begin_management_preparation_apply( + created.id, + expected_row_revision=ready.row_revision, + confirmation=True, + now=6, + ) + claimed_apply = await store.claim_operation_job( + "worker", now=7, lease_seconds=60, kind="repair" + ) + assert claimed_apply is not None + done = await preparation.run_claimed_apply(claimed_apply, "worker", "admin", now=8) + assert done.state == "succeeded" + assert done.succeeded_count == 1 + with sqlite3.connect(db_path) as connection: + identity = connection.execute( + "SELECT release_group_mbid, release_mbid, decision_source, " + "selected_by_user_id, attempt_id FROM local_album_external_identities " + "WHERE local_album_id = 'album-1'" + ).fetchone() + assert identity == ( + "rg-suggested", + "release-apply", + "manual", + "admin", + "attempt-apply", + ) + track = connection.execute( + "SELECT local_track_id, recording_mbid, release_mbid, release_track_mbid, " + "medium_position, release_track_position, decision_source, attempt_id " + "FROM local_track_external_identities WHERE local_track_id = 'track-1-1'" + ).fetchone() + assert track == ( + "track-1-1", + "recording-release-apply", + "release-apply", + "release-track-release-apply", + 1, + 1, + "manual", + "attempt-apply", + ) + review = connection.execute( + "SELECT state, reason_code, decided_by_user_id " + "FROM library_identification_reviews WHERE id = 'review-1'" + ).fetchone() + assert review == ("resolved", "SUGGESTED_EDITION_ACCEPTED", "admin") + action = connection.execute( + "SELECT action_kind, reason_code, before_json, after_json " + "FROM library_catalog_actions WHERE local_album_id = 'album-1'" + ).fetchone() + assert action is not None + assert action[0] == "accept_suggested_edition" + assert action[1] == "SUGGESTED_EDITION_ACCEPTED" + assert json.loads(action[2]) == {} + after_payload = json.loads(action[3]) + assert after_payload["release_group_mbid"] == "rg-suggested" + assert after_payload["release_mbid"] == "release-apply" + assert after_payload["tracks"] == [ + { + "local_track_id": "track-1-1", + "recording_mbid": "recording-release-apply", + "release_track_mbid": "release-track-release-apply", + "medium_position": 1, + "release_track_position": 1, + } + ] + finding_row = connection.execute( + "SELECT state, apply_result FROM library_identity_repair_findings " + "WHERE job_id = ?", + (created.id,), + ).fetchone() + assert finding_row == ("applied", "EDITION_ACCEPTED") + after_estimate = await store.estimate_management_identity_preparation([]) + assert after_estimate["exact_release_required_count"] == 0 + assert after_estimate["ready_album_count"] == 1 + + +@pytest.mark.asyncio +async def test_management_identity_preparation_apply_skips_stale_suggested_editions( + store: NativeLibraryStore, db_path: Path +) -> None: + for suffix in ("1", "2", "3", "4"): + await _seed_album(store, suffix) + context = await store.get_album_identification_context(f"album-{suffix}") + assert context is not None + revisions = album_input_revisions(context["tracks"]) + if suffix == "4": + _seed_stored_attempt( + db_path, + local_album_id="album-4", + attempt_id="attempt-unsafe-4", + revisions=revisions, + evidence=[ + ( + "evidence-unsafe-4", + _suggestion_evidence( + suffix="4", + release_mbid="release-stale-4", + reason_code="AMBIGUOUS", + ), + ) + ], + completed_at=1, + ) + _seed_stored_attempt( + db_path, + local_album_id=f"album-{suffix}", + attempt_id=f"attempt-stale-{suffix}", + revisions=album_input_revisions(context["tracks"]), + evidence=[ + ( + f"evidence-stale-{suffix}", + _suggestion_evidence( + suffix=suffix, release_mbid=f"release-stale-{suffix}" + ), + ) + ], + ) + preparation, created, ready = await _run_preparation( + store, _SuggestedEditionProvider(), idempotency_key="suggest-stale-apply" + ) + with sqlite3.connect(db_path) as connection: + connection.execute( + "UPDATE local_albums SET row_revision = row_revision + 1 " + "WHERE id = 'album-1'" + ) + connection.execute( + "UPDATE local_tracks SET tag_revision = 'tag-changed' " + "WHERE id = 'track-2-1'" + ) + connection.execute( + "UPDATE library_identity_repair_findings SET evidence_id = 'evidence-unsafe-4' " + "WHERE job_id = ? AND local_album_id = 'album-4'", + (created.id,), + ) + connection.commit() + sealed_context = await store.get_album_identification_context("album-3") + assert sealed_context is not None + await store.attach_album_identity( + LocalAlbumExternalIdentity( + local_album_id="album-3", + release_group_mbid="rg-other", + release_mbid="release-other", + decision_source="manual", + selected_at=6, + ), + expected_album_revision=int(sealed_context["album"]["row_revision"]), + ) + await preparation.begin_management_preparation_apply( + created.id, + expected_row_revision=ready.row_revision, + confirmation=True, + now=6, + ) + claimed_apply = await store.claim_operation_job( + "worker", now=7, lease_seconds=60, kind="repair" + ) + assert claimed_apply is not None + done = await preparation.run_claimed_apply(claimed_apply, "worker", "admin", now=8) + assert done.state == "succeeded" + assert done.succeeded_count == 0 + assert done.skipped_count == 4 + with sqlite3.connect(db_path) as connection: + findings = connection.execute( + "SELECT local_album_id, state, apply_result " + "FROM library_identity_repair_findings WHERE job_id = ? ORDER BY local_album_id", + (created.id,), + ).fetchall() + assert findings == [ + (f"album-{suffix}", "stale", "STALE_SUBJECT") + for suffix in ("1", "2", "3", "4") + ] + identities = connection.execute( + "SELECT local_album_id, release_mbid FROM local_album_external_identities " + "ORDER BY local_album_id" + ).fetchall() + assert identities == [("album-3", "release-other")] + assert ( + connection.execute( + "SELECT COUNT(*) FROM local_track_external_identities" + ).fetchone()[0] + == 0 + ) diff --git a/backend/tests/services/native/test_target_library_policy_service.py b/backend/tests/services/native/test_target_library_policy_service.py index a051fe09f..20028168d 100644 --- a/backend/tests/services/native/test_target_library_policy_service.py +++ b/backend/tests/services/native/test_target_library_policy_service.py @@ -470,6 +470,47 @@ async def test_empty_roots_save_allowed_when_catalog_is_empty() -> None: ) +@pytest.mark.asyncio +async def test_disabled_save_with_roots_succeeds() -> None: + previous = TypedLibrarySettings( + library_roots=[_root("root", "/music", "Music")] + ) + disabled = TypedLibrarySettings( + library_roots=[_root("root", "/music", "Music")], enabled=False + ) + proposed = LibraryPolicyResolver(disabled) + revision = LibraryPolicyResolver(previous).policy_revision + base = Mock() + base.current_settings.return_value = previous + base.current_settings_raw.return_value = previous + base.prepare_change.return_value = (proposed, []) + base.rebase_scopes.return_value = [] + base.collapse_scopes.return_value = [] + base.get_settings.return_value = LibrarySettingsResponse( + library_roots=disabled.library_roots, + policy_revision=proposed.policy_revision, + enabled=False, + ) + store = AsyncMock() + store.get_pending_policy.return_value = None + reconciliation = AsyncMock() + reconciliation.commit_boundary.return_value = {"changed": 0, "cancelled": 0} + service = TargetLibraryPolicyService(base, reconciliation, store) + + response = await service.save_settings( + disabled, expected_policy_revision=revision + ) + assert response.enabled is False + assert response.library_roots[0]["id"] == "root" + assert proposed.settings.enabled is False + base.persist_settings.assert_called_once_with( + proposed.settings, expected_policy_revision=revision + ) + reconciliation.commit_boundary.assert_awaited_once_with( + proposed_policy_revision=proposed.policy_revision + ) + + @pytest.mark.asyncio async def test_restorable_roots_excludes_configured_and_reports_derived_paths() -> None: current = TypedLibrarySettings( diff --git a/backend/tests/services/native/test_target_scan_runtime.py b/backend/tests/services/native/test_target_scan_runtime.py index f4c4993fb..8ccf2a752 100644 --- a/backend/tests/services/native/test_target_scan_runtime.py +++ b/backend/tests/services/native/test_target_scan_runtime.py @@ -1,6 +1,8 @@ from __future__ import annotations import asyncio +import errno +import logging import threading import time from contextlib import suppress @@ -35,7 +37,9 @@ run_library_contribution_verification_worker, run_target_identification_worker, run_target_operation_worker, + run_target_worker_watchdog, ) +from infrastructure.resilience.retry import CircuitState from services.native.background_workload_gate import BackgroundWorkloadGate @@ -122,7 +126,9 @@ async def test_supervisor_refreshes_scheduler_and_resolver_each_iteration() -> N coordinator = AsyncMock() coordinator.run_once.return_value = None scheduler = AsyncMock() - resolver = SimpleNamespace(policy_revision="one") + resolver = SimpleNamespace( + policy_revision="one", settings=SimpleNamespace(enabled=True) + ) calls = {"scheduler": 0, "resolver": 0, "settings": 0} def scheduler_getter(): @@ -154,10 +160,39 @@ def settings_getter(): settings_getter, ) - assert calls == {"scheduler": 2, "resolver": 2, "settings": 2} + # One extra resolver read comes from the startup recovery gate; the two + # loop iterations then read it once each. + assert calls == {"scheduler": 2, "resolver": 3, "settings": 2} assert scheduler.tick.await_count == 2 +@pytest.mark.asyncio +async def test_supervisor_skips_recover_tick_and_run_when_library_disabled() -> None: + coordinator = AsyncMock() + coordinator.run_once.return_value = None + scheduler = AsyncMock() + resolver = SimpleNamespace( + policy_revision="one", settings=SimpleNamespace(enabled=False) + ) + wakeups = SimpleNamespace( + revision=lambda _kind: 0, + wait=AsyncMock(side_effect=asyncio.CancelledError()), + ) + await supervise_target_scans( + lambda: coordinator, + lambda: {}, + wakeups, + lambda: scheduler, + lambda: resolver, + lambda: {"frequency": "manual", "daily_time": "03:00", "timezone_name": "UTC"}, + ) + + coordinator.recover.assert_not_awaited() + scheduler.tick.assert_not_awaited() + coordinator.run_once.assert_not_awaited() + wakeups.wait.assert_awaited_once() + + @pytest.mark.asyncio async def test_target_identification_worker_recovers_claims_and_survives_iterations() -> ( None @@ -181,6 +216,156 @@ async def test_target_identification_worker_recovers_claims_and_survives_iterati service.run_claimed_job.assert_awaited_once_with({"id": "job-1"}, "test-worker") +@pytest.mark.asyncio +async def test_identification_worker_defers_crashed_job_with_unexpected_error() -> None: + queue = AsyncMock() + queue.is_paused.return_value = False + queue.claim.side_effect = [{"id": "job-1", "row_revision": 1}, None] + service = AsyncMock() + service.run_claimed_job.side_effect = RuntimeError("boom") + wakeups = SimpleNamespace( + revision=lambda _kind: 0, + wait=AsyncMock(side_effect=asyncio.CancelledError()), + ) + await run_target_identification_worker( + lambda: queue, + lambda: service, + worker_id="test-worker", + work_wakeups=wakeups, + ) + + # The crashed job is deferred as UNEXPECTED_ERROR (feeding the deferral + # cap) instead of being re-claimed into an infinite crash loop. + queue.defer.assert_awaited_once_with( + {"id": "job-1", "row_revision": 1}, "test-worker", "UNEXPECTED_ERROR" + ) + + +def _idle_identification_harness(): + queue = AsyncMock() + queue.is_paused.return_value = False + queue.claim.return_value = None + service = AsyncMock() + wakeups = SimpleNamespace( + revision=lambda _kind: 0, + wait=AsyncMock( + side_effect=[None, asyncio.CancelledError(), asyncio.CancelledError()] + ), + ) + return queue, service, wakeups + + +@pytest.mark.asyncio +async def test_identification_worker_sweeps_provider_deferrals_when_breaker_closed() -> ( + None +): + queue, service, wakeups = _idle_identification_harness() + probe = AsyncMock() + + await run_target_identification_worker( + lambda: queue, + lambda: service, + worker_id="test-worker", + work_wakeups=wakeups, + provider_state_getter=lambda: CircuitState.CLOSED, + probe_provider=probe, + ) + + queue.reset_provider_deferrals.assert_awaited_once() + probe.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_identification_worker_probes_once_per_rate_limit_when_half_open() -> ( + None +): + queue, service, wakeups = _idle_identification_harness() + probe = AsyncMock() + + await run_target_identification_worker( + lambda: queue, + lambda: service, + worker_id="test-worker", + work_wakeups=wakeups, + provider_state_getter=lambda: CircuitState.HALF_OPEN, + probe_provider=probe, + ) + + # Two idle iterations inside the 60s sweep window: exactly one probe. + probe.assert_awaited_once_with() + queue.reset_provider_deferrals.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_identification_worker_skips_provider_sweep_when_breaker_open() -> None: + queue, service, wakeups = _idle_identification_harness() + probe = AsyncMock() + + await run_target_identification_worker( + lambda: queue, + lambda: service, + worker_id="test-worker", + work_wakeups=wakeups, + provider_state_getter=lambda: CircuitState.OPEN, + probe_provider=probe, + ) + + probe.assert_not_awaited() + queue.reset_provider_deferrals.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_worker_watchdog_restarts_dead_worker_only( + monkeypatch: pytest.MonkeyPatch, +) -> None: + registry = TaskRegistry() + monkeypatch.setattr(TaskRegistry, "get_instance", classmethod(lambda cls: registry)) + + async def run_forever() -> None: + await asyncio.Event().wait() + + alive_task = asyncio.get_running_loop().create_task(run_forever()) + registry.register("alive-worker", alive_task) + dead_task = asyncio.get_running_loop().create_task(run_forever()) + registry.register("dead-worker", dead_task) + dead_task.cancel() + with suppress(asyncio.CancelledError): + await dead_task + + restarted: list[asyncio.Task[None]] = [] + + def dead_starter() -> asyncio.Task[None]: + task = asyncio.get_running_loop().create_task(run_forever()) + registry.register("dead-worker", task) + restarted.append(task) + return task + + alive_starter_calls = 0 + + def alive_starter() -> asyncio.Task[None]: + nonlocal alive_starter_calls + alive_starter_calls += 1 + return alive_task + + async def stop_after_first_iteration(_seconds: float) -> None: + raise asyncio.CancelledError + + monkeypatch.setattr(asyncio, "sleep", stop_after_first_iteration) + try: + await run_target_worker_watchdog( + {"dead-worker": dead_starter, "alive-worker": alive_starter} + ) + assert len(restarted) == 1 + assert alive_starter_calls == 0 + assert registry.is_running("dead-worker") + finally: + alive_task.cancel() + for task in restarted: + task.cancel() + with suppress(asyncio.CancelledError): + await asyncio.gather(alive_task, *restarted) + + @pytest.mark.asyncio async def test_identification_worker_starts_no_new_unit_while_scan_is_active() -> None: queue = AsyncMock() @@ -241,6 +426,30 @@ async def activate_scan() -> bool: service.run_claimed_job.assert_not_awaited() +@pytest.mark.asyncio +async def test_identification_worker_skips_claims_when_library_disabled() -> None: + queue = AsyncMock() + queue.is_paused.return_value = False + queue.claim.return_value = None + service = AsyncMock() + wakeups = SimpleNamespace( + revision=lambda _kind: 0, + wait=AsyncMock(side_effect=asyncio.CancelledError()), + ) + await run_target_identification_worker( + lambda: queue, + lambda: service, + worker_id="test-worker", + work_wakeups=wakeups, + enabled_getter=lambda: False, + ) + + queue.recover.assert_not_awaited() + queue.claim.assert_not_awaited() + service.run_claimed_job.assert_not_awaited() + wakeups.wait.assert_awaited_once() + + @pytest.mark.asyncio async def test_target_operation_worker_recovers_and_dispatches_each_iteration() -> None: supervisor = AsyncMock() @@ -263,6 +472,28 @@ async def test_target_operation_worker_recovers_and_dispatches_each_iteration() supervisor.run_once.assert_awaited_with("test-worker") +@pytest.mark.asyncio +async def test_operation_worker_skips_claims_when_library_disabled() -> None: + supervisor = AsyncMock() + recovery = AsyncMock() + wakeups = SimpleNamespace( + revision=lambda _kind: 0, + wait=AsyncMock(side_effect=asyncio.CancelledError()), + ) + await run_target_operation_worker( + lambda: supervisor, + lambda: recovery, + worker_id="test-worker", + work_wakeups=wakeups, + enabled_getter=lambda: False, + ) + + supervisor.recover.assert_not_awaited() + recovery.recover_once.assert_not_awaited() + supervisor.run_once.assert_not_awaited() + wakeups.wait.assert_awaited_once() + + @pytest.mark.asyncio async def test_operation_worker_does_not_reclaim_repairs_during_scan() -> None: store = AsyncMock() @@ -487,6 +718,180 @@ def test_target_compat_module_has_no_legacy_scanner_dependency() -> None: assert "LibraryScanner" not in names +def _scan_run(run_id: str = "run-1") -> ScanRun: + return ScanRun( + id=run_id, + kind="incremental", + trigger="manual", + state="discovering", + phase="discovering", + ) + + +@pytest.mark.asyncio +async def test_walk_oserror_logs_path_and_records_failure_row( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + root = tmp_path / "music" + root.mkdir() + denied = root / "secret" + store = AsyncMock() + store.classify_scan_paths.return_value = {} + store.add_scan_inventory_batch.return_value = (2, 1) + + def denied_walk(*_args, **_kwargs): + raise PermissionError(errno.EACCES, "Permission denied", str(denied)) + yield + + scanner = LibraryInventoryScanner(store, directory_walker=denied_walk) + scope = ScanScope(root_id="root", policy_revision="policy-1") + + with caplog.at_level(logging.WARNING, logger="services.native.library_inventory_scanner"): + _updated, completed, failure_code = await scanner._walk_scope( + _scan_run(), + scope, + root, + root, + SimpleNamespace(resolve=lambda _path: None), + AsyncMock(return_value=True), + ) + + assert completed is False + assert failure_code == "ROOT_PERMISSION_DENIED" + records = store.record_scan_failures.await_args.args[1] + assert [ + (record.failure_code, record.relative_path, record.phase) + for record in records + ] == [("WALK_EACCES", "secret", "discovering")] + store.complete_scan_scope_discovery.assert_awaited_once_with( + "run-1", + "root", + ".", + state="partially_read", + error_code="ROOT_PERMISSION_DENIED", + ) + assert "event=walk_error" in caplog.text + assert "secret" in caplog.text + + +@pytest.mark.asyncio +async def test_wedged_walk_times_out_detaches_producer_and_recovers( + tmp_path: Path, +) -> None: + root = tmp_path / "music" + root.mkdir() + (root / "track.flac").touch() + wedged = threading.Event() + calls = 0 + + def walker(*_args, **_kwargs): + nonlocal calls + calls += 1 + if calls == 1: + yield (str(root), [], ["track.flac"]) + wedged.wait() + return + yield (str(root), [], ["track.flac"]) + + store = AsyncMock() + store.classify_scan_paths.return_value = {"track.flac": ("new", None)} + store.add_scan_inventory_batch.return_value = (2, 1) + scanner = LibraryInventoryScanner( + store, + directory_walker=walker, + walk_deadline_seconds=0.05, + ) + scope = ScanScope(root_id="root", policy_revision="policy-1") + resolver = SimpleNamespace(resolve=lambda _path: None) + checkpoint = AsyncMock(return_value=True) + + started = time.monotonic() + _updated, completed, failure_code = await scanner._walk_scope( + _scan_run(), scope, root, root, resolver, checkpoint + ) + elapsed = time.monotonic() - started + + assert completed is False + assert failure_code == "WALK_TIMEOUT" + assert elapsed < 2.0 + assert len(scanner._detached_walkers) == 1 + records = store.record_scan_failures.await_args.args[1] + assert [record.failure_code for record in records] == ["WALK_TIMEOUT"] + store.complete_scan_scope_discovery.assert_awaited_once_with( + "run-1", + "root", + ".", + state="partially_read", + error_code="WALK_TIMEOUT", + ) + + # Releasing the wedged syscall lets the detached producer finish cleanly. + wedged.set() + deadline = time.monotonic() + 2.0 + while scanner._detached_walkers and time.monotonic() < deadline: + await asyncio.sleep(0.01) + assert not scanner._detached_walkers + + # A subsequent walk on the same scanner is unaffected by the detached one. + _updated, completed, failure_code = await scanner._walk_scope( + _scan_run("run-2"), scope, root, root, resolver, checkpoint + ) + assert completed is True + assert failure_code is None + + +@pytest.mark.asyncio +async def test_root_probe_timeout_fails_the_run_with_walk_timeout( + tmp_path: Path, +) -> None: + root = tmp_path / "music" + root.mkdir() + wedged_probe = threading.Event() + + def probe(_path: Path) -> bool: + wedged_probe.wait() + return True + + store = AsyncMock() + store.get_scan_scope_discovery_state.return_value = "pending" + scanner = LibraryInventoryScanner( + store, + walk_deadline_seconds=0.05, + directory_probe=probe, + ) + scope = ScanScope(root_id="root", policy_revision="policy-1") + + try: + started = time.monotonic() + await asyncio.wait_for( + scanner.discover( + _scan_run(), + [scope], + {"root": root}, + SimpleNamespace(), + AsyncMock(return_value=True), + ), + timeout=2.0, + ) + elapsed = time.monotonic() - started + finally: + wedged_probe.set() + + assert elapsed < 2.0 + records = store.record_scan_failures.await_args.args[1] + assert [record.failure_code for record in records] == ["WALK_TIMEOUT"] + store.complete_scan_scope_discovery.assert_awaited_once_with( + "run-1", + "root", + ".", + state="unavailable", + error_code="WALK_TIMEOUT", + ) + assert ( + store.transition_scan_run.await_args.kwargs["terminal_code"] == "WALK_TIMEOUT" + ) + + @pytest.mark.asyncio async def test_inventory_file_stat_runs_outside_the_event_loop_thread( tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -523,7 +928,7 @@ def record_stat(path: Path, *args, **kwargs): ) scope = ScanScope(root_id="root", policy_revision="policy-1") - _updated, completed = await scanner._walk_scope( + _updated, completed, failure_code = await scanner._walk_scope( run, scope, root, @@ -533,6 +938,7 @@ def record_stat(path: Path, *args, **kwargs): ) assert completed is True + assert failure_code is None assert stat_threads assert event_loop_thread not in stat_threads diff --git a/backend/tests/services/test_artist_release_pagination.py b/backend/tests/services/test_artist_release_pagination.py index abb1a52c0..19104dfde 100644 --- a/backend/tests/services/test_artist_release_pagination.py +++ b/backend/tests/services/test_artist_release_pagination.py @@ -2,6 +2,7 @@ import os import tempfile +from typing import Any os.environ.setdefault("ROOT_APP_DIR", tempfile.mkdtemp()) @@ -9,6 +10,7 @@ from unittest.mock import AsyncMock, MagicMock from core.exceptions import ClientDisconnectedError +from infrastructure.cache.cache_keys import mb_artist_release_groups_key from services.artist_service import ArtistService @@ -46,10 +48,22 @@ def _make_prefs( return p +def _make_dict_cache() -> tuple[AsyncMock, dict[str, Any]]: + """AsyncMock cache backed by a real dict, for multi-request tests.""" + store: dict[str, Any] = {} + cache = AsyncMock() + cache.get = AsyncMock(side_effect=store.get) + cache.set = AsyncMock( + side_effect=lambda key, value, ttl_seconds: store.__setitem__(key, value) + ) + return cache, store + + def _make_service( *, mb_release_pages: list[tuple[list[dict], int]] | None = None, prefs: MagicMock | None = None, + memory_cache: AsyncMock | None = None, ) -> ArtistService: mb_repo = AsyncMock() if mb_release_pages is not None: @@ -65,9 +79,10 @@ def _make_service( wikidata_repo = AsyncMock() - memory_cache = AsyncMock() - memory_cache.get = AsyncMock(return_value=None) - memory_cache.set = AsyncMock() + if memory_cache is None: + memory_cache = AsyncMock() + memory_cache.get = AsyncMock(return_value=None) + memory_cache.set = AsyncMock() disk_cache = AsyncMock() disk_cache.get_artist = AsyncMock(return_value=None) @@ -84,22 +99,6 @@ def _make_service( class TestFilterAwarePagination: - @pytest.mark.asyncio - async def test_disconnect_after_first_upstream_page_prevents_second_stage(self): - batch = [_make_release_group("rg-1", "Album A", "Album")] - svc = _make_service(mb_release_pages=[(batch, 200), (batch, 200)]) - is_disconnected = AsyncMock(side_effect=[False, False, True]) - - with pytest.raises(ClientDisconnectedError): - await svc.get_artist_releases( - ARTIST_MBID, - offset=0, - limit=50, - is_disconnected=is_disconnected, - ) - - svc._mb_repo.get_artist_release_groups.assert_awaited_once() - @pytest.mark.asyncio async def test_single_page_fits_filter(self): rg1 = _make_release_group("rg-1", "Album A", "Album") @@ -135,75 +134,7 @@ async def test_sparse_filter_scans_multiple_batches(self): assert result.albums[0].title == "Real Album" assert result.returned_count == 1 - assert result.source_total_count == 6 - - @pytest.mark.asyncio - async def test_has_more_true_when_unscanned_raw_data_remains(self): - rgs = [_make_release_group("rg-1", "Album 1", "Album")] - svc = _make_service(mb_release_pages=[(rgs, 200), ([], 200), ([], 200)]) - - result = await svc.get_artist_releases(ARTIST_MBID, offset=0, limit=50) - - assert result.has_more is True - assert result.next_offset is not None - assert result.returned_count == 1 - - @pytest.mark.asyncio - async def test_next_offset_is_scan_position(self): - batch1 = [ - _make_release_group(f"rg-{i}", f"Album {i}", "Album") for i in range(100) - ] - svc = _make_service(mb_release_pages=[(batch1, 200)]) - - result = await svc.get_artist_releases(ARTIST_MBID, offset=0, limit=10) - - assert result.has_more is True - assert result.returned_count == 10 - assert result.next_offset == 10 - - @pytest.mark.asyncio - async def test_no_duplicates_within_scan(self): - batch1 = [ - _make_release_group(f"rg-{i}", f"Album {i}", "Album") for i in range(5) - ] - batch2 = [ - _make_release_group(f"rg-{i}", f"Album {i}", "Album") for i in range(3, 8) - ] - svc = _make_service( - mb_release_pages=[ - (batch1, 8), - (batch2, 8), - ] - ) - - result = await svc.get_artist_releases(ARTIST_MBID, offset=0, limit=50) - - all_ids = [r.id for r in result.albums] - assert len(all_ids) == len(set(all_ids)) - - @pytest.mark.asyncio - async def test_no_drops_across_sequential_pages(self): - rgs = [_make_release_group(f"rg-{i}", f"Album {i}", "Album") for i in range(10)] - svc = _make_service( - mb_release_pages=[ - (rgs, 10), - (rgs[3:], 10), - (rgs[6:], 10), - (rgs[9:], 10), - ] - ) - - page1 = await svc.get_artist_releases(ARTIST_MBID, offset=0, limit=3) - page2 = await svc.get_artist_releases(ARTIST_MBID, offset=3, limit=3) - page3 = await svc.get_artist_releases(ARTIST_MBID, offset=6, limit=3) - page4 = await svc.get_artist_releases(ARTIST_MBID, offset=9, limit=3) - - pages = [page1, page2, page3, page4] - assert [page.returned_count for page in pages] == [3, 3, 3, 1] - assert [page.next_offset for page in pages] == [3, 6, 9, None] - ids = [item.id for page in pages for item in page.albums] - assert ids == [item["id"] for item in rgs] - assert len(ids) == len(set(ids)) + assert result.source_total_count == 1 @pytest.mark.asyncio async def test_empty_result_set(self): @@ -262,44 +193,6 @@ async def test_limit_controls_returned_items(self): assert result.has_more is True assert result.next_offset == 3 - @pytest.mark.asyncio - async def test_sparse_pages_honor_limit_without_drops(self): - filtered = [ - _make_release_group(f"rg-b{i}", f"Broadcast {i}", "Broadcast") - for i in range(95) - ] - first_included = [ - _make_release_group(f"rg-a{i}", f"Album {i}", "Album") for i in range(5) - ] - later_included = [ - _make_release_group(f"rg-s{i}", f"Single {i}", "Single") for i in range(20) - ] - svc = _make_service( - mb_release_pages=[ - (filtered + first_included, 120), - (later_included, 120), - (later_included[5:], 120), - (later_included[15:], 120), - ] - ) - - page1 = await svc.get_artist_releases(ARTIST_MBID, offset=0, limit=10) - page2 = await svc.get_artist_releases(ARTIST_MBID, offset=105, limit=10) - page3 = await svc.get_artist_releases(ARTIST_MBID, offset=115, limit=10) - - pages = [page1, page2, page3] - assert [page.returned_count for page in pages] == [10, 10, 5] - assert [page.next_offset for page in pages] == [105, 115, None] - ids = [ - item.id - for page in pages - for items in (page.albums, page.singles, page.eps) - for item in items - ] - expected = [item["id"] for item in first_included + later_included] - assert set(ids) == set(expected) - assert len(ids) == len(set(ids)) - @pytest.mark.asyncio async def test_exception_returns_empty_page(self): svc = _make_service() @@ -342,7 +235,7 @@ async def test_all_types_filtered_out_except_one(self): assert result.returned_count == 1 assert result.albums[0].title == "Found Album" - assert result.source_total_count == 6 + assert result.source_total_count == 1 assert result.has_more is False @pytest.mark.asyncio @@ -363,15 +256,145 @@ async def test_global_sort_across_batches(self): assert result.albums[1].title == "Old Album" @pytest.mark.asyncio - async def test_scan_batch_cap_stops_early(self): - batches = [ - ([_make_release_group(f"rg-{i}", f"Album {i}", "Album")], 5000) - for i in range(25) + async def test_next_offset_is_arithmetic(self): + rgs = [ + _make_release_group(f"rg-{i}", f"Album {i}", "Album") for i in range(100) + ] + cache, _ = _make_dict_cache() + svc = _make_service(mb_release_pages=[(rgs, 100)], memory_cache=cache) + + page1 = await svc.get_artist_releases(ARTIST_MBID, offset=0, limit=10) + page2 = await svc.get_artist_releases(ARTIST_MBID, offset=10, limit=10) + + assert page1.has_more is True + assert page1.returned_count == 10 + assert page1.next_offset == 10 + assert page2.returned_count == 10 + assert page2.next_offset == 20 + + @pytest.mark.asyncio + async def test_no_drops_across_sequential_pages(self): + rgs = [_make_release_group(f"rg-{i}", f"Album {i}", "Album") for i in range(10)] + cache, _ = _make_dict_cache() + svc = _make_service(mb_release_pages=[(rgs, 10)], memory_cache=cache) + + page1 = await svc.get_artist_releases(ARTIST_MBID, offset=0, limit=3) + page2 = await svc.get_artist_releases(ARTIST_MBID, offset=3, limit=3) + page3 = await svc.get_artist_releases(ARTIST_MBID, offset=6, limit=3) + page4 = await svc.get_artist_releases(ARTIST_MBID, offset=9, limit=3) + + pages = [page1, page2, page3, page4] + assert [page.returned_count for page in pages] == [3, 3, 3, 1] + assert [page.next_offset for page in pages] == [3, 6, 9, None] + ids = [item.id for page in pages for item in page.albums] + assert ids == [item["id"] for item in rgs] + assert len(ids) == len(set(ids)) + + @pytest.mark.asyncio + async def test_disconnect_during_multi_page_fetch_raises(self): + batch1 = [ + _make_release_group(f"rg-{i}", f"Album {i}", "Album") for i in range(5) ] - svc = _make_service(mb_release_pages=batches) + batch2 = [ + _make_release_group(f"rg-{i + 5}", f"Album {i + 5}", "Album") + for i in range(5) + ] + svc = _make_service(mb_release_pages=[(batch1, 200), (batch2, 200)]) + is_disconnected = AsyncMock(side_effect=[False, False, True]) + + with pytest.raises(ClientDisconnectedError): + await svc.get_artist_releases( + ARTIST_MBID, + offset=0, + limit=50, + is_disconnected=is_disconnected, + ) + + svc._cache.set.assert_not_awaited() + + @pytest.mark.asyncio + async def test_complete_fetch_cached(self): + batch1 = [ + _make_release_group(f"rg-{i}", f"Album {i}", "Album") for i in range(100) + ] + batch2 = [ + _make_release_group(f"rg-{100 + i}", f"Album {100 + i}", "Album") + for i in range(9) + ] + cache, store = _make_dict_cache() + svc = _make_service( + mb_release_pages=[(batch1, 109), (batch2, 109)], + memory_cache=cache, + ) + + first = await svc.get_artist_releases(ARTIST_MBID, offset=0, limit=50) + second = await svc.get_artist_releases(ARTIST_MBID, offset=0, limit=50) + + assert first.returned_count == 50 + assert first.source_total_count == 109 + assert second.returned_count == 50 + assert second.source_total_count == 109 + assert svc._mb_repo.get_artist_release_groups.await_count == 2 + svc._cache.set.assert_awaited_once() + cached = store[mb_artist_release_groups_key(ARTIST_MBID)] + assert len(cached) == 109 + + @pytest.mark.asyncio + async def test_partial_fetch_not_cached(self): + batch1 = [ + _make_release_group(f"rg-{i}", f"Album {i}", "Album") for i in range(100) + ] + svc = _make_service(mb_release_pages=[(batch1, 200), ([], 200)]) result = await svc.get_artist_releases(ARTIST_MBID, offset=0, limit=50) - assert result.has_more is True - assert result.next_offset is not None - assert result.next_offset == 2 + assert result.returned_count == 50 + assert result.source_total_count == 100 + svc._cache.set.assert_not_awaited() + + @pytest.mark.asyncio + async def test_gid_sorted_pages_no_drop_regression(self): + # MB's browse endpoint pages by one order but its JSON serializer + # re-sorts each page by GID: the target RG (Negative Spaces) can land + # anywhere in a page, and scan-position pagination dropped it. + negative_spaces_id = "fe83cc29-01a9-4650-95ca-d3e135c07278" + page1 = [ + _make_release_group(f"aaaaaaaa-0000-4000-8000-{i:012d}", f"Album {i}", "Album") + for i in range(99) + ] + [ + _make_release_group( + negative_spaces_id, "Negative Spaces", "Album", "2024-11-15" + ) + ] + page2 = [ + _make_release_group(f"bbbbbbbb-0000-4000-8000-{i:012d}", f"Album B{i}", "Album") + for i in range(9) + ] + cache, _ = _make_dict_cache() + svc = _make_service( + mb_release_pages=[(page1, 109), (page2, 109)], + memory_cache=cache, + ) + + page_a = await svc.get_artist_releases(ARTIST_MBID, offset=0, limit=50) + page_b = await svc.get_artist_releases(ARTIST_MBID, offset=50, limit=50) + page_c = await svc.get_artist_releases(ARTIST_MBID, offset=100, limit=50) + + ids = [item.id for page in (page_a, page_b, page_c) for item in page.albums] + assert len(ids) == 109 + assert len(ids) == len(set(ids)) + assert ids.count(negative_spaces_id) == 1 + assert page_c.has_more is False + + @pytest.mark.asyncio + async def test_overlapping_pages_deduped(self): + rgs = [_make_release_group(f"rg-{i}", f"Album {i}", "Album") for i in range(15)] + page1 = rgs[:10] + page2 = rgs[5:] # overlaps page1 by 50% + svc = _make_service(mb_release_pages=[(page1, 15), (page2, 15)]) + + result = await svc.get_artist_releases(ARTIST_MBID, offset=0, limit=50) + + ids = [item.id for item in result.albums] + assert len(ids) == 15 + assert len(ids) == len(set(ids)) diff --git a/backend/tests/services/test_discovery_precache_lock.py b/backend/tests/services/test_discovery_precache_lock.py index 555f990a8..8e8595597 100644 --- a/backend/tests/services/test_discovery_precache_lock.py +++ b/backend/tests/services/test_discovery_precache_lock.py @@ -13,8 +13,12 @@ @pytest.fixture(autouse=True) def _reset_precache_flag(): _ads_module._discovery_precache_running = False + _ads_module._precache_consecutive_failures = 0 + _ads_module._precache_paused_until = 0.0 yield _ads_module._discovery_precache_running = False + _ads_module._precache_consecutive_failures = 0 + _ads_module._precache_paused_until = 0.0 def _make_service( diff --git a/backend/tests/services/test_discovery_precache_pause.py b/backend/tests/services/test_discovery_precache_pause.py new file mode 100644 index 000000000..d129b3b42 --- /dev/null +++ b/backend/tests/services/test_discovery_precache_pause.py @@ -0,0 +1,177 @@ +"""Tests for the discovery precache upstream-outage pause/backoff.""" + +import asyncio +import logging +from contextlib import ExitStack, contextmanager +from time import monotonic +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from services.artist_discovery_service import ArtistDiscoveryService +import services.artist_discovery_service as _ads_module + + +@pytest.fixture(autouse=True) +def _reset_precache_flag(): + _ads_module._discovery_precache_running = False + _ads_module._precache_consecutive_failures = 0 + _ads_module._precache_paused_until = 0.0 + yield + _ads_module._discovery_precache_running = False + _ads_module._precache_consecutive_failures = 0 + _ads_module._precache_paused_until = 0.0 + + +def _make_service( + *, lb_configured: bool = True, lastfm_enabled: bool = False, + client_factory=None, auth_store=None, workload_gate=None, +): + lb_repo = MagicMock() + lb_repo.is_configured.return_value = lb_configured + + lastfm_repo = MagicMock() if lastfm_enabled else None + prefs = MagicMock() + prefs.is_lastfm_enabled.return_value = lastfm_enabled + advanced = MagicMock() + advanced.artist_discovery_precache_concurrency = 2 + prefs.get_advanced_settings.return_value = advanced + + cache = AsyncMock() + cache.get = AsyncMock(return_value=None) + cache.set = AsyncMock() + + library_db = AsyncMock() + library_db.get_all_artist_mbids = AsyncMock(return_value=set()) + + svc = ArtistDiscoveryService( + listenbrainz_repo=lb_repo, + musicbrainz_repo=MagicMock(), + library_db=library_db, + library_repo=MagicMock(), + memory_cache=cache, + lastfm_repo=lastfm_repo, + preferences_service=prefs, + client_factory=client_factory, + auth_store=auth_store, + workload_gate=workload_gate, + ) + return svc + + +@contextmanager +def _patch_sources_hanging(svc): + async def hang(*args, **kwargs): + await asyncio.sleep(1) + return MagicMock() # pragma: no cover + + with ExitStack() as stack: + sim = stack.enter_context( + patch.object(svc, "get_similar_artists", new_callable=AsyncMock, side_effect=hang) + ) + stack.enter_context( + patch.object(svc, "get_top_songs", new_callable=AsyncMock, side_effect=hang) + ) + stack.enter_context( + patch.object(svc, "get_top_albums", new_callable=AsyncMock, side_effect=hang) + ) + yield sim + + +@contextmanager +def _patch_sources_working(svc): + with ExitStack() as stack: + stack.enter_context( + patch.object(svc, "get_similar_artists", new_callable=AsyncMock, return_value=MagicMock()) + ) + stack.enter_context( + patch.object(svc, "get_top_songs", new_callable=AsyncMock, return_value=MagicMock()) + ) + stack.enter_context( + patch.object(svc, "get_top_albums", new_callable=AsyncMock, return_value=MagicMock()) + ) + yield + + +@pytest.mark.asyncio +async def test_consecutive_unit_failures_pause_precache(caplog, monkeypatch): + """5 consecutive unit timeouts trip the pause; the next call does no work.""" + svc = _make_service() + monkeypatch.setattr(_ads_module, "_DISCOVERY_WORKER_TIMEOUT", 0.05) + + with caplog.at_level(logging.INFO), _patch_sources_hanging(svc): + for i in range(5): + result = await svc.precache_artist_discovery([f"mbid-{i}"], delay=0) + assert result == 0 + + assert _ads_module._precache_paused_until > monotonic() + assert "Discovery precache paused for 1800s after 5 consecutive unit failures" in caplog.text + + # The 6th call returns 0 in milliseconds without invoking any source. + with _patch_sources_hanging(svc) as sim: + result = await svc.precache_artist_discovery(["mbid-5"], delay=0) + assert result == 0 + assert sim.await_count == 0 + + +@pytest.mark.asyncio +async def test_pause_expiry_probe_success_resets(monkeypatch): + """After the pause expires, a successful probe resets the failure streak.""" + svc = _make_service() + monkeypatch.setattr(_ads_module, "_DISCOVERY_WORKER_TIMEOUT", 0.05) + + with _patch_sources_hanging(svc): + for i in range(5): + await svc.precache_artist_discovery([f"mbid-{i}"], delay=0) + assert _ads_module._precache_paused_until > monotonic() + + # Pause expires (or is cleared); sources recover. + _ads_module._precache_paused_until = 0.0 + with _patch_sources_working(svc): + result = await svc.precache_artist_discovery(["mbid-ok"], delay=0) + + assert result == 1 + + +@pytest.mark.asyncio +async def test_success_resets_failure_streak(monkeypatch): + """A single success between failures keeps the streak below the threshold.""" + svc = _make_service() + monkeypatch.setattr(_ads_module, "_DISCOVERY_WORKER_TIMEOUT", 0.05) + + # Two genuine unit failures (hanging sources time out). + with _patch_sources_hanging(svc): + await svc.precache_artist_discovery(["mbid-a"], delay=0) + await svc.precache_artist_discovery(["mbid-b"], delay=0) + assert _ads_module._precache_consecutive_failures == 2 + + # One success resets the streak. + with _patch_sources_working(svc): + result = await svc.precache_artist_discovery(["mbid-c"], delay=0) + assert result == 1 + assert _ads_module._precache_consecutive_failures == 0 + + # Two more failures: streak is 2, not 4 - the pause must not trip. + with _patch_sources_hanging(svc): + await svc.precache_artist_discovery(["mbid-d"], delay=0) + await svc.precache_artist_discovery(["mbid-e"], delay=0) + assert _ads_module._precache_consecutive_failures == 2 + assert _ads_module._precache_paused_until == 0.0 + + +@pytest.mark.asyncio +async def test_chunk_loop_aborts_mid_list_when_pause_trips(monkeypatch): + """Queued chunk units fast-complete once the pause trips instead of fetching.""" + svc = _make_service() + monkeypatch.setattr(_ads_module, "_DISCOVERY_WORKER_TIMEOUT", 0.05) + + with _patch_sources_hanging(svc) as sim: + result = await svc.precache_artist_discovery( + [f"mbid-{i}" for i in range(30)], delay=0 + ) + + assert result == 0 + assert _ads_module._precache_paused_until > monotonic() + # 5 recorded failures + at most 2 in-flight + slack: units queued behind the + # pause must not invoke sources. + assert sim.await_count <= 8 diff --git a/backend/tests/services/test_discovery_precache_progress.py b/backend/tests/services/test_discovery_precache_progress.py index 15332f5ff..af2c135e6 100644 --- a/backend/tests/services/test_discovery_precache_progress.py +++ b/backend/tests/services/test_discovery_precache_progress.py @@ -6,6 +6,17 @@ import pytest from services.artist_discovery_service import ArtistDiscoveryService +import services.artist_discovery_service as _ads_module + + +@pytest.fixture(autouse=True) +def _reset_pause_state(): + """No test process may see stale pause state from another test.""" + _ads_module._precache_consecutive_failures = 0 + _ads_module._precache_paused_until = 0.0 + yield + _ads_module._precache_consecutive_failures = 0 + _ads_module._precache_paused_until = 0.0 def _make_service(*, lb_configured: bool = True, lastfm_enabled: bool = False): diff --git a/backend/tests/services/test_download_service.py b/backend/tests/services/test_download_service.py index 70ce484be..ebf051e0e 100644 --- a/backend/tests/services/test_download_service.py +++ b/backend/tests/services/test_download_service.py @@ -1096,6 +1096,38 @@ async def test_import_held_places_and_resolves(tmp_path): svc._orchestrator.settle_after_manual_import.assert_awaited_once_with("t-1") +@pytest.mark.asyncio +async def test_import_held_without_library_root_propagates_and_stays_held(tmp_path): + """No library root configured: the ConfigurationError propagates to the route's + 400 mapping and the row stays held - the user restores a root and retries.""" + import threading + + from infrastructure.persistence.download_store import DownloadStore + + store = DownloadStore(db_path=tmp_path / "library.db", write_lock=threading.Lock()) + held_file = tmp_path / "held" / "x.flac" + held_file.parent.mkdir() + held_file.write_bytes(b"audio") + hid = await _record_held(store, held_file) + fp = MagicMock() + fp.place_held_file = AsyncMock( + side_effect=ConfigurationError( + "No library root is configured - restore one in Settings → Library, then try again." + ) + ) + svc = _held_service(store, fp) + store.resolve_held_import = AsyncMock() + + with pytest.raises(ConfigurationError, match="No library root is configured"): + await svc.import_held(hid, "user-a", "user") + + store.resolve_held_import.assert_not_awaited() # NOT resolved: retry stays possible + svc._orchestrator.settle_after_manual_import.assert_not_awaited() + held = await store.list_held_imports("user-a", "user") + assert [value.id for value in held] == [hid] + assert await store.has_unresolved_held_for_task("t-1") is True + + @pytest.mark.asyncio async def test_import_held_unknown_id_raises_not_found(tmp_path): import threading diff --git a/backend/tests/services/test_file_processor.py b/backend/tests/services/test_file_processor.py index 7e3037fb5..dcc076e61 100644 --- a/backend/tests/services/test_file_processor.py +++ b/backend/tests/services/test_file_processor.py @@ -1198,6 +1198,45 @@ async def test_place_held_file_imports_bypassing_verify(tmp_path: Path): assert await manager.has_album("rg-9") is True +@pytest.mark.asyncio +async def test_place_held_file_without_library_root_raises_configuration_error( + tmp_path: Path, +): + """Every root removed since the file was held: an actionable, recoverable 400 + (the user restores a root and retries) instead of an IndexError 500 - and the + publisher is never touched, so nothing lands in a half-imported state.""" + from core.exceptions import ConfigurationError + from models.held_import import HeldImport + + publisher = AsyncMock() + fp, _manager, _client, _library, _downloads = _make_processor( + tmp_path, publisher=publisher + ) + fp._library_paths = [] # roots cleared after the hold was recorded + held_dir = tmp_path / "held" + held_dir.mkdir() + held_file = held_dir / "src.flac" + shutil.copy(_FLAC, held_file) + held = HeldImport( + id=1, + user_id="user-a", + held_path=str(held_file), + reason="fingerprint_mismatch", + source="usenet", + status="held", + created_at=0.0, + release_group_mbid="rg-9", + recording_mbid="rec-3", + track_number=3, + naming_template=_TEMPLATE, + ) + + with pytest.raises(ConfigurationError, match="No library root is configured"): + await fp.place_held_file(held) + + publisher.assert_not_called() + + # -- P2: the file's OWN tags vs the requested identity (2026-07-05 wrong-single) -- diff --git a/backend/tests/services/test_jellyfin_library_service.py b/backend/tests/services/test_jellyfin_library_service.py index 720aaea0d..c844b4f5a 100644 --- a/backend/tests/services/test_jellyfin_library_service.py +++ b/backend/tests/services/test_jellyfin_library_service.py @@ -1,5 +1,6 @@ from __future__ import annotations +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock import pytest @@ -216,3 +217,145 @@ async def test_artist_summary_has_play_count(self): result = await service.get_favorites_expanded(limit=10) assert len(result.artists) == 1 assert result.artists[0].play_count == 77 + + +class TestImportPlaylistMapsAlbumMbids: + """import_playlist stores the MusicBrainz MBID as album_id, not the + Jellyfin album GUID, so the MBID-keyed local catalog can match (#150).""" + + @staticmethod + def _requesting(): + return SimpleNamespace(id="u1") + + @staticmethod + def _playlist_service(): + ps = MagicMock() + ps.get_by_source_ref = AsyncMock(return_value=None) + ps.create_playlist = AsyncMock(return_value=SimpleNamespace(id="dn-pl-1")) + ps.add_tracks = AsyncMock() + return ps + + @staticmethod + def _track_item(id: str, album_id: str, track_number: int = 1) -> JellyfinItem: + return JellyfinItem( + id=id, + name=f"Song {id}", + type="Audio", + artist_name="Artist", + album_name="Album", + album_id=album_id, + artist_id="artist-1", + index_number=track_number, + parent_index_number=1, + duration_ticks=200 * 10_000_000, + ) + + def _wire_playlist(self, repo, track_items): + repo.get_playlists = AsyncMock( + return_value=[_item(id="pl-1", name="Mix", type="Playlist")] + ) + repo.get_playlist = AsyncMock( + return_value=_item(id="pl-1", name="Mix", type="Playlist") + ) + repo.get_playlist_items = AsyncMock(return_value=track_items) + + @pytest.mark.asyncio + async def test_stores_release_group_mbid_as_album_id(self): + service, repo = _make_service() + self._wire_playlist(repo, [self._track_item("t1", "guid-a")]) + repo.get_album_detail = AsyncMock( + return_value=_item( + id="guid-a", + provider_ids={ + "MusicBrainzReleaseGroup": "rg-123", + "MusicBrainzAlbum": "rel-456", + }, + ) + ) + ps = self._playlist_service() + + await service.import_playlist("pl-1", ps, self._requesting()) + + track_dicts = ps.add_tracks.call_args[0][2] + assert track_dicts[0]["album_id"] == "rg-123" + assert track_dicts[0]["track_source_id"] == "t1" + assert track_dicts[0]["source_type"] == "jellyfin" + + @pytest.mark.asyncio + async def test_falls_back_to_musicbrainz_album_provider_id(self): + service, repo = _make_service() + self._wire_playlist(repo, [self._track_item("t1", "guid-a")]) + repo.get_album_detail = AsyncMock( + return_value=_item(id="guid-a", provider_ids={"MusicBrainzAlbum": "rel-456"}) + ) + ps = self._playlist_service() + + await service.import_playlist("pl-1", ps, self._requesting()) + + track_dicts = ps.add_tracks.call_args[0][2] + assert track_dicts[0]["album_id"] == "rel-456" + + @pytest.mark.asyncio + async def test_keeps_guid_when_album_detail_missing(self): + service, repo = _make_service() + self._wire_playlist(repo, [self._track_item("t1", "guid-a")]) + repo.get_album_detail = AsyncMock(return_value=None) + ps = self._playlist_service() + + await service.import_playlist("pl-1", ps, self._requesting()) + + track_dicts = ps.add_tracks.call_args[0][2] + assert track_dicts[0]["album_id"] == "guid-a" + + @pytest.mark.asyncio + async def test_keeps_guid_when_no_provider_ids(self): + service, repo = _make_service() + self._wire_playlist(repo, [self._track_item("t1", "guid-a")]) + repo.get_album_detail = AsyncMock(return_value=_item(id="guid-a")) + ps = self._playlist_service() + + await service.import_playlist("pl-1", ps, self._requesting()) + + track_dicts = ps.add_tracks.call_args[0][2] + assert track_dicts[0]["album_id"] == "guid-a" + + @pytest.mark.asyncio + async def test_fetches_each_distinct_album_once(self): + service, repo = _make_service() + self._wire_playlist( + repo, + [ + self._track_item("t1", "guid-a", track_number=1), + self._track_item("t2", "guid-a", track_number=2), + self._track_item("t3", "guid-b", track_number=1), + ], + ) + repo.get_album_detail = AsyncMock( + side_effect=lambda guid: _item( + id=guid, provider_ids={"MusicBrainzReleaseGroup": f"rg-{guid}"} + ) + ) + ps = self._playlist_service() + + await service.import_playlist("pl-1", ps, self._requesting()) + + assert repo.get_album_detail.await_count == 2 + awaited = {c.args[0] for c in repo.get_album_detail.await_args_list} + assert awaited == {"guid-a", "guid-b"} + track_dicts = ps.add_tracks.call_args[0][2] + assert [t["album_id"] for t in track_dicts] == [ + "rg-guid-a", + "rg-guid-a", + "rg-guid-b", + ] + + @pytest.mark.asyncio + async def test_already_imported_skips_album_fetches(self): + service, repo = _make_service() + ps = self._playlist_service() + ps.get_by_source_ref = AsyncMock(return_value=SimpleNamespace(id="dn-pl-1")) + + result = await service.import_playlist("pl-1", ps, self._requesting()) + + assert result.already_imported is True + repo.get_album_detail.assert_not_called() diff --git a/backend/tests/services/test_jellyfin_playback_service.py b/backend/tests/services/test_jellyfin_playback_service.py index b99727a9c..9a6bcfa53 100644 --- a/backend/tests/services/test_jellyfin_playback_service.py +++ b/backend/tests/services/test_jellyfin_playback_service.py @@ -1,3 +1,4 @@ +import logging import pytest from unittest.mock import AsyncMock, MagicMock @@ -225,3 +226,131 @@ async def test_start_playback_app_level_auth_failure_still_raises(self): with pytest.raises(JellyfinAuthError): await svc.start_playback("item-1") + + +class TestPerUserProxyStreaming: + """Streaming proxies through the user's linked repo, app account as fallback.""" + + @staticmethod + def _stream_result(): + from repositories.navidrome_models import StreamProxyResult + + async def _chunks(): + yield b"data" + + return StreamProxyResult( + status_code=200, + headers={"Content-Type": "audio/flac"}, + media_type="audio/flac", + body_chunks=_chunks(), + ) + + @staticmethod + def _factory(per_user_repo): + factory = MagicMock() + factory.resolve_jellyfin = AsyncMock(return_value=per_user_repo) + return factory + + @pytest.mark.asyncio + async def test_proxy_stream_uses_per_user_repo_when_linked(self): + app_repo = _make_repo() + per_user = _make_repo() + per_user.proxy_get_stream = AsyncMock(return_value=self._stream_result()) + svc = JellyfinPlaybackService( + jellyfin_repo=app_repo, client_factory=self._factory(per_user) + ) + + await svc.proxy_stream("item-1", range_header="bytes=0-", user_id="u1") + + per_user.proxy_get_stream.assert_awaited_once_with( + "item-1", range_header="bytes=0-" + ) + app_repo.proxy_get_stream.assert_not_called() + + @pytest.mark.asyncio + async def test_proxy_head_uses_per_user_repo_when_linked(self): + app_repo = _make_repo() + per_user = _make_repo() + per_user.proxy_head_stream = AsyncMock(return_value=self._stream_result()) + svc = JellyfinPlaybackService( + jellyfin_repo=app_repo, client_factory=self._factory(per_user) + ) + + await svc.proxy_head("item-1", user_id="u1") + + per_user.proxy_head_stream.assert_awaited_once_with("item-1") + app_repo.proxy_head_stream.assert_not_called() + + @pytest.mark.asyncio + async def test_proxy_stream_falls_back_to_app_repo_when_unlinked(self): + app_repo = _make_repo() + app_repo.proxy_get_stream = AsyncMock(return_value=self._stream_result()) + svc = JellyfinPlaybackService( + jellyfin_repo=app_repo, client_factory=self._factory(None) + ) + + await svc.proxy_stream("item-1", user_id="u1") + + app_repo.proxy_get_stream.assert_awaited_once_with( + "item-1", range_header=None + ) + + @pytest.mark.asyncio + async def test_proxy_stream_falls_back_to_app_repo_on_linked_auth_failure( + self, caplog + ): + from core.exceptions import JellyfinAuthError + + app_repo = _make_repo() + app_repo.proxy_get_stream = AsyncMock(return_value=self._stream_result()) + per_user = _make_repo() + per_user.proxy_get_stream = AsyncMock( + side_effect=JellyfinAuthError("revoked") + ) + svc = JellyfinPlaybackService( + jellyfin_repo=app_repo, client_factory=self._factory(per_user) + ) + + with caplog.at_level(logging.WARNING): + await svc.proxy_stream("item-1", user_id="u1") + + per_user.proxy_get_stream.assert_awaited_once() + app_repo.proxy_get_stream.assert_awaited_once_with( + "item-1", range_header=None + ) + assert any( + "app account" in record.message and "u1" in record.message + for record in caplog.records + ) + + @pytest.mark.asyncio + async def test_proxy_head_falls_back_to_app_repo_on_linked_auth_failure(self): + from core.exceptions import JellyfinAuthError + + app_repo = _make_repo() + app_repo.proxy_head_stream = AsyncMock(return_value=self._stream_result()) + per_user = _make_repo() + per_user.proxy_head_stream = AsyncMock( + side_effect=JellyfinAuthError("revoked") + ) + svc = JellyfinPlaybackService( + jellyfin_repo=app_repo, client_factory=self._factory(per_user) + ) + + await svc.proxy_head("item-1", user_id="u1") + + per_user.proxy_head_stream.assert_awaited_once() + app_repo.proxy_head_stream.assert_awaited_once_with("item-1") + + @pytest.mark.asyncio + async def test_proxy_stream_app_level_auth_failure_still_raises(self): + from core.exceptions import JellyfinAuthError + + app_repo = _make_repo() + app_repo.proxy_get_stream = AsyncMock( + side_effect=JellyfinAuthError("bad api key") + ) + svc = JellyfinPlaybackService(jellyfin_repo=app_repo) + + with pytest.raises(JellyfinAuthError): + await svc.proxy_stream("item-1") diff --git a/backend/tests/services/test_navidrome_stream_proxy.py b/backend/tests/services/test_navidrome_stream_proxy.py index 6ecd29019..c222e4688 100644 --- a/backend/tests/services/test_navidrome_stream_proxy.py +++ b/backend/tests/services/test_navidrome_stream_proxy.py @@ -92,3 +92,71 @@ async def test_proxy_stream_raises_416(): with pytest.raises(ExternalServiceError, match="416"): await service.proxy_stream("song-1", "bytes=9999-") + + +class TestPerUserProxy: + """Streaming proxies through the user's linked repo, app account as fallback.""" + + @staticmethod + def _stream_result(): + from repositories.navidrome_repository import StreamProxyResult + + async def fake_chunks(): + yield b"audio" + + return StreamProxyResult( + status_code=200, + headers={"Content-Type": "audio/flac"}, + media_type="audio/flac", + body_chunks=fake_chunks(), + ) + + @staticmethod + def _factory(per_user_repo): + factory = MagicMock() + factory.resolve_navidrome = AsyncMock(return_value=per_user_repo) + return factory + + @pytest.mark.asyncio + async def test_proxy_stream_uses_per_user_repo_when_linked(self): + app_repo = MagicMock() + per_user = MagicMock() + per_user.proxy_get_stream = AsyncMock(return_value=self._stream_result()) + service = NavidromePlaybackService( + navidrome_repo=app_repo, client_factory=self._factory(per_user) + ) + + await service.proxy_stream("song-1", "bytes=0-", user_id="u1") + + per_user.proxy_get_stream.assert_awaited_once_with( + "song-1", range_header="bytes=0-" + ) + app_repo.proxy_get_stream.assert_not_called() + + @pytest.mark.asyncio + async def test_proxy_head_uses_per_user_repo_when_linked(self): + app_repo = MagicMock() + per_user = MagicMock() + per_user.proxy_head_stream = AsyncMock(return_value=self._stream_result()) + service = NavidromePlaybackService( + navidrome_repo=app_repo, client_factory=self._factory(per_user) + ) + + await service.proxy_head("song-1", user_id="u1") + + per_user.proxy_head_stream.assert_awaited_once_with("song-1") + app_repo.proxy_head_stream.assert_not_called() + + @pytest.mark.asyncio + async def test_proxy_stream_falls_back_to_app_repo_when_unlinked(self): + app_repo = MagicMock() + app_repo.proxy_get_stream = AsyncMock(return_value=self._stream_result()) + service = NavidromePlaybackService( + navidrome_repo=app_repo, client_factory=self._factory(None) + ) + + await service.proxy_stream("song-1", None, user_id="u1") + + app_repo.proxy_get_stream.assert_awaited_once_with( + "song-1", range_header=None + ) diff --git a/backend/tests/services/test_playlist_source_resolution.py b/backend/tests/services/test_playlist_source_resolution.py index 928f7a74b..a9f37dd3d 100644 --- a/backend/tests/services/test_playlist_source_resolution.py +++ b/backend/tests/services/test_playlist_source_resolution.py @@ -480,3 +480,70 @@ async def _match(album_id, album_name, artist_name): assert sorted(result["t-a"]) == ["navidrome"] assert sorted(result["t-b"]) == ["navidrome"] + + +class TestGuidAlbumIdFallback: + """Pre-fix Jellyfin imports stored the Jellyfin album GUID as album_id, which + the MBID-keyed matchers can never hit. Resolution re-keys the GUID to the + album's MusicBrainz id via Jellyfin provider ids (no migration, #150).""" + + @pytest.mark.asyncio + async def test_guid_album_id_resolves_local_source(self, tmp_path): + service, repo = _make_service(tmp_path) + track = _make_track( + album_id="jf-guid-1", + source_type="jellyfin", + available_sources=["jellyfin"], + ) + repo.get_tracks = MagicMock(return_value=[track]) + + jf = _make_jf_service(found=False) + jf.resolve_album_mbid = AsyncMock(return_value="mbid-abc") + local = _make_local_service() + + result = await service.resolve_track_sources( + "p-1", jf_service=jf, local_service=local, + ) + + local.match_album_by_mbid.assert_called_once_with("mbid-abc") + assert sorted(result["t-1"]) == ["jellyfin", "local"] + repo.batch_link_library_files.assert_called_once_with("p-1", {"t-1": "789"}) + updates = repo.batch_update_available_sources.call_args[0][1] + assert sorted(updates["t-1"]) == ["jellyfin", "local"] + + @pytest.mark.asyncio + async def test_guid_fallback_does_not_poison_mbid_cache_key(self, tmp_path): + cache = AsyncMock() + cache.get = AsyncMock(return_value=None) + repo = MagicMock() + service = PlaylistService(repo=repo, cache_dir=tmp_path, cache=cache) + jf = _make_jf_service(found=False) + jf.resolve_album_mbid = AsyncMock(return_value="mbid-abc") + + await service._resolve_album_sources("jf-guid-1", jf, None) + + set_key = cache.set.call_args[0][0] + assert "jf-guid-1" in set_key + assert "mbid-abc" not in set_key + + @pytest.mark.asyncio + async def test_no_fallback_when_mbid_match_found(self, tmp_path): + service, _ = _make_service(tmp_path) + jf = _make_jf_service(found=False) + jf.match_album_by_mbid = AsyncMock( + return_value=SimpleNamespace( + found=True, + tracks=[ + SimpleNamespace( + track_number=1, title="Wall Street Shuffle", jellyfin_id="jf-9" + ) + ], + ) + ) + jf.resolve_album_mbid = AsyncMock(return_value="mbid-other") + local = _make_local_service() + + await service._resolve_album_sources("mbid-abc", jf, local) + + jf.resolve_album_mbid.assert_not_called() + local.match_album_by_mbid.assert_called_once_with("mbid-abc") diff --git a/backend/tests/services/test_plex_playback_service.py b/backend/tests/services/test_plex_playback_service.py index 8f53d29fc..24654320e 100644 --- a/backend/tests/services/test_plex_playback_service.py +++ b/backend/tests/services/test_plex_playback_service.py @@ -186,3 +186,69 @@ async def test_stopped_uses_per_user_repo_when_linked(self): assert result is True per_user.now_playing.assert_awaited_once_with("rk-1", state="stopped") app_repo.now_playing.assert_not_awaited() + + +class TestPerUserProxy: + """Streaming proxies through the user's linked repo, app account as fallback.""" + + @staticmethod + def _factory(per_user_repo): + factory = MagicMock() + factory.resolve_plex = AsyncMock(return_value=per_user_repo) + return factory + + @pytest.mark.asyncio + async def test_proxy_stream_uses_per_user_repo_when_linked(self): + app_repo = MagicMock() + per_user = MagicMock() + per_user.proxy_get_stream = AsyncMock( + return_value=StreamProxyResult( + status_code=200, headers={}, media_type="audio/mpeg", + body_chunks=iter([b"data"]), + ) + ) + service = PlexPlaybackService( + plex_repo=app_repo, client_factory=self._factory(per_user) + ) + + await service.proxy_stream("/part/key", range_header="bytes=0-", user_id="u1") + + per_user.proxy_get_stream.assert_awaited_once_with( + "/part/key", range_header="bytes=0-" + ) + app_repo.proxy_get_stream.assert_not_called() + + @pytest.mark.asyncio + async def test_proxy_head_uses_per_user_repo_when_linked(self): + app_repo = MagicMock() + per_user = MagicMock() + per_user.proxy_head_stream = AsyncMock( + return_value=StreamProxyResult(status_code=200, headers={}, media_type=None) + ) + service = PlexPlaybackService( + plex_repo=app_repo, client_factory=self._factory(per_user) + ) + + await service.proxy_head("/part/key", user_id="u1") + + per_user.proxy_head_stream.assert_awaited_once_with("/part/key") + app_repo.proxy_head_stream.assert_not_called() + + @pytest.mark.asyncio + async def test_proxy_stream_falls_back_to_app_repo_when_unlinked(self): + app_repo = MagicMock() + app_repo.proxy_get_stream = AsyncMock( + return_value=StreamProxyResult( + status_code=200, headers={}, media_type="audio/mpeg", + body_chunks=iter([b"data"]), + ) + ) + service = PlexPlaybackService( + plex_repo=app_repo, client_factory=self._factory(None) + ) + + await service.proxy_stream("/part/key", user_id="u1") + + app_repo.proxy_get_stream.assert_awaited_once_with( + "/part/key", range_header=None + ) diff --git a/backend/tests/services/test_settings_cache_invalidation.py b/backend/tests/services/test_settings_cache_invalidation.py index 40d39c91e..c50566a8f 100644 --- a/backend/tests/services/test_settings_cache_invalidation.py +++ b/backend/tests/services/test_settings_cache_invalidation.py @@ -295,3 +295,125 @@ async def test_youtube_settings_change_clears_home_cache(): mock_repo_fn.cache_clear.assert_called_once() for key in home_keys: assert await cache.get(key) is None + + +@pytest.fixture +def mb_live_state(): + """Snapshot and restore the live MusicBrainz module state around a test.""" + from repositories.musicbrainz_base import ( + get_mb_api_base, + set_mb_api_base, + mb_rate_limiter, + ) + + base = get_mb_api_base() + rate = mb_rate_limiter.rate + capacity = mb_rate_limiter.capacity + yield + set_mb_api_base(base) + mb_rate_limiter.update_rate(rate) + mb_rate_limiter.update_capacity(capacity) + + +def _mb_settings(api_url: str, rate_limit: float, concurrent_searches: int): + from api.v1.schemas.settings import MusicBrainzConnectionSettings + + return MusicBrainzConnectionSettings( + api_url=api_url, + rate_limit=rate_limit, + concurrent_searches=concurrent_searches, + ) + + +def _seed_mb_live_state(settings) -> None: + from repositories.musicbrainz_base import set_mb_api_base, mb_rate_limiter + + set_mb_api_base(settings.api_url) + mb_rate_limiter.update_rate(settings.rate_limit) + mb_rate_limiter.update_capacity(settings.concurrent_searches) + + +def _spy_mb_side_effects(monkeypatch): + from unittest.mock import MagicMock + + from repositories.musicbrainz_base import mb_circuit_breaker, mb_deduplicator + + reset = MagicMock() + clear = MagicMock() + monkeypatch.setattr(mb_circuit_breaker, "reset", reset) + monkeypatch.setattr(mb_deduplicator, "clear", clear) + return reset, clear + + +@pytest.mark.asyncio(loop_scope="function") +async def test_musicbrainz_settings_unchanged_skips_reset_and_cache_clear( + mb_live_state, monkeypatch +): + """Saving identical MusicBrainz settings is a no-op: no breaker reset, no + deduplicator clear, caches survive - endpoint behavior did not change.""" + service, cache = await _build_service() + settings = _mb_settings("https://mb.mirror.example/ws/2", 2.0, 4) + _seed_mb_live_state(settings) + reset, clear = _spy_mb_side_effects(monkeypatch) + + mb_keys = [f"{p}noop" for p in musicbrainz_prefixes()] + await _populate(cache, mb_keys) + + await service.on_musicbrainz_settings_changed(settings) + + reset.assert_not_called() + clear.assert_not_called() + for key in mb_keys: + assert await cache.get(key) == "v" + + +@pytest.mark.asyncio(loop_scope="function") +async def test_musicbrainz_settings_endpoint_change_resets_and_clears( + mb_live_state, monkeypatch +): + """An endpoint change resets the breaker, clears the deduplicator, and + clears the MusicBrainz cache prefixes.""" + service, cache = await _build_service() + _seed_mb_live_state(_mb_settings("https://mb-a.example/ws/2", 2.0, 4)) + reset, clear = _spy_mb_side_effects(monkeypatch) + + mb_keys = [f"{p}endpoint" for p in musicbrainz_prefixes()] + await _populate(cache, mb_keys) + + await service.on_musicbrainz_settings_changed( + _mb_settings("https://mb-b.example/ws/2", 2.0, 4) + ) + + from repositories.musicbrainz_base import get_mb_api_base + + reset.assert_called_once() + clear.assert_called_once() + assert get_mb_api_base() == "https://mb-b.example/ws/2" + for key in mb_keys: + assert await cache.get(key) is None + + +@pytest.mark.asyncio(loop_scope="function") +async def test_musicbrainz_settings_rate_only_change_resets_and_clears( + mb_live_state, monkeypatch +): + """A rate-only change re-arms the limiter, so it is behavior-changing and + must reset the breaker and clear caches too.""" + service, cache = await _build_service() + _seed_mb_live_state(_mb_settings("https://mb-a.example/ws/2", 2.0, 4)) + reset, clear = _spy_mb_side_effects(monkeypatch) + + mb_keys = [f"{p}rate" for p in musicbrainz_prefixes()] + await _populate(cache, mb_keys) + + await service.on_musicbrainz_settings_changed( + _mb_settings("https://mb-a.example/ws/2", 3.0, 4) + ) + + from repositories.musicbrainz_base import mb_rate_limiter + + reset.assert_called_once() + clear.assert_called_once() + assert mb_rate_limiter.rate == 3.0 + for key in mb_keys: + assert await cache.get(key) is None diff --git a/backend/tests/services/test_source_playlist_import.py b/backend/tests/services/test_source_playlist_import.py index 64a2b0669..7b9763bf3 100644 --- a/backend/tests/services/test_source_playlist_import.py +++ b/backend/tests/services/test_source_playlist_import.py @@ -93,6 +93,7 @@ def _jellyfin_service(playlists=None, items=None) -> JellyfinLibraryService: repo.get_most_played_albums = AsyncMock(return_value=[]) repo.get_library_stats = AsyncMock(return_value={"album_count": 0, "artist_count": 0, "track_count": 0}) repo.get_albums = AsyncMock(return_value=([], 0)) + repo.get_album_detail = AsyncMock(return_value=None) type(repo).stats_ttl = PropertyMock(return_value=600) prefs = MagicMock() conn = MagicMock() diff --git a/backend/tests/test_advanced_settings_roundtrip.py b/backend/tests/test_advanced_settings_roundtrip.py index 2c97432f5..0aafb7959 100644 --- a/backend/tests/test_advanced_settings_roundtrip.py +++ b/backend/tests/test_advanced_settings_roundtrip.py @@ -217,3 +217,27 @@ def test_defaults_match(self, field: str, default_val) -> None: frontend = AdvancedSettingsFrontend() assert getattr(backend, field) == default_val assert getattr(frontend, field) == default_val + + +class TestPreferLocalCoverArtRoundTrip: + def test_default_value_is_true(self) -> None: + settings = AdvancedSettings() + assert settings.prefer_local_cover_art is True + + def test_frontend_default_is_true(self) -> None: + frontend = AdvancedSettingsFrontend() + assert frontend.prefer_local_cover_art is True + + def test_roundtrip_preserves_true(self) -> None: + backend = AdvancedSettings(prefer_local_cover_art=True) + frontend = AdvancedSettingsFrontend.from_backend(backend) + assert frontend.prefer_local_cover_art is True + restored = frontend.to_backend() + assert restored.prefer_local_cover_art is True + + def test_roundtrip_preserves_false(self) -> None: + backend = AdvancedSettings(prefer_local_cover_art=False) + frontend = AdvancedSettingsFrontend.from_backend(backend) + assert frontend.prefer_local_cover_art is False + restored = frontend.to_backend() + assert restored.prefer_local_cover_art is False diff --git a/entrypoint.sh b/entrypoint.sh index 2184b18f6..b107698e7 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,6 +1,19 @@ #!/bin/sh set -e +# A bind mount over /app shadows the image's application code with the user's +# data directory; fail fast here instead of emitting confusing writability or +# runtime errors against the shadowing mount later. /app is always present in +# the image, so a missing /app means we are outside the container (e.g. tests +# or a hand-run shell) and the check is skipped. +if [ -d /app ] && { [ ! -f /app/main.py ] || [ ! -f /app/.droppedneedle-source-revision ] || [ ! -f /app/maintenance/automatic_upgrade.py ]; }; then + echo "[init] FATAL: /app does not contain the DroppedNeedle application code." + echo "[init] A bind mount over /app hides the application with your data." + echo "[init] Mount data subdirectories only: /app/config, /app/cache, /app/plugins, /app/imports." + echo "[init] Remove the /app bind mount and restart." + exit 1 +fi + REQUESTED_UMASK=${UMASK:-027} case "$REQUESTED_UMASK" in [0-7][0-7][0-7]|0[0-7][0-7][0-7]) ;; diff --git a/frontend/src/lib/api/playlists.ts b/frontend/src/lib/api/playlists.ts index 84b747ab8..5da63a40f 100644 --- a/frontend/src/lib/api/playlists.ts +++ b/frontend/src/lib/api/playlists.ts @@ -20,6 +20,7 @@ export interface PlaylistTrack { duration: number | null; created_at: string; plex_rating_key: string | null; + library_file_id: string | null; } export interface PlaylistSummary { diff --git a/frontend/src/lib/components/BaseImage.svelte b/frontend/src/lib/components/BaseImage.svelte index 6a55e5299..791b57223 100644 --- a/frontend/src/lib/components/BaseImage.svelte +++ b/frontend/src/lib/components/BaseImage.svelte @@ -153,7 +153,9 @@ } return responsiveCoverUrl ? `${coverUrl} 250w, ${responsiveCoverUrl} 500w` : undefined; }); - let visualSourceKey = $derived(useRemoteUrl && resolvedRemoteUrl ? resolvedRemoteUrl : coverUrl); + let visualSourceKey = $derived( + useRemoteUrl && resolvedRemoteUrl && !remoteError ? resolvedRemoteUrl : coverUrl + ); let sizeClasses = $derived(imageType === 'album' ? albumSizeClasses : artistSizeClasses); let sizeClass = $derived(sizeClasses[size]); let roundedClass = $derived(roundedClasses[rounded]); @@ -181,6 +183,13 @@ scheduleVisualSettlement(); } }); + $effect(() => { + // a stalled CDN image emits neither load nor error: at the settle deadline flip + // to the covers proxy (cached image or 202 warm) instead of a dead placeholder + if (useRemoteUrl && resolvedRemoteUrl && !remoteError && visualSettled && !imgLoaded) { + remoteError = true; + } + }); $effect(() => { const source = imageType === 'album' ? (canonicalAlbumCoverUrl ?? customUrl ?? mbid) : mbid; diff --git a/frontend/src/lib/components/BaseImage.svelte.spec.ts b/frontend/src/lib/components/BaseImage.svelte.spec.ts index de7bfcc87..ddeadf6e9 100644 --- a/frontend/src/lib/components/BaseImage.svelte.spec.ts +++ b/frontend/src/lib/components/BaseImage.svelte.spec.ts @@ -1,4 +1,4 @@ -import { page } from '@vitest/browser/context'; +import { cdp, page } from '@vitest/browser/context'; import { describe, expect, it, vi, beforeEach } from 'vitest'; import { render } from 'vitest-browser-svelte'; @@ -34,6 +34,22 @@ import BaseImage from './BaseImage.svelte'; const validMbid = 'b1392450-e666-3926-a536-22c65f834433'; const cdnUrl = 'https://r2.theaudiodb.com/images/media/artist/thumb/abc123.jpg'; +interface FetchCdpSession { + send( + method: 'Fetch.enable', + params: { patterns: Array<{ urlPattern: string }> } + ): Promise; + send(method: 'Fetch.disable'): Promise; +} + +// hold CDN requests so the remote img deterministically stalls: the fake key 404s whenever +// the real network beats the fake-timer clock, and then the stall path is never exercised +async function holdCdnRequests(): Promise<() => Promise> { + const session = cdp() as unknown as FetchCdpSession; + await session.send('Fetch.enable', { patterns: [{ urlPattern: 'https://r2.theaudiodb.com/*' }] }); + return () => session.send('Fetch.disable'); +} + function renderComponent( overrides: Partial<{ mbid: string; @@ -213,15 +229,44 @@ describe('BaseImage.svelte - warming skeleton', () => { vi.useRealTimers(); }); - it('settles an unresolved direct image even when the browser emits no load or error event', async () => { + it('falls back to the covers proxy when a direct image emits neither load nor error', async () => { + const releaseCdn = await holdCdnRequests(); vi.useFakeTimers(); - renderComponent({ remoteUrl: cdnUrl, imageType: 'artist', lazy: false }); - - await vi.advanceTimersByTimeAsync(6500); + try { + renderComponent({ remoteUrl: cdnUrl, imageType: 'artist', lazy: false }); + + await vi.advanceTimersByTimeAsync(6500); + + // the CDN branch must be gone: shimmer is back and the img points at the covers proxy + await expect.element(page.getByTestId('cover-fallback')).not.toBeInTheDocument(); + await expect.element(page.getByTestId('cover-skeleton')).toBeInTheDocument(); + await expect + .element(page.getByAltText('Test Image')) + .toHaveAttribute('src', `/api/v1/covers/artist/${validMbid}?size=250`); + } finally { + vi.useRealTimers(); + await releaseCdn(); + } + }); - await expect.element(page.getByTestId('cover-fallback')).toBeInTheDocument(); - await expect.element(page.getByTestId('cover-skeleton')).not.toBeInTheDocument(); - vi.useRealTimers(); + it('keeps the direct image when it loads before the settle deadline', async () => { + const releaseCdn = await holdCdnRequests(); + vi.useFakeTimers(); + try { + renderComponent({ remoteUrl: cdnUrl, imageType: 'artist', lazy: false }); + + await vi.advanceTimersByTimeAsync(4000); + page.getByAltText('Test Image').element().dispatchEvent(new Event('load')); + await vi.advanceTimersByTimeAsync(3000); + + await expect + .element(page.getByAltText('Test Image')) + .toHaveAttribute('src', `${cdnUrl}/small`); + await expect.element(page.getByTestId('cover-fallback')).not.toBeInTheDocument(); + } finally { + vi.useRealTimers(); + await releaseCdn(); + } }); it('replaces a settled fallback when shared warming succeeds later', async () => { diff --git a/frontend/src/lib/components/downloads/HeldTrackCard.svelte.spec.ts b/frontend/src/lib/components/downloads/HeldTrackCard.svelte.spec.ts index 4134eefcf..74396e93e 100644 --- a/frontend/src/lib/components/downloads/HeldTrackCard.svelte.spec.ts +++ b/frontend/src/lib/components/downloads/HeldTrackCard.svelte.spec.ts @@ -4,11 +4,28 @@ import { render } from 'vitest-browser-svelte'; import type { HeldImport } from '$lib/types'; -const h = vi.hoisted(() => ({ importMut: vi.fn(), discardMut: vi.fn() })); +const h = vi.hoisted(() => ({ + importMut: vi.fn(), + discardMut: vi.fn(), + importError: null as { message: string } | null, + discardError: null as { message: string } | null +})); vi.mock('$lib/queries/downloads/DownloadMutations.svelte', () => ({ - importHeldTrack: () => ({ mutate: h.importMut, isPending: false }), - discardHeldTrack: () => ({ mutate: h.discardMut, isPending: false }) + importHeldTrack: () => ({ + mutate: h.importMut, + isPending: false, + get error() { + return h.importError; + } + }), + discardHeldTrack: () => ({ + mutate: h.discardMut, + isPending: false, + get error() { + return h.discardError; + } + }) })); import HeldTrackCard from './HeldTrackCard.svelte'; @@ -52,6 +69,8 @@ describe('HeldTrackCard', () => { beforeEach(() => { h.importMut.mockReset(); h.discardMut.mockReset(); + h.importError = null; + h.discardError = null; }); it('shows the track, the couldn’t-verify state, and the AcoustID evidence', async () => { @@ -72,6 +91,23 @@ describe('HeldTrackCard', () => { expect(h.discardMut).not.toHaveBeenCalled(); }); + it('renders a server error inline and leaves the buttons enabled for retry', async () => { + h.importError = { + message: 'No library root is configured - restore one in Settings → Library, then try again.' + }; + renderCard(held()); + + const alert = page.getByRole('alert'); + await expect.element(alert).toHaveTextContent(/No library root is configured/); + const importButton = page.getByRole('button', { name: /Import anyway/ }); + const discardButton = page.getByRole('button', { name: /Discard/ }); + await expect.element(importButton).toBeEnabled(); + await expect.element(discardButton).toBeEnabled(); + + await importButton.click(); + expect(h.importMut).toHaveBeenCalledTimes(1); + }); + it('discards the held track on "Discard"', async () => { renderCard(held()); await page.getByRole('button', { name: /Discard/ }).click(); diff --git a/frontend/src/lib/components/downloads/HeldTrackReview.svelte b/frontend/src/lib/components/downloads/HeldTrackReview.svelte index 6d507925c..258a68b72 100644 --- a/frontend/src/lib/components/downloads/HeldTrackReview.svelte +++ b/frontend/src/lib/components/downloads/HeldTrackReview.svelte @@ -27,6 +27,15 @@ // without this a fast second click re-POSTs an already-consumed held id let done = $state(false); const busy = $derived(importMut.isPending || discardMut.isPending || done); + // server-side failure reason (e.g. no library root configured) shown inline so the + // review card itself says what to fix - the toast alone disappears too fast + const actionError = $derived.by(() => { + const err = importMut.error ?? discardMut.error; + if (!err) return null; + return typeof err === 'object' && 'message' in err && typeof err.message === 'string' + ? err.message + : null; + }); // what the rejecting check SAW - the reason we couldn't auto-confirm it, so the human // decides informed. The evidence source depends on the hold reason: AcoustID's @@ -166,6 +175,9 @@ Discard + {#if actionError} + + {/if}