feat: include migration storage in system overview

api/openapi.yaml

@@ -1236,10 +1236,11 @@ components:
         indexConsistent: { type: boolean }
     SystemOverview:
       type: object
-      required: [imageCount, storedBytes, aliasCount, heapAllocBytes, heapSysBytes, rssBytes, goroutines, indexes, indexConsistent, missingImageCount, missingImageIds, lastInspection, lastRebuild, lastDaily]
+      required: [imageCount, storedBytes, migrationStoredBytes, aliasCount, heapAllocBytes, heapSysBytes, rssBytes, goroutines, indexes, indexConsistent, missingImageCount, missingImageIds, lastInspection, lastRebuild, lastDaily]
       properties:
         imageCount: { type: integer, format: int64, minimum: 0 }
         storedBytes: { type: integer, format: int64, minimum: 0 }
+        migrationStoredBytes: { type: integer, format: int64, minimum: 0 }
         aliasCount: { type: integer, format: int64, minimum: 0 }
         heapAllocBytes: { type: integer, format: int64, minimum: 0 }
         heapSysBytes: { type: integer, format: int64, minimum: 0 }

internal/app/app.go

@@ -92,10 +92,10 @@ func Build(ctx context.Context, cfg config.Config, db *sql.DB, logger *slog.Logg
 		cancel()
 		return nil, fmt.Errorf("load application settings: %w", err)
 	}
+	migrationImageService := migrationimage.NewService(filesystem, cfg.MigrationMutations)
 	maintenanceService := maintenance.NewService(
-		maintenance.NewRepository(db), filesystem, rebuilder, deliveryIndex, authService, tokenService, logger,
+		maintenance.NewRepository(db), filesystem, rebuilder, deliveryIndex, authService, tokenService, migrationImageService, logger,
 	)
-	migrationImageService := migrationimage.NewService(filesystem, cfg.MigrationMutations)
 	unavailableIDs := append(append([]string(nil), loadResult.Delivery.MissingIDs...), loadResult.Delivery.InvalidSizeIDs...)
 	maintenanceService.RecordStartupMissing(unavailableIDs)
 	ui, err := webui.New()

internal/httpapi/maintenance_handlers.go

@@ -42,20 +42,21 @@ type rebuildResponse struct {
 }
 
 type overviewResponse struct {
-	ImageCount        int64               `json:"imageCount"`
-	StoredBytes       int64               `json:"storedBytes"`
-	AliasCount        int64               `json:"aliasCount"`
-	HeapAllocBytes    uint64              `json:"heapAllocBytes"`
-	HeapSysBytes      uint64              `json:"heapSysBytes"`
-	RSSBytes          uint64              `json:"rssBytes"`
-	Goroutines        int                 `json:"goroutines"`
-	Indexes           indexStatsResponse  `json:"indexes"`
-	IndexConsistent   bool                `json:"indexConsistent"`
-	MissingImageCount int                 `json:"missingImageCount"`
-	MissingImageIDs   []string            `json:"missingImageIds"`
-	LastInspection    *inspectionResponse `json:"lastInspection"`
-	LastRebuild       *rebuildResponse    `json:"lastRebuild"`
-	LastDaily         *dailyResponse      `json:"lastDaily"`
+	ImageCount           int64               `json:"imageCount"`
+	StoredBytes          int64               `json:"storedBytes"`
+	MigrationStoredBytes int64               `json:"migrationStoredBytes"`
+	AliasCount           int64               `json:"aliasCount"`
+	HeapAllocBytes       uint64              `json:"heapAllocBytes"`
+	HeapSysBytes         uint64              `json:"heapSysBytes"`
+	RSSBytes             uint64              `json:"rssBytes"`
+	Goroutines           int                 `json:"goroutines"`
+	Indexes              indexStatsResponse  `json:"indexes"`
+	IndexConsistent      bool                `json:"indexConsistent"`
+	MissingImageCount    int                 `json:"missingImageCount"`
+	MissingImageIDs      []string            `json:"missingImageIds"`
+	LastInspection       *inspectionResponse `json:"lastInspection"`
+	LastRebuild          *rebuildResponse    `json:"lastRebuild"`
+	LastDaily            *dailyResponse      `json:"lastDaily"`
 }
 
 type dailyResponse struct {
@@ -110,7 +111,8 @@ func (h *maintenanceHandler) inspect(w http.ResponseWriter, r *http.Request) {
 
 func toOverviewResponse(value maintenance.Overview) overviewResponse {
 	response := overviewResponse{
-		ImageCount: value.Persistent.ImageCount, StoredBytes: value.Persistent.StoredBytes, AliasCount: value.Persistent.AliasCount,
+		ImageCount: value.Persistent.ImageCount, StoredBytes: value.Persistent.StoredBytes,
+		MigrationStoredBytes: value.MigrationStoredBytes, AliasCount: value.Persistent.AliasCount,
 		HeapAllocBytes: value.Runtime.HeapAllocBytes, HeapSysBytes: value.Runtime.HeapSysBytes,
 		RSSBytes: value.Runtime.RSSBytes, Goroutines: value.Runtime.Goroutines,
 		Indexes: indexStatsResponse{

internal/httpapi/phase_five_test.go

@@ -91,11 +91,16 @@ func TestPhaseFiveOverviewInspectionAndManualRebuild(t *testing.T) {
 	fixture := newPhaseTwoFixture(t)
 	cookies, csrfToken, _ := fixture.login(nil, phaseTwoPassword)
 	image := fixture.uploadBytes(cookies, csrfToken, "public", "", "overview.jpg", phaseTwoJPEG(t))
+	migrationBytes := phaseTwoJPEG(t)
+	if err := os.WriteFile(filepath.Join(fixture.dataDirectory, "migrations", "overview.jpg"), migrationBytes, 0o640); err != nil {
+		t.Fatalf("write migration image: %v", err)
+	}
 
 	overview := fixture.request(http.MethodGet, "/api/v1/overview", nil, cookies, "", "")
 	var firstOverview overviewResponse
 	if err := json.Unmarshal(overview.Body.Bytes(), &firstOverview); overview.Code != http.StatusOK || err != nil ||
-		firstOverview.ImageCount != 1 || firstOverview.Indexes.Images != 1 || !firstOverview.IndexConsistent {
+		firstOverview.ImageCount != 1 || firstOverview.MigrationStoredBytes != int64(len(migrationBytes)) ||
+		firstOverview.Indexes.Images != 1 || !firstOverview.IndexConsistent {
 		t.Fatalf("overview status = %d, value = %+v, error = %v", overview.Code, firstOverview, err)
 	}
 	if err := os.Remove(filepath.Join(fixture.dataDirectory, "images", image.ID)); err != nil {

internal/httpapi/phase_two_test.go

@@ -253,8 +253,10 @@ func newHTTPFixture(t *testing.T, engine processor.Engine, gate *processor.Gate,
 	settingsService := settings.NewService(settings.NewRepository(db))
 	var logs bytes.Buffer
 	logger := slog.New(slog.NewJSONHandler(&logs, nil))
-	maintenanceService := maintenance.NewService(maintenance.NewRepository(db), filesystem, rebuilder, deliveryIndex, authService, tokenService, logger)
 	migrationImageService := migrationimage.NewService(filesystem, true)
+	maintenanceService := maintenance.NewService(
+		maintenance.NewRepository(db), filesystem, rebuilder, deliveryIndex, authService, tokenService, migrationImageService, logger,
+	)
 	router := NewRouter(Dependencies{
 		DB: db, Logger: logger, Auth: authService, APITokens: tokenService,
 		Aliases: aliasService, Images: imageService, Importer: importService, Settings: settingsService, DeliveryIndex: deliveryIndex, Storage: filesystem,

internal/maintenance/service.go

@@ -23,15 +23,16 @@ type IndexStats struct {
 }
 
 type Overview struct {
-	Persistent        PersistentStats
-	Runtime           RuntimeSnapshot
-	Indexes           IndexStats
-	IndexConsistent   bool
-	MissingImageCount int
-	MissingImageIDs   []string
-	LastInspection    *InspectionResult
-	LastRebuild       *RebuildResult
-	LastDaily         *DailyResult
+	Persistent           PersistentStats
+	MigrationStoredBytes int64
+	Runtime              RuntimeSnapshot
+	Indexes              IndexStats
+	IndexConsistent      bool
+	MissingImageCount    int
+	MissingImageIDs      []string
+	LastInspection       *InspectionResult
+	LastRebuild          *RebuildResult
+	LastDaily            *DailyResult
 }
 
 type InspectionResult struct {
@@ -72,6 +73,7 @@ type DailyResult struct {
 type Service struct {
 	repository        *Repository
 	storage           *storage.Filesystem
+	migrationStorage  migrationStorage
 	rebuilder         *indexstate.Rebuilder
 	delivery          *delivery.Index
 	sessions          *auth.Service
@@ -85,18 +87,23 @@ type Service struct {
 	lastDaily         *DailyResult
 }
 
+type migrationStorage interface {
+	StoredBytes(context.Context) (int64, error)
+}
+
 func NewService(
 	repository *Repository,
 	filesystem *storage.Filesystem,
 	rebuilder *indexstate.Rebuilder,
 	deliveryIndex *delivery.Index,
 	sessions *auth.Service,
 	tokens *apitoken.Service,
+	migrationStorage migrationStorage,
 	logger *slog.Logger,
 ) *Service {
 	return &Service{
 		repository: repository, storage: filesystem, rebuilder: rebuilder,
-		delivery: deliveryIndex, sessions: sessions, tokens: tokens, logger: logger,
+		delivery: deliveryIndex, sessions: sessions, tokens: tokens, migrationStorage: migrationStorage, logger: logger,
 	}
 }
 
@@ -113,6 +120,10 @@ func (s *Service) Overview(ctx context.Context) (Overview, error) {
 	if err != nil {
 		return Overview{}, err
 	}
+	migrationBytes, err := s.migrationStorage.StoredBytes(ctx)
+	if err != nil {
+		return Overview{}, err
+	}
 	s.mu.RLock()
 	missingCount := s.missingImageCount
 	missingIDs := append([]string(nil), s.missingImageIDs...)
@@ -121,7 +132,7 @@ func (s *Service) Overview(ctx context.Context) (Overview, error) {
 	lastDaily := cloneDaily(s.lastDaily)
 	s.mu.RUnlock()
 	return Overview{
-		Persistent: persistent, Runtime: CaptureRuntime(), Indexes: indexes, IndexConsistent: consistent,
+		Persistent: persistent, MigrationStoredBytes: migrationBytes, Runtime: CaptureRuntime(), Indexes: indexes, IndexConsistent: consistent,
 		MissingImageCount: missingCount, MissingImageIDs: missingIDs,
 		LastInspection: lastInspection, LastRebuild: lastRebuild, LastDaily: lastDaily,
 	}, nil

internal/maintenance/service_test.go

@@ -18,6 +18,7 @@ import (
 	images "github.com/Willxup/imagesilo/internal/image"
 	"github.com/Willxup/imagesilo/internal/indexbarrier"
 	"github.com/Willxup/imagesilo/internal/indexstate"
+	"github.com/Willxup/imagesilo/internal/migrationimage"
 	"github.com/Willxup/imagesilo/internal/platform/database"
 	"github.com/Willxup/imagesilo/internal/platform/storage"
 )
@@ -140,10 +141,24 @@ func TestSizeMismatchIsReportedUnavailableAndNeverCleanedAsOrphan(t *testing.T)
 	}
 }
 
+func TestOverviewIncludesMigrationDirectoryStorage(t *testing.T) {
+	directory, service, db := prepareMaintenanceTest(t)
+	defer db.Close()
+	content := []byte("migration directory bytes")
+	if err := os.WriteFile(filepath.Join(directory, "migrations", "notes.txt"), content, 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	overview, err := service.Overview(context.Background())
+	if err != nil || overview.MigrationStoredBytes != int64(len(content)) {
+		t.Fatalf("Overview() migration storage = %d, %v", overview.MigrationStoredBytes, err)
+	}
+}
+
 func prepareMaintenanceTest(t *testing.T) (string, *Service, *sql.DB) {
 	t.Helper()
 	directory := t.TempDir()
-	for _, path := range []string{"db", "images", filepath.Join("cache", "thumbnails"), "tmp"} {
+	for _, path := range []string{"db", "images", "migrations", filepath.Join("cache", "thumbnails"), "tmp"} {
 		if err := os.MkdirAll(filepath.Join(directory, path), 0o750); err != nil {
 			t.Fatal(err)
 		}
@@ -171,7 +186,10 @@ func prepareMaintenanceTest(t *testing.T) (string, *Service, *sql.DB) {
 	deliveryIndex := delivery.NewIndex()
 	rebuilder := indexstate.NewRebuilder(db, filesystem, authRepository, tokenRepository, deliveryIndex, sessionIndex, tokenIndex, barrier)
 	var logs bytes.Buffer
-	service := NewService(NewRepository(db), filesystem, rebuilder, deliveryIndex, authService, tokenService, slog.New(slog.NewJSONHandler(&logs, nil)))
+	service := NewService(
+		NewRepository(db), filesystem, rebuilder, deliveryIndex, authService, tokenService,
+		migrationimage.NewService(filesystem, false), slog.New(slog.NewJSONHandler(&logs, nil)),
+	)
 	return directory, service, db
 }
 

internal/migrationimage/service.go

@@ -73,6 +73,7 @@ type Service struct {
 type migrationSnapshot struct {
 	items        []Image
 	skippedFiles int
+	storedBytes  int64
 	scannedAt    time.Time
 }
 
@@ -94,6 +95,14 @@ func (s *Service) Refresh(ctx context.Context) error {
 	return err
 }
 
+func (s *Service) StoredBytes(ctx context.Context) (int64, error) {
+	snapshot, err := s.loadSnapshot(ctx, false)
+	if err != nil {
+		return 0, err
+	}
+	return snapshot.storedBytes, nil
+}
+
 func (s *Service) Search(ctx context.Context, filter ListFilter) (Page, error) {
 	if err := validateListFilter(filter); err != nil {
 		return Page{}, err
@@ -245,7 +254,9 @@ func (s *Service) scanSnapshot(ctx context.Context) (*migrationSnapshot, error)
 		}
 		return items[left].ModifiedAt.After(items[right].ModifiedAt)
 	})
-	return &migrationSnapshot{items: items, skippedFiles: skipped, scannedAt: s.now().UTC()}, nil
+	return &migrationSnapshot{
+		items: items, skippedFiles: skipped, storedBytes: scanned.StoredBytes, scannedAt: s.now().UTC(),
+	}, nil
 }
 
 func (s *Service) removeCachedPath(canonicalPath string) {
@@ -255,12 +266,17 @@ func (s *Service) removeCachedPath(canonicalPath string) {
 		return
 	}
 	items := make([]Image, 0, len(s.snapshot.items))
+	storedBytes := s.snapshot.storedBytes
 	for _, item := range s.snapshot.items {
 		if item.Path != canonicalPath {
 			items = append(items, item)
+		} else {
+			storedBytes -= item.StoredSize
 		}
 	}
-	s.snapshot = &migrationSnapshot{items: items, skippedFiles: s.snapshot.skippedFiles, scannedAt: s.snapshot.scannedAt}
+	s.snapshot = &migrationSnapshot{
+		items: items, skippedFiles: s.snapshot.skippedFiles, storedBytes: storedBytes, scannedAt: s.snapshot.scannedAt,
+	}
 }
 
 func validateListFilter(filter ListFilter) error {

internal/migrationimage/service_test.go

@@ -86,6 +86,10 @@ func TestSearchCachesUntilRefreshAndDeleteUpdatesSnapshot(t *testing.T) {
 	if err != nil || len(initial.Items) != 1 || initial.Items[0].Path != "/i/first.jpg" {
 		t.Fatalf("initial Search() = %+v, %v", initial, err)
 	}
+	initialBytes, err := service.StoredBytes(context.Background())
+	if err != nil || initialBytes != int64(len(migrationTestJPEG(t))) {
+		t.Fatalf("initial StoredBytes() = %d, %v", initialBytes, err)
+	}
 	writeMigrationTestImage(t, dataDirectory, "images/second.jpg", migrationTestJPEG(t), base.Add(time.Hour))
 	cached, err := service.Search(context.Background(), ListFilter{})
 	if err != nil || len(cached.Items) != 1 {
@@ -98,6 +102,10 @@ func TestSearchCachesUntilRefreshAndDeleteUpdatesSnapshot(t *testing.T) {
 	if err != nil || len(refreshed.Items) != 2 || refreshed.Items[0].Path != "/images/second.jpg" {
 		t.Fatalf("refreshed Search() = %+v, %v", refreshed, err)
 	}
+	refreshedBytes, err := service.StoredBytes(context.Background())
+	if err != nil || refreshedBytes != 2*initialBytes {
+		t.Fatalf("refreshed StoredBytes() = %d, %v", refreshedBytes, err)
+	}
 
 	if _, err := service.Delete(context.Background(), "/images/second.jpg"); err != nil {
 		t.Fatalf("Delete() error = %v", err)
@@ -106,6 +114,10 @@ func TestSearchCachesUntilRefreshAndDeleteUpdatesSnapshot(t *testing.T) {
 	if err != nil || len(afterDelete.Items) != 1 || afterDelete.Items[0].Path != "/i/first.jpg" {
 		t.Fatalf("Search() after deletion = %+v, %v", afterDelete, err)
 	}
+	afterDeleteBytes, err := service.StoredBytes(context.Background())
+	if err != nil || afterDeleteBytes != initialBytes {
+		t.Fatalf("StoredBytes() after deletion = %d, %v", afterDeleteBytes, err)
+	}
 }
 
 func TestDeleteEvictsExternallyRemovedImageFromSnapshot(t *testing.T) {

internal/platform/storage/filesystem.go

@@ -42,6 +42,7 @@ type MigrationFile struct {
 type MigrationList struct {
 	Files        []MigrationFile
 	SkippedFiles int
+	StoredBytes  int64
 }
 
 type MigrationDeleteResult struct {
@@ -97,6 +98,7 @@ func (f *Filesystem) ListMigrationImages(ctx context.Context) (MigrationList, er
 			result.SkippedFiles++
 			return nil
 		}
+		result.StoredBytes += info.Size()
 		file, mimeType, err := openMigrationImage(root, relativePath)
 		if err != nil {
 			result.SkippedFiles++

internal/platform/storage/filesystem_test.go

@@ -75,10 +75,12 @@ func TestListMigrationImagesSkipsInvalidContentAndSymlinks(t *testing.T) {
 	if err := os.WriteFile(validPath, jpegBytes, 0o640); err != nil {
 		t.Fatalf("WriteFile(valid): %v", err)
 	}
-	if err := os.WriteFile(filepath.Join(migrationsDirectory, "disguised.jpg"), []byte("not an image"), 0o640); err != nil {
+	disguisedBytes := []byte("not an image")
+	if err := os.WriteFile(filepath.Join(migrationsDirectory, "disguised.jpg"), disguisedBytes, 0o640); err != nil {
 		t.Fatalf("WriteFile(disguised): %v", err)
 	}
-	if err := os.WriteFile(filepath.Join(migrationsDirectory, "notes.txt"), []byte("not public"), 0o640); err != nil {
+	notesBytes := []byte("not public")
+	if err := os.WriteFile(filepath.Join(migrationsDirectory, "notes.txt"), notesBytes, 0o640); err != nil {
 		t.Fatalf("WriteFile(notes): %v", err)
 	}
 	if err := os.Symlink(validPath, filepath.Join(migrationsDirectory, "linked.jpg")); err != nil {
@@ -95,6 +97,9 @@ func TestListMigrationImagesSkipsInvalidContentAndSymlinks(t *testing.T) {
 	if listed.SkippedFiles < 2 {
 		t.Fatalf("SkippedFiles = %d, want at least invalid content and unsupported extension", listed.SkippedFiles)
 	}
+	if want := int64(len(jpegBytes) + len(disguisedBytes) + len(notesBytes)); listed.StoredBytes != want {
+		t.Fatalf("StoredBytes = %d, want %d for all regular files", listed.StoredBytes, want)
+	}
 }
 
 func TestListMigrationImagesHonorsCancellation(t *testing.T) {