M5.2: portable restore planning dry-run

README.md

@@ -3,7 +3,7 @@
 Self-hosted, local-first tools for numbers stations, shortwave monitoring and
 signal-analysis workflows.
 
-## Current milestone: M5.1
+## Current milestone: M5.2
 
 The application currently provides:
 
@@ -41,6 +41,8 @@ The application currently provides:
 - versioned metadata-only recording/annotation manifest export with checksums and provenance
 - full portable `tar.gz` archival export containing `manifest.json` plus verified managed WAV/FLAC files
 - safe restore/import of M5.0 archives with staged extraction, checksum validation and collision protection
+- dry-run restore planning with per-recording `import`, `duplicate`, `conflict` and `unmatched` classification
+- per-recording annotation import/skip counts before restore
 - local rebuild of waveform, frequency, spectrogram and fingerprint data from restored managed audio
 - local JSON persistence under `/data`
 - embedded web UI and JSON API
@@ -51,23 +53,22 @@ and `duplicate capture`. They are stored locally alongside recording metadata an
 do not require AI, an API key, or an external service.
 
 Managed playback reads audio directly from the appliance. Browser-native codec
-support determines whether a WAV/FLAC recording can be played; M5.1 does not
+support determines whether a WAV/FLAC recording can be played; M5.2 does not
 transcode audio or send it to an external service. Saved timestamp annotations
 persist in `recordings.json`, can be searched across the archive, exported as JSON
 or CSV, and safely imported from the versioned JSON format.
 
-M5.0 archives can now be restored through a conservative staged importer. Archive
-paths, manifest structure, file sizes, SHA-256 values and WAV/FLAC signatures are
-validated before commit. Existing recording IDs are never blindly overwritten;
-equivalent entries are treated as duplicates, different entries as conflicts, and
-recordings whose observation is unavailable locally are reported as unmatched.
-Derived preview/fingerprint payloads are rebuilt locally from restored audio rather
-than trusted from the archive manifest.
+Before committing an M5.0 archive restore, M5.2 can run the same structural,
+checksum, managed-audio and annotation validation as a dry-run planner. It reports
+which recordings would be imported, treated as duplicates, rejected as conflicts,
+or remain unmatched because their observation is not present locally. The planner
+does not append recording metadata, move managed audio into final paths, or save
+annotations; temporary validation staging is removed before the request completes.
 
 The application does not require an SDR, receiver, API key or external data
 provider for these workflows.
 
-See [docs/M5_1.md](docs/M5_1.md) for the current scope.
+See [docs/M5_2.md](docs/M5_2.md) for the current scope.
 
 ## Run with Go
 
@@ -100,6 +101,7 @@ Then open <http://localhost:8080>.
 - `GET/POST /api/recordings`
 - `GET /api/recording-bundle`
 - `GET /api/recording-archive`
+- `POST /api/recording-archive/plan`
 - `POST /api/recording-archive/import`
 - `GET /api/recordings/{id}/similar`
 - `GET /api/recording-clusters?threshold=98`
@@ -128,7 +130,7 @@ Number Station Tools is intended to grow into a workbench for:
 - waveform, frequency, spectrogram, fingerprint and signal-analysis workflows
 - categorized timestamp annotations, bookmarks and operator notes on recordings
 - cross-recording annotation search, portable transfer and timeline navigation
-- portable recording manifests, verified archival bundles and conservative local restore
+- portable recording manifests, verified archival bundles, dry-run planning and conservative local restore
 - human review and classification of signal matches and repeated transmissions
 - optional SDR and network-receiver integrations
 - optional data-provider imports

cmd/number-station-tools/m4_9_test.go

@@ -14,7 +14,6 @@ func TestM49RecordingBundleEmbedded(t *testing.T) {
 	for _, want := range []string{
 		"/api/recording-bundle",
 		"Export manifest only",
-		"recording metadata, checksums, annotations",
 	} {
 		if !strings.Contains(page, want) {
 			t.Fatalf("index missing %q", want)

cmd/number-station-tools/m5_1_test.go

@@ -15,7 +15,6 @@ func TestM51ArchiveRestoreEmbedded(t *testing.T) {
 		"Verify and restore archive",
 		"recording-archive-import-form",
 		"/archive-restore.js",
-		"missing observations are reported as unmatched",
 	} {
 		if !strings.Contains(page, want) {
 			t.Fatalf("index missing %q", want)

cmd/number-station-tools/m5_2_test.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestM52RestorePlanningEmbedded(t *testing.T) {
+	index, err := webFS.ReadFile("web/index.html")
+	if err != nil {
+		t.Fatal(err)
+	}
+	script, err := webFS.ReadFile("web/archive-restore.js")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{"recording-archive-plan", "Plan restore", "recording-archive-plan-results"} {
+		if !strings.Contains(string(index), want) {
+			t.Fatalf("index missing %q", want)
+		}
+	}
+	for _, want := range []string{"/api/recording-archive/plan", "annotations_import", "annotations_skip", "renderPlan"} {
+		if !strings.Contains(string(script), want) {
+			t.Fatalf("archive restore script missing %q", want)
+		}
+	}
+}

cmd/number-station-tools/portable_restore_plan.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type portableRestorePlanEntry struct {
+	RecordingID       string `json:"recording_id"`
+	ObservationID     string `json:"observation_id"`
+	Path              string `json:"path"`
+	Managed           bool   `json:"managed"`
+	Action            string `json:"action"`
+	Reason            string `json:"reason,omitempty"`
+	AnnotationsImport int    `json:"annotations_import"`
+	AnnotationsSkip   int    `json:"annotations_skip"`
+}
+
+type portableRestorePlan struct {
+	Import    int                        `json:"import"`
+	Duplicate int                        `json:"duplicate"`
+	Conflict  int                        `json:"conflict"`
+	Unmatched int                        `json:"unmatched"`
+	Entries   []portableRestorePlanEntry `json:"entries"`
+}
+
+func planPortableRestore(src io.Reader, db *store, rs *recordingStore, audioDir string) (portableRestorePlan, error) {
+	staged, err := readPortableArchive(src, audioDir)
+	if err != nil {
+		return portableRestorePlan{}, err
+	}
+	defer os.RemoveAll(staged.dir)
+
+	plan := portableRestorePlan{Entries: []portableRestorePlanEntry{}}
+	seenIDs := make(map[string]struct{}, len(staged.manifest.Items))
+	requiredFiles := make(map[string]struct{})
+
+	rs.mu.Lock()
+	existingRecordings := append([]recording(nil), rs.data.Recordings...)
+	existingAnnotations := append([]recordingAnnotation(nil), rs.data.Annotations...)
+	rs.mu.Unlock()
+	existingByID := make(map[string]recording, len(existingRecordings))
+	for _, rec := range existingRecordings {
+		existingByID[rec.ID] = rec
+	}
+
+	for _, item := range staged.manifest.Items {
+		rec := portableRecording(item.Recording)
+		rec.ID = strings.TrimSpace(rec.ID)
+		if rec.ID == "" {
+			return portableRestorePlan{}, errors.New("manifest contains recording without id")
+		}
+		if _, exists := seenIDs[rec.ID]; exists {
+			return portableRestorePlan{}, fmt.Errorf("manifest contains duplicate recording id %s", rec.ID)
+		}
+		seenIDs[rec.ID] = struct{}{}
+
+		entry := portableRestorePlanEntry{RecordingID: rec.ID, ObservationID: rec.ObservationID, Path: rec.Path, Managed: rec.Managed}
+		if rec.Managed {
+			archivePath, err := managedArchivePath(rec)
+			if err != nil {
+				return portableRestorePlan{}, fmt.Errorf("recording %s: %w", rec.ID, err)
+			}
+			file, ok := staged.files[archivePath]
+			if !ok {
+				return portableRestorePlan{}, fmt.Errorf("managed recording %s is missing %s", rec.ID, archivePath)
+			}
+			requiredFiles[archivePath] = struct{}{}
+			rebuilt, err := prepareRestoredRecording(rec, file)
+			if err != nil {
+				return portableRestorePlan{}, fmt.Errorf("recording %s: %w", rec.ID, err)
+			}
+			rec = rebuilt
+		} else if err := validateRecording(rec); err != nil {
+			return portableRestorePlan{}, fmt.Errorf("recording %s: %w", rec.ID, err)
+		}
+
+		if !observationExists(db, rec.ObservationID) {
+			entry.Action = "unmatched"
+			entry.Reason = "observation does not exist locally"
+			plan.Unmatched++
+			plan.Entries = append(plan.Entries, entry)
+			continue
+		}
+
+		if existing, ok := existingByID[rec.ID]; ok {
+			if samePortableRecording(existing, rec) {
+				entry.Action = "duplicate"
+				entry.Reason = "equivalent recording already exists"
+				plan.Duplicate++
+				rec = existing
+			} else {
+				entry.Action = "conflict"
+				entry.Reason = "recording id already exists with different content"
+				plan.Conflict++
+				plan.Entries = append(plan.Entries, entry)
+				continue
+			}
+		} else if rec.Managed {
+			dest := filepath.Join(audioDir, filepath.Base(rec.Path))
+			if _, err := os.Lstat(dest); err == nil {
+				entry.Action = "conflict"
+				entry.Reason = "managed audio destination already exists"
+				plan.Conflict++
+				plan.Entries = append(plan.Entries, entry)
+				continue
+			} else if !errors.Is(err, os.ErrNotExist) {
+				return portableRestorePlan{}, fmt.Errorf("cannot inspect destination for %s", rec.ID)
+			}
+			entry.Action = "import"
+			plan.Import++
+		} else {
+			entry.Action = "import"
+			plan.Import++
+		}
+
+		planned := append([]recordingAnnotation(nil), existingAnnotations...)
+		for _, incoming := range item.Annotations {
+			annotation := incoming
+			annotation.ID = ""
+			annotation.RecordingID = rec.ID
+			annotation.Type = normalizeAnnotationType(annotation.Type)
+			annotation.Label = strings.TrimSpace(annotation.Label)
+			annotation.Notes = strings.TrimSpace(annotation.Notes)
+			if err := validateRecordingAnnotation(rec, annotation); err != nil {
+				return portableRestorePlan{}, fmt.Errorf("invalid annotation for recording %s: %w", rec.ID, err)
+			}
+			duplicate := false
+			for _, existing := range planned {
+				if sameAnnotation(existing, annotation) {
+					duplicate = true
+					break
+				}
+			}
+			if duplicate {
+				entry.AnnotationsSkip++
+				continue
+			}
+			planned = append(planned, annotation)
+			entry.AnnotationsImport++
+		}
+		plan.Entries = append(plan.Entries, entry)
+	}
+
+	for name := range staged.files {
+		if _, ok := requiredFiles[name]; !ok {
+			return portableRestorePlan{}, fmt.Errorf("archive contains unreferenced audio entry %q", name)
+		}
+	}
+	return plan, nil
+}

cmd/number-station-tools/portable_restore_plan_test.go

@@ -0,0 +1,99 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestPortableRestorePlanDoesNotMutateState(t *testing.T) {
+	db := restoreTestDB(t)
+	rs, err := openRecordingStore(filepath.Join(t.TempDir(), "recordings.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	audioDir := filepath.Join(t.TempDir(), "audio")
+	archive := restoreTestArchive(t, testWAV(8000, 1, 1000), nil)
+
+	plan, err := planPortableRestore(bytes.NewReader(archive), db, rs, audioDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if plan.Import != 1 || plan.Duplicate != 0 || plan.Conflict != 0 || plan.Unmatched != 0 || len(plan.Entries) != 1 {
+		t.Fatalf("plan = %#v", plan)
+	}
+	if plan.Entries[0].Action != "import" || plan.Entries[0].AnnotationsImport != 1 {
+		t.Fatalf("entry = %#v", plan.Entries[0])
+	}
+	if len(rs.list()) != 0 {
+		t.Fatal("dry-run changed recording metadata")
+	}
+	if _, err := os.Stat(filepath.Join(audioDir, "r1.wav")); !os.IsNotExist(err) {
+		t.Fatalf("dry-run created destination audio: %v", err)
+	}
+}
+
+func TestPortableRestorePlanClassifiesDuplicateConflictAndUnmatched(t *testing.T) {
+	db := restoreTestDB(t)
+	audio := testWAV(8000, 1, 1000)
+	archive := restoreTestArchive(t, audio, nil)
+	audioDir := filepath.Join(t.TempDir(), "audio")
+	rs, _ := openRecordingStore(filepath.Join(t.TempDir(), "recordings.json"))
+
+	if _, err := restorePortableArchive(bytes.NewReader(archive), db, rs, audioDir); err != nil {
+		t.Fatal(err)
+	}
+	plan, err := planPortableRestore(bytes.NewReader(archive), db, rs, audioDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if plan.Duplicate != 1 || plan.Entries[0].Action != "duplicate" || plan.Entries[0].AnnotationsSkip != 1 {
+		t.Fatalf("duplicate plan = %#v", plan)
+	}
+
+	rs.data.Recordings[0].Notes = "different"
+	plan, err = planPortableRestore(bytes.NewReader(archive), db, rs, audioDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if plan.Conflict != 1 || plan.Entries[0].Action != "conflict" {
+		t.Fatalf("conflict plan = %#v", plan)
+	}
+
+	unmatched := restoreTestArchive(t, audio, func(manifest *recordingBundleManifest) {
+		manifest.Items[0].Recording.ObservationID = "missing"
+	})
+	plan, err = planPortableRestore(bytes.NewReader(unmatched), db, &recordingStore{data: recordingData{Recordings: []recording{}, Annotations: []recordingAnnotation{}}}, filepath.Join(t.TempDir(), "audio"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if plan.Unmatched != 1 || plan.Entries[0].Action != "unmatched" {
+		t.Fatalf("unmatched plan = %#v", plan)
+	}
+}
+
+func TestPortableRestorePlanAPI(t *testing.T) {
+	db := restoreTestDB(t)
+	rs, _ := openRecordingStore(filepath.Join(t.TempDir(), "recordings.json"))
+	audioDir := filepath.Join(t.TempDir(), "audio")
+	mux := http.NewServeMux()
+	registerRecordingRestoreHandler(mux, db, rs, audioDir)
+	archive := restoreTestArchive(t, testWAV(8000, 1, 1000), nil)
+	req := httptest.NewRequest(http.MethodPost, "/api/recording-archive/plan", bytes.NewReader(archive))
+	w := httptest.NewRecorder()
+	mux.ServeHTTP(w, req)
+	if w.Code != http.StatusOK {
+		t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
+	}
+	var plan portableRestorePlan
+	if err := json.Unmarshal(w.Body.Bytes(), &plan); err != nil {
+		t.Fatal(err)
+	}
+	if plan.Import != 1 || len(plan.Entries) != 1 {
+		t.Fatalf("plan=%#v", plan)
+	}
+}

cmd/number-station-tools/recording_bundle_api.go

@@ -29,6 +29,15 @@ func registerRecordingBundleHandlers(mux *http.ServeMux, rs *recordingStore, aud
 }
 
 func registerRecordingRestoreHandler(mux *http.ServeMux, db *store, rs *recordingStore, audioDir string) {
+	mux.HandleFunc("POST /api/recording-archive/plan", func(w http.ResponseWriter, r *http.Request) {
+		r.Body = http.MaxBytesReader(w, r.Body, maxPortableArchiveUploadBytes)
+		plan, err := planPortableRestore(r.Body, db, rs, audioDir)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		writeJSON(w, http.StatusOK, plan)
+	})
 	mux.HandleFunc("POST /api/recording-archive/import", func(w http.ResponseWriter, r *http.Request) {
 		r.Body = http.MaxBytesReader(w, r.Body, maxPortableArchiveUploadBytes)
 		result, err := restorePortableArchive(r.Body, db, rs, audioDir)