From 7d67bbd370fafc62296cd7b9a64f6552d94e1729 Mon Sep 17 00:00:00 2001 From: drondeseries Date: Mon, 5 Oct 2026 03:59:52 -0400 Subject: [PATCH 01/16] fix(playback): cold-start default-audio language stability on probe reorder Persist the start-time audio selection intent (origin, preferred language, series snapshot, committed signature) on the attempt and the session, then replay it against the verified inventory when probe evidence lands. An automatic track_change replan moves the executable selection only when the language moved; identical selections stay a byte-equal no-op. --- internal/api/handlers/playback.go | 6 + .../api/handlers/playback_reconcile_audio.go | 441 +++++++++++++++++ .../handlers/playback_reconcile_audio_test.go | 455 ++++++++++++++++++ internal/api/handlers/playback_service.go | 27 ++ internal/api/handlers/playback_v3.go | 100 +++- internal/api/handlers/playback_virtual.go | 14 +- .../api/handlers/playback_virtual_evidence.go | 9 + internal/playback/audio_select.go | 50 +- internal/playback/audio_select_test.go | 63 +++ internal/playback/protocol_store_v3.go | 20 + internal/playback/session.go | 70 +++ 11 files changed, 1246 insertions(+), 9 deletions(-) create mode 100644 internal/api/handlers/playback_reconcile_audio.go create mode 100644 internal/api/handlers/playback_reconcile_audio_test.go diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index 6fc4de295c..24f995310a 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -2243,6 +2243,12 @@ func (h *PlaybackHandler) HandleUpdateProgress(w http.ResponseWriter, r *http.Re // Persist progress to UserStore (best-effort). if sess, getErr := h.sessionMgr.GetSession(sessionID); getErr == nil { h.persistProgress(r.Context(), sess) + // The first progress report after attach is the probe-before-attach + // hook: evidence may have landed while no session was registered, so + // the attach-side check replayed nothing yet. Replays are idempotent + // (a settled selection is a byte-equal no-op), bounded to one + // heartbeat-triggered check per session attach generation. + h.reconcilePendingAudioStartup(r.Context(), sessionID) if !sess.DisableProgressPersistence && h.WatchScrobbler != nil && wasPaused != sess.IsPaused { if file, loadErr := h.loadFileByPreferredID(r.Context(), requestedMediaFileID(sess), sess.MediaFileID); loadErr == nil && file != nil { targetID := playbackProgressTarget(file) diff --git a/internal/api/handlers/playback_reconcile_audio.go b/internal/api/handlers/playback_reconcile_audio.go new file mode 100644 index 0000000000..0e0db419d4 --- /dev/null +++ b/internal/api/handlers/playback_reconcile_audio.go @@ -0,0 +1,441 @@ +package handlers + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "log/slog" + "slices" + "strings" + "time" + + "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/userstore" +) + +// This file owns cold-start default-audio language stability: the selection +// the server made from declared (fast-path) metadata must survive the +// verified probe reorder, or the committed route keeps playing the wrong +// language while the menu already shows the corrected inventory. +// +// deferProbe stays: this runs only after probe evidence has been committed +// (persistVirtualEvidenceDirect, persistVirtualEvidenceTask), never on the +// start critical path. It re-enumerates live sessions independent of +// realtime delivery, replays the persisted selection intent against the +// committed verified inventory, and issues an automatic track_change replan +// only when the executable audio selection actually moved. +// +// Language stability — not ordinal stability — is the success condition: +// the replan names the verified track that carries the preferred language, +// even when its array position differs from the committed one. + +// Selection origins persisted on the attempt record (see AttemptRecordV3). +const ( + // SelectionOriginAuto means the viewer sent no audio identity: the + // server resolved an omitted selection through SelectAudioTrack. + SelectionOriginAuto = "auto" + // SelectionOriginExplicit means the viewer named an audio track. + // Reconciliation must never override it; only verify it still exists. + SelectionOriginExplicit = "explicit" +) + +// AudioReconciliationReplanReason marks the automatic track_change body the +// reconcile path builds. track_change must not carry Failure (see +// ReplanRequestV3.Validate), so the reason travels in the deterministic +// replan-request id and the plan log, both additive and client-invisible. +const AudioReconciliationReplanReason = "default_audio_reconciliation" + +// reconcileVerifiedDefaultAudio replays persisted audio selection intent +// against the committed verified inventory for fileID. It is invoked after +// probe persistence commits (both persistVirtualEvidenceDirect branches and +// the buffered worker), keeps an old code path recording, and never fires +// inside PublishInventoryUpdated: this reconciles the executable recipe, not +// the menu, independent of whether any session holds a realtime connection. +func (h *PlaybackHandler) reconcileVerifiedDefaultAudio(ctx context.Context, fileID int) { + if h == nil || fileID <= 0 || h.sessionMgr == nil { + return + } + lookup, ok := h.sessionMgr.(mediaFileSessionLookup) + if !ok { + return + } + for _, session := range lookup.GetSessionsByMediaFileID(fileID) { + if session == nil || session.ID == "" { + continue + } + h.reconcileSessionDefaultAudio(ctx, session, fileID) + } +} + +// reconcileSessionDefaultAudio reconciles one live session against the +// verified catalog inventory of the file it is bound to. +func (h *PlaybackHandler) reconcileSessionDefaultAudio(ctx context.Context, session *playback.Session, fileID int) { + intent := h.audioSelectionIntentForSession(ctx, session) + switch intent.origin { + case SelectionOriginExplicit: + h.verifyExplicitAudioSelection(ctx, session, intent, fileID) + case SelectionOriginAuto: + h.reconcileAutoAudioSelection(ctx, session, intent, fileID) + default: + // Legacy sessions and records without intent fields predate this + // change: do nothing and never trigger a spurious correction. + slog.DebugContext(ctx, "default audio reconciliation skipped: no selection intent", + "component", "api", "session", session.ID, "file_id", fileID) + } +} + +// audioSelectionIntent is the persisted intent a session started with. +type audioSelectionIntent struct { + origin string + preferredLang string + seriesSignature *userstore.AudioTrackSignature + selectedOverride *userstore.AudioTrackSignature + seriesPref *playback.AudioTrackPreference +} + +// audioSelectionIntentForSession reads the durable intent: the attempt +// record first (it survives session-manager rebuilds), then the live +// session mirror for reconnects before the record is reachable. +func (h *PlaybackHandler) audioSelectionIntentForSession(ctx context.Context, session *playback.Session) audioSelectionIntent { + var intent audioSelectionIntent + if h != nil && h.PlanStoreV3 != nil { + if record, err := h.PlanStoreV3.GetAttempt(ctx, session.ID); err == nil && record != nil { + intent.origin = strings.TrimSpace(record.SelectionOrigin) + intent.preferredLang = strings.TrimSpace(record.PreferredAudioLanguage) + intent.seriesSignature = record.SeriesAudioPreferenceSignature + intent.selectedOverride = record.SelectedAudioSignature + } + _ = ctx + } + if intent.origin == "" { + intent.origin = strings.TrimSpace(session.SelectionOrigin) + } + if intent.preferredLang == "" { + intent.preferredLang = strings.TrimSpace(session.PreferredAudioLanguage) + } + if intent.seriesSignature == nil { + intent.seriesSignature = session.SeriesAudioPreferenceSignature + } + if intent.selectedOverride == nil { + intent.selectedOverride = session.SelectedAudioSignature + } + if intent.seriesSignature != nil || intent.preferredLang != "" { + intent.seriesPref = &playback.AudioTrackPreference{ + AudioLanguage: intent.preferredLang, + TrackSignature: intent.seriesSignature, + } + } + return intent +} + +// reconcileAutoAudioSelection re-runs SelectAudioTrack against the verified +// inventory and issues an automatic track_change replan when the executable +// selection moved. Identical executable selection is a byte-equal no-op. +func (h *PlaybackHandler) reconcileAutoAudioSelection(ctx context.Context, session *playback.Session, intent audioSelectionIntent, fileID int) { + if h == nil || h.fileResolver == nil || h.PlanStoreV3 == nil { + return + } + live, err := h.sessionMgr.GetSession(session.ID) + if err != nil || live == nil { + return + } + verified, err := h.fileResolver.GetByID(ctx, fileID) + if err != nil || verified == nil || len(verified.AudioTracks) == 0 { + return + } + if !virtualEvidenceMatchesBoundFile(verified, live) { + // Source rotation (or a sibling-row write) means this evidence is + // stale for the bound session: refuse, and surface the still-valid + // tracks in memory only, exactly like the refused-write path. + h.publishRefusedProbeInventory(ctx, fileID, strings.TrimSpace(live.VirtualSourceURI), verified) + slog.InfoContext(ctx, "default audio reconciliation refused: stale evidence for the bound candidate", + "component", "api", "session", session.ID, "file_id", fileID) + return + } + record, err := h.PlanStoreV3.GetAttempt(ctx, live.ID) + if err != nil || record == nil { + return + } + if strings.TrimSpace(record.SelectionOrigin) != "" && strings.TrimSpace(record.SelectionOrigin) != SelectionOriginAuto { + // A racing explicit change persisted first: never override. + return + } + // The live session must still carry the committed start selection: a + // viewer track_change between start and probe supersedes the intent + // (the race case), and reconciliation must not override the viewer's + // newer choice. Compare executable signature identity, not the ordinal, + // because the reorder is exactly what can shift ordinals. + if !liveSelectionStillCommitted(live, record, intent) { + return + } + committedIndex := committedAudioTrackIndexV3(record, live) + if committedIndex < 0 || committedIndex >= len(live.VirtualAudioTracks) { + // Non-virtual or legacy session without a plan-time inventory: the + // committed rows still map by verified order, which is exactly what + // SelectAudioTrack replays below. Fall through with the verified set + // as the plan-time set. + live = &playback.Session{ + ID: live.ID, + UserID: live.UserID, + ProfileID: live.ProfileID, + AudioTrackIndex: committedIndex, + VirtualAudioTracks: verified.AudioTracks, + } + if committedIndex < 0 || committedIndex >= len(verified.AudioTracks) { + return + } + } + recomputed := playback.SelectAudioTrack(verified.AudioTracks, intent.preferredLang, intent.seriesPref) + if recomputed < 0 || recomputed >= len(verified.AudioTracks) { + return + } + committed := live.VirtualAudioTracks[committedIndex] + selected := verified.AudioTracks[recomputed] + if audioSignatureMatchesCommittedTrack(selected, committed, intent, committedIndex, recomputed, live.VirtualAudioTracks, verified.AudioTracks) { + // Byte-equal no-op: the executable selection is identical. + return + } + if err := h.reconcileDefaultAudioReplan(ctx, live, record, verified, recomputed); err != nil { + slog.WarnContext(ctx, "default audio reconciliation replan failed", + "component", "api", "session", live.ID, "file_id", fileID, "error", err) + } +} + +// committedAudioTrackIndexV3 resolves the executable committed audio index: +// the plan's selected index when present, else the session's committed index. +func committedAudioTrackIndexV3(record *playback.AttemptRecordV3, session *playback.Session) int { + if record != nil && record.CurrentPlan.SelectedTracks.Audio != nil && record.CurrentPlan.SelectedTracks.Audio.Index != nil { + return *record.CurrentPlan.SelectedTracks.Audio.Index + } + if session != nil { + return session.AudioTrackIndex + } + return 0 +} + +// audioSignatureMatchesCommittedTrack reports whether the recomputed verified +// selection is the same executable selection the recipe committed: same audio +// ordinal rank, same transport-relevant facts (codec + channel layout), and +// the same language evidence the intent produced at start. Same ordinal alone +// is NOT enough (a reorder keeps ordinals while moving languages), and same +// language alone is not enough (a different stream would need a fresh recipe). +func audioSignatureMatchesCommittedTrack(selected, committed models.AudioTrack, intent audioSelectionIntent, committedIndex, recomputed int, planTime, verified []models.AudioTrack) bool { + if playback.AudioStreamOrdinal(planTime, committedIndex) != playback.AudioStreamOrdinal(verified, recomputed) { + return false + } + if !audioTransportFactsEqual(selected, committed) { + return false + } + if intent.selectedOverride != nil { + if !audioTrackSignatureEqual(selected, *intent.selectedOverride) { + return false + } + } + return defaultAudioLanguageStillSatisfied(selected, intent) +} + +// audioTransportFactsEqual is the transport reuse guard: the existing +// transport (and any fixed ffmpeg audio map on it) is only byte-identical +// when the chosen stream decodes the same codec with the same channel +// layout. When these differ the caller must not reuse the map; it issues a +// fresh recipe via the automatic replan instead. +func audioTransportFactsEqual(a, b models.AudioTrack) bool { + return strings.EqualFold(strings.TrimSpace(a.Codec), strings.TrimSpace(b.Codec)) && + a.Channels == b.Channels && + strings.EqualFold(strings.TrimSpace(a.Layout), strings.TrimSpace(b.Layout)) +} + +// audioTrackSignatureEqual compares two probed tracks by the same stable +// signature the series preference persists: language (canonical), title, +// codec, layout and channel count. +func audioTrackSignatureEqual(track models.AudioTrack, sig userstore.AudioTrackSignature) bool { + candidate := playback.AudioTrackSignatureFromTrack(track) + if candidate == nil || sig.IsZero() { + return candidate == nil && sig.IsZero() + } + return audioSignatureFieldsEqual(*candidate, sig) +} + +// audioSignatureFieldsEqual compares signature snapshots field by field: the +// type carries slices, so it is not directly comparable. +func audioSignatureFieldsEqual(a, b userstore.AudioTrackSignature) bool { + return a.Language == b.Language && + a.Title == b.Title && + a.EmbeddedTitle == b.EmbeddedTitle && + a.Codec == b.Codec && + a.Layout == b.Layout && + a.Channels == b.Channels && + slices.Equal(a.Languages, b.Languages) +} + +// defaultAudioLanguageStillSatisfied reports whether the recomputed track +// still carries the preferred language (or no preference was ever resolved). +// MULTi membership goes through the shared membership helper's authority: a +// bare MULTI/DUAL never counts as a concrete match, a member list entry does. +func defaultAudioLanguageStillSatisfied(track models.AudioTrack, intent audioSelectionIntent) bool { + return intent.preferredLang == "" || playback.TrackCarriesLanguage(track, intent.preferredLang) +} + +// reconcileDefaultAudioReplan issues the automatic track_change replan built +// from the recipe, remapped onto the verified order, preserving position, +// with no route exclusion and no failure classification (track_change must +// not carry one). The replan-request id names the reconciliation reason so +// the durable lease history attributes the change to the server, not the +// viewer; no new operation or client-visible field is introduced. +func (h *PlaybackHandler) reconcileDefaultAudioReplan(ctx context.Context, session *playback.Session, record *playback.AttemptRecordV3, verified *models.MediaFile, audioIndex int) error { + verifiedIndex := audioIndex + audioID := playback.TrackIDV3(verified.ID, "audio", verifiedIndex) + caller := PlaybackCaller{ + UserID: session.UserID, + ProfileID: session.ProfileID, + } + req := playback.ReplanRequestV3{ + ProtocolVersion: playback.ProtocolV3, + Operation: playback.ReplanOperationTrackChangeV3, + PlaybackAttemptID: record.PlaybackAttemptID, + ReplanRequestID: reconcileReplanRequestID(session, verifiedIndex, "req"), + FailedPlanID: record.CurrentPlanID, + PlanAttemptID: reconcileReplanRequestID(session, verifiedIndex, "plan"), + PlanAttemptKey: record.CurrentPlan.PlanAttemptKey, + AttemptedPlanKeys: append([]string(nil), record.CurrentPlan.PlanAttemptKey), + AttemptCount: 1, + QualityPreference: record.NormalizedRequest.QualityPreference, + PositionSeconds: session.Position, + Metered: record.NormalizedRequest.Metered, + SelectedTracks: playback.SelectedTracksV3{ + Audio: &playback.TrackIdentityV3{ID: audioID, Index: &verifiedIndex}, + Subtitle: record.CurrentPlan.SelectedTracks.Subtitle, + }, + Capabilities: record.NormalizedRequest.Capabilities, + ClientPlaybackContext: record.NormalizedRequest.ClientPlaybackContext, + } + body, err := json.Marshal(req) + if err != nil { + return err + } + response, err := h.ReplanPlaybackV2(ctx, caller, session.ID, PlaybackReplanCommand{Request: req, Digest: ReplanDigestV3(body)}) + if err != nil { + return err + } + _ = response + slog.InfoContext(ctx, "default audio reconciled to the verified inventory", + "component", "api", "session", session.ID, "file_id", verified.ID, + "audio_index", verifiedIndex, "reason", AudioReconciliationReplanReason) + return nil +} + +// verifyExplicitAudioSelection checks that an explicit viewer selection still +// exists in the verified inventory. It never overrides: a vanished track +// surfaces a terminal diagnostic (route event + log) so the viewer learns the +// selection is gone instead of hearing a silently substituted track. +func (h *PlaybackHandler) verifyExplicitAudioSelection(ctx context.Context, session *playback.Session, intent audioSelectionIntent, fileID int) { + if h == nil || h.fileResolver == nil { + return + } + verified, err := h.fileResolver.GetByID(ctx, fileID) + if err != nil || verified == nil || len(verified.AudioTracks) == 0 { + return + } + live, err := h.sessionMgr.GetSession(session.ID) + if err != nil || live == nil { + return + } + if !virtualEvidenceMatchesBoundFile(verified, live) { + return + } + var record *playback.AttemptRecordV3 + if h.PlanStoreV3 != nil { + record, _ = h.PlanStoreV3.GetAttempt(ctx, live.ID) + } + committedIndex := committedAudioTrackIndexV3(record, live) + track, ok := audioTrackAtCommittedIndex(verified.AudioTracks, committedIndex) + if ok && intent.selectedOverride != nil && !intent.selectedOverride.IsZero() { + ok = audioTrackSignatureEqual(track, *intent.selectedOverride) + } + if ok { + return + } + slog.WarnContext(ctx, "explicit audio selection missing from the verified inventory", + "component", "api", "session", live.ID, "file_id", fileID, "audio_index", committedIndex) + if record != nil { + h.enqueueRouteEventV3(playback.RouteEventRecordV3{ + RouteEventV3: playback.RouteEventV3{ + ProtocolVersion: playback.ProtocolV3, + PlaybackAttemptID: record.PlaybackAttemptID, + SessionID: live.ID, + PlanID: record.CurrentPlanID, + Event: playback.RouteEventTerminalV3, + OutputContextID: record.NormalizedRequest.ClientPlaybackContext.Output.OutputContextID, + }, + UserID: live.UserID, + ProfileID: live.ProfileID, + }) + } +} + +// audioTrackAtCommittedIndex returns the verified track at the committed +// index, or false when the reorder dropped it out of range. +func audioTrackAtCommittedIndex(tracks []models.AudioTrack, index int) (models.AudioTrack, bool) { + if index < 0 || index >= len(tracks) { + return models.AudioTrack{}, false + } + return tracks[index], true +} + +// importGuardReconcileAudio keeps ReplanDigestV3 discoverable for this file's +// command builder; the digest function lives in playback_service.go. +var _ = ReplanDigestV3 + +// reconcilePendingAudioStartup re-checks one session when it attaches after +// its probe already landed: probe landing with no registered session leaves +// nothing to reconcile, so the attach (and first heartbeats, which reuse the +// session lookup) replays the same per-session path once the attempt exists. +func (h *PlaybackHandler) reconcilePendingAudioStartup(ctx context.Context, sessionID string) { + if h == nil || h.sessionMgr == nil || h.PlanStoreV3 == nil || sessionID == "" { + return + } + session, err := h.sessionMgr.GetSession(sessionID) + if err != nil || session == nil { + return + } + record, err := h.PlanStoreV3.GetAttempt(ctx, sessionID) + if err != nil || record == nil { + return + } + if strings.TrimSpace(record.SelectionOrigin) == "" { + return + } + deadline, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + h.reconcileSessionDefaultAudio(deadline, session, record.EffectiveMediaFileID) +} + +// reconcileReplanRequestID mints a deterministic, human-readable replan +// identity for one automatic reconciliation: the reason prefix attributes +// the change, the session hash and target index keep it unique per +// (session, decision) without depending on the session id being a UUID. +func reconcileReplanRequestID(session *playback.Session, audioIndex int, kind string) string { + sum := sha256.Sum256([]byte(session.ID)) + return fmt.Sprintf("%s-%s-%s-%d", AudioReconciliationReplanReason, kind, hex.EncodeToString(sum[:])[:8], audioIndex) +} + +// liveSelectionStillCommitted reports whether the live session's committed +// audio selection still matches the start-time selected signature: the +// user-change race guard. A viewer track_change between start and probe +// moves the live selection off the start signature, and the automatic +// replan must then decline instead of overriding the viewer's choice. +func liveSelectionStillCommitted(session *playback.Session, record *playback.AttemptRecordV3, intent audioSelectionIntent) bool { + if session == nil || intent.selectedOverride == nil || intent.selectedOverride.IsZero() { + return true + } + committedIndex := committedAudioTrackIndexV3(record, session) + tracks := session.VirtualAudioTracks + track, ok := audioTrackAtCommittedIndex(tracks, committedIndex) + if !ok { + return true + } + return audioTrackSignatureEqual(track, *intent.selectedOverride) +} diff --git a/internal/api/handlers/playback_reconcile_audio_test.go b/internal/api/handlers/playback_reconcile_audio_test.go new file mode 100644 index 0000000000..6e168a652f --- /dev/null +++ b/internal/api/handlers/playback_reconcile_audio_test.go @@ -0,0 +1,455 @@ +package handlers + +import ( + "context" + "encoding/json" + "math/rand" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/userstore" +) + +// reconcileTestSessionManager is the minimal session surface the +// reconciliation path needs: live sessions plus lookup by file. +type reconcileTestSessionManager struct { + *playback.SessionManager +} + +// verifiedFileResolver serves one fixed catalog row for reconcile tests. +type verifiedFileResolver struct { + file *models.MediaFile +} + +func (r verifiedFileResolver) GetByID(_ context.Context, _ int) (*models.MediaFile, error) { + return r.file, nil +} + +// reconcileFixture wires a handler with a live session, an attempt record +// and a fixed verified inventory. The session's VirtualSourceURI is left +// empty so virtualEvidenceMatchesBoundFile refuses when there is no bound +// candidate; callers that exercise the reconcile path set the fields +// explicitly. InstallationID stays empty on purpose: the full automatic +// replan seam requires caller/installation wiring, so tests assert the +// reconcile decision surface, not the transport commit. +func reconcileFixture(t *testing.T, session *playback.Session, record *playback.AttemptRecordV3, verified *models.MediaFile) *PlaybackHandler { + t.Helper() + manager := playback.NewSessionManager(0, 0) + copy := *session + manager.RegisterReconstructed(©) + + handler := NewPlaybackHandler(manager, verifiedFileResolver{file: verified}) + if err := handler.PlanStoreV3.SaveAttempt(context.Background(), *record); err != nil { + t.Fatalf("save attempt: %v", err) + } + if err := manager.SetAudioSelectionIntent(session.ID, session.SelectionOrigin, session.PreferredAudioLanguage, session.SeriesAudioPreferenceSignature, session.SelectedAudioSignature); err != nil { + t.Fatalf("set intent: %v", err) + } + return handler +} + +func reconcileLiveSession(t *testing.T, handler *PlaybackHandler, sessionID string) *playback.Session { + t.Helper() + manager, ok := handler.sessionMgr.(*playback.SessionManager) + if !ok { + t.Fatal("handler session manager is not a *playback.SessionManager") + } + live, err := manager.GetSession(sessionID) + if err != nil { + t.Fatalf("get session: %v", err) + } + return live +} + +func reconcileIntent(preferred string, series *userstore.AudioTrackSignature, selected *userstore.AudioTrackSignature) *playback.AttemptRecordV3 { + return &playback.AttemptRecordV3{ + PlaybackAttemptID: "attempt-reconcile-1", + SessionID: "11111111-1111-1111-1111-111111111111", + ExpiresAt: time.Now().Add(time.Hour), + UserID: 1, + ProfileID: "profile-1", + RequestedMediaFileID: 7, + EffectiveMediaFileID: 7, + CurrentPlanID: "plan-reconcile-1", + SelectionOrigin: SelectionOriginAuto, + PreferredAudioLanguage: preferred, + SeriesAudioPreferenceSignature: series, + SelectedAudioSignature: selected, + } +} + +// The cold-start story: declared order [en, pt] committed index 0 while the +// preference is pt; the verified inventory reverses to [pt, en] with sparse +// absolute stream indexes. Reconciliation must select the preferred +// language at its verified position, not the same ordinal. +func TestReconcileVerifiedDefaultAudioReordersToPreferredLanguage(t *testing.T) { + planTime := []models.AudioTrack{ + {Index: 1, Language: "en", Codec: "aac", Channels: 2}, + {Index: 3, Language: "pt", Codec: "aac", Channels: 2}, + } + verified := &models.MediaFile{ID: 7, AudioTracks: []models.AudioTrack{ + {Index: 3, Language: "pt", Codec: "aac", Channels: 2}, + {Index: 1, Language: "en", Codec: "aac", Channels: 2}, + }} + session := &playback.Session{ + ID: "11111111-1111-1111-1111-111111111111", UserID: 1, ProfileID: "profile-1", + MediaFileID: 7, RequestedMediaFileID: 7, PlayMethod: playback.PlayDirect, + AudioTrackIndex: 0, + SelectionOrigin: SelectionOriginAuto, + PreferredAudioLanguage: "pt", + SelectedAudioSignature: playback.AudioTrackSignatureFromTrack(planTime[0]), + VirtualAudioTracks: planTime, + VirtualSourceURI: "virtual://movie/tt-reconcile", + VirtualSubtitleEvidenceURI: "virtual://movie/tt-reconcile", + VirtualSubtitleEvidenceFileID: 7, + VirtualSubtitleEvidenceSet: true, + } + record := reconcileIntent("pt", nil, playback.AudioTrackSignatureFromTrack(planTime[0])) + record.CurrentPlan.SelectedTracks.Audio = &playback.TrackIdentityV3{ID: playback.TrackIDV3(7, "audio", 0), Index: intPtrReconcile(0)} + verified.FilePath = "virtual://movie/tt-reconcile" + + handler := reconcileFixture(t, session, record, verified) + live := reconcileLiveSession(t, handler, session.ID) + + // Prove the executable comparison alone demands the verified position: + // index 0 in declared order committed en, while pt now sits at + // verified index 0. The same comparison must hold after the + // byte-equal check fails inside the reconcile path. + intent := handler.audioSelectionIntentForSession(context.Background(), live) + if got := playback.SelectAudioTrack(verified.AudioTracks, intent.preferredLang, intent.seriesPref); got != 0 { + t.Fatalf("verified selection = %d, want 0 (pt at its verified position)", got) + } + if audioSignatureMatchesCommittedTrack(verified.AudioTracks[0], planTime[0], intent, 0, 0, planTime, verified.AudioTracks) { + t.Fatal("reordered languages must not compare byte-equal") + } + // Simulate the evidence write binding the row to the bound candidate. + // The fixture keeps InstallationID empty, so the full replan seam + // refuses before planning; assert the reconcile path reaches that + // decision without mutating the committed attempt or pushing a menu. + handler.reconcileSessionDefaultAudio(context.Background(), live, 7) + + after, err := handler.PlanStoreV3.GetAttempt(context.Background(), session.ID) + if err != nil { + t.Fatalf("get attempt: %v", err) + } + if after.CurrentPlanID != record.CurrentPlanID { + t.Fatal("refused automatic replan must not commit a new plan") + } +} + +func intPtrReconcile(v int) *int { return &v } + +// sessionForReconcile is a UUID so the automatic replan seam (which +// requires canonical session ids) accepts the fixture session. +const sessionForReconcile = "11111111-1111-1111-1111-111111111111" + +// MULTi membership: a track whose Languages list carries eng beats a bare +// eng track only on rank, while a bare MULTi primary with no member list +// never counts as a concrete language match. +func TestReconcileMultiMembershipSemantics(t *testing.T) { + member := models.AudioTrack{Language: "mul", Languages: []string{"eng", "fre"}, Codec: "eac3", Channels: 6} + if !playback.TrackCarriesLanguage(member, "eng") { + t.Fatal("MULTi track with [eng,fre] must carry eng") + } + if !playback.TrackCarriesLanguage(member, "fre") { + t.Fatal("MULTi track with [eng,fre] must carry fre") + } + bare := models.AudioTrack{Language: "mul", Codec: "eac3", Channels: 6} + if playback.TrackCarriesLanguage(bare, "eng") { + t.Fatal("bare MULTi (no member list) must not match eng") + } + dual := models.AudioTrack{Language: "DUAL", Codec: "ac3", Channels: 6} + if playback.TrackCarriesLanguage(dual, "eng") { + t.Fatal("bare DUAL must not match eng") + } + tracks := []models.AudioTrack{ + bare, + {Language: "eng", Codec: "aac", Channels: 2}, + } + if got := playback.SelectAudioTrack(tracks, "eng", nil); got != 1 { + t.Fatalf("bare MULTi must not steal the preference: selected %d, want 1", got) + } + if got := playback.SelectAudioTrack(tracks, "deu", nil); got != 0 { + t.Fatalf("bare MULTi stays eligible as neutral fallback: selected %d, want 0", got) + } +} + +// Regional fidelity: a pt-BR preference selects the exact pt-BR track over +// pt-PT and bare pt, using the same rank chain that start uses. +func TestReconcileRegionalVariantFidelity(t *testing.T) { + tracks := []models.AudioTrack{ + {Index: 1, Language: "pt-PT", Codec: "aac", Channels: 2}, + {Index: 2, Language: "pt-BR", Codec: "aac", Channels: 2}, + {Index: 3, Language: "pt", Codec: "aac", Channels: 2}, + } + if got := playback.SelectAudioTrack(tracks, "pt-BR", nil); got != 1 { + t.Fatalf("pt-BR preference selected %d, want 1 (exact pt-BR)", got) + } + verified := &models.MediaFile{ID: 9, AudioTracks: []models.AudioTrack{ + {Index: 2, Language: "pt-BR", Codec: "aac", Channels: 2}, + {Index: 1, Language: "pt-PT", Codec: "aac", Channels: 2}, + }} + selected := verified.AudioTracks[0] + intent := audioSelectionIntent{origin: SelectionOriginAuto, preferredLang: "pt-BR"} + if !defaultAudioLanguageStillSatisfied(selected, intent) { + t.Fatal("reordered exact pt-BR track must still satisfy the pt-BR preference") + } + if defaultAudioLanguageStillSatisfied(verified.AudioTracks[1], intent) == false { + // pt-PT is a regional variant, not an exact tag; it still satisfies + // the language under the rank chain but must not win over exact. + t.Fatal("pt-PT must rank under the pt-BR preference, not fail it outright") + } +} + +// Omitted vs explicit: an explicit selection verified present is never +// moved; a legacy record with no intent fields never reconciles. +func TestReconcileExplicitSelectionSurvivesProbe(t *testing.T) { + verified := []models.AudioTrack{ + {Index: 1, Language: "en", Codec: "aac", Channels: 2}, + {Index: 2, Language: "fr", Codec: "aac", Channels: 2}, + } + intent := audioSelectionIntent{ + origin: SelectionOriginExplicit, + selectedOverride: playback.AudioTrackSignatureFromTrack(verified[1]), + } + if !audioTrackSignatureEqual(verified[1], *intent.selectedOverride) { + t.Fatal("explicit fr track must still verify present") + } + if audioTrackSignatureEqual(verified[0], *intent.selectedOverride) { + t.Fatal("explicit fr signature must not match the en track") + } + + legacy := audioSelectionIntent{} + if legacy.origin == SelectionOriginAuto || legacy.origin == SelectionOriginExplicit { + t.Fatal("legacy record without intent fields must carry no origin") + } + var raw map[string]any + if err := json.Unmarshal([]byte(`{"playback_attempt_id":"x"}`), &raw); err != nil { + t.Fatal(err) + } + old := playback.AttemptRecordV3{} + blob, _ := json.Marshal(old) + var revived playback.AttemptRecordV3 + if err := json.Unmarshal(blob, &revived); err != nil { + t.Fatalf("old record must deserialize safely: %v", err) + } + if revived.SelectionOrigin != "" || revived.SelectedAudioSignature != nil || revived.SeriesAudioPreferenceSignature != nil { + t.Fatal("old record must carry zero intent fields") + } +} + +// Duplicates: two ordinal-equal tracks with different absolute indexes keep +// the first stable winner; the byte-equal check must hold. +func TestReconcileDuplicateTracksFirstStableWins(t *testing.T) { + planTime := []models.AudioTrack{ + {Index: 1, Language: "en", Codec: "aac", Channels: 2}, + {Index: 3, Language: "en", Codec: "aac", Channels: 2}, + } + if got := playback.SelectAudioTrack(planTime, "en", nil); got != 0 { + t.Fatalf("duplicate en tracks: selected %d, want first stable 0", got) + } + verified := []models.AudioTrack{ + {Index: 1, Language: "en", Codec: "aac", Channels: 2}, + {Index: 3, Language: "en", Codec: "aac", Channels: 2}, + } + intent := audioSelectionIntent{ + origin: SelectionOriginAuto, + preferredLang: "en", + selectedOverride: playback.AudioTrackSignatureFromTrack(planTime[0]), + } + if !audioSignatureMatchesCommittedTrack(verified[0], planTime[0], intent, 0, 0, planTime, verified) { + t.Fatal("identical inventories must be a byte-equal no-op") + } + // Same ordinal but a different language after reorder is not identical: + // the language authority must reject it. + swapped := []models.AudioTrack{ + {Index: 1, Language: "fr", Codec: "aac", Channels: 2}, + {Index: 3, Language: "en", Codec: "aac", Channels: 2}, + } + if audioSignatureMatchesCommittedTrack(swapped[0], planTime[0], intent, 0, 0, planTime, swapped) { + t.Fatal("same ordinal with a different language is not the same executable selection") + } +} + +// Transport reuse guard: a stream that decodes differently (codec or +// channel layout changed) must never reuse a fixed ffmpeg audio map. +func TestReconcileTransportReuseGuard(t *testing.T) { + base := models.AudioTrack{Language: "en", Codec: "eac3", Channels: 6, Layout: "5.1"} + if !audioTransportFactsEqual(base, models.AudioTrack{Language: "en", Codec: "EAC3", Channels: 6, Layout: "5.1"}) { + t.Fatal("identical codec/layout must reuse") + } + for _, changed := range []models.AudioTrack{ + {Language: "en", Codec: "aac", Channels: 6, Layout: "5.1"}, + {Language: "en", Codec: "eac3", Channels: 2, Layout: "stereo"}, + {Language: "en", Codec: "eac3", Channels: 6, Layout: "stereo"}, + } { + if audioTransportFactsEqual(base, changed) { + t.Fatalf("changed transport facts must not reuse: %+v", changed) + } + } +} + +// Permutation fuzz: a unique best language match survives ~200 order +// permutations, so the reorder can never move the preference by position. +func TestReconcileSelectAudioTrackPermutationStable(t *testing.T) { + languages := []string{"en", "fr", "de", "es", "it", "ja"} + preferred := "fr" + rng := rand.New(rand.NewSource(42)) + for i := 0; i < 200; i++ { + order := rng.Perm(len(languages)) + tracks := make([]models.AudioTrack, len(languages)) + want := -1 + for pos, li := range order { + tracks[pos] = models.AudioTrack{Index: pos + 1, Language: languages[li], Codec: "aac", Channels: 2} + if languages[li] == preferred { + want = pos + } + } + if got := playback.SelectAudioTrack(tracks, preferred, nil); got != want || tracks[got].Language != preferred { + t.Fatalf("permutation %d: selected %d (%q), want %d (%q)", i, got, tracks[got].Language, want, preferred) + } + } +} + +// Canonical dedup: pt-BR and pt-PT are distinct provider hints; exact +// duplicates collapse, regional variants never do. +func TestMergeVirtualCandidateLanguagesPreservesRegionalVariants(t *testing.T) { + probed := &models.MediaFile{} + candidate := VirtualPlaybackStream{AudioLanguages: []string{"pt-BR", "pt-PT", "pt-br", "ENG", "eng"}} + mergeVirtualCandidateTracks(probed, candidate) + if len(probed.AudioTracks) != 3 { + t.Fatalf("tracks = %d, want 3 (pt-BR, pt-PT, en)", len(probed.AudioTracks)) + } + langs := map[string]bool{} + for _, track := range probed.AudioTracks { + langs[track.Language] = true + } + for _, want := range []string{"pt-BR", "pt-PT", "ENG"} { + if !langs[want] { + t.Errorf("missing regional variant %q: got %+v", want, probed.AudioTracks) + } + } +} + +// User-change race: when the committed selection no longer matches the +// start-time selected signature (the viewer switched tracks while the probe +// was in flight), the reconcile path must not override the viewer's choice. +// Exercised through the same race guard the automatic path checks before +// building its replan body. +func TestReconcileUserChangeRaceNeverOverrides(t *testing.T) { + verified := []models.AudioTrack{ + {Index: 1, Language: "en", Codec: "aac", Channels: 2}, + {Index: 2, Language: "fr", Codec: "aac", Channels: 2}, + } + // Start committed en (index 0); the viewer then switched to fr. + startSig := playback.AudioTrackSignatureFromTrack(verified[0]) + intent := audioSelectionIntent{origin: SelectionOriginAuto, preferredLang: "en", selectedOverride: startSig} + current := verified[1] + if audioTrackSignatureEqual(current, *intent.selectedOverride) { + t.Fatal("post-switch fr track must not equal the start-time en signature") + } + // The reconcile decision must therefore treat the live selection as + // superseding the intent and decline to build a replan. + committedIndex := 1 + committed := verified[committedIndex] + recomputed := playback.SelectAudioTrack(verified, intent.preferredLang, intent.seriesPref) + if recomputed != 0 { + t.Fatalf("en preference recomputes to %d, want 0", recomputed) + } + if audioSignatureMatchesCommittedTrack(verified[recomputed], committed, intent, committedIndex, recomputed, verified, verified) { + t.Fatal("a user-switched selection must never compare byte-equal to the start intent") + } + // The live-session race guard must also refuse: the session's committed + // index now names the fr track, not the start-time en signature. + race := &playback.Session{AudioTrackIndex: committedIndex, VirtualAudioTracks: verified} + if liveSelectionStillCommitted(race, nil, intent) { + t.Fatal("live race guard must refuse a viewer-switched selection") + } + // And accept an unmoved selection. + calm := &playback.Session{AudioTrackIndex: 0, VirtualAudioTracks: verified} + if !liveSelectionStillCommitted(calm, nil, intent) { + t.Fatal("live race guard must accept the unchanged start selection") + } +} + +// Source rotation: evidence naming a different candidate than the bound +// session must be refused (and surfaced in memory) instead of reconciled. +func TestReconcileSourceRotationRefusesStaleEvidence(t *testing.T) { + file := &models.MediaFile{ID: 11, FilePath: "virtual://movie/tt-new?result=2", AudioTracks: []models.AudioTrack{ + {Index: 1, Language: "en", Codec: "aac", Channels: 2}, + }} + session := &playback.Session{ + ID: "22222222-2222-2222-2222-222222222222", + UserID: 1, + ProfileID: "profile-1", + MediaFileID: 11, + AudioTrackIndex: 0, + SelectionOrigin: SelectionOriginAuto, + PreferredAudioLanguage: "en", + VirtualAudioTracks: file.AudioTracks, + VirtualSourceURI: "virtual://movie/tt-old?result=1", + VirtualSubtitleEvidenceURI: "virtual://movie/tt-old?result=1", + VirtualSubtitleEvidenceFileID: 11, + VirtualSubtitleEvidenceSet: true, + VirtualSourceRevision: "rev-old", + } + if virtualEvidenceMatchesBoundFile(file, session) { + t.Fatal("rotated candidate URI must not match the bound evidence") + } +} + +// Probe-before-registration: when the attempt does not exist yet (probe +// landed before the session attached), the attach-side hook must find no +// record and return without failing; legacy records without intent must +// likewise do nothing. +func TestReconcileProbeBeforeRegistrationDefersToAttach(t *testing.T) { + manager := playback.NewSessionManager(0, 0) + session := &playback.Session{ + ID: "33333333-3333-3333-3333-333333333333", UserID: 1, ProfileID: "profile-1", + MediaFileID: 13, + } + manager.RegisterReconstructed(session) + handler := NewPlaybackHandler(manager, verifiedFileResolver{file: &models.MediaFile{ID: 13}}) + // No attempt saved: the attach hook must no-op instead of erroring. + handler.reconcilePendingAudioStartup(context.Background(), session.ID) + + record := &playback.AttemptRecordV3{ + PlaybackAttemptID: "attempt-legacy-13", + SessionID: session.ID, + UserID: 1, + ProfileID: "profile-1", + EffectiveMediaFileID: 13, + CurrentPlanID: "plan-legacy-13", + ExpiresAt: nowPlusHour(), + // No SelectionOrigin: a record persisted before intent capture. + } + if err := handler.PlanStoreV3.SaveAttempt(context.Background(), *record); err != nil { + t.Fatalf("save attempt: %v", err) + } + live, err := manager.GetSession(session.ID) + if err != nil { + t.Fatalf("get session: %v", err) + } + before, err := json.Marshal(live) + if err != nil { + t.Fatal(err) + } + handler.reconcilePendingAudioStartup(context.Background(), session.ID) + afterLive, err := manager.GetSession(session.ID) + if err != nil { + t.Fatalf("get session: %v", err) + } + after, err := json.Marshal(afterLive) + if err != nil { + t.Fatal(err) + } + if string(before) != string(after) { + t.Fatal("legacy record without intent must leave the session untouched") + } +} + +func nowPlusHour() (t time.Time) { + return time.Now().Add(time.Hour) +} diff --git a/internal/api/handlers/playback_service.go b/internal/api/handlers/playback_service.go index 8694afa0c7..fd4b5ab0f0 100644 --- a/internal/api/handlers/playback_service.go +++ b/internal/api/handlers/playback_service.go @@ -22,6 +22,7 @@ import ( "github.com/Silo-Server/silo-server/internal/logredact" "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/userstore" ) // Playback service seams for the v2 adapter (internal/apiv2/playback.go). @@ -1491,6 +1492,32 @@ func (h *PlaybackHandler) publishInventoryUpdatedToSession(ctx context.Context, return inventory.InventoryRevision, true } +// AudioReconciliationReasonV3 names the automatic default-audio +// reconciliation replan: it is a server-initiated track_change with no user +// gesture behind it, preserved in ClientPlaybackContext-independent data so +// the replan lease history distinguishes it from a viewer's manual switch. +// The replan still flows through the same application path (see §2 of +// playback_reconcile_audio.go); the marker only attributes the decision. +const AudioReconciliationReasonV3 = "automatic_default_audio_reconciliation" + +// SetAudioSelectionIntent is the handler-owned entry point for recording one +// session's durable audio selection intent at start time. It binds the +// origin, the resolved preferred language, the series-preference snapshot and +// the committed track signature on the live session; the attempt record +// carries the same tuple for sessions rebuilt without the live copy. +func (h *PlaybackHandler) SetAudioSelectionIntent(sessionID, origin, preferredLang string, seriesSig, selectedSig *userstore.AudioTrackSignature) error { + if h == nil || h.sessionMgr == nil { + return errors.New("session manager is not configured") + } + setter, ok := h.sessionMgr.(interface { + SetAudioSelectionIntent(sessionID, origin, preferredLang string, seriesSig, selectedSig *userstore.AudioTrackSignature) error + }) + if !ok { + return errors.New("session manager cannot record audio selection intent") + } + return setter.SetAudioSelectionIntent(sessionID, origin, preferredLang, seriesSig, selectedSig) +} + // ReplanDigestV3 fingerprints the exact replan body so a reused request id with // different input is a detectable idempotency violation. func ReplanDigestV3(body []byte) string { diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index 93f9d58a13..5ed276ff53 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -41,6 +41,7 @@ import ( "github.com/Silo-Server/silo-server/internal/subtitles" "github.com/Silo-Server/silo-server/internal/tonemap" "github.com/Silo-Server/silo-server/internal/transcodenode" + "github.com/Silo-Server/silo-server/internal/userstore" ) const ( @@ -1946,6 +1947,15 @@ func (h *PlaybackHandler) startPlaybackApplicationV3(r *http.Request, body []byt if err != nil { return playback.DecisionResponseV3{}, playbackOperationError(http.StatusInternalServerError, "internal_error", "Failed to load the saved audio preference") } + if req.CarriedAudioTrackID == "" { + // A carried selection is an explicit viewer choice remapped onto + // this file; only a fully omitted selection is server-resolved. + virtualDecision.startAudioSelectionOrigin = SelectionOriginAuto + } else { + virtualDecision.startAudioSelectionOrigin = SelectionOriginExplicit + } + } else { + virtualDecision.startAudioSelectionOrigin = SelectionOriginExplicit } timings.mark("audio_preference") effectiveFile := requestedFile @@ -2828,7 +2838,7 @@ func (h *PlaybackHandler) rotateRejectedVirtualCandidateStartV3( if planResult.Terminal != nil || planResult.Plan == nil { return playback.DecisionResponseV3{}, false } - response, statusErr := h.startPlannedPlaybackV3(r, userID, profileID, req, requestDigests, catalogFile, &resolvedFile, audioIndex, virtualPlanDecisionV3{candidateRank: resolved.CandidateRank, candidateCount: resolved.CandidateCount, substitutionReason: substitutionReasonDecodeRejectedV3}, planResult, clientInfo, substitutionReasonDecodeRejectedV3) + response, statusErr := h.startPlannedPlaybackV3(r, userID, profileID, req, requestDigests, catalogFile, &resolvedFile, audioIndex, virtualPlanDecisionV3{candidateRank: resolved.CandidateRank, candidateCount: resolved.CandidateCount, substitutionReason: substitutionReasonDecodeRejectedV3, startAudioSelectionOrigin: virtualPlanSelectionOriginForRequestV3(req)}, planResult, clientInfo, substitutionReasonDecodeRejectedV3) if statusErr == nil { // The start committed a replacement candidate after excluding the // rejected ones. Persist the chain on the new attempt so a later @@ -3003,6 +3013,11 @@ type virtualPlanDecisionV3 struct { // substitutionReason carries the alternate-version or rotation walk's // substitution cause to the plan. Empty for a same-release resolve. substitutionReason string + // startAudioSelectionOrigin records whether the committed audio + // selection was omitted ("auto") or explicitly picked ("explicit"). + // Captured where preferredAudioTrackIndexV3 returns, before defaults + // erase the distinction, and consumed by intentForCommittedStartV3. + startAudioSelectionOrigin string } // selectedAudioTrackLogFieldsV3 names the audio track a plan will play: its @@ -3179,6 +3194,26 @@ func (h *PlaybackHandler) startPlannedPlaybackV3(r *http.Request, userID int, pr } response := playback.DecisionResponseV3{ProtocolVersion: playback.ProtocolV3, ServerFeatures: serverFeaturesForRequestV3(r.Context()), Outcome: playback.OutcomePlayableV3, SessionID: session.ID, PlaybackPlan: result.Plan} record := playback.AttemptRecordV3{PlaybackAttemptID: req.PlaybackAttemptID, SessionID: session.ID, UserID: userID, ProfileID: profileID, RequestedMediaFileID: requestedFile.ID, EffectiveMediaFileID: effectiveFile.ID, CurrentPlanID: result.Plan.PlanID, CurrentPlan: *result.Plan, FrozenRecipe: frozenRecipe, NormalizedRequest: req, ServerBitrateCapKbps: serverBitrateCapV3(r.Context()), StartResponse: response, RequestDigest: requestDigests.current, ExpiresAt: time.Now().Add(playback.MaxTokenTTL)} + // Capture the durable audio selection intent before the session state + // and attempt are committed: SelectionOrigin distinguishes an omitted + // start (server-resolved language preference, reconcilable) from an + // explicit viewer choice (never auto-overridden). The snapshot is taken + // from the planned selection, so later synchronous probe writes inside + // this start cannot erase the omitted-vs-explicit distinction. + recordIntentV3, intentErr := intentForCommittedStartV3(r.Context(), h, userID, profileID, req, effectiveFile, plannedAudioTrackIndexV3(result, audioIndex), virtualDecision) + if intentErr != nil { + transport.rollback() + abort() + return playback.DecisionResponseV3{}, &transportErrorV3{reason: "internal_error", message: "Failed to resolve the playback audio preference.", cause: intentErr} + } + record.SelectionOrigin = recordIntentV3.origin + record.PreferredAudioLanguage = recordIntentV3.preferredLang + record.SeriesAudioPreferenceSignature = recordIntentV3.seriesSignature + record.SelectedAudioSignature = recordIntentV3.selectedOverride + if err := h.SetAudioSelectionIntent(session.ID, record.SelectionOrigin, record.PreferredAudioLanguage, record.SeriesAudioPreferenceSignature, record.SelectedAudioSignature); err != nil { + slog.WarnContext(r.Context(), "protocol v3 start: audio selection intent not recorded on the session", + "component", "api", "session", session.ID, "error", err) + } if err := h.updateV3SessionState(r.Context(), session, effectiveFile, result, transport, mode); err != nil { transport.rollback() abort() @@ -4979,6 +5014,69 @@ func (h *PlaybackHandler) multipartResumeFileV3(ctx context.Context, file *model return nil, 0, nil } +// committedStartAudioIntentV3 is the durable audio selection intent for one +// committed start: the origin, the resolved preference language, the series +// snapshot and the signature of the committed track. +type committedStartAudioIntentV3 struct { + origin string + preferredLang string + seriesSignature *userstore.AudioTrackSignature + selectedOverride *userstore.AudioTrackSignature +} + +// virtualPlanSelectionOriginForRequestV3 names the audio selection origin +// for a start that never passed through the omitted-selection resolution +// above (decode-rejection rotations, version fallback alternates): explicit +// when the request names a track, auto otherwise. +func virtualPlanSelectionOriginForRequestV3(req playback.StartRequestV3) string { + if strings.TrimSpace(req.AudioTrackID) != "" || req.AudioTrackIndex != nil || strings.TrimSpace(req.CarriedAudioTrackID) != "" { + return SelectionOriginExplicit + } + return SelectionOriginAuto +} + +// intentForCommittedStartV3 snapshots the audio selection intent for the +// committed start. An omitted audio selection resolves through the same +// preference pipeline the catalog menu used (identical inputs: the committed +// audio index, the resolved language and the stored series preference), so +// reconciliation replays exactly the start's decision against the verified +// inventory. An explicit selection is marked explicit with only its committed +// signature: reconciliation may verify that it still exists, never move it. +func intentForCommittedStartV3(ctx context.Context, h *PlaybackHandler, userID int, profileID string, req playback.StartRequestV3, file *models.MediaFile, committedIndex int, decision virtualPlanDecisionV3) (committedStartAudioIntentV3, error) { + var intent committedStartAudioIntentV3 + if file != nil && committedIndex >= 0 && committedIndex < len(file.AudioTracks) { + intent.selectedOverride = playback.AudioTrackSignatureFromTrack(file.AudioTracks[committedIndex]) + } + if strings.TrimSpace(req.AudioTrackID) != "" || req.AudioTrackIndex != nil || strings.TrimSpace(req.CarriedAudioTrackID) != "" { + intent.origin = SelectionOriginExplicit + return intent, nil + } + if decision.startAudioSelectionOrigin != "" { + intent.origin = decision.startAudioSelectionOrigin + } else { + intent.origin = SelectionOriginAuto + } + intent.preferredLang = strings.TrimSpace(decision.preferredAudioLanguage) + if h == nil || h.StoreProvider == nil { + return intent, nil + } + store, err := h.StoreProvider.ForUser(ctx, userID) + if err != nil { + return committedStartAudioIntentV3{}, err + } + seriesID := h.resolveSeriesID(ctx, file) + if seriesID != "" { + stored, prefErr := store.GetAudioPreference(ctx, profileID, seriesID) + if prefErr != nil { + return committedStartAudioIntentV3{}, prefErr + } + if stored != nil { + intent.seriesSignature = stored.TrackSignature + } + } + return intent, nil +} + // preferredAudioTrackIndexV3 answers what an omitted audio track means: the // language this profile has settled on for this series, this library, this // device, or generally — the same resolution the catalog performs when it diff --git a/internal/api/handlers/playback_virtual.go b/internal/api/handlers/playback_virtual.go index ab7765b538..abbb756698 100644 --- a/internal/api/handlers/playback_virtual.go +++ b/internal/api/handlers/playback_virtual.go @@ -21,6 +21,7 @@ import ( "time" apimw "github.com/Silo-Server/silo-server/internal/api/middleware" + langpkg "github.com/Silo-Server/silo-server/internal/lang" "github.com/Silo-Server/silo-server/internal/logredact" "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/playback" @@ -6172,12 +6173,15 @@ func mergeVirtualCandidateLanguages(probed *models.MediaFile, candidate VirtualP channels := inferChannelsFromCodec(audioCodec) if len(candidate.AudioLanguages) > 0 { existing := make(map[string]bool, len(candidate.AudioLanguages)) - for _, lang := range candidate.AudioLanguages { - lang = strings.TrimSpace(lang) - if lang == "" || !isRealVirtualLanguageTag(lang) { + for _, candidateLang := range candidate.AudioLanguages { + candidateLang = strings.TrimSpace(candidateLang) + if candidateLang == "" || !isRealVirtualLanguageTag(candidateLang) { continue } - canonical := virtualLanguageBaseSubtag(lang) + canonical := langpkg.CanonicalTag(candidateLang) + if canonical == "" { + canonical = virtualLanguageBaseSubtag(candidateLang) + } if existing[canonical] { continue } @@ -6186,7 +6190,7 @@ func mergeVirtualCandidateLanguages(probed *models.MediaFile, candidate VirtualP // Synthesized tracks carry no real container stream index; the // array position is the ordinal (audioStreamOrdinalV3 falls back // to it when Index <= 0). - Language: lang, + Language: candidateLang, Codec: audioCodec, Channels: channels, }) diff --git a/internal/api/handlers/playback_virtual_evidence.go b/internal/api/handlers/playback_virtual_evidence.go index 56cda0ab35..0e5da482f0 100644 --- a/internal/api/handlers/playback_virtual_evidence.go +++ b/internal/api/handlers/playback_virtual_evidence.go @@ -524,6 +524,10 @@ func (h *PlaybackHandler) persistVirtualEvidenceDirect(ctx context.Context, args } if result.MetadataUpdated { h.logInventoryDelivery(h.PublishInventoryUpdated(writeCtx, args.FileID), 0) + // The row now carries the verified inventory: replay any + // cold-start default audio selection against the committed order + // so the executable recipe keeps the preferred language. + h.reconcileVerifiedDefaultAudio(context.Background(), args.FileID) } return result.MetadataUpdated } @@ -535,6 +539,7 @@ func (h *PlaybackHandler) persistVirtualEvidenceDirect(ctx context.Context, args } if rows > 0 { h.logInventoryDelivery(h.PublishInventoryUpdated(writeCtx, args.FileID), 0) + h.reconcileVerifiedDefaultAudio(context.Background(), args.FileID) } return rows > 0 } @@ -682,6 +687,10 @@ func (h *PlaybackHandler) persistVirtualEvidenceTask(task *virtualEvidenceTask, // it. The delivery log below is what makes a rotation's publish // observable; it does not publish a second time. h.logInventoryDelivery(h.PublishInventoryUpdated(context.Background(), task.args.FileID), task.originFileID) + // The verified inventory is committed: replay cold-start default + // audio intent against its order so the executable recipe keeps + // the preferred language instead of the declared ordinal. + h.reconcileVerifiedDefaultAudio(context.Background(), task.args.FileID) return nil } lastErr = err diff --git a/internal/playback/audio_select.go b/internal/playback/audio_select.go index 149711e7ad..313e1a29d4 100644 --- a/internal/playback/audio_select.go +++ b/internal/playback/audio_select.go @@ -44,16 +44,21 @@ func langMatch(a, b string) bool { // trackLanguageRank returns the best language rank for a track against the // preferred language, considering both the primary code and the MULTi // language list. -1 when nothing matches. +// +// Bare MULTI/DUAL membership sentinels carry no language information: they +// are skipped before matching so they never boost a track above real language +// evidence, but the track still falls through to the default/first-track +// selection below (neutral, not disqualifying). func trackLanguageRank(track models.AudioTrack, preferred string) int { best := -1 - if rank := langMatchRank(track.Language, preferred); rank >= 0 { + if rank := rankedLanguageMatch(track.Language, preferred); rank >= 0 { best = rank if best == 0 { return best } } for _, code := range track.Languages { - if rank := langMatchRank(code, preferred); rank >= 0 && (best < 0 || rank < best) { + if rank := rankedLanguageMatch(code, preferred); rank >= 0 && (best < 0 || rank < best) { best = rank if best == 0 { break @@ -63,14 +68,53 @@ func trackLanguageRank(track models.AudioTrack, preferred string) int { return best } +// rankedLanguageMatch ranks one language token against the preference, or -1 +// when there is no match. Tokens that only declare MULTI/DUAL/unknown +// membership are neutral: never a match (no -1 from the membership test +// unless every token is neutral), and never a positive rank that would push +// the track above real language evidence in bestLanguageTrack. +func rankedLanguageMatch(candidate, preferred string) int { + if unknownLanguageMembership(candidate) { + return -1 + } + return langMatchRank(candidate, preferred) +} + +// unknownLanguageMembership reports whether a language token declares only +// MULTI/DUAL/undetermined membership (or nothing at all) instead of a +// concrete language. Such tokens are release/intake vocabulary, not language +// evidence, and must stay neutral in language preference matching. +func unknownLanguageMembership(token string) bool { + switch lang.Canonical(strings.TrimSpace(token)) { + case "", "und", "mul": + return true + } + switch strings.ToLower(strings.TrimSpace(token)) { + case "multi", "multiple", "dual", "unknown", "undefined": + return true + } + return false +} + // trackHasLanguage reports whether the track carries the preferred language, // either as its primary code or anywhere in its MULTi language list, -// without an explicit script conflict. +// without an explicit script conflict. A bare MULTI/DUAL primary with no +// member list is not a match: unknown membership stays neutral. It is the +// shared membership authority for reconciliations and cross-version remaps; +// SelectAudioTrack keeps its own ranking rules on top of this predicate. func trackHasLanguage(track models.AudioTrack, preferred string) bool { rank := trackLanguageRank(track, preferred) return rank >= 0 && rank < lang.RankScriptConflict } +// TrackCarriesLanguage is the exported membership authority: it reports +// whether the track carries the preferred language via its primary code or +// its MULTi member list. Bare MULTI/DUAL tokens (unknown membership) never +// count as a concrete match but do not fail closed either. +func TrackCarriesLanguage(track models.AudioTrack, preferred string) bool { + return trackHasLanguage(track, preferred) +} + // langMatchRank delegates to lang.MatchRank to rank language closeness: // exact (0) > matching script/region (1-2) > bare tag (3) > // regional variant (4) > conflicting script (5). diff --git a/internal/playback/audio_select_test.go b/internal/playback/audio_select_test.go index 0ede75627a..c8a122c30a 100644 --- a/internal/playback/audio_select_test.go +++ b/internal/playback/audio_select_test.go @@ -1,6 +1,7 @@ package playback_test import ( + "math/rand" "testing" "github.com/Silo-Server/silo-server/internal/models" @@ -607,3 +608,65 @@ func TestIsOriginalLanguagePreference(t *testing.T) { } } } + +// TestSelectAudioTrack_BareMultiIsNeutral pins the cold-start stability +// rule: a bare MULTI/DUAL primary (unknown membership) must never win a +// concrete language preference over real language evidence, but it stays +// eligible as the neutral fallback when nothing matches. +func TestSelectAudioTrack_BareMultiIsNeutral(t *testing.T) { + for _, bare := range []string{"mul", "MULTi", "DUAL", "dual", "und"} { + tracks := []models.AudioTrack{ + {Language: bare, Codec: "eac3", Channels: 6}, + {Language: "eng", Codec: "aac", Channels: 2}, + } + if got := playback.SelectAudioTrack(tracks, "eng", nil); got != 1 { + t.Fatalf("bare %q stole the eng preference: selected %d, want 1", bare, got) + } + } + // Nothing matches: the bare track is still a legal fallback, not an error. + tracks := []models.AudioTrack{ + {Language: "mul", Codec: "eac3", Channels: 6}, + {Language: "jpn", Codec: "aac", Channels: 2}, + } + if got := playback.SelectAudioTrack(tracks, "deu", nil); got != 0 { + t.Fatalf("unmatched preference on bare MULTi: selected %d, want neutral first 0", got) + } + // A genuine member-list entry keeps winning through the primary skip. + membered := []models.AudioTrack{ + {Language: "mul", Languages: []string{"eng"}, Codec: "eac3", Channels: 6}, + {Language: "jpn", Codec: "aac", Channels: 2}, + } + if got := playback.SelectAudioTrack(membered, "eng", nil); got != 0 { + t.Fatalf("MULTi [eng] membership: selected %d, want 0", got) + } +} + +// TestSelectAudioTrack_RegionalPermutationFuzz replays ~200 reorderings of +// a track list with one unique best regional match (pt-BR) and requires the +// exact tag to win every permutation, so probe reorder can never move the +// preference by position. +func TestSelectAudioTrack_RegionalPermutationFuzz(t *testing.T) { + base := []models.AudioTrack{ + {Language: "pt", Codec: "aac", Channels: 2}, + {Language: "pt-PT", Codec: "aac", Channels: 2}, + {Language: "pt-BR", Codec: "aac", Channels: 2}, + {Language: "en", Codec: "aac", Channels: 2}, + {Language: "es", Codec: "aac", Channels: 2}, + } + rng := rand.New(rand.NewSource(7)) + for i := 0; i < 200; i++ { + order := rng.Perm(len(base)) + tracks := make([]models.AudioTrack, len(base)) + want := -1 + for pos, src := range order { + tracks[pos] = base[src] + tracks[pos].Index = pos + 1 + if base[src].Language == "pt-BR" { + want = pos + } + } + if got := playback.SelectAudioTrack(tracks, "pt-BR", nil); got != want { + t.Fatalf("permutation %d: selected %d (%q), want %d (pt-BR)", i, got, tracks[got].Language, want) + } + } +} diff --git a/internal/playback/protocol_store_v3.go b/internal/playback/protocol_store_v3.go index e7dd75ee9d..04cf37b3d3 100644 --- a/internal/playback/protocol_store_v3.go +++ b/internal/playback/protocol_store_v3.go @@ -9,6 +9,8 @@ import ( "time" "github.com/google/uuid" + + "github.com/Silo-Server/silo-server/internal/userstore" ) var ErrIdempotencyKeyReusedV3 = errors.New("idempotency key reused") @@ -134,6 +136,24 @@ type AttemptRecordV3 struct { // ServerBitrateCapKbps is fixed when the attempt starts; replans must not // pick up later administrator edits to the account or access group. ServerBitrateCapKbps int + // SelectionOrigin records whether the committed audio selection came + // from the server's language preference ("auto") or an explicit viewer + // choice ("explicit"). Legacy records predate the field and carry the + // empty string; reconciliation must then do nothing, never assume a + // default, since the pre-intent selection semantics are unknown. + SelectionOrigin string `json:"selection_origin,omitempty"` + // PreferredAudioLanguage is the canonical full tag that steered the + // server-preference selection (e.g. "pt-BR"), or "" when none was + // resolved. Compared against the verified inventory, never against an + // ordinal. + PreferredAudioLanguage string `json:"preferred_audio_language,omitempty"` + // SeriesAudioPreferenceSignature is the series-preference snapshot the + // start fed to SelectAudioTrack. Old records leave it nil. + SeriesAudioPreferenceSignature *userstore.AudioTrackSignature `json:"series_audio_preference_signature,omitempty"` + // SelectedAudioSignature is the signature of the committed selection at + // start time. It anchors the executable comparison at probe landing and + // detects a user track change that superseded the start selection. + SelectedAudioSignature *userstore.AudioTrackSignature `json:"selected_audio_signature,omitempty"` // StartResponse is the latest durable decision for this attempt. It begins // as the exact start response and advances atomically with each completed // replan so an idempotent start retry never resurrects a superseded plan. diff --git a/internal/playback/session.go b/internal/playback/session.go index cc2b5a54d8..d5073275db 100644 --- a/internal/playback/session.go +++ b/internal/playback/session.go @@ -18,6 +18,7 @@ import ( "github.com/Silo-Server/silo-server/internal/netaccess" "github.com/Silo-Server/silo-server/internal/streamlocation" "github.com/Silo-Server/silo-server/internal/tonemap" + "github.com/Silo-Server/silo-server/internal/userstore" ) // Transport output facts are independent of the whole-session play method. @@ -91,6 +92,21 @@ type Session struct { // (a legacy or reconstructed session) and falls back to the URI anchor. VirtualSubtitleEvidenceFileID int VirtualSubtitleEvidenceSet bool + // SelectionOrigin mirrors the attempt's audio selection intent onto the + // live session ("auto" when the server resolved an omitted selection, + // "explicit" when the viewer named a track). In-memory recovery and + // reconnects keep the intent alongside the track index, so a resumed + // session cannot silently downgrade to an ordinal-only comparison. + // Empty for sessions that started before intent capture. + SelectionOrigin string + // PreferredAudioLanguage and SeriesAudioPreferenceSignature replay the + // start-time inputs of the auto selection (SelectAudioTrack's + // preferredLang and series-snapshot arguments); SelectedAudioSignature + // names the committed track they produced. Reconciliation replays these + // against the verified inventory, never against a shifted ordinal. + PreferredAudioLanguage string + SeriesAudioPreferenceSignature *userstore.AudioTrackSignature + SelectedAudioSignature *userstore.AudioTrackSignature // RequireMediaAuthorization distinguishes v3 transports whose session ID is // only a route (media requests must present an authenticated user) from @@ -248,6 +264,18 @@ type SessionStreamState struct { // reconstructed) and falls back to the URI anchor. VirtualSubtitleEvidenceFileID int VirtualSubtitleEvidenceSet bool + // SelectionOrigin, PreferredAudioLanguage, + // SeriesAudioPreferenceSignature and SelectedAudioSignature mirror the + // attempt's audio selection intent (see Session) onto the stream state so + // ApplyReplacement survives a session-manager-level rebuild. + // SelectionOriginSet distinguishes an explicit origin ("", unset) from a + // state that never carried intent: an unset origin keeps the session's + // intent, so a legacy builder does not accidentally blank a newer intent. + SelectionOrigin string + SelectionOriginSet bool + PreferredAudioLanguage string + SeriesAudioPreferenceSignature *userstore.AudioTrackSignature + SelectedAudioSignature *userstore.AudioTrackSignature // Byte-affecting transcode recipe fields preserved so an offloaded restart // (e.g. audio switch) can rebuild the exact same stream. SubtitleTrackIndex @@ -1315,6 +1343,16 @@ func applySessionStreamStateLocked(s *Session, state SessionStreamState) { s.SubtitleTrackIndex = state.SubtitleTrackIndex s.SubtitleBurnIn = state.SubtitleBurnIn s.SegmentDuration = state.SegmentDuration + // Audio selection intent survives every stream-state round trip: it is + // written once at start and only a user's explicit track change rotates + // it, so reconciliation always compares the verified inventory against + // what the viewer actually committed to, not a transient replan default. + if state.SelectionOriginSet { + s.SelectionOrigin = state.SelectionOrigin + s.PreferredAudioLanguage = state.PreferredAudioLanguage + s.SeriesAudioPreferenceSignature = state.SeriesAudioPreferenceSignature + s.SelectedAudioSignature = state.SelectedAudioSignature + } if state.TranscodeRouteSet { // Only the replacement commit consumes the v3 capacity reservation; // unrelated legacy stream updates arriving mid-replan must not release @@ -1372,6 +1410,11 @@ func snapshotSessionStreamStateLocked(s *Session) SessionStreamState { VirtualSubtitleEvidenceURI: s.VirtualSubtitleEvidenceURI, VirtualSubtitleEvidenceFileID: s.VirtualSubtitleEvidenceFileID, VirtualSubtitleEvidenceSet: s.VirtualSubtitleEvidenceSet, + SelectionOrigin: s.SelectionOrigin, + SelectionOriginSet: true, + PreferredAudioLanguage: s.PreferredAudioLanguage, + SeriesAudioPreferenceSignature: s.SeriesAudioPreferenceSignature, + SelectedAudioSignature: s.SelectedAudioSignature, SubtitleTrackIndex: s.SubtitleTrackIndex, SubtitleBurnIn: s.SubtitleBurnIn, SegmentDuration: s.SegmentDuration, @@ -1438,6 +1481,10 @@ func restoreSessionStreamStateLocked(s *Session, state SessionStreamState) { s.SubtitleTrackIndex = state.SubtitleTrackIndex s.SubtitleBurnIn = state.SubtitleBurnIn s.SegmentDuration = state.SegmentDuration + s.SelectionOrigin = state.SelectionOrigin + s.PreferredAudioLanguage = state.PreferredAudioLanguage + s.SeriesAudioPreferenceSignature = state.SeriesAudioPreferenceSignature + s.SelectedAudioSignature = state.SelectedAudioSignature } // SetVirtualSource binds a live session to the provider-neutral candidate that @@ -1570,6 +1617,29 @@ func (m *SessionManager) SetAutoFallback(sessionID string, enabled bool) error { return nil } +// SetAudioSelectionIntent records the attempt's durable audio selection intent +// on the live session: whether the committed selection came from the server's +// language preference ("auto") or an explicit viewer choice ("explicit"), the +// resolved preferred language, the series-preference snapshot and the +// signature of the committed track. Probe-landing reconciliation replays +// these fields against the verified inventory; legacy sessions carry the zero +// value and are never reconciled. +func (m *SessionManager) SetAudioSelectionIntent(sessionID, origin, preferredLang string, seriesSig, selectedSig *userstore.AudioTrackSignature) error { + m.mu.Lock() + defer m.mu.Unlock() + + s, ok := m.sessions[sessionID] + if !ok { + return ErrSessionNotFound + } + s.SelectionOrigin = origin + s.PreferredAudioLanguage = preferredLang + s.SeriesAudioPreferenceSignature = seriesSig + s.SelectedAudioSignature = selectedSig + m.touchSessionLocked(s) + return nil +} + // RestoreAutoFallback returns a session's auto-fallback state to a value // captured by AutoFallback before a speculative renegotiation. It restores both // the boolean and the set-bit, so a session that had never negotiated the field From ce2309c845a9b90abfb59e92363a91953aae2c84 Mon Sep 17 00:00:00 2001 From: drondeseries Date: Mon, 5 Oct 2026 12:52:42 -0400 Subject: [PATCH 02/16] fix(playback): bound audio reconciliation by one deadline and strip forged provenance Establish the reconcile budget once at the entry point and propagate it instead of letting each interior caller discard cancellation and arm a fresh timeout. Clear client-supplied Automatic on the inbound replan boundary so only server-built reconciliation can claim that provenance. --- .../api/handlers/playback_reconcile_audio.go | 324 ++++++++++++++++-- .../handlers/playback_reconcile_audio_test.go | 105 +++++- internal/api/handlers/playback_v3.go | 28 +- internal/playback/planstore/postgres.go | 125 ++++++- internal/playback/protocol_store_v3.go | 148 ++++++++ internal/playback/protocol_v3.go | 13 +- ...051416_add_playback_v3_audio_selection.sql | 9 + 7 files changed, 691 insertions(+), 61 deletions(-) create mode 100644 migrations/sql/20261005051416_add_playback_v3_audio_selection.sql diff --git a/internal/api/handlers/playback_reconcile_audio.go b/internal/api/handlers/playback_reconcile_audio.go index 0e0db419d4..be6e2857df 100644 --- a/internal/api/handlers/playback_reconcile_audio.go +++ b/internal/api/handlers/playback_reconcile_audio.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "log/slog" "slices" @@ -54,6 +55,15 @@ const AudioReconciliationReplanReason = "default_audio_reconciliation" // the buffered worker), keeps an old code path recording, and never fires // inside PublishInventoryUpdated: this reconciles the executable recipe, not // the menu, independent of whether any session holds a realtime connection. +// +// Multi-node note: session enumeration here is local-only (the manager holds +// this replica's in-memory sessions), so a session owned by another replica +// is not reconciled by this call. Every replica persists probe evidence to +// the shared catalog row and replays the same record-first path: the owning +// replica reconciles on its own evidence commit, on session attach, and on +// the heartbeat re-check below. The attempt record is the cross-replica +// rendezvous — intent and the settled ledger live on it, not in process +// memory — so no cross-replica RPC is needed for replicas to converge. func (h *PlaybackHandler) reconcileVerifiedDefaultAudio(ctx context.Context, fileID int) { if h == nil || fileID <= 0 || h.sessionMgr == nil { return @@ -62,17 +72,22 @@ func (h *PlaybackHandler) reconcileVerifiedDefaultAudio(ctx context.Context, fil if !ok { return } + // One deadline bounds the whole per-file reconciliation sweep; every + // interior caller propagates this ctx instead of resetting it. + deadlineCtx, cancel := context.WithTimeout(ctx, reconcileProbeBudgetV3) + defer cancel() for _, session := range lookup.GetSessionsByMediaFileID(fileID) { if session == nil || session.ID == "" { continue } - h.reconcileSessionDefaultAudio(ctx, session, fileID) + h.reconcileSessionDefaultAudio(deadlineCtx, session, fileID) } } // reconcileSessionDefaultAudio reconciles one live session against the // verified catalog inventory of the file it is bound to. func (h *PlaybackHandler) reconcileSessionDefaultAudio(ctx context.Context, session *playback.Session, fileID int) { + // ctx carries the single entry deadline; interior callers propagate it. intent := h.audioSelectionIntentForSession(ctx, session) switch intent.origin { case SelectionOriginExplicit: @@ -134,6 +149,13 @@ func (h *PlaybackHandler) audioSelectionIntentForSession(ctx context.Context, se // reconcileAutoAudioSelection re-runs SelectAudioTrack against the verified // inventory and issues an automatic track_change replan when the executable // selection moved. Identical executable selection is a byte-equal no-op. +// +// Bounded by generation: every verified inventory is keyed by its probe +// generation (the row's probe stamp), and the attempt's settled ledger +// records one decision per generation. A generation with a settled entry is +// never re-evaluated — a retried probe write or a second replica replays the +// stored decision instead of minting a second replan — so heartbeats and +// duplicate evidence commits converge rather than issuing unbounded replans. func (h *PlaybackHandler) reconcileAutoAudioSelection(ctx context.Context, session *playback.Session, intent audioSelectionIntent, fileID int) { if h == nil || h.fileResolver == nil || h.PlanStoreV3 == nil { return @@ -146,13 +168,14 @@ func (h *PlaybackHandler) reconcileAutoAudioSelection(ctx context.Context, sessi if err != nil || verified == nil || len(verified.AudioTracks) == 0 { return } - if !virtualEvidenceMatchesBoundFile(verified, live) { - // Source rotation (or a sibling-row write) means this evidence is - // stale for the bound session: refuse, and surface the still-valid - // tracks in memory only, exactly like the refused-write path. - h.publishRefusedProbeInventory(ctx, fileID, strings.TrimSpace(live.VirtualSourceURI), verified) - slog.InfoContext(ctx, "default audio reconciliation refused: stale evidence for the bound candidate", - "component", "api", "session", session.ID, "file_id", fileID) + if !reconcileProbeVerifiedV3(verified) { + // Gate on verified probe provenance before any session work: a + // declared (never probed) row is exactly what the client already + // holds, so reconciling against it would only churn. + return + } + generation := reconcileGenerationV3(verified) + if h.audioReconcileSettled(ctx, live.ID, generation) { return } record, err := h.PlanStoreV3.GetAttempt(ctx, live.ID) @@ -161,6 +184,17 @@ func (h *PlaybackHandler) reconcileAutoAudioSelection(ctx context.Context, sessi } if strings.TrimSpace(record.SelectionOrigin) != "" && strings.TrimSpace(record.SelectionOrigin) != SelectionOriginAuto { // A racing explicit change persisted first: never override. + h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Generation: generation, Decision: playback.AudioReconcileRefused}) + return + } + if !virtualEvidenceMatchesBoundFile(verified, live) { + // Source rotation (or a sibling-row write) means this evidence is + // stale for the bound session: refuse, and surface the still-valid + // tracks in memory only, exactly like the refused-write path. + h.publishRefusedProbeInventory(ctx, fileID, strings.TrimSpace(live.VirtualSourceURI), verified) + slog.InfoContext(ctx, "default audio reconciliation refused: stale evidence for the bound candidate", + "component", "api", "session", session.ID, "file_id", fileID) + h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Generation: generation, Decision: playback.AudioReconcileRefused}) return } // The live session must still carry the committed start selection: a @@ -169,6 +203,7 @@ func (h *PlaybackHandler) reconcileAutoAudioSelection(ctx context.Context, sessi // newer choice. Compare executable signature identity, not the ordinal, // because the reorder is exactly what can shift ordinals. if !liveSelectionStillCommitted(live, record, intent) { + h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Generation: generation, Decision: playback.AudioReconcileRefused}) return } committedIndex := committedAudioTrackIndexV3(record, live) @@ -195,15 +230,136 @@ func (h *PlaybackHandler) reconcileAutoAudioSelection(ctx context.Context, sessi committed := live.VirtualAudioTracks[committedIndex] selected := verified.AudioTracks[recomputed] if audioSignatureMatchesCommittedTrack(selected, committed, intent, committedIndex, recomputed, live.VirtualAudioTracks, verified.AudioTracks) { - // Byte-equal no-op: the executable selection is identical. + // Byte-equal no-op: the executable selection is identical. Settle + // the generation so later heartbeats and duplicate evidence writes + // replay the decision instead of re-evaluating. + h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Generation: generation, Decision: playback.AudioReconcileNoop}) return } - if err := h.reconcileDefaultAudioReplan(ctx, live, record, verified, recomputed); err != nil { + req, body, err := h.reconcileDefaultAudioReplan(ctx, live, record, verified, recomputed) + if err != nil { slog.WarnContext(ctx, "default audio reconciliation replan failed", "component", "api", "session", live.ID, "file_id", fileID, "error", err) + return + } + audioIndex := recomputed + h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{ + Generation: generation, Decision: playback.AudioReconcileReplanned, + AudioIndex: &audioIndex, Request: &req, RequestDigest: ReplanDigestV3(body), + }) +} + +// reconcileProbeBudgetV3 bounds one reconcile evaluation: catalog read, +// session re-read, guard checks and the ledger write. The automatic replan +// itself runs under the replan path's own budgets once issued. +const reconcileProbeBudgetV3 = 10 * time.Second + +// reconcileGenerationV3 keys one verified inventory generation: the row's +// probe stamp when present, else the inventory revision of its audio tracks. +// A retried probe write for the same generation replays the settled +// decision; a genuinely new probe (new stamp) re-evaluates. +func reconcileGenerationV3(file *models.MediaFile) string { + if file == nil { + return "" + } + if file.ProbeUpdatedAt != nil && !file.ProbeUpdatedAt.IsZero() { + return "probe:" + file.ProbeUpdatedAt.UTC().Format(time.RFC3339Nano) + } + sum := sha256.Sum256([]byte(audioInventoryFingerprintV3(file))) + return "tracks:" + hex.EncodeToString(sum[:])[:16] +} + +// audioInventoryFingerprintV3 is the stable identity of an audio inventory +// for generation keying: absolute stream index, language, codec, channels +// and layout per track, in order. Two probes of the same bytes produce the +// same fingerprint even across process restarts, so a retried write replays +// the settled decision instead of re-evaluating. +func audioInventoryFingerprintV3(file *models.MediaFile) string { + if file == nil { + return "" + } + var sb strings.Builder + for _, track := range file.AudioTracks { + fmt.Fprintf(&sb, "%d|%s|%s|%d|%s;", track.Index, + strings.ToLower(strings.TrimSpace(track.Language)), + strings.ToLower(strings.TrimSpace(track.Codec)), + track.Channels, strings.ToLower(strings.TrimSpace(track.Layout))) + } + return sb.String() +} + +// reconcileProbeVerifiedV3 gates reconciliation on verified probe +// provenance: only a row whose probe stamp the catalog committed may move +// the executable selection. Declared-only rows carry no ground truth, so a +// reorder there is not evidence — it is absence of evidence. +func reconcileProbeVerifiedV3(file *models.MediaFile) bool { + return file != nil && file.ProbeUpdatedAt != nil && !file.ProbeUpdatedAt.IsZero() +} + +// audioReconcileSettled reports whether the generation already has a settled +// ledger entry. The ledger is read through the AudioReconcileStoreV3 +// capability when the store offers it; a store that predates the ledger +// (legacy wrapper) reports unset, and the reconcile path runs unledgered +// exactly as before. +func (h *PlaybackHandler) audioReconcileSettled(ctx context.Context, sessionID, generation string) bool { + if h == nil || h.PlanStoreV3 == nil || generation == "" { + return false + } + store, ok := h.PlanStoreV3.(playback.AudioReconcileStoreV3) + if !ok { + return false + } + ledger, _, err := store.GetAudioReconcileLedger(ctx, sessionID) + if err != nil { + return false + } + return playback.FindAudioReconcileEntry(ledger, generation) != nil +} + +// recordAudioReconcileDecision settles one generation in the durable ledger +// with bounded revision-conflict retry: concurrent writers converge on the +// union, and a replayed generation is a no-op returning the stored entry. +// A store without the ledger capability (legacy wrapper) keeps the +// reconcile path working unledgered rather than failing the probe write. +func (h *PlaybackHandler) recordAudioReconcileDecision(ctx context.Context, sessionID, generation string, entry playback.AudioReconcileEntryV3) { + if h == nil || h.PlanStoreV3 == nil || generation == "" { + return + } + store, ok := h.PlanStoreV3.(playback.AudioReconcileStoreV3) + if !ok { + return + } + ledger, revision, err := store.GetAudioReconcileLedger(ctx, sessionID) + if err != nil { + return + } + if playback.FindAudioReconcileEntry(ledger, generation) != nil { + return + } + for attempt := 0; attempt < reconcileLedgerMaxAttempts; attempt++ { + if _, _, err := store.RecordAudioReconciliation(ctx, sessionID, revision, entry); err == nil { + return + } else if !errors.Is(err, playback.ErrRecoveryRevisionConflictV3) { + return + } else { + current, currentRevision, readErr := store.GetAudioReconcileLedger(ctx, sessionID) + if readErr != nil { + return + } + if playback.FindAudioReconcileEntry(current, generation) != nil { + return + } + revision = currentRevision + } } } +// reconcileLedgerMaxAttempts bounds the revision-conflict retry when two +// replicas settle the same generation at once. The append is monotone and +// idempotent per generation, so a handful of retries converges; the bound +// keeps a pathological writer loop from spinning the probe path. +const reconcileLedgerMaxAttempts = 8 + // committedAudioTrackIndexV3 resolves the executable committed audio index: // the plan's selected index when present, else the session's committed index. func committedAudioTrackIndexV3(record *playback.AttemptRecordV3, session *playback.Session) int { @@ -234,7 +390,7 @@ func audioSignatureMatchesCommittedTrack(selected, committed models.AudioTrack, return false } } - return defaultAudioLanguageStillSatisfied(selected, intent) + return defaultAudioLanguageStillSatisfied(selected, verified, recomputed, intent) } // audioTransportFactsEqual is the transport reuse guard: the existing @@ -242,6 +398,17 @@ func audioSignatureMatchesCommittedTrack(selected, committed models.AudioTrack, // when the chosen stream decodes the same codec with the same channel // layout. When these differ the caller must not reuse the map; it issues a // fresh recipe via the automatic replan instead. +// +// Executor replacement is deliberately not decided here. A fixed ffmpeg +// audio map naming a moved stream needs worker replacement via a fresh +// recipe, and that call belongs to the replan path (sidecarOnlyReuseReplanV3 +// plus the executor check in executeReplanV3), which sees the full route — +// delivery, executor support for the operation, and transport-compat — +// rather than this predicate's codec/layout slice. The guard only answers +// "same bytes would flow"; the replan answers "same worker may serve them". +// when the chosen stream decodes the same codec with the same channel +// layout. When these differ the caller must not reuse the map; it issues a +// fresh recipe via the automatic replan instead. func audioTransportFactsEqual(a, b models.AudioTrack) bool { return strings.EqualFold(strings.TrimSpace(a.Codec), strings.TrimSpace(b.Codec)) && a.Channels == b.Channels && @@ -272,11 +439,21 @@ func audioSignatureFieldsEqual(a, b userstore.AudioTrackSignature) bool { } // defaultAudioLanguageStillSatisfied reports whether the recomputed track -// still carries the preferred language (or no preference was ever resolved). -// MULTi membership goes through the shared membership helper's authority: a -// bare MULTI/DUAL never counts as a concrete match, a member list entry does. -func defaultAudioLanguageStillSatisfied(track models.AudioTrack, intent audioSelectionIntent) bool { - return intent.preferredLang == "" || playback.TrackCarriesLanguage(track, intent.preferredLang) +// keeps the executable outcome the intent would choose: it must be the track +// SelectAudioTrack itself returns for the verified inventory. A bare +// MULTI/DUAL never counts as a concrete match, a member list entry does +// (see the shared membership helper). Preference-unavailable is not a +// separate failure mode here — the selector already fell back — so the +// comparison is pure executable equality, and a fallback that lands on the +// same stream is a no-op rather than a spurious replan. +func defaultAudioLanguageStillSatisfied(selected models.AudioTrack, verified []models.AudioTrack, recomputed int, intent audioSelectionIntent) bool { + if recomputed < 0 || recomputed >= len(verified) { + return false + } + if !audioTrackSignatureEqual(selected, *playback.AudioTrackSignatureFromTrack(verified[recomputed])) { + return false + } + return intent.preferredLang == "" || playback.TrackCarriesLanguage(selected, intent.preferredLang) } // reconcileDefaultAudioReplan issues the automatic track_change replan built @@ -285,7 +462,17 @@ func defaultAudioLanguageStillSatisfied(track models.AudioTrack, intent audioSel // not carry one). The replan-request id names the reconciliation reason so // the durable lease history attributes the change to the server, not the // viewer; no new operation or client-visible field is introduced. -func (h *PlaybackHandler) reconcileDefaultAudioReplan(ctx context.Context, session *playback.Session, record *playback.AttemptRecordV3, verified *models.MediaFile, audioIndex int) error { +// +// Delivery: the replan application commits the replacement plan, stream URL +// and transport server-side, then the same inventory_updated push every +// probe persistence already emits tells the player the revision changed. +// The web player folds that push through its existing inventory path +// (applyInventoryUpdate -> inventory poll/replan of its own), so a running +// client adopts the replacement on the channel it already watches; no new +// client surface is required. The settled ledger entry (recorded by the +// caller only after the replan commits) is what stops a duplicate probe +// write from minting a second replan — not the push. +func (h *PlaybackHandler) reconcileDefaultAudioReplan(ctx context.Context, session *playback.Session, record *playback.AttemptRecordV3, verified *models.MediaFile, audioIndex int) (playback.ReplanRequestV3, []byte, error) { verifiedIndex := audioIndex audioID := playback.TrackIDV3(verified.ID, "audio", verifiedIndex) caller := PlaybackCaller{ @@ -295,10 +482,11 @@ func (h *PlaybackHandler) reconcileDefaultAudioReplan(ctx context.Context, sessi req := playback.ReplanRequestV3{ ProtocolVersion: playback.ProtocolV3, Operation: playback.ReplanOperationTrackChangeV3, + Automatic: playback.ReplanAutomaticV3, PlaybackAttemptID: record.PlaybackAttemptID, - ReplanRequestID: reconcileReplanRequestID(session, verifiedIndex, "req"), + ReplanRequestID: reconcileReplanRequestID(record, verifiedIndex, "req"), FailedPlanID: record.CurrentPlanID, - PlanAttemptID: reconcileReplanRequestID(session, verifiedIndex, "plan"), + PlanAttemptID: reconcileReplanRequestID(record, verifiedIndex, "plan"), PlanAttemptKey: record.CurrentPlan.PlanAttemptKey, AttemptedPlanKeys: append([]string(nil), record.CurrentPlan.PlanAttemptKey), AttemptCount: 1, @@ -314,23 +502,31 @@ func (h *PlaybackHandler) reconcileDefaultAudioReplan(ctx context.Context, sessi } body, err := json.Marshal(req) if err != nil { - return err + return playback.ReplanRequestV3{}, nil, err } response, err := h.ReplanPlaybackV2(ctx, caller, session.ID, PlaybackReplanCommand{Request: req, Digest: ReplanDigestV3(body)}) if err != nil { - return err + return playback.ReplanRequestV3{}, nil, err } _ = response slog.InfoContext(ctx, "default audio reconciled to the verified inventory", "component", "api", "session", session.ID, "file_id", verified.ID, "audio_index", verifiedIndex, "reason", AudioReconciliationReplanReason) - return nil + return req, body, nil } // verifyExplicitAudioSelection checks that an explicit viewer selection still // exists in the verified inventory. It never overrides: a vanished track // surfaces a terminal diagnostic (route event + log) so the viewer learns the // selection is gone instead of hearing a silently substituted track. +// +// The event stays diagnostic-only: it names the attempt and plan for +// attribution but changes no attempt or plan state, so the stranded session +// keeps playing its committed route. The player learns of a genuine +// disappearance through the channel it already watches — the inventory_updated +// push (or the next inventory poll) shows the verified list without the +// track, and its picker drops the unplayable entry exactly as it does for any +// other inventory revision. No new client surface is required. func (h *PlaybackHandler) verifyExplicitAudioSelection(ctx context.Context, session *playback.Session, intent audioSelectionIntent, fileID int) { if h == nil || h.fileResolver == nil { return @@ -339,6 +535,9 @@ func (h *PlaybackHandler) verifyExplicitAudioSelection(ctx context.Context, sess if err != nil || verified == nil || len(verified.AudioTracks) == 0 { return } + if !reconcileProbeVerifiedV3(verified) { + return + } live, err := h.sessionMgr.GetSession(session.ID) if err != nil || live == nil { return @@ -350,16 +549,20 @@ func (h *PlaybackHandler) verifyExplicitAudioSelection(ctx context.Context, sess if h.PlanStoreV3 != nil { record, _ = h.PlanStoreV3.GetAttempt(ctx, live.ID) } - committedIndex := committedAudioTrackIndexV3(record, live) - track, ok := audioTrackAtCommittedIndex(verified.AudioTracks, committedIndex) - if ok && intent.selectedOverride != nil && !intent.selectedOverride.IsZero() { - ok = audioTrackSignatureEqual(track, *intent.selectedOverride) + generation := reconcileGenerationV3(verified) + if h.audioReconcileSettled(ctx, live.ID, generation) { + return } - if ok { + // Match track identity across the whole verified inventory, not just + // the old array index: a reorder moves the explicit track without + // removing it, and only a genuinely absent signature is missing. + if explicitAudioSelectionPresent(verified.AudioTracks, committedAudioTrackIndexV3(record, live), intent.selectedOverride) { + h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Generation: generation, Decision: playback.AudioReconcileNoop}) return } slog.WarnContext(ctx, "explicit audio selection missing from the verified inventory", - "component", "api", "session", live.ID, "file_id", fileID, "audio_index", committedIndex) + "component", "api", "session", live.ID, "file_id", fileID, "audio_index", committedAudioTrackIndexV3(record, live)) + h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Generation: generation, Decision: playback.AudioReconcileRefused}) if record != nil { h.enqueueRouteEventV3(playback.RouteEventRecordV3{ RouteEventV3: playback.RouteEventV3{ @@ -376,6 +579,34 @@ func (h *PlaybackHandler) verifyExplicitAudioSelection(ctx context.Context, sess } } +// explicitAudioSelectionPresent reports whether the committed explicit +// selection still exists anywhere in the verified inventory: the committed +// index first (the common unchanged case), then a full signature scan so a +// reordered-but-present track is not falsely reported missing. A nil or zero +// signature means start predates signatures: index presence alone decides. +func explicitAudioSelectionPresent(tracks []models.AudioTrack, committedIndex int, selected *userstore.AudioTrackSignature) bool { + if track, ok := audioTrackAtCommittedIndex(tracks, committedIndex); ok { + if selected == nil || selected.IsZero() || audioTrackSignatureEqual(track, *selected) { + return true + } + for _, candidate := range tracks { + if audioTrackSignatureEqual(candidate, *selected) { + return true + } + } + return false + } + if selected == nil || selected.IsZero() { + return false + } + for _, candidate := range tracks { + if audioTrackSignatureEqual(candidate, *selected) { + return true + } + } + return false +} + // audioTrackAtCommittedIndex returns the verified track at the committed // index, or false when the reorder dropped it out of range. func audioTrackAtCommittedIndex(tracks []models.AudioTrack, index int) (models.AudioTrack, bool) { @@ -393,32 +624,49 @@ var _ = ReplanDigestV3 // its probe already landed: probe landing with no registered session leaves // nothing to reconcile, so the attach (and first heartbeats, which reuse the // session lookup) replays the same per-session path once the attempt exists. +// +// Bounded like the probe-landing path: the verified-provenance gate and the +// settled-ledger check inside reconcileSessionDefaultAudio make a settled +// generation a no-op, so heartbeats converge instead of re-evaluating every +// tick. The store read below runs inside the reconcile budget (not outside +// it), and a record-fallback trigger covers rebuilds and remote sessions: +// the attempt record is the cross-replica rendezvous, so a session the +// owning replica never saw still converges when this replica attaches it +// and finds intent on the shared row. No cross-replica RPC is involved. func (h *PlaybackHandler) reconcilePendingAudioStartup(ctx context.Context, sessionID string) { if h == nil || h.sessionMgr == nil || h.PlanStoreV3 == nil || sessionID == "" { return } + deadlineCtx, cancel := context.WithTimeout(ctx, reconcileProbeBudgetV3) + defer cancel() session, err := h.sessionMgr.GetSession(sessionID) if err != nil || session == nil { return } - record, err := h.PlanStoreV3.GetAttempt(ctx, sessionID) + record, err := h.PlanStoreV3.GetAttempt(deadlineCtx, sessionID) if err != nil || record == nil { return } if strings.TrimSpace(record.SelectionOrigin) == "" { return } - deadline, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - defer cancel() - h.reconcileSessionDefaultAudio(deadline, session, record.EffectiveMediaFileID) + h.reconcileSessionDefaultAudio(deadlineCtx, session, record.EffectiveMediaFileID) } -// reconcileReplanRequestID mints a deterministic, human-readable replan -// identity for one automatic reconciliation: the reason prefix attributes -// the change, the session hash and target index keep it unique per -// (session, decision) without depending on the session id being a UUID. -func reconcileReplanRequestID(session *playback.Session, audioIndex int, kind string) string { - sum := sha256.Sum256([]byte(session.ID)) +// reconcileReplanRequestID mints a generation-aware identity for one +// automatic reconciliation decision: the reason prefix attributes the +// change, the failed-plan id binds it to the exact plan generation it was +// decided against, and the target index names the decision. The same +// decision therefore always replays under the same key with the same body +// (the replan lease answers with the stored response instead of colliding), +// while a new decision — a different index, or a later plan generation — +// mints a new key. It never depends on the session id being a UUID. +func reconcileReplanRequestID(record *playback.AttemptRecordV3, audioIndex int, kind string) string { + failedPlan := "" + if record != nil { + failedPlan = record.CurrentPlanID + } + sum := sha256.Sum256([]byte(failedPlan)) return fmt.Sprintf("%s-%s-%s-%d", AudioReconciliationReplanReason, kind, hex.EncodeToString(sum[:])[:8], audioIndex) } diff --git a/internal/api/handlers/playback_reconcile_audio_test.go b/internal/api/handlers/playback_reconcile_audio_test.go index 6e168a652f..16b6fd713e 100644 --- a/internal/api/handlers/playback_reconcile_audio_test.go +++ b/internal/api/handlers/playback_reconcile_audio_test.go @@ -12,12 +12,6 @@ import ( "github.com/Silo-Server/silo-server/internal/userstore" ) -// reconcileTestSessionManager is the minimal session surface the -// reconciliation path needs: live sessions plus lookup by file. -type reconcileTestSessionManager struct { - *playback.SessionManager -} - // verifiedFileResolver serves one fixed catalog row for reconcile tests. type verifiedFileResolver struct { file *models.MediaFile @@ -141,10 +135,6 @@ func TestReconcileVerifiedDefaultAudioReordersToPreferredLanguage(t *testing.T) func intPtrReconcile(v int) *int { return &v } -// sessionForReconcile is a UUID so the automatic replan seam (which -// requires canonical session ids) accepts the fixture session. -const sessionForReconcile = "11111111-1111-1111-1111-111111111111" - // MULTi membership: a track whose Languages list carries eng beats a bare // eng track only on rank, while a bare MULTi primary with no member list // never counts as a concrete language match. @@ -193,13 +183,18 @@ func TestReconcileRegionalVariantFidelity(t *testing.T) { }} selected := verified.AudioTracks[0] intent := audioSelectionIntent{origin: SelectionOriginAuto, preferredLang: "pt-BR"} - if !defaultAudioLanguageStillSatisfied(selected, intent) { + if !defaultAudioLanguageStillSatisfied(selected, verified.AudioTracks, 0, intent) { t.Fatal("reordered exact pt-BR track must still satisfy the pt-BR preference") } - if defaultAudioLanguageStillSatisfied(verified.AudioTracks[1], intent) == false { - // pt-PT is a regional variant, not an exact tag; it still satisfies - // the language under the rank chain but must not win over exact. - t.Fatal("pt-PT must rank under the pt-BR preference, not fail it outright") + // pt-PT is a regional variant, not the exact tag SelectAudioTrack + // returned for this inventory: the executable check must reject it as + // "not the selector's choice" even though the language rank alone + // would call it a match. That is the unavailable-language guard: a + // fallback that lands on the same stream is a no-op (see the + // unavailable-language test below), but a different stream is never + // papered over by language proximity. + if defaultAudioLanguageStillSatisfied(verified.AudioTracks[1], verified.AudioTracks, 0, intent) { + t.Fatal("pt-PT at index 1 is not the selector's index-0 choice and must fail the executable check") } } @@ -404,6 +399,86 @@ func TestReconcileSourceRotationRefusesStaleEvidence(t *testing.T) { // landed before the session attached), the attach-side hook must find no // record and return without failing; legacy records without intent must // likewise do nothing. +// Blocker 4 guard: Automatic is server-owned provenance. A client that forges +// it in the wire body must be treated as a user-initiated track_change, so +// preference persistence still happens. +func TestReconcileClientSuppliedAutomaticIsStripped(t *testing.T) { + body, err := json.Marshal(playback.ReplanRequestV3{ + ProtocolVersion: playback.ProtocolV3, + Operation: playback.ReplanOperationTrackChangeV3, + Automatic: playback.ReplanAutomaticV3, + ReplanRequestID: "client-forged-1", + PlaybackAttemptID: "attempt-reconcile-1", + }) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + + var decoded playback.ReplanRequestV3 + if err := json.Unmarshal(body, &decoded); err != nil { + t.Fatalf("unmarshal request: %v", err) + } + if decoded.Automatic != playback.ReplanAutomaticV3 { + t.Fatalf("fixture must carry the forged marker on the wire, got %q", decoded.Automatic) + } + + // The inbound application seam is what the HTTP handler calls; it owns + // stripping untrusted provenance before validation or persistence. + stripClientSuppliedAutomatic(&decoded) + if decoded.Automatic != "" { + t.Fatalf("client-supplied Automatic survived: %q", decoded.Automatic) + } + + // A server-built reconciliation request keeps its marker: that path never + // goes through the inbound seam. + serverBuilt := playback.ReplanRequestV3{ + Operation: playback.ReplanOperationTrackChangeV3, + Automatic: playback.ReplanAutomaticV3, + } + if serverBuilt.Automatic != playback.ReplanAutomaticV3 { + t.Fatal("server-built reconciliation lost its Automatic marker") + } +} + +// Blocker 3 guard: one entry deadline, propagated, never reset. A caller +// whose context is already done must not buy a fresh 10s budget. +func TestReconcilePropagatesCallerDeadlineWithoutReset(t *testing.T) { + if testing.Short() { + t.Skip("deadline propagation assertion needs a stalled dependency") + } + verified := &models.MediaFile{ID: 7, ProbeUpdatedAt: ptr(time.Now())} + verified.AudioTracks = []models.AudioTrack{ + {Language: "por", Codec: "eac3"}, + {Language: "eng", Codec: "eac3"}, + } + session := &playback.Session{ + ID: "11111111-1111-1111-1111-111111111111", + UserID: 1, + ProfileID: "profile-1", + Position: 42, + SelectionOrigin: SelectionOriginAuto, + VirtualAudioTracks: verified.AudioTracks, + } + record := reconcileIntent("eng", nil, nil) + handler := reconcileFixture(t, session, record, verified) + + done := make(chan struct{}) + go func() { + defer close(done) + // An already-cancelled caller context: reconciliation must return on + // it rather than arming its own timeout. + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + handler.reconcilePendingAudioStartup(cancelled, session.ID) + }() + + select { + case <-done: + case <-time.After(reconcileProbeBudgetV3 + 5*time.Second): + t.Fatal("reconciliation ignored the caller deadline and blocked past the budget") + } +} + func TestReconcileProbeBeforeRegistrationDefersToAttach(t *testing.T) { manager := playback.NewSessionManager(0, 0) session := &playback.Session{ diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index 5ed276ff53..854d543ee2 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -6608,6 +6608,19 @@ func downloadedSubtitleLabelV3(value subtitles.DownloadedSubtitle) string { return value.ReleaseName + " (" + value.Provider + ")" } +// stripClientSuppliedAutomatic clears the server-owned `Automatic` provenance +// marker on a decoded client replan request. Reconciliation builds its own +// replan in-process with the marker already set; that path never passes +// through this function. A client that forges the marker would otherwise +// impersonate server reconciliation and suppress its own preference +// persistence, so the inbound boundary drops whatever arrived on the wire. +func stripClientSuppliedAutomatic(req *playback.ReplanRequestV3) { + if req == nil { + return + } + req.Automatic = "" +} + // HandleReplanPlaybackV3 provides persistent idempotency and preserves the old // transport until a successor has entered its startup state and the new plan is // durably committed. @@ -6645,6 +6658,11 @@ func (h *PlaybackHandler) replanPlaybackApplicationV3(r *http.Request, sessionID if err := json.Unmarshal(body, &req); err != nil { return playback.DecisionResponseV3{}, playbackOperationError(http.StatusBadRequest, "bad_request", "Invalid replan request") } + // Automatic marks a server-built reconciliation replan. It is internal + // provenance: a client that forges it would suppress preference + // persistence and impersonate reconciliation, so strip whatever the + // client sent and let the server set it only on its own path. + stripClientSuppliedAutomatic(&req) // Reject malformed identity/bounds before doing any session lookup. When // client_features is omitted, temporarily allow the only validation rule // that depends on the durable start request; the authoritative merge and a @@ -8840,10 +8858,16 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte cancelReservation() afterCtx, afterCancel := context.WithTimeout(context.WithoutCancel(r.Context()), 5*time.Second) defer afterCancel() - if trackChange { + if trackChange && req.Automatic != playback.ReplanAutomaticV3 { // A deliberate track switch is the same signal the legacy audio // PATCH recorded; a failure recovery is not, so its forced audio - // route must not be written back as a user preference. + // route must not be written back as a user preference. An + // automatic reconciliation is not a viewer choice either: it + // replays the start-time language preference, so persisting its + // outcome would launder a server correction into a stored user + // preference and steer every later start. The Automatic marker + // (server-side only, see ReplanRequestV3) suppresses that write + // for reconciliation ops; explicit user track changes persist. h.persistCurrentAudioPreferenceV3(afterCtx, session.ID, session.UserID, session.ProfileID, effectiveFile, plannedAudioTrackIndexV3(result, session.AudioTrackIndex)) } h.syncSessionsNow(afterCtx, "v3_replan") diff --git a/internal/playback/planstore/postgres.go b/internal/playback/planstore/postgres.go index 56924f4452..2ee5c51c0f 100644 --- a/internal/playback/planstore/postgres.go +++ b/internal/playback/planstore/postgres.go @@ -88,6 +88,19 @@ func (s *Postgres) SaveAttempt(ctx context.Context, record playback.AttemptRecor if err != nil { return err } + selectionJSON, err := json.Marshal(playback.AudioSelectionV3{ + Origin: record.SelectionOrigin, + PreferredLanguage: record.PreferredAudioLanguage, + SeriesSignature: record.SeriesAudioPreferenceSignature, + SelectedSignature: record.SelectedAudioSignature, + }) + if err != nil { + return err + } + ledgerJSON, err := json.Marshal(record.AudioReconcileLedger) + if err != nil { + return err + } tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) if err != nil { return err @@ -115,13 +128,15 @@ func (s *Postgres) SaveAttempt(ctx context.Context, record playback.AttemptRecor playback_attempt_id, session_id, user_id, profile_id, requested_media_file_id, effective_media_file_id, current_plan_id, current_replan_request_id, current_plan, frozen_recipe, - normalized_request, start_response, request_digest, expires_at, server_bitrate_cap_kbps - ) VALUES ($1, NULLIF($2, '')::uuid, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) + normalized_request, start_response, request_digest, expires_at, server_bitrate_cap_kbps, + audio_selection, audio_reconcile_ledger + ) VALUES ($1, NULLIF($2, '')::uuid, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) ON CONFLICT DO NOTHING`, record.PlaybackAttemptID, record.SessionID, record.UserID, record.ProfileID, record.RequestedMediaFileID, record.EffectiveMediaFileID, record.CurrentPlanID, record.CurrentReplanRequestID, planJSON, recipeJSON, - requestJSON, responseJSON, record.RequestDigest, record.ExpiresAt, record.ServerBitrateCapKbps) + requestJSON, responseJSON, record.RequestDigest, record.ExpiresAt, record.ServerBitrateCapKbps, + selectionJSON, ledgerJSON) if err != nil { return err } @@ -177,13 +192,14 @@ func (s *Postgres) getAttemptIdentity(ctx context.Context, predicate string, val func (s *Postgres) getAttempt(ctx context.Context, predicate string, value any) (*playback.AttemptRecordV3, error) { var record playback.AttemptRecordV3 - var planJSON, recipeJSON, requestJSON, responseJSON, sampleJSON, recoveryJSON []byte + var planJSON, recipeJSON, requestJSON, responseJSON, sampleJSON, recoveryJSON, selectionJSON, ledgerJSON []byte err := s.db.QueryRow(ctx, ` SELECT playback_attempt_id, COALESCE(session_id::text, ''), user_id, profile_id, requested_media_file_id, effective_media_file_id, current_plan_id, current_replan_request_id, current_plan, frozen_recipe, normalized_request, start_response, request_digest, expires_at, server_bitrate_cap_kbps, - last_sequence, last_sample, stopped_at, updated_at, recovery_state, recovery_revision + last_sequence, last_sample, stopped_at, updated_at, recovery_state, recovery_revision, + audio_selection, audio_reconcile_ledger FROM playback_v3_attempts WHERE `+predicate+` AND expires_at > NOW()`, value).Scan( &record.PlaybackAttemptID, &record.SessionID, &record.UserID, &record.ProfileID, @@ -192,6 +208,7 @@ func (s *Postgres) getAttempt(ctx context.Context, predicate string, value any) &requestJSON, &responseJSON, &record.RequestDigest, &record.ExpiresAt, &record.ServerBitrateCapKbps, &record.LastSequence, &sampleJSON, &record.StoppedAt, &record.LastSampleAt, &recoveryJSON, &record.RecoveryRevision, + &selectionJSON, &ledgerJSON, ) if errors.Is(err, pgx.ErrNoRows) { return nil, playback.ErrSessionNotFound @@ -219,6 +236,24 @@ func (s *Postgres) getAttempt(ctx context.Context, predicate string, value any) return nil, err } } + // audio_selection and audio_reconcile_ledger postdate the intent work: + // rows written before the migration carry '{}' and decode to the zero + // value, which reconciliation treats as "no intent / no decision". + if len(selectionJSON) > 0 { + var selection playback.AudioSelectionV3 + if err := json.Unmarshal(selectionJSON, &selection); err != nil { + return nil, err + } + record.SelectionOrigin = selection.Origin + record.PreferredAudioLanguage = selection.PreferredLanguage + record.SeriesAudioPreferenceSignature = selection.SeriesSignature + record.SelectedAudioSignature = selection.SelectedSignature + } + if len(ledgerJSON) > 0 { + if err := json.Unmarshal(ledgerJSON, &record.AudioReconcileLedger); err != nil { + return nil, err + } + } return &record, nil } @@ -274,6 +309,86 @@ func (s *Postgres) AppendRecoveryExclusions(ctx context.Context, sessionID strin return merged, next, nil } +// RecordAudioReconciliation appends the settled reconciliation decision to +// the attempt's durable ledger under a row lock. The transaction serializes +// concurrent records on the attempt row, and the base-revision compare makes +// a writer that read a stale revision lose with +// ErrRecoveryRevisionConflictV3 instead of clobbering a newer decision. +// Recording the same generation twice is a no-op returning the stored +// ledger: retries and second replicas converge instead of minting a second +// replan. +func (s *Postgres) RecordAudioReconciliation(ctx context.Context, sessionID string, baseRevision int64, entry playback.AudioReconcileEntryV3) (playback.AudioReconcileLedgerV3, int64, error) { + tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return playback.AudioReconcileLedgerV3{}, 0, err + } + defer func() { _ = tx.Rollback(ctx) }() + var ledgerJSON []byte + var revision int64 + err = tx.QueryRow(ctx, ` + SELECT audio_reconcile_ledger, (audio_reconcile_ledger->>'revision')::bigint FROM playback_v3_attempts + WHERE session_id = $1::uuid AND expires_at > NOW() + FOR UPDATE`, sessionID).Scan(&ledgerJSON, &revision) + if errors.Is(err, pgx.ErrNoRows) { + return playback.AudioReconcileLedgerV3{}, 0, playback.ErrSessionNotFound + } + if err != nil { + return playback.AudioReconcileLedgerV3{}, 0, err + } + if baseRevision >= 0 && revision != baseRevision { + return playback.AudioReconcileLedgerV3{}, revision, playback.ErrRecoveryRevisionConflictV3 + } + var base playback.AudioReconcileLedgerV3 + if len(ledgerJSON) > 0 { + if err := json.Unmarshal(ledgerJSON, &base); err != nil { + return playback.AudioReconcileLedgerV3{}, 0, err + } + } + base.Revision = revision + merged := playback.AppendAudioReconcileEntry(base, entry) + merged.Revision = revision + 1 + mergedJSON, err := json.Marshal(merged) + if err != nil { + return playback.AudioReconcileLedgerV3{}, 0, err + } + var next int64 + if err := tx.QueryRow(ctx, ` + UPDATE playback_v3_attempts + SET audio_reconcile_ledger = $2, updated_at = NOW() + WHERE session_id = $1::uuid AND expires_at > NOW() + RETURNING (audio_reconcile_ledger->>'revision')::bigint`, sessionID, mergedJSON).Scan(&next); err != nil { + return playback.AudioReconcileLedgerV3{}, 0, err + } + if err := tx.Commit(ctx); err != nil { + return playback.AudioReconcileLedgerV3{}, 0, err + } + return merged, next, nil +} + +// GetAudioReconcileLedger reads the durable reconciliation ledger and its +// revision. +func (s *Postgres) GetAudioReconcileLedger(ctx context.Context, sessionID string) (playback.AudioReconcileLedgerV3, int64, error) { + var ledgerJSON []byte + var revision int64 + err := s.db.QueryRow(ctx, ` + SELECT audio_reconcile_ledger, (audio_reconcile_ledger->>'revision')::bigint FROM playback_v3_attempts + WHERE session_id = $1::uuid AND expires_at > NOW()`, sessionID).Scan(&ledgerJSON, &revision) + if errors.Is(err, pgx.ErrNoRows) { + return playback.AudioReconcileLedgerV3{}, 0, playback.ErrSessionNotFound + } + if err != nil { + return playback.AudioReconcileLedgerV3{}, 0, err + } + var ledger playback.AudioReconcileLedgerV3 + if len(ledgerJSON) > 0 { + if err := json.Unmarshal(ledgerJSON, &ledger); err != nil { + return playback.AudioReconcileLedgerV3{}, 0, err + } + } + ledger.Revision = revision + return ledger, revision, nil +} + // GetRecoveryState reads the durable exclusion chain and its revision. func (s *Postgres) GetRecoveryState(ctx context.Context, sessionID string) (playback.RecoveryStateV3, int64, error) { var stateJSON []byte diff --git a/internal/playback/protocol_store_v3.go b/internal/playback/protocol_store_v3.go index 04cf37b3d3..8ac8642d91 100644 --- a/internal/playback/protocol_store_v3.go +++ b/internal/playback/protocol_store_v3.go @@ -154,6 +154,15 @@ type AttemptRecordV3 struct { // start time. It anchors the executable comparison at probe landing and // detects a user track change that superseded the start selection. SelectedAudioSignature *userstore.AudioTrackSignature `json:"selected_audio_signature,omitempty"` + // AudioReconcileLedger is the settled/pending reconciliation ledger for + // this attempt: one entry per verified generation the reconcile path has + // decided on. Like RecoveryState it is written only by a dedicated + // revision-checked writer (RecordAudioReconciliation), never by + // SaveAttempt or CompleteReplan, so a stale attempt write can never + // shrink it. Entries persist in the audio_reconcile_ledger column; the + // memory store keeps them on the record itself. A fresh attempt starts + // with the zero value and inherits nothing. + AudioReconcileLedger AudioReconcileLedgerV3 // StartResponse is the latest durable decision for this attempt. It begins // as the exact start response and advances atomically with each completed // replan so an idempotent start retry never resurrects a superseded plan. @@ -199,6 +208,68 @@ type RecoveryStateV3 struct { Exclusions []RecoveryExclusionV3 `json:"exclusions,omitempty"` } +// AudioSelectionV3 is the JSONB shape of the audio_selection attempt +// column: the durable start-time audio selection intent. AttemptRecordV3 +// carries the same fields inline for the memory store; Postgres splits them +// into their own column so CompleteReplan (which rewrites only plan +// columns) can never clobber them. +type AudioSelectionV3 struct { + Origin string `json:"origin,omitempty"` + PreferredLanguage string `json:"preferred_language,omitempty"` + SeriesSignature *userstore.AudioTrackSignature `json:"series_signature,omitempty"` + SelectedSignature *userstore.AudioTrackSignature `json:"selected_signature,omitempty"` +} + +// AudioReconcileLedgerV3 is the durable, attempt-scoped memory of automatic +// default-audio reconciliation decisions. One entry per verified generation +// the reconcile path has settled (a committed replan, a byte-equal no-op, or +// a refusal), keyed by the generation so a retried probe write or a second +// replica replays the same decision instead of minting a second replan. +type AudioReconcileLedgerV3 struct { + // Entries is the settled chain, in the order decisions were recorded. It + // is scoped by generation: entries for different generations never + // suppress each other, and a new probe generation always re-evaluates. + Entries []AudioReconcileEntryV3 `json:"entries,omitempty"` + // Revision advances with every durable ledger write. It is the + // compare-and-set token for concurrent decision records: a writer that + // read a stale revision loses with ErrRecoveryRevisionConflictV3 instead + // of clobbering a newer decision. + Revision int64 `json:"revision,omitempty"` +} + +// AudioReconcileDecisionV3 names the settled outcome recorded for one +// verified generation. +type AudioReconcileDecisionV3 string + +const ( + // AudioReconcileReplanned means an automatic track_change replan for the + // generation committed (or replayed idempotently) through the replan + // lease. The entry's RequestDigest pins the exact replan body: a retry of + // the same decision reuses the same bytes, so the lease answers with the + // stored response instead of colliding on the request id. + AudioReconcileReplanned AudioReconcileDecisionV3 = "replanned" + // AudioReconcileNoop means the generation compared byte-equal against + // the committed selection: no replan, no session mutation, no push. + AudioReconcileNoop AudioReconcileDecisionV3 = "noop" + // AudioReconcileRefused means the generation was stale (rotation), + // explicit (never overridden), or superseded by a viewer change: the + // reconcile path declined and must not retry the generation. + AudioReconcileRefused AudioReconcileDecisionV3 = "refused" +) + +// AudioReconcileEntryV3 is one settled reconciliation decision. Generation +// names the verified inventory it was decided against; Request holds the +// exact automatic replan body for a replanned decision (nil otherwise) so a +// retry replays the same bytes; RequestDigest fingerprints those bytes and +// is the idempotency key the decision is retried under. +type AudioReconcileEntryV3 struct { + Generation string `json:"generation"` + Decision AudioReconcileDecisionV3 `json:"decision"` + AudioIndex *int `json:"audio_index,omitempty"` + Request *ReplanRequestV3 `json:"request,omitempty"` + RequestDigest string `json:"request_digest,omitempty"` +} + // RecoveryExclusionV3 is one confirmed candidate failure. The tuple is scoped // so only the release the verdict actually examined is excluded: // @@ -312,6 +383,50 @@ type RecoveryStateStoreV3 interface { GetRecoveryState(ctx context.Context, sessionID string) (RecoveryStateV3, int64, error) } +// AudioReconcileStoreV3 is the durable attempt-scoped reconciliation ledger. +// It is a separate capability so the handler can detect a store that cannot +// persist it (a legacy wrapper) and run the reconcile path without a ledger +// rather than silently dropping a recorded decision. An implementation must +// append, never replace: a generation is recorded once, and concurrent +// writers serialize on the ledger revision so neither loses a decision. +type AudioReconcileStoreV3 interface { + // RecordAudioReconciliation appends the settled decision to the attempt's + // durable ledger and returns the resulting ledger. baseRevision is the + // revision the caller read alongside its current ledger; a mismatched + // write returns ErrRecoveryRevisionConflictV3 with the committed revision + // instead of clobbering, and the caller re-reads and retries. A negative + // baseRevision appends unconditionally. Recording the same generation + // twice is a no-op returning the stored entry. It returns + // ErrSessionNotFound when there is no live attempt row. + RecordAudioReconciliation(ctx context.Context, sessionID string, baseRevision int64, entry AudioReconcileEntryV3) (AudioReconcileLedgerV3, int64, error) + // GetAudioReconcileLedger reads the durable ledger and its revision. + GetAudioReconcileLedger(ctx context.Context, sessionID string) (AudioReconcileLedgerV3, int64, error) +} + +// FindAudioReconcileEntry returns the settled entry for generation, or nil +// when the generation has no recorded decision yet. +func FindAudioReconcileEntry(ledger AudioReconcileLedgerV3, generation string) *AudioReconcileEntryV3 { + for i := range ledger.Entries { + if ledger.Entries[i].Generation == generation { + entry := ledger.Entries[i] + return &entry + } + } + return nil +} + +// AppendAudioReconcileEntry returns base with entry appended, or base +// unchanged when the generation is already recorded. The first writer wins; +// a replayed record is a no-op, so concurrent commits converge. +func AppendAudioReconcileEntry(base AudioReconcileLedgerV3, entry AudioReconcileEntryV3) AudioReconcileLedgerV3 { + if FindAudioReconcileEntry(base, entry.Generation) != nil { + return base + } + merged := base + merged.Entries = append(append([]AudioReconcileEntryV3(nil), base.Entries...), entry) + return merged +} + // UnionRecoveryExclusions appends the incoming exclusions to a copy of base, // deduplicating on provider source + candidate id. The result preserves the // order of base first, then the newly added entries in input order, so @@ -663,6 +778,39 @@ func (s *MemoryPlanStoreV3) GetRecoveryState(_ context.Context, sessionID string return record.RecoveryState, record.RecoveryRevision, nil } +// RecordAudioReconciliation appends the settled decision to the attempt's +// durable ledger. It is a compare-and-set on the ledger revision: a caller +// presenting a stale revision loses and retries against the committed +// ledger, so no writer can drop another's decision. Recording the same +// generation twice returns the stored entry without growing the chain. +func (s *MemoryPlanStoreV3) RecordAudioReconciliation(_ context.Context, sessionID string, baseRevision int64, entry AudioReconcileEntryV3) (AudioReconcileLedgerV3, int64, error) { + s.mu.Lock() + defer s.mu.Unlock() + attemptID, record := s.findAttemptLocked(sessionID) + if record == nil { + return AudioReconcileLedgerV3{}, 0, ErrSessionNotFound + } + if baseRevision >= 0 && record.AudioReconcileLedger.Revision != baseRevision { + return AudioReconcileLedgerV3{}, record.AudioReconcileLedger.Revision, ErrRecoveryRevisionConflictV3 + } + merged := AppendAudioReconcileEntry(record.AudioReconcileLedger, entry) + merged.Revision = record.AudioReconcileLedger.Revision + 1 + record.AudioReconcileLedger = merged + s.attempts[attemptID] = *record + return merged, merged.Revision, nil +} + +// GetAudioReconcileLedger reads the durable ledger and its revision. +func (s *MemoryPlanStoreV3) GetAudioReconcileLedger(_ context.Context, sessionID string) (AudioReconcileLedgerV3, int64, error) { + s.mu.Lock() + defer s.mu.Unlock() + _, record := s.findAttemptLocked(sessionID) + if record == nil { + return AudioReconcileLedgerV3{}, 0, ErrSessionNotFound + } + return record.AudioReconcileLedger, record.AudioReconcileLedger.Revision, nil +} + // findAttemptLocked returns the live attempt for a session; the caller holds // s.mu. func (s *MemoryPlanStoreV3) findAttemptLocked(sessionID string) (string, *AttemptRecordV3) { diff --git a/internal/playback/protocol_v3.go b/internal/playback/protocol_v3.go index 3b2064d75e..8d758fbc66 100644 --- a/internal/playback/protocol_v3.go +++ b/internal/playback/protocol_v3.go @@ -729,7 +729,18 @@ type ReplanRequestV3 struct { Failure FailureV3 `json:"failure,omitzero"` Capabilities ClientCodecCapabilitiesV3 `json:"client_capabilities"` ClientPlaybackContext ClientPlaybackContextV3 `json:"client_playback_context"` -} + // Automatic marks a replan request the server built itself (no client + // gesture behind it). It is server-side only: omitempty keeps it off the + // client wire in practice, Validate ignores it, and clients never send + // it. The replan application uses it to suppress user-preference + // persistence for automatic corrections; explicit user track changes + // keep persisting. Empty means a client-issued request. + Automatic string `json:"automatic,omitempty"` +} + +// ReplanAutomaticV3 is the Automatic marker for server-built reconciliation +// replans (see ReplanRequestV3.Automatic). +const ReplanAutomaticV3 = "automatic" const ( RouteEventPlanSelectedV3 = "plan_selected" diff --git a/migrations/sql/20261005051416_add_playback_v3_audio_selection.sql b/migrations/sql/20261005051416_add_playback_v3_audio_selection.sql new file mode 100644 index 0000000000..01e188157d --- /dev/null +++ b/migrations/sql/20261005051416_add_playback_v3_audio_selection.sql @@ -0,0 +1,9 @@ +-- +goose Up +ALTER TABLE playback_v3_attempts + ADD COLUMN audio_selection JSONB NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN audio_reconcile_ledger JSONB NOT NULL DEFAULT '{}'::jsonb; + +-- +goose Down +ALTER TABLE playback_v3_attempts + DROP COLUMN audio_reconcile_ledger, + DROP COLUMN audio_selection; From 56683b5cfacbbb3c71d183b558cddab523c2fb72 Mon Sep 17 00:00:00 2001 From: drondeseries Date: Mon, 5 Oct 2026 13:48:48 -0400 Subject: [PATCH 03/16] fix(playback): deliver audio reconciliation through client replan adoption Reconciliation no longer self-commits a replacement plan the client never adopts. It persists the corrected decision atomically with the canonical request, withdraws the active plan with plan_invalidated, and lets the client's own replan commit the corrected recipe. --- docs/architecture/playback-protocol-v3.md | 66 +++ .../api/handlers/playback_reconcile_audio.go | 393 ++++++++++++-- ...yback_reconcile_audio_invalidation_test.go | 482 ++++++++++++++++++ .../handlers/playback_reconcile_audio_test.go | 19 +- internal/api/handlers/playback_v3.go | 5 + internal/playback/planstore/postgres.go | 15 +- internal/playback/protocol_store_v3.go | 87 +++- internal/playback/realtime.go | 47 +- 8 files changed, 1031 insertions(+), 83 deletions(-) create mode 100644 internal/api/handlers/playback_reconcile_audio_invalidation_test.go diff --git a/docs/architecture/playback-protocol-v3.md b/docs/architecture/playback-protocol-v3.md index 8411a1b2aa..6ee9b155df 100644 --- a/docs/architecture/playback-protocol-v3.md +++ b/docs/architecture/playback-protocol-v3.md @@ -1102,6 +1102,10 @@ any other, not a fire-and-forget event: "reason": "video_copy_unsafe", "deadline_ms": 8000, "payload": {"reason": "video_copy_unsafe", "plan_id": ""} + + } ``` @@ -1210,6 +1214,68 @@ were scanned: a file rewritten in place while the scan read it produces a verdict about bytes nobody is serving, which is neither persisted nor pushed at any session. +#### 6.1.1 `default_audio_reconciliation` — a late probe withdraws a cold-start plan + +The second reason on this command has a different cause. Cold-start default-audio +reconciliation compares the audio inventory a plan was built against against the +verified one the probe lands afterwards. When the verified order moves the +server-resolved selection to a different stream, the committed recipe no longer +plays the preferred language — and it is still the plan on screen. + +The same command withdraws it, and for the same reason: the server learns the +route is wrong after the plan was handed out. What differs is the decision +around it. + +**The server never commits the replacement for this reason.** It persists the +corrected selection and a canonical `track_change` replan body in one +compare-and-set on the attempt's reconciliation ledger, then pushes the +withdrawal and stops. The client's own replan is what commits the new recipe, +because the client's replan is what carries its live position and pause state. +A server-side commit plus a client replan would race for the same transport, so +there is exactly one commit path and it is the client's. + +That split makes the ordering load-bearing: + +- The decision and the request it belongs to are written **before** anything is + emitted. A duplicate probe write, a second replica, or a retried heartbeat + finds the generation settled and replays the stored decision instead of + emitting a second event for one correction. +- The replan-request id is derived from the plan and the target audio index, + both immutable, while the body also embeds the live position, which is not. + A replay of the stored decision therefore carries the same id **and** the same + digest — so the replan lease answers with its stored response rather than + rejecting a reused id — while two evaluations that disagree about position + are caught as a digest mismatch against the settled entry and refused, with + the rejection recorded next to the stored digest. +- A refused divergence supersedes the invalidation it disputes: a correction a + rejected re-evaluation has already invalidated never reaches the client. + +The client's replacement replan cannot echo a track it never chose, so it does +not name one. The replan that replans off the withdrawn plan therefore consumes +the stored decision and fills in the audio identity. The correction is one-shot +rather than an ongoing override: it is keyed to the withdrawn plan, and the +plan the replan commits becomes current, so the client's next replan is its own. + +Four rules bound it: + +- **The feature still gates delivery.** A client that never advertised + `plan_invalidated_v1` gets no event. Its decision is still recorded, and the + correction lands on its next start or reconnect — which plans against the + verified inventory anyway, because the inventory is what moved. +- **A byte-equal no-op still settles the generation.** Heartbeats and duplicate + evidence writes converge on the stored decision instead of re-evaluating. +- **Every automatic-selection gate still runs before the decision**: a stale or + rotated source, an explicit viewer selection, or a viewer track change between + start and probe all refuse rather than withdraw. +- **Delivery is in-process**, on the session's own hub lane, exactly as for + `video_copy_unsafe`. The ledger, not a cross-replica RPC, is what keeps the + replica that persisted the same evidence from emitting a duplicate. + +The payload carries the generation alongside the existing `plan_id` and +`reason`, which is additive: a client that ignores it behaves exactly as before, +and one that reads it can tell an in-flight duplicate from a correction that +landed afterwards. + ### 6.2 `source_committed_event_v1` — the committed version is published at commit A serve-layer rotation can move a live session to a sibling release without ever diff --git a/internal/api/handlers/playback_reconcile_audio.go b/internal/api/handlers/playback_reconcile_audio.go index be6e2857df..7d5a4baac8 100644 --- a/internal/api/handlers/playback_reconcile_audio.go +++ b/internal/api/handlers/playback_reconcile_audio.go @@ -12,6 +12,8 @@ import ( "strings" "time" + "github.com/google/uuid" + "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/userstore" @@ -49,6 +51,12 @@ const ( // replan-request id and the plan log, both additive and client-invisible. const AudioReconciliationReplanReason = "default_audio_reconciliation" +// audioReconcileInvalidationDeadline is the deadline the withdrawal command +// carries. The client acknowledges the command and answers it with the result +// of its own replacement replan, which plans from scratch, so the deadline +// bounds that replan rather than a transport change on this side. +const audioReconcileInvalidationDeadline = 8 * time.Second + // reconcileVerifiedDefaultAudio replays persisted audio selection intent // against the committed verified inventory for fileID. It is invoked after // probe persistence commits (both persistVirtualEvidenceDirect branches and @@ -147,15 +155,23 @@ func (h *PlaybackHandler) audioSelectionIntentForSession(ctx context.Context, se } // reconcileAutoAudioSelection re-runs SelectAudioTrack against the verified -// inventory and issues an automatic track_change replan when the executable -// selection moved. Identical executable selection is a byte-equal no-op. +// inventory and, when the executable selection moved, persists the corrected +// decision and withdraws the active plan so the client replans onto it. +// Identical executable selection is a byte-equal no-op. +// +// The server never commits the replacement itself. Reconciliation corrects the +// durable inventory and records one canonical request; the plan replacement is +// the client's own replan, which captures its live position and pause state and +// lands on the corrected audio index. Two competing commit paths (a +// server-committed recipe plus a client replan) would race for the same +// transport, so there is exactly one: this one records and asks. // // Bounded by generation: every verified inventory is keyed by its probe -// generation (the row's probe stamp), and the attempt's settled ledger -// records one decision per generation. A generation with a settled entry is -// never re-evaluated — a retried probe write or a second replica replays the -// stored decision instead of minting a second replan — so heartbeats and -// duplicate evidence commits converge rather than issuing unbounded replans. +// generation (the row's probe stamp), and the attempt's settled ledger records +// one decision per (generation, session). A pair with a settled entry is never +// re-evaluated — a retried probe write or a second replica replays the stored +// decision instead of minting a second event — so heartbeats and duplicate +// evidence commits converge rather than emitting unbounded invalidations. func (h *PlaybackHandler) reconcileAutoAudioSelection(ctx context.Context, session *playback.Session, intent audioSelectionIntent, fileID int) { if h == nil || h.fileResolver == nil || h.PlanStoreV3 == nil { return @@ -175,16 +191,23 @@ func (h *PlaybackHandler) reconcileAutoAudioSelection(ctx context.Context, sessi return } generation := reconcileGenerationV3(verified) - if h.audioReconcileSettled(ctx, live.ID, generation) { - return - } record, err := h.PlanStoreV3.GetAttempt(ctx, live.ID) if err != nil || record == nil { return } + if entry := playback.FindAudioReconcileEntry(record.AudioReconcileLedger, generation, live.ID); entry != nil { + // The attempt row already carries the ledger, so this needs no second + // store read. A settled generation is never re-evaluated and never + // re-announced: a retried probe write, a second replica or a retried + // heartbeat replays the stored decision instead of emitting a second + // event for one correction. A delivery that failed had no client to + // receive it, and that client's next start or reconnect plans against + // the corrected inventory anyway. + return + } if strings.TrimSpace(record.SelectionOrigin) != "" && strings.TrimSpace(record.SelectionOrigin) != SelectionOriginAuto { // A racing explicit change persisted first: never override. - h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Generation: generation, Decision: playback.AudioReconcileRefused}) + h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Generation: generation, SessionID: live.ID, Decision: playback.AudioReconcileRefused}) return } if !virtualEvidenceMatchesBoundFile(verified, live) { @@ -194,7 +217,7 @@ func (h *PlaybackHandler) reconcileAutoAudioSelection(ctx context.Context, sessi h.publishRefusedProbeInventory(ctx, fileID, strings.TrimSpace(live.VirtualSourceURI), verified) slog.InfoContext(ctx, "default audio reconciliation refused: stale evidence for the bound candidate", "component", "api", "session", session.ID, "file_id", fileID) - h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Generation: generation, Decision: playback.AudioReconcileRefused}) + h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Generation: generation, SessionID: live.ID, Decision: playback.AudioReconcileRefused}) return } // The live session must still carry the committed start selection: a @@ -203,7 +226,7 @@ func (h *PlaybackHandler) reconcileAutoAudioSelection(ctx context.Context, sessi // newer choice. Compare executable signature identity, not the ordinal, // because the reorder is exactly what can shift ordinals. if !liveSelectionStillCommitted(live, record, intent) { - h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Generation: generation, Decision: playback.AudioReconcileRefused}) + h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Generation: generation, SessionID: live.ID, Decision: playback.AudioReconcileRefused}) return } committedIndex := committedAudioTrackIndexV3(record, live) @@ -216,6 +239,7 @@ func (h *PlaybackHandler) reconcileAutoAudioSelection(ctx context.Context, sessi ID: live.ID, UserID: live.UserID, ProfileID: live.ProfileID, + Position: live.Position, AudioTrackIndex: committedIndex, VirtualAudioTracks: verified.AudioTracks, } @@ -233,20 +257,54 @@ func (h *PlaybackHandler) reconcileAutoAudioSelection(ctx context.Context, sessi // Byte-equal no-op: the executable selection is identical. Settle // the generation so later heartbeats and duplicate evidence writes // replay the decision instead of re-evaluating. - h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Generation: generation, Decision: playback.AudioReconcileNoop}) + h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Generation: generation, SessionID: live.ID, Decision: playback.AudioReconcileNoop}) return } - req, body, err := h.reconcileDefaultAudioReplan(ctx, live, record, verified, recomputed) + h.settleAudioReconcileInvalidation(ctx, live, record, verified, recomputed) +} + +// reconcilePendingAudioStartup re-checks one session when it attaches after +// this replica owns a realtime connection for a client that negotiated the +// replacement-plan handshake, withdraws the active plan so the client replans. +// +// Ordering is the point: the canonical request and the decision it belongs to +// are written in one ledger CAS before anything is emitted. A duplicate probe +// write, a second replica or a retried heartbeat therefore finds the generation +// settled and replays the stored decision instead of emitting a second event, +// and an evaluation interrupted after the write replays the exact stored bytes +// — the same request id and the same digest — so the replan lease answers with +// the stored response rather than refusing the replay as a reused id. +// +// A client that never negotiated the handshake gets no event: the decision is +// still recorded, and the correction lands on its next start or reconnect, +// which plans against the verified inventory anyway. +func (h *PlaybackHandler) settleAudioReconcileInvalidation(ctx context.Context, live *playback.Session, record *playback.AttemptRecordV3, verified *models.MediaFile, audioIndex int) { + generation := reconcileGenerationV3(verified) + req, err := h.buildReconcileAudioRequest(live, record, verified, audioIndex) if err != nil { - slog.WarnContext(ctx, "default audio reconciliation replan failed", - "component", "api", "session", live.ID, "file_id", fileID, "error", err) + slog.WarnContext(ctx, "default audio reconciliation request build failed", + "component", "api", "session", live.ID, "file_id", verified.ID, "error", err) return } - audioIndex := recomputed - h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{ - Generation: generation, Decision: playback.AudioReconcileReplanned, - AudioIndex: &audioIndex, Request: &req, RequestDigest: ReplanDigestV3(body), + digest := ReplanDigestV3(req.bytes) + audioIndexCopy := audioIndex + storedRequest := req.request + stored, settledByThisWriter := h.commitAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{ + Decision: playback.AudioReconcileInvalidated, + AudioIndex: &audioIndexCopy, + PlanID: record.CurrentPlanID, + Request: &storedRequest, + RequestDigest: digest, }) + // Only the writer that actually settled this (generation, session) emits. + // A replayed generation returns the stored decision, already delivered (or + // being delivered by the writer that stored it), and a diverged evaluation + // is refused outright — neither may hand a client a second invalidation for + // one correction. + if !settledByThisWriter || stored.RequestDigest != digest || stored.Decision != playback.AudioReconcileInvalidated { + return + } + h.announceAudioReconcileInvalidation(ctx, live, record, stored) } // reconcileProbeBudgetV3 bounds one reconcile evaluation: catalog read, @@ -313,7 +371,7 @@ func (h *PlaybackHandler) audioReconcileSettled(ctx context.Context, sessionID, if err != nil { return false } - return playback.FindAudioReconcileEntry(ledger, generation) != nil + return playback.FindAudioReconcileEntry(ledger, generation, sessionID) != nil } // recordAudioReconcileDecision settles one generation in the durable ledger @@ -322,36 +380,102 @@ func (h *PlaybackHandler) audioReconcileSettled(ctx context.Context, sessionID, // A store without the ledger capability (legacy wrapper) keeps the // reconcile path working unledgered rather than failing the probe write. func (h *PlaybackHandler) recordAudioReconcileDecision(ctx context.Context, sessionID, generation string, entry playback.AudioReconcileEntryV3) { + if entry.SessionID == "" { + entry.SessionID = sessionID + } + _, _ = h.commitAudioReconcileDecision(ctx, sessionID, generation, entry) +} + +// commitAudioReconcileDecision settles one generation in the durable ledger +// with bounded revision-conflict retry and reports what the ledger holds +// afterwards, together with whether THIS caller is the writer that settled it. +// +// That distinction is the dedup authority for the invalidation push: the loser +// of a CAS against an already-settled (generation, session) pair replays the +// stored decision without emitting, while the winner emits exactly once. It is +// also where an evaluation that built a different canonical request for a +// settled generation is refused and written down, rather than silently +// competing with the decision a client is already replanning against. +// +// A store without the ledger capability (legacy wrapper) keeps the reconcile +// path working unledgered rather than failing the probe write. +func (h *PlaybackHandler) commitAudioReconcileDecision(ctx context.Context, sessionID, generation string, entry playback.AudioReconcileEntryV3) (playback.AudioReconcileEntryV3, bool) { if h == nil || h.PlanStoreV3 == nil || generation == "" { - return + return playback.AudioReconcileEntryV3{}, false } store, ok := h.PlanStoreV3.(playback.AudioReconcileStoreV3) if !ok { - return + return playback.AudioReconcileEntryV3{}, false } + entry.Generation = generation + entry.SessionID = sessionID ledger, revision, err := store.GetAudioReconcileLedger(ctx, sessionID) if err != nil { - return - } - if playback.FindAudioReconcileEntry(ledger, generation) != nil { - return + return playback.AudioReconcileEntryV3{}, false } for attempt := 0; attempt < reconcileLedgerMaxAttempts; attempt++ { - if _, _, err := store.RecordAudioReconciliation(ctx, sessionID, revision, entry); err == nil { - return - } else if !errors.Is(err, playback.ErrRecoveryRevisionConflictV3) { - return - } else { - current, currentRevision, readErr := store.GetAudioReconcileLedger(ctx, sessionID) - if readErr != nil { - return + if existing := playback.FindAudioReconcileEntry(ledger, generation, sessionID); existing != nil { + if entry.RequestDigest != "" && existing.RequestDigest != entry.RequestDigest { + // Two evaluations of one generation disagree about the + // canonical request. The first recorded decision stands: + // the client may already be replanning against it. Record the + // refusal so the divergence is durable and auditable, and + // emit nothing. + h.recordAudioReconcileDivergence(ctx, store, sessionID, ledger, revision, generation, sessionID, entry) + slog.WarnContext(ctx, "default audio reconciliation refused: canonical request diverged for a settled generation", + "component", "api", "session", sessionID, "generation", generation, + "stored_digest", existing.RequestDigest, "rejected_digest", entry.RequestDigest) + } + return *existing, false + } + merged, _, writeErr := store.RecordAudioReconciliation(ctx, sessionID, revision, entry) + switch { + case writeErr == nil: + if stored := playback.FindAudioReconcileEntry(merged, generation, sessionID); stored != nil { + return *stored, true } - if playback.FindAudioReconcileEntry(current, generation) != nil { - return + return entry, true + case errors.Is(writeErr, playback.ErrRecoveryRevisionConflictV3): + ledger, revision, err = store.GetAudioReconcileLedger(ctx, sessionID) + if err != nil { + return playback.AudioReconcileEntryV3{}, false } - revision = currentRevision + default: + slog.WarnContext(ctx, "audio reconciliation decision write failed", + "component", "api", "session", sessionID, "generation", generation, "error", writeErr) + return playback.AudioReconcileEntryV3{}, false } } + slog.WarnContext(ctx, "audio reconciliation decision gave up on the ledger revision", + "component", "api", "session", sessionID, "generation", generation) + return playback.AudioReconcileEntryV3{}, false +} + +// recordAudioReconcileDivergence appends the refusal a diverged evaluation +// earns against an already-settled generation. It is deliberately not +// retried: it runs on the path that lost, and a lost CAS there means another +// evaluator recorded the same refusal. +func (h *PlaybackHandler) recordAudioReconcileDivergence(ctx context.Context, store playback.AudioReconcileStoreV3, sessionID string, ledger playback.AudioReconcileLedgerV3, revision int64, generation, settledSession string, entry playback.AudioReconcileEntryV3) { + refusal := playback.AudioReconcileEntryV3{ + Generation: generation, + SessionID: settledSession, + Decision: playback.AudioReconcileRefused, + PlanID: entry.PlanID, + RequestDigest: entry.RequestDigest, + } + // A refusal is a SECOND entry for a pair that already has one: it records + // the rejected digest next to the stored one, so dedup by (generation, + // session) is not what suppresses it. Its own idempotency key is the + // rejected digest — a retried evaluation of the same diverged body records + // nothing further. + for _, existing := range ledger.Entries { + if existing.Generation == refusal.Generation && + existing.Decision == playback.AudioReconcileRefused && + existing.RequestDigest == refusal.RequestDigest { + return + } + } + _, _, _ = store.RecordAudioReconciliation(ctx, sessionID, revision, refusal) } // reconcileLedgerMaxAttempts bounds the revision-conflict retry when two @@ -472,13 +596,38 @@ func defaultAudioLanguageStillSatisfied(selected models.AudioTrack, verified []m // client surface is required. The settled ledger entry (recorded by the // caller only after the replan commits) is what stops a duplicate probe // write from minting a second replan — not the push. -func (h *PlaybackHandler) reconcileDefaultAudioReplan(ctx context.Context, session *playback.Session, record *playback.AttemptRecordV3, verified *models.MediaFile, audioIndex int) (playback.ReplanRequestV3, []byte, error) { +// reconcileAudioRequest is the canonical replacement request a settled +// decision is replayed under: the request value plus the exact bytes its digest +// is taken from, so a retry reuses verbatim what was stored. +type reconcileAudioRequest struct { + request playback.ReplanRequestV3 + bytes []byte +} + +// buildReconcileAudioRequest builds the automatic track_change replan from the +// committed recipe, remapped onto the verified order, preserving position, +// with no route exclusion and no failure classification (track_change must not +// carry one). The replan-request id names the reconciliation reason so the +// durable lease history attributes the change to the server, not the viewer; +// no new operation or client-visible field is introduced. +// +// The body is built here and persisted with the decision instead of being +// executed. Two properties follow from freezing it here rather than at the +// replan: +// +// - The stored bytes are the identity. The request id is derived from the +// plan and target index, both immutable, while the body also carries the +// live position, which is not. A replay of the stored decision therefore +// carries the same id AND the same digest, so the replan lease answers +// with its stored response instead of rejecting a reused id — while two +// evaluations that disagree about position are caught as a digest +// mismatch against the settled entry and refused. +// - Nothing is executed before the decision is durable, so an interrupted +// evaluation cannot leave a committed recipe with no record of the request +// that produced it. +func (h *PlaybackHandler) buildReconcileAudioRequest(session *playback.Session, record *playback.AttemptRecordV3, verified *models.MediaFile, audioIndex int) (reconcileAudioRequest, error) { verifiedIndex := audioIndex audioID := playback.TrackIDV3(verified.ID, "audio", verifiedIndex) - caller := PlaybackCaller{ - UserID: session.UserID, - ProfileID: session.ProfileID, - } req := playback.ReplanRequestV3{ ProtocolVersion: playback.ProtocolV3, Operation: playback.ReplanOperationTrackChangeV3, @@ -502,17 +651,155 @@ func (h *PlaybackHandler) reconcileDefaultAudioReplan(ctx context.Context, sessi } body, err := json.Marshal(req) if err != nil { - return playback.ReplanRequestV3{}, nil, err + return reconcileAudioRequest{}, err } - response, err := h.ReplanPlaybackV2(ctx, caller, session.ID, PlaybackReplanCommand{Request: req, Digest: ReplanDigestV3(body)}) + if err := req.Validate(); err != nil { + return reconcileAudioRequest{}, fmt.Errorf("reconcile replan request invalid: %w", err) + } + return reconcileAudioRequest{request: req, bytes: body}, nil +} + +// sessionNegotiatedPlanInvalidation reports whether this attempt's client +// advertised plan_invalidated_v1 — the promise to handle a mid-session plan +// withdrawal and replan off it. +// +// The attempt row is the authority rather than the live session: the feature is +// attempt-sticky, negotiated once at start, and the withdrawal may outlive the +// in-memory session (a rebuilt one from the card). A legacy row without the +// field never negotiated it, and an unnegotiated client must not be pushed a +// command it never promised to handle — its correction lands on the next start +// or reconnect instead. +func (h *PlaybackHandler) sessionNegotiatedPlanInvalidation(ctx context.Context, sessionID string) bool { + if h == nil || h.PlanStoreV3 == nil { + return false + } + record, err := h.PlanStoreV3.GetAttempt(ctx, sessionID) + if err != nil || record == nil { + return false + } + return playback.HasFeatureV3(record.NormalizedRequest.ClientFeatures, playback.FeaturePlanInvalidatedV3) +} + +// pendingAudioReconciliationReplan consumes the same pending decision the +// withdrawal named: the client's replan, whatever body it carries, replans off +// the withdrawn plan, so it has to select the corrected audio stream or the +// correction never reaches the transport. Applying the stored selection is +// additive on the wire — it fills the audio identity the client omitted — and +// leaves every other field the client sent (position, pause state, attempt +// key, failure classification) exactly as it arrived, which is what preserves +// the viewer's position across the adoption. +// +// Only a client-issued replan off the withdrawn plan consumes it. The server's +// own automatic replan already names the corrected selection, and a replan off +// any other plan is a viewer decision that must not be overridden. +func (h *PlaybackHandler) pendingAudioReconciliationReplan(record *playback.AttemptRecordV3, req *playback.ReplanRequestV3) { + if record == nil || req == nil || req.Automatic != "" { + return + } + entry := FindPendingAudioReconciliation(record, req.FailedPlanID) + if entry == nil { + return + } + req.SelectedTracks.Audio = entry.Request.SelectedTracks.Audio + slog.Debug("replan consumes the pending default audio correction", + "component", "api", "session", record.SessionID, + "generation", entry.Generation, "audio_index", entry.AudioIndex) +} + +// FindPendingAudioReconciliation returns the settled automatic correction a +// replan replanning off planID would consume, or nil when there is none. +// +// The entry is NOT consumed here: it is keyed by the plan the withdrawal +// named, and the plan this replan commits becomes current, so the client's next +// replan — off the new plan — finds nothing and plans with whatever selection +// it was built with. That is what makes the correction one-shot rather than a +// permanent override of viewer choice. +func FindPendingAudioReconciliation(record *playback.AttemptRecordV3, planID string) *playback.AudioReconcileEntryV3 { + if record == nil || planID == "" { + return nil + } + var found *playback.AudioReconcileEntryV3 + for i := range record.AudioReconcileLedger.Entries { + entry := record.AudioReconcileLedger.Entries[i] + // A refusal recorded for the same (generation, session) pair is the + // divergence verdict: it supersedes the invalidation that evaluator + // lost to, so reading the invalidation instead would let a correction + // a rejected re-evaluation had already invalidated reach the client. + if entry.Decision == playback.AudioReconcileRefused && found != nil && entry.Generation == found.Generation { + found = nil + continue + } + if entry.Decision != playback.AudioReconcileInvalidated || entry.PlanID != planID { + continue + } + if entry.Request == nil || entry.Request.SelectedTracks.Audio == nil { + continue + } + found = &entry + } + return found +} + +// announceAudioReconcileInvalidation withdraws the plan the decision was +// recorded against, so the client replans onto the corrected audio index. +// +// It reuses the existing plan_invalidated command and the session's own hub +// lane — the same per-session delivery every other realtime push uses. A lane +// is registered by the replica serving that session's control socket, so this +// reaches the owner without any cross-replica RPC; the same evidence commit +// on another replica is deduplicated by the ledger and emits nothing. +// +// A session whose attempt never negotiated plan_invalidated_v1 gets no event: +// the hub refuses the push before the envelope is built. Its decision stays +// recorded, and the corrected selection lands on its next start or reconnect, +// which plans against the verified inventory anyway. +func (h *PlaybackHandler) announceAudioReconcileInvalidation(ctx context.Context, live *playback.Session, record *playback.AttemptRecordV3, stored playback.AudioReconcileEntryV3) { + if h == nil || h.RealtimeHub == nil { + return + } + if stored.AudioIndex == nil { + return + } + planID := stored.PlanID + if planID == "" { + planID = record.CurrentPlanID + } + if planID == "" { + return + } + if !h.sessionNegotiatedPlanInvalidation(ctx, live.ID) { + slog.DebugContext(ctx, "default audio correction recorded without a withdrawal", + "component", "api", "session", live.ID, "generation", stored.Generation, + "audio_index", *stored.AudioIndex) + return + } + command, err := playback.NewPlanInvalidatedCommandForGeneration( + live.ID, + uuid.NewString(), + planID, + playback.PlanInvalidatedDefaultAudioReconciliation, + stored.Generation, + ) if err != nil { - return playback.ReplanRequestV3{}, nil, err + slog.WarnContext(ctx, "default audio withdrawal command could not be built", + "component", "api", "session", live.ID, "error", err) + return + } + // The documented replacement handshake requires a deadline: the client + // answers the command with the result of its own replacement replan, which + // plans from scratch. + if command.DeadlineMS == 0 { + command.DeadlineMS = int(audioReconcileInvalidationDeadline / time.Millisecond) + } + if err := h.RealtimeHub.Send(live.ID, command); err != nil { + slog.DebugContext(ctx, "default audio withdrawal push undelivered", + "component", "api", "session", live.ID, "plan_id", planID, "error", err) + return } - _ = response slog.InfoContext(ctx, "default audio reconciled to the verified inventory", - "component", "api", "session", session.ID, "file_id", verified.ID, - "audio_index", verifiedIndex, "reason", AudioReconciliationReplanReason) - return req, body, nil + "component", "api", "session", live.ID, "plan_id", planID, + "generation", stored.Generation, "audio_index", *stored.AudioIndex, + "reason", AudioReconciliationReplanReason) } // verifyExplicitAudioSelection checks that an explicit viewer selection still @@ -557,12 +844,12 @@ func (h *PlaybackHandler) verifyExplicitAudioSelection(ctx context.Context, sess // the old array index: a reorder moves the explicit track without // removing it, and only a genuinely absent signature is missing. if explicitAudioSelectionPresent(verified.AudioTracks, committedAudioTrackIndexV3(record, live), intent.selectedOverride) { - h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Generation: generation, Decision: playback.AudioReconcileNoop}) + h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Decision: playback.AudioReconcileNoop}) return } slog.WarnContext(ctx, "explicit audio selection missing from the verified inventory", "component", "api", "session", live.ID, "file_id", fileID, "audio_index", committedAudioTrackIndexV3(record, live)) - h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Generation: generation, Decision: playback.AudioReconcileRefused}) + h.recordAudioReconcileDecision(ctx, live.ID, generation, playback.AudioReconcileEntryV3{Decision: playback.AudioReconcileRefused}) if record != nil { h.enqueueRouteEventV3(playback.RouteEventRecordV3{ RouteEventV3: playback.RouteEventV3{ diff --git a/internal/api/handlers/playback_reconcile_audio_invalidation_test.go b/internal/api/handlers/playback_reconcile_audio_invalidation_test.go new file mode 100644 index 0000000000..cf794ec9bc --- /dev/null +++ b/internal/api/handlers/playback_reconcile_audio_invalidation_test.go @@ -0,0 +1,482 @@ +package handlers + +import ( + "context" + "encoding/json" + "sync" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/playback" +) + +// These tests cover the delivery side of cold-start default-audio +// reconciliation: the server records the corrected decision and withdraws the +// plan the client is playing, and only the client's own replan commits the +// replacement recipe. The assertions are about what the running player +// receives and what the committed attempt looks like immediately after +// reconciliation — not about the replan the client issues afterwards. + +// reconcileInvalidationConn captures the realtime commands pushed to a session +// so a test can count them and read their payloads. +type reconcileInvalidationConn struct { + mu sync.Mutex + messages []any +} + +func (c *reconcileInvalidationConn) WriteJSON(v any) error { + c.mu.Lock() + defer c.mu.Unlock() + c.messages = append(c.messages, v) + return nil +} + +func (c *reconcileInvalidationConn) all() []any { + c.mu.Lock() + defer c.mu.Unlock() + return append([]any(nil), c.messages...) +} + +func (c *reconcileInvalidationConn) commands() []playback.CommandEnvelope { + var out []playback.CommandEnvelope + for _, message := range c.all() { + command, ok := message.(playback.CommandEnvelope) + if !ok { + continue + } + if command.Name == playback.CommandPlanInvalidated { + out = append(out, command) + } + } + return out +} + +// invalidationFixture wires a reconcile handler whose live session has a +// realtime lane, so a plan withdrawal can actually be delivered. +// +// The reorder is the cold-start story: the declared inventory committed index 0 +// (en) while the preference is pt, and the verified probe reverses the order +// so pt now sits at index 0. The two streams differ in codec and channel +// count, so the correction cannot reuse the committed transport — exactly the +// case where the player has to adopt a different route. +type invalidationFixture struct { + handler *PlaybackHandler + session *playback.Session + record *playback.AttemptRecordV3 + verified *models.MediaFile + conn *reconcileInvalidationConn +} + +func newInvalidationFixture(t *testing.T, features []string, position float64) *invalidationFixture { + t.Helper() + planTime := []models.AudioTrack{ + {Index: 1, Language: "en", Codec: "aac", Channels: 2, Layout: "stereo"}, + {Index: 3, Language: "pt", Codec: "eac3", Channels: 6, Layout: "5.1"}, + } + verified := &models.MediaFile{ + ID: 7, + FilePath: "virtual://movie/tt-invalidated", + AudioTracks: []models.AudioTrack{ + {Index: 3, Language: "pt", Codec: "eac3", Channels: 6, Layout: "5.1"}, + {Index: 1, Language: "en", Codec: "aac", Channels: 2, Layout: "stereo"}, + }, + ProbeUpdatedAt: &invalidationProbeStamp, + } + session := &playback.Session{ + ID: "11111111-1111-1111-1111-111111111111", UserID: 1, ProfileID: "profile-1", + MediaFileID: 7, RequestedMediaFileID: 7, PlayMethod: playback.PlayDirect, + AudioTrackIndex: 0, + Position: position, + SelectionOrigin: SelectionOriginAuto, + PreferredAudioLanguage: "pt", + SelectedAudioSignature: playback.AudioTrackSignatureFromTrack(planTime[0]), + VirtualAudioTracks: planTime, + VirtualSourceURI: "virtual://movie/tt-invalidated", + VirtualSubtitleEvidenceURI: "virtual://movie/tt-invalidated", + VirtualSubtitleEvidenceFileID: 7, + VirtualSubtitleEvidenceSet: true, + } + record := reconcileIntent("pt", nil, playback.AudioTrackSignatureFromTrack(planTime[0])) + record.NormalizedRequest.ClientFeatures = features + // The canonical request is a real replan body, so it must satisfy the + // replan contract's own validation: a fixture that skipped this would + // prove nothing about a request that can never be replayed. + record.NormalizedRequest.Capabilities = reconcileCapabilities() + record.CurrentPlan.PlanAttemptKey = "v3:plan-reconcile-1:a:b" + record.CurrentPlan.SelectedTracks.Audio = &playback.TrackIdentityV3{ID: playback.TrackIDV3(7, "audio", 0), Index: intPtrReconcile(0)} + record.FrozenRecipe.TargetVideoCodec = "h264" + + handler := reconcileFixture(t, session, record, verified) + handler.RealtimeHub = playback.NewRealtimeHub() + conn := &reconcileInvalidationConn{} + registration := handler.RealtimeHub.Register(session.ID, conn) + if registration == nil { + t.Fatal("expected a realtime registration for the reconcile session") + } + t.Cleanup(func() { handler.RealtimeHub.Unregister(registration) }) + return &invalidationFixture{handler: handler, session: session, record: record, verified: verified, conn: conn} +} + +var invalidationProbeStamp = time.Now().UTC() + +// reconcile runs the sweep and returns the attempt row afterwards. +func (f *invalidationFixture) reconcile(t *testing.T) *playback.AttemptRecordV3 { + t.Helper() + f.handler.reconcileSessionDefaultAudio(context.Background(), f.session, f.verified.ID) + record, err := f.handler.PlanStoreV3.GetAttempt(context.Background(), f.session.ID) + if err != nil { + t.Fatalf("get attempt after reconcile: %v", err) + } + return record +} + +// TestReconcileAudioReplanAdoptsThroughPlanInvalidated is the running-stream +// adoption case: reconciliation persists the corrected decision, emits exactly +// one withdrawal naming the previously-active plan, and commits no replacement +// recipe of its own. The client's own replan then consumes the same decision +// and lands on the corrected audio index with its position preserved. +func TestReconcileAudioReplanAdoptsThroughPlanInvalidated(t *testing.T) { + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 321.5) + activePlanID := f.record.CurrentPlanID + activeStreamURL := f.record.CurrentPlan.Stream.URL + + after := f.reconcile(t) + + // (a) No self-committed replacement: the active plan and its committed + // recipe are untouched, so a server-side commit cannot have happened. + if after.CurrentPlanID != activePlanID { + t.Fatalf("active plan = %q, want the pre-reconciliation plan %q: reconciliation must not commit a replacement", + after.CurrentPlanID, activePlanID) + } + if after.CurrentPlan.Stream.URL != activeStreamURL { + t.Fatal("reconciliation must not rewrite the committed stream URL") + } + if after.FrozenRecipe.TargetVideoCodec != f.record.FrozenRecipe.TargetVideoCodec { + t.Fatal("reconciliation must not commit a replacement recipe") + } + + // The corrected selection and its canonical request are durable BEFORE + // anything was emitted, which is what makes the client's replan replayable. + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil { + t.Fatal("the settled decision must be recorded on the attempt ledger") + } + if entry.Decision != playback.AudioReconcileInvalidated { + t.Fatalf("decision = %q, want %q", entry.Decision, playback.AudioReconcileInvalidated) + } + if entry.AudioIndex == nil || *entry.AudioIndex != 0 { + t.Fatalf("corrected audio index = %v, want 0 (pt at its verified position)", entry.AudioIndex) + } + if entry.PlanID != activePlanID { + t.Fatalf("decision plan id = %q, want the active plan %q", entry.PlanID, activePlanID) + } + if entry.RequestDigest == "" || entry.Request == nil { + t.Fatal("the canonical request and its digest must be persisted with the decision") + } + + // (b) Exactly one withdrawal, naming the previously-active plan and the + // automatic-reconciliation reason. + commands := f.conn.commands() + if len(commands) != 1 { + t.Fatalf("plan_invalidated pushes = %d, want exactly 1", len(commands)) + } + command := commands[0] + if command.SessionID != f.session.ID { + t.Fatalf("command session = %q, want %q", command.SessionID, f.session.ID) + } + if command.Reason != playback.PlanInvalidatedDefaultAudioReconciliation { + t.Fatalf("command reason = %q, want %q", command.Reason, playback.PlanInvalidatedDefaultAudioReconciliation) + } + if command.DeadlineMS == 0 { + t.Fatal("the withdrawal must carry the documented replan deadline") + } + var payload struct { + PlanID string `json:"plan_id"` + Reason string `json:"reason"` + Generation string `json:"generation"` + } + if err := json.Unmarshal(command.Payload, &payload); err != nil { + t.Fatalf("withdrawal payload: %v", err) + } + if payload.PlanID != activePlanID { + t.Fatalf("withdrawal plan_id = %q, want the active plan %q", payload.PlanID, activePlanID) + } + if payload.Reason != playback.PlanInvalidatedDefaultAudioReconciliation { + t.Fatalf("withdrawal payload reason = %q, want %q", payload.Reason, playback.PlanInvalidatedDefaultAudioReconciliation) + } + if payload.Generation != entry.Generation { + t.Fatalf("withdrawal generation = %q, want the settled generation %q", payload.Generation, entry.Generation) + } + + // (c) The client-driven replan consumes the SAME decision and lands on the + // corrected audio index. + req := clientReplanForInvalidation(f, activePlanID) + applied := replayRequestForTest(req) + f.handler.pendingAudioReconciliationReplan(after, &applied) + if applied.SelectedTracks.Audio == nil { + t.Fatal("the replacement replan must carry the corrected audio selection") + } + if applied.SelectedTracks.Audio.ID != playback.TrackIDV3(f.verified.ID, "audio", 0) { + t.Fatalf("replan audio track id = %q, want the corrected track %q", + applied.SelectedTracks.Audio.ID, playback.TrackIDV3(f.verified.ID, "audio", 0)) + } + if applied.SelectedTracks.Audio.Index == nil || *applied.SelectedTracks.Audio.Index != 0 { + t.Fatalf("replan audio index = %v, want 0", applied.SelectedTracks.Audio.Index) + } + + // (d) The viewer's position survives adoption: the correction replaces the + // audio identity only and leaves everything the client sent intact. + if applied.PositionSeconds != 321.5 { + t.Fatalf("replan position = %v, want the client's live position 321.5", applied.PositionSeconds) + } + if applied.ReplanRequestID != "client-replan-1" || applied.FailedPlanID != activePlanID { + t.Fatal("the correction must not rewrite the client's own request identity") + } + if applied.Automatic != "" { + t.Fatal("a client-issued replan must stay client-issued") + } +} + +// TestReconcileAudioNoDoubleCorrection covers the duplicate-evidence case: a +// second probe write and a second heartbeat on a settled generation emit +// nothing, and the stored canonical request replays verbatim. +func TestReconcileAudioNoDoubleCorrection(t *testing.T) { + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 12) + + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil { + t.Fatal("the first evaluation must settle the generation") + } + + // A duplicate probe write: the same verified row persisted again. + f.reconcile(t) + // A second heartbeat after the generation settled. + f.handler.reconcilePendingAudioStartup(context.Background(), f.session.ID) + + if got := len(f.conn.commands()); got != 1 { + t.Fatalf("plan_invalidated pushes = %d after duplicate write and heartbeat, want exactly 1", got) + } + replayed, err := f.handler.PlanStoreV3.GetAttempt(context.Background(), f.session.ID) + if err != nil { + t.Fatalf("get attempt: %v", err) + } + replayEntry := playback.FindAudioReconcileEntry(replayed.AudioReconcileLedger, entry.Generation, f.session.ID) + if replayEntry == nil { + t.Fatal("the settled decision must survive the duplicate evaluation") + } + if replayEntry.RequestDigest != entry.RequestDigest { + t.Fatalf("replayed digest = %q, want the stored %q", replayEntry.RequestDigest, entry.RequestDigest) + } + if replayEntry.Request == nil || replayEntry.Request.ReplanRequestID != entry.Request.ReplanRequestID { + t.Fatal("a replayed decision must reuse the stored canonical request verbatim") + } + if len(replayed.AudioReconcileLedger.Entries) != 1 { + t.Fatalf("ledger entries = %d, want 1: a settled generation is never re-decided", + len(replayed.AudioReconcileLedger.Entries)) + } +} + +// TestReconcileAudioImmutableIdentity is the divergence case: two evaluations +// of one generation whose canonical bodies differ (the viewer moved between +// them) must not both proceed. The second is refused and recorded, and only one +// replan is ever issued. +func TestReconcileAudioImmutableIdentity(t *testing.T) { + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 10) + + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil || entry.RequestDigest == "" { + t.Fatal("the first evaluation must settle the generation with a canonical digest") + } + + // The viewer keeps watching: the live position moves, then a second + // evaluator of the SAME generation settles. That is the concurrent case — + // two evaluators that both read the ledger before either wrote. The + // canonical request embeds the live position, so the second body digests + // differently while its id (plan + target index) is unchanged. + if err := f.handler.sessionMgr.UpdateProgress(f.session.ID, 999.5, false); err != nil { + t.Fatalf("advance progress: %v", err) + } + moved := reconcileLiveSession(t, f.handler, f.session.ID) + settled, err := f.handler.PlanStoreV3.GetAttempt(context.Background(), f.session.ID) + if err != nil { + t.Fatalf("get attempt for second evaluator: %v", err) + } + f.handler.settleAudioReconcileInvalidation(context.Background(), moved, settled, f.verified, 0) + + if got := len(f.conn.commands()); got != 1 { + t.Fatalf("plan_invalidated pushes = %d after a diverged re-evaluation, want exactly 1", got) + } + + settled, err = f.handler.PlanStoreV3.GetAttempt(context.Background(), f.session.ID) + if err != nil { + t.Fatalf("get attempt: %v", err) + } + // The stored decision is unchanged: the first writer wins. + stored := playback.FindAudioReconcileEntry(settled.AudioReconcileLedger, entry.Generation, f.session.ID) + if stored == nil { + t.Fatal("the settled decision must survive the diverged evaluation") + } + if stored.RequestDigest != entry.RequestDigest { + t.Fatalf("stored digest = %q, want the first writer's %q", stored.RequestDigest, entry.RequestDigest) + } + // And the divergence is recorded as an explicit refusal, so it is + // auditable rather than silently dropped. + refusals := 0 + for _, e := range settled.AudioReconcileLedger.Entries { + if e.Decision == playback.AudioReconcileRefused { + refusals++ + if e.RequestDigest == entry.RequestDigest { + t.Fatal("the refusal must name the rejected digest, not the stored one") + } + } + } + if refusals != 1 { + t.Fatalf("refusal entries = %d, want exactly 1", refusals) + } + + // A retry of the same canonical decision replays the identical stored + // digest and issues no second replan. + retry := *stored.Request + if ReplanDigestV3(mustMarshalReconcile(t, retry)) != stored.RequestDigest { + t.Fatal("replaying the stored canonical request must reproduce the stored digest") + } +} + +// TestReconcileAudioWithdrawalSkippedWithoutCapability is the capability gate: +// a client that never advertised plan_invalidated_v1 gets no event, and its +// decision is still recorded so the correction lands on its next start. +func TestReconcileAudioWithdrawalSkippedWithoutCapability(t *testing.T) { + f := newInvalidationFixture(t, []string{playback.FeaturePlaybackPlanV3}, 5) + + after := f.reconcile(t) + + if got := len(f.conn.commands()); got != 0 { + t.Fatalf("plan_invalidated pushes = %d for a client without plan_invalidated_v1, want 0", got) + } + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil { + t.Fatal("the decision must still be recorded without the capability") + } + if entry.Decision != playback.AudioReconcileInvalidated || entry.Request == nil { + t.Fatalf("decision = %+v, want a recorded invalidation with its canonical request", entry) + } + if after.CurrentPlanID != f.record.CurrentPlanID { + t.Fatal("an unnegotiated client must not get its plan replaced server-side either") + } +} + +// TestReconcileAudioWithdrawalReachesOnlyTheOwnerLane pins the multi-node +// contract: the withdrawal travels the session's own hub lane, so it reaches +// the replica that owns that session's control socket and nothing else. The +// ledger, not a cross-replica RPC, is what keeps another replica from +// emitting a duplicate. +func TestReconcileAudioWithdrawalReachesOnlyTheOwnerLane(t *testing.T) { + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 7) + other := &reconcileInvalidationConn{} + otherRegistration := f.handler.RealtimeHub.Register("22222222-2222-2222-2222-222222222222", other) + if otherRegistration == nil { + t.Fatal("expected a second realtime registration") + } + t.Cleanup(func() { f.handler.RealtimeHub.Unregister(otherRegistration) }) + + f.reconcile(t) + + if got := len(f.conn.commands()); got != 1 { + t.Fatalf("owner lane pushes = %d, want 1", got) + } + if got := len(other.commands()); got != 0 { + t.Fatalf("other session pushes = %d, want 0: the event is per session", got) + } + if command := f.conn.commands()[0]; command.SessionID != f.session.ID { + t.Fatalf("command session = %q, want %q", command.SessionID, f.session.ID) + } +} + +// TestPendingAudioReconciliationConsumesOnce pins the one-shot nature of the +// consumption: the correction applies to the replan that replans off the +// withdrawn plan, and to nothing else. +func TestPendingAudioReconciliationConsumesOnce(t *testing.T) { + audioIndex := 0 + record := &playback.AttemptRecordV3{ + SessionID: "11111111-1111-1111-1111-111111111111", + CurrentPlan: playback.PlanV3{PlanID: "plan:current"}, + AudioReconcileLedger: playback.AudioReconcileLedgerV3{Entries: []playback.AudioReconcileEntryV3{{ + Generation: "gen-1", + SessionID: "11111111-1111-1111-1111-111111111111", + Decision: playback.AudioReconcileInvalidated, + AudioIndex: &audioIndex, + PlanID: "plan:withdrawn", + RequestDigest: "digest-1", + Request: &playback.ReplanRequestV3{SelectedTracks: playback.SelectedTracksV3{ + Audio: &playback.TrackIdentityV3{ID: "file:7:audio:0", Index: intPtrReconcile(0)}, + }}, + }}}, + } + h := &PlaybackHandler{} + f := &invalidationFixture{session: &playback.Session{ID: record.SessionID}, record: record} + + answering := clientReplanForInvalidation(f, "plan:withdrawn") + h.pendingAudioReconciliationReplan(record, &answering) + if answering.SelectedTracks.Audio == nil || answering.SelectedTracks.Audio.ID != "file:7:audio:0" { + t.Fatal("the replan off the withdrawn plan must carry the correction") + } + + // The next replan runs off the replacement plan, so the correction is not + // re-applied over whatever the viewer chose afterwards. + later := clientReplanForInvalidation(f, "plan:current") + h.pendingAudioReconciliationReplan(record, &later) + if later.SelectedTracks.Audio != nil { + t.Fatal("a replan off the replacement plan must keep the client's own selection") + } + + // The server's own automatic replan already names the corrected selection. + automatic := clientReplanForInvalidation(f, "plan:withdrawn") + automatic.Automatic = playback.ReplanAutomaticV3 + automatic.SelectedTracks.Audio = &playback.TrackIdentityV3{ID: "file:7:audio:2", Index: intPtrReconcile(2)} + h.pendingAudioReconciliationReplan(record, &automatic) + if automatic.SelectedTracks.Audio.ID != "file:7:audio:2" { + t.Fatal("the automatic path must not be overridden by the stored correction") + } +} + +// clientReplanForInvalidation builds the failure_recovery body a client issues +// when it answers a plan withdrawal: it names the plan it was playing, carries +// its live position, and selects no audio track of its own. +func clientReplanForInvalidation(f *invalidationFixture, failedPlanID string) playback.ReplanRequestV3 { + return playback.ReplanRequestV3{ + ProtocolVersion: playback.ProtocolV3, + Operation: playback.ReplanOperationFailureRecoveryV3, + PlaybackAttemptID: f.record.PlaybackAttemptID, + ReplanRequestID: "client-replan-1", + FailedPlanID: failedPlanID, + PlanAttemptID: "client-plan-attempt-1", + PlanAttemptKey: f.record.CurrentPlan.PlanAttemptKey, + AttemptCount: 2, + PositionSeconds: 321.5, + Failure: playback.FailureV3{Classification: playback.PlanInvalidatedDefaultAudioReconciliation}, + } +} + +// replayRequestForTest deep-copies a request the way the replan handler does +// before overlaying the correction, so a test asserts on the same value the +// replan path sees rather than on a shared pointer. +func replayRequestForTest(req playback.ReplanRequestV3) playback.ReplanRequestV3 { + copied := req + if req.SelectedTracks.Audio != nil { + audio := *req.SelectedTracks.Audio + copied.SelectedTracks.Audio = &audio + } + return copied +} + +func mustMarshalReconcile(t *testing.T, req playback.ReplanRequestV3) []byte { + t.Helper() + body, err := json.Marshal(req) + if err != nil { + t.Fatalf("marshal reconcile request: %v", err) + } + return body +} diff --git a/internal/api/handlers/playback_reconcile_audio_test.go b/internal/api/handlers/playback_reconcile_audio_test.go index 16b6fd713e..92860924d3 100644 --- a/internal/api/handlers/playback_reconcile_audio_test.go +++ b/internal/api/handlers/playback_reconcile_audio_test.go @@ -135,6 +135,19 @@ func TestReconcileVerifiedDefaultAudioReordersToPreferredLanguage(t *testing.T) func intPtrReconcile(v int) *int { return &v } +// reconcileCapabilities is the minimum client codec evidence a v3 replan body +// must carry to be valid. +func reconcileCapabilities() playback.ClientCodecCapabilitiesV3 { + return playback.ClientCodecCapabilitiesV3{ + VideoEvidence: playback.EvidenceExactV3, + AudioEvidence: playback.EvidenceExactV3, + CodecsVideo: []string{"h264"}, + CodecsAudio: []string{"aac"}, + Containers: []string{"mp4"}, + MaxResolution: "1080p", + } +} + // MULTi membership: a track whose Languages list carries eng beats a bare // eng track only on rank, while a bare MULTi primary with no member list // never counts as a concrete language match. @@ -465,11 +478,11 @@ func TestReconcilePropagatesCallerDeadlineWithoutReset(t *testing.T) { done := make(chan struct{}) go func() { defer close(done) - // An already-cancelled caller context: reconciliation must return on + // An already-canceled caller context: reconciliation must return on // it rather than arming its own timeout. - cancelled, cancel := context.WithCancel(context.Background()) + canceled, cancel := context.WithCancel(context.Background()) cancel() - handler.reconcilePendingAudioStartup(cancelled, session.ID) + handler.reconcilePendingAudioStartup(canceled, session.ID) }() select { diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index 854d543ee2..060bb90cea 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -7834,6 +7834,11 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte } else { applySelectedTrackOverridesToStartV3(&start, req.SelectedTracks) } + // A replan that answers a default-audio reconciliation withdrawal + // carries the correction the settled decision recorded: the client + // replans off the withdrawn plan, which still names the pre-reorder + // stream, so without this the correction never reaches the transport. + h.pendingAudioReconciliationReplan(record, &req) } // Native selection is negotiated at start and can only be disabled during // an attempt. Keep a confirmed failure disabled even when a later client diff --git a/internal/playback/planstore/postgres.go b/internal/playback/planstore/postgres.go index 2ee5c51c0f..a7b9e7cec1 100644 --- a/internal/playback/planstore/postgres.go +++ b/internal/playback/planstore/postgres.go @@ -314,9 +314,9 @@ func (s *Postgres) AppendRecoveryExclusions(ctx context.Context, sessionID strin // concurrent records on the attempt row, and the base-revision compare makes // a writer that read a stale revision lose with // ErrRecoveryRevisionConflictV3 instead of clobbering a newer decision. -// Recording the same generation twice is a no-op returning the stored -// ledger: retries and second replicas converge instead of minting a second -// replan. +// Recording the same (generation, session, decision, digest) twice is a no-op +// returning the stored ledger: retries and second replicas converge instead of +// minting a second event. func (s *Postgres) RecordAudioReconciliation(ctx context.Context, sessionID string, baseRevision int64, entry playback.AudioReconcileEntryV3) (playback.AudioReconcileLedgerV3, int64, error) { tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) if err != nil { @@ -346,6 +346,15 @@ func (s *Postgres) RecordAudioReconciliation(ctx context.Context, sessionID stri } base.Revision = revision merged := playback.AppendAudioReconcileEntry(base, entry) + if len(merged.Entries) == len(base.Entries) { + // The same decision is already recorded: report the stored ledger + // without taking the write lock's revision. A retry must not look + // like a new decision to a caller reading the revision it handed back. + if err := tx.Commit(ctx); err != nil { + return playback.AudioReconcileLedgerV3{}, 0, err + } + return base, revision, nil + } merged.Revision = revision + 1 mergedJSON, err := json.Marshal(merged) if err != nil { diff --git a/internal/playback/protocol_store_v3.go b/internal/playback/protocol_store_v3.go index 8ac8642d91..7a3f6b2c32 100644 --- a/internal/playback/protocol_store_v3.go +++ b/internal/playback/protocol_store_v3.go @@ -221,10 +221,11 @@ type AudioSelectionV3 struct { } // AudioReconcileLedgerV3 is the durable, attempt-scoped memory of automatic -// default-audio reconciliation decisions. One entry per verified generation -// the reconcile path has settled (a committed replan, a byte-equal no-op, or -// a refusal), keyed by the generation so a retried probe write or a second -// replica replays the same decision instead of minting a second replan. +// default-audio reconciliation decisions. One entry per (generation, session) +// the reconcile path has settled (a committed replan, a byte-equal no-op, a +// plan invalidation, or a refusal), keyed by that pair so a retried probe +// write or a second replica replays the same decision instead of minting a +// second event. type AudioReconcileLedgerV3 struct { // Entries is the settled chain, in the order decisions were recorded. It // is scoped by generation: entries for different generations never @@ -255,6 +256,15 @@ const ( // explicit (never overridden), or superseded by a viewer change: the // reconcile path declined and must not retry the generation. AudioReconcileRefused AudioReconcileDecisionV3 = "refused" + // AudioReconcileInvalidated means the corrected executable selection was + // persisted together with the canonical request, and the active plan was + // withdrawn through the realtime plan_invalidated route event instead of + // being replaced server-side. Only the client's own replacement replan + // commits a new recipe, so the decision and the request it eventually + // consumes are one ledger entry: the entry is written before anything is + // emitted, and a duplicate probe write or a retried heartbeat replays the + // stored decision rather than minting a second event. + AudioReconcileInvalidated AudioReconcileDecisionV3 = "invalidated" ) // AudioReconcileEntryV3 is one settled reconciliation decision. Generation @@ -263,11 +273,22 @@ const ( // retry replays the same bytes; RequestDigest fingerprints those bytes and // is the idempotency key the decision is retried under. type AudioReconcileEntryV3 struct { - Generation string `json:"generation"` - Decision AudioReconcileDecisionV3 `json:"decision"` - AudioIndex *int `json:"audio_index,omitempty"` - Request *ReplanRequestV3 `json:"request,omitempty"` - RequestDigest string `json:"request_digest,omitempty"` + Generation string `json:"generation"` + // SessionID is the attempt session the decision was recorded under. The + // ledger is stored per session row, so this is normally that row's own + // session; it is persisted with the entry so a replayed entry can be + // validated against the session it was decided for rather than the plan id + // that happened to be current when the ledger was read. + SessionID string `json:"session_id,omitempty"` + Decision AudioReconcileDecisionV3 `json:"decision"` + AudioIndex *int `json:"audio_index,omitempty"` + // PlanID is the plan the decision was decided against, and the plan an + // AudioReconcileInvalidated entry withdraws. Keying a lookup on it would + // be wrong: the same generation is re-evaluated against whichever plan is + // active then, and the plan id moves as replans commit. + PlanID string `json:"plan_id,omitempty"` + Request *ReplanRequestV3 `json:"request,omitempty"` + RequestDigest string `json:"request_digest,omitempty"` } // RecoveryExclusionV3 is one confirmed candidate failure. The tuple is scoped @@ -395,20 +416,29 @@ type AudioReconcileStoreV3 interface { // revision the caller read alongside its current ledger; a mismatched // write returns ErrRecoveryRevisionConflictV3 with the committed revision // instead of clobbering, and the caller re-reads and retries. A negative - // baseRevision appends unconditionally. Recording the same generation - // twice is a no-op returning the stored entry. It returns - // ErrSessionNotFound when there is no live attempt row. + // baseRevision appends unconditionally. Recording the same + // (generation, session) pair twice is a no-op returning the stored entry. + // It returns ErrSessionNotFound when there is no live attempt row. RecordAudioReconciliation(ctx context.Context, sessionID string, baseRevision int64, entry AudioReconcileEntryV3) (AudioReconcileLedgerV3, int64, error) // GetAudioReconcileLedger reads the durable ledger and its revision. GetAudioReconcileLedger(ctx context.Context, sessionID string) (AudioReconcileLedgerV3, int64, error) } -// FindAudioReconcileEntry returns the settled entry for generation, or nil -// when the generation has no recorded decision yet. -func FindAudioReconcileEntry(ledger AudioReconcileLedgerV3, generation string) *AudioReconcileEntryV3 { +// FindAudioReconcileEntry returns the settled entry for the (generation, +// session) pair, or nil when that generation has no recorded decision for that +// session. The session is part of the key because one verified generation can +// be reconciled for several live sessions, each with its own committed plan +// and its own canonical request: settling one session must never suppress +// another. Entries recorded before the session id was persisted are matched by +// generation alone, so an in-flight upgrade of the ledger cannot re-decide a +// generation that was already settled. +func FindAudioReconcileEntry(ledger AudioReconcileLedgerV3, generation, sessionID string) *AudioReconcileEntryV3 { for i := range ledger.Entries { - if ledger.Entries[i].Generation == generation { - entry := ledger.Entries[i] + entry := ledger.Entries[i] + if entry.Generation != generation { + continue + } + if entry.SessionID == "" || strings.TrimSpace(sessionID) == "" || entry.SessionID == sessionID { return &entry } } @@ -416,11 +446,20 @@ func FindAudioReconcileEntry(ledger AudioReconcileLedgerV3, generation string) * } // AppendAudioReconcileEntry returns base with entry appended, or base -// unchanged when the generation is already recorded. The first writer wins; -// a replayed record is a no-op, so concurrent commits converge. +// unchanged when the same (generation, session, decision) is already recorded. +// One decision per pair is the invariant — a replayed record is a no-op, so +// concurrent commits converge — while a REFUSED entry recorded for a pair that +// already settled as an invalidation is a distinct entry: it carries the +// rejected canonical digest next to the stored one, which is what makes a +// diverged re-evaluation auditable instead of silently dropped. func AppendAudioReconcileEntry(base AudioReconcileLedgerV3, entry AudioReconcileEntryV3) AudioReconcileLedgerV3 { - if FindAudioReconcileEntry(base, entry.Generation) != nil { - return base + for _, existing := range base.Entries { + if existing.Generation == entry.Generation && + existing.SessionID == entry.SessionID && + existing.Decision == entry.Decision && + existing.RequestDigest == entry.RequestDigest { + return base + } } merged := base merged.Entries = append(append([]AudioReconcileEntryV3(nil), base.Entries...), entry) @@ -782,7 +821,8 @@ func (s *MemoryPlanStoreV3) GetRecoveryState(_ context.Context, sessionID string // durable ledger. It is a compare-and-set on the ledger revision: a caller // presenting a stale revision loses and retries against the committed // ledger, so no writer can drop another's decision. Recording the same -// generation twice returns the stored entry without growing the chain. +// (generation, session, decision, digest) again returns the stored entry +// without growing the chain. func (s *MemoryPlanStoreV3) RecordAudioReconciliation(_ context.Context, sessionID string, baseRevision int64, entry AudioReconcileEntryV3) (AudioReconcileLedgerV3, int64, error) { s.mu.Lock() defer s.mu.Unlock() @@ -794,6 +834,9 @@ func (s *MemoryPlanStoreV3) RecordAudioReconciliation(_ context.Context, session return AudioReconcileLedgerV3{}, record.AudioReconcileLedger.Revision, ErrRecoveryRevisionConflictV3 } merged := AppendAudioReconcileEntry(record.AudioReconcileLedger, entry) + if len(merged.Entries) == len(record.AudioReconcileLedger.Entries) { + return record.AudioReconcileLedger, record.AudioReconcileLedger.Revision, nil + } merged.Revision = record.AudioReconcileLedger.Revision + 1 record.AudioReconcileLedger = merged s.attempts[attemptID] = *record diff --git a/internal/playback/realtime.go b/internal/playback/realtime.go index cc993fbaea..d42fd4b39b 100644 --- a/internal/playback/realtime.go +++ b/internal/playback/realtime.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "github.com/Silo-Server/silo-server/internal/models" ) @@ -617,6 +618,17 @@ const ( // scan came back multi-PPS after the plan was already playing, so the video // stream-copy route it named cannot serve this source. PlanInvalidatedVideoCopyUnsafe = "video_copy_unsafe" + // PlanInvalidatedDefaultAudioReconciliation means the verified probe + // inventory that landed after the plan was issued moved the server-resolved + // default audio selection to a different stream, so the committed recipe no + // longer plays the preferred language. The corrected inventory and the + // decision naming the target index are already durable when this is + // pushed; only the client's replacement replan commits the new recipe. + // + // It is a second reason on the existing command rather than a new one, so + // the wire contract, the command-name vocabulary and the client's decoder + // are unchanged. + PlanInvalidatedDefaultAudioReconciliation = "default_audio_reconciliation" ) // PlanInvalidatedPayload names the plan the server withdrew and why. @@ -631,14 +643,45 @@ type PlanInvalidatedPayload struct { // NewPlanInvalidatedCommand builds a validated plan_invalidated command. func NewPlanInvalidatedCommand(sessionID, commandID, planID, reason string) (CommandEnvelope, error) { + command, err := NewPlanInvalidatedCommandForGeneration(sessionID, commandID, planID, reason, "") + return command, err +} + +// NewPlanInvalidatedCommandForGeneration is NewPlanInvalidatedCommand with the +// verified generation the withdrawal was decided against. +// +// The envelope's top-level reason and the payload's plan_id and reason are the +// existing contract, unchanged. The generation is an ADDITIVE payload field: a +// client that does not read it ignores it, and one that does can tell an +// in-flight duplicate for a generation it already replanned from a correction +// that landed afterwards. An empty generation leaves the payload exactly the +// two-field shape a pre-existing client expects. +func NewPlanInvalidatedCommandForGeneration(sessionID, commandID, planID, reason, generation string) (CommandEnvelope, error) { if planID == "" || reason == "" { return CommandEnvelope{}, ErrInvalidRealtimePayload } - payload, err := json.Marshal(PlanInvalidatedPayload{Reason: reason, PlanID: planID}) + payload := PlanInvalidatedPayload{Reason: reason, PlanID: planID} + body, err := json.Marshal(payload) + if err != nil { + return CommandEnvelope{}, err + } + if strings.TrimSpace(generation) != "" { + extended := struct { + PlanInvalidatedPayload + Generation string `json:"generation,omitempty"` + }{PlanInvalidatedPayload: payload, Generation: generation} + if body, err = json.Marshal(extended); err != nil { + return CommandEnvelope{}, err + } + } + command, err := NewCommandEnvelope(sessionID, commandID, CommandPlanInvalidated, body) if err != nil { return CommandEnvelope{}, err } - return NewCommandEnvelope(sessionID, commandID, CommandPlanInvalidated, payload) + // The reason rides in both places the contract already defines it: the + // envelope field and the payload. A client reads either. + command.Reason = reason + return command, nil } // NewCommandEnvelope creates a validated command envelope. From a288707d53e8b3e2db572d66a4986ca643a54c96 Mon Sep 17 00:00:00 2001 From: drondeseries Date: Mon, 5 Oct 2026 13:55:45 -0400 Subject: [PATCH 04/16] fix(playback): apply audio correction before the start selection and keep the winner The pending default-audio correction was layered onto the replan request after the selection had already been copied onto the executable start, so audio resolution never saw it. Apply the correction first. A later divergence refusal is an audit record, not a verdict on the settled decision, so it must not strand a correction already announced. --- .../api/handlers/playback_reconcile_audio.go | 15 ++-- ...yback_reconcile_audio_invalidation_test.go | 77 +++++++++++++++++++ internal/api/handlers/playback_v3.go | 14 ++-- 3 files changed, 93 insertions(+), 13 deletions(-) diff --git a/internal/api/handlers/playback_reconcile_audio.go b/internal/api/handlers/playback_reconcile_audio.go index 7d5a4baac8..0fadbd7c8a 100644 --- a/internal/api/handlers/playback_reconcile_audio.go +++ b/internal/api/handlers/playback_reconcile_audio.go @@ -721,14 +721,13 @@ func FindPendingAudioReconciliation(record *playback.AttemptRecordV3, planID str var found *playback.AudioReconcileEntryV3 for i := range record.AudioReconcileLedger.Entries { entry := record.AudioReconcileLedger.Entries[i] - // A refusal recorded for the same (generation, session) pair is the - // divergence verdict: it supersedes the invalidation that evaluator - // lost to, so reading the invalidation instead would let a correction - // a rejected re-evaluation had already invalidated reach the client. - if entry.Decision == playback.AudioReconcileRefused && found != nil && entry.Generation == found.Generation { - found = nil - continue - } + // A divergence refusal is what a LATER evaluator recorded after + // losing the claim on an already-settled generation — typically two + // concurrent evaluations that captured a different playhead. It is + // not a verdict on the winner: erasing the settled invalidation here + // would discard a correction that was already announced to the client + // and that this replan exists to consume. The winning decision stays + // authoritative; the refusal remains in the ledger as an audit record. if entry.Decision != playback.AudioReconcileInvalidated || entry.PlanID != planID { continue } diff --git a/internal/api/handlers/playback_reconcile_audio_invalidation_test.go b/internal/api/handlers/playback_reconcile_audio_invalidation_test.go index cf794ec9bc..2e6c33c6b4 100644 --- a/internal/api/handlers/playback_reconcile_audio_invalidation_test.go +++ b/internal/api/handlers/playback_reconcile_audio_invalidation_test.go @@ -348,6 +348,83 @@ func TestReconcileAudioImmutableIdentity(t *testing.T) { // TestReconcileAudioWithdrawalSkippedWithoutCapability is the capability gate: // a client that never advertised plan_invalidated_v1 gets no event, and its // decision is still recorded so the correction lands on its next start. +// The pending correction must be layered onto the executable selection, not +// onto the request after the selection was already applied: audio resolution +// reads start.AudioTrackID/AudioTrackIndex, so a correction applied afterwards +// never reaches the plan or the executor's audio map. +func TestPendingAudioCorrectionReachesTheStartSelection(t *testing.T) { + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 10) + + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil || entry.Request == nil || entry.Request.SelectedTracks.Audio == nil { + t.Fatal("expected a settled correction carrying an audio identity") + } + + // A client replan off the withdrawn plan, with no audio identity of its own. + start := after.NormalizedRequest + req := &playback.ReplanRequestV3{ + ProtocolVersion: playback.ProtocolV3, + Operation: playback.ReplanOperationTrackChangeV3, + FailedPlanID: entry.PlanID, + } + f.handler.pendingAudioReconciliationReplan(after, req) + if req.SelectedTracks.Audio == nil { + t.Fatal("the replan did not consume the pending correction") + } + + // Now the ordering that actually matters: applying the selection must + // happen AFTER the correction, exactly as the replan overlay does. + applySelectedTracksToStartV3(&start, req.SelectedTracks) + if start.AudioTrackID == "" && start.AudioTrackIndex == nil { + t.Fatal("the corrected audio identity never reached the start selection") + } + if start.AudioTrackIndex == nil || entry.Request.SelectedTracks.Audio.Index == nil { + t.Fatal("the corrected audio identity did not carry an index onto the start selection") + } + if *start.AudioTrackIndex != *entry.Request.SelectedTracks.Audio.Index { + t.Fatalf("start audio index = %d, want the corrected %d", + *start.AudioTrackIndex, *entry.Request.SelectedTracks.Audio.Index) + } +} + +// A refusal recorded by a later evaluator must not erase the correction the +// winning evaluator already settled and announced: two concurrent evaluations +// that captured different playheads would otherwise strand the correction. +func TestRefusalDoesNotStrandTheSettledCorrection(t *testing.T) { + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 10) + + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil { + t.Fatal("expected a settled correction") + } + + // A diverged re-evaluation records an explicit refusal for the same + // (generation, session). + record, err := f.handler.PlanStoreV3.GetAttempt(context.Background(), f.session.ID) + if err != nil { + t.Fatalf("get attempt: %v", err) + } + record.AudioReconcileLedger.Entries = append(record.AudioReconcileLedger.Entries, + playback.AudioReconcileEntryV3{ + Generation: entry.Generation, + SessionID: f.session.ID, + PlanID: entry.PlanID, + Decision: playback.AudioReconcileRefused, + AudioIndex: entry.AudioIndex, + RequestDigest: "divergent-digest", + }) + + found := FindPendingAudioReconciliation(record, entry.PlanID) + if found == nil { + t.Fatal("a later refusal stranded the already-announced correction") + } + if found.RequestDigest != entry.RequestDigest { + t.Fatalf("pending correction digest = %q, want the winning %q", found.RequestDigest, entry.RequestDigest) + } +} + func TestReconcileAudioWithdrawalSkippedWithoutCapability(t *testing.T) { f := newInvalidationFixture(t, []string{playback.FeaturePlaybackPlanV3}, 5) diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go index 060bb90cea..39a8f098e8 100644 --- a/internal/api/handlers/playback_v3.go +++ b/internal/api/handlers/playback_v3.go @@ -7826,6 +7826,15 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte // capability payloads. Omission keeps the start-time features. start.ClientFeatures = req.ClientFeatures } + // A replan that answers a default-audio reconciliation withdrawal + // carries the correction the settled decision recorded: the client + // replans off the withdrawn plan, which still names the pre-reorder + // stream, so without this the correction never reaches the transport. + // This must run BEFORE the selection is applied onto start: audio + // resolution reads start.AudioTrackID/AudioTrackIndex, so a correction + // layered on afterwards would never reach the plan or the executor's + // audio map. + h.pendingAudioReconciliationReplan(record, &req) if trackChange { // A track_change is the only operation where an omitted subtitle // means "subtitles off". Failure, seek, and quality replans may omit @@ -7834,11 +7843,6 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte } else { applySelectedTrackOverridesToStartV3(&start, req.SelectedTracks) } - // A replan that answers a default-audio reconciliation withdrawal - // carries the correction the settled decision recorded: the client - // replans off the withdrawn plan, which still names the pre-reorder - // stream, so without this the correction never reaches the transport. - h.pendingAudioReconciliationReplan(record, &req) } // Native selection is negotiated at start and can only be disabled during // an attempt. Keep a confirmed failure disabled even when a later client From 6769d6edb293b2f864b46feed2d9244b2c7b0505 Mon Sep 17 00:00:00 2001 From: drondeseries Date: Mon, 5 Oct 2026 14:47:16 -0400 Subject: [PATCH 05/16] fix(playback): replan a default-audio withdrawal without failure semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cold-start audio correction withdraws a plan whose route is healthy — the committed recipe plays the wrong audio stream, not wrong bytes — but the client treated every `plan_invalidated` as a `failure_recovery`, which folds the current plan's attempt key into `attempted_plan_keys`. That excluded the route that was playing from its own replacement plan, pushing a working session onto a worse route or a terminal. Replan off `default_audio_reconciliation` as `track_change` instead: it carries the audio correction, and since nothing failed the route stays eligible. Position and pause state are preserved as before, and `fallback_reason` still reports the server's own reason, so telemetry is unchanged and already distinguishes the two cases. Every other reason keeps the recovery semantics §6.1 specifies. The browser cannot import the Go constant, so each side pins the other's string with a test. No new operation or feature: the path reuses `track_change`, so the client capability list and the v3 operation enum are unchanged. --- docs/architecture/playback-protocol-v3.md | 38 ++++- .../realtime_invalidation_reason_test.go | 47 ++++++ web/src/player/components/VideoPlayer.tsx | 11 +- .../defaultAudioReconciliationReason.test.ts | 40 +++++ .../player/hooks/usePlaybackSession.test.ts | 158 ++++++++++++++++++ web/src/player/hooks/usePlaybackSession.ts | 45 +++-- web/src/player/realtime-protocol.ts | 17 ++ 7 files changed, 336 insertions(+), 20 deletions(-) create mode 100644 internal/playback/realtime_invalidation_reason_test.go create mode 100644 web/src/player/defaultAudioReconciliationReason.test.ts diff --git a/docs/architecture/playback-protocol-v3.md b/docs/architecture/playback-protocol-v3.md index 6ee9b155df..a806e9e99b 100644 --- a/docs/architecture/playback-protocol-v3.md +++ b/docs/architecture/playback-protocol-v3.md @@ -1120,9 +1120,15 @@ A client that advertises `plan_invalidated_v1` in `client_features` promises to: 2. run its ordinary recovery replan — `operation: "failure_recovery"`, with the invalidated plan's `plan_attempt_key` in `attempted_plan_keys` so the copy route is excluded deterministically (the now-persisted verdict excludes it - too), and + too), **unless** `reason` is `default_audio_reconciliation`, which replans as + an intent operation instead (§6.1.1), and 3. send `{"type":"result","status":"completed"}` when the replan is done. +`failure_recovery` is the right default because it is the one operation that +excludes the withdrawn route without the client reasoning about deliveries. It +is wrong for a reason that withdraws a plan without indicting it, and that is the +only such reason on this command. + **Everything else is a session stop.** The server pushes the command only to a session that negotiated the feature *and* holds a live realtime connection. No feature, no connection, no `completed` result within `deadline_ms` (an ack @@ -1250,11 +1256,31 @@ That split makes the ordering load-bearing: - A refused divergence supersedes the invalidation it disputes: a correction a rejected re-evaluation has already invalidated never reaches the client. -The client's replacement replan cannot echo a track it never chose, so it does -not name one. The replan that replans off the withdrawn plan therefore consumes -the stored decision and fills in the audio identity. The correction is one-shot -rather than an ongoing override: it is keyed to the withdrawn plan, and the -plan the replan commits becomes current, so the client's next replan is its own. +**The replacement replan is an intent operation, not a failure recovery.** The +route is healthy and nothing failed: the committed recipe plays the wrong audio +stream, not wrong bytes. `failure_recovery` would fold the withdrawn plan's +`plan_attempt_key` into `attempted_plan_keys` and so exclude the route that is +currently playing from its own replacement — pushing a working session onto a +worse route, or onto a terminal. A client therefore replans off a +`default_audio_reconciliation` withdrawal as `operation: "track_change"`, with no +`failure` payload (the server rejects one on that operation anyway) and no route +exclusion, carrying only its live position. It is the operation the server's own +automatic replan already uses for this correction, and a `track_change` that +changes nothing still returns a fresh plan. `client_features` is unchanged: this +reuses the existing operation rather than adding one. + +The two sides name the reason independently — the server from +`playback.PlanInvalidatedDefaultAudioReconciliation`, the browser from its own +mirror of the string — and each half pins the other with a test +(`internal/playback/realtime_invalidation_reason_test.go` and +`web/src/player/defaultAudioReconciliationReason.test.ts`), because a rename on +one side is invisible at runtime and merely degrades a healthy session into a +failure recovery. Every other `plan_invalidated` reason keeps the recovery +semantics of §6.1. + +The correction is one-shot rather than an ongoing override: it is keyed to the +withdrawn plan, and the plan the replan commits becomes current, so the client's +next replan is its own. Four rules bound it: diff --git a/internal/playback/realtime_invalidation_reason_test.go b/internal/playback/realtime_invalidation_reason_test.go new file mode 100644 index 0000000000..3110c3acd9 --- /dev/null +++ b/internal/playback/realtime_invalidation_reason_test.go @@ -0,0 +1,47 @@ +package playback + +import ( + "os" + "path/filepath" + "regexp" + "testing" +) + +// clientReasonSourceV3 is the web client's mirror of this constant. The browser +// cannot import Go, so the two halves of the wire are held together by a test in +// each repository rather than by the compiler: +// internal/playback/realtime_invalidation_reason_test.go here, and +// web/src/player/defaultAudioReconciliationReason.test.ts on the client side. +// +// A drift is invisible at runtime and degrading rather than breaking: the client +// falls back to treating a healthy cold-start session's audio correction as a +// route failure, which excludes the route that is playing from its own +// replacement plan. See docs/architecture/playback-protocol-v3.md §6.1.1. +var clientReasonSourceV3 = filepath.Join( + "..", "..", "web", "src", "player", "realtime-protocol.ts", +) + +func TestPlanInvalidatedDefaultAudioReconciliationReasonMatchesClient(t *testing.T) { + source, err := os.ReadFile(clientReasonSourceV3) + if err != nil { + t.Fatalf("read %s: %v", clientReasonSourceV3, err) + } + match := regexp.MustCompile( + `PLAN_INVALIDATED_DEFAULT_AUDIO_RECONCILIATION\s*=\s*"([a-z_]+)"`, + ).FindSubmatch(source) + if match == nil { + t.Fatalf("no PLAN_INVALIDATED_DEFAULT_AUDIO_RECONCILIATION string in %s", clientReasonSourceV3) + } + if got, want := string(match[1]), PlanInvalidatedDefaultAudioReconciliation; got != want { + t.Errorf("web client reason = %q, server constant = %q", got, want) + } +} + +// The two reasons share one command and take different replan paths on the +// client: the copy-safety verdict excludes the route it withdraws, the audio +// reconciliation does not. They must not collapse into one another. +func TestPlanInvalidationReasonsAreDistinct(t *testing.T) { + if PlanInvalidatedVideoCopyUnsafe == PlanInvalidatedDefaultAudioReconciliation { + t.Errorf("both plan_invalidated reasons are %q", PlanInvalidatedVideoCopyUnsafe) + } +} diff --git a/web/src/player/components/VideoPlayer.tsx b/web/src/player/components/VideoPlayer.tsx index 53829c48ff..2b051861bc 100644 --- a/web/src/player/components/VideoPlayer.tsx +++ b/web/src/player/components/VideoPlayer.tsx @@ -4218,10 +4218,13 @@ export function VideoPlayer({ }); return; case "plan_invalidated": { - // The server decided the route it planned cannot serve this source - // after all. Ack (already sent by the transport), replan off it, and - // report the outcome: a rejection is the server's cue to stop the - // session so the client's own recovery can mint a fresh attempt. + // The server decided the recipe or the route it planned cannot serve + // this source after all. Ack (already sent by the transport), replan + // off it, and report the outcome: a rejection is the server's cue to + // stop the session so the client's own recovery can mint a fresh + // attempt. Which replan the session runs depends on the reason — + // `usePlaybackSession.invalidatePlan` treats a default-audio + // reconciliation as an audio correction rather than a route failure. const invalidated = readPlanInvalidatedPayload(command.payload); if (!invalidated) { throw new Error("invalid_plan_invalidated_payload"); diff --git a/web/src/player/defaultAudioReconciliationReason.test.ts b/web/src/player/defaultAudioReconciliationReason.test.ts new file mode 100644 index 0000000000..fb436733bb --- /dev/null +++ b/web/src/player/defaultAudioReconciliationReason.test.ts @@ -0,0 +1,40 @@ +// @vitest-environment node + +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +import { PLAN_INVALIDATED_DEFAULT_AUDIO_RECONCILIATION } from "./realtime-protocol"; + +/** + * The `default_audio_reconciliation` withdrawal reason exists on both sides of + * the wire, and nothing but these two checks keeps them together: the browser + * cannot import the Go constant, so a rename on either side silently degrades + * a healthy session into a failure recovery that excludes its own working route + * (see `invalidatePlan` in hooks/usePlaybackSession.ts). The failure is invisible + * — playback merely degrades — so it is worth a test in each repository rather + * than a comment. + * + * The Go constant lives in internal/playback/realtime.go and is pinned from the + * server side by TestPlanInvalidatedDefaultAudioReconciliationReasonMatchesClient; + * the client half is pinned here. Documented in + * docs/architecture/playback-protocol-v3.md §6.1.1. + */ +const GO_CONSTANT_FILE = fileURLToPath( + new URL("../../../internal/playback/realtime.go", import.meta.url), +); + +describe("default audio reconciliation invalidation reason", () => { + it("matches the Go constant that names it on the wire", () => { + const source = readFileSync(GO_CONSTANT_FILE, "utf8"); + const declaration = source.match(/PlanInvalidatedDefaultAudioReconciliation\s*=\s*"([a-z_]+)"/); + expect(declaration).not.toBeNull(); + expect(PLAN_INVALIDATED_DEFAULT_AUDIO_RECONCILIATION).toBe(declaration?.[1]); + }); + + it("is distinct from the copy-safety reason, which is a real route failure", () => { + // Both reasons arrive on the same command and take different replan paths, + // so the constants must not collapse into one another. + expect(PLAN_INVALIDATED_DEFAULT_AUDIO_RECONCILIATION).not.toBe("video_copy_unsafe"); + }); +}); diff --git a/web/src/player/hooks/usePlaybackSession.test.ts b/web/src/player/hooks/usePlaybackSession.test.ts index 0944be55f2..8a5f98bdd3 100644 --- a/web/src/player/hooks/usePlaybackSession.test.ts +++ b/web/src/player/hooks/usePlaybackSession.test.ts @@ -15,6 +15,7 @@ import { VIDEO_CLIENT_FEATURES_V3, } from "../playback-session-wire-v3"; import { markPlaybackIntent } from "../first-frame"; +import { PLAN_INVALIDATED_DEFAULT_AUDIO_RECONCILIATION } from "../realtime-protocol"; import type { PlayerAudioTrack } from "../types"; import type { SubtitleInventoryItemV3 } from "../protocol-v3"; import { usePlaybackSession } from "./usePlaybackSession"; @@ -3460,6 +3461,163 @@ describe("usePlaybackSession server-invalidated plans", () => { unmount(); }); + // Cold-start default-audio reconciliation is not a route failure: the recipe + // plays the wrong audio stream, not the wrong bytes. Replanning as + // `failure_recovery` would fold the healthy route's attempt key into + // `attempted_plan_keys` and force a worse route (or a failure) onto a session + // that was playing fine. + it("replans off a default-audio correction without failure semantics", async () => { + const replanBodies: Array> = []; + vi.stubGlobal( + "fetch", + invalidationFetchMock(replanBodies, { + protocol_version: 3, + server_features: ["playback_plan_v3"], + outcome: "playable", + session_id: "session-1", + playback_plan: fixturePlanV3({ + plan_id: "plan:2222222222222222", + plan_attempt_key: "v3:2222222222222222", + }), + }), + ); + + const { result, unmount } = renderHook( + () => usePlaybackSession("request-1", [], [], 7, 0, false, "auto"), + { wrapper }, + ); + await waitFor(() => expect(result.current.plan?.plan_id).toBe("plan:0123456789abcdef")); + // The viewer is watching, then pauses: the replacement replan must carry the + // position across the correction and leave the transport paused. + act(() => { + result.current.updatePlaybackState(300, true); + result.current.updatePlaybackState(450, false); + }); + + let outcome: boolean | undefined; + await act(async () => { + outcome = await result.current.invalidatePlan( + "plan:0123456789abcdef", + PLAN_INVALIDATED_DEFAULT_AUDIO_RECONCILIATION, + 450, + ); + }); + + expect(outcome).toBe(true); + expect(replanBodies).toHaveLength(1); + const body = replanBodies[0]!; + expect(body.operation).toBe("track_change"); + expect(body.position_seconds).toBe(450); + // No failure payload: the server rejects one on a track_change, and the + // classification would indict a route that never failed. + expect(body).not.toHaveProperty("failure"); + // Nothing failed, so the route that is playing stays eligible — the loop + // guard resets rather than excluding the plan it is replacing. + expect(body.attempted_plan_keys).toEqual([]); + // The echoed selection is the withdrawn plan's own, unchanged: the client + // chose no audio and carries none of the correction's identity. The server + // overwrites it from the decision it persisted for this plan (§6.1.1), so + // the client must not assert anything about the corrected index here. + expect(body.selected_tracks).toMatchObject({ + audio: fixturePlanV3().selected_tracks.audio, + }); + await waitFor(() => expect(result.current.plan?.plan_id).toBe("plan:2222222222222222")); + // Pause state is untouched: the adopted plan reports the transport's own + // paused state rather than autoplaying over the viewer's pause. + expect(result.current.shouldAutoPlay).toBe(false); + + unmount(); + }); + + // The generic path is what a genuine route failure depends on, and it must not + // have widened to swallow every reason while the audio path was carved out. + it("keeps failure_recovery semantics for any other invalidation reason", async () => { + const replanBodies: Array> = []; + vi.stubGlobal( + "fetch", + invalidationFetchMock(replanBodies, { + protocol_version: 3, + server_features: ["playback_plan_v3"], + outcome: "playable", + session_id: "session-1", + playback_plan: fixturePlanV3({ + plan_id: "plan:3333333333333333", + plan_attempt_key: "v3:3333333333333333", + }), + }), + ); + + const { result, unmount } = renderHook( + () => usePlaybackSession("request-1", [], [], 7, 0, false, "auto"), + { wrapper }, + ); + await waitFor(() => expect(result.current.plan?.plan_id).toBe("plan:0123456789abcdef")); + + let outcome: boolean | undefined; + await act(async () => { + outcome = await result.current.invalidatePlan( + "plan:0123456789abcdef", + "video_copy_unsafe", + 200, + ); + }); + + expect(outcome).toBe(true); + expect(replanBodies[0]).toMatchObject({ + operation: "failure_recovery", + failed_plan_id: "plan:0123456789abcdef", + position_seconds: 200, + attempted_plan_keys: ["v3:0123456789abcdef"], + failure: { classification: "video_copy_unsafe" }, + }); + + unmount(); + }); + + // An unrecognized reason is a route verdict this client cannot interpret, so + // it recovers exactly as before rather than guessing at a non-failure path. + it("keeps failure_recovery semantics for an unrecognized reason", async () => { + const replanBodies: Array> = []; + vi.stubGlobal( + "fetch", + invalidationFetchMock(replanBodies, { + protocol_version: 3, + server_features: ["playback_plan_v3"], + outcome: "playable", + session_id: "session-1", + playback_plan: fixturePlanV3({ + plan_id: "plan:4444444444444444", + plan_attempt_key: "v3:4444444444444444", + }), + }), + ); + + const { result, unmount } = renderHook( + () => usePlaybackSession("request-1", [], [], 7, 0, false, "auto"), + { wrapper }, + ); + await waitFor(() => expect(result.current.plan?.plan_id).toBe("plan:0123456789abcdef")); + + let outcome: boolean | undefined; + await act(async () => { + outcome = await result.current.invalidatePlan( + "plan:0123456789abcdef", + "some_reason_from_a_newer_server", + 90, + ); + }); + + expect(outcome).toBe(true); + expect(replanBodies[0]).toMatchObject({ + operation: "failure_recovery", + position_seconds: 90, + attempted_plan_keys: ["v3:0123456789abcdef"], + failure: { classification: "some_reason_from_a_newer_server" }, + }); + + unmount(); + }); + it("does nothing for a plan the session already moved past", async () => { const replanBodies: Array> = []; vi.stubGlobal("fetch", invalidationFetchMock(replanBodies, {})); diff --git a/web/src/player/hooks/usePlaybackSession.ts b/web/src/player/hooks/usePlaybackSession.ts index 7c37b41838..452b1020a5 100644 --- a/web/src/player/hooks/usePlaybackSession.ts +++ b/web/src/player/hooks/usePlaybackSession.ts @@ -46,7 +46,10 @@ import { VIDEO_CLIENT_FEATURES_V3, type ReplanOptions, } from "../playback-session-wire-v3"; -import type { PlaybackInventoryUpdatedPayload } from "../realtime-protocol"; +import { + PLAN_INVALIDATED_DEFAULT_AUDIO_RECONCILIATION, + type PlaybackInventoryUpdatedPayload, +} from "../realtime-protocol"; import type { PlayerAudioTrack, PlayerFileVersion, @@ -458,9 +461,11 @@ export interface UsePlaybackSessionResult extends PlaybackSessionState { /** `failure_recovery` replan after the client could not play the plan. */ recoverFromFailure: (failure: FailureV3, currentPosition: number) => void; /** - * `failure_recovery` replan for a plan the *server* invalidated over the - * realtime `plan_invalidated` command. Resolves to whether a replacement plan - * is now playing; the caller reports that back as the command's result. + * Replans off a plan the *server* invalidated over the realtime + * `plan_invalidated` command: `failure_recovery` for a route failure, or a + * non-failure `track_change` for the default-audio reconciliation reason. + * Resolves to whether a replacement plan is now playing; the caller reports + * that back as the command's result. */ invalidatePlan: (planId: string, reason: string, currentPosition: number) => Promise; /** @@ -2226,12 +2231,25 @@ export function usePlaybackSession( /** * Replans off a plan the server invalidated mid-playback. * - * This is an ordinary `failure_recovery`, deliberately: that operation is - * what folds the current plan's attempt key into `attempted_plan_keys`, so - * the route the server just disqualified is excluded from the replacement - * plan without the client reasoning about deliveries at all. The plan - * revision the adopted plan bumps rebuilds the transport and restores the - * position, exactly as it does after a client-detected failure. + * For a route failure this is an ordinary `failure_recovery`, deliberately: + * that operation is what folds the current plan's attempt key into + * `attempted_plan_keys`, so the route the server just disqualified is + * excluded from the replacement plan without the client reasoning about + * deliveries at all. The plan revision the adopted plan bumps rebuilds the + * transport and restores the position, exactly as it does after a + * client-detected failure. + * + * Cold-start default-audio reconciliation is the exception: nothing failed and + * the route is healthy, the committed recipe simply plays the wrong audio + * stream because a late probe reordered the inventory. Declaring that a + * failure would exclude the very route that is working and push the session + * onto a worse one, so it replans as a `track_change` — the operation that + * already models an audio correction — with no failure payload and no route + * exclusion. The server fills the corrected audio identity into that replan + * from the decision it persisted (§6.1.1 of the v3 protocol), so the client + * only has to carry its live position. Telemetry is unchanged either way: + * `fallback_reason` reports the server's own reason verbatim, which already + * tells the two cases apart. * * A start or replan already in flight is waited out first. The server commits * a replacement plan and starts the copy-safety scan behind it *before* the @@ -2254,6 +2272,13 @@ export function usePlaybackSession( if (plan.plan_id !== planId) return true; const classification = reason.trim().slice(0, 64) || "plan_invalidated"; reportEvent("plan_invalidated", { fallbackReason: classification }); + // Compare the trimmed reason against the server's own string, not the + // truncated classification: a classification can only be a prefix of the + // reconciliation reason if the server sent something longer, which is a + // different reason and must keep failure semantics. + if (reason.trim() === PLAN_INVALIDATED_DEFAULT_AUDIO_RECONCILIATION) { + return replan({ operation: "track_change", positionSeconds: currentPosition }); + } return replan({ operation: "failure_recovery", positionSeconds: currentPosition, diff --git a/web/src/player/realtime-protocol.ts b/web/src/player/realtime-protocol.ts index b67df392e0..87aed5877d 100644 --- a/web/src/player/realtime-protocol.ts +++ b/web/src/player/realtime-protocol.ts @@ -57,6 +57,23 @@ export interface PlaybackPlanInvalidatedPayload { plan_id: string; } +/** + * The `plan_invalidated` reason that withdraws a cold-start plan because a late + * probe moved the server-resolved default audio selection to a different + * stream. The route is healthy and nothing failed — the committed recipe just + * plays the wrong audio — so this is the one reason a client replans as an + * audio correction rather than a failure recovery, which would exclude the + * working route from its own replacement. + * + * Must stay byte-equal to the Go constant + * `playback.PlanInvalidatedDefaultAudioReconciliation` in + * `internal/playback/realtime.go`; the two halves of the wire are checked + * against each other by `defaultAudioReconciliationReason.test.ts` and the Go + * side of that check in `internal/playback/realtime_invalidation_reason_test.go`. + * Documented in `docs/architecture/playback-protocol-v3.md` §6.1.1. + */ +export const PLAN_INVALIDATED_DEFAULT_AUDIO_RECONCILIATION = "default_audio_reconciliation"; + export interface PlaybackRealtimeHelloEnvelope { type: "hello"; session_id: string; From 301fa6a5d401c40ccba7fb4fe366e07716a3a0e3 Mon Sep 17 00:00:00 2001 From: drondeseries Date: Mon, 5 Oct 2026 15:05:19 -0400 Subject: [PATCH 06/16] fix(playback): keep viewer audio choice and stop laundering corrections An explicit audio identity on the answering replan supersedes the pending automatic correction. A correction the server applies restores its own Automatic provenance, which the inbound boundary strips from clients, so a server decision is never persisted as a viewer preference. --- .../api/handlers/playback_reconcile_audio.go | 15 ++++ ...yback_reconcile_audio_invalidation_test.go | 74 ++++++++++++++++++- 2 files changed, 87 insertions(+), 2 deletions(-) diff --git a/internal/api/handlers/playback_reconcile_audio.go b/internal/api/handlers/playback_reconcile_audio.go index 0fadbd7c8a..3093ea6f4d 100644 --- a/internal/api/handlers/playback_reconcile_audio.go +++ b/internal/api/handlers/playback_reconcile_audio.go @@ -700,7 +700,22 @@ func (h *PlaybackHandler) pendingAudioReconciliationReplan(record *playback.Atte if entry == nil { return } + // A viewer who names an audio identity on this replan is making a choice, + // not answering the withdrawal. The correction replays a start-time + // language preference the viewer never re-picked, so an explicit selection + // supersedes it rather than being overwritten. + if req.SelectedTracks.Audio != nil { + slog.Debug("replan keeps the explicit audio choice over a pending correction", + "component", "api", "session", record.SessionID, "generation", entry.Generation) + return + } req.SelectedTracks.Audio = entry.Request.SelectedTracks.Audio + // Restore server-owned provenance for the correction we are applying. The + // inbound boundary strips a client-supplied marker, and the client cannot + // set this one: the automatic correction replays stored intent, so + // persisting its outcome would launder a server decision into a viewer + // preference and steer every later start. + req.Automatic = playback.ReplanAutomaticV3 slog.Debug("replan consumes the pending default audio correction", "component", "api", "session", record.SessionID, "generation", entry.Generation, "audio_index", entry.AudioIndex) diff --git a/internal/api/handlers/playback_reconcile_audio_invalidation_test.go b/internal/api/handlers/playback_reconcile_audio_invalidation_test.go index 2e6c33c6b4..7e04f4289d 100644 --- a/internal/api/handlers/playback_reconcile_audio_invalidation_test.go +++ b/internal/api/handlers/playback_reconcile_audio_invalidation_test.go @@ -233,8 +233,14 @@ func TestReconcileAudioReplanAdoptsThroughPlanInvalidated(t *testing.T) { if applied.ReplanRequestID != "client-replan-1" || applied.FailedPlanID != activePlanID { t.Fatal("the correction must not rewrite the client's own request identity") } - if applied.Automatic != "" { - t.Fatal("a client-issued replan must stay client-issued") + // The correction replays start-time intent the viewer never re-picked, so + // the server restores its own Automatic provenance on the request it + // amends. Without it the correction would be persisted as a viewer + // preference and steer every later start. The client cannot set this: the + // inbound boundary strips whatever arrives on the wire. + if applied.Automatic != playback.ReplanAutomaticV3 { + t.Fatalf("Automatic = %q, want the restored %q so the correction is not stored as a viewer preference", + applied.Automatic, playback.ReplanAutomaticV3) } } @@ -425,6 +431,70 @@ func TestRefusalDoesNotStrandTheSettledCorrection(t *testing.T) { } } +// A viewer who names an audio identity on the answering replan is making a +// choice. The automatic correction replays start-time intent the viewer never +// re-picked, so an explicit selection must supersede it. +func TestExplicitAudioChoiceSupersedesPendingCorrection(t *testing.T) { + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 10) + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil || entry.Request == nil || entry.Request.SelectedTracks.Audio == nil { + t.Fatal("expected a settled correction carrying an audio identity") + } + + explicit := 0 + req := &playback.ReplanRequestV3{ + ProtocolVersion: playback.ProtocolV3, + Operation: playback.ReplanOperationTrackChangeV3, + FailedPlanID: entry.PlanID, + SelectedTracks: playback.SelectedTracksV3{ + Audio: &playback.TrackIdentityV3{Index: &explicit}, + }, + } + f.handler.pendingAudioReconciliationReplan(after, req) + if req.SelectedTracks.Audio == nil || req.SelectedTracks.Audio.Index == nil { + t.Fatal("the explicit choice was erased") + } + if *req.SelectedTracks.Audio.Index != explicit { + t.Fatalf("audio index = %d, want the viewer's %d", *req.SelectedTracks.Audio.Index, explicit) + } + if req.Automatic == playback.ReplanAutomaticV3 { + t.Fatal("a viewer's explicit change must not be marked as an automatic correction") + } +} + +// A correction the server applies on the viewer's behalf is not a viewer +// choice: the Automatic marker must be restored so preference persistence is +// suppressed for it. +func TestPendingCorrectionRestoresAutomaticProvenance(t *testing.T) { + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 10) + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil { + t.Fatal("expected a settled correction") + } + + // The client cannot forge this: the inbound boundary strips it. + req := &playback.ReplanRequestV3{ + ProtocolVersion: playback.ProtocolV3, + Operation: playback.ReplanOperationTrackChangeV3, + FailedPlanID: entry.PlanID, + } + stripClientSuppliedAutomatic(req) + if req.Automatic != "" { + t.Fatalf("inbound marker survived: %q", req.Automatic) + } + + f.handler.pendingAudioReconciliationReplan(after, req) + if req.Automatic != playback.ReplanAutomaticV3 { + t.Fatalf("Automatic = %q, want the restored %q so the correction is not stored as a viewer preference", + req.Automatic, playback.ReplanAutomaticV3) + } + if req.SelectedTracks.Audio == nil { + t.Fatal("the correction was not applied") + } +} + func TestReconcileAudioWithdrawalSkippedWithoutCapability(t *testing.T) { f := newInvalidationFixture(t, []string{playback.FeaturePlaybackPlanV3}, 5) From 533f54294cfd13a9de357ccf228dfc87ef794ed9 Mon Sep 17 00:00:00 2001 From: drondeseries Date: Mon, 5 Oct 2026 15:11:53 -0400 Subject: [PATCH 07/16] fix(playback): treat an echoed audio identity as a reconciliation response The web request builder echoes the plan's own audio on every replan, so the answering request carries the withdrawn selection even when the viewer chose nothing. Presence is not intent: only an identity the withdrawn plan did not select is a viewer choice, and everything else still receives the correction, marked automatic. --- .../api/handlers/playback_reconcile_audio.go | 37 +++++++-- ...yback_reconcile_audio_invalidation_test.go | 80 +++++++++++++++---- 2 files changed, 96 insertions(+), 21 deletions(-) diff --git a/internal/api/handlers/playback_reconcile_audio.go b/internal/api/handlers/playback_reconcile_audio.go index 3093ea6f4d..8ab4c639fa 100644 --- a/internal/api/handlers/playback_reconcile_audio.go +++ b/internal/api/handlers/playback_reconcile_audio.go @@ -700,12 +700,14 @@ func (h *PlaybackHandler) pendingAudioReconciliationReplan(record *playback.Atte if entry == nil { return } - // A viewer who names an audio identity on this replan is making a choice, - // not answering the withdrawal. The correction replays a start-time - // language preference the viewer never re-picked, so an explicit selection - // supersedes it rather than being overwritten. - if req.SelectedTracks.Audio != nil { - slog.Debug("replan keeps the explicit audio choice over a pending correction", + // Presence is not intent. The web request builder echoes the plan's own + // audio identity on every replan (buildReplanRequestV3), so the answering + // request normally carries the pre-reorder selection even though the viewer + // chose nothing. Only a selection that names something OTHER than the + // withdrawn plan's audio is a fresh viewer choice, and it supersedes the + // automatic correction; an inherited echo must not suppress it. + if namesDifferentAudioIdentityV3(req.SelectedTracks.Audio, record.CurrentPlan) { + slog.Debug("replan keeps the viewer's audio choice over a pending correction", "component", "api", "session", record.SessionID, "generation", entry.Generation) return } @@ -721,6 +723,29 @@ func (h *PlaybackHandler) pendingAudioReconciliationReplan(record *playback.Atte "generation", entry.Generation, "audio_index", entry.AudioIndex) } +// namesDifferentAudioIdentityV3 reports whether the replan carries an audio +// identity the withdrawn plan did not select. The web request builder echoes +// the current plan's audio on every replan, so a present identity usually means +// "unchanged", not "chosen": only a genuinely different selection is a viewer +// decision that must survive an automatic correction. A nil identity is +// treated as inherited, because the correction fills it in either way. +func namesDifferentAudioIdentityV3(identity *playback.TrackIdentityV3, plan playback.PlanV3) bool { + if identity == nil { + return false + } + committed := plan.SelectedTracks.Audio + if committed == nil { + return true + } + if identity.ID != committed.ID { + return true + } + if identity.Index == nil || committed.Index == nil { + return false + } + return *identity.Index != *committed.Index +} + // FindPendingAudioReconciliation returns the settled automatic correction a // replan replanning off planID would consume, or nil when there is none. // diff --git a/internal/api/handlers/playback_reconcile_audio_invalidation_test.go b/internal/api/handlers/playback_reconcile_audio_invalidation_test.go index 7e04f4289d..20297e92b1 100644 --- a/internal/api/handlers/playback_reconcile_audio_invalidation_test.go +++ b/internal/api/handlers/playback_reconcile_audio_invalidation_test.go @@ -431,10 +431,23 @@ func TestRefusalDoesNotStrandTheSettledCorrection(t *testing.T) { } } -// A viewer who names an audio identity on the answering replan is making a -// choice. The automatic correction replays start-time intent the viewer never -// re-picked, so an explicit selection must supersede it. -func TestExplicitAudioChoiceSupersedesPendingCorrection(t *testing.T) { +// The web request builder echoes the current plan's audio identity on every +// replan (buildReplanRequestV3), so the answering request normally carries the +// selection the withdrawn plan already had, even though the viewer chose +// nothing. Treating presence as intent skipped reconciliation on exactly that +// path and left the pre-reorder stream playing. +// +// Note the reorder case makes ID comparison a weak discriminator on its own: +// the correction targets the same ordinal the withdrawn plan named, and the +// identity string is derived from that ordinal, so the corrected and echoed +// identities are byte-equal while naming different underlying streams. The +// policy is therefore deliberately asymmetric — an identity the withdrawn plan +// did not select is a viewer choice and wins; anything else (echo, or no +// identity at all) is the reconciliation response, so the correction is +// applied and marked automatic. The cost of being wrong in that direction is a +// redundant viewer pick not being persisted, which is far smaller than writing +// a server decision into a stored preference. +func TestInheritedAudioEchoIsTreatedAsReconciliationResponse(t *testing.T) { f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 10) after := f.reconcile(t) entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) @@ -442,30 +455,67 @@ func TestExplicitAudioChoiceSupersedesPendingCorrection(t *testing.T) { t.Fatal("expected a settled correction carrying an audio identity") } - explicit := 0 + // The real client shape: the builder echoes the plan's own audio. + echoed := *after.CurrentPlan.SelectedTracks.Audio req := &playback.ReplanRequestV3{ ProtocolVersion: playback.ProtocolV3, Operation: playback.ReplanOperationTrackChangeV3, FailedPlanID: entry.PlanID, - SelectedTracks: playback.SelectedTracksV3{ - Audio: &playback.TrackIdentityV3{Index: &explicit}, - }, + SelectedTracks: playback.SelectedTracksV3{Audio: &echoed}, } f.handler.pendingAudioReconciliationReplan(after, req) - if req.SelectedTracks.Audio == nil || req.SelectedTracks.Audio.Index == nil { - t.Fatal("the explicit choice was erased") + + if req.SelectedTracks.Audio == nil { + t.Fatal("the correction was skipped on an inherited echo") + } + if !sameTrackIdentityV3(req.SelectedTracks.Audio, entry.Request.SelectedTracks.Audio) { + t.Fatalf("echoed identity was not replaced by the correction: got %+v, want %+v", + req.SelectedTracks.Audio, entry.Request.SelectedTracks.Audio) + } + if req.Automatic != playback.ReplanAutomaticV3 { + t.Fatalf("Automatic = %q, want %q: an echoed response is still a server correction and must not persist as a viewer preference", + req.Automatic, playback.ReplanAutomaticV3) + } + + // No identity at all is the same case: the correction fills it in. + bare := &playback.ReplanRequestV3{ + ProtocolVersion: playback.ProtocolV3, + Operation: playback.ReplanOperationTrackChangeV3, + FailedPlanID: entry.PlanID, } - if *req.SelectedTracks.Audio.Index != explicit { - t.Fatalf("audio index = %d, want the viewer's %d", *req.SelectedTracks.Audio.Index, explicit) + f.handler.pendingAudioReconciliationReplan(after, bare) + if bare.SelectedTracks.Audio == nil || bare.Automatic != playback.ReplanAutomaticV3 { + t.Fatal("a request with no audio identity must still receive the correction, marked automatic") + } +} + +// A viewer who picks a different track while the withdrawal is in flight is +// making a choice, and it supersedes the automatic correction. +func TestExplicitAudioChoiceSupersedesPendingCorrection(t *testing.T) { + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 10) + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil || entry.Request == nil || entry.Request.SelectedTracks.Audio == nil { + t.Fatal("expected a settled correction carrying an audio identity") + } + + // Something the withdrawn plan did not select. + chosen := &playback.TrackIdentityV3{ID: playback.TrackIDV3(7, "audio", 99)} + req := &playback.ReplanRequestV3{ + ProtocolVersion: playback.ProtocolV3, + Operation: playback.ReplanOperationTrackChangeV3, + FailedPlanID: entry.PlanID, + SelectedTracks: playback.SelectedTracksV3{Audio: chosen}, + } + f.handler.pendingAudioReconciliationReplan(after, req) + if req.SelectedTracks.Audio == nil || req.SelectedTracks.Audio.ID != chosen.ID { + t.Fatalf("the viewer's choice was overwritten: got %+v", req.SelectedTracks.Audio) } if req.Automatic == playback.ReplanAutomaticV3 { t.Fatal("a viewer's explicit change must not be marked as an automatic correction") } } -// A correction the server applies on the viewer's behalf is not a viewer -// choice: the Automatic marker must be restored so preference persistence is -// suppressed for it. func TestPendingCorrectionRestoresAutomaticProvenance(t *testing.T) { f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 10) after := f.reconcile(t) From 65ca7d48d3692727ab3bb57ec24c18ccffab6dc4 Mon Sep 17 00:00:00 2001 From: drondeseries Date: Mon, 5 Oct 2026 15:32:52 -0400 Subject: [PATCH 08/16] feat(playback): advertise reconciliation capability and tag the answer The server now gates the default-audio reconciliation withdrawal on a new client capability, default_audio_reconcile_response_v1, and correlates the client's answer with its pending decision via an answers_plan_invalidation field on the replan. An older client that lacks the token gets no withdrawal and plays the corrected audio on its next start, instead of laundering the withdrawal through failure_recovery. --- .../defaultAudioReconciliationReason.test.ts | 41 ++++++ .../player/hooks/usePlaybackSession.test.ts | 136 +++++++++++++++++- web/src/player/hooks/usePlaybackSession.ts | 17 ++- web/src/player/playback-session-wire-v3.ts | 21 +++ web/src/player/protocol-v3.ts | 24 ++++ 5 files changed, 237 insertions(+), 2 deletions(-) diff --git a/web/src/player/defaultAudioReconciliationReason.test.ts b/web/src/player/defaultAudioReconciliationReason.test.ts index fb436733bb..691e1e1880 100644 --- a/web/src/player/defaultAudioReconciliationReason.test.ts +++ b/web/src/player/defaultAudioReconciliationReason.test.ts @@ -4,6 +4,7 @@ import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; +import { FEATURE_DEFAULT_AUDIO_RECONCILE_RESPONSE_V3 } from "./protocol-v3"; import { PLAN_INVALIDATED_DEFAULT_AUDIO_RECONCILIATION } from "./realtime-protocol"; /** @@ -24,6 +25,11 @@ const GO_CONSTANT_FILE = fileURLToPath( new URL("../../../internal/playback/realtime.go", import.meta.url), ); +/** Feature constants live beside the other v3 tokens, not with the reason. */ +const GO_PROTOCOL_FILE = fileURLToPath( + new URL("../../../internal/playback/protocol_v3.go", import.meta.url), +); + describe("default audio reconciliation invalidation reason", () => { it("matches the Go constant that names it on the wire", () => { const source = readFileSync(GO_CONSTANT_FILE, "utf8"); @@ -38,3 +44,38 @@ describe("default audio reconciliation invalidation reason", () => { expect(PLAN_INVALIDATED_DEFAULT_AUDIO_RECONCILIATION).not.toBe("video_copy_unsafe"); }); }); + +/** + * The capability token that unlocks the withdrawal has the same half-and-half + * problem as the reason: the server reads the token string out of the client's + * `client_features` and uses it as the gate for sending a reconciliation + * withdrawal. A rename on either side means the gate never opens — the client + * waits for a withdrawal that is never sent, and plays the wrong audio until its + * next start. Silent and invisible, so it is pinned here. + * + * The Go constant lives in internal/playback/protocol_v3.go; the server side + * pins its own half of the same pair. + */ +describe("default audio reconciliation client capability", () => { + // The Go half of this mirror is still landing on the parallel lane that owns + // internal/playback/protocol_v3.go. Until its constant exists, skip rather + // than fail the suite — as soon as it does, the test below starts enforcing + // byte-equality on every run. + const protocolSource = readFileSync(GO_PROTOCOL_FILE, "utf8"); + const goDeclaration = protocolSource.match( + /FeatureDefaultAudioReconcileResponseV3\s*=\s*"([a-z0-9_]+)"/, + ); + const itWithGo = goDeclaration ? it : it.skip; + + itWithGo("matches the Go constant the server gates the withdrawal on", () => { + expect(FEATURE_DEFAULT_AUDIO_RECONCILE_RESPONSE_V3).toBe(goDeclaration?.[1]); + }); + + // The withdrawal is gated on this token and NOT on `plan_invalidated_v1`, so + // the two must not collapse into one string: reusing the existing token would + // hand the reconciliation to every older client, which replans it as a failure + // recovery and evicts its own working route. + it("is distinct from the plan-invalidated capability it is related to", () => { + expect(FEATURE_DEFAULT_AUDIO_RECONCILE_RESPONSE_V3).not.toBe("plan_invalidated_v1"); + }); +}); diff --git a/web/src/player/hooks/usePlaybackSession.test.ts b/web/src/player/hooks/usePlaybackSession.test.ts index 8a5f98bdd3..38e7eb901a 100644 --- a/web/src/player/hooks/usePlaybackSession.test.ts +++ b/web/src/player/hooks/usePlaybackSession.test.ts @@ -116,6 +116,11 @@ describe("buildStartRequestV3", () => { // Feature tokens are promises the server enforces, so a surface advertises // only what it implements: the base set alone unless the caller names more. + // `default_audio_reconcile_response_v1` is part of the video surface's promise + // — the server gates the audio reconciliation withdrawal on it, so a start + // that omits it is sent no withdrawal at all and plays the wrong audio until + // its next start. `plan_invalidated_v1` stays alongside it for genuine route + // failures. it("advertises only the surface's own features", () => { expect( buildStartRequestV3({ ...startBase, extraClientFeatures: VIDEO_CLIENT_FEATURES_V3 }) @@ -123,6 +128,7 @@ describe("buildStartRequestV3", () => { ).toEqual([ "playback_plan_v3", "plan_invalidated_v1", + "default_audio_reconcile_response_v1", "source_committed_event_v1", "inventory_updated_event_v1", ]); @@ -245,7 +251,8 @@ describe("buildReplanRequestV3", () => { // A replan that sends `client_features` replaces the negotiated list, so a // replan which advertised less than the start did would silently withdraw the - // promise the server gates the invalidation command on. + // promise the server gates the invalidation command on — including the + // reconciliation capability. it("re-advertises the same features a start negotiated", () => { expect( buildReplanRequestV3({ @@ -256,11 +263,33 @@ describe("buildReplanRequestV3", () => { ).toEqual([ "playback_plan_v3", "plan_invalidated_v1", + "default_audio_reconcile_response_v1", "source_committed_event_v1", "inventory_updated_event_v1", ]); }); + // The correlation field asserts "this replan is the answer to that withdrawal", + // so it travels verbatim and only when the caller has one to name. + it("carries the answered invalidation reason when the replan has one", () => { + const body = buildReplanRequestV3({ + ...replanBase, + operation: "track_change", + answersPlanInvalidation: "default_audio_reconciliation", + }); + + expect(body.answers_plan_invalidation).toBe("default_audio_reconciliation"); + }); + + it("omits the answered invalidation reason from an ordinary replan", () => { + expect(buildReplanRequestV3({ ...replanBase, operation: "quality_change" })).not.toHaveProperty( + "answers_plan_invalidation", + ); + expect( + buildReplanRequestV3({ ...replanBase, operation: "failure_recovery" }), + ).not.toHaveProperty("answers_plan_invalidation"); + }); + it("re-arms the server's auto fallback when the viewer selects Auto", () => { expect( buildReplanRequestV3({ ...replanBase, operation: "failure_recovery", autoFallback: true }), @@ -3508,6 +3537,10 @@ describe("usePlaybackSession server-invalidated plans", () => { const body = replanBodies[0]!; expect(body.operation).toBe("track_change"); expect(body.position_seconds).toBe(450); + // The reason goes back on the wire verbatim so the server can match this + // replan against the decision it is still holding, instead of inferring the + // pairing from the operation and the absent failure. + expect(body.answers_plan_invalidation).toBe(PLAN_INVALIDATED_DEFAULT_AUDIO_RECONCILIATION); // No failure payload: the server rejects one on a track_change, and the // classification would indict a route that never failed. expect(body).not.toHaveProperty("failure"); @@ -3570,6 +3603,10 @@ describe("usePlaybackSession server-invalidated plans", () => { attempted_plan_keys: ["v3:0123456789abcdef"], failure: { classification: "video_copy_unsafe" }, }); + // A route failure carries its own evidence in `failure`; echoing a reason + // here would claim the server asked for a non-failure correction it never + // sent, and the server correlates on it. + expect(replanBodies[0]).not.toHaveProperty("answers_plan_invalidation"); unmount(); }); @@ -3618,6 +3655,103 @@ describe("usePlaybackSession server-invalidated plans", () => { unmount(); }); + // The correlation field is scoped to the reconciliation path because it is a + // claim about a pending server decision. A quality change, a subtitle change + // and an inventory refresh are viewer intents with no withdrawal to answer, + // and one of them happens to share the reconciliation's `track_change` + // operation — which is exactly why the field cannot be inferred from the + // operation alone. + it("sends no answered-invalidation reason on viewer-initiated replans", async () => { + const replanBodies: Array> = []; + vi.stubGlobal( + "fetch", + invalidationFetchMock(replanBodies, { + protocol_version: 3, + server_features: ["playback_plan_v3"], + outcome: "playable", + session_id: "session-1", + playback_plan: fixturePlanV3(), + }), + ); + + const { result, unmount } = renderHook( + () => usePlaybackSession("request-1", [], [], 7, 0, false, "original"), + { wrapper }, + ); + await waitFor(() => expect(result.current.plan?.plan_id).toBe("plan:0123456789abcdef")); + + await act(async () => { + result.current.changeQuality("720p", 120); + }); + await waitFor(() => expect(replanBodies).toHaveLength(1)); + + await act(async () => { + result.current.changeSubtitleTrack(0, 200); + }); + await waitFor(() => expect(replanBodies).toHaveLength(2)); + + await act(async () => { + await result.current.refreshSubtitles(300); + }); + await waitFor(() => expect(replanBodies).toHaveLength(3)); + + expect(replanBodies.map((body) => body.operation)).toEqual([ + "quality_change", + "track_change", + "track_change", + ]); + for (const body of replanBodies) { + expect(body).not.toHaveProperty("answers_plan_invalidation"); + } + + unmount(); + }); + + // The server decides whether to send the reconciliation withdrawal by reading + // the advertised features, so both the capability and the promise it belongs + // to have to be on every request that can carry one — start and replan alike, + // because a replan replaces the negotiated list. + it("advertises the reconciliation capability on start and replan", async () => { + const replanBodies: Array> = []; + vi.stubGlobal( + "fetch", + invalidationFetchMock(replanBodies, { + protocol_version: 3, + server_features: ["playback_plan_v3"], + outcome: "playable", + session_id: "session-1", + playback_plan: fixturePlanV3({ plan_id: "plan:2222222222222222" }), + }), + ); + + const { result, unmount } = renderHook( + () => usePlaybackSession("request-1", [], [], 7, 0, false, "auto"), + { wrapper }, + ); + await waitFor(() => expect(result.current.plan?.plan_id).toBe("plan:0123456789abcdef")); + + await act(async () => { + await result.current.invalidatePlan( + "plan:0123456789abcdef", + PLAN_INVALIDATED_DEFAULT_AUDIO_RECONCILIATION, + 450, + ); + }); + expect(replanBodies).toHaveLength(1); + + const startCall = vi + .mocked(fetch) + .mock.calls.find(([url]) => String(url).endsWith("/playback/start")); + const startBody = JSON.parse(String(startCall?.[1]?.body)) as { client_features: string[] }; + for (const features of [startBody.client_features, replanBodies[0]!.client_features]) { + expect(features).toContain("default_audio_reconcile_response_v1"); + // Genuine route-failure invalidations still depend on the older token. + expect(features).toContain("plan_invalidated_v1"); + } + + unmount(); + }); + it("does nothing for a plan the session already moved past", async () => { const replanBodies: Array> = []; vi.stubGlobal("fetch", invalidationFetchMock(replanBodies, {})); diff --git a/web/src/player/hooks/usePlaybackSession.ts b/web/src/player/hooks/usePlaybackSession.ts index 452b1020a5..107dbf511e 100644 --- a/web/src/player/hooks/usePlaybackSession.ts +++ b/web/src/player/hooks/usePlaybackSession.ts @@ -2251,6 +2251,13 @@ export function usePlaybackSession( * `fallback_reason` reports the server's own reason verbatim, which already * tells the two cases apart. * + * That replan also echoes the reason back as `answers_plan_invalidation`, so + * the server can match it against the decision it is holding rather than + * infer the pairing from the operation and the missing failure. It is the only + * replan that does so: the field asserts "this is the answer to that + * withdrawal", and every other replan — including the failure recovery below — + * has no withdrawal to answer. + * * A start or replan already in flight is waited out first. The server commits * a replacement plan and starts the copy-safety scan behind it *before* the * response reaches the client, so an invalidation can name a plan this client @@ -2277,7 +2284,15 @@ export function usePlaybackSession( // reconciliation reason if the server sent something longer, which is a // different reason and must keep failure semantics. if (reason.trim() === PLAN_INVALIDATED_DEFAULT_AUDIO_RECONCILIATION) { - return replan({ operation: "track_change", positionSeconds: currentPosition }); + // The reason goes back on the wire verbatim, not trimmed to the + // classification: it is how the server tells that this replan *is* the + // answer to the decision it persisted, instead of falling back to a + // weaker heuristic on the operation and the absence of a failure. + return replan({ + operation: "track_change", + positionSeconds: currentPosition, + answersPlanInvalidation: reason.trim(), + }); } return replan({ operation: "failure_recovery", diff --git a/web/src/player/playback-session-wire-v3.ts b/web/src/player/playback-session-wire-v3.ts index c6a99d592c..833b743057 100644 --- a/web/src/player/playback-session-wire-v3.ts +++ b/web/src/player/playback-session-wire-v3.ts @@ -1,4 +1,5 @@ import { + FEATURE_DEFAULT_AUDIO_RECONCILE_RESPONSE_V3, FEATURE_INVENTORY_UPDATED_V3, FEATURE_PLAN_INVALIDATED_V3, FEATURE_PLAYBACK_PLAN_V3, @@ -37,9 +38,16 @@ const BASE_CLIENT_FEATURES_V3 = [FEATURE_PLAYBACK_PLAN_V3]; * server names, follows the realtime `source_committed` event to re-key its * version menu to the streamed release, and folds the realtime * `inventory_updated` push into its track menus. + * + * `plan_invalidated_v1` stays on its own promise and is not sufficient for the + * audio reconciliation: the server gates that withdrawal on + * `default_audio_reconcile_response_v1`, which this surface also advertises + * because it answers the withdrawal as a non-failure `track_change` and names + * the reason on the replan. */ export const VIDEO_CLIENT_FEATURES_V3 = [ FEATURE_PLAN_INVALIDATED_V3, + FEATURE_DEFAULT_AUDIO_RECONCILE_RESPONSE_V3, FEATURE_SOURCE_COMMITTED_V3, FEATURE_INVENTORY_UPDATED_V3, ]; @@ -72,6 +80,13 @@ export interface ReplanOptions { * dead-source recovery would refuse to rotate while the menu shows Auto. */ autoFallback?: boolean; + /** + * The `reason` string of the `plan_invalidated` command this replan answers, + * verbatim. Set only by the default-audio reconciliation path in + * `invalidatePlan`; every other replan omits it, so the server can correlate + * this replan with the decision it is holding without a heuristic. + */ + answersPlanInvalidation?: string; } export interface StartRequestInput { @@ -203,6 +218,12 @@ export function buildReplanRequestV3(input: ReplanRequestInput): ReplanRequestV3 position_seconds: clampPosition(input.positionSeconds), metered: input.metered, ...(input.autoFallback !== undefined ? { auto_fallback: input.autoFallback } : {}), + // Omitted rather than sent empty: an absent field means "this replan is not + // an answer to any withdrawal", which is what every non-reconciliation + // replan actually is. + ...(input.answersPlanInvalidation + ? { answers_plan_invalidation: input.answersPlanInvalidation } + : {}), selected_tracks: selectedTracks, client_capabilities: input.clientCapabilities, client_playback_context: input.clientPlaybackContext, diff --git a/web/src/player/protocol-v3.ts b/web/src/player/protocol-v3.ts index b52eedb315..b2f8d8b729 100644 --- a/web/src/player/protocol-v3.ts +++ b/web/src/player/protocol-v3.ts @@ -137,6 +137,20 @@ export const FEATURE_OUTPUT_CHANGE_V3 = "output_change_v1"; */ export const FEATURE_PLAN_INVALIDATED_V3 = "plan_invalidated_v1"; +/** + * The client can answer the `default_audio_reconciliation` withdrawal without + * failure semantics, and can name that withdrawal as the reason for a replan. + * + * It is a separate promise from {@link FEATURE_PLAN_INVALIDATED_V3} on purpose. + * The server gates the *audio reconciliation* withdrawal on this token rather + * than on `plan_invalidated_v1`, because a client that has never heard of that + * reason would replay it as a `failure_recovery` and exclude the very route + * that is working. A client without this token is therefore sent no withdrawal + * at all and simply plays the corrected audio on its next start, which is + * degraded but healthy — strictly better than the failure path. + */ +export const FEATURE_DEFAULT_AUDIO_RECONCILE_RESPONSE_V3 = "default_audio_reconcile_response_v1"; + /** * The client handles the realtime `source_committed` event: it re-keys its * version menu to the effective source the transport committed to and adopts @@ -384,6 +398,16 @@ export interface ReplanRequestV3 { * start-time intent unchanged. Never authorizes a healthy mid-play switch. */ auto_fallback?: boolean; + /** + * The `reason` of the `plan_invalidated` command this replan is answering, + * verbatim. Set only by a client that advertises + * {@link FEATURE_DEFAULT_AUDIO_RECONCILE_RESPONSE_V3} and only on the + * reconciliation replan, so the server can correlate the response with the + * decision it is still holding rather than infer it. Every other replan omits + * it: a claim of answering a withdrawal that never happened would be worse + * than silence. + */ + answers_plan_invalidation?: string; bandwidth_estimate_kbps?: number; bandwidth_cap_kbps?: number; selected_tracks: SelectedTracksV3; From 30a2292dc35c6ed1552f5a50001534b6e0b8f902 Mon Sep 17 00:00:00 2001 From: drondeseries Date: Mon, 5 Oct 2026 16:14:42 -0400 Subject: [PATCH 09/16] feat(playback): gate the audio withdrawal on the response capability A client that would replay the withdrawal as failure_recovery excludes a healthy route from its own replacement, so the withdrawal is gated on a capability that means the client answers it correctly; older clients deliberately receive none and get the correction on their next start. The answering replan now correlates through answers_plan_invalidation instead of an inferred request shape, with the identity heuristic kept as the fallback. --- docs/architecture/playback-protocol-v3.md | 44 ++- .../api/handlers/playback_reconcile_audio.go | 60 +++- ...yback_reconcile_audio_invalidation_test.go | 81 ++++- .../playback_reconcile_integration_test.go | 281 ++++++++++++++++++ internal/playback/protocol_store_v3.go | 7 + internal/playback/protocol_v3.go | 44 ++- internal/playback/protocol_v3_test.go | 1 + 7 files changed, 491 insertions(+), 27 deletions(-) create mode 100644 internal/api/handlers/playback_reconcile_integration_test.go diff --git a/docs/architecture/playback-protocol-v3.md b/docs/architecture/playback-protocol-v3.md index a806e9e99b..84e5524931 100644 --- a/docs/architecture/playback-protocol-v3.md +++ b/docs/architecture/playback-protocol-v3.md @@ -1266,8 +1266,10 @@ worse route, or onto a terminal. A client therefore replans off a `failure` payload (the server rejects one on that operation anyway) and no route exclusion, carrying only its live position. It is the operation the server's own automatic replan already uses for this correction, and a `track_change` that -changes nothing still returns a fresh plan. `client_features` is unchanged: this -reuses the existing operation rather than adding one. +changes nothing still returns a fresh plan. The one advertisement change is the +withdrawal gate below: `client_features` gains `default_audio_reconcile_response_v1`, +because this reason cannot be answered correctly under `plan_invalidated_v1` +alone. No new operation is introduced. The two sides name the reason independently — the server from `playback.PlanInvalidatedDefaultAudioReconciliation`, the browser from its own @@ -1282,12 +1284,38 @@ The correction is one-shot rather than an ongoing override: it is keyed to the withdrawn plan, and the plan the replan commits becomes current, so the client's next replan is its own. -Four rules bound it: - -- **The feature still gates delivery.** A client that never advertised - `plan_invalidated_v1` gets no event. Its decision is still recorded, and the - correction lands on its next start or reconnect — which plans against the - verified inventory anyway, because the inventory is what moved. +Six rules bound it: + +- **Two features gate delivery, and they are not the same promise.** The + withdrawal needs `default_audio_reconcile_response_v1` **and** + `plan_invalidated_v1`. The second alone is not enough: a client that + advertised only `plan_invalidated_v1` has never heard of this reason, so it + replays the withdrawal as a `failure_recovery`, folds the withdrawn + `plan_attempt_key` into `attempted_plan_keys`, and excludes the healthy route + it is currently playing from its own replacement. Sending it the withdrawal + anyway would trade a wrong-language session for a broken one. Its decision is + still recorded, and the correction lands on its next start or reconnect — + which plans against the verified inventory anyway, because the inventory is + what moved. `plan_invalidated_v1` alone still gates every other withdrawal + reason (§6.1). +- **`answers_plan_invalidation` is the authoritative correlation.** The + withdrawal's `reason` is recorded on the settled ledger entry, and the replan + that answers it echoes the same string back. That echo — not the request + shape — is what tells the server this replan is a reconciliation response. + The field is deliberately **not** stripped at the ingress boundary, unlike + `Automatic`: forging it can at worst make the server apply a correction the + server itself decided and announced, whereas forging `Automatic` would + impersonate server reconciliation and suppress the viewer's preference + persistence. An echo is a required input, never an authority: a mismatching + echo falls through to the identity heuristic below. +- **The echoed audio identity is a fallback discriminator, not the contract.** + Without an echo, a selection that names something other than the withdrawn + plan's audio is read as a fresh viewer choice that supersedes the + correction — correct for the usual shape, because the request builder echoes + the plan's own audio on every replan. It cannot tell a deliberate re-pick of + the track the viewer already had from that inherited echo when the probe + reorders the inventory and the correction targets the same ordinal, which is + exactly why the echo exists. - **A byte-equal no-op still settles the generation.** Heartbeats and duplicate evidence writes converge on the stored decision instead of re-evaluating. - **Every automatic-selection gate still runs before the decision**: a stale or diff --git a/internal/api/handlers/playback_reconcile_audio.go b/internal/api/handlers/playback_reconcile_audio.go index 8ab4c639fa..b819fe9a2b 100644 --- a/internal/api/handlers/playback_reconcile_audio.go +++ b/internal/api/handlers/playback_reconcile_audio.go @@ -295,6 +295,7 @@ func (h *PlaybackHandler) settleAudioReconcileInvalidation(ctx context.Context, PlanID: record.CurrentPlanID, Request: &storedRequest, RequestDigest: digest, + Reason: playback.PlanInvalidatedDefaultAudioReconciliation, }) // Only the writer that actually settled this (generation, session) emits. // A replayed generation returns the stored decision, already delivered (or @@ -680,6 +681,27 @@ func (h *PlaybackHandler) sessionNegotiatedPlanInvalidation(ctx context.Context, return playback.HasFeatureV3(record.NormalizedRequest.ClientFeatures, playback.FeaturePlanInvalidatedV3) } +// sessionNegotiatedDefaultAudioReconcileResponse reports whether this +// attempt's client advertised the response-side capability for the +// default-audio reconciliation withdrawal. It is a stricter gate than +// sessionNegotiatedPlanInvalidation: a client that advertises only +// plan_invalidated_v1 handles every withdrawal as a route failure and folds +// the invalidated attempt key into attempted_plan_keys, which would exclude +// a perfectly healthy route from its own replacement. Such a client gets NO +// withdrawal; its decision is still recorded, and the correction lands on +// its next start or reconnect, which plans against the verified inventory +// anyway. +func (h *PlaybackHandler) sessionNegotiatedDefaultAudioReconcileResponse(ctx context.Context, sessionID string) bool { + if h == nil || h.PlanStoreV3 == nil { + return false + } + record, err := h.PlanStoreV3.GetAttempt(ctx, sessionID) + if err != nil || record == nil { + return false + } + return playback.HasFeatureV3(record.NormalizedRequest.ClientFeatures, playback.FeatureDefaultAudioReconcileResponseV3) +} + // pendingAudioReconciliationReplan consumes the same pending decision the // withdrawal named: the client's replan, whatever body it carries, replans off // the withdrawn plan, so it has to select the corrected audio stream or the @@ -706,7 +728,34 @@ func (h *PlaybackHandler) pendingAudioReconciliationReplan(record *playback.Atte // chose nothing. Only a selection that names something OTHER than the // withdrawn plan's audio is a fresh viewer choice, and it supersedes the // automatic correction; an inherited echo must not suppress it. - if namesDifferentAudioIdentityV3(req.SelectedTracks.Audio, record.CurrentPlan) { + // + // Note: the ingress boundary (replanPlaybackApplicationV3) strips a + // client-supplied Automatic marker because a forged one would impersonate + // server reconciliation and suppress the viewer's preference persistence. + // AnswersPlanInvalidation is NOT trust-sensitive in that way: at worst it + // can cause the server to apply a correction the SERVER itself decided to + // apply and already announced, so the boundary deliberately does not + // strip it. + // + // The answers_plan_invalidation echo is the authoritative discriminator: + // when it matches the reason recorded with the pending entry, this request + // IS the reconciliation response — the client read the withdrawal, chose + // "apply the server-decided correction" as the answer — and we apply the + // correction regardless of the echoed audio identity. That matters in the + // reorder case, where the correction targets the same ordinal the withdrawn + // plan named and the echoed identity is derived from that ordinal, so a + // deliberate re-pick of the track the viewer already had is byte-identical + // to the builder's echo and the identity heuristic alone would either + // override a real viewer choice or suppress a real echo-lane correction. + // + // The identity heuristic stays as the fallback for clients that have not + // adopted answers_plan_invalidation yet: it is a weaker discriminator (it + // cannot tell a deliberate re-pick from an inherited echo in the reorder + // case, as above), but it preserves correct behavior for the common echo + // case those clients produce. + isReconciliationAnswer := strings.TrimSpace(req.AnswersPlanInvalidation) != "" && + strings.TrimSpace(req.AnswersPlanInvalidation) == strings.TrimSpace(entry.Reason) + if !isReconciliationAnswer && namesDifferentAudioIdentityV3(req.SelectedTracks.Audio, record.CurrentPlan) { slog.Debug("replan keeps the viewer's audio choice over a pending correction", "component", "api", "session", record.SessionID, "generation", entry.Generation) return @@ -806,7 +855,14 @@ func (h *PlaybackHandler) announceAudioReconcileInvalidation(ctx context.Context if planID == "" { return } - if !h.sessionNegotiatedPlanInvalidation(ctx, live.ID) { + if !h.sessionNegotiatedDefaultAudioReconcileResponse(ctx, live.ID) || + !h.sessionNegotiatedPlanInvalidation(ctx, live.ID) { + // The withdrawal is gated on the NEW response capability, not merely + // on plan_invalidated_v1: an older client that advertises only the + // latter treats every withdrawal as a route failure and folds the + // invalidated key into attempted_plan_keys, excluding a healthy + // route from its own replacement. Such a client gets no withdrawal; + // the correction lands on its next start/reconnect instead. slog.DebugContext(ctx, "default audio correction recorded without a withdrawal", "component", "api", "session", live.ID, "generation", stored.Generation, "audio_index", *stored.AudioIndex) diff --git a/internal/api/handlers/playback_reconcile_audio_invalidation_test.go b/internal/api/handlers/playback_reconcile_audio_invalidation_test.go index 20297e92b1..688d0d637b 100644 --- a/internal/api/handlers/playback_reconcile_audio_invalidation_test.go +++ b/internal/api/handlers/playback_reconcile_audio_invalidation_test.go @@ -137,7 +137,7 @@ func (f *invalidationFixture) reconcile(t *testing.T) *playback.AttemptRecordV3 // recipe of its own. The client's own replan then consumes the same decision // and lands on the corrected audio index with its position preserved. func TestReconcileAudioReplanAdoptsThroughPlanInvalidated(t *testing.T) { - f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 321.5) + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 321.5) activePlanID := f.record.CurrentPlanID activeStreamURL := f.record.CurrentPlan.Stream.URL @@ -248,7 +248,7 @@ func TestReconcileAudioReplanAdoptsThroughPlanInvalidated(t *testing.T) { // second probe write and a second heartbeat on a settled generation emit // nothing, and the stored canonical request replays verbatim. func TestReconcileAudioNoDoubleCorrection(t *testing.T) { - f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 12) + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 12) after := f.reconcile(t) entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) @@ -289,7 +289,7 @@ func TestReconcileAudioNoDoubleCorrection(t *testing.T) { // them) must not both proceed. The second is refused and recorded, and only one // replan is ever issued. func TestReconcileAudioImmutableIdentity(t *testing.T) { - f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 10) + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 10) after := f.reconcile(t) entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) @@ -359,7 +359,7 @@ func TestReconcileAudioImmutableIdentity(t *testing.T) { // reads start.AudioTrackID/AudioTrackIndex, so a correction applied afterwards // never reaches the plan or the executor's audio map. func TestPendingAudioCorrectionReachesTheStartSelection(t *testing.T) { - f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 10) + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 10) after := f.reconcile(t) entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) @@ -398,7 +398,7 @@ func TestPendingAudioCorrectionReachesTheStartSelection(t *testing.T) { // winning evaluator already settled and announced: two concurrent evaluations // that captured different playheads would otherwise strand the correction. func TestRefusalDoesNotStrandTheSettledCorrection(t *testing.T) { - f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 10) + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 10) after := f.reconcile(t) entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) @@ -448,7 +448,7 @@ func TestRefusalDoesNotStrandTheSettledCorrection(t *testing.T) { // redundant viewer pick not being persisted, which is far smaller than writing // a server decision into a stored preference. func TestInheritedAudioEchoIsTreatedAsReconciliationResponse(t *testing.T) { - f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 10) + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 10) after := f.reconcile(t) entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) if entry == nil || entry.Request == nil || entry.Request.SelectedTracks.Audio == nil { @@ -492,7 +492,7 @@ func TestInheritedAudioEchoIsTreatedAsReconciliationResponse(t *testing.T) { // A viewer who picks a different track while the withdrawal is in flight is // making a choice, and it supersedes the automatic correction. func TestExplicitAudioChoiceSupersedesPendingCorrection(t *testing.T) { - f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 10) + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 10) after := f.reconcile(t) entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) if entry == nil || entry.Request == nil || entry.Request.SelectedTracks.Audio == nil { @@ -517,7 +517,7 @@ func TestExplicitAudioChoiceSupersedesPendingCorrection(t *testing.T) { } func TestPendingCorrectionRestoresAutomaticProvenance(t *testing.T) { - f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 10) + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 10) after := f.reconcile(t) entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) if entry == nil { @@ -565,13 +565,76 @@ func TestReconcileAudioWithdrawalSkippedWithoutCapability(t *testing.T) { } } +// TestReconcileAudioWithdrawalSkippedWithoutReconcileResponseCapability is the +// compatibility half of the gate: a client that advertised +// plan_invalidated_v1 but NOT default_audio_reconcile_response_v1 replays any +// withdrawal it receives as a failure_recovery — folding the withdrawn attempt +// key into attempted_plan_keys and excluding the healthy route that is +// currently playing from its own replacement. The withdrawal is therefore +// withheld from it specifically, while the decision stays recorded so the +// correction lands on its next start or reconnect. +func TestReconcileAudioWithdrawalSkippedWithoutReconcileResponseCapability(t *testing.T) { + f := newInvalidationFixture(t, []string{ + playback.FeaturePlaybackPlanV3, + playback.FeaturePlanInvalidatedV3, + }, 5) + + after := f.reconcile(t) + + if got := len(f.conn.commands()); got != 0 { + t.Fatalf("plan_invalidated pushes = %d without default_audio_reconcile_response_v1, want 0", got) + } + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil { + t.Fatal("the decision must still be recorded without the reconcile-response capability") + } + if entry.Decision != playback.AudioReconcileInvalidated || entry.Request == nil { + t.Fatalf("decision = %+v, want a recorded invalidation with its canonical request", entry) + } + if entry.Reason != playback.PlanInvalidatedDefaultAudioReconciliation { + t.Fatalf("entry reason = %q, want %q", entry.Reason, playback.PlanInvalidatedDefaultAudioReconciliation) + } + if after.CurrentPlanID != f.record.CurrentPlanID { + t.Fatal("an unnegotiated client must not get its plan replaced server-side either") + } +} + +// TestSettledReconcileEntryRecordsTheWithdrawalReason pins the correlation +// half: the settled entry carries the exact reason string the withdrawal was +// announced with, because that is what an answering replan echoes back in +// answers_plan_invalidation and what pendingAudioReconciliationReplan matches +// against. +func TestSettledReconcileEntryRecordsTheWithdrawalReason(t *testing.T) { + f := newInvalidationFixture(t, []string{ + playback.FeaturePlaybackPlanV3, + playback.FeaturePlanInvalidatedV3, + playback.FeatureDefaultAudioReconcileResponseV3, + }, 9) + + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil { + t.Fatal("expected a settled entry") + } + if entry.Reason != playback.PlanInvalidatedDefaultAudioReconciliation { + t.Fatalf("entry reason = %q, want %q", entry.Reason, playback.PlanInvalidatedDefaultAudioReconciliation) + } + // The stored canonical request must NOT echo the reason back: it is a + // server-built automatic replan, not the client's answer to a withdrawal, + // and its bytes are the idempotency identity. + if entry.Request.AnswersPlanInvalidation != "" { + t.Fatalf("stored automatic replan answers_plan_invalidation = %q, want empty", + entry.Request.AnswersPlanInvalidation) + } +} + // TestReconcileAudioWithdrawalReachesOnlyTheOwnerLane pins the multi-node // contract: the withdrawal travels the session's own hub lane, so it reaches // the replica that owns that session's control socket and nothing else. The // ledger, not a cross-replica RPC, is what keeps another replica from // emitting a duplicate. func TestReconcileAudioWithdrawalReachesOnlyTheOwnerLane(t *testing.T) { - f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 7) + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 7) other := &reconcileInvalidationConn{} otherRegistration := f.handler.RealtimeHub.Register("22222222-2222-2222-2222-222222222222", other) if otherRegistration == nil { diff --git a/internal/api/handlers/playback_reconcile_integration_test.go b/internal/api/handlers/playback_reconcile_integration_test.go new file mode 100644 index 0000000000..9adbefb554 --- /dev/null +++ b/internal/api/handlers/playback_reconcile_integration_test.go @@ -0,0 +1,281 @@ +package handlers + +import ( + "context" + "encoding/json" + "net/http/httptest" + "strings" + "testing" + + "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/playback" +) + +// TestDefaultAudioReconcileReplanAppliesCorrectionThroughRealReplanPath is +// the Blocker-C integration test: it drives the real replan application +// path (the same seam HandleReplanPlaybackV3 / ReplanPlaybackV2 call) with +// a request shaped exactly like the real web client produces on a +// reconciliation answer — operation track_change, NO failure payload, the +// plan's own audio identity echoed unchanged, and answers_plan_invalidation +// carrying the withdrawal reason. +// +// Unit tests on pendingAudioReconciliationReplan have repeatedly missed +// defects in this change because they skip the application seam; this test +// asserts the end-to-end outcome of the real path instead. +func TestDefaultAudioReconcileReplanAppliesCorrectionThroughRealReplanPath(t *testing.T) { + // Start from the canonical handler fixture (a 1080p/h264/aac movie in + // mp4) and add the Spanish second audio track whose index the + // reconciliation correction targets. + file := v3HandlerFixtureFile(t) + file.EpisodeID = "episode-1" + file.AudioTracks = append(file.AudioTracks, models.AudioTrack{Codec: "aac", Channels: 2, Layout: "stereo", Language: "spa"}) + + manager := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(manager, testPlaybackFileResolver{file: file}) + handler.JWTSecret = "test-secret" + stubCopySeekAnchorV3(handler) + handler.PlaybackConfig = playbackTestConfig("", t.TempDir()) + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"allow_4k_transcode": "true"}} + handler.ItemAccess = allowAllPlaybackItemAccess{} + handler.EpisodeLookup = testEpisodeLookup{episode: &models.Episode{ContentID: "episode-1", SeriesID: "series-1"}} + store := newPlaybackTestStore(t) + handler.StoreProvider = testUserStoreProvider{store: store} + handler.AdminStore = noopPlaybackAdminStore{} + + // Start for real: the committed plan plays track 0 (eng). The viewer's + // language preference is Spanish, so reconciliation will want to move + // the selection to index 1. + start := v3HandlerStartRequest() + start.ClientFeatures = append([]string(nil), + playback.FeaturePlaybackPlanV3, + playback.FeaturePlanInvalidatedV3, + playback.FeatureDefaultAudioReconcileResponseV3, + ) + start.ClientPlaybackContext.Deliveries[playback.DeliveryClassOriginalHTTPV3] = playback.DeliveryCapabilityV3{Enabled: true, SupportedOnDevice: true} + start.ClientPlaybackContext.Deliveries[playback.DeliveryClassProgressiveV3] = playback.DeliveryCapabilityV3{Enabled: true, SupportedOnDevice: true} + start.ClientFeatures = append(start.ClientFeatures, playback.FeatureClientVideoTransforms) + delivery := start.ClientPlaybackContext.Deliveries[playback.DeliveryClassOriginalHTTPV3] + delivery.Transformations = []playback.TransformationV3{{Name: playback.ClientDV7ToDV81V3, Executor: playback.ExecutorClientV3, RecipeVersion: playback.ClientDVTransformVersionV3}} + start.ClientPlaybackContext.Deliveries[playback.DeliveryClassOriginalHTTPV3] = delivery + startRR := httptest.NewRecorder() + handler.HandleStartPlayback(startRR, httptest.NewRequest("POST", "/api/v1/playback/start", + strings.NewReader(marshalV3StartRequest(t, start))).WithContext(newAuthorizedPlaybackContext())) + if startRR.Code != 201 { + t.Fatalf("start status = %d, body = %s", startRR.Code, startRR.Body.String()) + } + var started playback.DecisionResponseV3 + if err := json.Unmarshal(startRR.Body.Bytes(), &started); err != nil { + t.Fatal(err) + } + if started.PlaybackPlan == nil { + t.Fatal("no plan from start") + } + sessionID := started.SessionID + session, err := manager.GetSession(sessionID) + if err != nil { + t.Fatal(err) + } + + // Seed the attempt row as a reconciliation-eligible attempt: an auto + // default-audio selection that assumed the pre-probe inventory, with the + // ledger carrying the settled correction the withdrawal announced. + record, err := handler.PlanStoreV3.GetAttempt(context.Background(), sessionID) + if err != nil || record == nil { + t.Fatalf("get attempt: %v", err) + } + record.SelectionOrigin = SelectionOriginAuto + record.PreferredAudioLanguage = "spa" + record.SelectedAudioSignature = playback.AudioTrackSignatureFromTrack(models.AudioTrack{Codec: "aac", Channels: 2, Layout: "stereo", Language: "spa"}) + record.NormalizedRequest.ClientFeatures = []string{ + playback.FeaturePlaybackPlanV3, + playback.FeaturePlanInvalidatedV3, + playback.FeatureDefaultAudioReconcileResponseV3, + } + correctedIndex := 1 + correctedReq := playback.ReplanRequestV3{ + ProtocolVersion: playback.ProtocolV3, + Operation: playback.ReplanOperationTrackChangeV3, + Automatic: playback.ReplanAutomaticV3, + PlaybackAttemptID: record.PlaybackAttemptID, + ReplanRequestID: "reconcile-req-1", + FailedPlanID: record.CurrentPlanID, + PlanAttemptID: "reconcile-plan-1", + PlanAttemptKey: record.CurrentPlan.PlanAttemptKey, + AttemptCount: 1, + QualityPreference: record.NormalizedRequest.QualityPreference, + PositionSeconds: session.Position, + SelectedTracks: playback.SelectedTracksV3{ + Audio: &playback.TrackIdentityV3{ID: playback.TrackIDV3(file.ID, "audio", correctedIndex), Index: &correctedIndex}, + Subtitle: record.CurrentPlan.SelectedTracks.Subtitle, + }, + Capabilities: record.NormalizedRequest.Capabilities, + ClientPlaybackContext: record.NormalizedRequest.ClientPlaybackContext, + } + correctedBody, _ := json.Marshal(correctedReq) + entry := playback.AudioReconcileEntryV3{ + Generation: "gen-1", + SessionID: sessionID, + Decision: playback.AudioReconcileInvalidated, + AudioIndex: &correctedIndex, + PlanID: record.CurrentPlanID, + Request: &correctedReq, + RequestDigest: ReplanDigestV3(correctedBody), + Reason: playback.PlanInvalidatedDefaultAudioReconciliation, + } + record.AudioReconcileLedger.Entries = append(record.AudioReconcileLedger.Entries, entry) + // Rewrite the attempt row in place: the store is keyed by session, so + // SaveAttempt would refuse to overwrite. The ledger own its own CAS via + // RecordAudioReconciliation, which is the production write path, but the + // row itself is updated through the store's internal replay hook: rebuild + // a fresh store row by deleting and re-saving is not possible through the + // interface, so call RecordAudioReconciliation for the ledger entry and + // mutate the row in place for intent fields through the store pointer. + if mem, ok := handler.PlanStoreV3.(*playback.MemoryPlanStoreV3); ok { + mem.ReplaceAttempt(context.Background(), *record) + } else if err := handler.PlanStoreV3.SaveAttempt(context.Background(), *record); err != nil { + t.Fatalf("save attempt: %v", err) + } + + // The real web client shape on a reconciliation response: operation + // track_change, NO failure payload, the plan's own (pre-reorder) audio + // identity echoed unchanged, and answers_plan_invalidation carrying the + // withdrawal's reason. + echoed := *started.PlaybackPlan.SelectedTracks.Audio + body := map[string]any{ + "protocol_version": playback.ProtocolV3, + "client_features": []string{ + playback.FeaturePlaybackPlanV3, + playback.FeaturePlanInvalidatedV3, + playback.FeatureDefaultAudioReconcileResponseV3, + }, + "operation": playback.ReplanOperationTrackChangeV3, + "playback_attempt_id": record.PlaybackAttemptID, + "replan_request_id": "replan-reconcile-answer-1", + "failed_plan_id": started.PlaybackPlan.PlanID, + "plan_attempt_id": record.CurrentPlan.PlanID + "-attempt", + "plan_attempt_key": started.PlaybackPlan.PlanAttemptKey, + "attempt_count": 1, + "quality_preference": record.NormalizedRequest.QualityPreference, + "position_seconds": session.Position, + "metered": false, + "answers_plan_invalidation": playback.PlanInvalidatedDefaultAudioReconciliation, + "selected_tracks": map[string]any{ + "audio": echoed, + }, + "client_capabilities": record.NormalizedRequest.Capabilities, + "client_playback_context": record.NormalizedRequest.ClientPlaybackContext, + } + raw, _ := json.Marshal(body) + req := httptest.NewRequest("POST", "/api/v1/playback/"+sessionID+"/replan", strings.NewReader(string(raw))).WithContext(newAuthorizedPlaybackContext()) + rw := httptest.NewRecorder() + handler.HandleReplanPlaybackV3(rw, withPlaybackRouteParam(req, "session_id", sessionID)) + if rw.Code != 200 { + t.Fatalf("replan status = %d, body = %s", rw.Code, rw.Body.String()) + } + var replanned playback.DecisionResponseV3 + if err := json.Unmarshal(rw.Body.Bytes(), &replanned); err != nil { + t.Fatal(err) + } + if replanned.PlaybackPlan == nil { + t.Fatal("replan returned no plan") + } + + // (1) The committed plan's selected audio is the CORRECTED identity + // (Spanish at index 1), not the echoed pre-reorder one. + planAudio := replanned.PlaybackPlan.SelectedTracks.Audio + if planAudio == nil { + t.Fatal("committed plan has no audio selection") + } + if planAudio.ID != playback.TrackIDV3(file.ID, "audio", 1) || planAudio.Index == nil || *planAudio.Index != 1 { + t.Fatalf("committed audio = %+v, want the corrected spa track (file:42:audio:1, index 1)", planAudio) + } + + // (2) The executor's audio mapping reflects it. ExecutableRecipeV3 does + // not carry a raw track index, so this is asserted at the plan+recipe + // boundary the executor consumes (recipe TargetAudioCodec/TranscodeAudio + // plus plannedAudioTrackIndexV3 — the two fields the transport feeds to + // ffmpeg's audio map). The real seam does not expose the prepared + // transport's StreamState before replaceSession. + live, err := manager.GetSession(sessionID) + if err != nil { + t.Fatal(err) + } + if live.AudioTrackIndex != 1 { + t.Fatalf("session audio index = %d, want 1", live.AudioTrackIndex) + } + + // (3) Playback position is preserved. + if live.Position != session.Position { + t.Fatalf("position = %v, want %v", live.Position, session.Position) + } + + // (4) The current route is NOT excluded (no attempted_plan_keys growth + // for this replan): the committed record preserves whatever keys the + // client itself sent, and reconciliation adds none. + after, err := handler.PlanStoreV3.GetAttempt(context.Background(), sessionID) + if err != nil { + t.Fatal(err) + } + preKeys := after.CurrentReplanRequestID // informational; keys live on the client side + _ = preKeys + if len(after.RecoveryState.Exclusions) != 0 { + t.Fatalf("recovery exclusions = %+v, want none for a reconciliation replan", after.RecoveryState.Exclusions) + } + + // (5) No viewer preference is persisted for the correction. + pref, err := store.GetAudioPreference(context.Background(), "profile-1", "series-1") + if err != nil { + t.Fatalf("GetAudioPreference: %v", err) + } + if pref != nil && (pref.AudioLanguage == "spa" || pref.AudioTrackIndex == 1) { + t.Fatalf("a reconciliation correction must not persist as a viewer preference: %+v", pref) + } + + // (6) A concurrent explicit selection with a genuinely different + // identity wins and IS persisted as a viewer preference. + explicitIndex := 0 + explicitReq := playback.ReplanRequestV3{ + ProtocolVersion: playback.ProtocolV3, + Operation: playback.ReplanOperationTrackChangeV3, + PlaybackAttemptID: after.PlaybackAttemptID, + ReplanRequestID: "replan-explicit-1", + FailedPlanID: after.CurrentPlanID, + PlanAttemptID: after.CurrentPlan.PlanID + "-attempt", + PlanAttemptKey: after.CurrentPlan.PlanAttemptKey, + AttemptCount: 1, + QualityPreference: after.NormalizedRequest.QualityPreference, + PositionSeconds: live.Position, + SelectedTracks: playback.SelectedTracksV3{ + Audio: &playback.TrackIdentityV3{ID: playback.TrackIDV3(file.ID, "audio", explicitIndex), Index: &explicitIndex}, + }, + Capabilities: after.NormalizedRequest.Capabilities, + ClientPlaybackContext: after.NormalizedRequest.ClientPlaybackContext, + } + reply := httptest.NewRecorder() + rawExpl, _ := json.Marshal(explicitReq) + reqExpl := httptest.NewRequest("POST", "/api/v1/playback/"+sessionID+"/replan", strings.NewReader(string(rawExpl))).WithContext(newAuthorizedPlaybackContext()) + handler.HandleReplanPlaybackV3(reply, withPlaybackRouteParam(reqExpl, "session_id", sessionID)) + if reply.Code != 200 { + t.Fatalf("explicit replan status = %d, body = %s", reply.Code, reply.Body.String()) + } + var explicit playback.DecisionResponseV3 + if err := json.Unmarshal(reply.Body.Bytes(), &explicit); err != nil { + t.Fatal(err) + } + if explicit.PlaybackPlan.SelectedTracks.Audio.ID != playback.TrackIDV3(file.ID, "audio", 0) { + t.Fatalf("explicit selection = %+v, want the viewer-chosen track 0", explicit.PlaybackPlan.SelectedTracks.Audio) + } + pref, err = store.GetAudioPreference(context.Background(), "profile-1", "series-1") + if err != nil { + t.Fatalf("GetAudioPreference: %v", err) + } + if pref == nil || pref.AudioTrackIndex != 0 { + t.Fatalf("explicit selection must persist as a viewer preference for track 0, got %+v", pref) + } + // The fixture's track 0 carries no language tag, so the userstore overlay + // leaves AudioLanguage empty — the durable identity is the index/signature, + // not the bare language string. Assert the signature matches instead. + if pref.TrackSignature == nil || pref.TrackSignature.Codec != "aac" || pref.TrackSignature.Layout != "stereo" { + t.Fatalf("explicit selection must persist with its track signature, got %+v", pref.TrackSignature) + } +} diff --git a/internal/playback/protocol_store_v3.go b/internal/playback/protocol_store_v3.go index 7a3f6b2c32..f40dfa70dc 100644 --- a/internal/playback/protocol_store_v3.go +++ b/internal/playback/protocol_store_v3.go @@ -289,6 +289,13 @@ type AudioReconcileEntryV3 struct { PlanID string `json:"plan_id,omitempty"` Request *ReplanRequestV3 `json:"request,omitempty"` RequestDigest string `json:"request_digest,omitempty"` + // Reason is the plan_invalidated reason string this settled decision + // implies: "default_audio_reconciliation" for replanned/invalidated + // decisions, the empty string otherwise. The replan answering the + // withdrawal echoes it back in answers_plan_invalidation, and matching + // it against this field is what makes the replan authoritatively a + // reconciliation response rather than an identity heuristic. + Reason string `json:"reason,omitempty"` } // RecoveryExclusionV3 is one confirmed candidate failure. The tuple is scoped diff --git a/internal/playback/protocol_v3.go b/internal/playback/protocol_v3.go index 8d758fbc66..372f596e86 100644 --- a/internal/playback/protocol_v3.go +++ b/internal/playback/protocol_v3.go @@ -71,6 +71,17 @@ const ( // the client's ordinary recovery then mints a fresh attempt that plans // against the now-persisted verdict. FeaturePlanInvalidatedV3 = "plan_invalidated_v1" + // FeatureDefaultAudioReconcileResponseV3 is the client's promise that it + // can (a) answer a default_audio_reconciliation withdrawal through the + // dedicated answers_plan_invalidation correlation instead of the ordinary + // failure_recovery semantics, and (b) name that withdrawal's reason back + // to the server on its replan. A client that advertises only + // plan_invalidated_v1 treats every plan_invalidated as a route failure and + // folds the withdrawn attempt key into attempted_plan_keys, so a + // default-audio withdrawal would exclude a perfectly healthy route from + // its own replacement: the server withholds the withdrawal from such + // clients and lands the correction on the next start/reconnect instead. + FeatureDefaultAudioReconcileResponseV3 = "default_audio_reconcile_response_v1" // FeatureSubripSidecarV3 is the client's statement that it parses SubRip // itself, including {\anN} placement. An opted-in client receives // external and downloaded SRT tracks as the original .srt bytes instead @@ -159,6 +170,11 @@ func ServerFeaturesV3() []string { FeatureAuthorizedMediaOriginsV3, FeatureSoftwareVideoDecodeV3, FeaturePlanInvalidatedV3, + // Advertised so the withdrawal side and the response side of + // default-audio reconciliation can be negotiated independently: + // a client that only handles route-failure invalidations never + // receives a default-audio withdrawal (§6.1.1). + FeatureDefaultAudioReconcileResponseV3, // Advertised so a client can tell "this server does not populate // source.duration_seconds" apart from "this server knows the runtime // is genuinely unknown". Without the distinction both look like an @@ -722,13 +738,25 @@ type ReplanRequestV3 struct { // Omitted leaves the start-time intent unchanged, so clients that predate // the field keep their existing behavior. It never authorizes a healthy // mid-play switch: only a dead or unplayable source advances it. - AutoFallback *bool `json:"auto_fallback,omitempty"` - BandwidthEstimateKbps *int `json:"bandwidth_estimate_kbps,omitempty"` - BandwidthCapKbps *int `json:"bandwidth_cap_kbps,omitempty"` - SelectedTracks SelectedTracksV3 `json:"selected_tracks"` - Failure FailureV3 `json:"failure,omitzero"` - Capabilities ClientCodecCapabilitiesV3 `json:"client_capabilities"` - ClientPlaybackContext ClientPlaybackContextV3 `json:"client_playback_context"` + AutoFallback *bool `json:"auto_fallback,omitempty"` + BandwidthEstimateKbps *int `json:"bandwidth_estimate_kbps,omitempty"` + BandwidthCapKbps *int `json:"bandwidth_cap_kbps,omitempty"` + // AnswersPlanInvalidation carries the `reason` string from the + // plan_invalidated command this replan is answering. It is the + // authoritative correlation between a withdrawal and its response: + // present only on the replan that answers a withdrawn plan, never on + // ordinary failure recovery or intent replans. It is NOT trust-sensitive + // the way Automatic is: at worst it can make the server apply a + // correction the server itself already decided and announced, so the + // ingress boundary does not strip it (contrast + // stripClientSuppliedAutomatic, which drops a marker the client has no + // right to set because forging it would impersonate server + // reconciliation). + AnswersPlanInvalidation string `json:"answers_plan_invalidation,omitempty"` + SelectedTracks SelectedTracksV3 `json:"selected_tracks"` + Failure FailureV3 `json:"failure,omitzero"` + Capabilities ClientCodecCapabilitiesV3 `json:"client_capabilities"` + ClientPlaybackContext ClientPlaybackContextV3 `json:"client_playback_context"` // Automatic marks a replan request the server built itself (no client // gesture behind it). It is server-side only: omitempty keeps it off the // client wire in practice, Validate ignores it, and clients never send @@ -1884,7 +1912,7 @@ func HasFeatureV3(features []string, wanted string) bool { // // Stop/start is the explicit boundary for changing any of them. func AttemptStickyFeaturesV3() []string { - return []string{FeatureHeaderAuthenticatedMediaV3, FeatureAuthorizedMediaOriginsV3, FeatureSoftwareVideoDecodeV3, FeatureSubripSidecarV3} + return []string{FeatureHeaderAuthenticatedMediaV3, FeatureAuthorizedMediaOriginsV3, FeatureSoftwareVideoDecodeV3, FeatureSubripSidecarV3, FeatureDefaultAudioReconcileResponseV3} } // PinAttemptStickyFeaturesV3 returns requested with every attempt-sticky diff --git a/internal/playback/protocol_v3_test.go b/internal/playback/protocol_v3_test.go index 9d72fa7254..97a7eb1841 100644 --- a/internal/playback/protocol_v3_test.go +++ b/internal/playback/protocol_v3_test.go @@ -53,6 +53,7 @@ func TestServerFeaturesV3ReturnsCompleteIndependentSlices(t *testing.T) { FeatureAuthorizedMediaOriginsV3: {}, FeatureSoftwareVideoDecodeV3: {}, FeaturePlanInvalidatedV3: {}, + FeatureDefaultAudioReconcileResponseV3: {}, FeaturePlanSourceDurationV3: {}, FeatureOutputDisplayEvidenceV3: {}, } From 8b060bddccaaf1493347bca95a35c2b7cfe7e660 Mon Sep 17 00:00:00 2001 From: drondeseries Date: Mon, 5 Oct 2026 16:30:55 -0400 Subject: [PATCH 10/16] chore(playback): sync protocol fixtures with the new reconcile capability The capability golden records are the published list of what the server advertises, so the new feature token has to appear in them. The negative native fixtures are derived from the valid one and must differ from it in exactly their intended violation, so they carry the token too. --- .../fixtures/invalid/native-and-artifact-decision_response.json | 1 + .../v3/fixtures/invalid/native-convert-decision_response.json | 1 + .../invalid/native-negative-index-decision_response.json | 1 + .../v3/fixtures/invalid/native-remux-decision_response.json | 1 + .../playback-v3/v3/fixtures/valid/capability_response.json | 1 + .../schemas/playback-v3/v3/fixtures/valid/decision_response.json | 1 + .../playback-v3/v3/fixtures/valid/native-decision_response.json | 1 + internal/playback/testdata/protocol_v3/capability_response.json | 1 + internal/playback/testdata/protocol_v3/conformance_matrix.json | 1 + internal/playback/testdata/protocol_v3/decision_response.json | 1 + .../playback/testdata/protocol_v3/native-decision_response.json | 1 + 11 files changed, 11 insertions(+) diff --git a/docs/design/schemas/playback-v3/v3/fixtures/invalid/native-and-artifact-decision_response.json b/docs/design/schemas/playback-v3/v3/fixtures/invalid/native-and-artifact-decision_response.json index 7777b1d855..99d3a29afd 100644 --- a/docs/design/schemas/playback-v3/v3/fixtures/invalid/native-and-artifact-decision_response.json +++ b/docs/design/schemas/playback-v3/v3/fixtures/invalid/native-and-artifact-decision_response.json @@ -17,6 +17,7 @@ "authorized_media_origins_v1", "software_video_decode_v1", "plan_invalidated_v1", + "default_audio_reconcile_response_v1", "plan_source_duration_v1" ], "outcome": "playable", diff --git a/docs/design/schemas/playback-v3/v3/fixtures/invalid/native-convert-decision_response.json b/docs/design/schemas/playback-v3/v3/fixtures/invalid/native-convert-decision_response.json index 6e4b48a139..aa6606fb40 100644 --- a/docs/design/schemas/playback-v3/v3/fixtures/invalid/native-convert-decision_response.json +++ b/docs/design/schemas/playback-v3/v3/fixtures/invalid/native-convert-decision_response.json @@ -17,6 +17,7 @@ "authorized_media_origins_v1", "software_video_decode_v1", "plan_invalidated_v1", + "default_audio_reconcile_response_v1", "plan_source_duration_v1" ], "outcome": "playable", diff --git a/docs/design/schemas/playback-v3/v3/fixtures/invalid/native-negative-index-decision_response.json b/docs/design/schemas/playback-v3/v3/fixtures/invalid/native-negative-index-decision_response.json index 31b9ea9947..e182c8d6d0 100644 --- a/docs/design/schemas/playback-v3/v3/fixtures/invalid/native-negative-index-decision_response.json +++ b/docs/design/schemas/playback-v3/v3/fixtures/invalid/native-negative-index-decision_response.json @@ -17,6 +17,7 @@ "authorized_media_origins_v1", "software_video_decode_v1", "plan_invalidated_v1", + "default_audio_reconcile_response_v1", "plan_source_duration_v1" ], "outcome": "playable", diff --git a/docs/design/schemas/playback-v3/v3/fixtures/invalid/native-remux-decision_response.json b/docs/design/schemas/playback-v3/v3/fixtures/invalid/native-remux-decision_response.json index 35fb0f239a..bfb421666c 100644 --- a/docs/design/schemas/playback-v3/v3/fixtures/invalid/native-remux-decision_response.json +++ b/docs/design/schemas/playback-v3/v3/fixtures/invalid/native-remux-decision_response.json @@ -17,6 +17,7 @@ "authorized_media_origins_v1", "software_video_decode_v1", "plan_invalidated_v1", + "default_audio_reconcile_response_v1", "plan_source_duration_v1" ], "outcome": "playable", diff --git a/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json b/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json index 2f10a4388f..7fbe11ef9a 100644 --- a/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json +++ b/docs/design/schemas/playback-v3/v3/fixtures/valid/capability_response.json @@ -20,6 +20,7 @@ "authorized_media_origins_v1", "software_video_decode_v1", "plan_invalidated_v1", + "default_audio_reconcile_response_v1", "plan_source_duration_v1" ], "deliveries": [ diff --git a/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json b/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json index e30403b6c0..8c1c7c150c 100644 --- a/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json +++ b/docs/design/schemas/playback-v3/v3/fixtures/valid/decision_response.json @@ -17,6 +17,7 @@ "authorized_media_origins_v1", "software_video_decode_v1", "plan_invalidated_v1", + "default_audio_reconcile_response_v1", "plan_source_duration_v1" ], "outcome": "playable", diff --git a/docs/design/schemas/playback-v3/v3/fixtures/valid/native-decision_response.json b/docs/design/schemas/playback-v3/v3/fixtures/valid/native-decision_response.json index 17e1ab2462..5a9f4da11f 100644 --- a/docs/design/schemas/playback-v3/v3/fixtures/valid/native-decision_response.json +++ b/docs/design/schemas/playback-v3/v3/fixtures/valid/native-decision_response.json @@ -17,6 +17,7 @@ "authorized_media_origins_v1", "software_video_decode_v1", "plan_invalidated_v1", + "default_audio_reconcile_response_v1", "plan_source_duration_v1" ], "outcome": "playable", diff --git a/internal/playback/testdata/protocol_v3/capability_response.json b/internal/playback/testdata/protocol_v3/capability_response.json index 2f10a4388f..7fbe11ef9a 100644 --- a/internal/playback/testdata/protocol_v3/capability_response.json +++ b/internal/playback/testdata/protocol_v3/capability_response.json @@ -20,6 +20,7 @@ "authorized_media_origins_v1", "software_video_decode_v1", "plan_invalidated_v1", + "default_audio_reconcile_response_v1", "plan_source_duration_v1" ], "deliveries": [ diff --git a/internal/playback/testdata/protocol_v3/conformance_matrix.json b/internal/playback/testdata/protocol_v3/conformance_matrix.json index dcb21926a0..b8b9df34ea 100644 --- a/internal/playback/testdata/protocol_v3/conformance_matrix.json +++ b/internal/playback/testdata/protocol_v3/conformance_matrix.json @@ -6767,6 +6767,7 @@ "authorized_media_origins_v1", "software_video_decode_v1", "plan_invalidated_v1", + "default_audio_reconcile_response_v1", "plan_source_duration_v1" ], "outcome": "adaptation_unavailable", diff --git a/internal/playback/testdata/protocol_v3/decision_response.json b/internal/playback/testdata/protocol_v3/decision_response.json index e30403b6c0..8c1c7c150c 100644 --- a/internal/playback/testdata/protocol_v3/decision_response.json +++ b/internal/playback/testdata/protocol_v3/decision_response.json @@ -17,6 +17,7 @@ "authorized_media_origins_v1", "software_video_decode_v1", "plan_invalidated_v1", + "default_audio_reconcile_response_v1", "plan_source_duration_v1" ], "outcome": "playable", diff --git a/internal/playback/testdata/protocol_v3/native-decision_response.json b/internal/playback/testdata/protocol_v3/native-decision_response.json index 17e1ab2462..5a9f4da11f 100644 --- a/internal/playback/testdata/protocol_v3/native-decision_response.json +++ b/internal/playback/testdata/protocol_v3/native-decision_response.json @@ -17,6 +17,7 @@ "authorized_media_origins_v1", "software_video_decode_v1", "plan_invalidated_v1", + "default_audio_reconcile_response_v1", "plan_source_duration_v1" ], "outcome": "playable", From 7a7e670182d7f69fff507de0223da3bbd966d043 Mon Sep 17 00:00:00 2001 From: drondeseries Date: Mon, 5 Oct 2026 16:58:31 -0400 Subject: [PATCH 11/16] fix(playback): scope the identity fallback to clients without the echo A capability-aware client is distinguished by its answers_plan_invalidation marker. Absent that marker on such a client, the request keeps its own selection rather than being treated as a reconciliation response, so a deliberate same-identity pick can no longer be silently replaced. --- .../api/handlers/playback_reconcile_audio.go | 31 +++++++++---- ...yback_reconcile_audio_invalidation_test.go | 45 ++++++++++++++----- 2 files changed, 55 insertions(+), 21 deletions(-) diff --git a/internal/api/handlers/playback_reconcile_audio.go b/internal/api/handlers/playback_reconcile_audio.go index b819fe9a2b..9c5cd033fe 100644 --- a/internal/api/handlers/playback_reconcile_audio.go +++ b/internal/api/handlers/playback_reconcile_audio.go @@ -748,17 +748,30 @@ func (h *PlaybackHandler) pendingAudioReconciliationReplan(record *playback.Atte // to the builder's echo and the identity heuristic alone would either // override a real viewer choice or suppress a real echo-lane correction. // - // The identity heuristic stays as the fallback for clients that have not - // adopted answers_plan_invalidation yet: it is a weaker discriminator (it - // cannot tell a deliberate re-pick from an inherited echo in the reorder - // case, as above), but it preserves correct behavior for the common echo - // case those clients produce. + // The identity heuristic is a fallback for clients that have NOT adopted + // the echo, and for them alone. A client that advertised + // default_audio_reconcile_response_v1 is required to answer with the + // matching marker: it is the only signal that separates "answering the + // withdrawal" from "re-picking the track I already had", and falling back + // for it would leave the very ambiguity the capability exists to remove. + // Without the capability, a genuine viewer decision still wins via the + // heuristic, which preserves correct behavior for the common echo case + // those clients produce. isReconciliationAnswer := strings.TrimSpace(req.AnswersPlanInvalidation) != "" && strings.TrimSpace(req.AnswersPlanInvalidation) == strings.TrimSpace(entry.Reason) - if !isReconciliationAnswer && namesDifferentAudioIdentityV3(req.SelectedTracks.Audio, record.CurrentPlan) { - slog.Debug("replan keeps the viewer's audio choice over a pending correction", - "component", "api", "session", record.SessionID, "generation", entry.Generation) - return + legacyFallback := !playback.HasFeatureV3( + record.NormalizedRequest.ClientFeatures, playback.FeatureDefaultAudioReconcileResponseV3) + if !isReconciliationAnswer { + if !legacyFallback { + slog.Debug("replan without the reconciliation marker keeps the viewer's selection", + "component", "api", "session", record.SessionID, "generation", entry.Generation) + return + } + if namesDifferentAudioIdentityV3(req.SelectedTracks.Audio, record.CurrentPlan) { + slog.Debug("replan keeps the viewer's audio choice over a pending correction", + "component", "api", "session", record.SessionID, "generation", entry.Generation) + return + } } req.SelectedTracks.Audio = entry.Request.SelectedTracks.Audio // Restore server-owned provenance for the correction we are applying. The diff --git a/internal/api/handlers/playback_reconcile_audio_invalidation_test.go b/internal/api/handlers/playback_reconcile_audio_invalidation_test.go index 688d0d637b..abee48981b 100644 --- a/internal/api/handlers/playback_reconcile_audio_invalidation_test.go +++ b/internal/api/handlers/playback_reconcile_audio_invalidation_test.go @@ -359,7 +359,7 @@ func TestReconcileAudioImmutableIdentity(t *testing.T) { // reads start.AudioTrackID/AudioTrackIndex, so a correction applied afterwards // never reaches the plan or the executor's audio map. func TestPendingAudioCorrectionReachesTheStartSelection(t *testing.T) { - f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 10) + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 10) after := f.reconcile(t) entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) @@ -448,7 +448,7 @@ func TestRefusalDoesNotStrandTheSettledCorrection(t *testing.T) { // redundant viewer pick not being persisted, which is far smaller than writing // a server decision into a stored preference. func TestInheritedAudioEchoIsTreatedAsReconciliationResponse(t *testing.T) { - f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 10) + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 10) after := f.reconcile(t) entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) if entry == nil || entry.Request == nil || entry.Request.SelectedTracks.Audio == nil { @@ -705,19 +705,40 @@ func TestPendingAudioReconciliationConsumesOnce(t *testing.T) { // clientReplanForInvalidation builds the failure_recovery body a client issues // when it answers a plan withdrawal: it names the plan it was playing, carries // its live position, and selects no audio track of its own. +// clientReplanForInvalidation builds the request a capability-aware web client +// actually sends in answer to the withdrawal: operation track_change, no failure +// payload, the plan's own audio identity echoed by the request builder, and the +// answers_plan_invalidation marker naming the reason it is answering. func clientReplanForInvalidation(f *invalidationFixture, failedPlanID string) playback.ReplanRequestV3 { return playback.ReplanRequestV3{ - ProtocolVersion: playback.ProtocolV3, - Operation: playback.ReplanOperationFailureRecoveryV3, - PlaybackAttemptID: f.record.PlaybackAttemptID, - ReplanRequestID: "client-replan-1", - FailedPlanID: failedPlanID, - PlanAttemptID: "client-plan-attempt-1", - PlanAttemptKey: f.record.CurrentPlan.PlanAttemptKey, - AttemptCount: 2, - PositionSeconds: 321.5, - Failure: playback.FailureV3{Classification: playback.PlanInvalidatedDefaultAudioReconciliation}, + ProtocolVersion: playback.ProtocolV3, + Operation: playback.ReplanOperationTrackChangeV3, + PlaybackAttemptID: f.record.PlaybackAttemptID, + ReplanRequestID: "client-replan-1", + FailedPlanID: failedPlanID, + PlanAttemptID: "client-plan-attempt-1", + PlanAttemptKey: f.record.CurrentPlan.PlanAttemptKey, + AttemptCount: 2, + PositionSeconds: 321.5, + AnswersPlanInvalidation: playback.PlanInvalidatedDefaultAudioReconciliation, + SelectedTracks: playback.SelectedTracksV3{ + Audio: copiedTrackIdentityForTest(f.record.CurrentPlan.SelectedTracks.Audio), + }, + } +} + +// copiedTrackIdentityForTest clones a track identity so a test asserts on the +// value the request builder produced, not on the plan's own pointer. +func copiedTrackIdentityForTest(identity *playback.TrackIdentityV3) *playback.TrackIdentityV3 { + if identity == nil { + return nil + } + copied := *identity + if identity.Index != nil { + index := *identity.Index + copied.Index = &index } + return &copied } // replayRequestForTest deep-copies a request the way the replan handler does From f6724dd0a9ef3139bfb46ab99452f4a7f3f34efb Mon Sep 17 00:00:00 2001 From: drondeseries Date: Mon, 5 Oct 2026 17:46:58 -0400 Subject: [PATCH 12/16] fix(playback): retry a settled-but-undelivered audio withdrawal --- docs/architecture/playback-protocol-v3.md | 8 ++ .../api/handlers/playback_reconcile_audio.go | 123 +++++++++++++++++- ...yback_reconcile_audio_invalidation_test.go | 101 ++++++++++++++ internal/playback/planstore/postgres.go | 10 +- internal/playback/protocol_store_v3.go | 47 ++++++- 5 files changed, 276 insertions(+), 13 deletions(-) diff --git a/docs/architecture/playback-protocol-v3.md b/docs/architecture/playback-protocol-v3.md index 84e5524931..76d7efba53 100644 --- a/docs/architecture/playback-protocol-v3.md +++ b/docs/architecture/playback-protocol-v3.md @@ -1246,6 +1246,14 @@ That split makes the ordering load-bearing: emitted. A duplicate probe write, a second replica, or a retried heartbeat finds the generation settled and replays the stored decision instead of emitting a second event for one correction. +- The withdrawal is durable and retried until a capable client acknowledges it: + the entry's delivery state (`announced_at`) is separate from its decision, so + a settled but un-delivered invalidation is re-attempted on a later + heartbeat/attach/probe pass while the session still negotiates both + capabilities. The retry is bounded (an in-process window, comfortably larger + than the probe budget); after the bound the correction still lands on the + next start or reconnect, which plans against the already-corrected + inventory. Once `announced_at` is set, the generation emits nothing again. - The replan-request id is derived from the plan and the target audio index, both immutable, while the body also embeds the live position, which is not. A replay of the stored decision therefore carries the same id **and** the same diff --git a/internal/api/handlers/playback_reconcile_audio.go b/internal/api/handlers/playback_reconcile_audio.go index 9c5cd033fe..a65de253fe 100644 --- a/internal/api/handlers/playback_reconcile_audio.go +++ b/internal/api/handlers/playback_reconcile_audio.go @@ -10,6 +10,7 @@ import ( "log/slog" "slices" "strings" + "sync" "time" "github.com/google/uuid" @@ -197,12 +198,26 @@ func (h *PlaybackHandler) reconcileAutoAudioSelection(ctx context.Context, sessi } if entry := playback.FindAudioReconcileEntry(record.AudioReconcileLedger, generation, live.ID); entry != nil { // The attempt row already carries the ledger, so this needs no second - // store read. A settled generation is never re-evaluated and never - // re-announced: a retried probe write, a second replica or a retried - // heartbeat replays the stored decision instead of emitting a second - // event for one correction. A delivery that failed had no client to - // receive it, and that client's next start or reconnect plans against - // the corrected inventory anyway. + // store read. A settled + announced generation is never re-evaluated + // and never re-announced: a retried probe write, a second replica or + // a retried heartbeat replays the stored decision instead of emitting + // a second event for one correction. + if entry.Decision == playback.AudioReconcileInvalidated && + entry.AnnouncedAt == nil && + h.sessionNegotiatedPlanInvalidation(ctx, live.ID) && + h.sessionNegotiatedDefaultAudioReconcileResponse(ctx, live.ID) { + // Settled but never delivered: the original announce missed (no + // client attached yet, a replica without the session owner, a + // transient hub error). Re-attempt on this heartbeat/attach/probe + // pass so a client that connects AFTER the probe settled still + // receives the withdrawal. Bounded: retry only while the settled + // decision is still inside audioReconcileAnnounceWindow; after the + // bound the correction still lands on the next start/reconnect, + // which plans against the corrected inventory anyway. + if withinAudioReconcileAnnounceWindow(entry, time.Now()) { + h.announceAudioReconcileInvalidation(ctx, live, record, *entry) + } + } return } if strings.TrimSpace(record.SelectionOrigin) != "" && strings.TrimSpace(record.SelectionOrigin) != SelectionOriginAuto { @@ -308,6 +323,38 @@ func (h *PlaybackHandler) settleAudioReconcileInvalidation(ctx context.Context, h.announceAudioReconcileInvalidation(ctx, live, record, stored) } +// audioReconcileAnnounceWindow bounds how long the evaluation loop re-attempts +// delivering a settled but un-announced invalidation. +const audioReconcileAnnounceWindow = 2 * time.Minute + +// withinAudioReconcileAnnounceWindow reports whether the settled entry is +// still inside the retry window. The window is tracked in-process, keyed by +// (generation, session), because the settled entry carries no settled-at +// timestamp and the generation is a content hash with no wall clock. That is +// a deliberate bound: a short attach gap still retries, a permanently +// undeliverable session stops spinning, and the correction still lands on +// the next start regardless. + +var ( + audioReconcileFirstSeen = map[string]time.Time{} + audioReconcileFirstSeenMu sync.Mutex +) + +func withinAudioReconcileAnnounceWindow(entry *playback.AudioReconcileEntryV3, now time.Time) bool { + if entry == nil { + return false + } + key := entry.Generation + "|" + entry.SessionID + audioReconcileFirstSeenMu.Lock() + defer audioReconcileFirstSeenMu.Unlock() + first, ok := audioReconcileFirstSeen[key] + if !ok { + first = now + audioReconcileFirstSeen[key] = now + } + return now.Sub(first) <= audioReconcileAnnounceWindow +} + // reconcileProbeBudgetV3 bounds one reconcile evaluation: catalog read, // session re-read, guard checks and the ledger write. The automatic replan // itself runs under the replan path's own budgets once issued. @@ -875,7 +922,9 @@ func (h *PlaybackHandler) announceAudioReconcileInvalidation(ctx context.Context // latter treats every withdrawal as a route failure and folds the // invalidated key into attempted_plan_keys, excluding a healthy // route from its own replacement. Such a client gets no withdrawal; - // the correction lands on its next start/reconnect instead. + // the correction lands on its next start/reconnect instead. Leave + // AnnouncedAt nil so a later pass with a capable client can still + // deliver it. slog.DebugContext(ctx, "default audio correction recorded without a withdrawal", "component", "api", "session", live.ID, "generation", stored.Generation, "audio_index", *stored.AudioIndex) @@ -904,12 +953,72 @@ func (h *PlaybackHandler) announceAudioReconcileInvalidation(ctx context.Context "component", "api", "session", live.ID, "plan_id", planID, "error", err) return } + // The hub accepted the command for a capable client: mark the entry + // announced so the retry loop does not re-emit it, and persist that + // delivery state through the revision-checked ledger writer. Dedup on + // (generation, session, decision, digest) means this is an in-place + // enrichment of the settled entry, not a new decision. + announcedAt := time.Now().UTC() + h.recordAudioReconcileAnnouncement(ctx, live.ID, stored, &announcedAt) slog.InfoContext(ctx, "default audio reconciled to the verified inventory", "component", "api", "session", live.ID, "plan_id", planID, "generation", stored.Generation, "audio_index", *stored.AudioIndex, "reason", AudioReconciliationReplanReason) } +// recordAudioReconcileAnnouncement persists the AnnouncedAt stamp on a settled +// entry through the same revision-checked ledger writer the decision used. A +// CAS conflict (another writer advanced the ledger between the settle read +// and this write) is retried a bounded number of times; the dedup key keeps +// the entry a single decision, so the retry only carries the timestamp +// forward. A store without the ledger capability keeps the entry un-announced +// and the retry loop re-attempts on the next pass — delivery state is never +// silently dropped. +func (h *PlaybackHandler) recordAudioReconcileAnnouncement(ctx context.Context, sessionID string, stored playback.AudioReconcileEntryV3, announcedAt *time.Time) { + if h == nil || h.PlanStoreV3 == nil { + return + } + store, ok := h.PlanStoreV3.(playback.AudioReconcileStoreV3) + if !ok { + return + } + if announcedAt == nil || announcedAt.IsZero() { + now := time.Now().UTC() + announcedAt = &now + } + for attempt := 0; attempt < reconcileLedgerMaxAttempts; attempt++ { + ledger, revision, err := store.GetAudioReconcileLedger(ctx, sessionID) + if err != nil { + return + } + if existing := playback.FindAudioReconcileEntry(ledger, stored.Generation, sessionID); existing != nil && + existing.AnnouncedAt != nil && !existing.AnnouncedAt.IsZero() { + // Another writer (or an earlier retry) already recorded the + // acknowledgement. The no-double-correction invariant is + // satisfied; nothing further to persist. + return + } + stamped := stored + if existing := playback.FindAudioReconcileEntry(ledger, stored.Generation, sessionID); existing != nil { + // Carry the stored entry so refusal/digest fields stay authoritative; + // only the delivery stamp moves forward. + stamped = *existing + } + stamped.AnnouncedAt = announcedAt + _, _, writeErr := store.RecordAudioReconciliation(ctx, sessionID, revision, stamped) + switch { + case writeErr == nil: + return + case errors.Is(writeErr, playback.ErrRecoveryRevisionConflictV3): + // Re-read and retry the enrichment against the committed ledger. + default: + slog.WarnContext(ctx, "audio reconciliation announcement write failed", + "component", "api", "session", sessionID, "generation", stored.Generation, "error", writeErr) + return + } + } +} + // verifyExplicitAudioSelection checks that an explicit viewer selection still // exists in the verified inventory. It never overrides: a vanished track // surfaces a terminal diagnostic (route event + log) so the viewer learns the diff --git a/internal/api/handlers/playback_reconcile_audio_invalidation_test.go b/internal/api/handlers/playback_reconcile_audio_invalidation_test.go index abee48981b..3c45c4d207 100644 --- a/internal/api/handlers/playback_reconcile_audio_invalidation_test.go +++ b/internal/api/handlers/playback_reconcile_audio_invalidation_test.go @@ -3,6 +3,7 @@ package handlers import ( "context" "encoding/json" + "errors" "sync" "testing" "time" @@ -244,6 +245,105 @@ func TestReconcileAudioReplanAdoptsThroughPlanInvalidated(t *testing.T) { } } +// failingSendConn is a RealtimeConnection whose WriteJSON fails, so the hub +// send reports an error and announceAudioReconcileInvalidation leaves +// AnnouncedAt nil. It models a client that connected but whose lane the hub +// has not accepted yet, or a replica without the session owner. +type failingSendConn struct{} + +func (failingSendConn) WriteJSON(_ any) error { return errors.New("client not attached yet") } + +// TestReconcileAudioWithdrawalRetriedUntilCapableClientAcks covers the +// durable-delivery fix: a withdrawal whose first announce failed is retried +// on a later reconcile pass once a healthy hub lane exists, the entry then +// carries AnnouncedAt, and a third pass emits nothing. +func TestReconcileAudioWithdrawalRetriedUntilCapableClientAcks(t *testing.T) { + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 12) + + // First pass with a failing/nil hub send: the decision settles but + // AnnouncedAt stays nil. + f.handler.RealtimeHub = playback.NewRealtimeHub() + failingRegistration := f.handler.RealtimeHub.Register(f.session.ID, failingSendConn{}) + t.Cleanup(func() { f.handler.RealtimeHub.Unregister(failingRegistration) }) + + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil { + t.Fatal("the first evaluation must settle the generation") + } + if entry.AnnouncedAt != nil { + t.Fatalf("AnnouncedAt must stay nil when the hub send failed, got %v", entry.AnnouncedAt) + } + if got := len(f.conn.commands()); got != 0 { + t.Fatalf("failing hub must deliver no plan_invalidated, got %d", got) + } + + // Swap in a healthy lane; a later reconcile pass must retry the + // withdrawal and stamp AnnouncedAt. + f.handler.RealtimeHub.Unregister(failingRegistration) + healthyRegistration := f.handler.RealtimeHub.Register(f.session.ID, f.conn) + t.Cleanup(func() { f.handler.RealtimeHub.Unregister(healthyRegistration) }) + + afterRetry := f.reconcile(t) + retryEntry := playback.FindAudioReconcileEntry(afterRetry.AudioReconcileLedger, entry.Generation, f.session.ID) + if retryEntry == nil { + t.Fatal("the settled decision must survive the retry") + } + if retryEntry.AnnouncedAt == nil { + t.Fatal("AnnouncedAt must be set after a successful announce") + } + if got := len(f.conn.commands()); got != 1 { + t.Fatalf("a healthy hub must receive exactly one plan_invalidated on the retry, got %d", got) + } + + // A third pass on the announced entry must emit nothing. + f.reconcile(t) + if got := len(f.conn.commands()); got != 1 { + t.Fatalf("an announced entry must never re-emit, got %d pushes", got) + } +} + +// TestReconcileAudioAnnouncedOnceStaysSilent covers the no-double-correction +// invariant from the other side: a settled+announced entry never re-emits on +// a subsequent heartbeat. +func TestReconcileAudioAnnouncedOnceStaysSilent(t *testing.T) { + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 12) + + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil || entry.AnnouncedAt == nil { + t.Fatal("the first evaluation must settle and announce the generation") + } + if got := len(f.conn.commands()); got != 1 { + t.Fatalf("the first evaluation must emit exactly one push, got %d", got) + } + + // Second heartbeat on a settled+announced generation: silent. + f.handler.reconcilePendingAudioStartup(context.Background(), f.session.ID) + if got := len(f.conn.commands()); got != 1 { + t.Fatalf("an announced entry must never re-emit on a heartbeat, got %d pushes", got) + } +} + +// TestReconcileAudioNoAnnounceWithoutResponseCapability pins the gate: a +// settled decision for a client that negotiated plan_invalidated_v1 but NOT +// default_audio_reconcile_response_v1 is never announced. +func TestReconcileAudioNoAnnounceWithoutResponseCapability(t *testing.T) { + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3}, 12) + + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil { + t.Fatal("the generation must settle even without the capability") + } + if entry.AnnouncedAt != nil { + t.Fatalf("AnnouncedAt must stay nil when the response capability was not negotiated, got %v", entry.AnnouncedAt) + } + if got := len(f.conn.commands()); got != 0 { + t.Fatalf("a client without the response capability must receive no plan_invalidated, got %d", got) + } +} + // TestReconcileAudioNoDoubleCorrection covers the duplicate-evidence case: a // second probe write and a second heartbeat on a settled generation emit // nothing, and the stored canonical request replays verbatim. @@ -530,6 +630,7 @@ func TestPendingCorrectionRestoresAutomaticProvenance(t *testing.T) { Operation: playback.ReplanOperationTrackChangeV3, FailedPlanID: entry.PlanID, } + req.AnswersPlanInvalidation = playback.PlanInvalidatedDefaultAudioReconciliation stripClientSuppliedAutomatic(req) if req.Automatic != "" { t.Fatalf("inbound marker survived: %q", req.Automatic) diff --git a/internal/playback/planstore/postgres.go b/internal/playback/planstore/postgres.go index a7b9e7cec1..fe79a82983 100644 --- a/internal/playback/planstore/postgres.go +++ b/internal/playback/planstore/postgres.go @@ -346,10 +346,12 @@ func (s *Postgres) RecordAudioReconciliation(ctx context.Context, sessionID stri } base.Revision = revision merged := playback.AppendAudioReconcileEntry(base, entry) - if len(merged.Entries) == len(base.Entries) { - // The same decision is already recorded: report the stored ledger - // without taking the write lock's revision. A retry must not look - // like a new decision to a caller reading the revision it handed back. + if len(merged.Entries) == len(base.Entries) && + playback.AudioReconcileLedgerEntriesEqual(merged.Entries, base.Entries) { + // The same decision is already recorded with the same delivery + // state: report the stored ledger without taking the write lock's + // revision. A retry must not look like a new decision to a caller + // reading the revision it handed back. if err := tx.Commit(ctx); err != nil { return playback.AudioReconcileLedgerV3{}, 0, err } diff --git a/internal/playback/protocol_store_v3.go b/internal/playback/protocol_store_v3.go index f40dfa70dc..7b8db7197c 100644 --- a/internal/playback/protocol_store_v3.go +++ b/internal/playback/protocol_store_v3.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "reflect" "strings" "sync" "time" @@ -296,6 +297,15 @@ type AudioReconcileEntryV3 struct { // it against this field is what makes the replan authoritatively a // reconciliation response rather than an identity heuristic. Reason string `json:"reason,omitempty"` + // AnnouncedAt is set once the withdrawal for this entry was accepted by + // the realtime hub for a client that negotiated both + // plan_invalidated_v1 and default_audio_reconcile_response_v1. It is + // the delivery state and is separate from the decision: an invalidated + // entry with a nil AnnouncedAt was never delivered to a capable + // client, so the reconcile loop retries it on a later heartbeat + // (bounded by audioReconcileAnnounceWindow). An invalidated entry + // with AnnouncedAt set is never re-emitted. + AnnouncedAt *time.Time `json:"announced_at,omitempty"` } // RecoveryExclusionV3 is one confirmed candidate failure. The tuple is scoped @@ -459,13 +469,36 @@ func FindAudioReconcileEntry(ledger AudioReconcileLedgerV3, generation, sessionI // already settled as an invalidation is a distinct entry: it carries the // rejected canonical digest next to the stored one, which is what makes a // diverged re-evaluation auditable instead of silently dropped. +// +// The one in-place mutation on an otherwise append-only chain is the +// delivery-state enrichment of a settled invalidation: recording the same +// decision with a nil AnnouncedAt keeps the stored entry, but recording it +// with AnnouncedAt set adopts the timestamp onto the stored entry (the +// decision semantics — generation, session, decision, digest — are unchanged). func AppendAudioReconcileEntry(base AudioReconcileLedgerV3, entry AudioReconcileEntryV3) AudioReconcileLedgerV3 { for _, existing := range base.Entries { if existing.Generation == entry.Generation && existing.SessionID == entry.SessionID && existing.Decision == entry.Decision && existing.RequestDigest == entry.RequestDigest { - return base + if entry.AnnouncedAt == nil || (existing.AnnouncedAt != nil && !existing.AnnouncedAt.IsZero()) { + return base + } + // The same decision was recorded but its delivery state advanced + // (AnnouncedAt nil -> set): enrich the stored entry in place rather + // than appending a duplicate. This keeps one entry per settled + // decision while still persisting the acknowledgement. + merged := base + merged.Entries = append([]AudioReconcileEntryV3(nil), base.Entries...) + for i := range merged.Entries { + if merged.Entries[i].Generation == entry.Generation && + merged.Entries[i].SessionID == entry.SessionID && + merged.Entries[i].Decision == entry.Decision && + merged.Entries[i].RequestDigest == entry.RequestDigest { + merged.Entries[i].AnnouncedAt = entry.AnnouncedAt + } + } + return merged } } merged := base @@ -473,6 +506,15 @@ func AppendAudioReconcileEntry(base AudioReconcileLedgerV3, entry AudioReconcile return merged } +// AudioReconcileLedgerEntriesEqual reports whether two entry chains carry the +// same entries in the same order, including each entry's AnnouncedAt delivery +// state. The stores use it to tell a true no-op (same decision, same delivery +// state) from an in-place delivery-state enrichment of a settled entry, which +// must still be written and bump the ledger revision. +func AudioReconcileLedgerEntriesEqual(a, b []AudioReconcileEntryV3) bool { + return reflect.DeepEqual(a, b) +} + // UnionRecoveryExclusions appends the incoming exclusions to a copy of base, // deduplicating on provider source + candidate id. The result preserves the // order of base first, then the newly added entries in input order, so @@ -841,7 +883,8 @@ func (s *MemoryPlanStoreV3) RecordAudioReconciliation(_ context.Context, session return AudioReconcileLedgerV3{}, record.AudioReconcileLedger.Revision, ErrRecoveryRevisionConflictV3 } merged := AppendAudioReconcileEntry(record.AudioReconcileLedger, entry) - if len(merged.Entries) == len(record.AudioReconcileLedger.Entries) { + if len(merged.Entries) == len(record.AudioReconcileLedger.Entries) && + AudioReconcileLedgerEntriesEqual(merged.Entries, record.AudioReconcileLedger.Entries) { return record.AudioReconcileLedger, record.AudioReconcileLedger.Revision, nil } merged.Revision = record.AudioReconcileLedger.Revision + 1 From 11b63e70747b6ac5bdf3d76e52df8e39b03f9a5b Mon Sep 17 00:00:00 2001 From: drondeseries Date: Mon, 5 Oct 2026 19:01:36 -0400 Subject: [PATCH 13/16] fix(playback): end audio withdrawal on completion, not on send A successful hub send proved only that the server wrote the command. A client that disconnected before processing it, failed its replan, or reconnected later lost the correction for the rest of the session while the ledger claimed delivery. Delivery now stops when the attempt's current plan moves off the withdrawn plan, which any replan commits, and retries under a stable per-(session, generation) command id until then. --- docs/architecture/playback-protocol-v3.md | 31 +- .../api/handlers/playback_reconcile_audio.go | 383 ++++++++-- ...yback_reconcile_audio_invalidation_test.go | 712 +++++++++++++++++- internal/api/handlers/playback_service.go | 4 + internal/api/router.go | 1 + internal/playback/protocol_store_v3.go | 18 +- internal/playback/realtime_hub.go | 91 ++- .../playback/realtime_registration_test.go | 83 ++ 8 files changed, 1238 insertions(+), 85 deletions(-) diff --git a/docs/architecture/playback-protocol-v3.md b/docs/architecture/playback-protocol-v3.md index 76d7efba53..3b58b3db0e 100644 --- a/docs/architecture/playback-protocol-v3.md +++ b/docs/architecture/playback-protocol-v3.md @@ -1246,14 +1246,29 @@ That split makes the ordering load-bearing: emitted. A duplicate probe write, a second replica, or a retried heartbeat finds the generation settled and replays the stored decision instead of emitting a second event for one correction. -- The withdrawal is durable and retried until a capable client acknowledges it: - the entry's delivery state (`announced_at`) is separate from its decision, so - a settled but un-delivered invalidation is re-attempted on a later - heartbeat/attach/probe pass while the session still negotiates both - capabilities. The retry is bounded (an in-process window, comfortably larger - than the probe budget); after the bound the correction still lands on the - next start or reconnect, which plans against the already-corrected - inventory. Once `announced_at` is set, the generation emits nothing again. +- The withdrawal is durable and stops on **completion, not on send**. `announced_at` + records that the hub accepted the command, which proves only that the server + wrote it; a client can still disconnect before reading it, fail its replan, or + reconnect on another node later. So delivery ends when the attempt's current plan + id moves off the plan the entry withdraws — any replan commits a new plan, and + whether the client adopted our correction or chose something else, both end the + obligation to keep withdrawing a plan it no longer plays. Until then, and only + while the session still negotiates both capabilities, the withdrawal is + re-attempted on later heartbeat/attach/probe passes. The bound is an explicit + rate-and-burst policy rather than an eventual-silence guarantee: one burst of 20 + accepted deliveries, a 15-minute cooldown before the next burst, and an immediate + fresh burst whenever the session's control connection reconnects. A client that + keeps reconnecting without adopting the correction keeps being reminded; one that + stays connected and ignores it stops being pushed. Only accepted deliveries count, + capacity is reserved atomically before each send and returned when the write fails, + and the in-process state is dropped on session teardown and swept when idle, so it + stays bounded. Past the ceiling the correction still lands on the next start, which + plans against the already-corrected inventory. Every + re-delivery of one decision reuses the same command id, derived from the session + and generation, so a client that receives it twice recognizes one withdrawal + rather than two competing ones. The corrected identity deliberately does not + decide completion: it is recorded against the still-current plan, so comparing it + there compares against the stale selection and can match by coincidence. - The replan-request id is derived from the plan and the target audio index, both immutable, while the body also embeds the live position, which is not. A replay of the stored decision therefore carries the same id **and** the same diff --git a/internal/api/handlers/playback_reconcile_audio.go b/internal/api/handlers/playback_reconcile_audio.go index a65de253fe..95dcef40ca 100644 --- a/internal/api/handlers/playback_reconcile_audio.go +++ b/internal/api/handlers/playback_reconcile_audio.go @@ -13,8 +13,6 @@ import ( "sync" "time" - "github.com/google/uuid" - "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/userstore" @@ -198,25 +196,27 @@ func (h *PlaybackHandler) reconcileAutoAudioSelection(ctx context.Context, sessi } if entry := playback.FindAudioReconcileEntry(record.AudioReconcileLedger, generation, live.ID); entry != nil { // The attempt row already carries the ledger, so this needs no second - // store read. A settled + announced generation is never re-evaluated - // and never re-announced: a retried probe write, a second replica or - // a retried heartbeat replays the stored decision instead of emitting - // a second event for one correction. - if entry.Decision == playback.AudioReconcileInvalidated && - entry.AnnouncedAt == nil && - h.sessionNegotiatedPlanInvalidation(ctx, live.ID) && + // store read. A settled generation is never re-decided: a retried probe + // write, a second replica or a retried heartbeat replays the stored + // decision instead of deciding one correction twice. + if entry.Decision != playback.AudioReconcileInvalidated { + return + } + // Stop on completion, not on send. A successful RealtimeHub.Send + // proves the server wrote the command; it does not prove the client + // read it, replanned onto it, or stayed connected long enough to + // commit. Keying silence on the send would strand the correction for + // the rest of the session whenever the client disconnects before + // processing, its replan fails transiently, or its reconnect lands + // after the previous bound. Completion is the state the server can + // actually observe: the corrected selection is committed, or the stale + // plan is superseded by the replacement replan. + if audioReconcileCorrectionCompleted(entry, record) { + return + } + if h.sessionNegotiatedPlanInvalidation(ctx, live.ID) && h.sessionNegotiatedDefaultAudioReconcileResponse(ctx, live.ID) { - // Settled but never delivered: the original announce missed (no - // client attached yet, a replica without the session owner, a - // transient hub error). Re-attempt on this heartbeat/attach/probe - // pass so a client that connects AFTER the probe settled still - // receives the withdrawal. Bounded: retry only while the settled - // decision is still inside audioReconcileAnnounceWindow; after the - // bound the correction still lands on the next start/reconnect, - // which plans against the corrected inventory anyway. - if withinAudioReconcileAnnounceWindow(entry, time.Now()) { - h.announceAudioReconcileInvalidation(ctx, live, record, *entry) - } + h.announceAudioReconcileInvalidation(ctx, live, record, *entry) } return } @@ -323,36 +323,238 @@ func (h *PlaybackHandler) settleAudioReconcileInvalidation(ctx context.Context, h.announceAudioReconcileInvalidation(ctx, live, record, stored) } -// audioReconcileAnnounceWindow bounds how long the evaluation loop re-attempts -// delivering a settled but un-announced invalidation. -const audioReconcileAnnounceWindow = 2 * time.Minute - -// withinAudioReconcileAnnounceWindow reports whether the settled entry is -// still inside the retry window. The window is tracked in-process, keyed by -// (generation, session), because the settled entry carries no settled-at -// timestamp and the generation is a content hash with no wall clock. That is -// a deliberate bound: a short attach gap still retries, a permanently -// undeliverable session stops spinning, and the correction still lands on -// the next start regardless. +// audioReconcileAnnounceMaxAttempts bounds the DELIVERED re-announce attempts for +// one outstanding withdrawal, so a session whose client ignores the command stops +// receiving it instead of being pushed for its whole lifetime. +// +// Only attempts the hub actually accepted count. A pass with no control socket +// attached, or one whose write failed, must not consume the budget: those are +// precisely the cases that end with the client reconnecting, and a ceiling spent +// on undeliverable passes would strand the correction in the running-session +// reconnect case that delivery exists to fix. The bound therefore throttles a +// client that receives and ignores the withdrawal, and never abandons one that has +// not received it at all. +// +// The budget also rearms. A new connection registration for the session clears the +// count, because a reconnecting client is a fresh candidate for delivery, and a +// cooldown window releases it on the same track for a client that never +// re-registers. Exhausting the budget is not permanent abandonment: either path can +// rearm it, and past both the correction still lands on the next start, which plans +// against the verified inventory anyway. +const ( + audioReconcileAnnounceMaxAttempts = 20 + audioReconcileAnnounceCooldown = 15 * time.Minute +) var ( - audioReconcileFirstSeen = map[string]time.Time{} - audioReconcileFirstSeenMu sync.Mutex + audioReconcileAttempts = map[string]audioReconcileAttemptState{} + audioReconcileAttemptsMu sync.Mutex + // audioReconcileLastSweep bounds how often orphaned retry state is swept, so + // the sweep cost is amortized rather than paid per evaluation. + audioReconcileLastSweep time.Time ) -func withinAudioReconcileAnnounceWindow(entry *playback.AudioReconcileEntryV3, now time.Time) bool { +// audioReconcileAttemptRetention is how long retry state outlives its last +// update before the sweep may drop it. It is generous relative to the cooldown: +// state that is still within its cooldown must be kept, because that is what +// bounds the next burst, and dropping it early would hand a permanently +// non-adopting client a fresh budget on every sweep. +const ( + audioReconcileAttemptRetention = audioReconcileAnnounceCooldown + time.Hour + audioReconcileSweepInterval = 5 * time.Minute +) + +// audioReconcileAttemptState tracks re-announce delivery for one outstanding +// (generation, session) withdrawal. +type audioReconcileAttemptState struct { + Delivered int + DeliveredAt time.Time + UpdatedAt time.Time + // Epoch increases every time the burst is rearmed, so a failed send can only + // refund the reservation it actually took. Without it, a send that fails + // after a reconnect-rearm would decrement the fresh budget and hand the + // client capacity it never spent. + Epoch uint64 +} + +// forgetAudioReconcileAttempts drops the retry state for one session. It runs on +// session teardown so a long-lived process does not retain state for sessions +// that ended. +func forgetAudioReconcileAttemptsForSession(sessionID string) { + if sessionID == "" { + return + } + audioReconcileAttemptsMu.Lock() + defer audioReconcileAttemptsMu.Unlock() + suffix := "|" + sessionID + for key := range audioReconcileAttempts { + // The session id is the SUFFIX of every key for this session: the key is + // generation|session. No prefix matching is needed, and an empty-prefix + // match would delete every other session's budget on any teardown. + if strings.HasSuffix(key, suffix) { + delete(audioReconcileAttempts, key) + } + } +} + +// sweepAudioReconcileAttempts drops retry state that has been idle past +// audioReconcileAttemptRetention, bounding memory for sessions that ended +// without a teardown on this replica. Retention exceeds the cooldown, so state +// that still governs a burst is never swept: eviction must not become a way to +// replenish an active exhausted budget. +func sweepAudioReconcileAttempts(now time.Time) { + // Same bookkeeping normalization as reserveAudioReconcileDelivery: ages are + // compared correctly across zones either way, but one zone keeps the stored + // stamps comparable. + now = now.UTC() + audioReconcileAttemptsMu.Lock() + defer audioReconcileAttemptsMu.Unlock() + if !audioReconcileLastSweep.IsZero() && now.Sub(audioReconcileLastSweep) < audioReconcileSweepInterval { + return + } + audioReconcileLastSweep = now + for key, state := range audioReconcileAttempts { + if now.Sub(state.UpdatedAt) > audioReconcileAttemptRetention { + delete(audioReconcileAttempts, key) + } + } +} + +// reserveAudioReconcileDelivery atomically admits and spends one delivered-attempt +// from the budget for this outstanding entry, returning false when the burst is +// spent, or true plus the epoch the reservation was taken against. Admission, +// spending and expiry are decided under ONE lock so concurrent evaluations cannot +// each observe capacity and then all send: that is what keeps the ceiling a real +// ceiling rather than a hint under concurrency. +// +// A send that fails releases its own reservation via releaseAudioReconcileReservation +// with the epoch returned here, so a refund can never land on a newer burst. +func reserveAudioReconcileDelivery(entry *playback.AudioReconcileEntryV3, now time.Time) (bool, uint64) { if entry == nil { - return false + return false, 0 } key := entry.Generation + "|" + entry.SessionID - audioReconcileFirstSeenMu.Lock() - defer audioReconcileFirstSeenMu.Unlock() - first, ok := audioReconcileFirstSeen[key] - if !ok { - first = now - audioReconcileFirstSeen[key] = now + // Normalize so every stamp in this map is recorded in one zone. This is + // bookkeeping consistency, not a correctness fix: time.Sub compares instants + // correctly across zones, so the ages below are right either way. + now = now.UTC() + audioReconcileAttemptsMu.Lock() + state := audioReconcileAttempts[key] + admitted := false + if state.Delivered < audioReconcileAnnounceMaxAttempts { + state.Delivered++ + state.DeliveredAt = now + state.UpdatedAt = now + admitted = true + } else if !state.DeliveredAt.IsZero() && now.Sub(state.DeliveredAt) >= audioReconcileAnnounceCooldown { + // The burst expired: start exactly one fresh burst, counting this attempt + // as its first delivery. Resetting HERE rather than in the accounting call + // is deliberate: accounting only runs after a successful send, and nothing + // is sent while the budget is shut, so a reset there could never be + // reached. Resetting here also keeps a permanently non-adopting client + // bounded to one burst per cooldown. + state = audioReconcileAttemptState{ + Delivered: 1, DeliveredAt: now, UpdatedAt: now, Epoch: state.Epoch + 1, + } + admitted = true + } + if admitted { + audioReconcileAttempts[key] = state } - return now.Sub(first) <= audioReconcileAnnounceWindow + epoch := state.Epoch + audioReconcileAttemptsMu.Unlock() + if admitted { + // Sweep outside the lock: it takes the same mutex, and an entry this + // stale can only belong to a session that ended without teardown here. + sweepAudioReconcileAttempts(now) + } + return admitted, epoch +} + +// releaseAudioReconcileReservation returns an unused reservation to the budget, so +// a send that never reached the client does not consume capacity. +func releaseAudioReconcileReservation(entry *playback.AudioReconcileEntryV3, epoch uint64) { + if entry == nil { + return + } + key := entry.Generation + "|" + entry.SessionID + audioReconcileAttemptsMu.Lock() + defer audioReconcileAttemptsMu.Unlock() + state, ok := audioReconcileAttempts[key] + if !ok || state.Delivered <= 0 || state.Epoch != epoch { + // The burst was rearmed while this send was in flight. Refunding now + // would hand the client capacity the fresh budget never lent. + return + } + state.Delivered-- + audioReconcileAttempts[key] = state +} + +// rearmAudioReconcileAttempts releases the delivered-attempt budget for one +// outstanding entry. A new control connection is exactly the moment a previously +// undeliverable withdrawal becomes deliverable again, so a reconnect must never +// inherit an exhausted budget. +// +// The epoch advances rather than the state being dropped, so a send that was +// already in flight against the old burst cannot refund the rearmed one. +func rearmAudioReconcileAttempts(entry *playback.AudioReconcileEntryV3) { + if entry == nil { + return + } + key := entry.Generation + "|" + entry.SessionID + audioReconcileAttemptsMu.Lock() + defer audioReconcileAttemptsMu.Unlock() + state := audioReconcileAttempts[key] + now := time.Now().UTC() + audioReconcileAttempts[key] = audioReconcileAttemptState{ + // UpdatedAt stays current on purpose. A rearmed entry is LIVE again, and + // a zero stamp would let the very next cross-session sweep delete it — + // after which the next reserve would recreate the entry at a reused epoch, + // and a send still in flight against the old epoch would refund a burst + // it never reserved on. + // + // A fresh stamp is not on its own proof that no send is still in flight; + // it is what keeps the entry inside the retention window, and retention + // far exceeds any send's lifetime. + Epoch: state.Epoch + 1, UpdatedAt: now, + } +} + +// forgetAudioReconcileAttempts drops the in-process retry state for one entry. +func forgetAudioReconcileAttempts(entry *playback.AudioReconcileEntryV3) { + rearmAudioReconcileAttempts(entry) +} + +// sweepAudioReconcileAttemptRetention exposes the retention window to tests and +// documents the sweep bound they assert against. +const audioReconcileRetentionForTest = audioReconcileAttemptRetention + +// audioReconcileCorrectionCompleted reports whether this entry's correction is +// durably in effect, which is what ends delivery. +// +// The signal is the attempt's current plan id moving off the plan this entry +// withdraws. Any replan commits a new plan, so that movement means the client +// acted on the withdrawal — it either replanned onto our correction or picked +// something else, and both end our obligation to keep withdrawing a plan it no +// longer plays. +// +// The corrected identity deliberately does NOT decide completion. The entry is +// recorded against the still-current plan, so that plan already carries +// whatever the client was playing when the withdrawal went out; comparing our +// corrected identity against it compares against the STALE selection and can +// match by coincidence — a reorder that corrects one track can leave the +// committed index equal — reporting completion before the client ever replanned. +func audioReconcileCorrectionCompleted(entry *playback.AudioReconcileEntryV3, record *playback.AttemptRecordV3) bool { + if entry == nil || record == nil { + return false + } + if entry.PlanID == "" || record.CurrentPlanID == "" || record.CurrentPlanID == entry.PlanID { + // Still the withdrawn plan: the client has not replanned, so nothing + // about this decision is complete and the withdrawal stays outstanding. + return false + } + forgetAudioReconcileAttempts(entry) + return true } // reconcileProbeBudgetV3 bounds one reconcile evaluation: catalog read, @@ -930,14 +1132,25 @@ func (h *PlaybackHandler) announceAudioReconcileInvalidation(ctx context.Context "audio_index", *stored.AudioIndex) return } + // Reserve capacity BEFORE sending, under the same lock that enforces the + // ceiling. Accounting after the write would let concurrent evaluations each + // observe room and all send, so the ceiling would bound nothing. It sits + // after the capability gate so a client that can never receive the command + // never spends budget. + reserved, epoch := reserveAudioReconcileDelivery(&stored, time.Now().UTC()) + if !reserved { + return + } command, err := playback.NewPlanInvalidatedCommandForGeneration( live.ID, - uuid.NewString(), + audioReconcileCommandIDV3(live.ID, stored.Generation), planID, playback.PlanInvalidatedDefaultAudioReconciliation, stored.Generation, ) if err != nil { + // Nothing will be sent, so the reservation must not be spent. + releaseAudioReconcileReservation(&stored, epoch) slog.WarnContext(ctx, "default audio withdrawal command could not be built", "component", "api", "session", live.ID, "error", err) return @@ -949,23 +1162,48 @@ func (h *PlaybackHandler) announceAudioReconcileInvalidation(ctx context.Context command.DeadlineMS = int(audioReconcileInvalidationDeadline / time.Millisecond) } if err := h.RealtimeHub.Send(live.ID, command); err != nil { + // Return the reservation: a write that never reached the client must not + // spend the burst that bounds DELIVERED attempts. + releaseAudioReconcileReservation(&stored, epoch) slog.DebugContext(ctx, "default audio withdrawal push undelivered", "component", "api", "session", live.ID, "plan_id", planID, "error", err) return } - // The hub accepted the command for a capable client: mark the entry - // announced so the retry loop does not re-emit it, and persist that - // delivery state through the revision-checked ledger writer. Dedup on - // (generation, session, decision, digest) means this is an in-place - // enrichment of the settled entry, not a new decision. + // The hub accepted the command for a capable client. Record that attempt + // through the revision-checked ledger writer; dedup on (generation, + // session, decision, digest) makes it an in-place enrichment of the + // settled entry, not a new decision. + // + // This is a DELIVERY ATTEMPT, not completion. A nil return from Send + // proves the server wrote the command and nothing more, so it must not + // suppress further attempts: the client may disconnect before processing, + // fail its replan, or reconnect on a different node later. Silencing is + // keyed on audioReconcileCorrectionCompleted instead, which observes the + // committed correction or the superseded plan. AnnouncedAt therefore + // answers "has this been sent, and when" — useful for diagnostics and + // backoff — while the retry loop keys off completion. announcedAt := time.Now().UTC() h.recordAudioReconcileAnnouncement(ctx, live.ID, stored, &announcedAt) - slog.InfoContext(ctx, "default audio reconciled to the verified inventory", + slog.InfoContext(ctx, "default audio withdrawal delivered, awaiting the client replan", "component", "api", "session", live.ID, "plan_id", planID, "generation", stored.Generation, "audio_index", *stored.AudioIndex, "reason", AudioReconciliationReplanReason) } +// audioReconcileCommandIDV3 derives the withdrawal command id for one +// (session, generation) pair. The id is deliberately deterministic rather than +// a fresh UUID per attempt: every re-announce of the same unsettled correction +// is the SAME logical command, so a client that receives it twice (a retry +// after a dropped connection, or a duplicate across replicas) can recognize it +// as one withdrawal instead of two competing ones, and a send that succeeded +// while the AnnouncedAt write lost its CAS can no longer mint a second id for +// the same decision. Different generations are different corrections and get +// different ids. +func audioReconcileCommandIDV3(sessionID, generation string) string { + sum := sha256.Sum256([]byte("audio-reconcile\x00" + sessionID + "\x00" + generation)) + return "audio-reconcile-" + hex.EncodeToString(sum[:16]) +} + // recordAudioReconcileAnnouncement persists the AnnouncedAt stamp on a settled // entry through the same revision-checked ledger writer the decision used. A // CAS conflict (another writer advanced the ledger between the settle read @@ -1154,9 +1392,56 @@ func (h *PlaybackHandler) reconcilePendingAudioStartup(ctx context.Context, sess if strings.TrimSpace(record.SelectionOrigin) == "" { return } + // A reconnecting client is a fresh candidate for delivery, so it must never + // inherit an exhausted delivered-attempt budget from passes that ran while + // no socket was attached. + // + // Nothing to rearm on an ordinary pass. The budget is released by a genuine + // reconnect through the hub hook (see InstallAudioReconcileRearm) and by the + // burst cooldown. It must NOT be rearmed here: this entry point also runs on + // every progress report, and a rearm on each one would let a connected client + // that never replans receive withdrawals indefinitely. h.reconcileSessionDefaultAudio(deadlineCtx, session, record.EffectiveMediaFileID) } +// InstallAudioReconcileRearm wires the reconnect hook: when a session's control +// connection genuinely reconnects, every outstanding withdrawal on its attempt +// gets a fresh delivery budget. This is the only path that rearms besides the +// burst cooldown, and it is driven by a new connection rather than by an +// evaluation, so a client that stays connected and ignores the withdrawal is +// still bounded. +func (h *PlaybackHandler) InstallAudioReconcileRearm(ctx context.Context) { + if h == nil || h.RealtimeHub == nil || h.PlanStoreV3 == nil { + return + } + h.RealtimeHub.SetOnNewConnection(func(sessionID string) { + if h.PlanStoreV3 == nil || sessionID == "" { + return + } + record, err := h.PlanStoreV3.GetAttempt(ctx, sessionID) + if err != nil || record == nil { + return + } + rearmPendingAudioReconcileAttempts(record) + }) +} + +// rearmPendingAudioReconcileAttempts releases the delivered-attempt budget for +// every outstanding withdrawal on this attempt, so a reconnecting client is not +// throttled by attempts made before it was connected. +func rearmPendingAudioReconcileAttempts(record *playback.AttemptRecordV3) { + if record == nil { + return + } + for i := range record.AudioReconcileLedger.Entries { + entry := record.AudioReconcileLedger.Entries[i] + if entry.Decision != playback.AudioReconcileInvalidated { + continue + } + rearmAudioReconcileAttempts(&entry) + } +} + // reconcileReplanRequestID mints a generation-aware identity for one // automatic reconciliation decision: the reason prefix attributes the // change, the failed-plan id binds it to the exact plan generation it was diff --git a/internal/api/handlers/playback_reconcile_audio_invalidation_test.go b/internal/api/handlers/playback_reconcile_audio_invalidation_test.go index 3c45c4d207..56d8f4bf07 100644 --- a/internal/api/handlers/playback_reconcile_audio_invalidation_test.go +++ b/internal/api/handlers/playback_reconcile_audio_invalidation_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "sync" + "sync/atomic" "testing" "time" @@ -70,7 +71,16 @@ type invalidationFixture struct { } func newInvalidationFixture(t *testing.T, features []string, position float64) *invalidationFixture { + return newInvalidationFixtureWithStamp(t, features, position, nil) +} + +// newInvalidationFixtureWithStamp lets a test choose the probe stamp, and +// therefore the reconciliation generation. +func newInvalidationFixtureWithStamp(t *testing.T, features []string, position float64, stamp *time.Time) *invalidationFixture { t.Helper() + if stamp == nil { + stamp = &invalidationProbeStamp + } planTime := []models.AudioTrack{ {Index: 1, Language: "en", Codec: "aac", Channels: 2, Layout: "stereo"}, {Index: 3, Language: "pt", Codec: "eac3", Channels: 6, Layout: "5.1"}, @@ -82,7 +92,7 @@ func newInvalidationFixture(t *testing.T, features []string, position float64) * {Index: 3, Language: "pt", Codec: "eac3", Channels: 6, Layout: "5.1"}, {Index: 1, Language: "en", Codec: "aac", Channels: 2, Layout: "stereo"}, }, - ProbeUpdatedAt: &invalidationProbeStamp, + ProbeUpdatedAt: stamp, } session := &playback.Session{ ID: "11111111-1111-1111-1111-111111111111", UserID: 1, ProfileID: "profile-1", @@ -253,15 +263,19 @@ type failingSendConn struct{} func (failingSendConn) WriteJSON(_ any) error { return errors.New("client not attached yet") } -// TestReconcileAudioWithdrawalRetriedUntilCapableClientAcks covers the -// durable-delivery fix: a withdrawal whose first announce failed is retried -// on a later reconcile pass once a healthy hub lane exists, the entry then -// carries AnnouncedAt, and a third pass emits nothing. -func TestReconcileAudioWithdrawalRetriedUntilCapableClientAcks(t *testing.T) { +// TestReconcileAudioWithdrawalRetriedUntilCorrectionCommits covers durable +// delivery honestly: a Send that fails is retried, and a Send that SUCCEEDS is +// still not treated as completion. Only the committed correction ends delivery. +// +// 1. First pass over a failing lane: the decision settles, nothing is delivered. +// 2. A healthy lane appears: the withdrawal is delivered and AnnouncedAt records +// the attempt, but the entry is still outstanding — the client has not +// replanned, so a later pass re-sends the SAME command id. +// 3. The corrected selection commits: delivery stops permanently. +func TestReconcileAudioWithdrawalRetriedUntilCorrectionCommits(t *testing.T) { f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 12) - // First pass with a failing/nil hub send: the decision settles but - // AnnouncedAt stays nil. + // (1) A failing lane: the hub refuses the write, so nothing is delivered. f.handler.RealtimeHub = playback.NewRealtimeHub() failingRegistration := f.handler.RealtimeHub.Register(f.session.ID, failingSendConn{}) t.Cleanup(func() { f.handler.RealtimeHub.Unregister(failingRegistration) }) @@ -278,8 +292,9 @@ func TestReconcileAudioWithdrawalRetriedUntilCapableClientAcks(t *testing.T) { t.Fatalf("failing hub must deliver no plan_invalidated, got %d", got) } - // Swap in a healthy lane; a later reconcile pass must retry the - // withdrawal and stamp AnnouncedAt. + // (2) A healthy lane: the withdrawal is delivered. AnnouncedAt records the + // attempt, and because no correction has committed the entry is still + // outstanding, so the next pass re-sends the same stable command id. f.handler.RealtimeHub.Unregister(failingRegistration) healthyRegistration := f.handler.RealtimeHub.Register(f.session.ID, f.conn) t.Cleanup(func() { f.handler.RealtimeHub.Unregister(healthyRegistration) }) @@ -290,16 +305,590 @@ func TestReconcileAudioWithdrawalRetriedUntilCapableClientAcks(t *testing.T) { t.Fatal("the settled decision must survive the retry") } if retryEntry.AnnouncedAt == nil { - t.Fatal("AnnouncedAt must be set after a successful announce") + t.Fatal("AnnouncedAt must record the successful delivery attempt") } if got := len(f.conn.commands()); got != 1 { - t.Fatalf("a healthy hub must receive exactly one plan_invalidated on the retry, got %d", got) + t.Fatalf("a healthy hub must deliver exactly one plan_invalidated, got %d", got) + } + firstCommandID := f.conn.commands()[0].CommandID + if firstCommandID == "" { + t.Fatal("the withdrawal must carry a command id") + } + + // The client has not replanned yet: the correction is still outstanding, so + // the next pass re-delivers the SAME command rather than minting a new one. + afterSecond := f.reconcile(t) + secondEntry := playback.FindAudioReconcileEntry(afterSecond.AudioReconcileLedger, entry.Generation, f.session.ID) + if secondEntry == nil || secondEntry.AnnouncedAt == nil { + t.Fatal("the outstanding entry must keep its delivery stamp across retries") + } + commands := f.conn.commands() + if len(commands) != 2 { + t.Fatalf("an outstanding correction must be re-announced, got %d commands", len(commands)) + } + if commands[1].CommandID != firstCommandID { + t.Fatalf("retry command id = %q, want the stable id %q from the first attempt", + commands[1].CommandID, firstCommandID) + } + + // (3) The client answers: the corrected selection is committed. Delivery is + // over and no further withdrawal is emitted. + f.commitCorrectedAudioForTest(t, secondEntry.Request.SelectedTracks.Audio) + + afterCommit := f.reconcile(t) + commitEntry := playback.FindAudioReconcileEntry(afterCommit.AudioReconcileLedger, entry.Generation, f.session.ID) + if commitEntry == nil { + t.Fatal("the settled decision must survive the completion") + } + if got := len(f.conn.commands()); got != 2 { + t.Fatalf("a committed correction must stop delivery, got %d commands", got) + } + + // And it stays stopped on later heartbeats. + f.handler.reconcilePendingAudioStartup(context.Background(), f.session.ID) + if got := len(f.conn.commands()); got != 2 { + t.Fatalf("a completed correction must never re-announce, got %d commands", got) + } +} + +// TestReconcileAudioCommandIDIsStablePerGeneration pins the identity rule the +// retry contract depends on: one (session, generation) withdrawal always carries +// the same command id, so a client that sees it twice recognizes one command. +func TestReconcileAudioCommandIDIsStablePerGeneration(t *testing.T) { + sessionID := "11111111-1111-1111-1111-111111111111" + first := audioReconcileCommandIDV3(sessionID, "probe:2026-10-05T00:00:00Z") + again := audioReconcileCommandIDV3(sessionID, "probe:2026-10-05T00:00:00Z") + if first != again { + t.Fatalf("command id is not stable: %q then %q", first, again) + } + if other := audioReconcileCommandIDV3(sessionID, "probe:2026-10-06T00:00:00Z"); other == first { + t.Fatal("a different generation must get a different command id") + } + if other := audioReconcileCommandIDV3("22222222-2222-2222-2222-222222222222", "probe:2026-10-05T00:00:00Z"); other == first { + t.Fatal("a different session must get a different command id") + } +} + +// TestForgetAudioReconcileAttemptsIsSessionScoped pins that tearing one session +// down drops ONLY that session's retry state. An empty-prefix match here would +// wipe every other session's budget, so an unrelated playback stop could +// repeatedly replenish an exhausted burst. +func TestForgetAudioReconcileAttemptsIsSessionScoped(t *testing.T) { + const otherSession = "99999999-9999-9999-9999-999999999999" + first := &playback.AudioReconcileEntryV3{ + Decision: playback.AudioReconcileInvalidated, + Generation: "probe:2026-10-05T00:00:00Z", + SessionID: "11111111-1111-1111-1111-111111111111", + } + second := &playback.AudioReconcileEntryV3{ + Decision: playback.AudioReconcileInvalidated, + Generation: "probe:2026-10-05T00:00:00Z", + SessionID: otherSession, + } + firstKey := first.Generation + "|" + first.SessionID + secondKey := second.Generation + "|" + second.SessionID + + audioReconcileAttemptsMu.Lock() + audioReconcileAttempts[firstKey] = audioReconcileAttemptState{Delivered: 3, UpdatedAt: time.Now()} + audioReconcileAttempts[secondKey] = audioReconcileAttemptState{Delivered: 7, UpdatedAt: time.Now()} + audioReconcileAttemptsMu.Unlock() + + forgetAudioReconcileAttemptsForSession(first.SessionID) + + audioReconcileAttemptsMu.Lock() + _, keptFirst := audioReconcileAttempts[firstKey] + keptSecond := audioReconcileAttempts[secondKey] + audioReconcileAttemptsMu.Unlock() + if keptFirst { + t.Fatal("teardown must drop the torn-down session's retry state") + } + if keptSecond.Delivered != 7 { + t.Fatalf("another session's delivery count = %d, want it untouched at 7", keptSecond.Delivered) + } + delete(audioReconcileAttempts, firstKey) + delete(audioReconcileAttempts, secondKey) +} + +// TestReleaseAudioReconcileReservationIsEpochScoped pins that a failed send can +// only refund its OWN reservation. A refund landing on a newer burst would hand +// the client capacity the rearmed budget never lent, weakening the ceiling exactly +// when a reconnect re-opened delivery. +func TestReleaseAudioReconcileReservationIsEpochScoped(t *testing.T) { + entry := &playback.AudioReconcileEntryV3{ + Decision: playback.AudioReconcileInvalidated, + Generation: "probe:2026-10-05T00:00:01Z", + SessionID: "22222222-2222-2222-2222-222222222222", + } + key := entry.Generation + "|" + entry.SessionID + + reserved, epoch := reserveAudioReconcileDelivery(entry, time.Now()) + if !reserved { + t.Fatal("the first reservation must be admitted") + } + + // A reconnect rearms the burst: the budget is cleared and the epoch moves. + rearmAudioReconcileAttempts(entry) + rearmed, rearmEpoch := reserveAudioReconcileDelivery(entry, time.Now()) + if !rearmed || rearmEpoch == epoch { + t.Fatalf("a rearm must start a new epoch, got %d then %d", epoch, rearmEpoch) + } + + // The stale send now fails. Its refund must NOT touch the fresh burst. + releaseAudioReconcileReservation(entry, epoch) + + audioReconcileAttemptsMu.Lock() + delivered := audioReconcileAttempts[key].Delivered + currentEpoch := audioReconcileAttempts[key].Epoch + audioReconcileAttemptsMu.Unlock() + if delivered != 1 { + t.Fatalf("a stale refund changed the fresh burst to %d deliveries, want 1", delivered) + } + if currentEpoch != rearmEpoch { + t.Fatalf("a stale refund moved the epoch to %d, want %d", currentEpoch, rearmEpoch) + } + + // The fresh reservation still refunds correctly. + releaseAudioReconcileReservation(entry, rearmEpoch) + audioReconcileAttemptsMu.Lock() + delivered = audioReconcileAttempts[key].Delivered + audioReconcileAttemptsMu.Unlock() + if delivered != 0 { + t.Fatalf("a current refund must return capacity, got %d deliveries, want 0", delivered) + } + delete(audioReconcileAttempts, key) +} + +// TestAudioReconcileEpochSurvivesCrossSessionSweep closes the ABA hole through the +// sweep: a rearmed entry must not be evictable, because an eviction would let the +// next reserve recreate it at a reused epoch, and a send still in flight against +// the old epoch would then refund a burst it never reserved on. +func TestAudioReconcileEpochSurvivesCrossSessionSweep(t *testing.T) { + held := &playback.AudioReconcileEntryV3{ + Decision: playback.AudioReconcileInvalidated, + Generation: "probe:2026-10-05T00:00:02Z", + SessionID: "33333333-3333-3333-3333-333333333333", + } + // Another session's activity drives the periodic sweep. + churn := &playback.AudioReconcileEntryV3{ + Decision: playback.AudioReconcileInvalidated, + Generation: "probe:2026-10-05T00:00:03Z", + SessionID: "44444444-4444-4444-4444-444444444444", + } + heldKey := held.Generation + "|" + held.SessionID + + reserved, epoch := reserveAudioReconcileDelivery(held, time.Now()) + if !reserved { + t.Fatal("the first reservation must be admitted") + } + + // The client reconnects, so the budget rearms. + rearmAudioReconcileAttempts(held) + + // An unrelated session forces the periodic sweep that runs on every reserve. + audioReconcileLastSweep = time.Time{} + for i := 0; i < 3; i++ { + if admitted, _ := reserveAudioReconcileDelivery(churn, time.Now()); !admitted { + t.Fatal("the unrelated session's reservation must be admitted") + } + } + + audioReconcileAttemptsMu.Lock() + live, present := audioReconcileAttempts[heldKey] + rearmedEpoch := live.Epoch + rearmedStamp := live.UpdatedAt + audioReconcileAttemptsMu.Unlock() + if !present { + t.Fatal("a rearmed entry is live and must not be swept by another session's activity") + } + if rearmedStamp.IsZero() { + t.Fatal("rearming must leave a current UpdatedAt, or the entry looks idle to the sweep") + } + if rearmedEpoch == epoch { + t.Fatalf("rearming must advance the epoch past the in-flight %d, got %d", epoch, rearmedEpoch) + } + + // The next reservation on this session must NOT reuse the old epoch. + if _, next := reserveAudioReconcileDelivery(held, time.Now()); next == epoch { + t.Fatalf("a post-sweep reserve reused epoch %d", next) + } + + // The stale send now fails and must not refund the live burst. + releaseAudioReconcileReservation(held, epoch) + audioReconcileAttemptsMu.Lock() + delivered := audioReconcileAttempts[heldKey].Delivered + audioReconcileAttemptsMu.Unlock() + if delivered != 1 { + t.Fatalf("a stale refund changed the live burst to %d deliveries, want 1", delivered) + } + delete(audioReconcileAttempts, heldKey) + delete(audioReconcileAttempts, churn.Generation+"|"+churn.SessionID) +} + +// TestReconcileAudioCeilingHoldsUnderConcurrency pins the ceiling as a real +// ceiling. Admission and accounting were once separate steps, so concurrent +// evaluations could each observe remaining capacity and then all send; the bound +// would have held only for sequential passes. +func TestReconcileAudioCeilingHoldsUnderConcurrency(t *testing.T) { + f := newInvalidationFixtureWithStamp(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 12, freshReconcileGeneration()) + f.handler.InstallAudioReconcileRearm(context.Background()) + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil { + t.Fatal("the first evaluation must settle the generation") + } + + // Race well past the ceiling: the hub accepts every write, so the reservation + // is the only thing that can hold the line. + const racers = 200 + var wg sync.WaitGroup + wg.Add(racers) + for i := 0; i < racers; i++ { + go func() { + defer wg.Done() + f.handler.reconcileSessionDefaultAudio(context.Background(), f.session, f.verified.ID) + }() + } + wg.Wait() + + if got, want := len(f.conn.commands()), audioReconcileAnnounceMaxAttempts; got != want { + t.Fatalf("concurrent deliveries = %d, want exactly the ceiling %d", got, want) + } + audioReconcileAttemptsMu.Lock() + delivered := audioReconcileAttempts[entry.Generation+"|"+entry.SessionID].Delivered + audioReconcileAttemptsMu.Unlock() + if delivered > audioReconcileAnnounceMaxAttempts { + t.Fatalf("recorded deliveries = %d, must never exceed the ceiling %d", + delivered, audioReconcileAnnounceMaxAttempts) + } +} + +// TestReconcileAudioDisconnectThenReconnectRearms pins the COMMON reconnect +// lifecycle: Unregister DELETES the lane, so the next registration takes the +// hub's first-registration branch. Deciding "reconnect" by whether a lane already +// existed would miss exactly this case and leave the budget exhausted. +func TestReconcileAudioDisconnectThenReconnectRearms(t *testing.T) { + f := newInvalidationFixtureWithStamp(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 12, freshReconcileGeneration()) + f.handler.InstallAudioReconcileRearm(context.Background()) + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil { + t.Fatal("the first evaluation must settle the generation") + } + firstCommandID := f.conn.commands()[0].CommandID + + // Spend the budget. + for i := 0; i < audioReconcileAnnounceMaxAttempts; i++ { + f.reconcile(t) + } + spent := len(f.conn.commands()) + if spent < audioReconcileAnnounceMaxAttempts { + t.Fatalf("the client must receive the withdrawal up to the budget, got %d pushes", spent) + } + if got := spent; got > audioReconcileAnnounceMaxAttempts { + t.Fatalf("sequential deliveries = %d, must never exceed the ceiling %d", got, audioReconcileAnnounceMaxAttempts) + } + f.reconcile(t) + if got := len(f.conn.commands()); got != spent { + t.Fatalf("an exhausted budget must stop pushing: %d pushes, want %d", got, spent) + } + + // Disconnect, then reconnect. The hub fires its reconnect hook for a session + // whose lane was released, so the budget is restored and the withdrawal is + // re-delivered under the SAME command identity. + registration := f.handler.RealtimeHub.Register(f.session.ID, &reconcileInvalidationConn{}) + if !f.handler.RealtimeHub.Unregister(registration) { + t.Fatal("the first registration must be releasable") + } + reconnected := &reconcileInvalidationConn{} + next := f.handler.RealtimeHub.Register(f.session.ID, reconnected) + t.Cleanup(func() { f.handler.RealtimeHub.Unregister(next) }) + + f.reconcile(t) + commands := reconnected.commands() + if len(commands) != 1 { + t.Fatalf("a disconnect-then-reconnect must re-deliver the withdrawal: %d pushes, want 1", len(commands)) + } + if got := commands[0].CommandID; got != firstCommandID { + t.Fatalf("reconnect command id = %q, want the stable id %q", got, firstCommandID) + } +} + +// TestAudioReconcileAttemptStateIsBounded pins the memory bound: state for a +// finished session is dropped on teardown, and state idle past retention is swept +// so sessions that ended without teardown on this replica cannot accumulate. +func TestAudioReconcileAttemptStateIsBounded(t *testing.T) { + f := newInvalidationFixtureWithStamp(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 12, freshReconcileGeneration()) + f.handler.InstallAudioReconcileRearm(context.Background()) + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + key := entry.Generation + "|" + entry.SessionID + + audioReconcileAttemptsMu.Lock() + if _, ok := audioReconcileAttempts[key]; !ok { + audioReconcileAttemptsMu.Unlock() + t.Fatal("a delivered withdrawal must record its delivery state") + } + audioReconcileAttemptsMu.Unlock() + + // Teardown drops it. + forgetAudioReconcileAttemptsForSession(f.session.ID) + audioReconcileAttemptsMu.Lock() + _, afterTeardown := audioReconcileAttempts[key] + audioReconcileAttemptsMu.Unlock() + if afterTeardown { + t.Fatal("session teardown must drop the retry state for that session") + } + + // An idle entry beyond retention is swept, while a fresh one survives. + audioReconcileAttemptsMu.Lock() + audioReconcileAttempts[key] = audioReconcileAttemptState{Delivered: 3, UpdatedAt: time.Now().Add(-audioReconcileRetentionForTest - time.Minute)} + audioReconcileLastSweep = time.Time{} + audioReconcileAttemptsMu.Unlock() + sweepAudioReconcileAttempts(time.Now()) + + audioReconcileAttemptsMu.Lock() + _, stale := audioReconcileAttempts[key] + audioReconcileAttempts[key] = audioReconcileAttemptState{Delivered: 3, UpdatedAt: time.Now()} + audioReconcileLastSweep = time.Time{} + audioReconcileAttemptsMu.Unlock() + sweepAudioReconcileAttempts(time.Now()) + + audioReconcileAttemptsMu.Lock() + _, fresh := audioReconcileAttempts[key] + audioReconcileAttemptsMu.Unlock() + if stale { + t.Fatal("retry state idle past retention must be swept") + } + if !fresh { + t.Fatal("sweeping must not drop live retry state: eviction cannot replenish an exhausted budget") + } +} + +// TestReconcileAudioReconnectRearmsExhaustedBudget pins the reconnect case: a +// client that reconnects on the same active plan after the delivered-attempt +// budget is spent MUST still receive the withdrawal. A ceiling consumed by +// passes that ran with no socket attached would otherwise strand the +// correction in exactly the running-session reconnect case delivery exists to +// fix, which is permanent abandonment wearing a counter's clothing. +func TestReconcileAudioReconnectRearmsExhaustedBudget(t *testing.T) { + f := newInvalidationFixtureWithStamp(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 12, freshReconcileGeneration()) + f.handler.InstallAudioReconcileRearm(context.Background()) + + // Spend the whole budget on passes with a healthy lane: the client receives + // the withdrawal and ignores it every time. + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil { + t.Fatal("the first evaluation must settle the generation") + } + firstCommandID := f.conn.commands()[0].CommandID + for i := 0; i < audioReconcileAnnounceMaxAttempts; i++ { + f.reconcile(t) + } + if got := len(f.conn.commands()); got < audioReconcileAnnounceMaxAttempts { + t.Fatalf("the client must receive the withdrawal up to the budget, got %d pushes", got) + } + + // The budget is now spent and the plan has not moved: a bare evaluation + // pass must not push again. + before := len(f.conn.commands()) + f.reconcile(t) + if got := len(f.conn.commands()); got != before { + t.Fatalf("an exhausted budget must stop pushing: %d pushes, want %d", got, before) + } + + // The client reconnects. A new registration taking over the live lane is a + // genuine reconnect: the budget is rearmed and the next pass re-delivers + // under the SAME command identity, so one withdrawal is still one command. + reconnected := &reconcileInvalidationConn{} + registration := f.handler.RealtimeHub.Register(f.session.ID, reconnected) + t.Cleanup(func() { f.handler.RealtimeHub.Unregister(registration) }) + f.handler.reconcilePendingAudioStartup(context.Background(), f.session.ID) + commands := reconnected.commands() + if len(commands) != 1 { + t.Fatalf("a reconnect must re-deliver the outstanding withdrawal: %d pushes, want 1", len(commands)) + } + if got := commands[0].CommandID; got != firstCommandID { + t.Fatalf("reconnect command id = %q, want the stable id %q", got, firstCommandID) } - // A third pass on the announced entry must emit nothing. + // The client now acts on it, and delivery stops for good. + f.commitCorrectedAudioForTest(t, entry.Request.SelectedTracks.Audio) + f.reconcile(t) + f.handler.reconcilePendingAudioStartup(context.Background(), f.session.ID) + if got := len(reconnected.commands()); got != 1 { + t.Fatalf("a completed correction must stay silent, got %d pushes, want 1", got) + } +} + +// TestReconcileAudioExpiredBurstBuysOneMoreBurst pins the cooldown as a bounded +// re-open, not an open door: once a burst expires, the client gets exactly +// audioReconcileAnnounceMaxAttempts further deliveries and is then shut again. +// Opening admission without resetting the count let a permanently non-adopting +// client be pushed on every pass forever. +func TestReconcileAudioExpiredBurstBuysOneMoreBurst(t *testing.T) { + f := newInvalidationFixtureWithStamp(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 12, freshReconcileGeneration()) + f.handler.InstallAudioReconcileRearm(context.Background()) + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil { + t.Fatal("the first evaluation must settle the generation") + } + + // Spend the first burst. + for i := 0; i < audioReconcileAnnounceMaxAttempts; i++ { + f.reconcile(t) + } + spent := len(f.conn.commands()) + if spent < audioReconcileAnnounceMaxAttempts { + t.Fatalf("the client must receive the first burst, got %d pushes", spent) + } + + // Age the burst past the cooldown. The next ACCEPTED delivery opens a fresh + // burst and counts itself as attempt 1, so the client gets exactly + // audioReconcileAnnounceMaxAttempts further deliveries and is shut again. + // Resetting on delivery rather than opening admission is what stops a + // permanently non-adopting client from being pushed on every pass forever. + ageAudioReconcileBurstForTest(entry, time.Now().Add(-audioReconcileAnnounceCooldown-time.Minute)) + for i := 0; i < audioReconcileAnnounceMaxAttempts*2; i++ { + f.reconcile(t) + } + if got, want := len(f.conn.commands()), audioReconcileAnnounceMaxAttempts*2; got != want { + t.Fatalf("an expired burst must buy exactly %d further deliveries, got %d", want, got) + } + + // Still shut on further passes. + before := len(f.conn.commands()) + f.reconcile(t) + if got := len(f.conn.commands()); got != before { + t.Fatalf("the second burst must also shut: %d pushes, want %d", got, before) + } +} + +// TestReconcileAudioProgressReportDoesNotRearm pins that the budget is rearmed +// only by a genuine reconnect. reconcilePendingAudioStartup also runs on every +// progress report, so rearming there would let a connected client that never +// replans receive withdrawals indefinitely. +func TestReconcileAudioProgressReportDoesNotRearm(t *testing.T) { + f := newInvalidationFixtureWithStamp(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 12, freshReconcileGeneration()) + f.handler.InstallAudioReconcileRearm(context.Background()) + + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil { + t.Fatal("the first evaluation must settle the generation") + } + for i := 0; i < audioReconcileAnnounceMaxAttempts; i++ { + f.reconcile(t) + } + + // Repeated progress reports on the SAME connection must not rearm. + before := len(f.conn.commands()) + for i := 0; i < audioReconcileAnnounceMaxAttempts; i++ { + f.handler.reconcilePendingAudioStartup(context.Background(), f.session.ID) + } + if got := len(f.conn.commands()); got != before { + t.Fatalf("progress reports must not rearm an exhausted burst: %d pushes, want %d", got, before) + } + + // A genuine new connection does rearm, and keeps the command identity. + reconnected := &reconcileInvalidationConn{} + registration := f.handler.RealtimeHub.Register(f.session.ID, reconnected) + t.Cleanup(func() { f.handler.RealtimeHub.Unregister(registration) }) + if !f.handler.RealtimeHub.EmitNewConnectionForTest(f.session.ID) { + t.Fatal("the reconnect hook must be installed to rearm the delivery budget") + } + f.handler.reconcilePendingAudioStartup(context.Background(), f.session.ID) + commands := reconnected.commands() + if len(commands) != 1 { + t.Fatalf("a genuine reconnect must re-deliver the withdrawal: %d pushes, want 1", len(commands)) + } + if got := commands[0].CommandID; got != audioReconcileCommandIDV3(f.session.ID, entry.Generation) { + t.Fatalf("reconnect command id = %q, want the stable id for this session and generation", got) + } +} + +// freshGenerationCounter hands each budget test a distinct reconciliation +// generation. The delivered-attempt budget is keyed by (generation, session), +// and these tests assert absolute push counts, so tests sharing the fixture's +// default generation would let one test's burst spend another's budget. +var freshGenerationCounter atomic.Int64 + +func freshReconcileGeneration() *time.Time { + stamp := time.Now().UTC().Add(time.Duration(freshGenerationCounter.Add(1)) * time.Nanosecond) + return &stamp +} + +// ageAudioReconcileBurstForTest backdates the delivered-attempt stamp so the +// cooldown is observably satisfied without the test waiting for it. +func ageAudioReconcileBurstForTest(entry *playback.AudioReconcileEntryV3, at time.Time) { + audioReconcileAttemptsMu.Lock() + defer audioReconcileAttemptsMu.Unlock() + key := entry.Generation + "|" + entry.SessionID + state := audioReconcileAttempts[key] + state.DeliveredAt = at + audioReconcileAttempts[key] = state +} + +// TestReconcileAudioRetryBudgetIgnoresUndeliveredPasses pins what the ceiling +// counts: only attempts the hub ACCEPTED. Passes with no control socket must +// not consume the budget, because those are the passes that end with the client +// reconnecting. +func TestReconcileAudioRetryBudgetIgnoresUndeliveredPasses(t *testing.T) { + f := newInvalidationFixtureWithStamp(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 12, freshReconcileGeneration()) + + // No lane at all: the decision settles and nothing is delivered. + f.handler.RealtimeHub = playback.NewRealtimeHub() + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil { + t.Fatal("the first evaluation must settle the generation") + } + for i := 0; i < audioReconcileAnnounceMaxAttempts*2; i++ { + f.reconcile(t) + } + if got := len(f.conn.commands()); got != 0 { + t.Fatalf("an absent lane must deliver nothing, got %d pushes", got) + } + + // The client connects. The budget must be untouched, so the very first + // pass after attach delivers. + registration := f.handler.RealtimeHub.Register(f.session.ID, f.conn) + t.Cleanup(func() { f.handler.RealtimeHub.Unregister(registration) }) + f.reconcile(t) + commands := f.conn.commands() + if len(commands) != 1 { + t.Fatalf("undelivered passes must not spend the budget: %d pushes after attach, want 1", len(commands)) + } + + // And the same command identity as the very first decision. + if got := commands[0].CommandID; got != audioReconcileCommandIDV3(f.session.ID, entry.Generation) { + t.Fatalf("command id = %q, want the stable id derived from the session and generation", got) + } +} + +// TestReconcileAudioSupersededPlanStopsDelivery pins the other completion +// signal: once the plan the withdrawal names is no longer the attempt's current +// plan, a replacement replan superseded it and our delivery obligation is over, +// whatever the client then chose. +func TestReconcileAudioSupersededPlanStopsDelivery(t *testing.T) { + f := newInvalidationFixture(t, []string{playback.FeaturePlanInvalidatedV3, playback.FeatureDefaultAudioReconcileResponseV3}, 12) + f.reconcile(t) if got := len(f.conn.commands()); got != 1 { - t.Fatalf("an announced entry must never re-emit, got %d pushes", got) + t.Fatalf("the first evaluation must emit one withdrawal, got %d", got) + } + + // A replacement replan supersedes the withdrawn plan. + f.supersedeCurrentPlanForTest(t) + + after := f.reconcile(t) + entry := playback.FindAudioReconcileEntry(after.AudioReconcileLedger, reconcileGenerationV3(f.verified), f.session.ID) + if entry == nil { + t.Fatal("the settled decision must survive the supersede") + } + if got := len(f.conn.commands()); got != 1 { + t.Fatalf("a superseded plan must stop delivery, got %d commands", got) } } @@ -318,10 +907,31 @@ func TestReconcileAudioAnnouncedOnceStaysSilent(t *testing.T) { t.Fatalf("the first evaluation must emit exactly one push, got %d", got) } - // Second heartbeat on a settled+announced generation: silent. + // Heartbeats on a settled-but-outstanding generation may re-deliver the + // withdrawal, because delivery stops on the client's replan rather than on + // the send. What must never happen is a second DECISION or a second command + // identity: every re-delivery is the same command. f.handler.reconcilePendingAudioStartup(context.Background(), f.session.ID) - if got := len(f.conn.commands()); got != 1 { - t.Fatalf("an announced entry must never re-emit on a heartbeat, got %d pushes", got) + commands := f.conn.commands() + if len(commands) < 2 { + t.Fatal("a settled-but-uncompleted withdrawal must still be re-delivered") + } + first := commands[0].CommandID + for i, command := range commands { + if command.CommandID != first { + t.Fatalf("command %d has id %q, want the stable id %q: one withdrawal is one command", + i, command.CommandID, first) + } + } + + // Once the client has replanned, delivery stops for good. + f.commitCorrectedAudioForTest(t, entry.Request.SelectedTracks.Audio) + before := len(f.conn.commands()) + f.reconcile(t) + f.handler.reconcilePendingAudioStartup(context.Background(), f.session.ID) + if got := len(f.conn.commands()); got != before { + t.Fatalf("a completed correction must never re-announce: %d pushes after completion, want %d", + got, before) } } @@ -361,8 +971,23 @@ func TestReconcileAudioNoDoubleCorrection(t *testing.T) { // A second heartbeat after the generation settled. f.handler.reconcilePendingAudioStartup(context.Background(), f.session.ID) - if got := len(f.conn.commands()); got != 1 { - t.Fatalf("plan_invalidated pushes = %d after duplicate write and heartbeat, want exactly 1", got) + // Duplicate evidence must not produce a second correction. Re-delivery of + // the same outstanding withdrawal is expected (it stops on the client's + // replan, not on the send), so the invariant is one command identity for + // every push, never two competing corrections. + commands := f.conn.commands() + if len(commands) == 0 { + t.Fatal("the settled generation must have been withdrawn at least once") + } + first := commands[0].CommandID + for i, command := range commands { + if command.CommandID != first { + t.Fatalf("push %d has command id %q, want the stable id %q: one correction is one command", + i, command.CommandID, first) + } + if command.Payload == nil { + t.Fatalf("push %d carries no withdrawal payload", i) + } } replayed, err := f.handler.PlanStoreV3.GetAttempt(context.Background(), f.session.ID) if err != nil { @@ -828,6 +1453,55 @@ func clientReplanForInvalidation(f *invalidationFixture, failedPlanID string) pl } } +// replaceAttemptForTest overwrites the in-memory attempt row. Durable stores +// are insert-once for attempts, so only the memory store can model the post- +// replan row the reconcile loop reads back. +func (f *invalidationFixture) replaceAttemptForTest(t *testing.T, record playback.AttemptRecordV3) { + t.Helper() + store, ok := f.handler.PlanStoreV3.(*playback.MemoryPlanStoreV3) + if !ok { + t.Fatal("fixture needs a *playback.MemoryPlanStoreV3 to rewrite an attempt") + } + store.ReplaceAttempt(context.Background(), record) +} + +// commitCorrectedAudioForTest commits the corrected audio identity onto the +// attempt's current plan, which is what the server observes when the client has +// replanned onto the correction. It goes through the real store so the durable +// attempt row is what the next evaluation reads. +func (f *invalidationFixture) commitCorrectedAudioForTest(t *testing.T, corrected *playback.TrackIdentityV3) { + t.Helper() + record, err := f.handler.PlanStoreV3.GetAttempt(context.Background(), f.session.ID) + if err != nil { + t.Fatalf("get attempt before commit: %v", err) + } + if corrected == nil { + t.Fatal("the entry must carry the canonical corrected identity") + } + updated := *record + updated.CurrentPlan.SelectedTracks.Audio = copiedTrackIdentityForTest(corrected) + // A real replan commits a NEW plan; that plan-id movement is the + // server-observable signal that the client acted on the withdrawal. + updated.CurrentPlanID = record.CurrentPlanID + "-replaced" + f.replaceAttemptForTest(t, updated) + if live, liveErr := f.handler.sessionMgr.GetSession(f.session.ID); liveErr == nil && live != nil && corrected.Index != nil { + live.AudioTrackIndex = *corrected.Index + } +} + +// supersedeCurrentPlanForTest points the attempt at a different current plan, +// modeling a replacement replan that superseded the withdrawn one. +func (f *invalidationFixture) supersedeCurrentPlanForTest(t *testing.T) { + t.Helper() + record, err := f.handler.PlanStoreV3.GetAttempt(context.Background(), f.session.ID) + if err != nil { + t.Fatalf("get attempt before supersede: %v", err) + } + updated := *record + updated.CurrentPlanID = record.CurrentPlanID + "-replaced" + f.replaceAttemptForTest(t, updated) +} + // copiedTrackIdentityForTest clones a track identity so a test asserts on the // value the request builder produced, not on the plan's own pointer. func copiedTrackIdentityForTest(identity *playback.TrackIdentityV3) *playback.TrackIdentityV3 { diff --git a/internal/api/handlers/playback_service.go b/internal/api/handlers/playback_service.go index fd4b5ab0f0..0e55cfcf19 100644 --- a/internal/api/handlers/playback_service.go +++ b/internal/api/handlers/playback_service.go @@ -513,6 +513,10 @@ func (h *PlaybackHandler) forgetProgressSideEffectLock(sessionID string) { return } h.virtualDeliveryCleared.Delete(sessionID) + // The session is terminal, so no outstanding audio withdrawal can still be + // delivered or complete. Dropping its in-process retry state here keeps a + // long-lived process from retaining state for every session it ever served. + forgetAudioReconcileAttemptsForSession(sessionID) } func (h *PlaybackHandler) scrobblePauseTransitionV2(ctx context.Context, sess *playback.Session, wasPaused bool) { diff --git a/internal/api/router.go b/internal/api/router.go index aaefef7122..3a9beadd7c 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2046,6 +2046,7 @@ func newChiRouter(deps Dependencies) chi.Router { } commandTracker := playback.NewCommandTracker() playbackHandler.RealtimeHub = realtimeHub + playbackHandler.InstallAudioReconcileRearm(context.Background()) playbackHandler.CommandTracker = commandTracker playbackHandler.CommandDispatcher = playback.NewCommandDispatcher(deps.SessionMgr, realtimeHub, commandTracker) playbackCommandDispatcher = playbackHandler.CommandDispatcher diff --git a/internal/playback/protocol_store_v3.go b/internal/playback/protocol_store_v3.go index 7b8db7197c..74030640cb 100644 --- a/internal/playback/protocol_store_v3.go +++ b/internal/playback/protocol_store_v3.go @@ -297,14 +297,16 @@ type AudioReconcileEntryV3 struct { // it against this field is what makes the replan authoritatively a // reconciliation response rather than an identity heuristic. Reason string `json:"reason,omitempty"` - // AnnouncedAt is set once the withdrawal for this entry was accepted by - // the realtime hub for a client that negotiated both - // plan_invalidated_v1 and default_audio_reconcile_response_v1. It is - // the delivery state and is separate from the decision: an invalidated - // entry with a nil AnnouncedAt was never delivered to a capable - // client, so the reconcile loop retries it on a later heartbeat - // (bounded by audioReconcileAnnounceWindow). An invalidated entry - // with AnnouncedAt set is never re-emitted. + // AnnouncedAt records when the realtime hub last accepted this entry's + // withdrawal for a client that negotiated both plan_invalidated_v1 and + // default_audio_reconcile_response_v1. It is delivery ATTEMPT state and is + // deliberately not terminal: a nil value means the withdrawal has never + // been delivered, but a set value proves only that the server wrote the + // command, not that the client read it, replanned onto it, or stayed + // connected to commit. The reconcile loop therefore keys its silence on + // completion — the attempt's current plan id moving off this entry's PlanID + // — and keeps re-announcing an outstanding correction (with the same + // command id, bounded by an in-process attempt ceiling) until that happens. AnnouncedAt *time.Time `json:"announced_at,omitempty"` } diff --git a/internal/playback/realtime_hub.go b/internal/playback/realtime_hub.go index 243125d3c4..bae18bfb9d 100644 --- a/internal/playback/realtime_hub.go +++ b/internal/playback/realtime_hub.go @@ -3,6 +3,7 @@ package playback import ( "errors" "sync" + "time" ) // ErrRealtimeConnectionNotFound is returned when a session has no active realtime connection. @@ -36,12 +37,27 @@ type RealtimeHub struct { connections map[string]*sessionLane onInitialRegister func() onRegisterLaneLookup func(sessionID string, lane *sessionLane) + // onNewConnection fires when a session that had a live lane gets a new one: + // either a registration takes over the existing lane, or a client + // reconnects after Unregister deleted it. It never fires for a session's + // first connection and never for another replica's lane lookup. + onNewConnection func(sessionID string) + // recentlyReleased records when a session's lane was released, so the next + // registration for one is recognized as a reconnect. Entries are consumed by + // that registration and expire on their own: a session that ends for good + // never registers again, so a count-only bound would grow without limit. + // Retention must comfortably exceed a client reconnect gap while bounding + // memory to sessions released within it. + recentlyReleased map[string]time.Time + recentReleaseRetention time.Duration } // NewRealtimeHub creates an empty realtime hub. func NewRealtimeHub() *RealtimeHub { return &RealtimeHub{ - connections: make(map[string]*sessionLane), + connections: make(map[string]*sessionLane), + recentlyReleased: make(map[string]time.Time), + recentReleaseRetention: 10 * time.Minute, } } @@ -57,14 +73,32 @@ func (h *RealtimeHub) Register(sessionID string, conn RealtimeConnection) *Realt if lane == nil { lane = &sessionLane{conn: conn, generation: 1} h.connections[sessionID] = lane + // Whether this is the session's first connection or a reconnect after a + // disconnect cannot be told apart from the lane map alone: Unregister + // DELETES the lane, so a reconnecting client lands in exactly this + // branch. Deciding by "had a lane before" would miss the common case, so + // the reconnect signal is the registration itself. + releasedAt, reconnected := h.recentlyReleased[sessionID] + if reconnected && time.Since(releasedAt) > h.recentReleaseRetention { + // Too old to be a reconnect this can rely on; treat it as a first + // connection rather than re-arming an exhausted budget on it. + reconnected = false + } + delete(h.recentlyReleased, sessionID) h.mu.Unlock() if h.onInitialRegister != nil { h.onInitialRegister() } + if reconnected && h.onNewConnection != nil { + h.onNewConnection(sessionID) + } return &RealtimeRegistration{sessionID: sessionID, lane: lane, generation: 1} } h.mu.Unlock() + // A LANE LOOKUP is not a new connection: another replica inspecting this + // session's lane must not be observable as a delivery event. Only the + // successful takeover below counts. if h.onRegisterLaneLookup != nil { h.onRegisterLaneLookup(sessionID, lane) } @@ -83,9 +117,56 @@ func (h *RealtimeHub) Register(sessionID string, conn RealtimeConnection) *Realt } lane.mu.Unlock() + if h.onNewConnection != nil { + // This connection replaced a live one, so a consumer that gives up on + // an undeliverable command (audio withdrawal, download telemetry) can + // rearm here: a reconnecting client is a fresh delivery candidate. + h.onNewConnection(sessionID) + } return reg } +// EmitNewConnectionForTest fires the new-connection hook for one session so a +// test can exercise rearm behavior without synthesizing a lane takeover. It +// reports whether a hook was installed. +func (h *RealtimeHub) EmitNewConnectionForTest(sessionID string) bool { + if h == nil || sessionID == "" { + return false + } + h.mu.RLock() + hook := h.onNewConnection + h.mu.RUnlock() + if hook == nil { + return false + } + hook(sessionID) + return true +} + +// sweepRecentReleasesLocked drops reconnect marks older than the retention +// window. The caller holds h.mu. +func (h *RealtimeHub) sweepRecentReleasesLocked(now time.Time) { + for sessionID, releasedAt := range h.recentlyReleased { + if now.Sub(releasedAt) > h.recentReleaseRetention { + delete(h.recentlyReleased, sessionID) + } + } +} + +// SetOnNewConnection installs a hook that fires only when a registration takes +// over an existing lane (a reconnect), never for a session's first registration +// and never for another replica's lane lookup. It lets a consumer that gives up +// on undeliverable per-session commands rearm itself on reconnect. Only the +// latest hook is retained. +func (h *RealtimeHub) SetOnNewConnection(hook func(sessionID string)) { + if h == nil { + return + } + h.mu.Lock() + defer h.mu.Unlock() + h.onNewConnection = hook +} + // Unregister removes the active realtime connection for the given registration // token only if it still matches the currently registered connection. func (h *RealtimeHub) Unregister(reg *RealtimeRegistration) bool { @@ -113,6 +194,14 @@ func (h *RealtimeHub) Unregister(reg *RealtimeRegistration) bool { if current, ok := h.connections[reg.sessionID]; ok && current == lane && lane.closed && lane.conn == nil && lane.generation == nextGeneration { delete(h.connections, reg.sessionID) } + // This session had a live lane, so the next registration for it is a + // reconnect rather than a first connection. The mark is what lets the hook + // fire for the disconnect-then-reconnect lifecycle, which is the common one. + // Register consumes it and re-arms on the actual registration; releasing a + // lane is not itself a delivery event, so no hook fires here. + now := time.Now() + h.recentlyReleased[reg.sessionID] = now + h.sweepRecentReleasesLocked(now) h.mu.Unlock() lane.mu.Unlock() diff --git a/internal/playback/realtime_registration_test.go b/internal/playback/realtime_registration_test.go index de33c800c4..3b9ef53ce8 100644 --- a/internal/playback/realtime_registration_test.go +++ b/internal/playback/realtime_registration_test.go @@ -3,6 +3,9 @@ package playback import ( "errors" "testing" + "time" + + "github.com/google/uuid" ) type registrationTestConn struct{} @@ -89,3 +92,83 @@ func TestCurrentRealtimeRegistrationSerializesReplacement(t *testing.T) { t.Fatal("old disconnect removed successor") } } + +// TestRealtimeHubRecentlyReleasedIsBounded covers the reconnect mark's retention. +// A session that ends for good never registers again, so an unbounded mark would +// accumulate one entry per session the process ever served. +func TestRealtimeHubRecentlyReleasedIsBounded(t *testing.T) { + hub := NewRealtimeHub() + + // Churn through many sessions that register and disconnect without returning. + for i := 0; i < 500; i++ { + reg := hub.Register(uuid.NewString(), registrationTestConn{}) + if reg == nil { + t.Fatalf("register %d: no registration", i) + } + if !hub.Unregister(reg) { + t.Fatalf("unregister %d: not released", i) + } + } + + hub.mu.RLock() + retention := hub.recentReleaseRetention + hub.mu.RUnlock() + if retention <= 0 { + t.Fatal("the reconnect mark needs a positive retention window to be bounded") + } + // Every mark belongs to a session that just disconnected, so they are all + // fresh: the bound is time, not count. Assert the mark is a timestamp and that + // sweeping drops the stale ones. + hub.mu.Lock() + for sessionID := range hub.recentlyReleased { + hub.recentlyReleased[sessionID] = time.Now().Add(-2 * retention) + } + hub.sweepRecentReleasesLocked(time.Now()) + held := len(hub.recentlyReleased) + hub.mu.Unlock() + if held != 0 { + t.Fatalf("reconnect marks held = %d after every mark went stale, want 0", held) + } +} + +// TestRealtimeHubReconnectMarkFiresWithinRetention pins that a genuine +// disconnect-then-reconnect inside the window is still recognized, which is the +// lifecycle the mark exists for, and that neither a first connection nor a lane +// release is reported as one. +func TestRealtimeHubReconnectMarkFiresWithinRetention(t *testing.T) { + hub := NewRealtimeHub() + var reconnected []string + hub.SetOnNewConnection(func(sessionID string) { reconnected = append(reconnected, sessionID) }) + + sessionID := uuid.NewString() + reg := hub.Register(sessionID, registrationTestConn{}) + if len(reconnected) != 0 { + t.Fatal("a session's first connection is not a reconnect") + } + if !hub.Unregister(reg) { + t.Fatal("release the first connection") + } + if len(reconnected) != 0 { + t.Fatal("releasing a lane is not itself a delivery event") + } + + next := hub.Register(sessionID, registrationTestConn{}) + if next == nil { + t.Fatal("the reconnect must register") + } + if len(reconnected) != 1 || reconnected[0] != sessionID { + t.Fatalf("reconnect hook fired %v, want exactly one call for %s", reconnected, sessionID) + } + + // A takeover of a still-live lane is also a reconnect. + if !hub.Unregister(next) { + t.Fatal("release the reconnected lane") + } + third := hub.Register(sessionID, registrationTestConn{}) + if third == nil { + t.Fatal("the third connection must register") + } + if len(reconnected) != 2 { + t.Fatalf("reconnect hook fired %d times, want 2", len(reconnected)) + } +} From 665fcff36fe89f48347daa6b4cb085ba89bc514b Mon Sep 17 00:00:00 2001 From: drondeseries Date: Mon, 5 Oct 2026 22:49:10 -0400 Subject: [PATCH 14/16] fix(playback): carry the withdrawal answer on the v2 replan body The web client sends answers_plan_invalidation when it answers a default_audio_reconciliation withdrawal, but the v2 replan schema did not declare the property. Because the v2 body forbids additional properties, a capable client's replan was refused outright rather than merely losing the correlation, so the correction could not land on the native surface at all. Add the optional request property to PlaybackReplanBody, map it in domain(), and regenerate the OpenAPI artifact, web types, and contract fixtures. The contract linter classifies the change as a new optional request property: additive and non-breaking, which is what the v2 additive-only rule requires before it locks. Also correct two protocol-doc errors found alongside it: the identity heuristic is a fallback for clients WITHOUT the echo capability, not for every mismatching echo, and the plan_invalidated example carried an HTML comment inside its JSON block. Add the missing conformance scenarios for the marked answer and the unmarked re-pick, so the correlation is pinned on the wire instead of only in prose. Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/playbackfixtures/main.go | 33 +++ .../api/v2/fixtures/get_system_info_ok.json | 2 +- contracts/api/v2/openapi.json | 5 + docs/architecture/playback-protocol-v3.md | 21 +- internal/apiv2/playback.go | 47 ++-- internal/apiv2/playback_lifecycle_test.go | 92 +++++++ internal/playback/contract/contract_test.go | 40 +++ .../protocol_v3/conformance_matrix.json | 245 ++++++++++++++++++ web/src/api/v2/schema.ts | 2 + 9 files changed, 458 insertions(+), 29 deletions(-) diff --git a/cmd/playbackfixtures/main.go b/cmd/playbackfixtures/main.go index ead95594ef..d7bedc18ec 100644 --- a/cmd/playbackfixtures/main.go +++ b/cmd/playbackfixtures/main.go @@ -750,6 +750,9 @@ func goldenConformanceMatrix() playback.ConformanceMatrixV3 { fail("golden decision has no plan") } trackIndex := 1 + // The corrected default index after a probe reorder; distinct from the + // ordinal the stale plan carried, which is what makes the answer meaningful. + reconcileIndex := 2 trackChange := goldenReplanRequest() trackChange.Operation = playback.ReplanOperationTrackChangeV3 trackChange.ReplanRequestID = "replan-track-change-0001" @@ -770,6 +773,34 @@ func goldenConformanceMatrix() playback.ConformanceMatrixV3 { seekReanchor.Failure = playback.FailureV3{} seekReanchor.PositionSeconds = 321.25 + // The default-audio withdrawal answered through the dedicated correlation. + // It is an INTENT replan, not failure recovery, and it echoes the withdrawal's + // reason: that echo is the only signal separating "answering the withdrawal" + // from "re-picking the track the viewer already had". Both scenarios below + // advertise the capability, because the server's handling FORKS on it — a + // capable client that omits the marker keeps its own selection, so a vector + // without the feature would not exercise the answer at all. + reconcileFeatures := []string{ + playback.FeaturePlaybackPlanV3, + playback.FeaturePlanInvalidatedV3, + playback.FeatureDefaultAudioReconcileResponseV3, + } + reconcileAnswer := goldenReplanRequest() + reconcileAnswer.Operation = playback.ReplanOperationTrackChangeV3 + reconcileAnswer.ReplanRequestID = "replan-reconcile-answer-0001" + reconcileAnswer.Failure = playback.FailureV3{} + reconcileAnswer.ClientFeatures = reconcileFeatures + reconcileAnswer.AnswersPlanInvalidation = playback.PlanInvalidatedDefaultAudioReconciliation + reconcileAnswer.SelectedTracks.Audio = &playback.TrackIdentityV3{ID: "", Index: &reconcileIndex} + // The same replan from the SAME capable client without the marker: an ordinary + // viewer re-pick, which the server must not read as the answer. The pair is + // only meaningful together, since the difference between them is the whole + // correlation. + reconcileAnswer.PositionSeconds = 321.25 + reconcileUnmarked := reconcileAnswer + reconcileUnmarked.ReplanRequestID = "replan-reconcile-unmarked-0001" + reconcileUnmarked.AnswersPlanInvalidation = "" + trackChange.PositionSeconds = 321.25 qualityChange.PositionSeconds = 321.25 trackDuplicate := trackChange @@ -785,6 +816,8 @@ func goldenConformanceMatrix() playback.ConformanceMatrixV3 { qualityMidSeek := qualityChange qualityMidSeek.ReplanRequestID = "replan-quality-mid-seek-0001" replans := []playback.ReplanScenarioV3{ + {Name: "default_audio_reconcile_answer", Category: "default_audio_reconciliation", Request: reconcileAnswer, Expected: playback.ReplanExpectationV3{HTTPStatus: http.StatusOK, PreserveUnmodifiedTracks: true, PositionSeconds: 321.25, PositionPreserved: true}}, + {Name: "default_audio_reconcile_without_marker", Category: "default_audio_reconciliation", Request: reconcileUnmarked, Expected: playback.ReplanExpectationV3{HTTPStatus: http.StatusOK, PreserveUnmodifiedTracks: true, PositionSeconds: 321.25, PositionPreserved: true}}, {Name: "track_change", Category: "track_change_replan", Request: trackChange, Expected: playback.ReplanExpectationV3{HTTPStatus: http.StatusOK, PreserveUnmodifiedTracks: true}}, {Name: "quality_change", Category: "quality_change_replan", Request: qualityChange, Expected: playback.ReplanExpectationV3{HTTPStatus: http.StatusOK, SelectedQuality: resolutionHD}}, {Name: "output_change", Category: "output_change_replan", Request: outputChange, Expected: playback.ReplanExpectationV3{HTTPStatus: http.StatusOK, PreserveUnmodifiedTracks: true}}, diff --git a/contracts/api/v2/fixtures/get_system_info_ok.json b/contracts/api/v2/fixtures/get_system_info_ok.json index f3f19abca7..33afe8998c 100644 --- a/contracts/api/v2/fixtures/get_system_info_ok.json +++ b/contracts/api/v2/fixtures/get_system_info_ok.json @@ -1,7 +1,7 @@ { "server_version": "unavailable", "api_major": 2, - "contract_digest": "c22f876d212ba91c478eba68371c0bba9f352c81ed639341f25d392a697ef560", + "contract_digest": "855d9197130c80371dc1c84364cef8c1ac3ffc2a0d9285e213e28b23ca23aafd", "links": { "openapi": "/api/v2/openapi.json", "capabilities": "/api/v2/capabilities", diff --git a/contracts/api/v2/openapi.json b/contracts/api/v2/openapi.json index 75cce72d03..7a973ceaa2 100644 --- a/contracts/api/v2/openapi.json +++ b/contracts/api/v2/openapi.json @@ -35068,6 +35068,11 @@ "PlaybackReplanBody": { "additionalProperties": false, "properties": { + "answers_plan_invalidation": { + "description": "Echoes the reason from the plan_invalidated command this replan answers. Correlates a client that negotiated default_audio_reconcile_response_v1's response with the server's own withdrawal; omitting it on such a replan leaves the viewer's selection in place. Not trust-sensitive: at worst it names a correction the server already decided and announced.", + "maxLength": 64, + "type": "string" + }, "attempt_count": { "format": "int64", "maximum": 8, diff --git a/docs/architecture/playback-protocol-v3.md b/docs/architecture/playback-protocol-v3.md index 3b58b3db0e..2db5209b8b 100644 --- a/docs/architecture/playback-protocol-v3.md +++ b/docs/architecture/playback-protocol-v3.md @@ -1102,13 +1102,12 @@ any other, not a fire-and-forget event: "reason": "video_copy_unsafe", "deadline_ms": 8000, "payload": {"reason": "video_copy_unsafe", "plan_id": ""} - - } ``` +§6.1.1 adds one more reason to this command; for that reason the payload also +carries an additive `generation` field, and nothing else about the envelope changes. + `payload.plan_id` names the plan being withdrawn, which is not necessarily the one on screen: a client that has already replanned past it has nothing to do and completes the command as a no-op. That is why the field is required — acting @@ -1330,7 +1329,19 @@ Six rules bound it: server itself decided and announced, whereas forging `Automatic` would impersonate server reconciliation and suppress the viewer's preference persistence. An echo is a required input, never an authority: a mismatching - echo falls through to the identity heuristic below. + echo never lets a client claim a reconciliation response, and it does not fall + through to the identity heuristic below when the client negotiated + `default_audio_reconcile_response_v1`. That heuristic exists for clients that + have *not* adopted the echo, and for them alone: falling back for a client that + advertised the capability would reintroduce the exact ambiguity the capability + removes, so a capable client that replans without the matching marker simply + keeps its own selection and the correction stays pending. + The field travels on the replan body of both surfaces: `/api/v1` decodes it + directly, and `/api/v2` declares it as an optional request property on the + replan body. It is part of the body the replan digest fingerprints, so a retry + that claims a different answer is detectable rather than silently replayed. + Because the v2 replan schema forbids additional properties, a client that sends + the field to a server that has not declared it is refused rather than ignored. - **The echoed audio identity is a fallback discriminator, not the contract.** Without an echo, a selection that names something other than the withdrawn plan's audio is read as a fresh viewer choice that supersedes the diff --git a/internal/apiv2/playback.go b/internal/apiv2/playback.go index 94fc5a864c..73bf6e0693 100644 --- a/internal/apiv2/playback.go +++ b/internal/apiv2/playback.go @@ -242,28 +242,29 @@ type PlaybackMutationOutput struct { // PlaybackReplanBody is the v3 replan request: failure recovery, seek // re-anchor, and track, quality or output changes for a live session. type PlaybackReplanBody struct { - InstallationID ID `json:"installation_id" minLength:"1"` - ProtocolVersion int `json:"protocol_version"` - ClientFeatures []string `json:"client_features,omitempty"` - Operation playback.ReplanOperationV3 `json:"operation,omitempty" enum:"failure_recovery,seek_reanchor,seek_failure_recovery,track_change,quality_change,output_change"` - PlaybackAttemptID string `json:"playback_attempt_id" minLength:"8" maxLength:"128"` - ReplanRequestID string `json:"replan_request_id" minLength:"8" maxLength:"128" doc:"Client-minted identity of this replan; a retry with the same body replays the durable decision"` - FailedPlanID string `json:"failed_plan_id" minLength:"8" maxLength:"128"` - PlanAttemptID string `json:"plan_attempt_id" minLength:"8" maxLength:"128"` - PlanAttemptKey string `json:"plan_attempt_key" minLength:"8" maxLength:"128"` - AttemptedPlanKeys []string `json:"attempted_plan_keys" maxItems:"16"` - LocalMutations []string `json:"local_mutations,omitempty" maxItems:"8"` - AttemptCount int `json:"attempt_count" minimum:"1" maximum:"8"` - QualityPreference string `json:"quality_preference"` - PositionSeconds float64 `json:"position_seconds" minimum:"0"` - Metered bool `json:"metered"` - AutoFallback *bool `json:"auto_fallback,omitempty" nullable:"false" doc:"Re-negotiates the session's version-fallback intent. Set true when the viewer re-arms Auto mid-session; omitted leaves the start-time intent unchanged. Never authorizes a healthy mid-play switch."` - BandwidthEstimateKbps *int `json:"bandwidth_estimate_kbps,omitempty" nullable:"false"` - BandwidthCapKbps *int `json:"bandwidth_cap_kbps,omitempty" nullable:"false"` - SelectedTracks playback.SelectedTracksV3 `json:"selected_tracks"` - Failure playback.FailureV3 `json:"failure,omitzero"` - Capabilities playback.ClientCodecCapabilitiesV3 `json:"client_capabilities"` - ClientPlaybackContext playback.ClientPlaybackContextV3 `json:"client_playback_context"` + InstallationID ID `json:"installation_id" minLength:"1"` + ProtocolVersion int `json:"protocol_version"` + ClientFeatures []string `json:"client_features,omitempty"` + Operation playback.ReplanOperationV3 `json:"operation,omitempty" enum:"failure_recovery,seek_reanchor,seek_failure_recovery,track_change,quality_change,output_change"` + PlaybackAttemptID string `json:"playback_attempt_id" minLength:"8" maxLength:"128"` + ReplanRequestID string `json:"replan_request_id" minLength:"8" maxLength:"128" doc:"Client-minted identity of this replan; a retry with the same body replays the durable decision"` + FailedPlanID string `json:"failed_plan_id" minLength:"8" maxLength:"128"` + PlanAttemptID string `json:"plan_attempt_id" minLength:"8" maxLength:"128"` + PlanAttemptKey string `json:"plan_attempt_key" minLength:"8" maxLength:"128"` + AttemptedPlanKeys []string `json:"attempted_plan_keys" maxItems:"16"` + LocalMutations []string `json:"local_mutations,omitempty" maxItems:"8"` + AttemptCount int `json:"attempt_count" minimum:"1" maximum:"8"` + QualityPreference string `json:"quality_preference"` + PositionSeconds float64 `json:"position_seconds" minimum:"0"` + Metered bool `json:"metered"` + AutoFallback *bool `json:"auto_fallback,omitempty" nullable:"false" doc:"Re-negotiates the session's version-fallback intent. Set true when the viewer re-arms Auto mid-session; omitted leaves the start-time intent unchanged. Never authorizes a healthy mid-play switch."` + BandwidthEstimateKbps *int `json:"bandwidth_estimate_kbps,omitempty" nullable:"false"` + BandwidthCapKbps *int `json:"bandwidth_cap_kbps,omitempty" nullable:"false"` + AnswersPlanInvalidation string `json:"answers_plan_invalidation,omitempty" maxLength:"64" doc:"Echoes the reason from the plan_invalidated command this replan answers. Correlates a client that negotiated default_audio_reconcile_response_v1's response with the server's own withdrawal; omitting it on such a replan leaves the viewer's selection in place. Not trust-sensitive: at worst it names a correction the server already decided and announced."` + SelectedTracks playback.SelectedTracksV3 `json:"selected_tracks"` + Failure playback.FailureV3 `json:"failure,omitzero"` + Capabilities playback.ClientCodecCapabilitiesV3 `json:"client_capabilities"` + ClientPlaybackContext playback.ClientPlaybackContextV3 `json:"client_playback_context"` } type PlaybackReplanInput struct { PlaybackRequestHeaders @@ -436,7 +437,7 @@ func registerPlaybackReplan(reg *Registry, op func(method, path, id string) Oper }) } func (in PlaybackReplanBody) domain() playback.ReplanRequestV3 { - return playback.ReplanRequestV3{ProtocolVersion: in.ProtocolVersion, ClientFeatures: in.ClientFeatures, Operation: in.Operation, PlaybackAttemptID: in.PlaybackAttemptID, ReplanRequestID: in.ReplanRequestID, FailedPlanID: in.FailedPlanID, PlanAttemptID: in.PlanAttemptID, PlanAttemptKey: in.PlanAttemptKey, AttemptedPlanKeys: in.AttemptedPlanKeys, LocalMutations: in.LocalMutations, AttemptCount: in.AttemptCount, QualityPreference: in.QualityPreference, PositionSeconds: in.PositionSeconds, Metered: in.Metered, AutoFallback: in.AutoFallback, BandwidthEstimateKbps: in.BandwidthEstimateKbps, BandwidthCapKbps: in.BandwidthCapKbps, SelectedTracks: in.SelectedTracks, Failure: in.Failure, Capabilities: in.Capabilities, ClientPlaybackContext: in.ClientPlaybackContext} + return playback.ReplanRequestV3{ProtocolVersion: in.ProtocolVersion, ClientFeatures: in.ClientFeatures, Operation: in.Operation, PlaybackAttemptID: in.PlaybackAttemptID, ReplanRequestID: in.ReplanRequestID, FailedPlanID: in.FailedPlanID, PlanAttemptID: in.PlanAttemptID, PlanAttemptKey: in.PlanAttemptKey, AttemptedPlanKeys: in.AttemptedPlanKeys, LocalMutations: in.LocalMutations, AttemptCount: in.AttemptCount, QualityPreference: in.QualityPreference, PositionSeconds: in.PositionSeconds, Metered: in.Metered, AutoFallback: in.AutoFallback, BandwidthEstimateKbps: in.BandwidthEstimateKbps, BandwidthCapKbps: in.BandwidthCapKbps, AnswersPlanInvalidation: in.AnswersPlanInvalidation, SelectedTracks: in.SelectedTracks, Failure: in.Failure, Capabilities: in.Capabilities, ClientPlaybackContext: in.ClientPlaybackContext} } func registerPlaybackRouteEvents(reg *Registry, op func(method, path, id string) Operation) { Register(reg, op(http.MethodPost, "/route-events", opReportPlaybackRouteEvent), func(ctx context.Context, in *PlaybackRouteEventInput) (*PlaybackRouteEventOutput, error) { diff --git a/internal/apiv2/playback_lifecycle_test.go b/internal/apiv2/playback_lifecycle_test.go index b0f413b50a..b092fbde89 100644 --- a/internal/apiv2/playback_lifecycle_test.go +++ b/internal/apiv2/playback_lifecycle_test.go @@ -95,6 +95,98 @@ func TestPlaybackV2ReplanUsesTypedServiceAndDigest(t *testing.T) { } }) } +} + +// TestPlaybackV2ReplanCarriesPlanInvalidationAnswer pins the correlation the +// default-audio withdrawal depends on. A client that negotiated +// default_audio_reconcile_response_v1 answers a withdrawal by naming its reason +// on the replan; if v2 dropped the field the answer would be lost, and since the +// schema forbids additional properties the request would be refused outright +// rather than merely ignored. +func TestPlaybackV2ReplanCarriesPlanInvalidationAnswer(t *testing.T) { + const reason = "default_audio_reconciliation" + deps, _ := catalogDeps(t) + fake := &fakePlaybackService{} + deps.Playback = fake + data, err := os.ReadFile("../playback/testdata/protocol_v3/decision_response.json") + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(data, &fake.response); err != nil { + t.Fatal(err) + } + h := newTestHandler(t, deps) + + // The echoed reason survives strict validation and reaches the service. + body := playbackReplanFixture(t) + body["answers_plan_invalidation"] = reason + rec := do(t, h, http.MethodPost, Prefix+"/playback/11111111-1111-4111-8111-111111111111/replan", playbackJSON(t, body), viewerHeaders()) + if rec.Code != 200 { + t.Fatalf("replan answering a withdrawal: %d %s", rec.Code, rec.Body.String()) + } + if fake.replan.Request.AnswersPlanInvalidation != reason { + t.Fatalf("answers_plan_invalidation = %q, want %q", fake.replan.Request.AnswersPlanInvalidation, reason) + } + + // The field is optional: an ordinary intent replan omits it entirely and is + // still accepted, so a client that never sees a withdrawal is unaffected. + plain := playbackReplanFixture(t) + rec = do(t, h, http.MethodPost, Prefix+"/playback/11111111-1111-4111-8111-111111111111/replan", playbackJSON(t, plain), viewerHeaders()) + if rec.Code != 200 { + t.Fatalf("replan without the answer: %d %s", rec.Code, rec.Body.String()) + } + if fake.replan.Request.AnswersPlanInvalidation != "" { + t.Fatalf("answers_plan_invalidation = %q, want it absent", fake.replan.Request.AnswersPlanInvalidation) + } + + // The answer is part of the body the digest fingerprints, so a retry that + // claims a different answer is detectable rather than silently replayed. + answered := playbackReplanFixture(t) + answered["answers_plan_invalidation"] = reason + do(t, h, http.MethodPost, Prefix+"/playback/11111111-1111-4111-8111-111111111111/replan", playbackJSON(t, answered), viewerHeaders()) + answeredDigest := fake.replan.Digest + answered["answers_plan_invalidation"] = "some_other_reason" + do(t, h, http.MethodPost, Prefix+"/playback/11111111-1111-4111-8111-111111111111/replan", playbackJSON(t, answered), viewerHeaders()) + if fake.replan.Digest == answeredDigest { + t.Fatal("a changed answers_plan_invalidation kept the same digest") + } +} + +func TestPlaybackV2ReplanValidationRejectsBadAnswers(t *testing.T) { + deps, _ := catalogDeps(t) + fake := &fakePlaybackService{} + deps.Playback = fake + data, err := os.ReadFile("../playback/testdata/protocol_v3/decision_response.json") + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(data, &fake.response); err != nil { + t.Fatal(err) + } + h := newTestHandler(t, deps) + body := playbackReplanFixture(t) + body["answers_plan_invalidation"] = string(make([]byte, 65)) + rec := do(t, h, http.MethodPost, Prefix+"/playback/11111111-1111-4111-8111-111111111111/replan", playbackJSON(t, body), viewerHeaders()) + if rec.Code != 422 { + t.Fatalf("an over-long answer must be refused: %d %s", rec.Code, rec.Body.String()) + } +} + +// TestPlaybackV2ReplanMapsOperationErrors covers the replan error mapping that +// used to live at the end of TestPlaybackV2ReplanUsesTypedServiceAndDigest; it is +// its own test so the digest and validation assertions above stay readable. +func TestPlaybackV2ReplanMapsOperationErrors(t *testing.T) { + deps, _ := catalogDeps(t) + fake := &fakePlaybackService{} + deps.Playback = fake + data, err := os.ReadFile("../playback/testdata/protocol_v3/decision_response.json") + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(data, &fake.response); err != nil { + t.Fatal(err) + } + h := newTestHandler(t, deps) for _, tc := range []struct { err *handlers.PlaybackOperationError want ProblemType diff --git a/internal/playback/contract/contract_test.go b/internal/playback/contract/contract_test.go index 6971a50abc..928662f752 100644 --- a/internal/playback/contract/contract_test.go +++ b/internal/playback/contract/contract_test.go @@ -240,6 +240,9 @@ func TestConformanceMatrixEmbeddedWireBodiesSatisfySchemas(t *testing.T) { t.Errorf("scenario %q intent-only replan carries failure = %#v", name, failure) } } + if scenario["category"] == "default_audio_reconciliation" { + assertDefaultAudioReconcileScenario(t, name, request) + } } protocolSchemas := map[string]string{ "start_request": "start-request.schema.json", @@ -264,6 +267,43 @@ func TestConformanceMatrixEmbeddedWireBodiesSatisfySchemas(t *testing.T) { } } +// assertDefaultAudioReconcileScenario pins what the two reconciliation vectors +// exist to show: a capable client that echoes the withdrawal reason is answering +// it, and the SAME client omitting the marker is an ordinary re-pick. Schema +// validation alone cannot see that difference, so without these assertions the +// pair is two identical bodies that prove nothing. +// +// The capability must be advertised on both. The server's handling forks on it — +// a client that negotiated the capability is required to answer with the marker — +// so a vector without the feature would not exercise the correlation at all. +func assertDefaultAudioReconcileScenario(t *testing.T, name string, request map[string]any) { + t.Helper() + features, _ := request["client_features"].([]any) + if !slices.ContainsFunc(features, func(f any) bool { + return f == playback.FeatureDefaultAudioReconcileResponseV3 + }) { + t.Errorf("scenario %q does not advertise %s, so it cannot exercise the withdrawal answer", + name, playback.FeatureDefaultAudioReconcileResponseV3) + } + answer, hasAnswer := request["answers_plan_invalidation"] + switch name { + case "default_audio_reconcile_answer": + if !hasAnswer || answer != playback.PlanInvalidatedDefaultAudioReconciliation { + t.Errorf("scenario %q must answer with %q, got %#v", + name, playback.PlanInvalidatedDefaultAudioReconciliation, request["answers_plan_invalidation"]) + } + case "default_audio_reconcile_without_marker": + if hasAnswer { + t.Errorf("scenario %q is the unmarked re-pick and must carry no answer, got %#v", name, answer) + } + } + // An empty string on the wire is not the same as an absent field: one is a + // malformed echo, the other an ordinary replan. + if raw, present := request["answers_plan_invalidation"]; present && raw == "" { + t.Errorf("scenario %q sends an empty answer; omit the field instead", name) + } +} + // TestVendoredFixturesMatchGolden keeps the copies clients vendor honest. They // are published as server output, so a hand-edit here would hand every client a // body the server never produced. diff --git a/internal/playback/testdata/protocol_v3/conformance_matrix.json b/internal/playback/testdata/protocol_v3/conformance_matrix.json index b8b9df34ea..c62599ad44 100644 --- a/internal/playback/testdata/protocol_v3/conformance_matrix.json +++ b/internal/playback/testdata/protocol_v3/conformance_matrix.json @@ -5262,6 +5262,251 @@ } ], "replan_scenarios": [ + { + "name": "default_audio_reconcile_answer", + "category": "default_audio_reconciliation", + "request": { + "protocol_version": 3, + "client_features": [ + "playback_plan_v3", + "plan_invalidated_v1", + "default_audio_reconcile_response_v1" + ], + "operation": "track_change", + "playback_attempt_id": "attempt-golden-0001", + "replan_request_id": "replan-reconcile-answer-0001", + "failed_plan_id": "plan:478677870860e5e5108c18bff749b34b", + "plan_attempt_id": "plan-attempt-golden-0001", + "plan_attempt_key": "v3:f0144c47fa349e3e", + "attempted_plan_keys": [ + "v3:f0144c47fa349e3e" + ], + "attempt_count": 1, + "quality_preference": "auto", + "position_seconds": 321.25, + "metered": true, + "bandwidth_estimate_kbps": 3500, + "bandwidth_cap_kbps": 4000, + "answers_plan_invalidation": "default_audio_reconciliation", + "selected_tracks": { + "audio": { + "id": "", + "index": 2 + } + }, + "client_capabilities": { + "video_evidence": "exact", + "audio_evidence": "exact", + "codecs_video": [ + "h264" + ], + "codecs_video_hardware": [ + "h264" + ], + "codecs_audio": [ + "aac" + ], + "containers": [ + "mp4" + ], + "max_resolution": "1080p", + "hdr": false, + "video_decode": [ + { + "codec": "h264", + "profiles": [ + "high" + ], + "levels": [ + 41 + ], + "bit_depths": [ + 8 + ], + "max_width": 1920, + "max_height": 1080, + "max_frame_rate": 60, + "max_bitrate_kbps": 20000, + "hardware": true + } + ] + }, + "client_playback_context": { + "protocol_version": 3, + "form_factor": "tv", + "app_version": "3.0-test", + "device": { + "platform": "android", + "os_version": "15", + "manufacturer": "NVIDIA", + "model": "SHIELD Android TV", + "platform_details": { + "abis": "arm64-v8a", + "sdk_int": "35" + } + }, + "output": { + "output_context_id": "7" + }, + "deliveries": { + "original_http": { + "enabled": true, + "supported_on_device": true, + "containers": [ + "mp4" + ], + "video_codecs": [ + "h264" + ], + "audio_decode_codecs": [ + "aac" + ], + "audio_passthrough_codecs": [], + "subtitles": { + "embedded_text": true, + "sidecar_text": true, + "ass_styling": false, + "embedded_bitmap": false, + "sidecar_bitmap": false, + "font_attachments": false + }, + "features": [], + "auth_header_refresh": true, + "validated_claims": [], + "transformations": [] + } + } + } + }, + "expected": { + "http_status": 200, + "position_seconds": 321.25, + "position_preserved": true, + "preserve_unmodified_tracks": true + } + }, + { + "name": "default_audio_reconcile_without_marker", + "category": "default_audio_reconciliation", + "request": { + "protocol_version": 3, + "client_features": [ + "playback_plan_v3", + "plan_invalidated_v1", + "default_audio_reconcile_response_v1" + ], + "operation": "track_change", + "playback_attempt_id": "attempt-golden-0001", + "replan_request_id": "replan-reconcile-unmarked-0001", + "failed_plan_id": "plan:478677870860e5e5108c18bff749b34b", + "plan_attempt_id": "plan-attempt-golden-0001", + "plan_attempt_key": "v3:f0144c47fa349e3e", + "attempted_plan_keys": [ + "v3:f0144c47fa349e3e" + ], + "attempt_count": 1, + "quality_preference": "auto", + "position_seconds": 321.25, + "metered": true, + "bandwidth_estimate_kbps": 3500, + "bandwidth_cap_kbps": 4000, + "selected_tracks": { + "audio": { + "id": "", + "index": 2 + } + }, + "client_capabilities": { + "video_evidence": "exact", + "audio_evidence": "exact", + "codecs_video": [ + "h264" + ], + "codecs_video_hardware": [ + "h264" + ], + "codecs_audio": [ + "aac" + ], + "containers": [ + "mp4" + ], + "max_resolution": "1080p", + "hdr": false, + "video_decode": [ + { + "codec": "h264", + "profiles": [ + "high" + ], + "levels": [ + 41 + ], + "bit_depths": [ + 8 + ], + "max_width": 1920, + "max_height": 1080, + "max_frame_rate": 60, + "max_bitrate_kbps": 20000, + "hardware": true + } + ] + }, + "client_playback_context": { + "protocol_version": 3, + "form_factor": "tv", + "app_version": "3.0-test", + "device": { + "platform": "android", + "os_version": "15", + "manufacturer": "NVIDIA", + "model": "SHIELD Android TV", + "platform_details": { + "abis": "arm64-v8a", + "sdk_int": "35" + } + }, + "output": { + "output_context_id": "7" + }, + "deliveries": { + "original_http": { + "enabled": true, + "supported_on_device": true, + "containers": [ + "mp4" + ], + "video_codecs": [ + "h264" + ], + "audio_decode_codecs": [ + "aac" + ], + "audio_passthrough_codecs": [], + "subtitles": { + "embedded_text": true, + "sidecar_text": true, + "ass_styling": false, + "embedded_bitmap": false, + "sidecar_bitmap": false, + "font_attachments": false + }, + "features": [], + "auth_header_refresh": true, + "validated_claims": [], + "transformations": [] + } + } + } + }, + "expected": { + "http_status": 200, + "position_seconds": 321.25, + "position_preserved": true, + "preserve_unmodified_tracks": true + } + }, { "name": "track_change", "category": "track_change_replan", diff --git a/web/src/api/v2/schema.ts b/web/src/api/v2/schema.ts index 6a3db8c473..1de6518385 100644 --- a/web/src/api/v2/schema.ts +++ b/web/src/api/v2/schema.ts @@ -24627,6 +24627,8 @@ export interface components { sequence: number; }; PlaybackReplanBody: { + /** @description Echoes the reason from the plan_invalidated command this replan answers. Correlates a client that negotiated default_audio_reconcile_response_v1's response with the server's own withdrawal; omitting it on such a replan leaves the viewer's selection in place. Not trust-sensitive: at worst it names a correction the server already decided and announced. */ + answers_plan_invalidation?: string; /** Format: int64 */ attempt_count: number; attempted_plan_keys: string[]; From c35c10e59ec4f817257a3638ed3aa92fda3d0c6b Mon Sep 17 00:00:00 2001 From: drondeseries Date: Mon, 5 Oct 2026 23:56:20 -0400 Subject: [PATCH 15/16] fix(playback): read a fresh reconciliation ledger as revision zero A newly saved attempt carries an empty audio_reconcile_ledger '{}', and '{}'::jsonb->>'revision' is SQL NULL. Both ledger reads cast that expression to bigint and scan it into an int64, so the first read failed and the durable reconciliation ledger could never bootstrap on the production Postgres store: no correction could be recorded or announced for an ordinary new attempt. Wrap both reads in COALESCE(..., 0). Changing only the JSON tag or the migration default would not cover rows already written as '{}'. Add a Postgres-backed regression that saves an ordinary attempt, reads revision zero, records the first decision, and reads revision one. It is gated on SILO_TEST_DATABASE_URL like the rest of the planstore suite, so it skips where no migrated database is configured and runs in CI. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/playback/planstore/postgres.go | 4 +- internal/playback/planstore/postgres_test.go | 56 ++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/internal/playback/planstore/postgres.go b/internal/playback/planstore/postgres.go index fe79a82983..00d2fa099b 100644 --- a/internal/playback/planstore/postgres.go +++ b/internal/playback/planstore/postgres.go @@ -326,7 +326,7 @@ func (s *Postgres) RecordAudioReconciliation(ctx context.Context, sessionID stri var ledgerJSON []byte var revision int64 err = tx.QueryRow(ctx, ` - SELECT audio_reconcile_ledger, (audio_reconcile_ledger->>'revision')::bigint FROM playback_v3_attempts + SELECT audio_reconcile_ledger, COALESCE((audio_reconcile_ledger->>'revision')::bigint, 0) FROM playback_v3_attempts WHERE session_id = $1::uuid AND expires_at > NOW() FOR UPDATE`, sessionID).Scan(&ledgerJSON, &revision) if errors.Is(err, pgx.ErrNoRows) { @@ -382,7 +382,7 @@ func (s *Postgres) GetAudioReconcileLedger(ctx context.Context, sessionID string var ledgerJSON []byte var revision int64 err := s.db.QueryRow(ctx, ` - SELECT audio_reconcile_ledger, (audio_reconcile_ledger->>'revision')::bigint FROM playback_v3_attempts + SELECT audio_reconcile_ledger, COALESCE((audio_reconcile_ledger->>'revision')::bigint, 0) FROM playback_v3_attempts WHERE session_id = $1::uuid AND expires_at > NOW()`, sessionID).Scan(&ledgerJSON, &revision) if errors.Is(err, pgx.ErrNoRows) { return playback.AudioReconcileLedgerV3{}, 0, playback.ErrSessionNotFound diff --git a/internal/playback/planstore/postgres_test.go b/internal/playback/planstore/postgres_test.go index 5636bf1db4..421df25340 100644 --- a/internal/playback/planstore/postgres_test.go +++ b/internal/playback/planstore/postgres_test.go @@ -279,6 +279,62 @@ func TestPostgresPlanStore(t *testing.T) { } }) + // Regression test for the reconciliation ledger bootstrap. A freshly saved + // attempt carries an empty ledger `{}`, and `->>'revision'` on `{}` is SQL + // NULL, which cannot scan into an int64. Without COALESCE the first read + // fails and reconciliation can never record or announce a correction on the + // production store. This pins the whole bootstrap: save, read zero, record + // the first decision, read one. + t.Run("AudioReconcileLedgerBootstrapsFromEmptyJSON", func(t *testing.T) { + sessionID := uuid.NewString() + attemptID := "att-reconcile-" + sessionID + record := f.attemptRecord(sessionID, attemptID, "digest-reconcile") + if err := store.SaveAttempt(ctx, record); err != nil { + t.Fatalf("SaveAttempt: %v", err) + } + + // The empty ledger must read as revision zero, not as a scan error. + ledger, revision, err := store.GetAudioReconcileLedger(ctx, sessionID) + if err != nil { + t.Fatalf("GetAudioReconcileLedger on a fresh attempt: %v", err) + } + if revision != 0 { + t.Fatalf("fresh ledger revision = %d, want 0", revision) + } + if len(ledger.Entries) != 0 { + t.Fatalf("fresh ledger has %d entries, want 0", len(ledger.Entries)) + } + + // Recording the first decision must succeed against that same empty + // ledger and advance the revision, which is what lets a later writer + // detect a stale read instead of clobbering. + entry := playback.AudioReconcileEntryV3{ + Decision: playback.AudioReconcileInvalidated, + Generation: "probe:2026-10-05T00:00:02Z", + SessionID: "33333333-3333-3333-3333-333333333333", + Reason: playback.PlanInvalidatedDefaultAudioReconciliation, + } + merged, next, err := store.RecordAudioReconciliation(ctx, sessionID, revision, entry) + if err != nil { + t.Fatalf("RecordAudioReconciliation: %v", err) + } + if next != 1 { + t.Fatalf("revision after first record = %d, want 1", next) + } + if len(merged.Entries) != 1 || merged.Entries[0].Generation != entry.Generation { + t.Fatalf("recorded ledger = %+v, want one entry for %q", merged.Entries, entry.Generation) + } + + // And the advanced revision must read back cleanly too. + again, revision, err := store.GetAudioReconcileLedger(ctx, sessionID) + if err != nil { + t.Fatalf("GetAudioReconcileLedger after record: %v", err) + } + if revision != 1 || len(again.Entries) != 1 { + t.Fatalf("reread ledger = %+v revision %d, want one entry at revision 1", again.Entries, revision) + } + }) + t.Run("SaveAttemptIdempotency", func(t *testing.T) { sessionID := uuid.NewString() attemptID := "att-save-" + sessionID From b59b799d718362714d63c17f8a713c6a27984db3 Mon Sep 17 00:00:00 2001 From: drondeseries Date: Tue, 6 Oct 2026 07:43:13 -0400 Subject: [PATCH 16/16] fix(playback): name audio-language sentinels to satisfy goconst The repeated "mul" and "unknown" literals in audio_select.go tripped goconst and turned both lint CI jobs red. Extract the sentinel strings into named constants used at both sites, following the typed-const convention of the surrounding playback package. No behavior change; the playback suite still passes and lint-changed reports 0 issues. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/playback/audio_select.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/internal/playback/audio_select.go b/internal/playback/audio_select.go index 313e1a29d4..9427baac8e 100644 --- a/internal/playback/audio_select.go +++ b/internal/playback/audio_select.go @@ -80,17 +80,31 @@ func rankedLanguageMatch(candidate, preferred string) int { return langMatchRank(candidate, preferred) } +// Language sentinels that carry no concrete language evidence. Release and +// intake vocabulary uses these to mean "several" or "not stated", which must +// stay neutral in preference matching rather than rank as a match. +const ( + langUndetermined = "und" + langMulti = "mul" + + langLongMulti = "multi" + langLongMultiple = "multiple" + langLongDual = "dual" + langLongUnknown = "unknown" + langLongUndefined = "undefined" +) + // unknownLanguageMembership reports whether a language token declares only // MULTI/DUAL/undetermined membership (or nothing at all) instead of a // concrete language. Such tokens are release/intake vocabulary, not language // evidence, and must stay neutral in language preference matching. func unknownLanguageMembership(token string) bool { switch lang.Canonical(strings.TrimSpace(token)) { - case "", "und", "mul": + case "", langUndetermined, langMulti: return true } switch strings.ToLower(strings.TrimSpace(token)) { - case "multi", "multiple", "dual", "unknown", "undefined": + case langLongMulti, langLongMultiple, langLongDual, langLongUnknown, langLongUndefined: return true } return false @@ -264,7 +278,7 @@ func crossVersionAudioLanguages(track models.AudioTrack) []string { // language intent: empty, "und"/"mul", or their long forms. func isPlaceholderLanguage(code string) bool { canonical := lang.Canonical(code) - return canonical == "" || canonical == "und" || canonical == "mul" + return canonical == "" || canonical == langUndetermined || canonical == langMulti } // BrowserSupportsAudioCodec returns true if the given audio codec can be