From a667846d0c7f70241f6554ca0c22a8ee9dee2bb7 Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Wed, 23 Sep 2026 22:08:51 +0000 Subject: [PATCH 01/20] fix(watchsync): classify Trakt and Simkl rate limits and pace writes Trakt and Simkl answered every non-2xx the same way, so a 429 never became a RateLimitedError and the sync service never deferred the account. Both providers now map 429 (and Simkl's documented rate_limit/RATE_LIMIT bodies) to RateLimitedError with the provider's Retry-After, retry short waits in place, and pace authenticated writes to one per second per access token as the providers document. Retry-After parsing and the per-credential write limiter move into internal/watchsync so MDBList, Trakt and Simkl share them. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../watchsync/providers/mdblist/provider.go | 27 +- .../providers/mdblist/provider_test.go | 33 +-- .../watchsync/providers/simkl/provider.go | 137 ++++++++- .../providers/simkl/ratelimit_test.go | 276 ++++++++++++++++++ .../watchsync/providers/trakt/provider.go | 99 ++++++- .../providers/trakt/ratelimit_test.go | 196 +++++++++++++ internal/watchsync/ratelimit.go | 122 ++++++++ internal/watchsync/ratelimit_test.go | 95 ++++++ 8 files changed, 931 insertions(+), 54 deletions(-) create mode 100644 internal/watchsync/providers/simkl/ratelimit_test.go create mode 100644 internal/watchsync/providers/trakt/ratelimit_test.go create mode 100644 internal/watchsync/ratelimit.go create mode 100644 internal/watchsync/ratelimit_test.go diff --git a/internal/watchsync/providers/mdblist/provider.go b/internal/watchsync/providers/mdblist/provider.go index 1eab01605c..b6fd7b89eb 100644 --- a/internal/watchsync/providers/mdblist/provider.go +++ b/internal/watchsync/providers/mdblist/provider.go @@ -608,8 +608,10 @@ func (p *Provider) doOnce(ctx context.Context, method, path, target string, payl } defer resp.Body.Close() if resp.StatusCode == http.StatusTooManyRequests { - return parseRetryAfter(resp.Header.Get("Retry-After"), time.Now()), - fmt.Errorf("mdblist request %s %s rate limited: status 429", method, path) + // An absent, malformed, or elapsed Retry-After yields 0, which do + // replaces with defaultRetryAfter. + wait, _ := watchsync.ParseRetryAfter(resp.Header.Get("Retry-After"), time.Now()) + return wait, fmt.Errorf("mdblist request %s %s rate limited: status 429", method, path) } if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { return -1, fmt.Errorf("mdblist request %s %s rejected: status %d (check api key): %w", method, path, resp.StatusCode, watchsync.ErrInvalidCredential) @@ -652,27 +654,6 @@ func responseErrorDetail(body io.Reader) string { return strings.Join(strings.Fields(string(raw)), " ") } -// parseRetryAfter reads an RFC 7231 Retry-After value (delay-seconds or -// HTTP-date). It returns 0 when the header is absent or unparseable. -func parseRetryAfter(value string, now time.Time) time.Duration { - value = strings.TrimSpace(value) - if value == "" { - return 0 - } - if seconds, err := strconv.Atoi(value); err == nil { - if seconds < 0 { - return 0 - } - return time.Duration(seconds) * time.Second - } - if at, err := http.ParseTime(value); err == nil { - if wait := at.Sub(now); wait > 0 { - return wait - } - } - return 0 -} - // --- ID & payload helpers --- type mdblistIDs struct { diff --git a/internal/watchsync/providers/mdblist/provider_test.go b/internal/watchsync/providers/mdblist/provider_test.go index 66dc2d96c1..35069849b9 100644 --- a/internal/watchsync/providers/mdblist/provider_test.go +++ b/internal/watchsync/providers/mdblist/provider_test.go @@ -774,23 +774,22 @@ func TestDoIncludesMDBListValidationError(t *testing.T) { } } -func TestParseRetryAfter(t *testing.T) { - now := time.Date(2026, time.July, 5, 12, 0, 0, 0, time.UTC) - cases := []struct { - value string - want time.Duration - }{ - {"", 0}, - {"garbage", 0}, - {"-5", 0}, - {"7", 7 * time.Second}, - {now.Add(90 * time.Second).Format(http.TimeFormat), 90 * time.Second}, - {now.Add(-time.Minute).Format(http.TimeFormat), 0}, - } - for _, tc := range cases { - if got := parseRetryAfter(tc.value, now); got != tc.want { - t.Fatalf("parseRetryAfter(%q) = %s, want %s", tc.value, got, tc.want) - } +func TestDoParsesHTTPDateRetryAfter(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", time.Now().Add(2*time.Hour).UTC().Format(http.TimeFormat)) + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + p := NewProvider(server.Client(), server.URL) + _, err := p.fetchUser(context.Background(), "key") + rle, ok := watchsync.AsRateLimited(err) + if !ok { + t.Fatalf("expected RateLimitedError, got %v", err) + } + // HTTP-dates have one-second resolution, so allow for truncation. + if rle.RetryAfter <= 2*time.Hour-5*time.Second || rle.RetryAfter > 2*time.Hour { + t.Fatalf("got retry-after %s, want about 2h", rle.RetryAfter) } } diff --git a/internal/watchsync/providers/simkl/provider.go b/internal/watchsync/providers/simkl/provider.go index 8c3db07068..594060cf7d 100644 --- a/internal/watchsync/providers/simkl/provider.go +++ b/internal/watchsync/providers/simkl/provider.go @@ -36,9 +36,45 @@ const ( simklCursorRemovedAnime = "simkl.inbound.anime.removed_from_list" ) +// Simkl rate limits, from its rate-limits and errors docs: 10 GETs and 1 POST +// per second, plus a per-user daily quota. Sync GETs are sequential and few, +// so only POSTs are paced. Repeated POST overages can also earn the token or +// client_id a temporary throttling block (412 client_id_failed), which is +// not retried, so pacing is the main defense. +const ( + writeInterval = time.Second + writeBurst = 1 + + // Simkl answers per-second overages with 429 {"error":"rate_limit"} and + // says to retry in about a second. Its Retry-After on that response + // carries the daily reset, so it is ignored. + perSecondRetryWait = time.Second + // A 400 {"error":"RATE_LIMIT"} is not a quota: it is a 20-second per-user + // lock on /scrobble and POST /sync/history that clears as soon as the + // user's in-flight write finishes. Simkl says to retry shortly, without + // exponential backoff. + writeLockRetryWait = 5 * time.Second + + maxInPlaceRetryWait = 10 * time.Second + maxRetryAttempts = 2 + + // Daily-quota 429s (user_limit_exceeded, app_limit_exceeded) carry + // Retry-After, and per-second 429s clear in about a second, so a 429 + // without a usable Retry-After matches neither documented case. A minute + // clears any per-second throttle or write lock with a wide margin without + // parking the connection for long on a guess. + defaultRetryAfter = time.Minute + + maxErrorBodyBytes = 4 << 10 +) + type Provider struct { client *http.Client baseURL string + // writes paces POSTs per access token. + writes *watchsync.CredentialLimiter + // sleep waits between in-place rate-limit retries; tests replace it. + sleep func(context.Context, time.Duration) error } func NewProvider(client *http.Client, baseURL string) *Provider { @@ -48,7 +84,12 @@ func NewProvider(client *http.Client, baseURL string) *Provider { if strings.TrimSpace(baseURL) == "" { baseURL = defaultBaseURL } - return &Provider{client: client, baseURL: strings.TrimRight(baseURL, "/")} + return &Provider{ + client: client, + baseURL: strings.TrimRight(baseURL, "/"), + writes: watchsync.NewCredentialLimiter(writeInterval, writeBurst), + sleep: watchsync.SleepContext, + } } func (p *Provider) Key() string { @@ -467,9 +508,52 @@ func (p *Provider) scrobble(ctx context.Context, path string, cfg watchsync.Serv } func (p *Provider) do(ctx context.Context, method string, path string, cfg watchsync.ServerConfig, token string, body io.Reader, out any) error { + // Buffer the body so a rate-limited request can be replayed. + var payload []byte + if body != nil { + buffered, err := io.ReadAll(body) + if err != nil { + return fmt.Errorf("read simkl request body: %w", err) + } + payload = buffered + } + paced := token != "" && method != http.MethodGet + for attempt := 0; ; attempt++ { + if paced { + if err := p.writes.Wait(ctx, token); err != nil { + return fmt.Errorf("wait for simkl write limiter: %w", err) + } + } + wait, limited, err := p.doOnce(ctx, method, path, cfg, token, payload, out) + if !limited { + return err + } + if attempt < maxRetryAttempts && wait <= maxInPlaceRetryWait { + if err := p.sleep(ctx, wait); err != nil { + return err + } + continue + } + // Repeated short waits that still end rate limited are not + // trustworthy, so back off for a full fallback window instead. + if attempt >= maxRetryAttempts && wait < defaultRetryAfter { + wait = defaultRetryAfter + } + return watchsync.RateLimitedError{Provider: p.Key(), RetryAfter: wait} + } +} + +// doOnce performs a single HTTP attempt. A rate-limit response reports +// limited with how long to wait before retrying; every other outcome reports +// its error, if any. +func (p *Provider) doOnce(ctx context.Context, method string, path string, cfg watchsync.ServerConfig, token string, payload []byte, out any) (wait time.Duration, limited bool, err error) { + var body io.Reader + if payload != nil { + body = bytes.NewReader(payload) + } req, err := http.NewRequestWithContext(ctx, method, p.baseURL+path, body) if err != nil { - return fmt.Errorf("create simkl request: %w", err) + return 0, false, fmt.Errorf("create simkl request: %w", err) } req.Header.Set("Content-Type", "application/json") req.Header.Set("simkl-api-key", cfg.ClientID) @@ -478,22 +562,59 @@ func (p *Provider) do(ctx context.Context, method string, path string, cfg watch } resp, err := p.client.Do(req) if err != nil { - return fmt.Errorf("send simkl request: %w", err) + return 0, false, fmt.Errorf("send simkl request: %w", err) } defer resp.Body.Close() if resp.StatusCode == http.StatusConflict { - return simklConflictError{method: method, path: path} + return 0, false, simklConflictError{method: method, path: path} + } + if wait, limited := rateLimitWait(resp); limited { + return wait, true, nil } if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - return fmt.Errorf("simkl request %s %s failed: status %d", method, path, resp.StatusCode) + return 0, false, fmt.Errorf("simkl request %s %s failed: status %d", method, path, resp.StatusCode) } if out == nil || resp.StatusCode == http.StatusNoContent { - return nil + return 0, false, nil } if err := json.NewDecoder(resp.Body).Decode(out); err != nil { - return fmt.Errorf("decode simkl response: %w", err) + return 0, false, fmt.Errorf("decode simkl response: %w", err) } - return nil + return 0, false, nil +} + +// rateLimitWait classifies Simkl's throttling responses, which share status +// codes with unrelated errors and are told apart by the body's error field. +func rateLimitWait(resp *http.Response) (time.Duration, bool) { + if resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusBadRequest { + return 0, false + } + code := errorCode(resp.Body) + switch { + case resp.StatusCode == http.StatusTooManyRequests && strings.EqualFold(code, "rate_limit"): + return perSecondRetryWait, true + case resp.StatusCode == http.StatusTooManyRequests: + wait, ok := watchsync.ParseRetryAfter(resp.Header.Get("Retry-After"), time.Now()) + if !ok { + wait = defaultRetryAfter + } + return wait, true + case strings.EqualFold(code, "rate_limit"): + return writeLockRetryWait, true + default: + return 0, false + } +} + +// errorCode reads the machine-readable error field from a Simkl error body. +func errorCode(body io.Reader) string { + var envelope struct { + Error string `json:"error"` + } + if err := json.NewDecoder(io.LimitReader(body, maxErrorBodyBytes)).Decode(&envelope); err != nil { + return "" + } + return strings.TrimSpace(envelope.Error) } type simklConflictError struct { diff --git a/internal/watchsync/providers/simkl/ratelimit_test.go b/internal/watchsync/providers/simkl/ratelimit_test.go new file mode 100644 index 0000000000..86674df060 --- /dev/null +++ b/internal/watchsync/providers/simkl/ratelimit_test.go @@ -0,0 +1,276 @@ +package simkl + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/historyimport" + "github.com/Silo-Server/silo-server/internal/watchsync" +) + +var rateLimitTestConfig = watchsync.ServerConfig{ClientID: "client-id", ClientSecret: "client-secret"} + +var rateLimitTestEvent = watchsync.ScrobbleEvent{ + Kind: historyimport.KindMovie, + IMDbID: "tt1375666", + PositionSeconds: 60, + DurationSeconds: 6000, +} + +// recordSleeps replaces the provider's in-place retry wait so tests observe +// the requested durations without sleeping. +func recordSleeps(p *Provider) *[]time.Duration { + var waits []time.Duration + p.sleep = func(_ context.Context, d time.Duration) error { + waits = append(waits, d) + return nil + } + return &waits +} + +// respondingServer answers every request with status, Retry-After (when +// set), and body, counting attempts. +func respondingServer(t *testing.T, status int, retryAfter, body string, attempts *atomic.Int32) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + if retryAfter != "" { + w.Header().Set("Retry-After", retryAfter) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + return server +} + +func requireRateLimited(t *testing.T, err error) watchsync.RateLimitedError { + t.Helper() + var limited watchsync.RateLimitedError + if !errors.As(err, &limited) { + t.Fatalf("expected RateLimitedError, got %v", err) + } + if limited.Provider != "simkl" { + t.Fatalf("got provider %q, want simkl", limited.Provider) + } + return limited +} + +func TestDailyQuotaRateLimitUsesRetryAfterSeconds(t *testing.T) { + var attempts atomic.Int32 + server := respondingServer(t, http.StatusTooManyRequests, "7200", + `{"error":"user_limit_exceeded","code":429,"message":"This user has reached their daily API request limit"}`, &attempts) + provider := NewProvider(server.Client(), server.URL) + waits := recordSleeps(provider) + + _, err := provider.FetchProgress(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token"}) + limited := requireRateLimited(t, err) + if limited.RetryAfter != 2*time.Hour { + t.Fatalf("got retry-after %s, want 2h", limited.RetryAfter) + } + if attempts.Load() != 1 || len(*waits) != 0 { + t.Fatalf("got %d attempts and waits %v; a daily quota must defer, not retry in place", attempts.Load(), *waits) + } +} + +func TestRateLimitUsesRetryAfterHTTPDate(t *testing.T) { + var attempts atomic.Int32 + server := respondingServer(t, http.StatusTooManyRequests, + time.Now().Add(10*time.Minute).UTC().Format(http.TimeFormat), "", &attempts) + provider := NewProvider(server.Client(), server.URL) + + _, err := provider.FetchProgress(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token"}) + limited := requireRateLimited(t, err) + // HTTP-dates have one-second resolution, so allow for truncation. + if limited.RetryAfter <= 10*time.Minute-5*time.Second || limited.RetryAfter > 10*time.Minute { + t.Fatalf("got retry-after %s, want about 10m", limited.RetryAfter) + } +} + +func TestRateLimitWithoutRetryAfterUsesFallback(t *testing.T) { + var attempts atomic.Int32 + server := respondingServer(t, http.StatusTooManyRequests, "", "", &attempts) + provider := NewProvider(server.Client(), server.URL) + waits := recordSleeps(provider) + + _, err := provider.FetchProgress(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token"}) + limited := requireRateLimited(t, err) + if limited.RetryAfter != defaultRetryAfter { + t.Fatalf("got retry-after %s, want fallback %s", limited.RetryAfter, defaultRetryAfter) + } + if attempts.Load() != 1 || len(*waits) != 0 { + t.Fatalf("got %d attempts and waits %v, want one attempt", attempts.Load(), *waits) + } +} + +func TestPerSecondRateLimitRetriesInPlaceIgnoringDailyRetryAfter(t *testing.T) { + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if attempts.Add(1) == 1 { + // Simkl documents that this Retry-After carries the daily reset. + w.Header().Set("Retry-After", "50000") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"error":"rate_limit","code":429}`)) + return + } + _, _ = w.Write([]byte(`[]`)) + })) + defer server.Close() + provider := NewProvider(server.Client(), server.URL) + waits := recordSleeps(provider) + + if _, err := provider.fetchPlayback(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token"}, "/sync/playback/movies", ""); err != nil { + t.Fatalf("fetch playback after in-place retry: %v", err) + } + if attempts.Load() != 2 || len(*waits) != 1 || (*waits)[0] != perSecondRetryWait { + t.Fatalf("got %d attempts and waits %v, want 2 attempts after one %s wait", attempts.Load(), *waits, perSecondRetryWait) + } +} + +func TestPerSecondRateLimitExhaustedDefersForFallback(t *testing.T) { + var attempts atomic.Int32 + server := respondingServer(t, http.StatusTooManyRequests, "50000", `{"error":"rate_limit","code":429}`, &attempts) + provider := NewProvider(server.Client(), server.URL) + waits := recordSleeps(provider) + + _, err := provider.FetchProgress(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token"}) + limited := requireRateLimited(t, err) + if attempts.Load() != maxRetryAttempts+1 || len(*waits) != maxRetryAttempts { + t.Fatalf("got %d attempts and waits %v, want %d attempts", attempts.Load(), *waits, maxRetryAttempts+1) + } + if limited.RetryAfter != defaultRetryAfter { + t.Fatalf("got retry-after %s, want floored %s", limited.RetryAfter, defaultRetryAfter) + } +} + +func TestWriteLockRetriesInPlaceWithSameBody(t *testing.T) { + var bodies []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + bodies = append(bodies, string(raw)) + if len(bodies) == 1 { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"RATE_LIMIT","code":400}`)) + return + } + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + provider := NewProvider(server.Client(), server.URL) + // Keep the write limiter out of the way so only the lock wait runs. + provider.writes = watchsync.NewCredentialLimiter(time.Nanosecond, 10) + waits := recordSleeps(provider) + + if err := provider.Start(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token"}, rateLimitTestEvent); err != nil { + t.Fatalf("Start after write lock cleared: %v", err) + } + if len(bodies) != 2 || bodies[0] == "" || bodies[0] != bodies[1] { + t.Fatalf("body not replayed identically: %#v", bodies) + } + if len(*waits) != 1 || (*waits)[0] != writeLockRetryWait { + t.Fatalf("got in-place waits %v, want [%s]", *waits, writeLockRetryWait) + } +} + +func TestPersistentWriteLockReturnsRateLimitedError(t *testing.T) { + var attempts atomic.Int32 + server := respondingServer(t, http.StatusBadRequest, "", `{"error":"RATE_LIMIT","code":400}`, &attempts) + provider := NewProvider(server.Client(), server.URL) + provider.writes = watchsync.NewCredentialLimiter(time.Nanosecond, 10) + waits := recordSleeps(provider) + + err := provider.Start(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token"}, rateLimitTestEvent) + limited := requireRateLimited(t, err) + if attempts.Load() != maxRetryAttempts+1 || len(*waits) != maxRetryAttempts { + t.Fatalf("got %d attempts and waits %v, want %d attempts", attempts.Load(), *waits, maxRetryAttempts+1) + } + if limited.RetryAfter != defaultRetryAfter { + t.Fatalf("got retry-after %s, want %s", limited.RetryAfter, defaultRetryAfter) + } +} + +func TestOtherBadRequestIsNotRateLimited(t *testing.T) { + var attempts atomic.Int32 + server := respondingServer(t, http.StatusBadRequest, "", `{"error":"wrong_parameter","code":400}`, &attempts) + provider := NewProvider(server.Client(), server.URL) + waits := recordSleeps(provider) + + err := provider.Start(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token"}, rateLimitTestEvent) + if err == nil { + t.Fatal("expected an error for a malformed request") + } + if _, ok := watchsync.AsRateLimited(err); ok { + t.Fatalf("400 wrong_parameter classified as rate limited: %v", err) + } + if attempts.Load() != 1 || len(*waits) != 0 { + t.Fatalf("got %d attempts and waits %v, want no retry", attempts.Load(), *waits) + } +} + +func TestConflictIsNotRateLimited(t *testing.T) { + var attempts atomic.Int32 + server := respondingServer(t, http.StatusConflict, "", `{"error":"already_watched"}`, &attempts) + provider := NewProvider(server.Client(), server.URL) + + err := provider.Pause(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token"}, rateLimitTestEvent) + var conflict simklConflictError + if !errors.As(err, &conflict) { + t.Fatalf("expected simklConflictError, got %v", err) + } + if attempts.Load() != 1 { + t.Fatalf("got %d attempts, want 1", attempts.Load()) + } +} + +func TestWriteLimiterPacesPerTokenAndLeavesReadsAlone(t *testing.T) { + var writes, reads atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + reads.Add(1) + _, _ = w.Write([]byte(`[]`)) + return + } + writes.Add(1) + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + provider := NewProvider(server.Client(), server.URL) + // One write per hour: a second write for the same token can only + // proceed by waiting, which the one-minute deadline below refuses. + provider.writes = watchsync.NewCredentialLimiter(time.Hour, 1) + tokenA := watchsync.Connection{AccessToken: "token-a"} + + if err := provider.Start(context.Background(), rateLimitTestConfig, tokenA, rateLimitTestEvent); err != nil { + t.Fatalf("first write for token-a: %v", err) + } + // The limiter refuses at once when the next slot is past the deadline, + // so the request never reaches the server. Without pacing it would. + deadline, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := provider.Start(deadline, rateLimitTestConfig, tokenA, rateLimitTestEvent); err == nil || !strings.Contains(err.Error(), "write limiter") { + t.Fatalf("second write for token-a = %v, want the write limiter to refuse it", err) + } + if err := provider.Start(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token-b"}, rateLimitTestEvent); err != nil { + t.Fatalf("token-b must not wait behind token-a: %v", err) + } + if writes.Load() != 2 { + t.Fatalf("server saw %d writes, want 2", writes.Load()) + } + + for range 3 { + if _, err := provider.fetchPlayback(context.Background(), rateLimitTestConfig, tokenA, "/sync/playback/movies", ""); err != nil { + t.Fatalf("reads must not be paced: %v", err) + } + } + if reads.Load() != 3 { + t.Fatalf("server saw %d reads, want 3", reads.Load()) + } +} diff --git a/internal/watchsync/providers/trakt/provider.go b/internal/watchsync/providers/trakt/provider.go index 851f8f99fe..545937b184 100644 --- a/internal/watchsync/providers/trakt/provider.go +++ b/internal/watchsync/providers/trakt/provider.go @@ -23,9 +23,34 @@ const defaultBaseURL = "https://api.trakt.tv" const traktMediaShows = "shows" +// Trakt rate limits, from its API rate-limiting guide: authenticated users get +// one POST/PUT/DELETE per second (AUTHED_API_POST_LIMIT) and 500 GETs per +// five minutes (AUTHED_API_GET_LIMIT). A sync's sequential GETs stay well +// inside the GET budget, so only writes are paced. +const ( + writeInterval = time.Second + writeBurst = 1 + + // A 429 whose Retry-After is this short, which is typical of the + // one-second write limit, is retried in place. Longer waits defer the + // connection instead of holding a sync run or scrobble open. + maxInPlaceRetryWait = 10 * time.Second + maxRetryAttempts = 2 + + // Trakt's limiter sends Retry-After, but 429s from its security layer may + // not. Without a hint, wait out one full window of the longest documented + // bucket (AUTHED_API_GET_LIMIT, 300 seconds) so whichever bucket tripped + // has reset. Trakt has no daily quota that would call for longer. + defaultRetryAfter = 5 * time.Minute +) + type Provider struct { client *http.Client baseURL string + // writes paces authenticated writes per access token. + writes *watchsync.CredentialLimiter + // sleep waits between in-place rate-limit retries; tests replace it. + sleep func(context.Context, time.Duration) error } func NewProvider(client *http.Client, baseURL string) *Provider { @@ -39,6 +64,8 @@ func NewProvider(client *http.Client, baseURL string) *Provider { return &Provider{ client: client, baseURL: strings.TrimRight(baseURL, "/"), + writes: watchsync.NewCredentialLimiter(writeInterval, writeBurst), + sleep: watchsync.SleepContext, } } @@ -605,26 +632,86 @@ func (p *Provider) do( body io.Reader, out any, ) error { + // Buffer the body so a rate-limited request can be replayed. + var payload []byte + if body != nil { + buffered, err := io.ReadAll(body) + if err != nil { + return fmt.Errorf("read trakt request body: %w", err) + } + payload = buffered + } + // Trakt's write limit is per authenticated user. The OAuth endpoints are + // unauthenticated and count against the application instead. + paced := token != "" && method != http.MethodGet + for attempt := 0; ; attempt++ { + if paced { + if err := p.writes.Wait(ctx, token); err != nil { + return fmt.Errorf("wait for trakt write limiter: %w", err) + } + } + wait, limited, err := p.doOnce(ctx, method, path, cfg, token, payload, out) + if !limited { + return err + } + if attempt < maxRetryAttempts && wait <= maxInPlaceRetryWait { + if err := p.sleep(ctx, wait); err != nil { + return err + } + continue + } + // Repeated short hints that still end in 429 are not trustworthy, so + // back off for a full fallback window rather than the last hint. + if attempt >= maxRetryAttempts && wait < defaultRetryAfter { + wait = defaultRetryAfter + } + return watchsync.RateLimitedError{Provider: p.Key(), RetryAfter: wait} + } +} + +// doOnce performs a single HTTP attempt. A 429 reports limited with the wait +// from Retry-After, or defaultRetryAfter when the header is absent or +// malformed; every other outcome reports its error, if any. +func (p *Provider) doOnce( + ctx context.Context, + method string, + path string, + cfg watchsync.ServerConfig, + token string, + payload []byte, + out any, +) (wait time.Duration, limited bool, err error) { + var body io.Reader + if payload != nil { + body = bytes.NewReader(payload) + } req, err := http.NewRequestWithContext(ctx, method, p.baseURL+path, body) if err != nil { - return fmt.Errorf("create trakt request: %w", err) + return 0, false, fmt.Errorf("create trakt request: %w", err) } p.addHeaders(req, cfg, token) resp, err := p.client.Do(req) if err != nil { - return fmt.Errorf("send trakt request: %w", err) + return 0, false, fmt.Errorf("send trakt request: %w", err) } defer resp.Body.Close() + if resp.StatusCode == http.StatusTooManyRequests { + wait, ok := watchsync.ParseRetryAfter(resp.Header.Get("Retry-After"), time.Now()) + if !ok { + wait = defaultRetryAfter + } + return wait, true, nil + } if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - return fmt.Errorf("trakt request %s %s failed: status %d", method, path, resp.StatusCode) + return 0, false, fmt.Errorf("trakt request %s %s failed: status %d", method, path, resp.StatusCode) } if out == nil { - return nil + return 0, false, nil } if err := json.NewDecoder(resp.Body).Decode(out); err != nil { - return fmt.Errorf("decode trakt response: %w", err) + return 0, false, fmt.Errorf("decode trakt response: %w", err) } - return nil + return 0, false, nil } type tokenResponse struct { diff --git a/internal/watchsync/providers/trakt/ratelimit_test.go b/internal/watchsync/providers/trakt/ratelimit_test.go new file mode 100644 index 0000000000..9055814a06 --- /dev/null +++ b/internal/watchsync/providers/trakt/ratelimit_test.go @@ -0,0 +1,196 @@ +package trakt + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/historyimport" + "github.com/Silo-Server/silo-server/internal/watchsync" +) + +var rateLimitTestConfig = watchsync.ServerConfig{ClientID: "client-id", ClientSecret: "client-secret"} + +// recordSleeps replaces the provider's in-place retry wait so tests observe +// the requested durations without sleeping. +func recordSleeps(p *Provider) *[]time.Duration { + var waits []time.Duration + p.sleep = func(_ context.Context, d time.Duration) error { + waits = append(waits, d) + return nil + } + return &waits +} + +func rateLimitedServer(t *testing.T, retryAfter string, attempts *atomic.Int32) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + if retryAfter != "" { + w.Header().Set("Retry-After", retryAfter) + } + w.WriteHeader(http.StatusTooManyRequests) + })) + t.Cleanup(server.Close) + return server +} + +func requireRateLimited(t *testing.T, err error) watchsync.RateLimitedError { + t.Helper() + var limited watchsync.RateLimitedError + if !errors.As(err, &limited) { + t.Fatalf("expected RateLimitedError, got %v", err) + } + if limited.Provider != "trakt" { + t.Fatalf("got provider %q, want trakt", limited.Provider) + } + return limited +} + +func TestRateLimitUsesRetryAfterSeconds(t *testing.T) { + var attempts atomic.Int32 + server := rateLimitedServer(t, "120", &attempts) + provider := NewProvider(server.Client(), server.URL) + waits := recordSleeps(provider) + + _, err := provider.FetchProgress(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token"}) + limited := requireRateLimited(t, err) + if limited.RetryAfter != 2*time.Minute { + t.Fatalf("got retry-after %s, want 2m", limited.RetryAfter) + } + if attempts.Load() != 1 || len(*waits) != 0 { + t.Fatalf("got %d attempts and waits %v; a long Retry-After must defer, not retry in place", attempts.Load(), *waits) + } +} + +func TestRateLimitUsesRetryAfterHTTPDate(t *testing.T) { + var attempts atomic.Int32 + server := rateLimitedServer(t, time.Now().Add(10*time.Minute).UTC().Format(http.TimeFormat), &attempts) + provider := NewProvider(server.Client(), server.URL) + + _, err := provider.FetchProgress(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token"}) + limited := requireRateLimited(t, err) + // HTTP-dates have one-second resolution, so allow for truncation. + if limited.RetryAfter <= 10*time.Minute-5*time.Second || limited.RetryAfter > 10*time.Minute { + t.Fatalf("got retry-after %s, want about 10m", limited.RetryAfter) + } +} + +func TestRateLimitWithoutRetryAfterUsesFallback(t *testing.T) { + var attempts atomic.Int32 + server := rateLimitedServer(t, "", &attempts) + provider := NewProvider(server.Client(), server.URL) + waits := recordSleeps(provider) + + _, err := provider.FetchProgress(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token"}) + limited := requireRateLimited(t, err) + if limited.RetryAfter != defaultRetryAfter { + t.Fatalf("got retry-after %s, want fallback %s", limited.RetryAfter, defaultRetryAfter) + } + if attempts.Load() != 1 || len(*waits) != 0 { + t.Fatalf("got %d attempts and waits %v, want one attempt", attempts.Load(), *waits) + } +} + +func TestRateLimitShortRetryAfterRetriesInPlaceWithSameBody(t *testing.T) { + var bodies []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + bodies = append(bodies, string(raw)) + if len(bodies) == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + provider := NewProvider(server.Client(), server.URL) + // Keep the write limiter out of the way so only the Retry-After wait runs. + provider.writes = watchsync.NewCredentialLimiter(time.Nanosecond, 10) + waits := recordSleeps(provider) + + err := provider.Start(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token"}, watchsync.ScrobbleEvent{ + Kind: historyimport.KindMovie, + IMDbID: "tt123", + PositionSeconds: 60, + DurationSeconds: 600, + }) + if err != nil { + t.Fatalf("Start after in-place retry: %v", err) + } + if len(bodies) != 2 || bodies[0] == "" || bodies[0] != bodies[1] { + t.Fatalf("body not replayed identically: %#v", bodies) + } + if len(*waits) != 1 || (*waits)[0] != time.Second { + t.Fatalf("got in-place waits %v, want [1s]", *waits) + } +} + +func TestRateLimitExhaustedInPlaceRetriesDeferForFallback(t *testing.T) { + var attempts atomic.Int32 + server := rateLimitedServer(t, "1", &attempts) + provider := NewProvider(server.Client(), server.URL) + waits := recordSleeps(provider) + + _, err := provider.FetchProgress(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token"}) + limited := requireRateLimited(t, err) + if attempts.Load() != maxRetryAttempts+1 || len(*waits) != maxRetryAttempts { + t.Fatalf("got %d attempts and waits %v, want %d attempts", attempts.Load(), *waits, maxRetryAttempts+1) + } + if limited.RetryAfter != defaultRetryAfter { + t.Fatalf("got retry-after %s, want floored %s", limited.RetryAfter, defaultRetryAfter) + } +} + +func TestWriteLimiterPacesPerTokenAndLeavesReadsAlone(t *testing.T) { + var writes, reads atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + reads.Add(1) + _, _ = w.Write([]byte(`[]`)) + return + } + writes.Add(1) + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + provider := NewProvider(server.Client(), server.URL) + // One write per hour: a second write for the same token can only + // proceed by waiting, which the one-minute deadline below refuses. + provider.writes = watchsync.NewCredentialLimiter(time.Hour, 1) + event := watchsync.ScrobbleEvent{Kind: historyimport.KindMovie, IMDbID: "tt123", DurationSeconds: 600} + tokenA := watchsync.Connection{AccessToken: "token-a"} + + if err := provider.Start(context.Background(), rateLimitTestConfig, tokenA, event); err != nil { + t.Fatalf("first write for token-a: %v", err) + } + // The limiter refuses at once when the next slot is past the deadline, + // so the request never reaches the server. Without pacing it would. + deadline, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := provider.Start(deadline, rateLimitTestConfig, tokenA, event); err == nil || !strings.Contains(err.Error(), "write limiter") { + t.Fatalf("second write for token-a = %v, want the write limiter to refuse it", err) + } + if err := provider.Start(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token-b"}, event); err != nil { + t.Fatalf("token-b must not wait behind token-a: %v", err) + } + if writes.Load() != 2 { + t.Fatalf("server saw %d writes, want 2", writes.Load()) + } + + for range 3 { + if _, err := provider.FetchProgress(context.Background(), rateLimitTestConfig, tokenA); err != nil { + t.Fatalf("reads must not be paced: %v", err) + } + } + if reads.Load() != 3 { + t.Fatalf("server saw %d reads, want 3", reads.Load()) + } +} diff --git a/internal/watchsync/ratelimit.go b/internal/watchsync/ratelimit.go new file mode 100644 index 0000000000..db8d8015da --- /dev/null +++ b/internal/watchsync/ratelimit.go @@ -0,0 +1,122 @@ +package watchsync + +import ( + "context" + "crypto/sha256" + "math" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/time/rate" +) + +// ParseRetryAfter reads an RFC 9110 Retry-After value: delay-seconds or an +// HTTP-date. ok is false when the value is absent or malformed. An HTTP-date +// that has already passed parses as zero, meaning "retry now". +func ParseRetryAfter(value string, now time.Time) (time.Duration, bool) { + value = strings.TrimSpace(value) + if value == "" { + return 0, false + } + if seconds, err := strconv.ParseInt(value, 10, 64); err == nil { + if seconds < 0 { + return 0, false + } + if seconds > int64(math.MaxInt64/time.Second) { + return time.Duration(math.MaxInt64), true + } + return time.Duration(seconds) * time.Second, true + } + if at, err := http.ParseTime(value); err == nil { + return max(at.Sub(now), 0), true + } + return 0, false +} + +// SleepContext waits for d, returning early with ctx.Err() when ctx ends. +func SleepContext(ctx context.Context, d time.Duration) error { + if d <= 0 { + return ctx.Err() + } + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +// credentialLimiterIdleTTL is how long a credential's limiter may sit unused +// before it becomes eligible for removal. +const credentialLimiterIdleTTL = 10 * time.Minute + +// CredentialLimiter paces requests per provider credential (an access token +// or API key), so one account's burst never delays another account's +// requests. Limiters are keyed by a SHA-256 digest of the credential: the map +// never holds a raw secret, and its keys never appear in logs. +type CredentialLimiter struct { + every time.Duration + burst int + now func() time.Time + + mu sync.Mutex + limiters map[[sha256.Size]byte]*credentialLimiterEntry + lastSweep time.Time +} + +type credentialLimiterEntry struct { + limiter *rate.Limiter + lastUsed time.Time +} + +// NewCredentialLimiter allows each credential burst requests at once and one +// more every interval after that. +func NewCredentialLimiter(every time.Duration, burst int) *CredentialLimiter { + return &CredentialLimiter{ + every: every, + burst: max(burst, 1), + now: time.Now, + limiters: make(map[[sha256.Size]byte]*credentialLimiterEntry), + } +} + +// Wait blocks until credential may send its next request. It returns an error +// without waiting when ctx is already done or its deadline would pass first. +func (l *CredentialLimiter) Wait(ctx context.Context, credential string) error { + return l.limiter(credential).Wait(ctx) +} + +func (l *CredentialLimiter) limiter(credential string) *rate.Limiter { + key := sha256.Sum256([]byte(credential)) + now := l.now() + l.mu.Lock() + defer l.mu.Unlock() + if now.Sub(l.lastSweep) >= credentialLimiterIdleTTL { + l.lastSweep = now + l.sweepLocked(now) + } + entry, ok := l.limiters[key] + if !ok { + entry = &credentialLimiterEntry{limiter: rate.NewLimiter(rate.Every(l.every), l.burst)} + l.limiters[key] = entry + } + entry.lastUsed = now + return entry.limiter +} + +// sweepLocked drops idle limiters so the map stays bounded as access tokens +// rotate. An idle limiter with a full bucket behaves exactly like a new one, +// so removing it cannot let a credential exceed its rate. +func (l *CredentialLimiter) sweepLocked(now time.Time) { + for key, entry := range l.limiters { + if now.Sub(entry.lastUsed) >= credentialLimiterIdleTTL && + entry.limiter.TokensAt(now) >= float64(l.burst) { + delete(l.limiters, key) + } + } +} diff --git a/internal/watchsync/ratelimit_test.go b/internal/watchsync/ratelimit_test.go new file mode 100644 index 0000000000..074357597c --- /dev/null +++ b/internal/watchsync/ratelimit_test.go @@ -0,0 +1,95 @@ +package watchsync + +import ( + "context" + "errors" + "net/http" + "testing" + "time" +) + +func TestParseRetryAfter(t *testing.T) { + now := time.Date(2026, time.July, 5, 12, 0, 0, 0, time.UTC) + cases := []struct { + value string + want time.Duration + wantOK bool + }{ + {"", 0, false}, + {" ", 0, false}, + {"garbage", 0, false}, + {"-5", 0, false}, + {"1.5", 0, false}, + {"0", 0, true}, + {"7", 7 * time.Second, true}, + {" 30 ", 30 * time.Second, true}, + {now.Add(90 * time.Second).Format(http.TimeFormat), 90 * time.Second, true}, + {now.Add(-time.Minute).Format(http.TimeFormat), 0, true}, + } + for _, tc := range cases { + got, ok := ParseRetryAfter(tc.value, now) + if got != tc.want || ok != tc.wantOK { + t.Fatalf("ParseRetryAfter(%q) = %s, %t; want %s, %t", tc.value, got, ok, tc.want, tc.wantOK) + } + } +} + +func TestParseRetryAfterSaturatesHugeDelays(t *testing.T) { + got, ok := ParseRetryAfter("99999999999999999", time.Now()) + if !ok || got <= 0 { + t.Fatalf("ParseRetryAfter(huge) = %s, %t; want a positive saturated duration", got, ok) + } +} + +func TestSleepContextReturnsWhenContextEnds(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := SleepContext(ctx, time.Hour); !errors.Is(err, context.Canceled) { + t.Fatalf("SleepContext error = %v, want context.Canceled", err) + } + if err := SleepContext(context.Background(), 0); err != nil { + t.Fatalf("SleepContext(0) = %v, want nil", err) + } +} + +func TestCredentialLimiterPacesEachCredentialSeparately(t *testing.T) { + // One request per hour makes any second request for the same credential + // block, without the test waiting for real time to pass. + limiter := NewCredentialLimiter(time.Hour, 1) + + if err := limiter.Wait(context.Background(), "token-a"); err != nil { + t.Fatalf("first wait for token-a: %v", err) + } + if err := limiter.Wait(context.Background(), "token-b"); err != nil { + t.Fatalf("token-b must not wait behind token-a: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := limiter.Wait(ctx, "token-a"); !errors.Is(err, context.Canceled) { + t.Fatalf("second wait for token-a = %v, want context.Canceled", err) + } + + deadline, cancelDeadline := context.WithTimeout(context.Background(), time.Minute) + defer cancelDeadline() + if err := limiter.Wait(deadline, "token-a"); err == nil { + t.Fatal("second wait for token-a returned before its next slot") + } +} + +func TestCredentialLimiterDropsIdleCredentials(t *testing.T) { + clock := time.Now() + limiter := NewCredentialLimiter(time.Second, 1) + limiter.now = func() time.Time { return clock } + + if err := limiter.Wait(context.Background(), "rotated-token"); err != nil { + t.Fatalf("wait: %v", err) + } + clock = clock.Add(credentialLimiterIdleTTL + time.Second) + if err := limiter.Wait(context.Background(), "current-token"); err != nil { + t.Fatalf("wait: %v", err) + } + if len(limiter.limiters) != 1 { + t.Fatalf("got %d limiters after sweep, want 1", len(limiter.limiters)) + } +} From d53738bb54ab71320badab9b5fd8f72199abfffc Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Wed, 23 Sep 2026 22:09:37 +0000 Subject: [PATCH 02/20] fix(watchsync): page every Trakt list read Since April 2026 Trakt serves only the first 100 items when page and limit are omitted. Favorites, watchlist, and history reads sent neither, so users with longer lists got partial imports, and ExportWatched compared local plays against a truncated history and re-sent older plays, which Trakt stores as duplicates. Generalize the watched-only pager into fetchTraktPages and use it for watched, favorites, watchlist, and history. It always sends limit=250, stops on X-Pagination-Page-Count, a page shorter than the applied X-Pagination-Limit, or an empty page, caps the walk, and returns no rows when any page fails. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../watchsync/providers/trakt/history_test.go | 246 +++++++++++++++++ .../providers/trakt/pagination_test.go | 257 ++++++++++++++++++ .../watchsync/providers/trakt/provider.go | 163 ++++++++--- .../providers/trakt/provider_test.go | 1 + .../watchsync/providers/trakt/watched_test.go | 18 +- 5 files changed, 632 insertions(+), 53 deletions(-) create mode 100644 internal/watchsync/providers/trakt/history_test.go create mode 100644 internal/watchsync/providers/trakt/pagination_test.go diff --git a/internal/watchsync/providers/trakt/history_test.go b/internal/watchsync/providers/trakt/history_test.go new file mode 100644 index 0000000000..d4986a239c --- /dev/null +++ b/internal/watchsync/providers/trakt/history_test.go @@ -0,0 +1,246 @@ +package trakt + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/historyimport" + "github.com/Silo-Server/silo-server/internal/userstore" + "github.com/Silo-Server/silo-server/internal/watchsync" +) + +func TestFetchHistoryImportsEveryPage(t *testing.T) { + var pages []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/sync/history" { + t.Errorf("unexpected path %s", r.URL.Path) + http.NotFound(w, r) + return + } + page := r.URL.Query().Get("page") + pages = append(pages, page) + if r.URL.Query().Get("limit") != "250" { + t.Errorf("limit = %q, want 250", r.URL.Query().Get("limit")) + } + if r.Header.Get("Authorization") != "Bearer test-token" { + t.Errorf("missing authorization") + } + w.Header().Set("Content-Type", "application/json") + // Short pages without pagination headers: only the empty page ends the history. + switch page { + case "1", "2": + writeTraktFixture(t, w, `[ + {"type":"movie","watched_at":"2026-05-0%sT12:00:00.000Z","movie":{"title":"Movie %s","year":2020,"ids":{"tmdb":10%s}}}, + {"type":"episode","watched_at":"2026-06-0%sT12:00:00.000Z","episode":{"season":1,"number":%s,"ids":{"tvdb":50%s}},"show":{"title":"Show","year":2021,"ids":{"tvdb":300}}} + ]`, page, page, page, page, page, page) + case "3": + writeTraktFixture(t, w, `[]`) + default: + t.Errorf("unexpected page %q", page) + http.Error(w, "unexpected page", http.StatusInternalServerError) + } + })) + defer server.Close() + + rows, err := NewProvider(server.Client(), server.URL).FetchHistory(context.Background(), watchsync.ServerConfig{}, watchsync.Connection{AccessToken: "test-token"}) + if err != nil { + t.Fatal(err) + } + if want := []string{"1", "2", "3"}; !reflect.DeepEqual(pages, want) { + t.Fatalf("pages = %v, want %v", pages, want) + } + want := []struct { + key, kind string + watchedAt time.Time + }{ + {"tmdb:101", historyimport.KindMovie, time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)}, + {"tvdb:501", historyimport.KindEpisode, time.Date(2026, 6, 1, 12, 0, 0, 0, time.UTC)}, + {"tmdb:102", historyimport.KindMovie, time.Date(2026, 5, 2, 12, 0, 0, 0, time.UTC)}, + {"tvdb:502", historyimport.KindEpisode, time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC)}, + } + if len(rows) != len(want) { + t.Fatalf("got %d rows, want %d: %#v", len(rows), len(want), rows) + } + for i, exp := range want { + row := rows[i] + if row.ProviderItemKey != exp.key || row.Kind != exp.kind || !row.WatchedAt.Equal(exp.watchedAt) { + t.Errorf("row %d = %#v, want key %s kind %s watched %s", i, row, exp.key, exp.kind, exp.watchedAt) + } + } +} + +func TestFetchHistoryDoesNotReturnPartialHistoryOnLaterPageFailure(t *testing.T) { + for _, failure := range []string{"http", "json"} { + t.Run(failure, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Query().Get("page") == "1" { + writeTraktFixture(t, w, `[{"type":"movie","watched_at":"2026-05-01T12:00:00.000Z","movie":{"ids":{"tmdb":123}}}]`) + return + } + if failure == "http" { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + } else { + writeTraktFixture(t, w, `[{`) + } + })) + defer server.Close() + + rows, err := NewProvider(server.Client(), server.URL).FetchHistory(context.Background(), watchsync.ServerConfig{}, watchsync.Connection{}) + if err == nil || rows != nil { + t.Fatalf("got rows=%#v, err=%v; want no partial history and an error", rows, err) + } + }) + } +} + +// TestExportWatchedDoesNotResendPlayFromLaterHistoryPage runs the real export +// reconciliation against a Trakt history that spans two pages. A play Trakt +// lists only on page 2 must count as present remotely, not be sent again. +func TestExportWatchedDoesNotResendPlayFromLaterHistoryPage(t *testing.T) { + var sent traktHistoryPayload + posts := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodGet && r.URL.Path == "/sync/history": + page := r.URL.Query().Get("page") + if page == "" { + page = "1" // Trakt serves only the first page when page is omitted. + } + w.Header().Set("X-Pagination-Page", page) + w.Header().Set("X-Pagination-Page-Count", "2") + switch page { + case "1": + writeTraktFixture(t, w, `[{"type":"movie","watched_at":"2026-05-01T12:00:00.000Z","movie":{"ids":{"tmdb":101}}}]`) + case "2": + writeTraktFixture(t, w, `[{"type":"movie","watched_at":"2026-05-02T12:00:00.000Z","movie":{"ids":{"tmdb":102}}}]`) + default: + t.Errorf("unexpected history page %q", page) + http.Error(w, "unexpected page", http.StatusInternalServerError) + } + case r.Method == http.MethodPost && r.URL.Path == "/sync/history": + posts++ + if err := json.NewDecoder(r.Body).Decode(&sent); err != nil { + t.Errorf("decode export body: %v", err) + } + w.WriteHeader(http.StatusCreated) + writeTraktFixture(t, w, `{}`) + default: + t.Errorf("unexpected request %s %s", r.Method, r.URL) + http.NotFound(w, r) + } + })) + defer server.Close() + + store := completedHistoryStore{rows: []userstore.WatchHistoryEntry{ + localMoviePlay("history-1", "101", "2026-05-01T12:00:00Z"), // remote page 1 + localMoviePlay("history-2", "102", "2026-05-02T12:00:00Z"), // remote page 2 + localMoviePlay("history-3", "103", "2026-05-03T12:00:00Z"), // not on Trakt + }} + repo := &historyExportRepo{} + service := watchsync.NewService(repo, watchsync.NewRegistry()).WithUserStoreProvider(staticUserStores{store: store}) + conn := watchsync.Connection{ID: "conn-1", Provider: "trakt", UserID: 7, ProfileID: "profile-1", AccessToken: "test-token"} + + result, err := service.ExportWatched(context.Background(), conn, watchsync.ServerConfig{}, NewProvider(server.Client(), server.URL)) + if err != nil { + t.Fatalf("ExportWatched: %v", err) + } + if result.RemoteFound != 2 || result.RemotePresent != 2 || result.Queued != 1 || result.Sent != 1 { + t.Fatalf("result = %+v, want 2 remote plays found and present, 1 queued and sent", result) + } + if posts != 1 || len(sent.Movies) != 1 || sent.Movies[0].IDs.TMDB != 103 || len(sent.Episodes) != 0 || len(sent.Shows) != 0 { + t.Fatalf("posts = %d, sent = %+v; want one export of tmdb 103 only", posts, sent) + } + wantStatus := map[string]string{"history-1": "remote_present", "history-2": "remote_present", "history-3": "sent"} + gotStatus := map[string]string{} + for _, export := range repo.exports { + gotStatus[export.HistoryID] = export.Status + } + if !reflect.DeepEqual(gotStatus, wantStatus) { + t.Fatalf("export statuses = %v, want %v", gotStatus, wantStatus) + } +} + +func localMoviePlay(id, tmdbID, watchedAt string) userstore.WatchHistoryEntry { + return userstore.WatchHistoryEntry{ + ID: id, + ProfileID: "profile-1", + MediaItemID: "movie-" + tmdbID, + WatchedAt: watchedAt, + DurationSeconds: 7200, + Completed: true, + Source: userstore.WatchHistorySourcePlayback, + Identity: userstore.WatchIdentity{ + StableType: historyimport.KindMovie, + ProviderIDs: map[string]string{"tmdb": tmdbID}, + }, + } +} + +// completedHistoryStore serves the completed history ExportWatched reads. Any +// other UserStore method panics through the nil embedded interface. +type completedHistoryStore struct { + userstore.UserStore + rows []userstore.WatchHistoryEntry +} + +func (s completedHistoryStore) ListCompletedHistory(_ context.Context, query userstore.CompletedHistoryQuery) ([]userstore.WatchHistoryEntry, error) { + if query.Offset > 0 { + return nil, nil + } + return s.rows, nil +} + +type staticUserStores struct { + store userstore.UserStore +} + +func (p staticUserStores) ForUser(context.Context, int) (userstore.UserStore, error) { + return p.store, nil +} + +func (staticUserStores) Close() error { return nil } + +// historyExportRepo keeps the history export rows of one ExportWatched run. +// Any other Repository method panics through the nil embedded interface. +type historyExportRepo struct { + watchsync.Repository + exports []watchsync.HistoryExport +} + +func (r *historyExportRepo) UpsertHistoryExports(_ context.Context, exports []watchsync.HistoryExport) error { + for _, export := range exports { + export.ID = "export-" + export.HistoryID + r.exports = append(r.exports, export) + } + return nil +} + +func (r *historyExportRepo) ListPendingHistoryExports(_ context.Context, connectionID string, limit int) ([]watchsync.HistoryExport, error) { + var pending []watchsync.HistoryExport + for _, export := range r.exports { + if export.ConnectionID == connectionID && export.Status == "pending" && len(pending) < limit { + pending = append(pending, export) + } + } + return pending, nil +} + +func (r *historyExportRepo) MarkHistoryExportStatus(_ context.Context, id string, status string, _ string) error { + for i := range r.exports { + if r.exports[i].ID == id { + r.exports[i].Status = status + } + } + return nil +} + +func (r *historyExportRepo) UpsertConnection(_ context.Context, conn watchsync.Connection) (watchsync.Connection, error) { + return conn, nil +} diff --git a/internal/watchsync/providers/trakt/pagination_test.go b/internal/watchsync/providers/trakt/pagination_test.go new file mode 100644 index 0000000000..6c6a1b79be --- /dev/null +++ b/internal/watchsync/providers/trakt/pagination_test.go @@ -0,0 +1,257 @@ +package trakt + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/historyimport" + "github.com/Silo-Server/silo-server/internal/watchsync" +) + +type traktListCase struct { + name string + moviesPath string + showsPath string + fetch func(*Provider, watchsync.Connection) ([]watchsync.RemoteFavorite, error) +} + +func traktListCases() []traktListCase { + return []traktListCase{ + { + name: "favorites", + moviesPath: "/users/me/favorites/movies/added", + showsPath: "/users/me/favorites/shows/added", + fetch: func(p *Provider, conn watchsync.Connection) ([]watchsync.RemoteFavorite, error) { + return p.FetchFavorites(context.Background(), watchsync.ServerConfig{}, conn) + }, + }, + { + name: "watchlist", + moviesPath: "/sync/watchlist/movies", + showsPath: "/sync/watchlist/shows", + fetch: func(p *Provider, conn watchsync.Connection) ([]watchsync.RemoteFavorite, error) { + return p.FetchWatchlist(context.Background(), watchsync.ServerConfig{}, conn) + }, + }, + } +} + +func TestFetchListsImportEveryPage(t *testing.T) { + for _, tc := range traktListCases() { + t.Run(tc.name, func(t *testing.T) { + var requests []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + kind := map[string]string{tc.moviesPath: "movies", tc.showsPath: "shows"}[r.URL.Path] + if kind == "" { + t.Errorf("unexpected path %s", r.URL.Path) + http.NotFound(w, r) + return + } + page := r.URL.Query().Get("page") + requests = append(requests, kind+":"+page) + if r.URL.Query().Get("limit") != "250" { + t.Errorf("limit = %q, want 250", r.URL.Query().Get("limit")) + } + if r.Header.Get("Authorization") != "Bearer test-token" { + t.Errorf("missing authorization") + } + w.Header().Set("Content-Type", "application/json") + // Short pages without pagination headers: only the empty page ends the list. + switch page { + case "1", "2": + if kind == "movies" { + writeTraktFixture(t, w, `[{"listed_at":"2026-05-0%sT12:00:00Z","movie":{"title":"Movie %s","year":2020,"ids":{"tmdb":10%s}}}]`, page, page, page) + } else { + writeTraktFixture(t, w, `[{"listed_at":"2026-06-0%sT12:00:00Z","show":{"title":"Show %s","year":2021,"ids":{"tvdb":30%s,"tmdb":20%s}}}]`, page, page, page, page) + } + case "3": + writeTraktFixture(t, w, `[]`) + default: + t.Errorf("unexpected page %q", page) + http.Error(w, "unexpected page", http.StatusInternalServerError) + } + })) + defer server.Close() + + rows, err := tc.fetch(NewProvider(server.Client(), server.URL), watchsync.Connection{AccessToken: "test-token"}) + if err != nil { + t.Fatal(err) + } + wantRequests := []string{"movies:1", "movies:2", "movies:3", "shows:1", "shows:2", "shows:3"} + if !reflect.DeepEqual(requests, wantRequests) { + t.Fatalf("requests = %v, want %v", requests, wantRequests) + } + want := []struct { + key, kind, title string + listedAt time.Time + }{ + {"tmdb:101", historyimport.KindMovie, "Movie 1", time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)}, + {"tmdb:102", historyimport.KindMovie, "Movie 2", time.Date(2026, 5, 2, 12, 0, 0, 0, time.UTC)}, + {"tvdb:301", historyimport.KindSeries, "Show 1", time.Date(2026, 6, 1, 12, 0, 0, 0, time.UTC)}, + {"tvdb:302", historyimport.KindSeries, "Show 2", time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC)}, + } + if len(rows) != len(want) { + t.Fatalf("got %d rows, want %d: %#v", len(rows), len(want), rows) + } + for i, exp := range want { + row := rows[i] + if row.ProviderItemKey != exp.key || row.Kind != exp.kind || row.Title != exp.title || !row.FavoritedAt.Equal(exp.listedAt) { + t.Errorf("row %d = %#v, want key %s kind %s title %s listed %s", i, row, exp.key, exp.kind, exp.title, exp.listedAt) + } + } + }) + } +} + +func TestFetchListsDoNotReturnPartialResultsOnLaterPageFailure(t *testing.T) { + for _, tc := range traktListCases() { + for _, kind := range []string{"movies", "shows"} { + for _, failure := range []string{"http", "json"} { + t.Run(tc.name+"/"+kind+"/"+failure, func(t *testing.T) { + failingPath := tc.moviesPath + if kind == "shows" { + failingPath = tc.showsPath + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Query().Get("page") == "1" { + writeTraktFixture(t, w, `[{"listed_at":"2026-05-01T12:00:00Z","movie":{"ids":{"tmdb":123}},"show":{"ids":{"tvdb":456}}}]`) + return + } + if r.URL.Path == failingPath { + if failure == "http" { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + } else { + writeTraktFixture(t, w, `[{`) + } + return + } + writeTraktFixture(t, w, `[]`) + })) + defer server.Close() + + rows, err := tc.fetch(NewProvider(server.Client(), server.URL), watchsync.Connection{}) + if err == nil || rows != nil { + t.Fatalf("got rows=%#v, err=%v; want no partial list and an error", rows, err) + } + }) + } + } + } +} + +func TestFetchTraktPagesStopsOnPaginationHeaders(t *testing.T) { + for _, tc := range []struct { + name string + headers map[string]string + bodies map[string]string + wantPages []string + wantRows int + }{ + { + // Both pages are shorter than the limit; the page count decides. + name: "page count", + headers: map[string]string{"X-Pagination-Limit": "250", "X-Pagination-Page-Count": "2", "X-Pagination-Item-Count": "2"}, + bodies: map[string]string{"1": `[{"movie":{"ids":{"tmdb":1}}}]`, "2": `[{"movie":{"ids":{"tmdb":2}}}]`}, + wantPages: []string{"1", "2"}, + wantRows: 2, + }, + { + // Trakt applied a smaller limit than requested; a page shorter than it is the last. + name: "applied limit", + headers: map[string]string{"X-Pagination-Limit": "2"}, + bodies: map[string]string{"1": `[{"movie":{"ids":{"tmdb":1}}},{"movie":{"ids":{"tmdb":2}}}]`, "2": `[{"movie":{"ids":{"tmdb":3}}}]`}, + wantPages: []string{"1", "2"}, + wantRows: 3, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var pages []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + page := r.URL.Query().Get("page") + pages = append(pages, page) + if got := r.URL.Query().Get("extended"); got != "full" { + t.Errorf("extended = %q, want full", got) + } + if got := r.URL.Query().Get("limit"); got != "250" { + t.Errorf("limit = %q, want 250", got) + } + body, ok := tc.bodies[page] + if !ok { + t.Errorf("unexpected page %q", page) + http.Error(w, "unexpected page", http.StatusInternalServerError) + return + } + for key, value := range tc.headers { + w.Header().Set(key, value) + } + w.Header().Set("X-Pagination-Page", page) + w.Header().Set("Content-Type", "application/json") + writeTraktFixture(t, w, "%s", body) + })) + defer server.Close() + + query := url.Values{"extended": {"full"}, "page": {"9"}, "limit": {"5"}} + rows, err := fetchTraktPages[traktFavoriteMovie](context.Background(), NewProvider(server.Client(), server.URL), watchsync.ServerConfig{}, watchsync.Connection{}, "/sync/watchlist/movies", query) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(pages, tc.wantPages) { + t.Fatalf("pages = %v, want %v", pages, tc.wantPages) + } + if len(rows) != tc.wantRows { + t.Fatalf("got %d rows, want %d", len(rows), tc.wantRows) + } + if query.Get("page") != "9" || query.Get("limit") != "5" { + t.Fatalf("caller query was modified: %v", query) + } + }) + } +} + +func TestFetchTraktPagesFailsAtPageCap(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + // A server that ignores page and sends no pagination headers never ends. + w.Header().Set("Content-Type", "application/json") + writeTraktFixture(t, w, `[{"movie":{"ids":{"tmdb":1}}}]`) + })) + defer server.Close() + + rows, err := fetchTraktPages[traktFavoriteMovie](context.Background(), NewProvider(server.Client(), server.URL), watchsync.ServerConfig{}, watchsync.Connection{}, "/sync/watchlist/movies", nil) + if err == nil || rows != nil { + t.Fatalf("got %d rows, err=%v; want no rows and an error", len(rows), err) + } + if requests != traktMaxPages { + t.Fatalf("requests = %d, want %d", requests, traktMaxPages) + } +} + +func TestFetchTraktPagesFailsWhenTheListChangesMidRead(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // The list shrinks between page 1 and page 2, which shifts offsets. + switch r.URL.Query().Get("page") { + case "1": + w.Header().Set("X-Pagination-Item-Count", "251") + w.Header().Set("X-Pagination-Page-Count", "2") + writeTraktFixture(t, w, `[{"listed_at":"2026-01-01T00:00:00Z","movie":{"title":"A","ids":{"trakt":1,"tmdb":1}}}]`) + default: + w.Header().Set("X-Pagination-Item-Count", "250") + w.Header().Set("X-Pagination-Page-Count", "1") + writeTraktFixture(t, w, `[{"listed_at":"2026-01-01T00:00:00Z","movie":{"title":"B","ids":{"trakt":2,"tmdb":2}}}]`) + } + })) + defer server.Close() + + rows, err := fetchTraktPages[traktFavoriteMovie](context.Background(), NewProvider(server.Client(), server.URL), + watchsync.ServerConfig{}, watchsync.Connection{AccessToken: "t"}, "/sync/watchlist/movies", nil) + if err == nil || rows != nil { + t.Fatalf("rows=%v err=%v, want an error and no rows", rows, err) + } +} diff --git a/internal/watchsync/providers/trakt/provider.go b/internal/watchsync/providers/trakt/provider.go index 545937b184..54b6a917c3 100644 --- a/internal/watchsync/providers/trakt/provider.go +++ b/internal/watchsync/providers/trakt/provider.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "log/slog" + "maps" "net/http" "net/url" "strconv" @@ -21,7 +22,8 @@ import ( const defaultBaseURL = "https://api.trakt.tv" -const traktMediaShows = "shows" +// traktExtendedProgress asks watched shows for per-episode season progress. +const traktExtendedProgress = "progress" // Trakt rate limits, from its API rate-limiting guide: authenticated users get // one POST/PUT/DELETE per second (AUTHED_API_POST_LIMIT) and 500 GETs per @@ -239,11 +241,12 @@ func (p *Provider) FetchWatched( cfg watchsync.ServerConfig, conn watchsync.Connection, ) ([]watchsync.RemoteWatch, error) { - movies, err := fetchWatchedPages[traktWatchedMovie](ctx, p, cfg, conn, "movies") + movies, err := fetchTraktPages[traktWatchedMovie](ctx, p, cfg, conn, "/sync/watched/movies", nil) if err != nil { return nil, err } - shows, err := fetchWatchedPages[traktWatchedShow](ctx, p, cfg, conn, traktMediaShows) + // Season and episode watched data is no longer included by default. + shows, err := fetchTraktPages[traktWatchedShow](ctx, p, cfg, conn, "/sync/watched/shows", url.Values{"extended": {traktExtendedProgress}}) if err != nil { return nil, err } @@ -288,27 +291,81 @@ func (p *Provider) FetchWatched( return rows, nil } -// fetchWatchedPages requests explicit pagination for Trakt's watched endpoints. -// Stop on an empty page: Trakt may apply a smaller limit than requested, -// particularly for shows with season progress, so a short page is not the end. -func fetchWatchedPages[T any](ctx context.Context, p *Provider, cfg watchsync.ServerConfig, conn watchsync.Connection, kind string) ([]T, error) { - query := url.Values{"limit": {"250"}} - if kind == traktMediaShows { - // Season and episode watched data is no longer included by default. - query.Set("extended", "progress") - } +const ( + // traktPageLimit is Trakt's maximum page size. Larger limits are clamped. + traktPageLimit = 250 + // traktMaxPages bounds a listing whose last page is never detected, such + // as a server that ignores page and sends no pagination headers. + traktMaxPages = 1000 +) + +// fetchTraktPages loads every page of a paginated Trakt GET endpoint. Trakt +// serves only a short first page when page and limit are omitted, so both are +// always sent; they replace any page or limit in query, and other parameters +// such as extended are kept. A failure on any page returns an error and no +// rows, so callers never import a partial listing. +func fetchTraktPages[T any]( + ctx context.Context, + p *Provider, + cfg watchsync.ServerConfig, + conn watchsync.Connection, + path string, + query url.Values, +) ([]T, error) { + params := url.Values{} + maps.Copy(params, query) + params.Set("limit", strconv.Itoa(traktPageLimit)) var rows []T - for page := 1; ; page++ { - query.Set("page", strconv.Itoa(page)) + itemCount := 0 + for page := 1; page <= traktMaxPages; page++ { + params.Set("page", strconv.Itoa(page)) var batch []T - if err := p.do(ctx, http.MethodGet, "/sync/watched/"+kind+"?"+query.Encode(), cfg, conn.AccessToken, nil, &batch); err != nil { + header, err := p.doWithHeader(ctx, http.MethodGet, path+"?"+params.Encode(), cfg, conn.AccessToken, nil, &batch) + if err != nil { return nil, err } - if len(batch) == 0 { - return rows, nil + // Offset pages shift when the list changes mid-read, which can skip + // an item. A changed item count reveals that, and a skipped item + // would read as removed, so the read fails and is retried next sync. + if count, ok := positiveHeaderInt(header, "X-Pagination-Item-Count"); ok { + if itemCount != 0 && count != itemCount { + return nil, fmt.Errorf("trakt %s changed while it was read (%d items, then %d)", path, itemCount, count) + } + itemCount = count } rows = append(rows, batch...) + if lastTraktPage(header, page, len(batch)) { + return rows, nil + } + } + return nil, fmt.Errorf("trakt %s did not reach its last page within %d pages", path, traktMaxPages) +} + +// lastTraktPage reports whether page, holding items rows, ends the listing. +// X-Pagination-Page-Count is authoritative when present. Otherwise a page +// shorter than the applied X-Pagination-Limit is the last one. The requested +// limit is not a safe comparison: Trakt can apply a smaller one, particularly +// for shows with season progress, so without headers only an empty page ends +// the listing. +func lastTraktPage(header http.Header, page, items int) bool { + if items == 0 { + return true + } + if count, ok := positiveHeaderInt(header, "X-Pagination-Page-Count"); ok { + return page >= count + } + if limit, ok := positiveHeaderInt(header, "X-Pagination-Limit"); ok { + return items < limit } + return false +} + +func positiveHeaderInt(header http.Header, key string) (int, bool) { + value, err := strconv.Atoi(strings.TrimSpace(header.Get(key))) + if err != nil || value <= 0 { + return 0, false + } + return value, true } func (p *Provider) FetchProgress( @@ -366,12 +423,12 @@ func (p *Provider) FetchFavorites( cfg watchsync.ServerConfig, conn watchsync.Connection, ) ([]watchsync.RemoteFavorite, error) { - var movies []traktFavoriteMovie - if err := p.do(ctx, http.MethodGet, "/users/me/favorites/movies/added", cfg, conn.AccessToken, nil, &movies); err != nil { + movies, err := fetchTraktPages[traktFavoriteMovie](ctx, p, cfg, conn, "/users/me/favorites/movies/added", nil) + if err != nil { return nil, err } - var shows []traktFavoriteShow - if err := p.do(ctx, http.MethodGet, "/users/me/favorites/shows/added", cfg, conn.AccessToken, nil, &shows); err != nil { + shows, err := fetchTraktPages[traktFavoriteShow](ctx, p, cfg, conn, "/users/me/favorites/shows/added", nil) + if err != nil { return nil, err } return p.remoteListItems(movies, shows), nil @@ -384,12 +441,12 @@ func (p *Provider) FetchWatchlist( cfg watchsync.ServerConfig, conn watchsync.Connection, ) ([]watchsync.RemoteFavorite, error) { - var movies []traktFavoriteMovie - if err := p.do(ctx, http.MethodGet, "/sync/watchlist/movies", cfg, conn.AccessToken, nil, &movies); err != nil { + movies, err := fetchTraktPages[traktFavoriteMovie](ctx, p, cfg, conn, "/sync/watchlist/movies", nil) + if err != nil { return nil, err } - var shows []traktFavoriteShow - if err := p.do(ctx, http.MethodGet, "/sync/watchlist/shows", cfg, conn.AccessToken, nil, &shows); err != nil { + shows, err := fetchTraktPages[traktFavoriteShow](ctx, p, cfg, conn, "/sync/watchlist/shows", nil) + if err != nil { return nil, err } return p.remoteListItems(movies, shows), nil @@ -433,8 +490,10 @@ func (p *Provider) FetchHistory( cfg watchsync.ServerConfig, conn watchsync.Connection, ) ([]watchsync.RemotePlay, error) { - var payload []traktHistoryItem - if err := p.do(ctx, http.MethodGet, "/sync/history", cfg, conn.AccessToken, nil, &payload); err != nil { + // ExportWatched reconciles against every remote play, so a missing page + // would resend plays Trakt already has; Trakt does not deduplicate them. + payload, err := fetchTraktPages[traktHistoryItem](ctx, p, cfg, conn, "/sync/history", nil) + if err != nil { return nil, err } rows := make([]watchsync.RemotePlay, 0, len(payload)) @@ -632,12 +691,27 @@ func (p *Provider) do( body io.Reader, out any, ) error { + _, err := p.doWithHeader(ctx, method, path, cfg, token, body, out) + return err +} + +// doWithHeader is do that also returns the response headers, which carry +// Trakt's X-Pagination-* values. +func (p *Provider) doWithHeader( + ctx context.Context, + method string, + path string, + cfg watchsync.ServerConfig, + token string, + body io.Reader, + out any, +) (http.Header, error) { // Buffer the body so a rate-limited request can be replayed. var payload []byte if body != nil { buffered, err := io.ReadAll(body) if err != nil { - return fmt.Errorf("read trakt request body: %w", err) + return nil, fmt.Errorf("read trakt request body: %w", err) } payload = buffered } @@ -647,16 +721,16 @@ func (p *Provider) do( for attempt := 0; ; attempt++ { if paced { if err := p.writes.Wait(ctx, token); err != nil { - return fmt.Errorf("wait for trakt write limiter: %w", err) + return nil, fmt.Errorf("wait for trakt write limiter: %w", err) } } - wait, limited, err := p.doOnce(ctx, method, path, cfg, token, payload, out) + header, wait, limited, err := p.doOnce(ctx, method, path, cfg, token, payload, out) if !limited { - return err + return header, err } if attempt < maxRetryAttempts && wait <= maxInPlaceRetryWait { if err := p.sleep(ctx, wait); err != nil { - return err + return nil, err } continue } @@ -665,13 +739,14 @@ func (p *Provider) do( if attempt >= maxRetryAttempts && wait < defaultRetryAfter { wait = defaultRetryAfter } - return watchsync.RateLimitedError{Provider: p.Key(), RetryAfter: wait} + return nil, watchsync.RateLimitedError{Provider: p.Key(), RetryAfter: wait} } } -// doOnce performs a single HTTP attempt. A 429 reports limited with the wait -// from Retry-After, or defaultRetryAfter when the header is absent or -// malformed; every other outcome reports its error, if any. +// doOnce performs a single HTTP attempt and returns the response headers. A +// 429 reports limited with the wait from Retry-After, or defaultRetryAfter +// when the header is absent or malformed; every other outcome reports its +// error, if any. func (p *Provider) doOnce( ctx context.Context, method string, @@ -680,19 +755,19 @@ func (p *Provider) doOnce( token string, payload []byte, out any, -) (wait time.Duration, limited bool, err error) { +) (header http.Header, wait time.Duration, limited bool, err error) { var body io.Reader if payload != nil { body = bytes.NewReader(payload) } req, err := http.NewRequestWithContext(ctx, method, p.baseURL+path, body) if err != nil { - return 0, false, fmt.Errorf("create trakt request: %w", err) + return nil, 0, false, fmt.Errorf("create trakt request: %w", err) } p.addHeaders(req, cfg, token) resp, err := p.client.Do(req) if err != nil { - return 0, false, fmt.Errorf("send trakt request: %w", err) + return nil, 0, false, fmt.Errorf("send trakt request: %w", err) } defer resp.Body.Close() if resp.StatusCode == http.StatusTooManyRequests { @@ -700,18 +775,18 @@ func (p *Provider) doOnce( if !ok { wait = defaultRetryAfter } - return wait, true, nil + return nil, wait, true, nil } if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - return 0, false, fmt.Errorf("trakt request %s %s failed: status %d", method, path, resp.StatusCode) + return nil, 0, false, fmt.Errorf("trakt request %s %s failed: status %d", method, path, resp.StatusCode) } if out == nil { - return 0, false, nil + return resp.Header, 0, false, nil } if err := json.NewDecoder(resp.Body).Decode(out); err != nil { - return 0, false, fmt.Errorf("decode trakt response: %w", err) + return nil, 0, false, fmt.Errorf("decode trakt response: %w", err) } - return 0, false, nil + return resp.Header, 0, false, nil } type tokenResponse struct { diff --git a/internal/watchsync/providers/trakt/provider_test.go b/internal/watchsync/providers/trakt/provider_test.go index 44f1438438..6960dc4610 100644 --- a/internal/watchsync/providers/trakt/provider_test.go +++ b/internal/watchsync/providers/trakt/provider_test.go @@ -226,6 +226,7 @@ func TestFetchFavoritesGetsMoviesAndShows(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { paths = append(paths, r.URL.Path) w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Pagination-Page-Count", "1") switch r.URL.Path { case "/users/me/favorites/movies/added": _, _ = w.Write([]byte(`[{"listed_at":"2026-05-04T12:00:00Z","movie":{"title":"Movie","year":2026,"ids":{"imdb":"tt123","tmdb":456}}}]`)) diff --git a/internal/watchsync/providers/trakt/watched_test.go b/internal/watchsync/providers/trakt/watched_test.go index 7a9518d3b5..a24bdd86c5 100644 --- a/internal/watchsync/providers/trakt/watched_test.go +++ b/internal/watchsync/providers/trakt/watched_test.go @@ -33,20 +33,20 @@ func TestFetchWatchedImportsEveryPageAndEpisodeProgress(t *testing.T) { case "1", "2": switch kind { case "movies": - writeWatchedFixture(t, w, `[{"plays":2,"last_watched_at":"2026-09-01T12:00:00Z","movie":{"title":"Movie %s","year":2020,"ids":{"tmdb":10%s}}}]`, page, page) + writeTraktFixture(t, w, `[{"plays":2,"last_watched_at":"2026-09-01T12:00:00Z","movie":{"title":"Movie %s","year":2020,"ids":{"tmdb":10%s}}}]`, page, page) case "shows": if r.URL.Query().Get("extended") != "progress" { // Current Trakt default: no seasons or episodes without progress. - writeWatchedFixture(t, w, `[{"show":{"title":"Show","ids":{"tmdb":200}}}]`) + writeTraktFixture(t, w, `[{"show":{"title":"Show","ids":{"tmdb":200}}}]`) return } - writeWatchedFixture(t, w, `[{"show":{"title":"Show %s","year":2021,"ids":{"tmdb":20%s,"tvdb":30%s,"imdb":"tt40%s"}},"seasons":[{"number":0,"episodes":[{"number":1,"plays":3,"last_watched_at":"2026-09-02T12:00:00Z"}]},{"number":2,"episodes":[{"number":5,"plays":1,"last_watched_at":"2026-09-03T12:00:00Z"}]}]}]`, page, page, page, page) + writeTraktFixture(t, w, `[{"show":{"title":"Show %s","year":2021,"ids":{"tmdb":20%s,"tvdb":30%s,"imdb":"tt40%s"}},"seasons":[{"number":0,"episodes":[{"number":1,"plays":3,"last_watched_at":"2026-09-02T12:00:00Z"}]},{"number":2,"episodes":[{"number":5,"plays":1,"last_watched_at":"2026-09-03T12:00:00Z"}]}]}]`, page, page, page, page) default: t.Errorf("unexpected path %s", r.URL.Path) http.NotFound(w, r) } case "3": - writeWatchedFixture(t, w, `[]`) + writeTraktFixture(t, w, `[]`) default: t.Errorf("unexpected page %q", page) http.Error(w, "unexpected page", 500) @@ -89,18 +89,18 @@ func TestFetchWatchedDoesNotReturnPartialHistoryOnLaterPageFailure(t *testing.T) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") if r.URL.Query().Get("page") == "1" { - writeWatchedFixture(t, w, `[{"plays":1,"last_watched_at":"2026-09-01T12:00:00Z","movie":{"ids":{"tmdb":123}},"show":{"ids":{"tmdb":456}},"seasons":[{"number":1,"episodes":[{"number":1,"plays":1,"last_watched_at":"2026-09-01T12:00:00Z"}]}]}]`) + writeTraktFixture(t, w, `[{"plays":1,"last_watched_at":"2026-09-01T12:00:00Z","movie":{"ids":{"tmdb":123}},"show":{"ids":{"tmdb":456}},"seasons":[{"number":1,"episodes":[{"number":1,"plays":1,"last_watched_at":"2026-09-01T12:00:00Z"}]}]}]`) return } if r.URL.Path == "/sync/watched/"+kind { if failure == "http" { http.Error(w, "unavailable", http.StatusServiceUnavailable) } else { - writeWatchedFixture(t, w, `[{`) + writeTraktFixture(t, w, `[{`) } return } - writeWatchedFixture(t, w, `[]`) + writeTraktFixture(t, w, `[]`) })) defer server.Close() rows, err := NewProvider(server.Client(), server.URL).FetchWatched(context.Background(), watchsync.ServerConfig{}, watchsync.Connection{}) @@ -112,9 +112,9 @@ func TestFetchWatchedDoesNotReturnPartialHistoryOnLaterPageFailure(t *testing.T) } } -func writeWatchedFixture(t *testing.T, w http.ResponseWriter, format string, args ...any) { +func writeTraktFixture(t *testing.T, w http.ResponseWriter, format string, args ...any) { t.Helper() if _, err := fmt.Fprintf(w, format, args...); err != nil { - t.Errorf("write watched fixture: %v", err) + t.Errorf("write trakt fixture: %v", err) } } From de0dfdedd6910877fe72bde5eedec78f79f0f6ad Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Wed, 23 Sep 2026 22:09:04 +0000 Subject: [PATCH 03/20] fix(watchsync): keep the MDBList API key out of error text The MDBList watch-sync provider puts the API key in the request query string. Transport errors wrap *url.Error, whose message includes that URL, and those errors reach the connection's last_error, sync run errors shown in the web UI, and logs. Build the keyed URL only inside the single request helper, sanitize transport and request-build errors with logredact.SanitizeURLError, and mask the key (raw or escaped) in error-body excerpts. Cancellation and timeout classification is kept. Same bug class as #692, which covers the separate internal/mdblist discovery client. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../watchsync/providers/mdblist/provider.go | 98 +++++- .../providers/mdblist/provider_test.go | 279 ++++++++++++++++++ 2 files changed, 362 insertions(+), 15 deletions(-) diff --git a/internal/watchsync/providers/mdblist/provider.go b/internal/watchsync/providers/mdblist/provider.go index b6fd7b89eb..13499246ac 100644 --- a/internal/watchsync/providers/mdblist/provider.go +++ b/internal/watchsync/providers/mdblist/provider.go @@ -20,6 +20,7 @@ import ( "golang.org/x/time/rate" "github.com/Silo-Server/silo-server/internal/historyimport" + "github.com/Silo-Server/silo-server/internal/logredact" "github.com/Silo-Server/silo-server/internal/userstore" "github.com/Silo-Server/silo-server/internal/watchsync" ) @@ -534,12 +535,6 @@ func (p *Provider) do(ctx context.Context, method string, path string, apiKey st if strings.TrimSpace(apiKey) == "" { return errors.New("mdblist api key is missing") } - target := p.baseURL + path - separator := "?" - if strings.Contains(path, "?") { - separator = "&" - } - target += separator + "apikey=" + url.QueryEscape(apiKey) // Buffer the body so rate-limited attempts can be replayed. var payload []byte @@ -556,7 +551,7 @@ func (p *Provider) do(ctx context.Context, method string, path string, apiKey st if err := limiter.Wait(ctx); err != nil { return fmt.Errorf("wait for mdblist rate limiter: %w", err) } - retryAfter, err := p.doOnce(ctx, method, path, target, payload, out) + retryAfter, err := p.doOnce(ctx, method, path, apiKey, payload, out) if err == nil { return nil } @@ -588,14 +583,18 @@ func (p *Provider) do(ctx context.Context, method string, path string, apiKey st // doOnce performs a single HTTP attempt. On a 429 it returns the wait hinted // by Retry-After (0 when absent) alongside the error; every other failure // returns -1 to signal "not retryable". -func (p *Provider) doOnce(ctx context.Context, method, path, target string, payload []byte, out any) (time.Duration, error) { +// +// The request URL carries the API key, so it must stay inside this function: +// errors returned here name the request by method and path, or by a +// sanitized URL. +func (p *Provider) doOnce(ctx context.Context, method, path, apiKey string, payload []byte, out any) (time.Duration, error) { var body io.Reader if payload != nil { body = bytes.NewReader(payload) } - req, err := http.NewRequestWithContext(ctx, method, target, body) + req, err := http.NewRequestWithContext(ctx, method, p.requestURL(path, apiKey), body) if err != nil { - return -1, fmt.Errorf("create mdblist request: %w", err) + return -1, requestError("create", apiKey, err) } if payload != nil { req.Header.Set("Content-Type", "application/json") @@ -604,7 +603,7 @@ func (p *Provider) doOnce(ctx context.Context, method, path, target string, payl resp, err := p.client.Do(req) if err != nil { - return -1, fmt.Errorf("send mdblist request: %w", err) + return -1, requestError("send", apiKey, err) } defer resp.Body.Close() if resp.StatusCode == http.StatusTooManyRequests { @@ -617,7 +616,7 @@ func (p *Provider) doOnce(ctx context.Context, method, path, target string, payl return -1, fmt.Errorf("mdblist request %s %s rejected: status %d (check api key): %w", method, path, resp.StatusCode, watchsync.ErrInvalidCredential) } if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - detail := responseErrorDetail(resp.Body) + detail := responseErrorDetail(resp.Body, apiKey) if detail != "" { return -1, fmt.Errorf("mdblist request %s %s failed: status %d: %s", method, path, resp.StatusCode, detail) } @@ -632,12 +631,81 @@ func (p *Provider) doOnce(ctx context.Context, method, path, target string, payl return -1, nil } -func responseErrorDetail(body io.Reader) string { - raw, err := io.ReadAll(io.LimitReader(body, maxErrorBodyBytes)) +// requestURL appends the API key MDBList expects as a query parameter. The +// result is a credential and must never reach error text or logs. +func (p *Provider) requestURL(path, apiKey string) string { + separator := "?" + if strings.Contains(path, "?") { + separator = "&" + } + return p.baseURL + path + separator + "apikey=" + url.QueryEscape(apiKey) +} + +// requestError reports a failure to build or send a request without the API +// key. http.NewRequestWithContext and http.Client.Do both return a *url.Error +// whose message embeds the request URL, key included; SanitizeURLError drops +// the query string while keeping the cause chain, so errors.Is still matches +// context.Canceled and context.DeadlineExceeded and errors.As still finds +// net.Error timeouts. +func requestError(stage, apiKey string, err error) error { + err = logredact.SanitizeURLError(err) + // The cause can still quote a URL outside the *url.Error field: a + // redirect with an unparseable Location header reports that + // server-supplied value verbatim. + if msg := err.Error(); redactAPIKey(msg, apiKey) != msg { + err = errors.New(redactAPIKey(msg, apiKey)) + } + return fmt.Errorf("%s mdblist request: %w", stage, err) +} + +// redactAPIKey masks every occurrence of the API key, raw or query-escaped. +func redactAPIKey(text, apiKey string) string { + for _, form := range apiKeyForms(apiKey) { + text = strings.ReplaceAll(text, form, logredact.Placeholder) + } + return text +} + +// trimPartialAPIKey drops a trailing fragment of the API key that truncation +// separated from the rest of the key, which redactAPIKey cannot recognize. +func trimPartialAPIKey(text, apiKey string) string { + for _, form := range apiKeyForms(apiKey) { + for n := len(form) - 1; n > 0; n-- { + if strings.HasSuffix(text, form[:n]) { + text = text[:len(text)-n] + break + } + } + } + return text +} + +func apiKeyForms(apiKey string) []string { + if apiKey == "" { + return nil + } + if escaped := url.QueryEscape(apiKey); escaped != apiKey { + return []string{apiKey, escaped} + } + return []string{apiKey} +} + +// responseErrorDetail returns a bounded excerpt of an error response body with +// the API key masked, since a server error page can echo the request URL. +func responseErrorDetail(body io.Reader, apiKey string) string { + raw, err := io.ReadAll(io.LimitReader(body, maxErrorBodyBytes+1)) if err != nil { return "" } - raw = bytes.TrimSpace(raw) + truncated := len(raw) > maxErrorBodyBytes + if truncated { + raw = raw[:maxErrorBodyBytes] + } + text := redactAPIKey(string(raw), apiKey) + if truncated { + text = trimPartialAPIKey(text, apiKey) + } + raw = bytes.TrimSpace([]byte(text)) if len(raw) == 0 { return "" } diff --git a/internal/watchsync/providers/mdblist/provider_test.go b/internal/watchsync/providers/mdblist/provider_test.go index 35069849b9..cd3109eb44 100644 --- a/internal/watchsync/providers/mdblist/provider_test.go +++ b/internal/watchsync/providers/mdblist/provider_test.go @@ -5,13 +5,16 @@ import ( "encoding/json" "errors" "io" + "net" "net/http" "net/http/httptest" + "net/url" "strings" "testing" "time" "github.com/Silo-Server/silo-server/internal/historyimport" + "github.com/Silo-Server/silo-server/internal/logredact" "github.com/Silo-Server/silo-server/internal/watchsync" ) @@ -829,3 +832,279 @@ func TestDoFloorsRetryAfterWhenRetriesExhausted(t *testing.T) { t.Fatalf("got retry-after %s, want floored %s", rle.RetryAfter, defaultRetryAfter) } } + +const sentinelAPIKey = "SENTINEL-KEY-123" + +var errInjectedTransport = errors.New("injected transport failure") + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +// assertErrorOmitsAPIKey fails when the key appears anywhere in err's chain, +// not only in its top-level message. +func assertErrorOmitsAPIKey(t *testing.T, err error) { + t.Helper() + if err == nil { + t.Fatal("expected an error") + } + for e := err; e != nil; e = errors.Unwrap(e) { + if strings.Contains(e.Error(), sentinelAPIKey) { + t.Fatalf("error chain leaks the API key: %q", e.Error()) + } + } +} + +// keyedRequests covers each request shape the provider sends with the API key: +// GET with and without a query string, and POST with a JSON body. +var keyedRequests = []struct { + name string + call func(context.Context, *Provider) error +}{ + {"GET user", func(ctx context.Context, p *Provider) error { + _, _, err := p.ConnectWithAPIKey(ctx, sentinelAPIKey) + return err + }}, + {"GET watched page", func(ctx context.Context, p *Provider) error { + _, err := p.FetchWatched(ctx, watchsync.ServerConfig{}, sentinelConnection()) + return err + }}, + {"GET playback", func(ctx context.Context, p *Provider) error { + _, err := p.FetchProgress(ctx, watchsync.ServerConfig{}, sentinelConnection()) + return err + }}, + {"POST watched", func(ctx context.Context, p *Provider) error { + _, err := p.ExportHistory(ctx, watchsync.ServerConfig{}, sentinelConnection(), []watchsync.LocalPlay{ + {HistoryID: "h1", Kind: historyimport.KindMovie, IMDbID: "tt0111161"}, + }) + return err + }}, + {"POST watchlist", func(ctx context.Context, p *Provider) error { + _, err := p.ExportWatchlist(ctx, watchsync.ServerConfig{}, sentinelConnection(), []watchsync.LocalFavorite{ + {MediaItemID: "m1", Kind: historyimport.KindMovie, IMDbID: "tt0111161"}, + }) + return err + }}, + {"POST scrobble", func(ctx context.Context, p *Provider) error { + return p.Start(ctx, watchsync.ServerConfig{}, sentinelConnection(), watchsync.ScrobbleEvent{ + Kind: historyimport.KindMovie, + IMDbID: "tt0111161", + PositionSeconds: 60, + DurationSeconds: 600, + }) + }}, +} + +func sentinelConnection() watchsync.Connection { + return watchsync.Connection{ID: "conn-1", AccessToken: sentinelAPIKey} +} + +func TestRequestFailuresOmitAPIKey(t *testing.T) { + sources := []struct { + name string + newProvider func(t *testing.T) *Provider + check func(t *testing.T, err error) + }{ + { + name: "transport error", + newProvider: func(*testing.T) *Provider { + client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, errInjectedTransport + })} + return NewProvider(client, "http://mdblist.test") + }, + check: func(t *testing.T, err error) { + if !errors.Is(err, errInjectedTransport) { + t.Fatalf("lost the transport cause: %v", err) + } + }, + }, + { + name: "connection refused", + newProvider: func(*testing.T) *Provider { + server := httptest.NewServer(http.NotFoundHandler()) + server.Close() + return NewProvider(&http.Client{}, server.URL) + }, + check: func(t *testing.T, err error) { + var opErr *net.OpError + if !errors.As(err, &opErr) { + t.Fatalf("lost the dial error: %v", err) + } + }, + }, + { + name: "malformed base URL", + newProvider: func(*testing.T) *Provider { + return NewProvider(&http.Client{}, "http://mdblist.test/\x7f") + }, + }, + { + name: "redirect loop", + newProvider: func(t *testing.T) *Provider { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, r.URL.RequestURI(), http.StatusFound) + })) + t.Cleanup(server.Close) + return NewProvider(server.Client(), server.URL) + }, + }, + { + name: "unparseable redirect", + newProvider: func(t *testing.T) *Provider { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", "http://%zz/?apikey="+r.URL.Query().Get("apikey")) + w.WriteHeader(http.StatusFound) + })) + t.Cleanup(server.Close) + return NewProvider(server.Client(), server.URL) + }, + }, + } + for _, source := range sources { + for _, request := range keyedRequests { + t.Run(source.name+"/"+request.name, func(t *testing.T) { + err := request.call(context.Background(), source.newProvider(t)) + assertErrorOmitsAPIKey(t, err) + if source.check != nil { + source.check(t, err) + } + }) + } + } +} + +func TestRateLimitRetryFailureOmitsAPIKey(t *testing.T) { + attempts := 0 + client := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + attempts++ + if attempts == 1 { + return &http.Response{ + StatusCode: http.StatusTooManyRequests, + Header: http.Header{"Retry-After": {"1"}}, + Body: io.NopCloser(strings.NewReader("")), + Request: r, + }, nil + } + return nil, errInjectedTransport + })} + p := NewProvider(client, "http://mdblist.test") + + err := p.do(context.Background(), http.MethodPost, "/watchlist/items/add", sentinelAPIKey, strings.NewReader(`{"movies":[]}`), nil) + assertErrorOmitsAPIKey(t, err) + if attempts != 2 { + t.Fatalf("got %d attempts, want the in-place retry to run", attempts) + } + if !errors.Is(err, errInjectedTransport) { + t.Fatalf("lost the transport cause: %v", err) + } +} + +func TestCanceledRequestStillMatchesContextCanceled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + cancel() + select { + case <-r.Context().Done(): + case <-release: + } + })) + defer server.Close() + defer close(release) + + p := NewProvider(server.Client(), server.URL) + _, _, err := p.ConnectWithAPIKey(ctx, sentinelAPIKey) + assertErrorOmitsAPIKey(t, err) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got %v", err) + } +} + +func TestTimedOutRequestStaysDetectable(t *testing.T) { + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + select { + case <-r.Context().Done(): + case <-release: + } + })) + defer server.Close() + defer close(release) + + cases := []struct { + name string + client *http.Client + ctx func() (context.Context, context.CancelFunc) + }{ + { + name: "client timeout", + client: &http.Client{Timeout: 50 * time.Millisecond}, + ctx: func() (context.Context, context.CancelFunc) { return context.WithCancel(context.Background()) }, + }, + { + name: "context deadline", + client: &http.Client{}, + ctx: func() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), 50*time.Millisecond) + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := tc.ctx() + defer cancel() + p := NewProvider(tc.client, server.URL) + _, _, err := p.ConnectWithAPIKey(ctx, sentinelAPIKey) + assertErrorOmitsAPIKey(t, err) + var netErr net.Error + if !errors.As(err, &netErr) || !netErr.Timeout() { + t.Fatalf("expected a net.Error timeout, got %v", err) + } + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected context.DeadlineExceeded, got %v", err) + } + }) + } +} + +func TestDoRedactsAPIKeyEchoedInErrorBody(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusInternalServerError) + _ = json.NewEncoder(w).Encode(map[string]string{"error": "unexpected failure for " + r.URL.RequestURI()}) + })) + defer server.Close() + + p := NewProvider(server.Client(), server.URL) + _, _, err := p.ConnectWithAPIKey(context.Background(), sentinelAPIKey) + assertErrorOmitsAPIKey(t, err) + if !strings.Contains(err.Error(), "unexpected failure for /user?apikey="+logredact.Placeholder) { + t.Fatalf("expected the redacted body excerpt, got %v", err) + } +} + +func TestResponseErrorDetailDropsKeyFragmentCutByLimit(t *testing.T) { + filler := strings.Repeat("x", maxErrorBodyBytes-10) + detail := responseErrorDetail(strings.NewReader(filler+sentinelAPIKey+" tail"), sentinelAPIKey) + if strings.Contains(detail, sentinelAPIKey[:10]) { + t.Fatalf("excerpt keeps the key fragment cut by the limit: %q", detail[len(detail)-20:]) + } + if detail != filler { + t.Fatalf("got %d-byte excerpt, want the %d filler bytes", len(detail), len(filler)) + } +} + +func TestRedactAPIKeyMasksRawAndEscapedForms(t *testing.T) { + const key = "k/y+z 1" + text := "raw=" + key + " escaped=" + url.QueryEscape(key) + got := redactAPIKey(text, key) + want := "raw=" + logredact.Placeholder + " escaped=" + logredact.Placeholder + if got != want { + t.Fatalf("redactAPIKey = %q, want %q", got, want) + } + if redactAPIKey("unchanged", "") != "unchanged" { + t.Fatal("empty key must not alter text") + } +} From 7fd6898c05dbd0d85225f84a7dd8428b47577a7d Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Wed, 23 Sep 2026 22:10:26 +0000 Subject: [PATCH 04/20] fix(watchsync): match Trakt not_found echoes by any shared id Trakt favorites and watchlist writes mapped not_found echoes back to request items by one derived key per side. Silo keys items IMDb-first while the provider keyed shows TVDB-first, so a show sent with both ids and echoed as missing was still reported as sent. Match echoes by any shared identifier (Trakt id, slug, IMDb, TMDB, TVDB), namespaced by kind because TMDB and TVDB number movies and shows separately. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/watchsync/lists.go | 14 +- .../watchsync/providers/trakt/provider.go | 115 +++++++++++-- .../providers/trakt/provider_test.go | 154 ++++++++++++++++++ internal/watchsync/service.go | 22 ++- internal/watchsync/service_test.go | 22 +++ 5 files changed, 294 insertions(+), 33 deletions(-) diff --git a/internal/watchsync/lists.go b/internal/watchsync/lists.go index 1abc8f5809..094a5ddeeb 100644 --- a/internal/watchsync/lists.go +++ b/internal/watchsync/lists.go @@ -494,9 +494,8 @@ func (s *Service) exportList(ctx context.Context, conn Connection, cfg ServerCon return result, err } now := s.now() - sent := exportResultSentSet(exportResult) for _, item := range toSend { - if sent[item.MediaItemID] || sent[item.ProviderItemKey] { + if sent, _ := exportItemOutcome(exportResult, item.MediaItemID, item.ProviderItemKey); sent { if err := s.repo.MarkListItemExported(ctx, conn.ID, b.kind, item.MediaItemID, now); err != nil { return result, err } @@ -562,13 +561,11 @@ func (s *Service) removePendingListItems(ctx context.Context, conn Connection, c return removed, err } now := s.now() - sent := exportResultSentSet(result) for _, item := range items { attempted[item.MediaItemID] = true // Sent (removed) and NotFound (already absent remotely) both reconcile // the row; true failures stay pending for the next run. - if sent[item.MediaItemID] || sent[item.ProviderItemKey] || - containsString(result.NotFound, item.MediaItemID) || containsString(result.NotFound, item.ProviderItemKey) { + if sent, notFound := exportItemOutcome(result, item.MediaItemID, item.ProviderItemKey); sent || notFound { if err := s.repo.MarkListItemRemoteRemoved(ctx, conn.ID, b.kind, item.MediaItemID, now); err != nil { return removed, err } @@ -709,10 +706,8 @@ func (s *Service) processLocalListEvent(ctx context.Context, event LocalListEven s.recordLocalWatchEventError(ctx, conn, err) continue } - sent := exportResultSentSet(result) for _, item := range event.Items { - if sent[item.MediaItemID] || sent[item.ProviderItemKey] || - containsString(result.NotFound, item.MediaItemID) || containsString(result.NotFound, item.ProviderItemKey) { + if sent, notFound := exportItemOutcome(result, item.MediaItemID, item.ProviderItemKey); sent || notFound { if err := s.repo.MarkListItemRemoteRemoved(ctx, conn.ID, b.kind, item.MediaItemID, now); err != nil { return err } @@ -764,9 +759,8 @@ func (s *Service) exportLocalListItems(ctx context.Context, conn Connection, cfg return err } now := s.now() - sent := exportResultSentSet(result) for _, item := range toSend { - if sent[item.MediaItemID] || sent[item.ProviderItemKey] { + if sent, _ := exportItemOutcome(result, item.MediaItemID, item.ProviderItemKey); sent { if err := s.repo.MarkListItemExported(ctx, conn.ID, b.kind, item.MediaItemID, now); err != nil { return err } diff --git a/internal/watchsync/providers/trakt/provider.go b/internal/watchsync/providers/trakt/provider.go index 54b6a917c3..f9a244aeaa 100644 --- a/internal/watchsync/providers/trakt/provider.go +++ b/internal/watchsync/providers/trakt/provider.go @@ -816,6 +816,72 @@ type traktIDs struct { TVDB int `json:"tvdb"` } +// traktIDIndex matches items Trakt echoes back in a response, such as its +// not_found lists, to the request items that produced them. An echo matches an +// item when the two share ANY identifier (Trakt id, slug, IMDb, TMDB, or TVDB): +// Trakt may echo a different id subset than Silo sent, and Silo keys an item by +// its own preferred id, so comparing one derived key per side misses matches. +// Identifiers are namespaced by Silo item kind (historyimport.Kind*) because +// TMDB and TVDB number movies, shows, and episodes independently. Zero ids +// never match. Create one with traktIDIndex{}. +// +// Limitation: an echo that carries only identifiers the item lacks (for +// example a bare Trakt id for an item Silo knows only by IMDb) cannot be +// matched, so callers treat that item as accepted. +type traktIDIndex map[traktIDRef]struct{} + +// ID schemes, as used in provider item keys ("tmdb:949") and traktIDRef. +const ( + idSchemeTrakt = "trakt" + idSchemeSlug = "slug" + idSchemeIMDb = "imdb" + idSchemeTMDB = "tmdb" + idSchemeTVDB = "tvdb" +) + +type traktIDRef struct { + kind string + scheme string + value string +} + +// add records every non-zero identifier in ids under kind. +func (idx traktIDIndex) add(kind string, ids traktIDs) { + for _, ref := range traktIDRefs(kind, ids) { + idx[ref] = struct{}{} + } +} + +// matches reports whether any non-zero identifier in ids was added under kind. +func (idx traktIDIndex) matches(kind string, ids traktIDs) bool { + for _, ref := range traktIDRefs(kind, ids) { + if _, ok := idx[ref]; ok { + return true + } + } + return false +} + +func traktIDRefs(kind string, ids traktIDs) []traktIDRef { + refs := make([]traktIDRef, 0, 5) + if ids.Trakt > 0 { + refs = append(refs, traktIDRef{kind: kind, scheme: idSchemeTrakt, value: strconv.Itoa(ids.Trakt)}) + } + if ids.Slug != "" { + refs = append(refs, traktIDRef{kind: kind, scheme: idSchemeSlug, value: ids.Slug}) + } + if ids.IMDb != "" { + refs = append(refs, traktIDRef{kind: kind, scheme: idSchemeIMDb, value: ids.IMDb}) + } + if ids.TMDB > 0 { + refs = append(refs, traktIDRef{kind: kind, scheme: idSchemeTMDB, value: strconv.Itoa(ids.TMDB)}) + } + if ids.TVDB > 0 { + refs = append(refs, traktIDRef{kind: kind, scheme: idSchemeTVDB, value: strconv.Itoa(ids.TVDB)}) + } + return refs +} + type traktMovie struct { Title string `json:"title"` Year int `json:"year"` @@ -1156,11 +1222,8 @@ func appendNestedRemoveEpisode(shows []traktHistoryRemoveShow, showIDs traktIDs, func buildFavoritesPayload(favorites []watchsync.LocalFavorite) traktFavoritesPayload { var payload traktFavoritesPayload for _, favorite := range favorites { - ids := traktIDs{IMDb: favorite.IMDbID, TMDB: parseInt(favorite.TMDBID), TVDB: parseInt(favorite.TVDBID)} - if ids.IMDb == "" && ids.TMDB == 0 && ids.TVDB == 0 { - ids = idsFromProviderItemKey(favorite.ProviderItemKey) - } - if ids.IMDb == "" && ids.TMDB == 0 && ids.TVDB == 0 { + ids := favoriteIDs(favorite) + if !sendableIDs(ids) { continue } switch favorite.Kind { @@ -1173,24 +1236,32 @@ func buildFavoritesPayload(favorites []watchsync.LocalFavorite) traktFavoritesPa return payload } +// favoriteExportResult maps a favorites or watchlist response back to the +// request items as (MediaItemID, key) pairs. An item goes to NotFound when a +// not_found echo of the same kind shares any id with the ids it was sent with +// (see traktIDIndex for the limitation), otherwise to Sent. Items with no key +// are left out of both lists. func favoriteExportResult(favorites []watchsync.LocalFavorite, notFound traktFavoritesPayload) watchsync.ExportResult { result := watchsync.ExportResult{Sent: make([]string, 0, len(favorites))} - notFoundKeys := map[string]bool{} + missing := traktIDIndex{} for _, movie := range notFound.Movies { - notFoundKeys[movieKey(movie.IDs)] = true + missing.add(historyimport.KindMovie, movie.IDs) } for _, show := range notFound.Shows { - notFoundKeys[showKey(show.IDs)] = true + missing.add(historyimport.KindSeries, show.IDs) } for _, favorite := range favorites { key := favorite.ProviderItemKey if key == "" { key = favoriteKey(favorite) } - if key == "" { + ids := favoriteIDs(favorite) + // An item without a sendable id was left out of the request, so it + // is neither sent nor reported missing. + if key == "" || !sendableIDs(ids) { continue } - if notFoundKeys[key] { + if missing.matches(favorite.Kind, ids) { result.NotFound = append(result.NotFound, favorite.MediaItemID, key) continue } @@ -1199,11 +1270,25 @@ func favoriteExportResult(favorites []watchsync.LocalFavorite, notFound traktFav return result } -func favoriteKey(favorite watchsync.LocalFavorite) string { +// favoriteIDs returns the ids a favorite or watchlist item is sent to Trakt +// with: its own external ids, falling back to the id its provider item key +// encodes. +func favoriteIDs(favorite watchsync.LocalFavorite) traktIDs { ids := traktIDs{IMDb: favorite.IMDbID, TMDB: parseInt(favorite.TMDBID), TVDB: parseInt(favorite.TVDBID)} if ids.IMDb == "" && ids.TMDB == 0 && ids.TVDB == 0 { ids = idsFromProviderItemKey(favorite.ProviderItemKey) } + return ids +} + +// sendableIDs reports whether ids can identify a title in a Trakt sync write. +// Trakt accepts its own id as well as IMDb, TMDB, and TVDB ids. +func sendableIDs(ids traktIDs) bool { + return hasAnyID(ids) || ids.Trakt > 0 +} + +func favoriteKey(favorite watchsync.LocalFavorite) string { + ids := favoriteIDs(favorite) if favorite.Kind == historyimport.KindSeries { return showKey(ids) } @@ -1216,13 +1301,13 @@ func idsFromProviderItemKey(key string) traktIDs { return traktIDs{} } switch prefix { - case "imdb": + case idSchemeIMDb: return traktIDs{IMDb: value} - case "tmdb": + case idSchemeTMDB: return traktIDs{TMDB: parseInt(value)} - case "tvdb": + case idSchemeTVDB: return traktIDs{TVDB: parseInt(value)} - case "trakt": + case idSchemeTrakt: return traktIDs{Trakt: parseInt(value)} default: return traktIDs{} diff --git a/internal/watchsync/providers/trakt/provider_test.go b/internal/watchsync/providers/trakt/provider_test.go index 6960dc4610..45f6a80a39 100644 --- a/internal/watchsync/providers/trakt/provider_test.go +++ b/internal/watchsync/providers/trakt/provider_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "slices" "testing" "time" @@ -331,6 +332,159 @@ func TestRemoveFavoritesCanUseProviderItemKeys(t *testing.T) { } } +type listSyncCall func(*Provider, context.Context, watchsync.ServerConfig, watchsync.Connection, []watchsync.LocalFavorite) (watchsync.ExportResult, error) + +// runListSync sends items through call against a fake Trakt that answers +// wantPath with the given not_found body. +func runListSync(t *testing.T, call listSyncCall, wantPath, notFound string, items []watchsync.LocalFavorite) watchsync.ExportResult { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != wantPath { + t.Errorf("got path %q, want %s", r.URL.Path, wantPath) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"not_found":` + notFound + `}`)) + })) + defer server.Close() + + result, err := call(NewProvider(server.Client(), server.URL), context.Background(), watchsync.ServerConfig{ + ClientID: "client-id", + ClientSecret: "client-secret", + }, watchsync.Connection{AccessToken: "token"}, items) + if err != nil { + t.Fatalf("list sync: %v", err) + } + return result +} + +func TestListSyncMatchesNotFoundShowByAnySharedID(t *testing.T) { + // Silo keys the show by IMDb, as providerItemKeyForLocalFavorite does; + // Trakt echoes it back by TVDB alone. + show := watchsync.LocalFavorite{ + MediaItemID: "show-1", + Kind: historyimport.KindSeries, + ProviderItemKey: "imdb:tt0903747", + IMDbID: "tt0903747", + TVDBID: "81189", + } + notFound := `{"movies":[],"shows":[{"ids":{"trakt":0,"slug":"","imdb":"","tmdb":0,"tvdb":81189}}]}` + + for _, tc := range []struct { + name string + path string + call listSyncCall + }{ + {"export favorites", "/sync/favorites", (*Provider).ExportFavorites}, + {"remove favorites", "/sync/favorites/remove", (*Provider).RemoveFavorites}, + {"export watchlist", "/sync/watchlist", (*Provider).ExportWatchlist}, + {"remove watchlist", "/sync/watchlist/remove", (*Provider).RemoveWatchlist}, + } { + t.Run(tc.name, func(t *testing.T) { + result := runListSync(t, tc.call, tc.path, notFound, []watchsync.LocalFavorite{show}) + if !slices.Equal(result.NotFound, []string{"show-1", "imdb:tt0903747"}) { + t.Fatalf("not found = %#v, want show-1 and its provider key", result.NotFound) + } + if len(result.Sent) != 0 { + t.Fatalf("sent = %#v, want none", result.Sent) + } + }) + } +} + +func TestExportFavoritesMatchesNotFoundTMDBOnlyMovie(t *testing.T) { + result := runListSync(t, (*Provider).ExportFavorites, "/sync/favorites", + `{"movies":[{"ids":{"trakt":0,"slug":"","imdb":"","tmdb":550,"tvdb":0}}],"shows":[]}`, + []watchsync.LocalFavorite{{MediaItemID: "movie-1", Kind: historyimport.KindMovie, TMDBID: "550"}}) + if !slices.Equal(result.NotFound, []string{"movie-1", "tmdb:550"}) { + t.Fatalf("not found = %#v, want movie-1 and tmdb:550", result.NotFound) + } + if len(result.Sent) != 0 { + t.Fatalf("sent = %#v, want none", result.Sent) + } +} + +func TestExportFavoritesMixedBatchReportsOnlyTheMissingItem(t *testing.T) { + result := runListSync(t, (*Provider).ExportFavorites, "/sync/favorites", + `{"movies":[{"ids":{"tmdb":550}}],"shows":[]}`, + []watchsync.LocalFavorite{ + {MediaItemID: "movie-found", Kind: historyimport.KindMovie, ProviderItemKey: "imdb:tt0133093", IMDbID: "tt0133093", TMDBID: "603"}, + {MediaItemID: "movie-missing", Kind: historyimport.KindMovie, ProviderItemKey: "tmdb:550", TMDBID: "550"}, + // Same TMDB number as the missing movie, but TMDB numbers shows + // separately, so the movie echo must not match it. + {MediaItemID: "show-found", Kind: historyimport.KindSeries, ProviderItemKey: "tmdb:550", TMDBID: "550", TVDBID: "81189"}, + // No usable ids or key: not sent, and reported in neither list. + {MediaItemID: "movie-no-ids", Kind: historyimport.KindMovie}, + }) + if !slices.Equal(result.NotFound, []string{"movie-missing", "tmdb:550"}) { + t.Fatalf("not found = %#v, want only movie-missing", result.NotFound) + } + wantSent := []string{"movie-found", "imdb:tt0133093", "show-found", "tmdb:550"} + if !slices.Equal(result.Sent, wantSent) { + t.Fatalf("sent = %#v, want %#v", result.Sent, wantSent) + } +} + +func TestExportFavoritesEchoWithOnlyUnknownIDStaysSent(t *testing.T) { + // Documented traktIDIndex limitation: the echo shares no id with the + // item, so it cannot be attributed and the item counts as sent. + result := runListSync(t, (*Provider).ExportFavorites, "/sync/favorites", + `{"movies":[{"ids":{"trakt":12601}}],"shows":[]}`, + []watchsync.LocalFavorite{{MediaItemID: "movie-1", Kind: historyimport.KindMovie, IMDbID: "tt0133093"}}) + if len(result.NotFound) != 0 { + t.Fatalf("not found = %#v, want none", result.NotFound) + } + if !slices.Equal(result.Sent, []string{"movie-1", "imdb:tt0133093"}) { + t.Fatalf("sent = %#v, want movie-1 and imdb:tt0133093", result.Sent) + } +} + +func TestRemoveFavoritesSendsItemsKnownOnlyByTraktID(t *testing.T) { + var gotBody map[string][]map[string]map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Errorf("decode request body: %v", err) + } + _, _ = w.Write([]byte(`{"deleted":{"movies":1},"not_found":{"movies":[],"shows":[]}}`)) + })) + defer server.Close() + + result, err := NewProvider(server.Client(), server.URL).RemoveFavorites(context.Background(), + watchsync.ServerConfig{ClientID: "client-id"}, watchsync.Connection{AccessToken: "token"}, + []watchsync.LocalFavorite{{MediaItemID: "movie-1", Kind: historyimport.KindMovie, ProviderItemKey: "trakt:5"}}) + if err != nil { + t.Fatal(err) + } + if len(gotBody["movies"]) != 1 || gotBody["movies"][0]["ids"]["trakt"] != float64(5) { + t.Fatalf("payload = %#v, want the movie by its Trakt id", gotBody) + } + if !slices.Equal(result.Sent, []string{"movie-1", "trakt:5"}) { + t.Fatalf("sent = %#v", result.Sent) + } +} + +func TestTraktIDIndexMatchesAnySharedIDPerKind(t *testing.T) { + idx := traktIDIndex{} + idx.add(historyimport.KindSeries, traktIDs{TVDB: 81189, Slug: "breaking-bad"}) + idx.add(historyimport.KindMovie, traktIDs{}) + + for _, tc := range []struct { + name string + kind string + ids traktIDs + want bool + }{ + {"shared tvdb", historyimport.KindSeries, traktIDs{IMDb: "tt0903747", TVDB: 81189}, true}, + {"shared slug", historyimport.KindSeries, traktIDs{Slug: "breaking-bad"}, true}, + {"other kind", historyimport.KindEpisode, traktIDs{TVDB: 81189}, false}, + {"no shared id", historyimport.KindSeries, traktIDs{IMDb: "tt0903747"}, false}, + {"zero ids never match", historyimport.KindMovie, traktIDs{}, false}, + } { + if got := idx.matches(tc.kind, tc.ids); got != tc.want { + t.Errorf("%s: matches = %v, want %v", tc.name, got, tc.want) + } + } +} + func TestHistoryPayloadsIncludeTVDBOnlyMovieIDs(t *testing.T) { play := watchsync.LocalPlay{ HistoryID: "history-tvdb", diff --git a/internal/watchsync/service.go b/internal/watchsync/service.go index 763d36d364..da85f097bd 100644 --- a/internal/watchsync/service.go +++ b/internal/watchsync/service.go @@ -1793,14 +1793,20 @@ func providerItemKeyForRemoteFavorite(favorite RemoteFavorite) string { } } -func exportResultSentSet(result ExportResult) map[string]bool { - sent := make(map[string]bool, len(result.Sent)) - for _, value := range result.Sent { - if value != "" { - sent[value] = true - } - } - return sent +// exportItemOutcome reports whether a provider answered for one item as sent or +// as not found. Providers name items by media item id, provider key, or both. +// The id decides whenever the result names it, because items of different +// kinds can share a key such as tmdb:550; the key is only a fallback. +func exportItemOutcome(result ExportResult, mediaItemID, key string) (sent, notFound bool) { + if mediaItemID != "" { + _, failed := result.Failed[mediaItemID] + sent = containsString(result.Sent, mediaItemID) + notFound = containsString(result.NotFound, mediaItemID) + if sent || notFound || failed { + return sent, notFound + } + } + return containsString(result.Sent, key), containsString(result.NotFound, key) } func containsString(values []string, candidate string) bool { diff --git a/internal/watchsync/service_test.go b/internal/watchsync/service_test.go index d55c4882be..380923d4de 100644 --- a/internal/watchsync/service_test.go +++ b/internal/watchsync/service_test.go @@ -3774,3 +3774,25 @@ func (r *serviceFakeRepo) UpdateConnectionSettings(ctx context.Context, provider } return r.UpsertConnection(ctx, current) } + +func TestExportItemOutcomeDecidesByMediaItemIDBeforeKey(t *testing.T) { + // A movie and a show share the key tmdb:550; the provider reports the + // movie missing and the show sent. + result := ExportResult{ + Sent: []string{"show-1", "tmdb:550"}, + NotFound: []string{"movie-1", "tmdb:550"}, + } + if sent, notFound := exportItemOutcome(result, "movie-1", "tmdb:550"); sent || !notFound { + t.Fatalf("movie outcome = sent %v notFound %v, want not found", sent, notFound) + } + if sent, notFound := exportItemOutcome(result, "show-1", "tmdb:550"); !sent || notFound { + t.Fatalf("show outcome = sent %v notFound %v, want sent", sent, notFound) + } + // A result that names items only by key still works. + if sent, _ := exportItemOutcome(ExportResult{Sent: []string{"imdb:tt1"}}, "movie-2", "imdb:tt1"); !sent { + t.Fatal("a key-only result must still confirm the item") + } + if sent, _ := exportItemOutcome(ExportResult{Failed: map[string]string{"movie-3": "x"}, Sent: []string{"imdb:tt3"}}, "movie-3", "imdb:tt3"); sent { + t.Fatal("a failed media item must not be confirmed through its key") + } +} From 58c1636d6572321d9b1399f23368d4072c4346ad Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Wed, 23 Sep 2026 22:13:21 +0000 Subject: [PATCH 05/20] feat(watchsync): sync movie and series ratings with watch providers Silo profiles rate movies and series with 1-5 stars, but watch-provider sync never carried ratings. Add rating sync with separate import and export settings per connection; new connections get both on, existing connections start with both off. Each item is a three-way merge of the local rating, the provider rating (integers 1-10, converted to stars by rounding half up), and the last rating both sides agreed on, stored in watch_provider_rating_items. The side that changed wins; a conflict never deletes and otherwise the newer change wins. Missing items count as provider removals only in complete snapshots of their kind, when no read row shares one of their ids, and when a previous read confirmed the provider held the rating. Imports are compare-and-set against the observed local value, so concurrent edits and multi-node runs stay safe without a lock. Local rating changes are sent through a value-less event from the shared rating seam. Settings and run counters are exposed on /api/v2 and in the web settings page; the frozen v1 responses and settings update are unchanged apart from two additive capability flags. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/silo/main.go | 3 +- .../api/v2/fixtures/get_system_info_ok.json | 2 +- .../fixtures/watch_provider_api_key_ok.json | 6 +- .../watch_provider_connection_ok.json | 6 +- .../v2/fixtures/watch_provider_poll_ok.json | 6 +- .../fixtures/watch_provider_settings_ok.json | 4 +- .../v2/fixtures/watch_provider_sync_ok.json | 4 + .../v2/fixtures/watch_provider_update_ok.json | 4 +- contracts/api/v2/openapi.json | 58 +- docs/architecture/api-contract.md | 11 +- .../watch-provider-rating-sync.md | 115 +++ internal/api/handlers/ratings.go | 27 + internal/api/handlers/watch_providers.go | 4 + internal/api/router.go | 3 + internal/apiv2/request_lifecycle.go | 16 + internal/apiv2/request_lifecycle_test.go | 4 +- internal/catalog/orphan_cleanup.go | 4 + internal/catalog/orphan_cleanup_test.go | 1 + internal/catalog/ratings_repo.go | 65 ++ internal/catalog/ratings_repo_db_test.go | 88 ++ .../reattribute/rating_sync_move_db_test.go | 75 ++ internal/catalog/reattribute/reattribute.go | 30 + internal/metadata/provider_id_integrity.go | 10 + .../metadata/provider_id_integrity_test.go | 17 + internal/watchsync/connection_settings.go | 4 +- internal/watchsync/lists.go | 23 +- internal/watchsync/ratings.go | 883 +++++++++++++++++ internal/watchsync/ratings_db_test.go | 202 ++++ internal/watchsync/ratings_test.go | 916 ++++++++++++++++++ internal/watchsync/repository.go | 178 +++- internal/watchsync/service.go | 35 +- internal/watchsync/service_test.go | 83 ++ internal/watchsync/types.go | 102 ++ .../20260923214332_watch_provider_ratings.sql | 55 ++ web/src/api/v2/schema.ts | 24 + web/src/hooks/queries/watchProviders.ts | 13 +- .../pages/settings/WatchProvidersSettings.tsx | 52 +- 37 files changed, 3096 insertions(+), 37 deletions(-) create mode 100644 docs/architecture/watch-provider-rating-sync.md create mode 100644 internal/catalog/ratings_repo_db_test.go create mode 100644 internal/catalog/reattribute/rating_sync_move_db_test.go create mode 100644 internal/watchsync/ratings.go create mode 100644 internal/watchsync/ratings_db_test.go create mode 100644 internal/watchsync/ratings_test.go create mode 100644 migrations/sql/20260923214332_watch_provider_ratings.sql diff --git a/cmd/silo/main.go b/cmd/silo/main.go index c0ca50c972..3fdac09c0a 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -2268,7 +2268,8 @@ func main() { watchProviderService. WithMatcher(historyimport.NewMatcher(historyRepo)). WithWatchState(watchstate.NewService(userStoreProvider).WithStableIdentityResolver(historyIdentity)). - WithUserStoreProvider(userStoreProvider) + WithUserStoreProvider(userStoreProvider). + WithRatingStore(catalog.NewRatingsRepo(deps.DB), recommendations.NewRepo(deps.DB)) backgroundInit = append(backgroundInit, func(ctx context.Context) { if compatTerminalRecoveryReady != nil { select { diff --git a/contracts/api/v2/fixtures/get_system_info_ok.json b/contracts/api/v2/fixtures/get_system_info_ok.json index dad8906a93..ba015d5223 100644 --- a/contracts/api/v2/fixtures/get_system_info_ok.json +++ b/contracts/api/v2/fixtures/get_system_info_ok.json @@ -1,7 +1,7 @@ { "server_version": "unavailable", "api_major": 2, - "contract_digest": "cbf349c60d581f6f9b18aca98a8cf0b4a26d6933ef1699a6d2971faae9aac631", + "contract_digest": "a9b7ed3aa6fc7800e9f19253166c4018af5c0d31973e0eb812366391212d60c7", "links": { "openapi": "/api/v2/openapi.json", "capabilities": "/api/v2/capabilities", diff --git a/contracts/api/v2/fixtures/watch_provider_api_key_ok.json b/contracts/api/v2/fixtures/watch_provider_api_key_ok.json index 51add3b4ac..5ae5e96080 100644 --- a/contracts/api/v2/fixtures/watch_provider_api_key_ok.json +++ b/contracts/api/v2/fixtures/watch_provider_api_key_ok.json @@ -13,7 +13,9 @@ "export_watchlist": false, "remove_watchlist": false, "provides_watchlist_order": false, - "scrobble_playback": false + "scrobble_playback": false, + "import_ratings": false, + "export_ratings": false }, "auth_method": "", "connected": true, @@ -29,5 +31,7 @@ "sync_watchlist_removals_enabled": false, "sync_watchlist_order_enabled": false, "scrobble_enabled": false, + "import_ratings_enabled": false, + "export_ratings_enabled": false, "credentials_configured": false } diff --git a/contracts/api/v2/fixtures/watch_provider_connection_ok.json b/contracts/api/v2/fixtures/watch_provider_connection_ok.json index 51add3b4ac..5ae5e96080 100644 --- a/contracts/api/v2/fixtures/watch_provider_connection_ok.json +++ b/contracts/api/v2/fixtures/watch_provider_connection_ok.json @@ -13,7 +13,9 @@ "export_watchlist": false, "remove_watchlist": false, "provides_watchlist_order": false, - "scrobble_playback": false + "scrobble_playback": false, + "import_ratings": false, + "export_ratings": false }, "auth_method": "", "connected": true, @@ -29,5 +31,7 @@ "sync_watchlist_removals_enabled": false, "sync_watchlist_order_enabled": false, "scrobble_enabled": false, + "import_ratings_enabled": false, + "export_ratings_enabled": false, "credentials_configured": false } diff --git a/contracts/api/v2/fixtures/watch_provider_poll_ok.json b/contracts/api/v2/fixtures/watch_provider_poll_ok.json index 51add3b4ac..5ae5e96080 100644 --- a/contracts/api/v2/fixtures/watch_provider_poll_ok.json +++ b/contracts/api/v2/fixtures/watch_provider_poll_ok.json @@ -13,7 +13,9 @@ "export_watchlist": false, "remove_watchlist": false, "provides_watchlist_order": false, - "scrobble_playback": false + "scrobble_playback": false, + "import_ratings": false, + "export_ratings": false }, "auth_method": "", "connected": true, @@ -29,5 +31,7 @@ "sync_watchlist_removals_enabled": false, "sync_watchlist_order_enabled": false, "scrobble_enabled": false, + "import_ratings_enabled": false, + "export_ratings_enabled": false, "credentials_configured": false } diff --git a/contracts/api/v2/fixtures/watch_provider_settings_ok.json b/contracts/api/v2/fixtures/watch_provider_settings_ok.json index 38fe37c07b..35b55c648b 100644 --- a/contracts/api/v2/fixtures/watch_provider_settings_ok.json +++ b/contracts/api/v2/fixtures/watch_provider_settings_ok.json @@ -10,5 +10,7 @@ "export_watchlist_enabled": false, "sync_watchlist_removals_enabled": false, "sync_watchlist_order_enabled": false, - "scrobble_enabled": false + "scrobble_enabled": false, + "import_ratings_enabled": false, + "export_ratings_enabled": false } diff --git a/contracts/api/v2/fixtures/watch_provider_sync_ok.json b/contracts/api/v2/fixtures/watch_provider_sync_ok.json index 85a1f9535a..50173510c5 100644 --- a/contracts/api/v2/fixtures/watch_provider_sync_ok.json +++ b/contracts/api/v2/fixtures/watch_provider_sync_ok.json @@ -21,6 +21,10 @@ "outbound_watchlist_found": 0, "outbound_watchlist_sent": 0, "watchlist_removals_sent": 0, + "inbound_ratings_found": 0, + "inbound_ratings_imported": 0, + "outbound_ratings_found": 0, + "outbound_ratings_sent": 0, "started_at": "2026-01-02T03:04:05.678Z", "created_at": "2026-01-02T03:04:05.678Z" }, diff --git a/contracts/api/v2/fixtures/watch_provider_update_ok.json b/contracts/api/v2/fixtures/watch_provider_update_ok.json index 38fe37c07b..35b55c648b 100644 --- a/contracts/api/v2/fixtures/watch_provider_update_ok.json +++ b/contracts/api/v2/fixtures/watch_provider_update_ok.json @@ -10,5 +10,7 @@ "export_watchlist_enabled": false, "sync_watchlist_removals_enabled": false, "sync_watchlist_order_enabled": false, - "scrobble_enabled": false + "scrobble_enabled": false, + "import_ratings_enabled": false, + "export_ratings_enabled": false } diff --git a/contracts/api/v2/openapi.json b/contracts/api/v2/openapi.json index 3435fd7e59..730dce7a91 100644 --- a/contracts/api/v2/openapi.json +++ b/contracts/api/v2/openapi.json @@ -19798,6 +19798,9 @@ "export_favorites_enabled": { "type": "boolean" }, + "export_ratings_enabled": { + "type": "boolean" + }, "export_unwatched_enabled": { "type": "boolean" }, @@ -19813,6 +19816,9 @@ "import_progress_enabled": { "type": "boolean" }, + "import_ratings_enabled": { + "type": "boolean" + }, "import_watched_enabled": { "type": "boolean" }, @@ -40069,6 +40075,9 @@ "export_favorites": { "type": "boolean" }, + "export_ratings": { + "type": "boolean" + }, "export_unwatched": { "type": "boolean" }, @@ -40084,6 +40093,9 @@ "import_progress": { "type": "boolean" }, + "import_ratings": { + "type": "boolean" + }, "import_watched": { "type": "boolean" }, @@ -40115,7 +40127,9 @@ "export_watchlist", "remove_watchlist", "provides_watchlist_order", - "scrobble_playback" + "scrobble_playback", + "import_ratings", + "export_ratings" ], "type": "object" }, @@ -40146,6 +40160,9 @@ "export_favorites_enabled": { "type": "boolean" }, + "export_ratings_enabled": { + "type": "boolean" + }, "export_unwatched_enabled": { "type": "boolean" }, @@ -40161,6 +40178,9 @@ "import_progress_enabled": { "type": "boolean" }, + "import_ratings_enabled": { + "type": "boolean" + }, "import_watched_enabled": { "type": "boolean" }, @@ -40249,6 +40269,8 @@ "sync_watchlist_removals_enabled", "sync_watchlist_order_enabled", "scrobble_enabled", + "import_ratings_enabled", + "export_ratings_enabled", "credentials_configured" ], "type": "object" @@ -40314,6 +40336,10 @@ "export_favorites_enabled": { "type": "boolean" }, + "export_ratings_enabled": { + "description": "Send the profile's star ratings to the provider (stars times two) and clear removed ones.", + "type": "boolean" + }, "export_unwatched_enabled": { "type": "boolean" }, @@ -40329,6 +40355,10 @@ "import_progress_enabled": { "type": "boolean" }, + "import_ratings_enabled": { + "description": "Import the provider's movie and series ratings as stars (1-2 is 1 star, 9-10 is 5 stars).", + "type": "boolean" + }, "import_watched_enabled": { "type": "boolean" }, @@ -40360,7 +40390,9 @@ "export_watchlist_enabled", "sync_watchlist_removals_enabled", "sync_watchlist_order_enabled", - "scrobble_enabled" + "scrobble_enabled", + "import_ratings_enabled", + "export_ratings_enabled" ], "type": "object" }, @@ -40463,6 +40495,14 @@ "format": "int64", "type": "integer" }, + "inbound_ratings_found": { + "format": "int64", + "type": "integer" + }, + "inbound_ratings_imported": { + "format": "int64", + "type": "integer" + }, "inbound_watched_found": { "format": "int64", "type": "integer" @@ -40491,6 +40531,16 @@ "format": "int64", "type": "integer" }, + "outbound_ratings_found": { + "description": "Movie and series ratings the profile holds.", + "format": "int64", + "type": "integer" + }, + "outbound_ratings_sent": { + "description": "Ratings set or cleared on the provider.", + "format": "int64", + "type": "integer" + }, "outbound_sent": { "format": "int64", "type": "integer" @@ -40549,6 +40599,10 @@ "outbound_watchlist_found", "outbound_watchlist_sent", "watchlist_removals_sent", + "inbound_ratings_found", + "inbound_ratings_imported", + "outbound_ratings_found", + "outbound_ratings_sent", "started_at", "created_at" ], diff --git a/docs/architecture/api-contract.md b/docs/architecture/api-contract.md index e0ccc36b71..7ae7c745a3 100644 --- a/docs/architecture/api-contract.md +++ b/docs/architecture/api-contract.md @@ -1270,8 +1270,8 @@ paged with `limit` plus an opaque cursor whose cards are the shared `CatalogItem (`{item_id, added_at}`, or `{item_id, rating, rated_at}`) or `404`, a bodiless `PUT` add (ratings take `{rating}`) answering `204`, and a `DELETE` answering `204` whether or not the entry existed. All six mutations are `non_retryable`: the shared seams dispatch provider -list events and recommendation refresh without change gating, and rating updates replace -`rated_at` even when unchanged. Their ledger `DEFECT` notes retain the durable-dispatch work +list and rating events and recommendation refresh without change gating, and rating updates +replace `rated_at` even when unchanged. Their ledger `DEFECT` notes retain the durable-dispatch work required before clients can retry automatically. The mutations are not demo-restricted: v1's demo guard only blocks its listed routes, so these writes pass in demo mode and v2 matches. The three lists page by keyset, not offset: the cursor is the (`added_at`, `item_id`) — for ratings @@ -1858,6 +1858,13 @@ job-dispatch mechanism. Cooldown problems retain their Retry-After header and th web settings page displays the delay. No watch-provider consumers were found in the Apple and Android source inventory. +Rating sync settings (`import_ratings_enabled`, `export_ratings_enabled`) and the +rating run counters exist only on v2. The frozen v1 connection and run responses omit +them, and a v1 settings update ignores them. The `import_ratings` and `export_ratings` +capability flags are the exception: v1 and v2 share the capabilities object, so v1 +responses gain those two additive fields. See +[watch-provider-rating-sync.md](watch-provider-rating-sync.md) for the sync rules. + ### Webhook connection management Eight v2 account operations manage webhook connections, secret rotation, external diff --git a/docs/architecture/watch-provider-rating-sync.md b/docs/architecture/watch-provider-rating-sync.md new file mode 100644 index 0000000000..05b4cf5949 --- /dev/null +++ b/docs/architecture/watch-provider-rating-sync.md @@ -0,0 +1,115 @@ +# Watch-provider rating sync + +A profile's movie and series ratings sync with a connected watch provider in either +direction or both. Each connection has two settings: `import_ratings_enabled` brings +provider ratings into Silo, and `export_ratings_enabled` sends Silo ratings to the +provider and clears ones the profile removes. New connections start with both on; +connections that existed before rating sync start with both off. A provider takes part +only when it advertises the matching `import_ratings` or `export_ratings` capability. +Season and episode ratings are not synced, because Silo rates only movies and series. + +## Scale + +Silo stores integer stars from 1 to 5. Providers and the plugin contract use integers +from 1 to 10. Import rounds half up (`stars = (rating + 1) / 2`, so 7 and 8 are both +4 stars) and export doubles (`rating = stars × 2`). Every sync decision compares stars, +so a provider change within one star, such as 7 to 8, is not a change, and Silo never +overwrites a provider's 7 with the 8 its 4 stars map to. + +## Agreed rating + +`watch_provider_rating_items` stores, per connection and item, the last rating both +sides agreed on, in stars. No row means both sides agreed the item is unrated, so the +first sync is a union of both sides. Each item is a three-way merge of the local +rating, the provider rating, and the agreed rating: + +- Both sides equal: record the agreement. +- Only one side moved from the agreed rating: that side wins, including removals. +- Both moved to different values: a rating beats a removal, so a conflict never + deletes; otherwise the newer change wins, and ties or unknown provider times go to + Silo. + +A decision the connection's direction settings do not allow is skipped without +recording agreement, so it is reconsidered when the direction is turned on. + +`remote_seen` records that a provider read confirmed the agreed rating. A sent rating +is agreed but not seen until a later read returns it, and it is agreed only if the local +rating still has the value that was sent. A sent removal keeps its agreed row until a +read confirms the title is unrated; if the provider still reports a rating (for example +a second entry for the same title), the removal is sent again instead of the rating +being imported back. + +Rows are scoped to the provider account they were agreed with. A connection that moves +to another account ignores the old rows, and so does a sync still running for the old +account, so no stale agreement can read as a removal. + +The row's `provider_item_key` is the provider's own key for the title once a read has +returned one, and Silo's key (`imdb:`, `tmdb:`, or `tvdb:`) before that. Rating writes +carry the same key, so a provider can name the title in later tombstones and writes. + +## Provider reads + +A read returns rows plus `SnapshotKinds`, the kinds for which the rows are the +provider's complete set. Rows match the catalog by TMDB, IMDb, or TVDB identifier as +[history-import-execution.md](history-import-execution.md) describes; several rows for +one item resolve to the newest. An explicit tombstone removes the rating whose agreed +row carries the same provider key. + +An item missing from a read counts as removed on the provider only when all hold: + +- its kind is in `SnapshotKinds`; +- no row in the read shares one of its identifiers within the same kind, so a row the + matcher could not place, or a row with an out-of-range rating, never looks like a + removal; +- its agreed row is `remote_seen`. + +A missing item whose rating was sent but never seen is sent again instead of removed +locally. A read row with no IMDb, TMDB, or TVDB id could be any title, so its kind is +dropped from `SnapshotKinds`. A complete snapshot that returns nothing for a kind while +Silo holds two or more confirmed ratings of that kind is treated as a failed read: +removals for that kind are skipped and the run records a warning. Any other missing item +keeps its agreed value. + +Silo reads the profile's ratings in one query, and a rated item the sync skips (no longer +in the catalog, or without external ids) never counts as a local removal. + +## Concurrency + +Scheduled syncs run on every API node and the per-connection lock is local to a +process, so correctness does not depend on locking. Imports use compare-and-set +writes (`RatingsRepo.SetIfUnchanged` and `DeleteIfUnchanged`) against the local value +the sync read, so a concurrent user edit wins and is reconsidered next run. Imported +ratings keep the provider's rating time. Provider writes are idempotent desired-state +writes. A stale agreed row heals on the next run, because equal values on both sides +simply record agreement again. Read cursors are skip hints only. They are saved in +send-only mode too, so a cursor-gated provider does not re-read everything each run, and +turning import on forces one full read so no change skipped while sending only is lost. + +## Local changes + +`RatingsHandler.SetRating` and `DeleteRating`, the seam shared by v1 and v2, dispatch +a `LocalRatingEvent` that names the changed items but carries no values. The handler +re-reads the current rating and sends a new or changed rating right away. A removal waits +for the next scheduled merge, which can see whether the provider changed the rating in +the meantime; a rating beats a removal. Imports write through the ratings repository +directly and never dispatch events, so an imported rating is not echoed back. Imports +mark the profile's recommendations stale once per run. + +## Provider-specific rules + +A provider that rates only some kinds implements `RatingKindFilter`; items of other +kinds are left out of its sync entirely, so they are neither sent nor read as removed. + +A provider that records a rated title as watched implements `RatingExportWatchGate`. +Silo then sends a new rating of that kind only once the profile has a completed play of +the title; until then the rating stays pending and each run records a warning. A title +the provider already holds a rating for is not held back, and neither are removals. + +## Identity changes + +Re-binding a connection to a different provider account clears its agreed ratings and +drops its rating read cursors, whose keys contain `.ratings`. Whenever ratings move to +another media item (a duplicate merge or a reattribution), the agreed rows move with them; +the destination's row wins a collision. A moved row is unconfirmed and forgets its +provider key: for the same title the next read confirms it again, and for a different +title the rating is sent for that title. A move therefore never reads as a removal. diff --git a/internal/api/handlers/ratings.go b/internal/api/handlers/ratings.go index caf722022e..0543bc4284 100644 --- a/internal/api/handlers/ratings.go +++ b/internal/api/handlers/ratings.go @@ -9,6 +9,7 @@ import ( apimw "github.com/Silo-Server/silo-server/internal/api/middleware" "github.com/Silo-Server/silo-server/internal/catalog" + "github.com/Silo-Server/silo-server/internal/watchsync" ) // ratingsRepository defines the data access interface for user ratings. @@ -20,12 +21,19 @@ type ratingsRepository interface { ListPage(ctx context.Context, userID int, profileID string, after *catalog.RatingKey, limit int) ([]catalog.UserRating, error) } +// LocalRatingEventDispatcher sends a profile's rating changes to its watch +// providers. +type LocalRatingEventDispatcher interface { + HandleLocalRatingEvent(ctx context.Context, event watchsync.LocalRatingEvent) error +} + // RatingsHandler handles user rating operations. type RatingsHandler struct { ratingsRepo ratingsRepository itemRepo personalDataItemRepository profileStaler ProfileStaler profileRefreshRequester ProfileRefreshRequester + ratingDispatcher LocalRatingEventDispatcher } // NewRatingsHandler creates a new RatingsHandler. @@ -43,10 +51,27 @@ func (h *RatingsHandler) SetProfileRefreshRequester(requester ProfileRefreshRequ h.profileRefreshRequester = requester } +// SetLocalRatingEventDispatcher configures where rating changes are sent for +// watch-provider sync. +func (h *RatingsHandler) SetLocalRatingEventDispatcher(dispatcher LocalRatingEventDispatcher) { + h.ratingDispatcher = dispatcher +} + func (h *RatingsHandler) markStale(ctx context.Context, userID int, profileID string) { triggerProfileRefresh(ctx, h.profileStaler, h.profileRefreshRequester, userID, profileID) } +func (h *RatingsHandler) dispatchRatingChange(ctx context.Context, userID int, profileID, itemID string) { + if h.ratingDispatcher == nil { + return + } + _ = h.ratingDispatcher.HandleLocalRatingEvent(ctx, watchsync.LocalRatingEvent{ + UserID: userID, + ProfileID: profileID, + MediaItemIDs: []string{itemID}, + }) +} + // --- Response types --- type ratingResponse struct { @@ -111,6 +136,7 @@ func (h *RatingsHandler) SetRating(ctx context.Context, userID int, profileID, i return apiError(http.StatusInternalServerError, "internal_error", "Failed to set rating") } h.markStale(ctx, userID, profileID) + h.dispatchRatingChange(ctx, userID, profileID, itemID) return nil } @@ -140,6 +166,7 @@ func (h *RatingsHandler) DeleteRating(ctx context.Context, userID int, profileID return apiError(http.StatusInternalServerError, "internal_error", "Failed to delete rating") } h.markStale(ctx, userID, profileID) + h.dispatchRatingChange(ctx, userID, profileID, itemID) return nil } diff --git a/internal/api/handlers/watch_providers.go b/internal/api/handlers/watch_providers.go index c9b7e212ad..55007191c1 100644 --- a/internal/api/handlers/watch_providers.go +++ b/internal/api/handlers/watch_providers.go @@ -77,6 +77,10 @@ func (h *WatchProviderHandler) HandleUpdateConnection(w http.ResponseWriter, r * writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body") return } + // Rating sync settings exist only on /api/v2; the frozen v1 contract keeps + // its original settings. + update.ImportRatingsEnabled = nil + update.ExportRatingsEnabled = nil status, err := h.service.UpdateConnection(r.Context(), userID, profileID, provider, update) if err != nil { writeError(w, http.StatusBadRequest, "watch_provider_error", err.Error()) diff --git a/internal/api/router.go b/internal/api/router.go index 4488c29dfc..485bea4e50 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -1062,6 +1062,9 @@ func newChiRouter(deps Dependencies) chi.Router { var recsRepoForStale *recommendations.Repo if ratingsRepo != nil && itemRepo != nil { ratingsHandler = handlers.NewRatingsHandler(ratingsRepo, itemRepo) + if dispatcher, ok := deps.WatchProviderService.(handlers.LocalRatingEventDispatcher); ok { + ratingsHandler.SetLocalRatingEventDispatcher(dispatcher) + } if deps.DB != nil { recsRepoForStale = recommendations.NewRepo(deps.DB) ratingsHandler.SetProfileStaler(recsRepoForStale) diff --git a/internal/apiv2/request_lifecycle.go b/internal/apiv2/request_lifecycle.go index 774aff7da5..3261b6abc9 100644 --- a/internal/apiv2/request_lifecycle.go +++ b/internal/apiv2/request_lifecycle.go @@ -96,6 +96,8 @@ type WatchProviderSettings struct { SyncWatchlistRemovalsEnabled bool `json:"sync_watchlist_removals_enabled"` SyncWatchlistOrderEnabled bool `json:"sync_watchlist_order_enabled"` ScrobbleEnabled bool `json:"scrobble_enabled"` + ImportRatingsEnabled bool `json:"import_ratings_enabled" doc:"Import the provider's movie and series ratings as stars (1-2 is 1 star, 9-10 is 5 stars)."` + ExportRatingsEnabled bool `json:"export_ratings_enabled" doc:"Send the profile's star ratings to the provider (stars times two) and clear removed ones."` } func watchProviderSettingsOf(status watchsync.ConnectionStatus) WatchProviderSettings { @@ -112,6 +114,8 @@ func watchProviderSettingsOf(status watchsync.ConnectionStatus) WatchProviderSet SyncWatchlistRemovalsEnabled: status.SyncWatchlistRemovalsEnabled, SyncWatchlistOrderEnabled: status.SyncWatchlistOrderEnabled, ScrobbleEnabled: status.ScrobbleEnabled, + ImportRatingsEnabled: status.ImportRatingsEnabled, + ExportRatingsEnabled: status.ExportRatingsEnabled, } } @@ -401,6 +405,8 @@ type WatchProviderConnection struct { SyncWatchlistRemovalsEnabled bool `json:"sync_watchlist_removals_enabled"` SyncWatchlistOrderEnabled bool `json:"sync_watchlist_order_enabled"` ScrobbleEnabled bool `json:"scrobble_enabled"` + ImportRatingsEnabled bool `json:"import_ratings_enabled"` + ExportRatingsEnabled bool `json:"export_ratings_enabled"` CredentialsConfigured bool `json:"credentials_configured"` ConnectionConfigSchema []AdminPluginConfigSchema `json:"connection_config_schema,omitempty"` LastInboundSyncAt *Instant `json:"last_inbound_sync_at,omitempty"` @@ -436,6 +442,8 @@ func watchProviderConnectionOf(s watchsync.ConnectionStatus) (WatchProviderConne SyncWatchlistRemovalsEnabled: s.SyncWatchlistRemovalsEnabled, SyncWatchlistOrderEnabled: s.SyncWatchlistOrderEnabled, ScrobbleEnabled: s.ScrobbleEnabled, + ImportRatingsEnabled: s.ImportRatingsEnabled, + ExportRatingsEnabled: s.ExportRatingsEnabled, CredentialsConfigured: s.CredentialsConfigured, ConnectionConfigSchema: schemas, LastInboundSyncAt: instantPtr(s.LastInboundSyncAt), @@ -470,6 +478,10 @@ type WatchProviderSyncRun struct { OutboundWatchlistFound int `json:"outbound_watchlist_found"` OutboundWatchlistSent int `json:"outbound_watchlist_sent"` WatchlistRemovalsSent int `json:"watchlist_removals_sent"` + InboundRatingsFound int `json:"inbound_ratings_found"` + InboundRatingsImported int `json:"inbound_ratings_imported"` + OutboundRatingsFound int `json:"outbound_ratings_found" doc:"Movie and series ratings the profile holds."` + OutboundRatingsSent int `json:"outbound_ratings_sent" doc:"Ratings set or cleared on the provider."` Warning string `json:"warning,omitempty"` Error string `json:"error,omitempty"` StartedAt Instant `json:"started_at"` @@ -500,6 +512,10 @@ func watchProviderSyncRunOf(s watchsync.SyncRun) WatchProviderSyncRun { OutboundWatchlistFound: s.OutboundWatchlistFound, OutboundWatchlistSent: s.OutboundWatchlistSent, WatchlistRemovalsSent: s.WatchlistRemovalsSent, + InboundRatingsFound: s.InboundRatingsFound, + InboundRatingsImported: s.InboundRatingsImported, + OutboundRatingsFound: s.OutboundRatingsFound, + OutboundRatingsSent: s.OutboundRatingsSent, Warning: s.Warning, Error: s.Error, StartedAt: NewInstant(s.StartedAt), diff --git a/internal/apiv2/request_lifecycle_test.go b/internal/apiv2/request_lifecycle_test.go index e215c305b2..8b11bf49e9 100644 --- a/internal/apiv2/request_lifecycle_test.go +++ b/internal/apiv2/request_lifecycle_test.go @@ -281,7 +281,7 @@ func TestWatchProviderMetadataHasNoSettingsValidator(t *testing.T) { if err := json.Unmarshal(after.Body.Bytes(), &fields); err != nil { t.Fatal(err) } - if len(fields) != 12 { + if len(fields) != 14 { t.Fatalf("settings fields=%v", fields) } for key, value := range fields { @@ -299,7 +299,7 @@ func TestWatchProviderMetadataHasNoSettingsValidator(t *testing.T) { if err := json.Unmarshal(patched.Body.Bytes(), &fields); err != nil { t.Fatal(err) } - if len(fields) != 12 { + if len(fields) != 14 { t.Fatalf("PATCH returned metadata: %v", fields) } } diff --git a/internal/catalog/orphan_cleanup.go b/internal/catalog/orphan_cleanup.go index 937d5fd1e7..2abb24c187 100644 --- a/internal/catalog/orphan_cleanup.go +++ b/internal/catalog/orphan_cleanup.go @@ -110,6 +110,10 @@ const orphanedMediaItemSafetyConditions = `NOT EXISTS ( SELECT 1 FROM public.watch_provider_list_items wpli WHERE wpli.media_item_id = mi.content_id ) + AND NOT EXISTS ( + SELECT 1 FROM public.watch_provider_rating_items wpri + WHERE wpri.media_item_id = mi.content_id + ) AND NOT EXISTS ( SELECT 1 FROM public.watch_provider_history_exports wphe WHERE wphe.media_item_id = mi.content_id diff --git a/internal/catalog/orphan_cleanup_test.go b/internal/catalog/orphan_cleanup_test.go index 03af2ab35b..00c8597164 100644 --- a/internal/catalog/orphan_cleanup_test.go +++ b/internal/catalog/orphan_cleanup_test.go @@ -25,6 +25,7 @@ func TestOrphanedProvisionalPredicatePreservesDurableMediaItemReferences(t *test "public.user_series_playback_preferences uspp WHERE uspp.series_id = mi.content_id", "public.user_subtitle_preferences usp WHERE usp.series_id = mi.content_id", "public.watch_provider_list_items wpli WHERE wpli.media_item_id = mi.content_id", + "public.watch_provider_rating_items wpri WHERE wpri.media_item_id = mi.content_id", } { if !strings.Contains(predicate, normalizePredicateSQL(want)) { t.Fatalf("cleanup predicate missing durable reference guard %q", want) diff --git a/internal/catalog/ratings_repo.go b/internal/catalog/ratings_repo.go index 7befafde2e..88255a1c47 100644 --- a/internal/catalog/ratings_repo.go +++ b/internal/catalog/ratings_repo.go @@ -53,6 +53,45 @@ func (r *RatingsRepo) Set(ctx context.Context, userID int, profileID, mediaItemI return nil } +// SetIfUnchanged writes a rating observed elsewhere (a watch provider) only if +// the item's rating still equals expected, where 0 means unrated. ratedAt is +// the time the rating was made. It reports whether the write applied; false +// means a concurrent change won, and the caller should re-read before retrying. +func (r *RatingsRepo) SetIfUnchanged(ctx context.Context, userID int, profileID, mediaItemID string, expected, rating int, ratedAt time.Time) (bool, error) { + var query string + args := []any{userID, profileID, mediaItemID, rating, ratedAt} + if expected == 0 { + query = ` + INSERT INTO user_ratings (user_id, profile_id, media_item_id, rating, rated_at) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (user_id, profile_id, media_item_id) DO NOTHING` + } else { + query = ` + UPDATE user_ratings SET rating = $4, rated_at = $5 + WHERE user_id = $1 AND profile_id = $2 AND media_item_id = $3 AND rating = $6` + args = append(args, expected) + } + tag, err := r.pool.Exec(ctx, query, args...) + if err != nil { + return false, fmt.Errorf("set rating if unchanged: %w", err) + } + return tag.RowsAffected() == 1, nil +} + +// DeleteIfUnchanged removes a rating only if it still equals expected. It +// reports whether a row was deleted. +func (r *RatingsRepo) DeleteIfUnchanged(ctx context.Context, userID int, profileID, mediaItemID string, expected int) (bool, error) { + tag, err := r.pool.Exec(ctx, ` + DELETE FROM user_ratings + WHERE user_id = $1 AND profile_id = $2 AND media_item_id = $3 AND rating = $4`, + userID, profileID, mediaItemID, expected, + ) + if err != nil { + return false, fmt.Errorf("delete rating if unchanged: %w", err) + } + return tag.RowsAffected() == 1, nil +} + // Get retrieves a user's rating for an item. Returns nil if not rated. func (r *RatingsRepo) Get(ctx context.Context, userID int, profileID, mediaItemID string) (*UserRating, error) { var ur UserRating @@ -147,6 +186,32 @@ func (r *RatingsRepo) List(ctx context.Context, userID int, profileID string, li return ratings, rows.Err() } +// ListAll returns every rating of a profile in one query, so the caller reads a +// single consistent snapshot. +func (r *RatingsRepo) ListAll(ctx context.Context, userID int, profileID string) ([]UserRating, error) { + rows, err := r.pool.Query(ctx, ` + SELECT user_id, profile_id, media_item_id, rating, rated_at + FROM user_ratings + WHERE user_id = $1 AND profile_id = $2 + ORDER BY media_item_id`, + userID, profileID, + ) + if err != nil { + return nil, fmt.Errorf("list all ratings: %w", err) + } + defer rows.Close() + + var ratings []UserRating + for rows.Next() { + var ur UserRating + if err := rows.Scan(&ur.UserID, &ur.ProfileID, &ur.MediaItemID, &ur.Rating, &ur.RatedAt); err != nil { + return nil, fmt.Errorf("scan rating: %w", err) + } + ratings = append(ratings, ur) + } + return ratings, rows.Err() +} + // ListForItems returns ratings for a specific set of item IDs (used by recommendation filtering). func (r *RatingsRepo) ListForItems(ctx context.Context, userID int, profileID string, itemIDs []string) (map[string]int, error) { rows, err := r.pool.Query(ctx, ` diff --git a/internal/catalog/ratings_repo_db_test.go b/internal/catalog/ratings_repo_db_test.go new file mode 100644 index 0000000000..d83a9812cb --- /dev/null +++ b/internal/catalog/ratings_repo_db_test.go @@ -0,0 +1,88 @@ +package catalog + +import ( + "fmt" + "os" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// TestRatingsRepoCompareAndSetDB covers the writes watch-provider rating sync +// uses: they apply only while the rating still has the value the sync read. +func TestRatingsRepoCompareAndSetDB(t *testing.T) { + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + ctx := t.Context() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + var userID int + if err := pool.QueryRow(ctx, "INSERT INTO users(username,role) VALUES($1,'user') RETURNING id", + fmt.Sprintf("ratings-cas-%d", time.Now().UnixNano())).Scan(&userID); err != nil { + t.Fatal(err) + } + defer func() { _, _ = pool.Exec(ctx, "DELETE FROM users WHERE id=$1", userID) }() + repo := NewRatingsRepo(pool) + const profile, item = "cas-profile", "cas-movie" + ratedAt := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + + applied, err := repo.SetIfUnchanged(ctx, userID, profile, item, 0, 4, ratedAt) + if err != nil || !applied { + t.Fatalf("insert when unrated: applied=%v err=%v", applied, err) + } + got, err := repo.Get(ctx, userID, profile, item) + if err != nil || got == nil || got.Rating != 4 || !got.RatedAt.Equal(ratedAt) { + t.Fatalf("stored rating = %#v err=%v, want 4 at the provider time", got, err) + } + if applied, _ := repo.SetIfUnchanged(ctx, userID, profile, item, 0, 5, ratedAt); applied { + t.Fatal("expecting unrated must not overwrite an existing rating") + } + if applied, _ := repo.SetIfUnchanged(ctx, userID, profile, item, 3, 5, ratedAt); applied { + t.Fatal("a stale expected value must not apply") + } + if applied, _ := repo.DeleteIfUnchanged(ctx, userID, profile, item, 3); applied { + t.Fatal("a stale expected value must not delete") + } + + // Two writers expecting the same value: exactly one wins. + var wg sync.WaitGroup + results := make(chan bool, 2) + for _, rating := range []int{1, 2} { + wg.Go(func() { + ok, err := repo.SetIfUnchanged(ctx, userID, profile, item, 4, rating, ratedAt) + if err != nil { + t.Error(err) + } + results <- ok + }) + } + wg.Wait() + close(results) + wins := 0 + for ok := range results { + if ok { + wins++ + } + } + if wins != 1 { + t.Fatalf("concurrent compare-and-set winners = %d, want 1", wins) + } + + current, err := repo.Get(ctx, userID, profile, item) + if err != nil || current == nil { + t.Fatalf("current = %#v err=%v", current, err) + } + if applied, err := repo.DeleteIfUnchanged(ctx, userID, profile, item, current.Rating); err != nil || !applied { + t.Fatalf("delete with the current value: applied=%v err=%v", applied, err) + } + if gone, _ := repo.Get(ctx, userID, profile, item); gone != nil { + t.Fatalf("rating survived delete: %#v", gone) + } +} diff --git a/internal/catalog/reattribute/rating_sync_move_db_test.go b/internal/catalog/reattribute/rating_sync_move_db_test.go new file mode 100644 index 0000000000..7c6b89b97a --- /dev/null +++ b/internal/catalog/reattribute/rating_sync_move_db_test.go @@ -0,0 +1,75 @@ +package reattribute + +import ( + "context" + "fmt" + "testing" +) + +// TestMoveRatingSyncPairsDB checks that watch-provider agreed ratings follow a +// reattribution unconfirmed and without their provider key, and that the +// destination's own row wins a collision. +func TestMoveRatingSyncPairsDB(t *testing.T) { + env := newTestEnv(t) + ctx := context.Background() + if _, err := env.pool.Exec(ctx, `INSERT INTO user_profiles (user_id, id, name) VALUES ($1, $2, 'Reattr')`, env.userID, env.profileID); err != nil { + t.Fatalf("seed profile: %v", err) + } + var connectionID string + if err := env.pool.QueryRow(ctx, ` + INSERT INTO watch_provider_connections (provider, user_id, profile_id) + VALUES ('reattr-test', $1, $2) RETURNING id::text + `, env.userID, env.profileID).Scan(&connectionID); err != nil { + t.Fatalf("seed connection: %v", err) + } + t.Cleanup(func() { + _, _ = env.pool.Exec(ctx, `DELETE FROM watch_provider_connections WHERE id = $1::uuid`, connectionID) + }) + from := fmt.Sprintf("reattr-from-%d", env.suffix) + to := fmt.Sprintf("reattr-to-%d", env.suffix) + collidingFrom := fmt.Sprintf("reattr-cfrom-%d", env.suffix) + collidingTo := fmt.Sprintf("reattr-cto-%d", env.suffix) + if _, err := env.pool.Exec(ctx, ` + INSERT INTO watch_provider_rating_items (connection_id, media_item_id, kind, provider_item_key, synced_rating, remote_seen) + VALUES ($1::uuid, $2, 'movie', 'imdb:tt1', 4, true), + ($1::uuid, $3, 'movie', 'imdb:tt2', 2, true), + ($1::uuid, $4, 'movie', 'imdb:tt3', 5, true) + `, connectionID, from, collidingFrom, collidingTo); err != nil { + t.Fatalf("seed agreed ratings: %v", err) + } + + tx, err := env.pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback(ctx) }() + if err := moveRatingSyncPairs(ctx, tx, []string{from, collidingFrom}, []string{to, collidingTo}); err != nil { + t.Fatal(err) + } + + var rating int + var seen bool + var key string + if err := tx.QueryRow(ctx, ` + SELECT synced_rating, remote_seen, provider_item_key FROM watch_provider_rating_items + WHERE connection_id = $1::uuid AND media_item_id = $2 + `, connectionID, to).Scan(&rating, &seen, &key); err != nil { + t.Fatalf("moved row: %v", err) + } + if rating != 4 || seen || key != "" { + t.Fatalf("moved row = rating %d seen %v key %q, want 4, unconfirmed, no key", rating, seen, key) + } + if err := tx.QueryRow(ctx, ` + SELECT synced_rating FROM watch_provider_rating_items + WHERE connection_id = $1::uuid AND media_item_id = $2 + `, connectionID, collidingTo).Scan(&rating); err != nil || rating != 5 { + t.Fatalf("destination row = %d (%v), want its own 5 to win", rating, err) + } + var left int + if err := tx.QueryRow(ctx, ` + SELECT count(*) FROM watch_provider_rating_items + WHERE connection_id = $1::uuid AND media_item_id = ANY($2) + `, connectionID, []string{from, collidingFrom}).Scan(&left); err != nil || left != 0 { + t.Fatalf("source rows left = %d (%v), want 0", left, err) + } +} diff --git a/internal/catalog/reattribute/reattribute.go b/internal/catalog/reattribute/reattribute.go index 305b71c4ae..d7dd4237e7 100644 --- a/internal/catalog/reattribute/reattribute.go +++ b/internal/catalog/reattribute/reattribute.go @@ -258,6 +258,33 @@ func movePairs(ctx context.Context, tx pgx.Tx, pairs []IDPair, report *Report) ( } report.IntentMoved += movedRows } + return moveRatingSyncPairs(ctx, tx, fromIDs, toIDs) +} + +// moveRatingSyncPairs moves watch-provider agreed ratings along with the +// user_ratings rows they describe; the destination's row wins a collision. A +// moved row is unconfirmed and forgets its provider key: when the destination +// is the same title the next sync reads the rating again and confirms it, and +// when it is a different title the rating is sent for that title. Either way +// the move never reads as a provider removal. The rows are not counted in +// IntentMoved, which reports user data. +func moveRatingSyncPairs(ctx context.Context, tx pgx.Tx, fromIDs, toIDs []string) error { + if _, err := tx.Exec(ctx, ` + DELETE FROM watch_provider_rating_items src + USING `+pairsCTE+`, watch_provider_rating_items dest + WHERE src.media_item_id = p.from_id + AND dest.media_item_id = p.to_id + AND dest.connection_id = src.connection_id + `, fromIDs, toIDs); err != nil { + return fmt.Errorf("reattribute: watch_provider_rating_items dedupe: %w", err) + } + if _, err := tx.Exec(ctx, ` + UPDATE watch_provider_rating_items t + SET media_item_id = p.to_id, remote_seen = false, provider_item_key = '', updated_at = now() + FROM `+pairsCTE+` WHERE t.media_item_id = p.from_id + `, fromIDs, toIDs); err != nil { + return fmt.Errorf("reattribute: watch_provider_rating_items move: %w", err) + } return nil } @@ -398,6 +425,9 @@ func moveFileSubset(ctx context.Context, tx pgx.Tx, opts Options, report *Report } report.IntentMoved += movedRows } + if err := moveRatingSyncPairs(ctx, tx, []string{opts.FromContentID}, []string{opts.ToContentID}); err != nil { + return err + } } return nil } diff --git a/internal/metadata/provider_id_integrity.go b/internal/metadata/provider_id_integrity.go index f6add64733..2d69c19993 100644 --- a/internal/metadata/provider_id_integrity.go +++ b/internal/metadata/provider_id_integrity.go @@ -897,6 +897,16 @@ var mediaItemMergeSteps = []mediaItemMergeStep{ AND src.list_kind = dest.list_kind AND (src.provider_item_key = dest.provider_item_key OR dest.media_item_id = $2)`}, {"move remaining watch provider list items", `UPDATE watch_provider_list_items SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`}, + // The agreed rating follows the ratings it describes. When both items carry + // one, the canonical row wins. A moved row is unconfirmed until the next read + // finds the rating again, so the move can never read as a provider removal. + {"merge watch provider rating items", ` + INSERT INTO watch_provider_rating_items (connection_id, provider_account_id, media_item_id, kind, provider_item_key, synced_rating, remote_seen, updated_at) + SELECT connection_id, provider_account_id, $2, kind, '', synced_rating, false, NOW() + FROM watch_provider_rating_items + WHERE media_item_id = $1 + ON CONFLICT (connection_id, media_item_id) DO NOTHING`}, + {"delete source watch provider rating items", `DELETE FROM watch_provider_rating_items WHERE media_item_id = $1`}, } func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID, canonicalID string) error { diff --git a/internal/metadata/provider_id_integrity_test.go b/internal/metadata/provider_id_integrity_test.go index be961339ba..9d44d69856 100644 --- a/internal/metadata/provider_id_integrity_test.go +++ b/internal/metadata/provider_id_integrity_test.go @@ -141,6 +141,23 @@ func TestMergeStepsUseCurrentWatchProviderListItemsTable(t *testing.T) { } } +func TestMergeStepsCarryWatchProviderRatingItems(t *testing.T) { + var hasMerge, hasDelete bool + for _, step := range mediaItemMergeSteps { + stepSQL := normalizeMergeStepSQL(step.sql) + if strings.Contains(stepSQL, "INSERT INTO watch_provider_rating_items") && + strings.Contains(stepSQL, "ON CONFLICT (connection_id, media_item_id) DO NOTHING") { + hasMerge = true + } + if strings.Contains(stepSQL, "DELETE FROM watch_provider_rating_items WHERE media_item_id = $1") { + hasDelete = true + } + } + if !hasMerge || !hasDelete { + t.Fatalf("media item merge steps must move agreed ratings to the canonical item (merge=%v delete=%v)", hasMerge, hasDelete) + } +} + func TestMergeStepsPreserveEbookReaderProgress(t *testing.T) { var hasMerge bool var hasDelete bool diff --git a/internal/watchsync/connection_settings.go b/internal/watchsync/connection_settings.go index 2d2e5c1003..ecf73d30db 100644 --- a/internal/watchsync/connection_settings.go +++ b/internal/watchsync/connection_settings.go @@ -49,8 +49,10 @@ func (r *PostgresRepository) UpdateConnectionSettings(ctx context.Context, provi sync_watchlist_removals_enabled=COALESCE($11,sync_watchlist_removals_enabled), sync_watchlist_order_enabled=COALESCE($12,sync_watchlist_order_enabled), scrobble_enabled=COALESCE($13,scrobble_enabled), + import_ratings_enabled=COALESCE($14,import_ratings_enabled), + export_ratings_enabled=COALESCE($15,export_ratings_enabled), updated_at=GREATEST(clock_timestamp(),updated_at+interval '1 microsecond') - WHERE id=$1 RETURNING `+connectionColumns, current.ID, update.ImportWatchedEnabled, update.ImportProgressEnabled, update.ExportWatchedEnabled, update.ExportUnwatchedEnabled, update.ImportFavoritesEnabled, update.ExportFavoritesEnabled, update.SyncFavoriteRemovalsEnabled, update.ImportWatchlistEnabled, update.ExportWatchlistEnabled, update.SyncWatchlistRemovalsEnabled, update.SyncWatchlistOrderEnabled, update.ScrobbleEnabled)) + WHERE id=$1 RETURNING `+connectionColumns, current.ID, update.ImportWatchedEnabled, update.ImportProgressEnabled, update.ExportWatchedEnabled, update.ExportUnwatchedEnabled, update.ImportFavoritesEnabled, update.ExportFavoritesEnabled, update.SyncFavoriteRemovalsEnabled, update.ImportWatchlistEnabled, update.ExportWatchlistEnabled, update.SyncWatchlistRemovalsEnabled, update.SyncWatchlistOrderEnabled, update.ScrobbleEnabled, update.ImportRatingsEnabled, update.ExportRatingsEnabled)) if err != nil { return Connection{}, fmt.Errorf("update watch provider settings: %w", err) } diff --git a/internal/watchsync/lists.go b/internal/watchsync/lists.go index 094a5ddeeb..39a5b472ef 100644 --- a/internal/watchsync/lists.go +++ b/internal/watchsync/lists.go @@ -584,14 +584,7 @@ func (s *Service) localItemsFromRows(ctx context.Context, conn Connection, b lis addedAtByID[row.MediaItemID] = addedAt } } - type listMediaResolver interface { - GetListMediaItems(ctx context.Context, mediaItemIDs []string) (map[string]LocalFavorite, error) - } - resolver, ok := s.repo.(listMediaResolver) - if !ok { - return nil, nil, nil, fmt.Errorf("list media resolver is not configured") - } - resolved, err := resolver.GetListMediaItems(ctx, ids) + resolved, err := s.resolveListMediaItems(ctx, ids) if err != nil { return nil, nil, nil, err } @@ -634,6 +627,20 @@ func (s *Service) localItemsFromRows(ctx context.Context, conn Connection, b lis return items, states, warnings, nil } +type listMediaResolver interface { + GetListMediaItems(ctx context.Context, mediaItemIDs []string) (map[string]LocalFavorite, error) +} + +// resolveListMediaItems loads the identity (kind, title, external ids) of +// movies and series by media item id. Unknown ids are absent from the result. +func (s *Service) resolveListMediaItems(ctx context.Context, ids []string) (map[string]LocalFavorite, error) { + resolver, ok := s.repo.(listMediaResolver) + if !ok { + return nil, fmt.Errorf("list media resolver is not configured") + } + return resolver.GetListMediaItems(ctx, ids) +} + // HandleLocalListEvent mirrors a real-time local list change (add/remove of a // favorite or watchlist item) to the providers bound to that list kind. It is // fire-and-forget so the originating API request is never blocked on provider diff --git a/internal/watchsync/ratings.go b/internal/watchsync/ratings.go new file mode 100644 index 0000000000..a2adabe01e --- /dev/null +++ b/internal/watchsync/ratings.go @@ -0,0 +1,883 @@ +package watchsync + +import ( + "context" + "errors" + "fmt" + "log/slog" + "maps" + "strings" + "time" + + "github.com/Silo-Server/silo-server/internal/catalog" + "github.com/Silo-Server/silo-server/internal/historyimport" + "github.com/Silo-Server/silo-server/internal/userstore" +) + +// This file syncs a profile's movie and series ratings with a provider. Silo +// stores 1 to 5 stars; providers use integers from 1 to 10. Every decision is +// made in stars, so a remote change inside one star (7 to 8) is not a change, +// and Silo never overwrites a remote 7 with the 8 its 4 stars map to. +// +// Each item is a three-way merge of the local rating, the remote rating, and +// the last rating both sides agreed on (watch_provider_rating_items). Whichever +// side moved away from the agreed rating wins; when both moved to different +// values, a rating beats a removal and otherwise the newer change wins, with +// ties going to Silo. +// +// Writes are safe without a cross-node lock. Imports are compare-and-set on the +// local value this run observed, so a concurrent user edit always wins and is +// reconsidered next run. Provider writes are idempotent desired-state pushes. A +// stale agreed rating heals on the next run, because equal local and remote +// values simply rebase. + +const ( + ratingExportBatchSize = 100 + // ratingCursorSegment marks provider cursor keys that belong to rating + // reads (for example "simkl.ratings.movies"), so they reset together with + // the agreed ratings when a connection moves to another provider account. + ratingCursorSegment = ".ratings" + // ratingImportCursorKey records that the rating read cursors were saved + // while import was on. Cursors saved in send-only mode may have skipped + // changes that were never imported, so turning import on reads everything. + ratingImportCursorKey = "watchsync.ratings.import" +) + +// ratingStore is the profile rating table (catalog.RatingsRepo). Imports use the +// compare-and-set writes, which never dispatch rating events, so an imported +// rating is not echoed back to providers. +type ratingStore interface { + ListAll(ctx context.Context, userID int, profileID string) ([]catalog.UserRating, error) + Get(ctx context.Context, userID int, profileID, mediaItemID string) (*catalog.UserRating, error) + SetIfUnchanged(ctx context.Context, userID int, profileID, mediaItemID string, expected, rating int, ratedAt time.Time) (bool, error) + DeleteIfUnchanged(ctx context.Context, userID int, profileID, mediaItemID string, expected int) (bool, error) +} + +// ratingProfileStaler marks a profile's recommendations stale after imports +// changed its ratings. +type ratingProfileStaler interface { + MarkProfileStale(ctx context.Context, userID int, profileID string) error +} + +func (s *Service) WithRatingStore(store ratingStore, staler ratingProfileStaler) *Service { + if s != nil { + s.ratings = store + s.ratingStaler = staler + } + return s +} + +// starsFromProviderRating converts a provider rating (1 to 10) to stars, +// rounding half up: 1-2 is 1 star, 3-4 is 2 stars, and 9-10 is 5 stars. +func starsFromProviderRating(rating int) int { + rating = min(max(rating, 1), 10) + return (rating + 1) / 2 +} + +// providerRatingFromStars converts stars to the provider scale. +func providerRatingFromStars(stars int) int { + return stars * 2 +} + +type ratingAction uint8 + +const ( + ratingKeep ratingAction = iota + ratingRebase + ratingExport + ratingImport +) + +// decideRating merges one item. local, remote and base are stars, with 0 for +// unrated; base is the last agreed rating. A zero remoteAt is unknown. +func decideRating(local, remote, base int, localAt, remoteAt time.Time) ratingAction { + switch { + case local == remote && local == base: + return ratingKeep + case local == remote: + return ratingRebase + case remote == base: + return ratingExport + case local == base: + return ratingImport + // Both sides changed to different values. A rating beats a removal so a + // conflict never deletes; otherwise the newer change wins and ties go to Silo. + case local == 0: + return ratingImport + case remote == 0: + return ratingExport + case !remoteAt.IsZero() && remoteAt.After(localAt): + return ratingImport + default: + return ratingExport + } +} + +// ratingItem is one movie or series in a rating sync. +type ratingItem struct { + identity LocalFavorite + local int + localAt time.Time + // stored is the agreed-rating row, nil when there is none. + stored *RatingSyncState + // base is the agreed rating used for the decision. It is 0 when there is no + // row, and also when a complete snapshot shows the provider never held a + // rating Silo sent, so the rating is sent again instead of deleted. + base int + remote int + remoteAt time.Time + // observed means this run read the remote value from the provider. + observed bool + // remoteKey is the provider's own key for the item from this run's read. + remoteKey string +} + +// providerKey is the key recorded for an item and sent with its writes: the +// provider's own key once a read returned one, so the provider's tombstones and +// writes can name it, and otherwise Silo's key. +func (item *ratingItem) providerKey() string { + if item.remoteKey != "" { + return item.remoteKey + } + if item.stored != nil && item.stored.ProviderItemKey != "" { + return item.stored.ProviderItemKey + } + return item.identity.ProviderItemKey +} + +// sendIdentity is the item identity sent to the provider, carrying its +// providerKey. +func (item *ratingItem) sendIdentity() LocalFavorite { + identity := item.identity + identity.ProviderItemKey = item.providerKey() + return identity +} + +// dropUnsyncedRatingKinds removes the items of kinds the provider does not rate. +func dropUnsyncedRatingKinds(items map[string]*ratingItem, provider Provider) { + filter, ok := provider.(RatingKindFilter) + if !ok { + return + } + for id, item := range items { + if !filter.SyncsRatingKind(item.identity.Kind) { + delete(items, id) + } + } +} + +// SyncRatingsResult summarizes one rating sync. +type SyncRatingsResult struct { + RemoteFound int + Imported int + LocalFound int + Sent int + Warnings []string +} + +// syncRatings runs the scheduled rating sync for one connection: read the +// provider's ratings, merge each item, apply imports locally, then send local +// changes. +func (s *Service) syncRatings(ctx context.Context, conn Connection, cfg ServerConfig, provider Provider) (SyncRatingsResult, error) { + var result SyncRatingsResult + if s.ratings == nil { + return result, fmt.Errorf("rating store is not configured") + } + caps := provider.Capabilities() + importer, canImport := provider.(RatingImporter) + _, canExport := provider.(RatingExporter) + canImport = canImport && caps.ImportRatings + canExport = canExport && caps.ExportRatings + importAllowed := conn.ImportRatingsEnabled && canImport + exportAllowed := conn.ExportRatingsEnabled && canExport + if !importAllowed && !exportAllowed { + return result, nil + } + + items, warnings, err := s.loadRatingItems(ctx, conn, nil) + if err != nil { + return result, err + } + dropUnsyncedRatingKinds(items, provider) + result.Warnings = append(result.Warnings, warnings...) + for _, item := range items { + if item.local > 0 { + result.LocalFound++ + } + } + + // A provider that can read ratings is read even when only sending, so Silo + // does not push values the provider already holds. + var batch RatingImportBatch + if canImport { + if s.matcher == nil { + return result, fmt.Errorf("watch provider matcher is not configured") + } + fetchConn := conn + if importAllowed && conn.SyncCursors[ratingImportCursorKey] == "" { + fetchConn.SyncCursors = withoutRatingCursors(conn.SyncCursors) + } + batch, err = importer.FetchRatings(ctx, cfg, fetchConn) + if err != nil { + return result, err + } + result.Warnings = append(result.Warnings, batch.Warnings...) + for _, row := range batch.Rows { + if !row.Removed { + result.RemoteFound++ + } + } + warnings, err := s.resolveRemoteRatings(ctx, items, batch) + if err != nil { + return result, err + } + dropUnsyncedRatingKinds(items, provider) + result.Warnings = append(result.Warnings, warnings...) + } else { + markRemoteUnknown(items) + } + + applied, err := s.reconcileRatings(ctx, conn, cfg, provider, items, importAllowed, exportAllowed, func() error { + if !canImport { + return nil + } + return s.saveRatingCursors(ctx, conn, batch.UpdatedCursors, importAllowed) + }) + result.Imported = applied.imported + result.Sent = applied.sent + result.Warnings = append(result.Warnings, applied.warnings...) + return result, err +} + +// saveRatingCursors stores the read cursors in send-only mode too, so a +// cursor-gated provider does not re-read every rating on every run, and marks +// whether import was on so turning it on later forces one full read. +func (s *Service) saveRatingCursors(ctx context.Context, conn Connection, updated map[string]string, importAllowed bool) error { + fresh, err := s.reloadConnection(ctx, conn) + if err != nil { + return err + } + if fresh.ProviderAccountID != conn.ProviderAccountID { + return nil + } + cursors := fresh.SyncCursors + if importAllowed && cursors[ratingImportCursorKey] == "" { + cursors = withoutRatingCursors(cursors) + } + cursors = mergeSyncCursors(cursors, updated) + if importAllowed { + cursors[ratingImportCursorKey] = "1" + } else { + delete(cursors, ratingImportCursorKey) + } + if maps.Equal(cursors, fresh.SyncCursors) { + return nil + } + fresh.SyncCursors = cursors + _, err = s.repo.UpsertConnection(ctx, fresh) + return err +} + +// HandleLocalRatingEvent sends a profile's rating changes to the providers +// that receive its ratings. It is fire-and-forget so the originating API +// request never waits on provider I/O. +func (s *Service) HandleLocalRatingEvent(ctx context.Context, event LocalRatingEvent) error { + if event.UserID == 0 || event.ProfileID == "" || len(event.MediaItemIDs) == 0 { + return nil + } + go func() { + bg, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if err := s.processLocalRatingEvent(bg, event); err != nil { + slog.WarnContext(ctx, "failed to dispatch local rating provider event", "component", "watchsync", "user_id", event.UserID, "profile_id", event.ProfileID, "error", err) + } + }() + return nil +} + +func (s *Service) processLocalRatingEvent(ctx context.Context, event LocalRatingEvent) error { + if s.ratings == nil { + return nil + } + conns, err := s.repo.ListRatingEventConnections(ctx, event.UserID, event.ProfileID) + if err != nil { + return err + } + for _, conn := range conns { + provider, ok := s.registry.Get(conn.Provider) + if !ok || !provider.Capabilities().ExportRatings { + continue + } + if _, ok := provider.(RatingExporter); !ok { + continue + } + cfg, err := s.serverConfig(ctx, conn.Provider) + if err != nil { + s.recordLocalWatchEventError(ctx, conn, err) + continue + } + conn, err = s.refreshConnectionIfNeeded(ctx, provider, cfg, conn) + if err != nil { + s.recordLocalWatchEventError(ctx, conn, err) + continue + } + items, _, err := s.loadRatingItems(ctx, conn, event.MediaItemIDs) + if err != nil { + s.recordLocalWatchEventError(ctx, conn, err) + continue + } + dropUnsyncedRatingKinds(items, provider) + // Only the local side is known here. A removal waits for the scheduled + // merge, which can see whether the provider changed the rating since + // (a rating beats a removal); a new rating is sent now. + for id, item := range items { + if item.local == 0 { + delete(items, id) + } + } + markRemoteUnknown(items) + if _, err := s.reconcileRatings(ctx, conn, cfg, provider, items, false, true, nil); err != nil { + if limited, ok := AsRateLimited(err); ok { + if deferErr := s.deferRateLimitedConnection(ctx, conn, limited); deferErr != nil { + s.recordLocalWatchEventError(ctx, conn, errors.Join(err, deferErr)) + } + continue + } + s.recordLocalWatchEventError(ctx, conn, err) + } + } + return nil +} + +// loadRatingItems gathers the profile's movie and series ratings and the +// connection's agreed ratings, keyed by media item. onlyIDs limits both to the +// listed items; nil loads everything. +func (s *Service) loadRatingItems(ctx context.Context, conn Connection, onlyIDs []string) (map[string]*ratingItem, []string, error) { + var local []catalog.UserRating + if onlyIDs != nil { + for _, id := range onlyIDs { + rating, err := s.ratings.Get(ctx, conn.UserID, conn.ProfileID, id) + if err != nil { + return nil, nil, err + } + if rating != nil { + local = append(local, *rating) + } + } + } else { + // One read gives a consistent snapshot; paging could miss a rating + // that a concurrent edit moves, which would read as a local removal. + var err error + if local, err = s.ratings.ListAll(ctx, conn.UserID, conn.ProfileID); err != nil { + return nil, nil, err + } + } + states, err := s.repo.ListRatingSyncStates(ctx, conn.ID, conn.ProviderAccountID, onlyIDs) + if err != nil { + return nil, nil, err + } + + ids := make([]string, 0, len(local)+len(states)) + for _, rating := range local { + ids = append(ids, rating.MediaItemID) + } + for _, state := range states { + ids = append(ids, state.MediaItemID) + } + resolved, err := s.resolveListMediaItems(ctx, ids) + if err != nil { + return nil, nil, err + } + + items := make(map[string]*ratingItem, len(ids)) + var warnings []string + ratedLocally := make(map[string]bool, len(local)) + for _, rating := range local { + ratedLocally[rating.MediaItemID] = true + identity, ok := resolved[rating.MediaItemID] + if !ok || !ratingSyncKind(identity.Kind) { + continue + } + if identity.ProviderItemKey == "" { + warnings = append(warnings, "rated item has no provider ids: "+rating.MediaItemID) + continue + } + items[rating.MediaItemID] = &ratingItem{identity: identity, local: rating.Rating, localAt: rating.RatedAt} + } + for i := range states { + state := states[i] + item, ok := items[state.MediaItemID] + if !ok { + // A rated item this sync skipped (no longer in the catalog, or + // without external ids) must not read as a local removal. + if ratedLocally[state.MediaItemID] { + continue + } + // Agreed but no longer rated locally. The media item may be gone, so + // fall back to the identity the agreed row recorded. + identity, found := resolved[state.MediaItemID] + if !found || identity.ProviderItemKey == "" { + identity = LocalFavorite{MediaItemID: state.MediaItemID, Kind: state.Kind, ProviderItemKey: state.ProviderItemKey} + } + if !ratingSyncKind(identity.Kind) || identity.ProviderItemKey == "" { + continue + } + item = &ratingItem{identity: identity} + items[state.MediaItemID] = item + } + item.stored = &state + item.base = state.SyncedRating + } + return items, warnings, nil +} + +// markRemoteUnknown treats every remote value as unchanged since the agreed +// rating, for merges that did not read the provider. +func markRemoteUnknown(items map[string]*ratingItem) { + for _, item := range items { + item.remote = item.base + } +} + +// resolveRemoteRatings sets each item's remote value from a provider read. +// Matched rows give the value directly and explicit tombstones remove it. An +// item missing from the read counts as unrated only when the read is a complete +// snapshot of the item's kind, no row shares one of its ids (a row the matcher +// could not place may be this item), and a previous read confirmed the provider +// held the agreed rating. Every other item is unknown and keeps its agreed value. +func (s *Service) resolveRemoteRatings(ctx context.Context, items map[string]*ratingItem, batch RatingImportBatch) ([]string, error) { + var warnings []string + snapshot := make(map[string]bool, len(batch.SnapshotKinds)) + for _, kind := range batch.SnapshotKinds { + snapshot[kind] = true + } + byKey := make(map[string][]*ratingItem, len(items)) + for _, item := range items { + if item.stored != nil && item.stored.ProviderItemKey != "" { + byKey[item.stored.ProviderItemKey] = append(byKey[item.stored.ProviderItemKey], item) + } + } + + type remoteMatch struct { + row RemoteRating + id string + } + seenTokens := make(map[string]bool) + rowsPerKind := make(map[string]int) + unidentified := make(map[string]bool) + var matches []remoteMatch + var unresolved []string + for _, row := range batch.Rows { + if row.Removed { + candidates := byKey[strings.TrimSpace(row.ProviderItemKey)] + if len(candidates) != 1 { + warnings = append(warnings, "watch sync provider returned a rating removal that matches no single title") + continue + } + item := candidates[0] + item.remote, item.remoteAt, item.observed = 0, time.Time{}, true + continue + } + if !ratingSyncKind(row.Kind) { + continue + } + // Record the row's ids before any check, so a row Silo cannot use + // still keeps its title from reading as removed. + rowsPerKind[row.Kind]++ + for _, token := range ratingIdentityTokens(row.Kind, row.IMDbID, row.TMDBID, row.TVDBID, row.ProviderItemKey) { + seenTokens[token] = true + } + if row.IMDbID == "" && row.TMDBID == "" && row.TVDBID == "" { + // A title known only by provider ids could be any local item, so + // absence from this read proves nothing for its kind. + unidentified[row.Kind] = true + continue + } + if row.Rating < 1 || row.Rating > 10 { + warnings = append(warnings, fmt.Sprintf("watch sync provider returned an out-of-range rating %d", row.Rating)) + continue + } + match, reason, err := s.matcher.Match(ctx, row.HistoryRecord()) + if err != nil { + return warnings, err + } + if match == nil { + if reason != "" { + warnings = append(warnings, reason) + } + continue + } + matches = append(matches, remoteMatch{row: row, id: match.MediaItemID}) + if _, ok := items[match.MediaItemID]; !ok { + unresolved = append(unresolved, match.MediaItemID) + } + } + + // Remote-only ratings become items with no local rating and no agreement. + if len(unresolved) > 0 { + resolved, err := s.resolveListMediaItems(ctx, unresolved) + if err != nil { + return warnings, err + } + for _, id := range unresolved { + identity, ok := resolved[id] + if !ok || !ratingSyncKind(identity.Kind) || items[id] != nil { + continue + } + items[id] = &ratingItem{identity: identity} + } + } + for _, match := range matches { + item := items[match.id] + if item == nil { + continue + } + // Several rows can match one item; the newest rating wins. + if item.observed && !match.row.RatedAt.After(item.remoteAt) { + continue + } + item.remote = starsFromProviderRating(match.row.Rating) + item.remoteAt = match.row.RatedAt + item.observed = true + item.remoteKey = strings.TrimSpace(match.row.ProviderItemKey) + } + + for kind := range unidentified { + if snapshot[kind] { + delete(snapshot, kind) + warnings = append(warnings, "watch sync provider returned "+kind+" ratings without an IMDb, TMDB, or TVDB id; skipped rating removals for that kind") + } + } + // A complete snapshot that returned nothing for a kind is more likely a + // failed read than a mass removal when it would remove several confirmed + // ratings Silo still holds, so it is not trusted. One such rating, or a + // removal Silo already sent, is an ordinary change and goes through. + for kind := range snapshot { + if rowsPerKind[kind] > 0 { + continue + } + held := 0 + for _, item := range items { + if item.identity.Kind == kind && item.local > 0 && item.stored != nil && item.stored.RemoteSeen { + held++ + } + } + if held > 1 { + delete(snapshot, kind) + warnings = append(warnings, "watch sync provider returned no "+kind+" ratings; skipped rating removals for that kind") + } + } + + for _, item := range items { + if item.observed { + continue + } + absent := snapshot[item.identity.Kind] + if absent { + identity := item.identity + for _, token := range ratingIdentityTokens(identity.Kind, identity.IMDbID, identity.TMDBID, identity.TVDBID, identity.ProviderItemKey, item.providerKey()) { + if seenTokens[token] { + absent = false + break + } + } + } + switch { + case !absent: + item.remote = item.base + case item.stored != nil && item.stored.RemoteSeen: + item.remote = 0 + default: + // The provider never confirmed holding this rating, so it was not + // agreed: send it again rather than delete it locally. + item.base = 0 + item.remote = 0 + } + } + return warnings, nil +} + +// ratingIdentityTokens are the kind-qualified ids and keys of an item. TMDB and +// TVDB number movies and series separately, so ids only compare within a kind. +func ratingIdentityTokens(kind, imdbID, tmdbID, tvdbID string, keys ...string) []string { + tokens := make([]string, 0, 3+len(keys)) + for _, id := range append([]string{ + prefixedID("imdb", imdbID), prefixedID("tmdb", tmdbID), prefixedID("tvdb", tvdbID), + }, keys...) { + if id = strings.TrimSpace(id); id != "" { + tokens = append(tokens, kind+":"+id) + } + } + return tokens +} + +func prefixedID(namespace, id string) string { + if strings.TrimSpace(id) == "" { + return "" + } + return namespace + ":" + strings.TrimSpace(id) +} + +func ratingSyncKind(kind string) bool { + return kind == historyimport.KindMovie || kind == historyimport.KindSeries +} + +type ratingReconcileResult struct { + imported int + sent int + warnings []string +} + +// reconcileRatings applies the merge decision for every item: imports first, +// then the agreed-rating bookkeeping, then afterImport (the scheduled run's +// cursor update), then provider writes. Decisions a direction does not allow +// are skipped without recording agreement, so they are reconsidered when the +// direction is turned on. +func (s *Service) reconcileRatings( + ctx context.Context, + conn Connection, + cfg ServerConfig, + provider Provider, + items map[string]*ratingItem, + importAllowed, exportAllowed bool, + afterImport func() error, +) (ratingReconcileResult, error) { + var result ratingReconcileResult + var upserts []RatingSyncState + var deletes []string + var sets []*ratingItem + var removals []*ratingItem + agree := func(item *ratingItem, stars int, seen bool) { + switch { + case stars == 0 && item.stored != nil: + deletes = append(deletes, item.identity.MediaItemID) + case stars == 0: + case item.stored == nil || item.stored.SyncedRating != stars || item.stored.RemoteSeen != seen: + upserts = append(upserts, RatingSyncState{ + ConnectionID: conn.ID, + ProviderAccountID: conn.ProviderAccountID, + MediaItemID: item.identity.MediaItemID, + Kind: item.identity.Kind, + ProviderItemKey: item.providerKey(), + SyncedRating: stars, + RemoteSeen: seen, + }) + } + } + + for _, item := range items { + switch decideRating(item.local, item.remote, item.base, item.localAt, item.remoteAt) { + case ratingKeep: + // Record a first confirmation that the provider holds the rating. + seen := item.stored != nil && item.stored.RemoteSeen + agree(item, item.base, seen || (item.observed && item.base > 0)) + case ratingRebase: + agree(item, item.local, item.observed) + case ratingImport: + if !importAllowed { + continue + } + applied, err := s.importRating(ctx, conn, item) + if err != nil { + return result, err + } + if applied { + result.imported++ + agree(item, item.remote, item.observed) + } + case ratingExport: + if !exportAllowed { + continue + } + if item.local > 0 { + sets = append(sets, item) + } else { + removals = append(removals, item) + } + } + } + if err := s.repo.UpsertRatingSyncStates(ctx, upserts); err != nil { + return result, err + } + if err := s.repo.DeleteRatingSyncStates(ctx, conn.ID, deletes); err != nil { + return result, err + } + if result.imported > 0 && s.ratingStaler != nil { + if err := s.ratingStaler.MarkProfileStale(ctx, conn.UserID, conn.ProfileID); err != nil { + slog.WarnContext(ctx, "failed to mark profile stale after rating import", "component", "watchsync", "user_id", conn.UserID, "profile_id", conn.ProfileID, "error", err) + } + } + if afterImport != nil { + if err := afterImport(); err != nil { + return result, err + } + } + if len(sets) == 0 && len(removals) == 0 { + return result, nil + } + + exporter, ok := provider.(RatingExporter) + if !ok { + return result, fmt.Errorf("provider %q does not implement rating export", conn.Provider) + } + sets, deferred, err := s.gateRatingExports(ctx, conn, provider, sets) + if err != nil { + return result, err + } + result.warnings = append(result.warnings, deferred...) + sent, warnings, err := s.sendRatings(ctx, conn, cfg, exporter, sets, removals) + result.sent = sent + result.warnings = append(result.warnings, warnings...) + return result, err +} + +// importRating writes the remote value locally if the local rating is still the +// value this run observed. +func (s *Service) importRating(ctx context.Context, conn Connection, item *ratingItem) (bool, error) { + id := item.identity.MediaItemID + if item.remote == 0 { + return s.ratings.DeleteIfUnchanged(ctx, conn.UserID, conn.ProfileID, id, item.local) + } + ratedAt := item.remoteAt + if ratedAt.IsZero() { + ratedAt = s.now() + } + return s.ratings.SetIfUnchanged(ctx, conn.UserID, conn.ProfileID, id, item.local, item.remote, ratedAt) +} + +// gateRatingExports holds back new ratings that a provider would record as a +// watch until the profile has a completed play of the title. A title the +// provider already holds a rating for is already on its list, so changing that +// rating records nothing new and is not held back. +func (s *Service) gateRatingExports(ctx context.Context, conn Connection, provider Provider, sets []*ratingItem) ([]*ratingItem, []string, error) { + gate, ok := provider.(RatingExportWatchGate) + if !ok || len(sets) == 0 { + return sets, nil, nil + } + gated := func(item *ratingItem) bool { + heldRemotely := (item.observed && item.remote > 0) || + (item.stored != nil && item.stored.RemoteSeen && item.stored.SyncedRating > 0) + return !heldRemotely && gate.RatingExportRequiresWatched(item.identity.Kind) + } + var gatedIDs []string + for _, item := range sets { + if gated(item) { + gatedIDs = append(gatedIDs, item.identity.MediaItemID) + } + } + if len(gatedIDs) == 0 { + return sets, nil, nil + } + if s.storeProvider == nil { + return nil, nil, fmt.Errorf("user store provider is not configured") + } + store, err := s.storeProvider.ForUser(ctx, conn.UserID) + if err != nil { + return nil, nil, fmt.Errorf("open user store: %w", err) + } + history, err := listAllCompletedHistory(ctx, store, userstore.CompletedHistoryQuery{ProfileID: conn.ProfileID, MediaItemIDs: gatedIDs}) + if err != nil { + return nil, nil, err + } + watched := make(map[string]bool, len(history)) + for _, entry := range history { + watched[entry.MediaItemID] = true + } + kept := sets[:0] + var warnings []string + for _, item := range sets { + if gated(item) && !watched[item.identity.MediaItemID] { + warnings = append(warnings, "rating waits for a completed play before it is sent: "+item.identity.MediaItemID) + continue + } + kept = append(kept, item) + } + return kept, warnings, nil +} + +// sendRatings pushes sets and removals in batches. A confirmed set is recorded +// as agreed only if the local rating still has the value sent, so a change +// made while the write was in flight is not masked; it is agreed but not seen +// until a later read confirms the provider kept it. A confirmed removal keeps +// its agreed row: the next read either finds the title unrated, which clears +// the row, or finds another provider entry still rated, which is removed in +// turn instead of being imported back. +func (s *Service) sendRatings(ctx context.Context, conn Connection, cfg ServerConfig, exporter RatingExporter, sets, removals []*ratingItem) (int, []string, error) { + sent := 0 + var warnings []string + for start := 0; start < len(sets); start += ratingExportBatchSize { + batch := sets[start:min(start+ratingExportBatchSize, len(sets))] + payload := make([]LocalRating, 0, len(batch)) + for _, item := range batch { + payload = append(payload, LocalRating{ + LocalFavorite: item.sendIdentity(), + Rating: providerRatingFromStars(item.local), + RatedAt: item.localAt, + }) + } + result, err := exporter.ExportRatings(ctx, cfg, conn, payload) + if err != nil { + return sent, warnings, err + } + states := make([]RatingSyncState, 0, len(batch)) + for _, item := range batch { + identity := item.sendIdentity() + if confirmed, _ := exportItemOutcome(result, identity.MediaItemID, identity.ProviderItemKey); !confirmed { + warnings = append(warnings, exportFailureReason(result, identity, "rating")+": "+identity.MediaItemID) + continue + } + sent++ + current, err := s.ratings.Get(ctx, conn.UserID, conn.ProfileID, identity.MediaItemID) + if err != nil { + return sent, warnings, err + } + if current == nil || current.Rating != item.local { + continue + } + states = append(states, RatingSyncState{ + ConnectionID: conn.ID, + ProviderAccountID: conn.ProviderAccountID, + MediaItemID: identity.MediaItemID, + Kind: identity.Kind, + ProviderItemKey: identity.ProviderItemKey, + SyncedRating: item.local, + }) + } + if err := s.repo.UpsertRatingSyncStates(ctx, states); err != nil { + return sent, warnings, err + } + } + for start := 0; start < len(removals); start += ratingExportBatchSize { + batch := removals[start:min(start+ratingExportBatchSize, len(removals))] + payload := make([]LocalFavorite, 0, len(batch)) + for _, item := range batch { + payload = append(payload, item.sendIdentity()) + } + result, err := exporter.RemoveRatings(ctx, cfg, conn, payload) + if err != nil { + return sent, warnings, err + } + for _, item := range batch { + identity := item.sendIdentity() + // A provider that no longer knows the title has no rating to clear. + if confirmed, missing := exportItemOutcome(result, identity.MediaItemID, identity.ProviderItemKey); confirmed || missing { + sent++ + continue + } + warnings = append(warnings, exportFailureReason(result, identity, "rating removal")+": "+identity.MediaItemID) + } + } + return sent, warnings, nil +} + +// withoutRatingCursors drops the rating read cursors, used together with +// ClearRatingSyncStates when a connection changes provider account. +func withoutRatingCursors(cursors map[string]string) map[string]string { + kept := make(map[string]string, len(cursors)) + for key, value := range cursors { + if !strings.Contains(key, ratingCursorSegment) { + kept[key] = value + } + } + return kept +} diff --git a/internal/watchsync/ratings_db_test.go b/internal/watchsync/ratings_db_test.go new file mode 100644 index 0000000000..e60a663446 --- /dev/null +++ b/internal/watchsync/ratings_db_test.go @@ -0,0 +1,202 @@ +package watchsync + +import ( + "os" + "sort" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/secret" +) + +func TestRatingSyncRepositoryDB(t *testing.T) { + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + ctx := t.Context() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + var userID int + if err := pool.QueryRow(ctx, "INSERT INTO users(username,role) VALUES($1,'user') RETURNING id", "watch-ratings-"+uuid.NewString()).Scan(&userID); err != nil { + t.Fatal(err) + } + defer func() { _, _ = pool.Exec(ctx, "DELETE FROM users WHERE id=$1", userID) }() + if _, err := pool.Exec(ctx, "INSERT INTO user_profiles(user_id,id,name) VALUES($1,'ratings-p','Ratings')", userID); err != nil { + t.Fatal(err) + } + cipher, err := secret.New([]byte("watch-ratings-test-key-with-enough-entropy")) + if err != nil { + t.Fatal(err) + } + repo := NewPostgresRepository(pool, cipher) + + conn, err := repo.UpsertConnection(ctx, Connection{ + Provider: "ratings", UserID: userID, ProfileID: "ratings-p", AccessToken: "token", + ImportRatingsEnabled: true, ExportRatingsEnabled: true, + }) + if err != nil { + t.Fatal(err) + } + if !conn.ImportRatingsEnabled || !conn.ExportRatingsEnabled { + t.Fatalf("inserted rating toggles = %v/%v, want on", conn.ImportRatingsEnabled, conn.ExportRatingsEnabled) + } + + t.Run("toggles are insert-only on upsert", func(t *testing.T) { + again := conn + again.ImportRatingsEnabled, again.ExportRatingsEnabled = false, false + saved, err := repo.UpsertConnection(ctx, again) + if err != nil { + t.Fatal(err) + } + if !saved.ImportRatingsEnabled || !saved.ExportRatingsEnabled { + t.Fatal("a token upsert must not change rating toggles") + } + }) + + t.Run("settings update and event connections", func(t *testing.T) { + updated, err := repo.UpdateConnectionSettings(ctx, "ratings", userID, "ratings-p", nil, ConnectionUpdate{ExportRatingsEnabled: new(false)}, nil) + if err != nil { + t.Fatal(err) + } + if !updated.ImportRatingsEnabled || updated.ExportRatingsEnabled || !updated.UpdatedAt.After(conn.UpdatedAt) { + t.Fatalf("updated = import %v export %v", updated.ImportRatingsEnabled, updated.ExportRatingsEnabled) + } + conns, err := repo.ListRatingEventConnections(ctx, userID, "ratings-p") + if err != nil { + t.Fatal(err) + } + if len(conns) != 0 { + t.Fatalf("event connections with export off = %d, want 0", len(conns)) + } + if _, err := repo.UpdateConnectionSettings(ctx, "ratings", userID, "ratings-p", nil, ConnectionUpdate{ExportRatingsEnabled: new(true)}, nil); err != nil { + t.Fatal(err) + } + conns, err = repo.ListRatingEventConnections(ctx, userID, "ratings-p") + if err != nil { + t.Fatal(err) + } + if len(conns) != 1 || conns[0].ID != conn.ID { + t.Fatalf("event connections = %#v, want the rating connection", conns) + } + due, err := repo.ListConnectionsDueForSync(ctx, conn.UpdatedAt) + if err != nil { + t.Fatal(err) + } + found := false + for _, candidate := range due { + found = found || candidate.ID == conn.ID + } + if !found { + t.Fatal("a connection with only rating sync on must be due for sync") + } + }) + + t.Run("sync run counters round-trip", func(t *testing.T) { + run, err := repo.CreateSyncRun(ctx, SyncRun{ConnectionID: conn.ID, Trigger: "test", Provider: "ratings", InboundRatingsFound: 1}) + if err != nil { + t.Fatal(err) + } + run.InboundRatingsFound, run.InboundRatingsImported, run.OutboundRatingsFound, run.OutboundRatingsSent = 4, 3, 2, 1 + run.Status = string(SyncRunStatusSuccess) + completed, err := repo.CompleteSyncRun(ctx, run) + if err != nil { + t.Fatal(err) + } + if completed.InboundRatingsFound != 4 || completed.InboundRatingsImported != 3 || completed.OutboundRatingsFound != 2 || completed.OutboundRatingsSent != 1 { + t.Fatalf("completed run counters = %#v", completed) + } + }) + + t.Run("agreed ratings", func(t *testing.T) { + if err := repo.UpsertRatingSyncStates(ctx, []RatingSyncState{ + {ConnectionID: conn.ID, MediaItemID: "m-1", Kind: "movie", ProviderItemKey: "imdb:tt1", SyncedRating: 4}, + {ConnectionID: conn.ID, MediaItemID: "s-1", Kind: "series", ProviderItemKey: "tvdb:1", SyncedRating: 2, RemoteSeen: true}, + }); err != nil { + t.Fatal(err) + } + // Updating keeps the recorded identity when the update carries none. + if err := repo.UpsertRatingSyncStates(ctx, []RatingSyncState{{ConnectionID: conn.ID, MediaItemID: "m-1", SyncedRating: 5, RemoteSeen: true}}); err != nil { + t.Fatal(err) + } + all, err := repo.ListRatingSyncStates(ctx, conn.ID, "", nil) + if err != nil { + t.Fatal(err) + } + sort.Slice(all, func(i, j int) bool { return all[i].MediaItemID < all[j].MediaItemID }) + want := []RatingSyncState{ + {ConnectionID: conn.ID, MediaItemID: "m-1", Kind: "movie", ProviderItemKey: "imdb:tt1", SyncedRating: 5, RemoteSeen: true}, + {ConnectionID: conn.ID, MediaItemID: "s-1", Kind: "series", ProviderItemKey: "tvdb:1", SyncedRating: 2, RemoteSeen: true}, + } + if len(all) != 2 || all[0] != want[0] || all[1] != want[1] { + t.Fatalf("states = %#v, want %#v", all, want) + } + only, err := repo.ListRatingSyncStates(ctx, conn.ID, "", []string{"s-1"}) + if err != nil { + t.Fatal(err) + } + if len(only) != 1 || only[0].MediaItemID != "s-1" { + t.Fatalf("filtered states = %#v", only) + } + if err := repo.DeleteRatingSyncStates(ctx, conn.ID, []string{"m-1"}); err != nil { + t.Fatal(err) + } + if states, _ := repo.ListRatingSyncStates(ctx, conn.ID, "", nil); len(states) != 1 { + t.Fatalf("states after delete = %#v", states) + } + if err := repo.ClearRatingSyncStates(ctx, conn.ID); err != nil { + t.Fatal(err) + } + if states, _ := repo.ListRatingSyncStates(ctx, conn.ID, "", nil); len(states) != 0 { + t.Fatalf("states after clear = %#v", states) + } + }) + + t.Run("agreed ratings are scoped to the provider account", func(t *testing.T) { + if err := repo.UpsertRatingSyncStates(ctx, []RatingSyncState{ + {ConnectionID: conn.ID, ProviderAccountID: "account-a", MediaItemID: "m-3", Kind: "movie", SyncedRating: 3}, + }); err != nil { + t.Fatal(err) + } + if states, _ := repo.ListRatingSyncStates(ctx, conn.ID, "account-b", nil); len(states) != 0 { + t.Fatalf("another account's rows = %#v, want none", states) + } + states, err := repo.ListRatingSyncStates(ctx, conn.ID, "account-a", nil) + if err != nil || len(states) != 1 || states[0].ProviderAccountID != "account-a" { + t.Fatalf("account rows = %#v (%v)", states, err) + } + // Re-agreeing under the new account takes the row over. + if err := repo.UpsertRatingSyncStates(ctx, []RatingSyncState{ + {ConnectionID: conn.ID, ProviderAccountID: "account-b", MediaItemID: "m-3", Kind: "movie", SyncedRating: 5}, + }); err != nil { + t.Fatal(err) + } + if states, _ := repo.ListRatingSyncStates(ctx, conn.ID, "account-a", nil); len(states) != 0 { + t.Fatalf("old account still sees %#v", states) + } + if err := repo.ClearRatingSyncStates(ctx, conn.ID); err != nil { + t.Fatal(err) + } + }) + + t.Run("connection delete cascades", func(t *testing.T) { + if err := repo.UpsertRatingSyncStates(ctx, []RatingSyncState{{ConnectionID: conn.ID, MediaItemID: "m-2", Kind: "movie", SyncedRating: 3}}); err != nil { + t.Fatal(err) + } + if err := repo.DeleteConnection(ctx, "ratings", userID, "ratings-p"); err != nil { + t.Fatal(err) + } + var remaining int + if err := pool.QueryRow(ctx, "SELECT count(*) FROM watch_provider_rating_items WHERE connection_id=$1::uuid", conn.ID).Scan(&remaining); err != nil { + t.Fatal(err) + } + if remaining != 0 { + t.Fatalf("agreed ratings left after connection delete: %d", remaining) + } + }) +} diff --git a/internal/watchsync/ratings_test.go b/internal/watchsync/ratings_test.go new file mode 100644 index 0000000000..629938fd65 --- /dev/null +++ b/internal/watchsync/ratings_test.go @@ -0,0 +1,916 @@ +package watchsync + +import ( + "context" + "sort" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/catalog" + "github.com/Silo-Server/silo-server/internal/historyimport" + "github.com/Silo-Server/silo-server/internal/userstore" +) + +const ( + ratingTestUserID = 7 + ratingTestProfileID = "profile-1" + ratingTestConnID = "conn-ratings" + ratingTestMovieA = "movie-a" + ratingTestMovieB = "movie-b" + ratingTestSeries = "series-a" +) + +func TestRatingScaleConversion(t *testing.T) { + want := map[int]int{1: 1, 2: 1, 3: 2, 4: 2, 5: 3, 6: 3, 7: 4, 8: 4, 9: 5, 10: 5, 0: 1, 11: 5} + for rating, stars := range want { + if got := starsFromProviderRating(rating); got != stars { + t.Errorf("starsFromProviderRating(%d) = %d, want %d", rating, got, stars) + } + } + for stars := 1; stars <= 5; stars++ { + if got := starsFromProviderRating(providerRatingFromStars(stars)); got != stars { + t.Errorf("stars %d do not round-trip through the provider scale (got %d)", stars, got) + } + } +} + +func TestDecideRating(t *testing.T) { + older := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + newer := older.Add(time.Hour) + for _, tc := range []struct { + name string + local, remote, base int + localAt, remoteAt time.Time + want ratingAction + }{ + {name: "all agree", local: 4, remote: 4, base: 4, want: ratingKeep}, + {name: "all unrated", want: ratingKeep}, + {name: "first sync equal values rebase", local: 4, remote: 4, want: ratingRebase}, + {name: "both removed rebase", base: 3, want: ratingRebase}, + {name: "only local changed", local: 2, remote: 4, base: 4, want: ratingExport}, + {name: "local removed", remote: 4, base: 4, want: ratingExport}, + {name: "only remote changed", local: 4, remote: 5, base: 4, want: ratingImport}, + {name: "remote removed", local: 4, base: 4, want: ratingImport}, + {name: "first sync local only", local: 3, want: ratingExport}, + {name: "first sync remote only", remote: 3, want: ratingImport}, + {name: "conflict local removal loses", remote: 5, base: 4, localAt: newer, remoteAt: older, want: ratingImport}, + {name: "conflict remote removal loses", local: 5, base: 4, localAt: older, remoteAt: newer, want: ratingExport}, + {name: "conflict newer remote wins", local: 2, remote: 5, base: 4, localAt: older, remoteAt: newer, want: ratingImport}, + {name: "conflict newer local wins", local: 2, remote: 5, base: 4, localAt: newer, remoteAt: older, want: ratingExport}, + {name: "conflict tie goes to silo", local: 2, remote: 5, base: 4, localAt: older, remoteAt: older, want: ratingExport}, + {name: "conflict unknown remote time goes to silo", local: 2, remote: 5, base: 4, localAt: older, want: ratingExport}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := decideRating(tc.local, tc.remote, tc.base, tc.localAt, tc.remoteAt); got != tc.want { + t.Fatalf("decideRating(%d, %d, %d) = %d, want %d", tc.local, tc.remote, tc.base, got, tc.want) + } + }) + } +} + +func TestSyncRatingsFirstSyncUnionsBothSides(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 3) // Silo only + h.provider.batch = RatingImportBatch{ + Rows: []RemoteRating{ + h.remoteRow(ratingTestMovieB, 7), // provider only + h.remoteRow(ratingTestSeries, 10), + }, + SnapshotKinds: []string{historyimport.KindMovie, historyimport.KindSeries}, + } + + result := h.sync() + + if got := h.store.stars(ratingTestMovieB); got != 4 { + t.Fatalf("imported movie B = %d stars, want 4", got) + } + if got := h.store.stars(ratingTestSeries); got != 5 { + t.Fatalf("imported series = %d stars, want 5", got) + } + if len(h.provider.exported) != 1 || h.provider.exported[0].MediaItemID != ratingTestMovieA || h.provider.exported[0].Rating != 6 { + t.Fatalf("exported = %#v, want movie A at 6", h.provider.exported) + } + if result.Imported != 2 || result.Sent != 1 || result.RemoteFound != 2 || result.LocalFound != 1 { + t.Fatalf("result = %#v", result) + } + if !h.stale { + t.Fatal("imports must mark the profile's recommendations stale") + } + if s := h.state(ratingTestMovieB); s == nil || s.SyncedRating != 4 || !s.RemoteSeen { + t.Fatalf("imported base = %#v, want 4 stars seen", s) + } + if s := h.state(ratingTestMovieA); s == nil || s.SyncedRating != 3 || s.RemoteSeen { + t.Fatalf("exported base = %#v, want 3 stars not yet seen", s) + } +} + +func TestSyncRatingsEquivalentValuesRebaseWithoutWrites(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 4) + h.provider.batch = RatingImportBatch{ + Rows: []RemoteRating{h.remoteRow(ratingTestMovieA, 7)}, + SnapshotKinds: []string{historyimport.KindMovie}, + } + + h.sync() + + if len(h.provider.exported) != 0 || len(h.provider.removed) != 0 { + t.Fatalf("a remote 7 and 4 stars agree; provider writes = %#v / %#v", h.provider.exported, h.provider.removed) + } + if s := h.state(ratingTestMovieA); s == nil || s.SyncedRating != 4 || !s.RemoteSeen { + t.Fatalf("base = %#v, want 4 stars seen", s) + } + + // A remote change inside the same star is not a change. + h.provider.batch.Rows = []RemoteRating{h.remoteRow(ratingTestMovieA, 8)} + h.sync() + if len(h.provider.exported) != 0 || h.store.stars(ratingTestMovieA) != 4 { + t.Fatalf("7 to 8 must not write either side; exported=%#v local=%d", h.provider.exported, h.store.stars(ratingTestMovieA)) + } +} + +func TestSyncRatingsPropagatesChangesAfterAgreement(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 4) + h.store.set(ratingTestMovieB, 2) + h.agree(ratingTestMovieA, 4, true) + h.agree(ratingTestMovieB, 2, true) + h.store.set(ratingTestMovieA, 1) // changed in Silo + h.provider.batch = RatingImportBatch{ + Rows: []RemoteRating{ + h.remoteRow(ratingTestMovieA, 8), + h.remoteRow(ratingTestMovieB, 10), // changed on the provider + }, + SnapshotKinds: []string{historyimport.KindMovie}, + } + + h.sync() + + if len(h.provider.exported) != 1 || h.provider.exported[0].MediaItemID != ratingTestMovieA || h.provider.exported[0].Rating != 2 { + t.Fatalf("exported = %#v, want movie A at 2", h.provider.exported) + } + if got := h.store.stars(ratingTestMovieB); got != 5 { + t.Fatalf("movie B = %d stars, want 5", got) + } +} + +func TestSyncRatingsRemovalsFollowTheChangedSide(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 4) + h.store.set(ratingTestMovieB, 3) + h.agree(ratingTestMovieA, 4, true) + h.agree(ratingTestMovieB, 3, true) + h.store.remove(ratingTestMovieB) // removed in Silo + // Movie A is absent from a complete snapshot: removed on the provider. + h.provider.batch = RatingImportBatch{ + Rows: []RemoteRating{h.remoteRow(ratingTestMovieB, 6)}, + SnapshotKinds: []string{historyimport.KindMovie}, + } + + h.sync() + + if got := h.store.stars(ratingTestMovieA); got != 0 { + t.Fatalf("movie A = %d stars, want removed", got) + } + if len(h.provider.removed) != 1 || h.provider.removed[0].MediaItemID != ratingTestMovieB { + t.Fatalf("removed = %#v, want movie B", h.provider.removed) + } + if h.state(ratingTestMovieA) != nil { + t.Fatalf("an imported removal must clear the agreed rating: %#v", h.state(ratingTestMovieA)) + } + // A sent removal stays agreed until a read confirms the title is unrated. + if h.state(ratingTestMovieB) == nil { + t.Fatal("a sent removal must keep its agreed rating until a read confirms it") + } + + h.provider.batch = RatingImportBatch{ + Rows: []RemoteRating{h.remoteRow(ratingTestSeries, 2)}, + SnapshotKinds: []string{historyimport.KindMovie}, + } + h.sync() + if h.state(ratingTestMovieB) != nil || len(h.provider.removed) != 0 { + t.Fatalf("a confirmed removal must clear the agreed rating without resending: state=%#v removed=%#v", h.state(ratingTestMovieB), h.provider.removed) + } +} + +func TestSyncRatingsResendsARemovalTheProviderDidNotFullyApply(t *testing.T) { + h := newRatingHarness(t) + h.agree(ratingTestMovieA, 4, true) + // Silo removed the rating and sent the removal, but another provider entry + // for the same title is still rated. + h.provider.batch = RatingImportBatch{ + Rows: []RemoteRating{h.remoteRow(ratingTestMovieA, 8)}, + SnapshotKinds: []string{historyimport.KindMovie}, + } + h.sync() + if got := h.store.stars(ratingTestMovieA); got != 0 { + t.Fatalf("the removed rating came back as %d stars", got) + } + if len(h.provider.removed) != 1 { + t.Fatalf("removed = %#v, want the removal sent again", h.provider.removed) + } +} + +func TestSyncRatingsUnseenAbsenceResendsInsteadOfDeleting(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 4) + // Silo sent this rating, but no provider read has confirmed it yet. + h.agree(ratingTestMovieA, 4, false) + h.provider.batch = RatingImportBatch{ + Rows: []RemoteRating{h.remoteRow(ratingTestMovieB, 6)}, + SnapshotKinds: []string{historyimport.KindMovie}, + } + + h.sync() + + if got := h.store.stars(ratingTestMovieA); got != 4 { + t.Fatalf("an unconfirmed rating must not be deleted locally; got %d stars", got) + } + if len(h.provider.exported) != 1 || h.provider.exported[0].MediaItemID != ratingTestMovieA { + t.Fatalf("exported = %#v, want movie A resent", h.provider.exported) + } +} + +func TestSyncRatingsAbsenceOutsideSnapshotKindsIsUnknown(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestSeries, 4) + h.agree(ratingTestSeries, 4, true) + // The provider skipped series (unchanged), so their absence means nothing. + h.provider.batch = RatingImportBatch{ + Rows: []RemoteRating{h.remoteRow(ratingTestMovieB, 6)}, + SnapshotKinds: []string{historyimport.KindMovie}, + } + + h.sync() + + if got := h.store.stars(ratingTestSeries); got != 4 { + t.Fatalf("series = %d stars, want unchanged 4", got) + } + if len(h.provider.exported) != 0 || len(h.provider.removed) != 0 { + t.Fatalf("provider writes = %#v / %#v, want none", h.provider.exported, h.provider.removed) + } +} + +func TestSyncRatingsEmptySnapshotDoesNotDelete(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 4) + h.store.set(ratingTestMovieB, 2) + h.agree(ratingTestMovieA, 4, true) + h.agree(ratingTestMovieB, 2, true) + h.provider.batch = RatingImportBatch{SnapshotKinds: []string{historyimport.KindMovie}} + + result := h.sync() + + if h.store.stars(ratingTestMovieA) != 4 || h.store.stars(ratingTestMovieB) != 2 { + t.Fatalf("an empty snapshot must not delete several ratings: A=%d B=%d", h.store.stars(ratingTestMovieA), h.store.stars(ratingTestMovieB)) + } + if len(result.Warnings) == 0 { + t.Fatal("an ignored empty snapshot should warn") + } +} + +func TestSyncRatingsEmptySnapshotRemovesTheLastRating(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 4) + h.agree(ratingTestMovieA, 4, true) + h.provider.batch = RatingImportBatch{SnapshotKinds: []string{historyimport.KindMovie}} + + h.sync() + + if got := h.store.stars(ratingTestMovieA); got != 0 { + t.Fatalf("removing the only rating on the provider must import; got %d stars", got) + } +} + +func TestSyncRatingsUnmatchedRowSharingAnIDIsNotARemoval(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 4) + h.agree(ratingTestMovieA, 4, true) + // The provider still lists the title, but the matcher cannot place the row: + // it carries only an IMDb id, which this matcher does not use. + row := h.remoteRow(ratingTestMovieA, 8) + row.TMDBID = "" + row.ProviderItemKey = "" + row.IMDbID = "tt0101" + h.provider.batch = RatingImportBatch{Rows: []RemoteRating{row}, SnapshotKinds: []string{historyimport.KindMovie}} + + h.sync() + + if got := h.store.stars(ratingTestMovieA); got != 4 { + t.Fatalf("movie A = %d stars, want unchanged 4", got) + } + if len(h.provider.removed) != 0 { + t.Fatalf("removed = %#v, want none", h.provider.removed) + } +} + +func TestSyncRatingsTombstoneRemovesByProviderKey(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 4) + h.agree(ratingTestMovieA, 4, false) + tombstone := RemoteRating{RemoteFavorite: RemoteFavorite{ProviderItemKey: h.media[ratingTestMovieA].ProviderItemKey, Removed: true}} + h.provider.batch = RatingImportBatch{Rows: []RemoteRating{tombstone}} + + h.sync() + + if got := h.store.stars(ratingTestMovieA); got != 0 { + t.Fatalf("movie A = %d stars, want removed by tombstone", got) + } +} + +func TestSyncRatingsRemembersTheProviderKeyForTombstonesAndWrites(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 4) + row := h.remoteRow(ratingTestMovieA, 8) + row.ProviderItemKey = "floppy:movie:101" + h.provider.batch = RatingImportBatch{Rows: []RemoteRating{row}, SnapshotKinds: []string{historyimport.KindMovie}} + h.sync() + if s := h.state(ratingTestMovieA); s == nil || s.ProviderItemKey != "floppy:movie:101" { + t.Fatalf("agreed row = %#v, want the provider's own key", s) + } + + // A later local change is sent under the provider's key. + h.store.set(ratingTestMovieA, 2) + h.provider.batch = RatingImportBatch{} + h.sync() + if len(h.provider.exported) != 1 || h.provider.exported[0].ProviderItemKey != "floppy:movie:101" { + t.Fatalf("exported = %#v, want the provider's key", h.provider.exported) + } + + // An incremental tombstone naming that key removes the rating. + h.agree(ratingTestMovieA, 2, true) + h.repo.ratingStates[0].ProviderItemKey = "floppy:movie:101" + tombstone := RemoteRating{RemoteFavorite: RemoteFavorite{ProviderItemKey: "floppy:movie:101", Removed: true}} + h.provider.batch = RatingImportBatch{Rows: []RemoteRating{tombstone}} + h.sync() + if got := h.store.stars(ratingTestMovieA); got != 0 { + t.Fatalf("movie A = %d stars, want removed by the provider's tombstone", got) + } +} + +func TestSyncRatingsSkipsKindsTheProviderDoesNotRate(t *testing.T) { + h := newRatingHarness(t) + h.provider.kinds = map[string]bool{historyimport.KindMovie: true} + h.store.set(ratingTestSeries, 3) + h.store.set(ratingTestMovieA, 4) + h.agree(ratingTestMovieA, 4, true) + + result := h.sync() + + if len(h.provider.exported) != 0 || len(result.Warnings) != 0 { + t.Fatalf("a movie-only provider must not be sent series ratings: exported=%#v warnings=%v", h.provider.exported, result.Warnings) + } + if result.LocalFound != 1 { + t.Fatalf("local found = %d, want only the movie", result.LocalFound) + } +} + +func TestSyncRatingsConcurrentLocalEditWins(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 4) + h.agree(ratingTestMovieA, 4, true) + h.store.conflicts[ratingTestMovieA] = true // the user edits between read and write + h.provider.batch = RatingImportBatch{ + Rows: []RemoteRating{h.remoteRow(ratingTestMovieA, 2)}, + SnapshotKinds: []string{historyimport.KindMovie}, + } + + result := h.sync() + + if result.Imported != 0 { + t.Fatalf("a lost compare-and-set must not count as imported: %#v", result) + } + if s := h.state(ratingTestMovieA); s == nil || s.SyncedRating != 4 { + t.Fatalf("a lost compare-and-set must leave the agreed rating: %#v", s) + } +} + +func TestSyncRatingsDirectionToggles(t *testing.T) { + t.Run("import only", func(t *testing.T) { + h := newRatingHarness(t) + h.conn.ExportRatingsEnabled = false + h.store.set(ratingTestMovieA, 3) + h.provider.batch = RatingImportBatch{Rows: []RemoteRating{h.remoteRow(ratingTestMovieB, 7)}, SnapshotKinds: []string{historyimport.KindMovie}} + h.sync() + if len(h.provider.exported) != 0 || h.store.stars(ratingTestMovieB) != 4 { + t.Fatalf("exported=%#v movieB=%d", h.provider.exported, h.store.stars(ratingTestMovieB)) + } + if h.state(ratingTestMovieA) != nil { + t.Fatal("a blocked export must not record agreement") + } + }) + t.Run("export only", func(t *testing.T) { + h := newRatingHarness(t) + h.conn.ImportRatingsEnabled = false + h.store.set(ratingTestMovieA, 3) + h.provider.batch = RatingImportBatch{ + Rows: []RemoteRating{h.remoteRow(ratingTestMovieB, 7)}, + SnapshotKinds: []string{historyimport.KindMovie}, + UpdatedCursors: map[string]string{"test.ratings.movies": "c1"}, + } + h.sync() + if h.store.stars(ratingTestMovieB) != 0 || len(h.provider.exported) != 1 { + t.Fatalf("exported=%#v movieB=%d", h.provider.exported, h.store.stars(ratingTestMovieB)) + } + cursors := h.repo.connections[connectionKey(h.conn.Provider, h.conn.UserID, h.conn.ProfileID)].SyncCursors + if cursors["test.ratings.movies"] != "c1" || cursors[ratingImportCursorKey] != "" { + t.Fatalf("send-only cursors = %#v, want the read cursor saved without the import marker", cursors) + } + + // Turning import on reads everything again: cursors saved in + // send-only mode may have skipped changes that were never imported. + h.conn = h.repo.connections[connectionKey(h.conn.Provider, h.conn.UserID, h.conn.ProfileID)] + h.conn.ImportRatingsEnabled = true + h.sync() + if h.provider.fetchedCursors["test.ratings.movies"] != "" { + t.Fatalf("the first import read got cursors %#v, want none", h.provider.fetchedCursors) + } + if marker := h.repo.connections[connectionKey(h.conn.Provider, h.conn.UserID, h.conn.ProfileID)].SyncCursors[ratingImportCursorKey]; marker == "" { + t.Fatal("an import read must set the import marker") + } + }) +} + +func TestSyncRatingsPersistsCursorsWhenImporting(t *testing.T) { + h := newRatingHarness(t) + h.provider.batch = RatingImportBatch{UpdatedCursors: map[string]string{"test.ratings.movies": "c1"}} + h.sync() + if cursor := h.repo.connections[connectionKey(h.conn.Provider, h.conn.UserID, h.conn.ProfileID)].SyncCursors["test.ratings.movies"]; cursor != "c1" { + t.Fatalf("cursor = %q, want c1", cursor) + } +} + +func TestSyncRatingsWatchGateHoldsUnwatchedMovies(t *testing.T) { + h := newRatingHarness(t) + h.provider.gateMovies = true + h.store.set(ratingTestMovieA, 4) + h.store.set(ratingTestMovieB, 5) + h.watched[ratingTestMovieB] = true + h.provider.batch = RatingImportBatch{SnapshotKinds: []string{historyimport.KindMovie}} + + result := h.sync() + + if len(h.provider.exported) != 1 || h.provider.exported[0].MediaItemID != ratingTestMovieB { + t.Fatalf("exported = %#v, want only the watched movie", h.provider.exported) + } + if h.state(ratingTestMovieA) != nil || len(result.Warnings) == 0 { + t.Fatalf("a held rating stays pending with a warning: state=%#v warnings=%v", h.state(ratingTestMovieA), result.Warnings) + } +} + +func TestSyncRatingsRateLimitStopsSending(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 4) + h.provider.exportErr = RateLimitedError{Provider: "test", RetryAfter: time.Minute} + + _, err := h.service.syncRatings(context.Background(), h.conn, ServerConfig{}, h.provider) + + if _, ok := AsRateLimited(err); !ok { + t.Fatalf("err = %v, want rate limited", err) + } + if h.state(ratingTestMovieA) != nil { + t.Fatal("a rate-limited rating stays pending") + } +} + +func TestLocalRatingEventSendsCurrentValue(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 4) + h.agree(ratingTestMovieA, 4, true) + h.store.set(ratingTestMovieA, 2) + h.store.set(ratingTestMovieB, 5) // not part of the event + + err := h.service.processLocalRatingEvent(context.Background(), LocalRatingEvent{ + UserID: ratingTestUserID, ProfileID: ratingTestProfileID, MediaItemIDs: []string{ratingTestMovieA}, + }) + if err != nil { + t.Fatal(err) + } + if len(h.provider.exported) != 1 || h.provider.exported[0].MediaItemID != ratingTestMovieA || h.provider.exported[0].Rating != 4 { + t.Fatalf("exported = %#v, want only movie A at 4", h.provider.exported) + } + if h.provider.fetches != 0 { + t.Fatal("a local event must not read the provider") + } + + // A removal waits for the scheduled merge, which can see whether the + // provider changed the rating in the meantime. + h.store.remove(ratingTestMovieA) + if err := h.service.processLocalRatingEvent(context.Background(), LocalRatingEvent{ + UserID: ratingTestUserID, ProfileID: ratingTestProfileID, MediaItemIDs: []string{ratingTestMovieA}, + }); err != nil { + t.Fatal(err) + } + if len(h.provider.removed) != 0 { + t.Fatalf("removed = %#v, want the removal left to the scheduled merge", h.provider.removed) + } +} + +func TestSyncRatingsNeverRemovesARatingItSkipped(t *testing.T) { + h := newRatingHarness(t) + // Rated and agreed, but the media item has left the catalog. + h.store.set("gone-movie", 4) + h.repo.ratingStates = append(h.repo.ratingStates, RatingSyncState{ + ConnectionID: h.conn.ID, ProviderAccountID: h.conn.ProviderAccountID, MediaItemID: "gone-movie", + Kind: historyimport.KindMovie, ProviderItemKey: "tmdb:999", SyncedRating: 4, RemoteSeen: true, + }) + h.provider.batch = RatingImportBatch{SnapshotKinds: []string{historyimport.KindMovie}} + h.sync() + if len(h.provider.removed) != 0 { + t.Fatalf("removed = %#v, want no removal for a rating Silo still holds", h.provider.removed) + } +} + +func TestSyncRatingsIgnoresAgreedRatingsOfAnotherAccount(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 4) + // Agreed with the connection's previous account, then absent here. + h.repo.ratingStates = append(h.repo.ratingStates, RatingSyncState{ + ConnectionID: h.conn.ID, ProviderAccountID: "old-account", MediaItemID: ratingTestMovieA, + Kind: historyimport.KindMovie, ProviderItemKey: "tmdb:101", SyncedRating: 4, RemoteSeen: true, + }) + h.provider.batch = RatingImportBatch{Rows: []RemoteRating{h.remoteRow(ratingTestMovieB, 6)}, SnapshotKinds: []string{historyimport.KindMovie}} + h.sync() + if got := h.store.stars(ratingTestMovieA); got != 4 { + t.Fatalf("movie A = %d stars; another account's agreement must not delete it", got) + } + if len(h.provider.exported) != 1 || h.provider.exported[0].MediaItemID != ratingTestMovieA { + t.Fatalf("exported = %#v, want movie A sent to the new account", h.provider.exported) + } +} + +func TestSyncRatingsDoesNotAgreeOnAValueChangedDuringTheSend(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 3) + h.provider.onExport = func() { h.store.set(ratingTestMovieA, 5) } + h.provider.batch = RatingImportBatch{SnapshotKinds: []string{historyimport.KindMovie}} + h.sync() + if s := h.state(ratingTestMovieA); s != nil { + t.Fatalf("agreed row = %#v; a rating changed during the send must stay pending", s) + } +} + +func TestSyncRatingsKeepsTitlesOfUnusableRows(t *testing.T) { + for _, tc := range []struct { + name string + row func(h *ratingHarness) RemoteRating + }{ + {"out of range", func(h *ratingHarness) RemoteRating { + row := h.remoteRow(ratingTestMovieA, 11) + return row + }}, + {"provider ids only", func(h *ratingHarness) RemoteRating { + return RemoteRating{RemoteFavorite: RemoteFavorite{ProviderItemKey: "mdblist:abc", Kind: historyimport.KindMovie}, Rating: 7} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 4) + h.agree(ratingTestMovieA, 4, true) + h.provider.batch = RatingImportBatch{Rows: []RemoteRating{tc.row(h)}, SnapshotKinds: []string{historyimport.KindMovie}} + h.sync() + if got := h.store.stars(ratingTestMovieA); got != 4 { + t.Fatalf("movie A = %d stars; an unusable row must not turn its title into a removal", got) + } + }) + } +} + +func TestSyncRatingsSkipsAmbiguousTombstones(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 4) + h.store.set(ratingTestSeries, 3) + h.agree(ratingTestMovieA, 4, true) + h.agree(ratingTestSeries, 3, true) + // The movie and the series both recorded the key tmdb:101. + tombstone := RemoteRating{RemoteFavorite: RemoteFavorite{ProviderItemKey: "tmdb:101", Removed: true}} + h.provider.batch = RatingImportBatch{Rows: []RemoteRating{tombstone}} + result := h.sync() + if h.store.stars(ratingTestMovieA) != 4 || h.store.stars(ratingTestSeries) != 3 || len(result.Warnings) == 0 { + t.Fatalf("an ambiguous tombstone must remove nothing: movie=%d series=%d warnings=%v", + h.store.stars(ratingTestMovieA), h.store.stars(ratingTestSeries), result.Warnings) + } +} + +func TestSyncRatingsWatchGateLetsHeldTitlesChange(t *testing.T) { + h := newRatingHarness(t) + h.provider.gateMovies = true + h.store.set(ratingTestMovieA, 4) + h.agree(ratingTestMovieA, 4, true) + h.store.set(ratingTestMovieA, 2) + h.provider.batch = RatingImportBatch{Rows: []RemoteRating{h.remoteRow(ratingTestMovieA, 8)}, SnapshotKinds: []string{historyimport.KindMovie}} + h.sync() + if len(h.provider.exported) != 1 { + t.Fatalf("exported = %#v; a title the provider already rates is not held for a watch", h.provider.exported) + } +} + +func TestExecuteSyncRunRecordsRatingCounters(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 3) + h.provider.batch = RatingImportBatch{Rows: []RemoteRating{h.remoteRow(ratingTestMovieB, 7)}, SnapshotKinds: []string{historyimport.KindMovie}} + + created, err := h.repo.CreateSyncRun(context.Background(), SyncRun{ConnectionID: h.conn.ID, Provider: h.conn.Provider}) + if err != nil { + t.Fatal(err) + } + run, err := h.service.executeSyncRun(context.Background(), h.conn, created) + if err != nil { + t.Fatal(err) + } + if run.InboundRatingsFound != 1 || run.InboundRatingsImported != 1 || run.OutboundRatingsFound != 1 || run.OutboundRatingsSent != 1 { + t.Fatalf("run counters = %#v", run) + } +} + +func TestPersistConnectionEnablesRatingsForNewConnections(t *testing.T) { + repo := newServiceFakeRepo() + service := NewService(repo, NewRegistry()) + conn, err := service.persistConnection(context.Background(), "test", 1, "p", TokenSet{AccessToken: "a"}, ProviderAccount{ID: "acct"}) + if err != nil { + t.Fatal(err) + } + if !conn.ImportRatingsEnabled || !conn.ExportRatingsEnabled { + t.Fatalf("new connection rating toggles = %v/%v, want on", conn.ImportRatingsEnabled, conn.ExportRatingsEnabled) + } +} + +func TestPersistConnectionAccountChangeClearsAgreedRatings(t *testing.T) { + repo := newServiceFakeRepo() + existing := Connection{ + ID: ratingTestConnID, Provider: "test", UserID: 1, ProfileID: "p", ProviderAccountID: "old", + SyncCursors: map[string]string{"test.ratings.movies": "c1", "test.watched": "w1"}, + } + repo.connections[connectionKey("test", 1, "p")] = existing + repo.ratingStates = []RatingSyncState{{ConnectionID: ratingTestConnID, MediaItemID: ratingTestMovieA, SyncedRating: 4, RemoteSeen: true}} + service := NewService(repo, NewRegistry()) + + conn, err := service.persistConnection(context.Background(), "test", 1, "p", TokenSet{AccessToken: "a"}, ProviderAccount{ID: "new"}) + if err != nil { + t.Fatal(err) + } + if len(repo.ratingStates) != 0 { + t.Fatalf("agreed ratings survived an account change: %#v", repo.ratingStates) + } + if conn.SyncCursors["test.ratings.movies"] != "" || conn.SyncCursors["test.watched"] != "w1" { + t.Fatalf("cursors = %#v, want only rating cursors dropped", conn.SyncCursors) + } + + // Reconnecting the same account keeps the agreement. + repo.ratingStates = []RatingSyncState{{ConnectionID: ratingTestConnID, MediaItemID: ratingTestMovieA, SyncedRating: 4, RemoteSeen: true}} + if _, err := service.persistConnection(context.Background(), "test", 1, "p", TokenSet{AccessToken: "a"}, ProviderAccount{ID: "new"}); err != nil { + t.Fatal(err) + } + if len(repo.ratingStates) != 1 { + t.Fatal("reconnecting the same account must keep agreed ratings") + } +} + +// --- harness --- + +type ratingHarness struct { + t *testing.T + repo *serviceFakeRepo + store *fakeRatingStore + provider *ratingProviderStub + service *Service + conn Connection + media map[string]LocalFavorite + watched map[string]bool + stale bool +} + +func newRatingHarness(t *testing.T) *ratingHarness { + t.Helper() + h := &ratingHarness{ + t: t, + repo: newServiceFakeRepo(), + store: newFakeRatingStore(), + provider: &ratingProviderStub{}, + watched: map[string]bool{}, + media: map[string]LocalFavorite{ + ratingTestMovieA: {MediaItemID: ratingTestMovieA, Kind: historyimport.KindMovie, IMDbID: "tt0101", TMDBID: "101", ProviderItemKey: "tmdb:101"}, + ratingTestMovieB: {MediaItemID: ratingTestMovieB, Kind: historyimport.KindMovie, TMDBID: "102", ProviderItemKey: "tmdb:102"}, + ratingTestSeries: {MediaItemID: ratingTestSeries, Kind: historyimport.KindSeries, TMDBID: "101", ProviderItemKey: "tmdb:101"}, + }, + } + h.repo.listMedia = h.media + h.conn = Connection{ + ID: ratingTestConnID, Provider: h.provider.Key(), UserID: ratingTestUserID, ProfileID: ratingTestProfileID, + AccessToken: "token", ProviderAccountID: "acct", ImportRatingsEnabled: true, ExportRatingsEnabled: true, + } + h.repo.connections[connectionKey(h.conn.Provider, h.conn.UserID, h.conn.ProfileID)] = h.conn + registry := NewRegistry() + if err := registry.Register(h.provider); err != nil { + t.Fatal(err) + } + h.service = NewService(h.repo, registry). + WithMatcher(ratingMatcherStub{media: h.media}). + WithUserStoreProvider(ratingHistoryStoreProvider{watched: h.watched}). + WithRatingStore(h.store, ratingStalerFunc(func() { h.stale = true })) + return h +} + +func (h *ratingHarness) sync() SyncRatingsResult { + h.t.Helper() + h.provider.exported, h.provider.removed = nil, nil + result, err := h.service.syncRatings(context.Background(), h.conn, ServerConfig{}, h.provider) + if err != nil { + h.t.Fatal(err) + } + return result +} + +func (h *ratingHarness) remoteRow(mediaItemID string, rating int) RemoteRating { + item := h.media[mediaItemID] + return RemoteRating{ + RemoteFavorite: RemoteFavorite{Provider: "test", ProviderItemKey: item.ProviderItemKey, Kind: item.Kind, TMDBID: item.TMDBID}, + Rating: rating, + RatedAt: time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC), + } +} + +func (h *ratingHarness) agree(mediaItemID string, stars int, seen bool) { + item := h.media[mediaItemID] + _ = h.repo.UpsertRatingSyncStates(context.Background(), []RatingSyncState{{ + ConnectionID: h.conn.ID, ProviderAccountID: h.conn.ProviderAccountID, MediaItemID: mediaItemID, Kind: item.Kind, + ProviderItemKey: item.ProviderItemKey, SyncedRating: stars, RemoteSeen: seen, + }}) +} + +func (h *ratingHarness) state(mediaItemID string) *RatingSyncState { + for i := range h.repo.ratingStates { + if h.repo.ratingStates[i].MediaItemID == mediaItemID { + return &h.repo.ratingStates[i] + } + } + return nil +} + +type fakeRatingStore struct { + ratings map[string]catalog.UserRating + conflicts map[string]bool + clock time.Time +} + +func newFakeRatingStore() *fakeRatingStore { + return &fakeRatingStore{ + ratings: map[string]catalog.UserRating{}, + conflicts: map[string]bool{}, + clock: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), + } +} + +func (s *fakeRatingStore) set(id string, stars int) { + s.clock = s.clock.Add(time.Minute) + s.ratings[id] = catalog.UserRating{UserID: ratingTestUserID, ProfileID: ratingTestProfileID, MediaItemID: id, Rating: stars, RatedAt: s.clock} +} + +func (s *fakeRatingStore) remove(id string) { delete(s.ratings, id) } + +func (s *fakeRatingStore) stars(id string) int { return s.ratings[id].Rating } + +func (s *fakeRatingStore) ListAll(_ context.Context, _ int, _ string) ([]catalog.UserRating, error) { + all := make([]catalog.UserRating, 0, len(s.ratings)) + for _, rating := range s.ratings { + all = append(all, rating) + } + sort.Slice(all, func(i, j int) bool { return all[i].MediaItemID < all[j].MediaItemID }) + return all, nil +} + +func (s *fakeRatingStore) Get(_ context.Context, _ int, _ string, id string) (*catalog.UserRating, error) { + rating, ok := s.ratings[id] + if !ok { + return nil, nil + } + return &rating, nil +} + +func (s *fakeRatingStore) SetIfUnchanged(_ context.Context, _ int, _ string, id string, expected, rating int, ratedAt time.Time) (bool, error) { + if s.conflicts[id] || s.stars(id) != expected { + return false, nil + } + s.ratings[id] = catalog.UserRating{UserID: ratingTestUserID, ProfileID: ratingTestProfileID, MediaItemID: id, Rating: rating, RatedAt: ratedAt} + return true, nil +} + +func (s *fakeRatingStore) DeleteIfUnchanged(_ context.Context, _ int, _ string, id string, expected int) (bool, error) { + if s.conflicts[id] || s.stars(id) != expected { + return false, nil + } + delete(s.ratings, id) + return true, nil +} + +type ratingStalerFunc func() + +func (f ratingStalerFunc) MarkProfileStale(context.Context, int, string) error { + f() + return nil +} + +// ratingMatcherStub matches remote rows by kind and TMDB id. +type ratingMatcherStub struct { + media map[string]LocalFavorite +} + +func (m ratingMatcherStub) Match(_ context.Context, record historyimport.Record) (*historyimport.Match, string, error) { + for _, item := range m.media { + if record.TMDBID != "" && item.Kind == record.Kind && item.TMDBID == record.TMDBID { + return &historyimport.Match{MediaItemID: item.MediaItemID}, "", nil + } + } + return nil, "no match", nil +} + +type ratingHistoryStoreProvider struct { + watched map[string]bool +} + +func (p ratingHistoryStoreProvider) ForUser(context.Context, int) (userstore.UserStore, error) { + return ratingHistoryStore{watched: p.watched}, nil +} + +func (ratingHistoryStoreProvider) Close() error { return nil } + +// ratingHistoryStore answers completed-history lookups for the watch gate. +type ratingHistoryStore struct { + userstore.UserStore + watched map[string]bool +} + +func (s ratingHistoryStore) ListCompletedHistory(_ context.Context, query userstore.CompletedHistoryQuery) ([]userstore.WatchHistoryEntry, error) { + if query.Offset > 0 { + return nil, nil + } + var rows []userstore.WatchHistoryEntry + for _, id := range query.MediaItemIDs { + if s.watched[id] { + rows = append(rows, userstore.WatchHistoryEntry{MediaItemID: id}) + } + } + return rows, nil +} + +type ratingProviderStub struct { + batch RatingImportBatch + fetches int + exported []LocalRating + removed []LocalFavorite + exportErr error + gateMovies bool + onExport func() + // fetchedCursors are the cursors the last FetchRatings call received. + fetchedCursors map[string]string + // kinds limits the rated kinds when set; nil rates every kind. + kinds map[string]bool +} + +func (p *ratingProviderStub) SyncsRatingKind(kind string) bool { + return p.kinds == nil || p.kinds[kind] +} + +func (*ratingProviderStub) Key() string { return "test" } +func (*ratingProviderStub) DisplayName() string { return "Test" } +func (*ratingProviderStub) Capabilities() Capabilities { + return Capabilities{ImportRatings: true, ExportRatings: true} +} + +func (*ratingProviderStub) ConnectWithAPIKey(context.Context, string) (TokenSet, ProviderAccount, error) { + return TokenSet{}, ProviderAccount{}, nil +} + +func (p *ratingProviderStub) FetchRatings(_ context.Context, _ ServerConfig, conn Connection) (RatingImportBatch, error) { + p.fetches++ + p.fetchedCursors = conn.SyncCursors + return p.batch, nil +} + +func (p *ratingProviderStub) ExportRatings(_ context.Context, _ ServerConfig, _ Connection, items []LocalRating) (ExportResult, error) { + if p.exportErr != nil { + return ExportResult{}, p.exportErr + } + p.exported = append(p.exported, items...) + if p.onExport != nil { + p.onExport() + } + var result ExportResult + for _, item := range items { + result.Sent = append(result.Sent, item.MediaItemID, item.ProviderItemKey) + } + return result, nil +} + +func (p *ratingProviderStub) RemoveRatings(_ context.Context, _ ServerConfig, _ Connection, items []LocalFavorite) (ExportResult, error) { + p.removed = append(p.removed, items...) + var result ExportResult + for _, item := range items { + result.Sent = append(result.Sent, item.MediaItemID, item.ProviderItemKey) + } + return result, nil +} + +func (p *ratingProviderStub) RatingExportRequiresWatched(kind string) bool { + return p.gateMovies && kind == historyimport.KindMovie +} diff --git a/internal/watchsync/repository.go b/internal/watchsync/repository.go index fa224328a6..cce24aae9e 100644 --- a/internal/watchsync/repository.go +++ b/internal/watchsync/repository.go @@ -46,6 +46,11 @@ type Repository interface { MarkListItemRemoteRemoved(ctx context.Context, connectionID string, kind ListKind, mediaItemID string, removedAt time.Time) error MarkListItemLocalRemoved(ctx context.Context, connectionID string, kind ListKind, mediaItemID string, removedAt time.Time) error MarkListItemError(ctx context.Context, connectionID string, kind ListKind, mediaItemID, lastError string) error + ListRatingEventConnections(ctx context.Context, userID int, profileID string) ([]Connection, error) + ListRatingSyncStates(ctx context.Context, connectionID, providerAccountID string, mediaItemIDs []string) ([]RatingSyncState, error) + UpsertRatingSyncStates(ctx context.Context, states []RatingSyncState) error + DeleteRatingSyncStates(ctx context.Context, connectionID string, mediaItemIDs []string) error + ClearRatingSyncStates(ctx context.Context, connectionID string) error ListScrobbleConnections(ctx context.Context, userID int, profileID string) ([]Connection, error) UpsertScrobbleSession(ctx context.Context, event ScrobbleEvent, connectionID string, action string) error PrepareConfirmedScrobbleStop(ctx context.Context, event ScrobbleEvent, connectionID string, staleBefore time.Time) (confirmedStopPreparation, time.Time, error) @@ -76,7 +81,8 @@ const connectionColumns = ` import_progress_enabled, export_watched_enabled, export_unwatched_enabled, import_favorites_enabled, export_favorites_enabled, sync_favorite_removals_enabled, import_watchlist_enabled, export_watchlist_enabled, sync_watchlist_removals_enabled, - sync_watchlist_order_enabled, scrobble_enabled, last_inbound_sync_at, + sync_watchlist_order_enabled, scrobble_enabled, import_ratings_enabled, + export_ratings_enabled, last_inbound_sync_at, last_progress_sync_at, last_outbound_sync_at, last_favorites_sync_at, last_watchlist_sync_at, last_scrobble_error_at, last_error, rate_limited_until, sync_cursors, created_at, updated_at` @@ -92,6 +98,8 @@ const syncRunColumns = ` outbound_favorites_sent, favorite_removals_sent, inbound_watchlist_found, inbound_watchlist_imported, outbound_watchlist_found, outbound_watchlist_sent, watchlist_removals_sent, + inbound_ratings_found, inbound_ratings_imported, + outbound_ratings_found, outbound_ratings_sent, warning, error, started_at, completed_at, created_at` // listItemStateColumns is the canonical select column list for @@ -230,12 +238,13 @@ func (r *PostgresRepository) UpsertConnection(ctx context.Context, conn Connecti import_watchlist_enabled, export_watchlist_enabled, sync_watchlist_removals_enabled, sync_watchlist_order_enabled, scrobble_enabled, last_inbound_sync_at, last_progress_sync_at, last_outbound_sync_at, last_favorites_sync_at, last_watchlist_sync_at, last_scrobble_error_at, - last_error, rate_limited_until, sync_cursors + last_error, rate_limited_until, sync_cursors, import_ratings_enabled, export_ratings_enabled ) VALUES ( COALESCE(NULLIF($1, '')::uuid, gen_random_uuid()), $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, - $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31::jsonb + $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31::jsonb, + $32, $33 ) ON CONFLICT (provider, user_id, profile_id) DO UPDATE SET provider_account_id = EXCLUDED.provider_account_id, @@ -287,6 +296,8 @@ func (r *PostgresRepository) UpsertConnection(ctx context.Context, conn Connecti conn.LastError, conn.RateLimitedUntil, encodeSyncCursors(conn.SyncCursors), + conn.ImportRatingsEnabled, + conn.ExportRatingsEnabled, ) saved, err := r.scanConnection(row) if err != nil { @@ -369,6 +380,8 @@ func (r *PostgresRepository) ListConnectionsDueForSync( OR export_watchlist_enabled OR sync_watchlist_removals_enabled OR scrobble_enabled + OR import_ratings_enabled + OR export_ratings_enabled ) ORDER BY provider, user_id, profile_id `, now) @@ -433,12 +446,15 @@ func (r *PostgresRepository) CreateSyncRun(ctx context.Context, run SyncRun) (Sy outbound_favorites_sent, favorite_removals_sent, inbound_watchlist_found, inbound_watchlist_imported, outbound_watchlist_found, outbound_watchlist_sent, watchlist_removals_sent, - warning, error, started_at, completed_at + warning, error, started_at, completed_at, + inbound_ratings_found, inbound_ratings_imported, + outbound_ratings_found, outbound_ratings_sent ) VALUES ( $1::uuid, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, - $16, $17, $18, $19, $20, $21, $22, $23, $24 + $16, $17, $18, $19, $20, $21, $22, $23, $24, + $25, $26, $27, $28 ) RETURNING `+syncRunColumns+` `, run.ConnectionID, run.Trigger, run.Status, run.Provider, @@ -449,7 +465,9 @@ func (r *PostgresRepository) CreateSyncRun(ctx context.Context, run SyncRun) (Sy run.OutboundFavoritesSent, run.FavoriteRemovalsSent, run.InboundWatchlistFound, run.InboundWatchlistImported, run.OutboundWatchlistFound, run.OutboundWatchlistSent, run.WatchlistRemovalsSent, - run.Warning, run.Error, run.StartedAt, run.CompletedAt) + run.Warning, run.Error, run.StartedAt, run.CompletedAt, + run.InboundRatingsFound, run.InboundRatingsImported, + run.OutboundRatingsFound, run.OutboundRatingsSent) created, err := scanSyncRun(row) if err != nil { return SyncRun{}, fmt.Errorf("scan created watch provider sync run: %w", err) @@ -479,7 +497,11 @@ func (r *PostgresRepository) CompleteSyncRun(ctx context.Context, run SyncRun) ( watchlist_removals_sent = $18, warning = $19, error = $20, - completed_at = $21 + completed_at = $21, + inbound_ratings_found = $22, + inbound_ratings_imported = $23, + outbound_ratings_found = $24, + outbound_ratings_sent = $25 WHERE id = $1::uuid RETURNING `+syncRunColumns+` `, run.ID, run.Status, run.InboundWatchedFound, run.InboundWatchedImported, @@ -488,7 +510,9 @@ func (r *PostgresRepository) CompleteSyncRun(ctx context.Context, run SyncRun) ( run.OutboundFavoritesSent, run.FavoriteRemovalsSent, run.InboundWatchlistFound, run.InboundWatchlistImported, run.OutboundWatchlistFound, run.OutboundWatchlistSent, run.WatchlistRemovalsSent, - run.Warning, run.Error, run.CompletedAt) + run.Warning, run.Error, run.CompletedAt, + run.InboundRatingsFound, run.InboundRatingsImported, + run.OutboundRatingsFound, run.OutboundRatingsSent) completed, err := scanSyncRun(row) if err != nil { return SyncRun{}, fmt.Errorf("complete watch provider sync run: %w", err) @@ -641,6 +665,138 @@ func (r *PostgresRepository) ListListEventConnections( return conns, nil } +// ListRatingEventConnections returns the profile's connections that send +// ratings, i.e. should mirror a local rating change to the provider. +func (r *PostgresRepository) ListRatingEventConnections(ctx context.Context, userID int, profileID string) ([]Connection, error) { + rows, err := r.pool.Query(ctx, ` + SELECT `+connectionColumns+` + FROM watch_provider_connections + WHERE user_id = $1 AND profile_id = $2 AND export_ratings_enabled = true + AND (rate_limited_until IS NULL OR rate_limited_until <= now()) + ORDER BY provider + `, userID, profileID) + if err != nil { + return nil, fmt.Errorf("list rating event connections: %w", err) + } + defer rows.Close() + + var conns []Connection + for rows.Next() { + conn, scanErr := r.scanConnection(rows) + if scanErr != nil { + return nil, fmt.Errorf("scan rating event connection: %w", scanErr) + } + conns = append(conns, conn) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate rating event connections: %w", err) + } + return conns, nil +} + +// ListRatingSyncStates returns a connection's agreed ratings with the given +// provider account. Rows agreed with another account are ignored, so a sync +// still running for an account the connection has since left cannot feed its +// rows into the new account's merge. A nil mediaItemIDs returns every state; +// otherwise only the listed items. +func (r *PostgresRepository) ListRatingSyncStates(ctx context.Context, connectionID, providerAccountID string, mediaItemIDs []string) ([]RatingSyncState, error) { + query := ` + SELECT connection_id::text, provider_account_id, media_item_id, kind, provider_item_key, synced_rating, remote_seen + FROM watch_provider_rating_items + WHERE connection_id = $1::uuid AND provider_account_id = $2` + args := []any{connectionID, providerAccountID} + if mediaItemIDs != nil { + query += ` AND media_item_id = ANY($3)` + args = append(args, mediaItemIDs) + } + rows, err := r.pool.Query(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("list rating sync states: %w", err) + } + defer rows.Close() + var states []RatingSyncState + for rows.Next() { + var state RatingSyncState + if err := rows.Scan(&state.ConnectionID, &state.ProviderAccountID, &state.MediaItemID, &state.Kind, &state.ProviderItemKey, &state.SyncedRating, &state.RemoteSeen); err != nil { + return nil, fmt.Errorf("scan rating sync state: %w", err) + } + states = append(states, state) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate rating sync states: %w", err) + } + return states, nil +} + +func (r *PostgresRepository) UpsertRatingSyncStates(ctx context.Context, states []RatingSyncState) error { + if len(states) == 0 { + return nil + } + connectionIDs := make([]string, len(states)) + accountIDs := make([]string, len(states)) + mediaItemIDs := make([]string, len(states)) + kinds := make([]string, len(states)) + keys := make([]string, len(states)) + ratings := make([]int32, len(states)) + seen := make([]bool, len(states)) + for i, state := range states { + connectionIDs[i] = state.ConnectionID + accountIDs[i] = state.ProviderAccountID + mediaItemIDs[i] = state.MediaItemID + kinds[i] = state.Kind + keys[i] = state.ProviderItemKey + ratings[i] = int32(state.SyncedRating) + seen[i] = state.RemoteSeen + } + _, err := r.pool.Exec(ctx, ` + INSERT INTO watch_provider_rating_items ( + connection_id, provider_account_id, media_item_id, kind, provider_item_key, synced_rating, remote_seen + ) + SELECT input.connection_id::uuid, input.provider_account_id, input.media_item_id, input.kind, + input.provider_item_key, input.synced_rating, input.remote_seen + FROM unnest($1::text[], $2::text[], $3::text[], $4::text[], $5::text[], $6::smallint[], $7::boolean[]) + AS input(connection_id, provider_account_id, media_item_id, kind, provider_item_key, synced_rating, remote_seen) + ON CONFLICT (connection_id, media_item_id) DO UPDATE SET + provider_account_id = EXCLUDED.provider_account_id, + kind = CASE WHEN EXCLUDED.kind <> '' THEN EXCLUDED.kind ELSE watch_provider_rating_items.kind END, + provider_item_key = CASE + WHEN EXCLUDED.provider_item_key <> '' THEN EXCLUDED.provider_item_key + ELSE watch_provider_rating_items.provider_item_key + END, + synced_rating = EXCLUDED.synced_rating, + remote_seen = EXCLUDED.remote_seen, + updated_at = now() + `, connectionIDs, accountIDs, mediaItemIDs, kinds, keys, ratings, seen) + if err != nil { + return fmt.Errorf("upsert rating sync states: %w", err) + } + return nil +} + +func (r *PostgresRepository) DeleteRatingSyncStates(ctx context.Context, connectionID string, mediaItemIDs []string) error { + if len(mediaItemIDs) == 0 { + return nil + } + _, err := r.pool.Exec(ctx, ` + DELETE FROM watch_provider_rating_items + WHERE connection_id = $1::uuid AND media_item_id = ANY($2) + `, connectionID, mediaItemIDs) + if err != nil { + return fmt.Errorf("delete rating sync states: %w", err) + } + return nil +} + +// ClearRatingSyncStates forgets every agreed rating of a connection, used when +// the connection is re-bound to a different provider account. +func (r *PostgresRepository) ClearRatingSyncStates(ctx context.Context, connectionID string) error { + _, err := r.pool.Exec(ctx, `DELETE FROM watch_provider_rating_items WHERE connection_id = $1::uuid`, connectionID) + if err != nil { + return fmt.Errorf("clear rating sync states: %w", err) + } + return nil +} + func (r *PostgresRepository) GetMediaDuration(ctx context.Context, mediaItemID string) (float64, error) { var duration float64 err := r.pool.QueryRow(ctx, mediaDurationQuery, mediaItemID).Scan(&duration) @@ -1339,6 +1495,10 @@ func scanSyncRun(row pgx.Row) (SyncRun, error) { &run.OutboundWatchlistFound, &run.OutboundWatchlistSent, &run.WatchlistRemovalsSent, + &run.InboundRatingsFound, + &run.InboundRatingsImported, + &run.OutboundRatingsFound, + &run.OutboundRatingsSent, &run.Warning, &run.Error, &run.StartedAt, @@ -1378,6 +1538,8 @@ func (r *PostgresRepository) scanConnection(row pgx.Row) (Connection, error) { &conn.SyncWatchlistRemovalsEnabled, &conn.SyncWatchlistOrderEnabled, &conn.ScrobbleEnabled, + &conn.ImportRatingsEnabled, + &conn.ExportRatingsEnabled, &conn.LastInboundSyncAt, &conn.LastProgressSyncAt, &conn.LastOutboundSyncAt, diff --git a/internal/watchsync/service.go b/internal/watchsync/service.go index da85f097bd..dfe6b7cae5 100644 --- a/internal/watchsync/service.go +++ b/internal/watchsync/service.go @@ -23,6 +23,8 @@ type Service struct { matcher mediaMatcher watchState watchStateImporter storeProvider userstore.UserStoreProvider + ratings ratingStore + ratingStaler ratingProfileStaler locks sync.Map scrobbleQueues sync.Map } @@ -139,6 +141,8 @@ func (s *Service) GetConnectionStatus(ctx context.Context, userID int, profileID SyncWatchlistRemovalsEnabled: false, SyncWatchlistOrderEnabled: true, ScrobbleEnabled: true, + ImportRatingsEnabled: true, + ExportRatingsEnabled: true, } if configurable, ok := provider.(connectionConfigProvider); ok { status.ConnectionConfigSchema = configurable.ConnectionConfigSchema() @@ -158,6 +162,8 @@ func (s *Service) GetConnectionStatus(ctx context.Context, userID int, profileID status.SyncWatchlistRemovalsEnabled = conn.SyncWatchlistRemovalsEnabled status.SyncWatchlistOrderEnabled = conn.SyncWatchlistOrderEnabled status.ScrobbleEnabled = conn.ScrobbleEnabled + status.ImportRatingsEnabled = conn.ImportRatingsEnabled + status.ExportRatingsEnabled = conn.ExportRatingsEnabled status.LastInboundSyncAt = conn.LastInboundSyncAt status.LastProgressSyncAt = conn.LastProgressSyncAt status.LastOutboundSyncAt = conn.LastOutboundSyncAt @@ -599,7 +605,16 @@ func (s *Service) persistConnection( ExportWatchlistEnabled: true, SyncWatchlistOrderEnabled: true, ScrobbleEnabled: true, + ImportRatingsEnabled: true, + ExportRatingsEnabled: true, } + } else if conn.ProviderAccountID != "" && account.ID != "" && account.ID != conn.ProviderAccountID { + // The agreed ratings describe the previous account. Keeping them would + // read every rating missing from the new account as a removal. + if err := s.repo.ClearRatingSyncStates(ctx, conn.ID); err != nil { + return Connection{}, err + } + conn.SyncCursors = withoutRatingCursors(conn.SyncCursors) } conn.Provider = providerKey conn.UserID = userID @@ -837,6 +852,22 @@ func (s *Service) executeSyncRun(ctx context.Context, conn Connection, run SyncR } } } + if rateLimited == nil && (conn.ImportRatingsEnabled || conn.ExportRatingsEnabled) && + (provider.Capabilities().ImportRatings || provider.Capabilities().ExportRatings) { + result, err := s.syncRatings(ctx, conn, cfg, provider) + run.InboundRatingsFound = result.RemoteFound + run.InboundRatingsImported = result.Imported + run.OutboundRatingsFound = result.LocalFound + run.OutboundRatingsSent = result.Sent + run.Warning = appendWarning(run.Warning, result.Warnings) + if err != nil { + recordFlowError("ratings", err) + } else if refreshed, refreshErr := s.reloadConnection(ctx, conn); refreshErr != nil { + flowErrors = append(flowErrors, "ratings connection refresh: "+refreshErr.Error()) + } else { + conn = refreshed + } + } if rateLimited != nil { if err := s.deferRateLimitedConnection(ctx, conn, *rateLimited); err != nil { @@ -923,7 +954,9 @@ func providerSyncNeedsAccessToken(caps Capabilities) bool { caps.ImportWatchlist || caps.ExportWatchlist || caps.RemoveWatchlist || - caps.ScrobblePlayback + caps.ScrobblePlayback || + caps.ImportRatings || + caps.ExportRatings } func (s *Service) completeSyncRun(ctx context.Context, run SyncRun) (SyncRun, error) { diff --git a/internal/watchsync/service_test.go b/internal/watchsync/service_test.go index 380923d4de..399d199602 100644 --- a/internal/watchsync/service_test.go +++ b/internal/watchsync/service_test.go @@ -50,6 +50,8 @@ type serviceFakeRepo struct { historyLookupIDs []string historyLookupLimit int listItemStates []ListItemState + ratingStates []RatingSyncState + listMedia map[string]LocalFavorite scrobbleConnections []Connection scrobbleSessions []ScrobbleSession pendingReconciliations []ScrobbleSession @@ -513,6 +515,87 @@ func (r *serviceFakeRepo) MarkListItemError(_ context.Context, connectionID stri return nil } +func (r *serviceFakeRepo) ListRatingEventConnections(_ context.Context, userID int, profileID string) ([]Connection, error) { + var conns []Connection + for _, conn := range r.connections { + if conn.UserID == userID && conn.ProfileID == profileID && conn.ExportRatingsEnabled { + conns = append(conns, cloneConnectionForTest(conn)) + } + } + return conns, nil +} + +func (r *serviceFakeRepo) ListRatingSyncStates(_ context.Context, connectionID, providerAccountID string, mediaItemIDs []string) ([]RatingSyncState, error) { + var states []RatingSyncState + for _, state := range r.ratingStates { + if state.ConnectionID != connectionID || state.ProviderAccountID != providerAccountID { + continue + } + if mediaItemIDs != nil && !containsString(mediaItemIDs, state.MediaItemID) { + continue + } + states = append(states, state) + } + return states, nil +} + +func (r *serviceFakeRepo) UpsertRatingSyncStates(_ context.Context, states []RatingSyncState) error { + for _, state := range states { + replaced := false + for i := range r.ratingStates { + existing := &r.ratingStates[i] + if existing.ConnectionID == state.ConnectionID && existing.MediaItemID == state.MediaItemID { + if state.Kind == "" { + state.Kind = existing.Kind + } + if state.ProviderItemKey == "" { + state.ProviderItemKey = existing.ProviderItemKey + } + *existing = state + replaced = true + break + } + } + if !replaced { + r.ratingStates = append(r.ratingStates, state) + } + } + return nil +} + +func (r *serviceFakeRepo) DeleteRatingSyncStates(_ context.Context, connectionID string, mediaItemIDs []string) error { + kept := r.ratingStates[:0] + for _, state := range r.ratingStates { + if state.ConnectionID == connectionID && containsString(mediaItemIDs, state.MediaItemID) { + continue + } + kept = append(kept, state) + } + r.ratingStates = kept + return nil +} + +func (r *serviceFakeRepo) ClearRatingSyncStates(_ context.Context, connectionID string) error { + kept := r.ratingStates[:0] + for _, state := range r.ratingStates { + if state.ConnectionID != connectionID { + kept = append(kept, state) + } + } + r.ratingStates = kept + return nil +} + +func (r *serviceFakeRepo) GetListMediaItems(_ context.Context, mediaItemIDs []string) (map[string]LocalFavorite, error) { + result := make(map[string]LocalFavorite, len(mediaItemIDs)) + for _, id := range mediaItemIDs { + if item, ok := r.listMedia[id]; ok { + result[id] = item + } + } + return result, nil +} + func (r *serviceFakeRepo) ListScrobbleConnections(_ context.Context, _ int, _ string) ([]Connection, error) { conns := make([]Connection, 0, len(r.scrobbleConnections)) for _, conn := range r.scrobbleConnections { diff --git a/internal/watchsync/types.go b/internal/watchsync/types.go index 1855440680..aa0ac078a5 100644 --- a/internal/watchsync/types.go +++ b/internal/watchsync/types.go @@ -26,6 +26,10 @@ type Capabilities struct { // user-configurable order that Silo can mirror locally. ProvidesWatchlistOrder bool `json:"provides_watchlist_order"` ScrobblePlayback bool `json:"scrobble_playback"` + // ImportRatings and ExportRatings cover movie and series ratings. + // ExportRatings means the provider can both set and clear a rating. + ImportRatings bool `json:"import_ratings"` + ExportRatings bool `json:"export_ratings"` } // ListKind identifies which personal list a sync operates on. The favorites and @@ -175,6 +179,46 @@ type WatchlistRemover interface { RemoveWatchlist(ctx context.Context, cfg ServerConfig, conn Connection, items []LocalFavorite) (ExportResult, error) } +// RatingImporter reads the provider's movie and series ratings. Ratings use the +// provider scale shared by every watch-sync provider: integers from 1 to 10. +type RatingImporter interface { + FetchRatings(ctx context.Context, cfg ServerConfig, conn Connection) (RatingImportBatch, error) +} + +// RatingImportBatch is one rating read. SnapshotKinds names the item kinds +// (historyimport.KindMovie, historyimport.KindSeries) for which Rows is the +// provider's complete set of ratings: an item of such a kind that is absent +// from Rows is unrated remotely. Absent items of any other kind are unknown, +// which is how a provider reports that it skipped an unchanged kind. +type RatingImportBatch struct { + Rows []RemoteRating + SnapshotKinds []string + UpdatedCursors map[string]string + Warnings []string +} + +// RatingExporter sets and clears ratings on the provider. Both calls are +// desired-state writes: resending an unchanged rating must succeed. +type RatingExporter interface { + ExportRatings(ctx context.Context, cfg ServerConfig, conn Connection, items []LocalRating) (ExportResult, error) + RemoveRatings(ctx context.Context, cfg ServerConfig, conn Connection, items []LocalFavorite) (ExportResult, error) +} + +// RatingKindFilter is implemented by providers that rate only some of the item +// kinds Silo syncs (historyimport.KindMovie, historyimport.KindSeries). Items +// of any other kind are left out of the sync: never sent and never read as +// removed. +type RatingKindFilter interface { + SyncsRatingKind(kind string) bool +} + +// RatingExportWatchGate is implemented by providers where rating a title also +// records it as watched. Silo then sends a new rating of such a kind only once +// the profile has a completed play of the title. +type RatingExportWatchGate interface { + RatingExportRequiresWatched(kind string) bool +} + type Scrobbler interface { Start(ctx context.Context, cfg ServerConfig, conn Connection, event ScrobbleEvent) error Pause(ctx context.Context, cfg ServerConfig, conn Connection, event ScrobbleEvent) error @@ -219,6 +263,8 @@ type Connection struct { SyncWatchlistRemovalsEnabled bool SyncWatchlistOrderEnabled bool ScrobbleEnabled bool + ImportRatingsEnabled bool + ExportRatingsEnabled bool LastInboundSyncAt *time.Time LastProgressSyncAt *time.Time LastOutboundSyncAt *time.Time @@ -261,6 +307,13 @@ type SyncRun struct { StartedAt time.Time `json:"started_at"` CompletedAt *time.Time `json:"completed_at,omitempty"` CreatedAt time.Time `json:"created_at"` + + // The rating counters are served only by /api/v2, which maps them + // explicitly; the frozen v1 response keeps its original fields. + InboundRatingsFound int `json:"-"` + InboundRatingsImported int `json:"-"` + OutboundRatingsFound int `json:"-"` + OutboundRatingsSent int `json:"-"` } type SyncRunStatus string @@ -479,6 +532,48 @@ type LocalFavorite struct { FavoritedAt time.Time } +// RemoteRating is one rated movie or series reported by a provider. The +// embedded RemoteFavorite carries the item identity; its Removed flag is an +// explicit tombstone resolved through ProviderItemKey. Rating is on the +// provider scale, an integer from 1 to 10. +type RemoteRating struct { + RemoteFavorite + Rating int + RatedAt time.Time +} + +// LocalRating is a Silo rating to send to a provider. The embedded +// LocalFavorite carries the item identity. Rating is already converted to the +// provider scale, an integer from 1 to 10. +type LocalRating struct { + LocalFavorite + Rating int + RatedAt time.Time +} + +// LocalRatingEvent reports that a profile set or cleared ratings. It carries +// no values: the handler reads the current rating so that events processed out +// of order never send a stale value. +type LocalRatingEvent struct { + UserID int + ProfileID string + MediaItemIDs []string +} + +// RatingSyncState is the last rating Silo and a provider agreed on for one item, +// in Silo stars (1 to 5). No state means the sides agreed the item is unrated. +// RemoteSeen records that a provider read confirmed the agreed rating. +// ProviderAccountID scopes the row to the provider account it was agreed with. +type RatingSyncState struct { + ConnectionID string + ProviderAccountID string + MediaItemID string + Kind string + ProviderItemKey string + SyncedRating int + RemoteSeen bool +} + type LocalWatchEventKind string const ( @@ -718,6 +813,11 @@ type ConnectionStatus struct { LastWatchlistSyncAt *time.Time `json:"last_watchlist_sync_at,omitempty"` LastScrobbleErrorAt *time.Time `json:"last_scrobble_error_at,omitempty"` LastError string `json:"last_error,omitempty"` + + // The rating toggles are served only by /api/v2, which maps them + // explicitly; the frozen v1 response keeps its original fields. + ImportRatingsEnabled bool `json:"-"` + ExportRatingsEnabled bool `json:"-"` } type ConnectionUpdate struct { @@ -733,6 +833,8 @@ type ConnectionUpdate struct { SyncWatchlistRemovalsEnabled *bool `json:"sync_watchlist_removals_enabled,omitempty"` SyncWatchlistOrderEnabled *bool `json:"sync_watchlist_order_enabled,omitempty"` ScrobbleEnabled *bool `json:"scrobble_enabled,omitempty"` + ImportRatingsEnabled *bool `json:"import_ratings_enabled,omitempty"` + ExportRatingsEnabled *bool `json:"export_ratings_enabled,omitempty"` } // UnknownProviderError reports a provider key the registry does not know. diff --git a/migrations/sql/20260923214332_watch_provider_ratings.sql b/migrations/sql/20260923214332_watch_provider_ratings.sql new file mode 100644 index 0000000000..91ecca60e8 --- /dev/null +++ b/migrations/sql/20260923214332_watch_provider_ratings.sql @@ -0,0 +1,55 @@ +-- +goose Up +-- +goose StatementBegin + +-- Per-connection rating sync toggles. Existing connections start with rating +-- sync off: turning it on writes to the user's external account, so it must be +-- an explicit choice. New connections are created with both on by the service. +ALTER TABLE public.watch_provider_connections + ADD COLUMN import_ratings_enabled boolean NOT NULL DEFAULT false, + ADD COLUMN export_ratings_enabled boolean NOT NULL DEFAULT false; + +-- Per-run rating counters. Removals count as sent. +ALTER TABLE public.watch_provider_sync_runs + ADD COLUMN inbound_ratings_found integer NOT NULL DEFAULT 0, + ADD COLUMN inbound_ratings_imported integer NOT NULL DEFAULT 0, + ADD COLUMN outbound_ratings_found integer NOT NULL DEFAULT 0, + ADD COLUMN outbound_ratings_sent integer NOT NULL DEFAULT 0; + +-- The last rating Silo and the provider agreed on, per connection and item, in +-- Silo stars. It is the base of the three-way merge that decides which side +-- changed. No row means the two sides agreed the item is unrated. +-- remote_seen records that a provider read confirmed the agreed rating, so an +-- item missing from a later complete snapshot counts as a remote removal only +-- when the provider is known to have held it. provider_account_id scopes the +-- row to the account it was agreed with; rows of another account are ignored. +CREATE TABLE public.watch_provider_rating_items ( + connection_id uuid NOT NULL + REFERENCES public.watch_provider_connections(id) ON DELETE CASCADE, + provider_account_id text NOT NULL DEFAULT '', + media_item_id text NOT NULL, + kind text NOT NULL, + provider_item_key text NOT NULL DEFAULT '', + synced_rating smallint NOT NULL CHECK (synced_rating BETWEEN 1 AND 5), + remote_seen boolean NOT NULL DEFAULT false, + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (connection_id, media_item_id) +); + +CREATE INDEX idx_watch_provider_rating_items_media + ON public.watch_provider_rating_items (media_item_id); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS public.watch_provider_rating_items; + +ALTER TABLE public.watch_provider_sync_runs + DROP COLUMN IF EXISTS outbound_ratings_sent, + DROP COLUMN IF EXISTS outbound_ratings_found, + DROP COLUMN IF EXISTS inbound_ratings_imported, + DROP COLUMN IF EXISTS inbound_ratings_found; + +ALTER TABLE public.watch_provider_connections + DROP COLUMN IF EXISTS export_ratings_enabled, + DROP COLUMN IF EXISTS import_ratings_enabled; +-- +goose StatementEnd diff --git a/web/src/api/v2/schema.ts b/web/src/api/v2/schema.ts index 92f8e6c074..ac54b76215 100644 --- a/web/src/api/v2/schema.ts +++ b/web/src/api/v2/schema.ts @@ -17239,11 +17239,13 @@ export interface components { }; ConnectionUpdate: { export_favorites_enabled?: boolean; + export_ratings_enabled?: boolean; export_unwatched_enabled?: boolean; export_watched_enabled?: boolean; export_watchlist_enabled?: boolean; import_favorites_enabled?: boolean; import_progress_enabled?: boolean; + import_ratings_enabled?: boolean; import_watched_enabled?: boolean; import_watchlist_enabled?: boolean; scrobble_enabled?: boolean; @@ -25527,11 +25529,13 @@ export interface components { }; WatchProviderCapabilities: { export_favorites: boolean; + export_ratings: boolean; export_unwatched: boolean; export_watched: boolean; export_watchlist: boolean; import_favorites: boolean; import_progress: boolean; + import_ratings: boolean; import_watched: boolean; import_watchlist: boolean; provides_watchlist_order: boolean; @@ -25547,11 +25551,13 @@ export interface components { credentials_configured: boolean; display_name: string; export_favorites_enabled: boolean; + export_ratings_enabled: boolean; export_unwatched_enabled: boolean; export_watched_enabled: boolean; export_watchlist_enabled: boolean; import_favorites_enabled: boolean; import_progress_enabled: boolean; + import_ratings_enabled: boolean; import_watched_enabled: boolean; import_watchlist_enabled: boolean; last_error?: string; @@ -25615,11 +25621,15 @@ export interface components { }; WatchProviderSettings: { export_favorites_enabled: boolean; + /** @description Send the profile's star ratings to the provider (stars times two) and clear removed ones. */ + export_ratings_enabled: boolean; export_unwatched_enabled: boolean; export_watched_enabled: boolean; export_watchlist_enabled: boolean; import_favorites_enabled: boolean; import_progress_enabled: boolean; + /** @description Import the provider's movie and series ratings as stars (1-2 is 1 star, 9-10 is 5 stars). */ + import_ratings_enabled: boolean; import_watched_enabled: boolean; import_watchlist_enabled: boolean; scrobble_enabled: boolean; @@ -25671,6 +25681,10 @@ export interface components { /** Format: int64 */ inbound_progress_imported: number; /** Format: int64 */ + inbound_ratings_found: number; + /** Format: int64 */ + inbound_ratings_imported: number; + /** Format: int64 */ inbound_watched_found: number; /** Format: int64 */ inbound_watched_imported: number; @@ -25684,6 +25698,16 @@ export interface components { outbound_favorites_sent: number; /** Format: int64 */ outbound_found: number; + /** + * Format: int64 + * @description Movie and series ratings the profile holds. + */ + outbound_ratings_found: number; + /** + * Format: int64 + * @description Ratings set or cleared on the provider. + */ + outbound_ratings_sent: number; /** Format: int64 */ outbound_sent: number; /** Format: int64 */ diff --git a/web/src/hooks/queries/watchProviders.ts b/web/src/hooks/queries/watchProviders.ts index eed63825d6..ae2bbd5586 100644 --- a/web/src/hooks/queries/watchProviders.ts +++ b/web/src/hooks/queries/watchProviders.ts @@ -1,7 +1,7 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { captureProfileRequestContext } from "@/api/client"; import { v2, V2ProblemError } from "@/api/v2/request"; -import { favoriteKeys, watchlistKeys, watchProviderKeys } from "./keys"; +import { favoriteKeys, ratingKeys, watchlistKeys, watchProviderKeys } from "./keys"; import { toast } from "sonner"; import { storage } from "@/utils/storage"; import type { PluginConfigSchema } from "@/api/types"; @@ -35,6 +35,8 @@ export interface WatchProviderCapabilities { remove_watchlist: boolean; provides_watchlist_order: boolean; scrobble_playback: boolean; + import_ratings: boolean; + export_ratings: boolean; } export interface WatchProviderConnection { @@ -57,6 +59,8 @@ export interface WatchProviderConnection { sync_watchlist_removals_enabled: boolean; sync_watchlist_order_enabled: boolean; scrobble_enabled: boolean; + import_ratings_enabled: boolean; + export_ratings_enabled: boolean; credentials_configured: boolean; connection_config_schema?: PluginConfigSchema[]; last_inbound_sync_at?: string; @@ -99,6 +103,10 @@ export interface WatchProviderSyncRun { outbound_watchlist_found: number; outbound_watchlist_sent: number; watchlist_removals_sent: number; + inbound_ratings_found: number; + inbound_ratings_imported: number; + outbound_ratings_found: number; + outbound_ratings_sent: number; warning?: string; error?: string; started_at: string; @@ -126,6 +134,8 @@ export type UpdateWatchProviderConnection = Partial< | "sync_watchlist_removals_enabled" | "sync_watchlist_order_enabled" | "scrobble_enabled" + | "import_ratings_enabled" + | "export_ratings_enabled" > >; @@ -359,6 +369,7 @@ export function useTriggerWatchProviderSync(provider: string) { }); queryClient.invalidateQueries({ queryKey: favoriteKeys.list() }); queryClient.invalidateQueries({ queryKey: watchlistKeys.list() }); + queryClient.invalidateQueries({ queryKey: ratingKeys.all }); toast.success("Watch provider sync started"); }, onError: (err) => { diff --git a/web/src/pages/settings/WatchProvidersSettings.tsx b/web/src/pages/settings/WatchProvidersSettings.tsx index 17b418f973..0a49b71339 100644 --- a/web/src/pages/settings/WatchProvidersSettings.tsx +++ b/web/src/pages/settings/WatchProvidersSettings.tsx @@ -308,13 +308,20 @@ function APIKeyBlock({ } interface ConnectedRunInfo { - imported: { watched: number; progress: number; favorites: number; watchlist: number }; + imported: { + watched: number; + progress: number; + favorites: number; + watchlist: number; + ratings: number; + }; exported: { watched: number; favorites: number; favoriteRemovals: number; watchlist: number; watchlistRemovals: number; + ratings: number; }; errorMessage?: string; errorHint?: string; @@ -329,6 +336,7 @@ function deriveRunInfo( progress: latestRun?.inbound_progress_imported ?? 0, favorites: latestRun?.inbound_favorites_imported ?? 0, watchlist: latestRun?.inbound_watchlist_imported ?? 0, + ratings: latestRun?.inbound_ratings_imported ?? 0, }; const exported = { watched: latestRun?.outbound_sent ?? 0, @@ -336,6 +344,7 @@ function deriveRunInfo( favoriteRemovals: latestRun?.favorite_removals_sent ?? 0, watchlist: latestRun?.outbound_watchlist_sent ?? 0, watchlistRemovals: latestRun?.watchlist_removals_sent ?? 0, + ratings: latestRun?.outbound_ratings_sent ?? 0, }; let errorMessage: string | undefined; let errorHint: string | undefined; @@ -429,6 +438,8 @@ function WatchProviderCard({ providerKey }: { providerKey: string }) { connection.import_favorites_enabled || connection.export_favorites_enabled; const watchlistSyncEnabled = connection.import_watchlist_enabled || connection.export_watchlist_enabled; + const supportsRatings = + connection.capabilities.import_ratings || connection.capabilities.export_ratings; let statusVariant: StatusVariant; let statusLabel: string; @@ -615,14 +626,19 @@ function WatchProviderCard({ providerKey }: { providerKey: string }) {
-
+
+ {supportsRatings ? ( + + ) : null}
@@ -782,6 +804,26 @@ function WatchProviderCard({ providerKey }: { providerKey: string }) { } /> ) : null} + {connection.capabilities.import_ratings ? ( + updateConnection.mutate({ import_ratings_enabled: checked })} + /> + ) : null} + {connection.capabilities.export_ratings ? ( + updateConnection.mutate({ export_ratings_enabled: checked })} + /> + ) : null} Date: Wed, 23 Sep 2026 22:14:51 +0000 Subject: [PATCH 06/20] feat(watchsync): sync Trakt movie and show ratings Read every Trakt movie and show rating through the paged reader and report both as complete snapshots. Send ratings to /sync/ratings with the local rating time and clear them through /sync/ratings/remove, mapping not_found echoes back to items by any shared id. Trakt already uses the 1-10 scale, so values pass through unchanged. Season and episode ratings are left alone. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../watchsync/providers/trakt/provider.go | 2 + .../providers/trakt/provider_test.go | 2 + internal/watchsync/providers/trakt/ratings.go | 166 ++++++++++++++++++ .../watchsync/providers/trakt/ratings_test.go | 121 +++++++++++++ 4 files changed, 291 insertions(+) create mode 100644 internal/watchsync/providers/trakt/ratings.go create mode 100644 internal/watchsync/providers/trakt/ratings_test.go diff --git a/internal/watchsync/providers/trakt/provider.go b/internal/watchsync/providers/trakt/provider.go index f9a244aeaa..0982d3a4db 100644 --- a/internal/watchsync/providers/trakt/provider.go +++ b/internal/watchsync/providers/trakt/provider.go @@ -92,6 +92,8 @@ func (p *Provider) Capabilities() watchsync.Capabilities { ExportWatchlist: true, RemoveWatchlist: true, ScrobblePlayback: true, + ImportRatings: true, + ExportRatings: true, } } diff --git a/internal/watchsync/providers/trakt/provider_test.go b/internal/watchsync/providers/trakt/provider_test.go index 45f6a80a39..7cfab3055a 100644 --- a/internal/watchsync/providers/trakt/provider_test.go +++ b/internal/watchsync/providers/trakt/provider_test.go @@ -35,6 +35,8 @@ func TestProviderIdentityAndCapabilities(t *testing.T) { ExportWatchlist: true, RemoveWatchlist: true, ScrobblePlayback: true, + ImportRatings: true, + ExportRatings: true, }) { t.Fatalf("unexpected capabilities: %#v", provider.Capabilities()) } diff --git a/internal/watchsync/providers/trakt/ratings.go b/internal/watchsync/providers/trakt/ratings.go new file mode 100644 index 0000000000..cfcd0f61cf --- /dev/null +++ b/internal/watchsync/providers/trakt/ratings.go @@ -0,0 +1,166 @@ +package trakt + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/Silo-Server/silo-server/internal/historyimport" + "github.com/Silo-Server/silo-server/internal/watchsync" +) + +// Trakt rates on the integer 1 to 10 scale watchsync uses, so ratings pass +// through unchanged. Only movie and show ratings are read and written: Silo +// rates movies and series, not seasons or episodes. + +type traktRatedMovie struct { + RatedAt time.Time `json:"rated_at"` + Rating int `json:"rating"` + Movie traktMovie `json:"movie"` +} + +type traktRatedShow struct { + RatedAt time.Time `json:"rated_at"` + Rating int `json:"rating"` + Show traktShow `json:"show"` +} + +type traktRatingPayload struct { + Rating int `json:"rating"` + RatedAt *time.Time `json:"rated_at,omitempty"` + IDs traktIDs `json:"ids"` +} + +type traktRatingsPayload struct { + Movies []traktRatingPayload `json:"movies,omitempty"` + Shows []traktRatingPayload `json:"shows,omitempty"` +} + +// FetchRatings reads every movie and show rating. Both lists are complete, so a +// title absent from them is unrated on Trakt. +func (p *Provider) FetchRatings( + ctx context.Context, + cfg watchsync.ServerConfig, + conn watchsync.Connection, +) (watchsync.RatingImportBatch, error) { + movies, err := fetchTraktPages[traktRatedMovie](ctx, p, cfg, conn, "/sync/ratings/movies", nil) + if err != nil { + return watchsync.RatingImportBatch{}, err + } + shows, err := fetchTraktPages[traktRatedShow](ctx, p, cfg, conn, "/sync/ratings/shows", nil) + if err != nil { + return watchsync.RatingImportBatch{}, err + } + rows := make([]watchsync.RemoteRating, 0, len(movies)+len(shows)) + for _, item := range movies { + rows = append(rows, watchsync.RemoteRating{ + RemoteFavorite: watchsync.RemoteFavorite{ + Provider: p.Key(), + ProviderItemKey: movieKey(item.Movie.IDs), + Kind: historyimport.KindMovie, + Title: item.Movie.Title, + Year: item.Movie.Year, + IMDbID: item.Movie.IDs.IMDb, + TMDBID: intString(item.Movie.IDs.TMDB), + TVDBID: intString(item.Movie.IDs.TVDB), + }, + Rating: item.Rating, + RatedAt: item.RatedAt, + }) + } + for _, item := range shows { + rows = append(rows, watchsync.RemoteRating{ + RemoteFavorite: watchsync.RemoteFavorite{ + Provider: p.Key(), + ProviderItemKey: showKey(item.Show.IDs), + Kind: historyimport.KindSeries, + Title: item.Show.Title, + Year: item.Show.Year, + IMDbID: item.Show.IDs.IMDb, + TMDBID: intString(item.Show.IDs.TMDB), + TVDBID: intString(item.Show.IDs.TVDB), + }, + Rating: item.Rating, + RatedAt: item.RatedAt, + }) + } + return watchsync.RatingImportBatch{ + Rows: rows, + SnapshotKinds: []string{historyimport.KindMovie, historyimport.KindSeries}, + }, nil +} + +// ExportRatings sets movie and show ratings. Trakt replaces an existing rating, +// so resending one is harmless. +func (p *Provider) ExportRatings( + ctx context.Context, + cfg watchsync.ServerConfig, + conn watchsync.Connection, + items []watchsync.LocalRating, +) (watchsync.ExportResult, error) { + var payload traktRatingsPayload + favorites := make([]watchsync.LocalFavorite, 0, len(items)) + for _, item := range items { + ids := favoriteIDs(item.LocalFavorite) + if !sendableIDs(ids) || item.Rating < 1 || item.Rating > 10 { + continue + } + entry := traktRatingPayload{Rating: item.Rating, IDs: ids} + if !item.RatedAt.IsZero() { + ratedAt := item.RatedAt.UTC() + entry.RatedAt = &ratedAt + } + switch item.Kind { + case historyimport.KindMovie: + payload.Movies = append(payload.Movies, entry) + case historyimport.KindSeries: + payload.Shows = append(payload.Shows, entry) + default: + continue + } + favorites = append(favorites, item.LocalFavorite) + } + if len(favorites) == 0 { + return watchsync.ExportResult{}, nil + } + return p.sendRatings(ctx, "/sync/ratings", cfg, conn, payload, favorites) +} + +// RemoveRatings clears movie and show ratings. Trakt reports a title it does +// not know in not_found, which the caller treats as already cleared. +func (p *Provider) RemoveRatings( + ctx context.Context, + cfg watchsync.ServerConfig, + conn watchsync.Connection, + items []watchsync.LocalFavorite, +) (watchsync.ExportResult, error) { + ids := buildFavoritesPayload(items) + if len(ids.Movies) == 0 && len(ids.Shows) == 0 { + return watchsync.ExportResult{}, nil + } + return p.sendRatings(ctx, "/sync/ratings/remove", cfg, conn, ids, items) +} + +func (p *Provider) sendRatings( + ctx context.Context, + path string, + cfg watchsync.ServerConfig, + conn watchsync.Connection, + payload any, + items []watchsync.LocalFavorite, +) (watchsync.ExportResult, error) { + var body bytes.Buffer + if err := json.NewEncoder(&body).Encode(payload); err != nil { + return watchsync.ExportResult{}, fmt.Errorf("encode trakt ratings payload: %w", err) + } + // The not_found lists echo {ids} (plus the rating on a set), the same + // shape favorites use. + var response traktFavoritesResponse + if err := p.do(ctx, http.MethodPost, path, cfg, conn.AccessToken, &body, &response); err != nil { + return watchsync.ExportResult{}, err + } + return favoriteExportResult(items, response.NotFound), nil +} diff --git a/internal/watchsync/providers/trakt/ratings_test.go b/internal/watchsync/providers/trakt/ratings_test.go new file mode 100644 index 0000000000..fbe14a43c5 --- /dev/null +++ b/internal/watchsync/providers/trakt/ratings_test.go @@ -0,0 +1,121 @@ +package trakt + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/historyimport" + "github.com/Silo-Server/silo-server/internal/watchsync" +) + +func TestFetchRatingsReadsMoviesAndShowsAsCompleteSnapshots(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("limit") != "250" { + t.Errorf("%s limit = %q, want 250", r.URL.Path, r.URL.Query().Get("limit")) + } + w.Header().Set("X-Pagination-Page-Count", "1") + switch r.URL.Path { + case "/sync/ratings/movies": + writeTraktFixture(t, w, `[{"rated_at":"2026-03-01T10:00:00.000Z","rating":7,"type":"movie","movie":{"title":"Heat","year":1995,"ids":{"trakt":1,"imdb":"tt0113277","tmdb":949}}}]`) + case "/sync/ratings/shows": + writeTraktFixture(t, w, `[{"rated_at":"2026-03-02T10:00:00.000Z","rating":10,"type":"show","show":{"title":"The Wire","year":2002,"ids":{"trakt":2,"imdb":"tt0306414","tmdb":1438,"tvdb":79126}}}]`) + default: + t.Errorf("unexpected path %s", r.URL.Path) + http.NotFound(w, r) + } + })) + defer server.Close() + + batch, err := NewProvider(server.Client(), server.URL).FetchRatings(context.Background(), watchsync.ServerConfig{ClientID: "c"}, watchsync.Connection{AccessToken: "t"}) + if err != nil { + t.Fatal(err) + } + if len(batch.SnapshotKinds) != 2 { + t.Fatalf("snapshot kinds = %v, want movie and series", batch.SnapshotKinds) + } + if len(batch.Rows) != 2 { + t.Fatalf("rows = %#v", batch.Rows) + } + movie, show := batch.Rows[0], batch.Rows[1] + if movie.Kind != historyimport.KindMovie || movie.Rating != 7 || movie.IMDbID != "tt0113277" || movie.TMDBID != "949" || + !movie.RatedAt.Equal(time.Date(2026, 3, 1, 10, 0, 0, 0, time.UTC)) { + t.Fatalf("movie row = %#v", movie) + } + if show.Kind != historyimport.KindSeries || show.Rating != 10 || show.TVDBID != "79126" || show.ProviderItemKey != "tvdb:79126" { + t.Fatalf("show row = %#v", show) + } +} + +func TestExportRatingsSendsRatingsAndMapsNotFound(t *testing.T) { + ratedAt := time.Date(2026, 4, 5, 6, 7, 8, 0, time.UTC) + var got traktRatingsPayload + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/sync/ratings" { + t.Errorf("request = %s %s", r.Method, r.URL.Path) + } + if err := json.NewDecoder(r.Body).Decode(&got); err != nil { + t.Error(err) + } + w.WriteHeader(http.StatusCreated) + // Trakt echoes the missing show by TVDB id only. + writeTraktFixture(t, w, `{"added":{"movies":1,"shows":0},"not_found":{"movies":[],"shows":[{"rating":8,"ids":{"tvdb":79126}}]}}`) + })) + defer server.Close() + + result, err := NewProvider(server.Client(), server.URL).ExportRatings(context.Background(), watchsync.ServerConfig{ClientID: "c"}, watchsync.Connection{AccessToken: "t"}, []watchsync.LocalRating{ + {LocalFavorite: watchsync.LocalFavorite{MediaItemID: "m1", Kind: historyimport.KindMovie, IMDbID: "tt0113277", ProviderItemKey: "imdb:tt0113277"}, Rating: 6, RatedAt: ratedAt}, + {LocalFavorite: watchsync.LocalFavorite{MediaItemID: "s1", Kind: historyimport.KindSeries, IMDbID: "tt0306414", TVDBID: "79126", ProviderItemKey: "imdb:tt0306414"}, Rating: 8}, + }) + if err != nil { + t.Fatal(err) + } + if len(got.Movies) != 1 || got.Movies[0].Rating != 6 || got.Movies[0].IDs.IMDb != "tt0113277" || got.Movies[0].RatedAt == nil || !got.Movies[0].RatedAt.Equal(ratedAt) { + t.Fatalf("movie payload = %#v", got.Movies) + } + if len(got.Shows) != 1 || got.Shows[0].Rating != 8 || got.Shows[0].RatedAt != nil { + t.Fatalf("show payload = %#v", got.Shows) + } + if !containsValue(result.Sent, "m1") || containsValue(result.Sent, "s1") || !containsValue(result.NotFound, "s1") { + t.Fatalf("result = %#v, want movie sent and show not found", result) + } +} + +func TestRemoveRatingsSendsIDsOnly(t *testing.T) { + var got map[string][]map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/sync/ratings/remove" { + t.Errorf("request = %s %s", r.Method, r.URL.Path) + } + if err := json.NewDecoder(r.Body).Decode(&got); err != nil { + t.Error(err) + } + writeTraktFixture(t, w, `{"deleted":{"movies":1},"not_found":{"movies":[],"shows":[]}}`) + })) + defer server.Close() + + result, err := NewProvider(server.Client(), server.URL).RemoveRatings(context.Background(), watchsync.ServerConfig{ClientID: "c"}, watchsync.Connection{AccessToken: "t"}, []watchsync.LocalFavorite{ + {MediaItemID: "m1", Kind: historyimport.KindMovie, ProviderItemKey: "tmdb:949"}, + }) + if err != nil { + t.Fatal(err) + } + if len(got["movies"]) != 1 || got["movies"][0]["rating"] != nil { + t.Fatalf("remove payload = %#v, want ids without a rating", got) + } + if !containsValue(result.Sent, "m1") { + t.Fatalf("result = %#v", result) + } +} + +func containsValue(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} From 0aac5d0ae0a66371169e192d3245cd01b6c4322f Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Wed, 23 Sep 2026 22:39:32 +0000 Subject: [PATCH 07/20] chore(deps): require the plugin SDK rating contract Require the silo-plugin-sdk commit that adds rating state and the series media type. Bump to the v0.17.0 tag once it is released. Co-Authored-By: Claude Opus 5.5 (1M context) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6b98f98626..6a16fcc3ad 100644 --- a/go.mod +++ b/go.mod @@ -128,7 +128,7 @@ require ( ) require ( - github.com/Silo-Server/silo-plugin-sdk v0.16.1 + github.com/Silo-Server/silo-plugin-sdk v0.16.2-0.20260923231249-95b81a099ca7 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 // indirect diff --git a/go.sum b/go.sum index 73cc96960c..ac5e206a67 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0g github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/SherClockHolmes/webpush-go v1.4.0 h1:ocnzNKWN23T9nvHi6IfyrQjkIc0oJWv1B1pULsf9i3s= github.com/SherClockHolmes/webpush-go v1.4.0/go.mod h1:XSq8pKX11vNV8MJEMwjrlTkxhAj1zKfxmyhdV7Pd6UA= -github.com/Silo-Server/silo-plugin-sdk v0.16.1 h1:bctPlzlsr75Z9ARQ86VnYY6UhyQ9YPwgoP+mqSRESPA= -github.com/Silo-Server/silo-plugin-sdk v0.16.1/go.mod h1:abwsCEKuPAAgeAqpNGbwoaut2eQlC/Kj97u89Vvg9qM= +github.com/Silo-Server/silo-plugin-sdk v0.16.2-0.20260923231249-95b81a099ca7 h1:7aVKMCncxENZ5QNXbDa99q/M12Y1KZSp/h/eQlFS3yU= +github.com/Silo-Server/silo-plugin-sdk v0.16.2-0.20260923231249-95b81a099ca7/go.mod h1:abwsCEKuPAAgeAqpNGbwoaut2eQlC/Kj97u89Vvg9qM= github.com/TwiN/go-color v1.4.1 h1:mqG0P/KBgHKVqmtL5ye7K0/Gr4l6hTksPgTgMk3mUzc= github.com/TwiN/go-color v1.4.1/go.mod h1:WcPf/jtiW95WBIsEeY1Lc/b8aaWoiqQpu5cf8WFxu+s= github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4= From f9fce035209b89c7734ce61bb9fc3c658489e3e8 Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Wed, 23 Sep 2026 22:39:39 +0000 Subject: [PATCH 08/20] feat(watchsync): sync ratings with watch-sync plugins Plugin providers can now advertise import_ratings and export_ratings. The adapter reads RATING remote state (a complete snapshot covers the rateable kinds the plugin supports; an incremental read covers none, so absent titles stay unknown), decodes key-only tombstones, and sends SET_RATING and REMOVE_RATING events. A SET event id carries the rating and rating time, so a retry reuses it and a later change does not. Series now cross the plugin boundary as the SDK's SERIES media type, for ratings and for favorites and watchlist; plugins that do not list SERIES see no change, and SERIES watched or progress rows are rejected because the contract does not define them. A plugin rates only the kinds it supports, so a movie-only plugin is never sent series ratings. Media types from a newer SDK are ignored instead of rejecting the plugin, so a future plugin release that opts into a new type keeps syncing on servers that predate it. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../watch-provider-rating-sync.md | 5 + internal/watchsync/plugin_provider.go | 36 +- internal/watchsync/plugin_provider_state.go | 215 +++++++- internal/watchsync/plugin_provider_test.go | 500 ++++++++++++++++++ 4 files changed, 740 insertions(+), 16 deletions(-) diff --git a/docs/architecture/watch-provider-rating-sync.md b/docs/architecture/watch-provider-rating-sync.md index 05b4cf5949..87944f8d96 100644 --- a/docs/architecture/watch-provider-rating-sync.md +++ b/docs/architecture/watch-provider-rating-sync.md @@ -99,6 +99,11 @@ mark the profile's recommendations stale once per run. A provider that rates only some kinds implements `RatingKindFilter`; items of other kinds are left out of its sync entirely, so they are neither sent nor read as removed. +Plugin providers rate the kinds they list in `supported_media_types`. Removing an +absent rating must answer `APPLIED` or `NO_CHANGE`, so a plugin's `REJECTED` answer to +a rating removal counts as a failure and the removal is retried on the next run. A +complete plugin snapshot with an unreadable rating row covers no kind, because the +row's kind is unknown, and the run records a warning. A provider that records a rated title as watched implements `RatingExportWatchGate`. Silo then sends a new rating of that kind only once the profile has a completed play of diff --git a/internal/watchsync/plugin_provider.go b/internal/watchsync/plugin_provider.go index afc27d232e..c7d56dfdaf 100644 --- a/internal/watchsync/plugin_provider.go +++ b/internal/watchsync/plugin_provider.go @@ -63,11 +63,13 @@ type PluginProvider struct { resolveClient WatchSyncPluginClientResolver resolveConfig WatchSyncPluginConfigResolver repository PluginCredentialRepository + now func() time.Time } const ( watchSyncUnsupportedMovieMediaMessage = "watch sync plugin does not support movie media" watchSyncUnsupportedEpisodeMediaMessage = "watch sync plugin does not support episode media" + watchSyncUnsupportedSeriesMediaMessage = "watch sync plugin does not support series media" watchSyncUnsupportedMediaMessage = "watch sync plugin does not support this media type" watchSyncJSONSchemaNumberType = "number" watchSyncJSONSchemaBooleanType = "boolean" @@ -112,6 +114,7 @@ func NewPluginProvider(options PluginProviderOptions) (*PluginProvider, error) { resolveClient: options.ResolveClient, resolveConfig: options.ResolveConfig, repository: options.Repository, + now: time.Now, }, nil } @@ -173,6 +176,8 @@ func (p *PluginProvider) Capabilities() Capabilities { RemoveWatchlist: p.descriptor.GetRemoveWatchlist(), ProvidesWatchlistOrder: p.descriptor.GetProvidesWatchlistOrder(), ScrobblePlayback: p.descriptor.GetScrobblePlayback(), + ImportRatings: p.descriptor.GetImportRatings(), + ExportRatings: p.descriptor.GetExportRatings(), } } @@ -1152,12 +1157,29 @@ func mediaFromIdentity(mediaItemID, kind, title string, year int, imdbID, tmdbID } } +// SyncsRatingKind reports whether the plugin rates items of kind, from the +// media types it supports. Without it, the host would send a movie-only plugin +// every series rating on each sync and log the rejection. +func (p *PluginProvider) SyncsRatingKind(kind string) bool { + return p.supportsMedia(watchSyncMediaType(kind)) +} + +// mediaFromLocalFavorite builds list and rating media. A series item carries +// its own ids, so its SERIES media has them in external_ids and no series_*. +func mediaFromLocalFavorite(item LocalFavorite) *pluginv1.WatchSyncMedia { + return mediaFromIdentity(item.MediaItemID, item.Kind, item.Title, item.Year, + item.IMDbID, item.TMDBID, item.TVDBID, "", 0, + item.SeriesIMDbID, item.SeriesTMDBID, item.SeriesTVDBID, 0, 0) +} + func watchSyncMediaType(kind string) pluginv1.WatchSyncMediaType { switch strings.ToLower(strings.TrimSpace(kind)) { case historyimport.KindMovie: return pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE case historyimport.KindEpisode: return pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_EPISODE + case historyimport.KindSeries: + return pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES default: return pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_UNSPECIFIED } @@ -1263,12 +1285,20 @@ func supportedWatchSyncMediaTypes(descriptor *pluginv1.WatchSyncProviderDescript for _, mediaType := range media { switch mediaType { case pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, - pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_EPISODE: + pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_EPISODE, + pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES: supported[mediaType] = struct{}{} - default: + case pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_UNSPECIFIED: return nil, fmt.Errorf("advertises unsupported media type %q", mediaType.String()) + default: + // A media type added by a newer SDK is ignored rather than + // rejecting the plugin, so a plugin release that opts into a new + // type keeps its existing sync on servers that predate the type. } } + if len(supported) == 0 { + return nil, errors.New("advertises no media type this server supports") + } return supported, nil } @@ -1286,6 +1316,8 @@ func unsupportedWatchSyncMediaMessage(mediaType pluginv1.WatchSyncMediaType) str return watchSyncUnsupportedMovieMediaMessage case pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_EPISODE: return watchSyncUnsupportedEpisodeMediaMessage + case pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES: + return watchSyncUnsupportedSeriesMediaMessage default: return watchSyncUnsupportedMediaMessage } diff --git a/internal/watchsync/plugin_provider_state.go b/internal/watchsync/plugin_provider_state.go index 360ce50282..73e7650ca1 100644 --- a/internal/watchsync/plugin_provider_state.go +++ b/internal/watchsync/plugin_provider_state.go @@ -17,8 +17,11 @@ const ( pluginProgressCursorKey = "plugin.remote.progress" pluginFavoritesCursorKey = "plugin.remote.favorites" pluginWatchlistCursorKey = "plugin.remote.watchlist" + pluginRatingsCursorKey = "plugin.remote.ratings" maxRemoteStatePages = 10_000 maxRemoteStateItems = 100_000 + + watchSyncIncompleteRatingSnapshotWarning = "watch sync plugin returned unreadable ratings, so ratings missing from this read are left unchanged" ) type pluginRemoteTraversal struct { @@ -138,6 +141,71 @@ func (p *PluginProvider) FetchWatchlistBatch( pluginv1.WatchSyncRemoteStateKind_WATCH_SYNC_REMOTE_STATE_KIND_WATCHLIST) } +// FetchRatings reads the plugin's RATING states. A complete snapshot is the +// full set for every rateable kind the plugin supports; an incremental +// traversal covers no kind, so a rating missing from it stays unknown. A +// complete snapshot with an unreadable rating row also covers no kind: the +// dropped row's kind is unknowable, and its title would otherwise read as +// unrated. +func (p *PluginProvider) FetchRatings( + ctx context.Context, + _ ServerConfig, + conn Connection, +) (RatingImportBatch, error) { + traversal, err := p.listRemoteState(ctx, conn, pluginRatingsCursorKey, + pluginv1.WatchSyncRemoteStateKind_WATCH_SYNC_REMOTE_STATE_KIND_RATING) + if err != nil { + return RatingImportBatch{}, err + } + batch := RatingImportBatch{ + UpdatedCursors: cursorUpdate(pluginRatingsCursorKey, traversal.nextCursor), + Warnings: traversal.warnings, + } + droppedRating := false + for _, state := range traversal.items { + if state.GetRating() == nil { + continue + } + row, err := remoteRatingFromProto(p.Key(), state) + if err != nil { + batch.Warnings = append(batch.Warnings, err.Error()) + // A dropped tombstone reads as absent, which a complete snapshot + // already means removed. Only a dropped rating can hide a title + // that is still rated. + if !state.GetRating().GetRemoved() { + droppedRating = true + } + continue + } + // Silo rates only movies and series, so an episode rating is not an + // error, just nothing to sync. + if !row.Removed && !ratingSyncKind(row.Kind) { + continue + } + batch.Rows = append(batch.Rows, row) + } + if traversal.completeSnapshot { + if droppedRating { + batch.Warnings = append(batch.Warnings, watchSyncIncompleteRatingSnapshotWarning) + } else { + batch.SnapshotKinds = p.rateableKinds() + } + } + return batch, nil +} + +// rateableKinds lists the rating kinds the plugin supports, movies first. +func (p *PluginProvider) rateableKinds() []string { + var kinds []string + if p.supportsMedia(pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE) { + kinds = append(kinds, historyimport.KindMovie) + } + if p.supportsMedia(pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES) { + kinds = append(kinds, historyimport.KindSeries) + } + return kinds +} + func (p *PluginProvider) fetchListState( ctx context.Context, conn Connection, @@ -295,9 +363,7 @@ func (p *PluginProvider) applyListEvents( events := make([]*pluginv1.WatchSyncEvent, 0, len(items)) keys := make([]string, 0, len(items)) for _, item := range items { - media := mediaFromIdentity(item.MediaItemID, item.Kind, item.Title, item.Year, - item.IMDbID, item.TMDBID, item.TVDBID, "", 0, - item.SeriesIMDbID, item.SeriesTMDBID, item.SeriesTVDBID, 0, 0) + media := mediaFromLocalFavorite(item) if !p.supportsMedia(media.GetMediaType()) { result.Failed[item.MediaItemID] = unsupportedWatchSyncMediaMessage(media.GetMediaType()) continue @@ -321,6 +387,70 @@ func (p *PluginProvider) applyListEvents( return mergeExportFailures(applied, result.Failed), err } +// ExportRatings sends SET_RATING events. The event ID carries the rating and +// the local rating time, so a retry of one change reuses its ID while a later +// re-rate to the same value, which moves the rating time, gets a new one. +func (p *PluginProvider) ExportRatings(ctx context.Context, _ ServerConfig, conn Connection, items []LocalRating) (ExportResult, error) { + return p.applyRatingEvents(ctx, conn, items, pluginv1.WatchSyncOperation_WATCH_SYNC_OPERATION_SET_RATING) +} + +// RemoveRatings sends REMOVE_RATING events. The host does not record when a +// rating was cleared, so the event ID carries the send time instead: it never +// repeats for a later removal, and a retry in a later run gets a fresh ID, +// which is safe because clearing an absent rating is a no-op for the plugin. +// For the same reason a REJECTED removal is reported as failed, not not-found. +func (p *PluginProvider) RemoveRatings(ctx context.Context, _ ServerConfig, conn Connection, items []LocalFavorite) (ExportResult, error) { + removals := make([]LocalRating, 0, len(items)) + for _, item := range items { + removals = append(removals, LocalRating{LocalFavorite: item}) + } + return p.applyRatingEvents(ctx, conn, removals, pluginv1.WatchSyncOperation_WATCH_SYNC_OPERATION_REMOVE_RATING) +} + +func (p *PluginProvider) applyRatingEvents( + ctx context.Context, + conn Connection, + items []LocalRating, + operation pluginv1.WatchSyncOperation, +) (ExportResult, error) { + failed := make(map[string]string) + events := make([]*pluginv1.WatchSyncEvent, 0, len(items)) + keys := make([]string, 0, len(items)) + sentAt := p.now().UnixNano() + for _, item := range items { + media := mediaFromLocalFavorite(item.LocalFavorite) + if !p.supportsMedia(media.GetMediaType()) { + failed[item.MediaItemID] = unsupportedWatchSyncMediaMessage(media.GetMediaType()) + continue + } + event := &pluginv1.WatchSyncEvent{ + Operation: operation, + Origin: pluginv1.WatchSyncOrigin_WATCH_SYNC_ORIGIN_MANUAL, + Media: media, + ProviderItemKey: item.ProviderItemKey, + } + if operation == pluginv1.WatchSyncOperation_WATCH_SYNC_OPERATION_SET_RATING { + if item.Rating < 1 || item.Rating > 10 { + failed[item.MediaItemID] = "watch sync rating must be from 1 to 10" + continue + } + ratedAt := int64(0) + if !item.RatedAt.IsZero() { + ratedAt = item.RatedAt.UnixNano() + } + event.EventId = fmt.Sprintf("%s:%s:%d:%d", operation.String(), item.MediaItemID, item.Rating, ratedAt) + event.OccurredAt = timestampOrNil(item.RatedAt) + event.Rating = int32(item.Rating) + } else { + event.EventId = fmt.Sprintf("%s:%s:%d", operation.String(), item.MediaItemID, sentAt) + } + events = append(events, event) + keys = append(keys, item.MediaItemID) + } + applied, err := p.applyPluginEvents(ctx, conn, events, keys) + return mergeExportFailures(applied, failed), err +} + func (p *PluginProvider) applyPluginEvents( ctx context.Context, conn Connection, @@ -377,7 +507,14 @@ func (p *PluginProvider) applyPluginEventsDetailed( pluginv1.WatchSyncApplyStatus_WATCH_SYNC_APPLY_STATUS_NO_CHANGE: result.Sent = append(result.Sent, key) case pluginv1.WatchSyncApplyStatus_WATCH_SYNC_APPLY_STATUS_REJECTED: - result.NotFound = append(result.NotFound, key) + // Clearing an absent rating must be APPLIED or NO_CHANGE, so a + // rejected removal is a failure to retry, not a missing title: + // reading it as cleared would let the rating come back. + if event.GetOperation() == pluginv1.WatchSyncOperation_WATCH_SYNC_OPERATION_REMOVE_RATING { + result.Failed[key] = safeApplyMessage(apply, conn.AccessToken, conn.RefreshToken) + } else { + result.NotFound = append(result.NotFound, key) + } case pluginv1.WatchSyncApplyStatus_WATCH_SYNC_APPLY_STATUS_RETRY: fault := apply.GetFault() if fault.GetCode() == pluginv1.WatchSyncFaultCode_WATCH_SYNC_FAULT_CODE_RATE_LIMITED { @@ -411,6 +548,11 @@ func remoteWatchFromProto(provider string, state *pluginv1.WatchSyncRemoteState) if err != nil { return RemoteWatch{}, err } + // The plugin contract defines SERIES for list and rating state only. A + // series-level watched row would otherwise expand to every local episode. + if identity.kind == historyimport.KindSeries { + return RemoteWatch{}, errors.New("watch sync plugin returned series-level watched state") + } watched := state.GetWatched() if watched.GetPlayCount() < 1 { return RemoteWatch{}, errors.New("watch sync plugin returned watched state with no plays") @@ -431,6 +573,9 @@ func remoteProgressFromProto(provider string, state *pluginv1.WatchSyncRemoteSta if err != nil { return RemoteProgress{}, err } + if identity.kind == historyimport.KindSeries { + return RemoteProgress{}, errors.New("watch sync plugin returned series-level progress state") + } progress := state.GetProgress() if progress.GetProgressPercent() < 0 || progress.GetProgressPercent() >= 100 { return RemoteProgress{}, errors.New("watch sync plugin returned progress outside [0,100)") @@ -469,18 +614,47 @@ func remoteFavoriteFromProto(provider string, state *pluginv1.WatchSyncRemoteSta if err != nil { return RemoteFavorite{}, err } - listedAt := time.Now().UTC() + row := identity.favorite(provider, providerItemKey) + row.FavoritedAt = time.Now().UTC() if value := timePointer(listed.GetListedAt()); value != nil { - listedAt = *value + row.FavoritedAt = *value } - return RemoteFavorite{ - Provider: provider, ProviderItemKey: providerItemKey, - Kind: identity.kind, Title: identity.title, Year: identity.year, - IMDbID: identity.imdbID, TMDBID: identity.tmdbID, TVDBID: identity.tvdbID, - SeriesTitle: identity.seriesTitle, SeriesYear: identity.seriesYear, - SeriesIMDbID: identity.seriesIMDbID, SeriesTMDBID: identity.seriesTMDBID, SeriesTVDBID: identity.seriesTVDBID, - SeasonNumber: identity.season, EpisodeNumber: identity.episode, FavoritedAt: listedAt, - }, nil + return row, nil +} + +// remoteRatingFromProto decodes one RATING state. A tombstone needs only its +// provider key; any other state needs media and a rating from 1 to 10. +func remoteRatingFromProto(provider string, state *pluginv1.WatchSyncRemoteState) (RemoteRating, error) { + rating := state.GetRating() + if rating == nil { + return RemoteRating{}, errors.New("watch sync plugin returned remote state without a rating") + } + providerItemKey := strings.TrimSpace(state.GetProviderItemKey()) + if rating.GetRemoved() { + if providerItemKey == "" { + return RemoteRating{}, errors.New("watch sync plugin returned a rating tombstone without provider identity") + } + return RemoteRating{RemoteFavorite: RemoteFavorite{ + Provider: provider, + ProviderItemKey: providerItemKey, + Removed: true, + }}, nil + } + identity, err := remoteIdentityFromProto(state) + if err != nil { + return RemoteRating{}, err + } + if rating.GetRating() < 1 || rating.GetRating() > 10 { + return RemoteRating{}, fmt.Errorf("watch sync plugin returned an out-of-range rating %d", rating.GetRating()) + } + row := RemoteRating{ + RemoteFavorite: identity.favorite(provider, providerItemKey), + Rating: int(rating.GetRating()), + } + if value := timePointer(rating.GetRatedAt()); value != nil { + row.RatedAt = *value + } + return row, nil } type remoteIdentity struct { @@ -489,6 +663,17 @@ type remoteIdentity struct { year, seriesYear, season, episode int } +func (identity remoteIdentity) favorite(provider, providerItemKey string) RemoteFavorite { + return RemoteFavorite{ + Provider: provider, ProviderItemKey: providerItemKey, + Kind: identity.kind, Title: identity.title, Year: identity.year, + IMDbID: identity.imdbID, TMDBID: identity.tmdbID, TVDBID: identity.tvdbID, + SeriesTitle: identity.seriesTitle, SeriesYear: identity.seriesYear, + SeriesIMDbID: identity.seriesIMDbID, SeriesTMDBID: identity.seriesTMDBID, SeriesTVDBID: identity.seriesTVDBID, + SeasonNumber: identity.season, EpisodeNumber: identity.episode, + } +} + func remoteIdentityFromProto(state *pluginv1.WatchSyncRemoteState) (remoteIdentity, error) { if state == nil || state.GetMedia() == nil || strings.TrimSpace(state.GetProviderItemKey()) == "" { return remoteIdentity{}, errors.New("watch sync plugin returned remote state without identity") @@ -500,6 +685,8 @@ func remoteIdentityFromProto(state *pluginv1.WatchSyncRemoteState) (remoteIdenti kind = historyimport.KindMovie case pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_EPISODE: kind = historyimport.KindEpisode + case pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES: + kind = historyimport.KindSeries default: return remoteIdentity{}, errors.New("watch sync plugin returned unsupported remote media") } diff --git a/internal/watchsync/plugin_provider_test.go b/internal/watchsync/plugin_provider_test.go index ff6249166b..5cb9f7d05c 100644 --- a/internal/watchsync/plugin_provider_test.go +++ b/internal/watchsync/plugin_provider_test.go @@ -3,13 +3,17 @@ package watchsync import ( "context" "encoding/base64" + "encoding/json" "errors" + "fmt" + "slices" "strings" "testing" "time" pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1" "github.com/Silo-Server/silo-server/internal/historyimport" + hostplugins "github.com/Silo-Server/silo-server/internal/plugins" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -32,6 +36,8 @@ type fakeWatchSyncPluginClient struct { devicePollResponse *pluginv1.WatchSyncDeviceAuthorizationServicePollResponse listResponse *pluginv1.WatchSyncListRemoteStateResponse listResponses []*pluginv1.WatchSyncListRemoteStateResponse + applyStatus pluginv1.WatchSyncApplyStatus // answers every event when applyResponse is nil + applyFault *pluginv1.WatchSyncFault // attached to each applyStatus answer applyErr error applyRequest *pluginv1.WatchSyncApplyEventsRequest exchangeRequest *pluginv1.WatchSyncExchangeAPIKeyRequest @@ -78,6 +84,13 @@ func (f *fakeWatchSyncPluginClient) GetAccount(_ context.Context, req *pluginv1. } func (f *fakeWatchSyncPluginClient) ApplyEvents(_ context.Context, req *pluginv1.WatchSyncApplyEventsRequest) (*pluginv1.WatchSyncApplyEventsResponse, error) { f.applyRequest = req + if f.applyResponse == nil && f.applyStatus != pluginv1.WatchSyncApplyStatus_WATCH_SYNC_APPLY_STATUS_UNSPECIFIED { + response := &pluginv1.WatchSyncApplyEventsResponse{} + for _, event := range req.GetEvents() { + response.Results = append(response.Results, &pluginv1.WatchSyncApplyResult{EventId: event.GetEventId(), Status: f.applyStatus, Fault: f.applyFault}) + } + return response, f.applyErr + } return f.applyResponse, f.applyErr } @@ -1280,6 +1293,7 @@ func TestPluginProviderMapsAllCapabilitiesAndListOperations(t *testing.T) { ImportFavorites: true, ExportFavorites: true, RemoveFavorites: true, ImportWatchlist: true, ExportWatchlist: true, RemoveWatchlist: true, ProvidesWatchlistOrder: true, ScrobblePlayback: true, MaxBatchSize: 25, + ImportRatings: true, ExportRatings: true, } client := &fakeWatchSyncPluginClient{} provider := testPluginProviderWithDescriptor(t, client, descriptor) @@ -1288,6 +1302,7 @@ func TestPluginProviderMapsAllCapabilitiesAndListOperations(t *testing.T) { ImportFavorites: true, ExportFavorites: true, RemoveFavorites: true, ImportWatchlist: true, ExportWatchlist: true, RemoveWatchlist: true, ProvidesWatchlistOrder: true, ScrobblePlayback: true, + ImportRatings: true, ExportRatings: true, }) { t.Fatalf("capabilities = %#v", provider.Capabilities()) } @@ -1411,3 +1426,488 @@ func TestPluginProviderForwardsAuthoritativeScrobbleCompletion(t *testing.T) { t.Fatal("incomplete event = true, want false") } } + +const testSeriesMediaID = "series-1" + +func ratingTestDescriptor(media ...pluginv1.WatchSyncMediaType) *pluginv1.WatchSyncProviderDescriptor { + return &pluginv1.WatchSyncProviderDescriptor{ + AuthMethods: []pluginv1.WatchSyncAuthMethod{pluginv1.WatchSyncAuthMethod_WATCH_SYNC_AUTH_METHOD_API_KEY}, + ImportRatings: true, + ExportRatings: true, + SupportedMediaTypes: media, + MaxBatchSize: 25, + } +} + +func remoteRatingState(key string, mediaType pluginv1.WatchSyncMediaType, imdbID string, rating int32, ratedAt *timestamppb.Timestamp) *pluginv1.WatchSyncRemoteState { + return &pluginv1.WatchSyncRemoteState{ + ProviderItemKey: key, + Media: &pluginv1.WatchSyncMedia{MediaType: mediaType, Title: "Title", ExternalIds: map[string]string{"imdb": imdbID}}, + Rating: &pluginv1.WatchSyncRemoteRatingState{Rating: rating, RatedAt: ratedAt}, + } +} + +func TestPluginProviderDecodesRatingSnapshot(t *testing.T) { + ratedAt := time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC) + client := &fakeWatchSyncPluginClient{listResponse: &pluginv1.WatchSyncListRemoteStateResponse{ + CompleteSnapshot: true, + NextCursor: "cursor-2", + Items: []*pluginv1.WatchSyncRemoteState{ + remoteRatingState("m1", pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, "tt1", 8, timestamppb.New(ratedAt)), + remoteRatingState("s1", pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES, "tt2", 7, nil), + // Silo does not sync episode ratings, and a state without rating + // data is not a rating. + remoteRatingState("e1", pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_EPISODE, "tt3", 9, nil), + {ProviderItemKey: "f1", Favorite: &pluginv1.WatchSyncRemoteListState{}}, + }, + }} + provider := testPluginProviderWithDescriptor(t, client, ratingTestDescriptor( + pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, + pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_EPISODE, + pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES, + )) + batch, err := provider.FetchRatings(context.Background(), ServerConfig{}, Connection{ + SyncCursors: map[string]string{pluginRatingsCursorKey: testCursorOne}, + }) + if err != nil { + t.Fatal(err) + } + if len(client.listRequests) != 1 || client.listRequests[0].GetCursor() != testCursorOne || + len(client.listRequests[0].GetStateKinds()) != 1 || + client.listRequests[0].GetStateKinds()[0] != pluginv1.WatchSyncRemoteStateKind_WATCH_SYNC_REMOTE_STATE_KIND_RATING { + t.Fatalf("requests = %#v", client.listRequests) + } + if !slices.Equal(batch.SnapshotKinds, []string{historyimport.KindMovie, historyimport.KindSeries}) || + batch.UpdatedCursors[pluginRatingsCursorKey] != "cursor-2" || len(batch.Warnings) != 0 || len(batch.Rows) != 2 { + t.Fatalf("batch = %#v", batch) + } + movie, series := batch.Rows[0], batch.Rows[1] + if movie.Provider != testPluginProviderKey || movie.ProviderItemKey != "m1" || movie.Kind != historyimport.KindMovie || + movie.IMDbID != "tt1" || movie.Rating != 8 || !movie.RatedAt.Equal(ratedAt) || movie.Removed { + t.Fatalf("movie row = %#v", movie) + } + if series.ProviderItemKey != "s1" || series.Kind != historyimport.KindSeries || series.IMDbID != "tt2" || + series.Rating != 7 || !series.RatedAt.IsZero() { + t.Fatalf("series row = %#v", series) + } + + // The cursor resets with the agreed ratings on an account change. + kept := withoutRatingCursors(map[string]string{pluginRatingsCursorKey: "a", pluginWatchedCursorKey: "b"}) + if _, ok := kept[pluginRatingsCursorKey]; ok || kept[pluginWatchedCursorKey] != "b" { + t.Fatalf("cursors kept after account change = %#v", kept) + } +} + +func TestPluginProviderRatingSnapshotKindsFollowSupportedMedia(t *testing.T) { + for _, tc := range []struct { + name string + media []pluginv1.WatchSyncMediaType + want []string + }{ + {name: "default media", want: []string{historyimport.KindMovie}}, + {name: "series only", media: []pluginv1.WatchSyncMediaType{pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES}, want: []string{historyimport.KindSeries}}, + {name: "episodes only", media: []pluginv1.WatchSyncMediaType{pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_EPISODE}}, + } { + t.Run(tc.name, func(t *testing.T) { + client := &fakeWatchSyncPluginClient{listResponse: &pluginv1.WatchSyncListRemoteStateResponse{CompleteSnapshot: true}} + provider := testPluginProviderWithDescriptor(t, client, ratingTestDescriptor(tc.media...)) + batch, err := provider.FetchRatings(context.Background(), ServerConfig{}, Connection{}) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(batch.SnapshotKinds, tc.want) { + t.Fatalf("SnapshotKinds = %#v, want %#v", batch.SnapshotKinds, tc.want) + } + }) + } +} + +func TestPluginProviderPaginatesIncrementalRatingsWithTombstone(t *testing.T) { + client := &fakeWatchSyncPluginClient{listResponses: []*pluginv1.WatchSyncListRemoteStateResponse{ + { + Items: []*pluginv1.WatchSyncRemoteState{remoteRatingState("m1", pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, "tt1", 6, nil)}, + NextPageToken: "page-2", + }, + { + Items: []*pluginv1.WatchSyncRemoteState{{ProviderItemKey: "m2", Rating: &pluginv1.WatchSyncRemoteRatingState{Removed: true}}}, + NextCursor: "cursor-2", + }, + }} + provider := testPluginProviderWithDescriptor(t, client, ratingTestDescriptor( + pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, + pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES, + )) + batch, err := provider.FetchRatings(context.Background(), ServerConfig{}, Connection{ + SyncCursors: map[string]string{pluginRatingsCursorKey: testCursorOne}, + }) + if err != nil { + t.Fatal(err) + } + if len(client.listRequests) != 2 || client.listRequests[1].GetCursor() != testCursorOne || + client.listRequests[1].GetPageToken() != "page-2" { + t.Fatalf("requests = %#v", client.listRequests) + } + // An incremental read is complete for no kind, so absent ratings stay unknown. + if len(batch.SnapshotKinds) != 0 || batch.UpdatedCursors[pluginRatingsCursorKey] != "cursor-2" || len(batch.Rows) != 2 { + t.Fatalf("batch = %#v", batch) + } + tombstone := batch.Rows[1] + if !tombstone.Removed || tombstone.ProviderItemKey != "m2" || tombstone.Kind != "" || tombstone.Rating != 0 { + t.Fatalf("tombstone = %#v", tombstone) + } +} + +func TestPluginProviderWarnsAndSkipsInvalidRatings(t *testing.T) { + client := &fakeWatchSyncPluginClient{listResponse: &pluginv1.WatchSyncListRemoteStateResponse{ + CompleteSnapshot: true, + Items: []*pluginv1.WatchSyncRemoteState{ + remoteRatingState("zero", pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, "tt0", 0, nil), + remoteRatingState("eleven", pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, "tt11", 11, nil), + remoteRatingState("ten", pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, "tt10", 10, nil), + {Rating: &pluginv1.WatchSyncRemoteRatingState{Removed: true}}, + }, + }} + provider := testPluginProviderWithDescriptor(t, client, ratingTestDescriptor()) + batch, err := provider.FetchRatings(context.Background(), ServerConfig{}, Connection{}) + if err != nil { + t.Fatal(err) + } + if len(batch.Rows) != 1 || batch.Rows[0].Rating != 10 { + t.Fatalf("rows = %#v", batch.Rows) + } + want := []string{ + "watch sync plugin returned an out-of-range rating 0", + "watch sync plugin returned an out-of-range rating 11", + "watch sync plugin returned a rating tombstone without provider identity", + watchSyncIncompleteRatingSnapshotWarning, + } + if !slices.Equal(batch.Warnings, want) { + t.Fatalf("warnings = %#v", batch.Warnings) + } +} + +// A complete snapshot that drops an unreadable rating cannot say which kind the +// row was, so it covers no kind and absent ratings stay unknown. A dropped +// tombstone reads as absent, which the snapshot already treats as removed. +func TestPluginProviderRatingSnapshotWithUnreadableRatingCoversNoKind(t *testing.T) { + valid := remoteRatingState("m1", pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, "tt1", 8, nil) + for _, tc := range []struct { + name string + complete bool + bad *pluginv1.WatchSyncRemoteState + wantKinds []string + wantWarn []string + }{ + { + name: "bad media", complete: true, + bad: remoteRatingState("x", pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_UNSPECIFIED, "tt2", 7, nil), + wantWarn: []string{"watch sync plugin returned unsupported remote media", watchSyncIncompleteRatingSnapshotWarning}, + }, + { + name: "missing key", complete: true, + bad: remoteRatingState("", pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES, "tt2", 7, nil), + wantWarn: []string{"watch sync plugin returned remote state without identity", watchSyncIncompleteRatingSnapshotWarning}, + }, + { + name: "out of range", complete: true, + bad: remoteRatingState("x", pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, "tt2", 11, nil), + wantWarn: []string{"watch sync plugin returned an out-of-range rating 11", watchSyncIncompleteRatingSnapshotWarning}, + }, + { + name: "bad tombstone keeps the snapshot", complete: true, + bad: &pluginv1.WatchSyncRemoteState{Rating: &pluginv1.WatchSyncRemoteRatingState{Removed: true}}, + wantKinds: []string{historyimport.KindMovie, historyimport.KindSeries}, + wantWarn: []string{"watch sync plugin returned a rating tombstone without provider identity"}, + }, + { + name: "incremental read", + bad: remoteRatingState("", pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, "tt2", 7, nil), + wantWarn: []string{"watch sync plugin returned remote state without identity"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + client := &fakeWatchSyncPluginClient{listResponse: &pluginv1.WatchSyncListRemoteStateResponse{ + CompleteSnapshot: tc.complete, + Items: []*pluginv1.WatchSyncRemoteState{valid, tc.bad}, + }} + provider := testPluginProviderWithDescriptor(t, client, ratingTestDescriptor( + pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, + pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES, + )) + batch, err := provider.FetchRatings(context.Background(), ServerConfig{}, Connection{}) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(batch.SnapshotKinds, tc.wantKinds) { + t.Fatalf("SnapshotKinds = %#v, want %#v", batch.SnapshotKinds, tc.wantKinds) + } + if !slices.Equal(batch.Warnings, tc.wantWarn) { + t.Fatalf("warnings = %#v, want %#v", batch.Warnings, tc.wantWarn) + } + if len(batch.Rows) != 1 || batch.Rows[0].ProviderItemKey != "m1" { + t.Fatalf("rows = %#v", batch.Rows) + } + }) + } +} + +func TestPluginProviderMapsSeriesMediaBothWays(t *testing.T) { + series := LocalFavorite{ + MediaItemID: testSeriesMediaID, Kind: historyimport.KindSeries, Title: "Show", Year: 2020, + IMDbID: "tt9", TMDBID: "99", TVDBID: "77", ProviderItemKey: "imdb:tt9", + } + media := mediaFromLocalFavorite(series) + if media.GetMediaType() != pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES || + media.GetTitle() != "Show" || media.GetYear() != 2020 || + media.GetExternalIds()["imdb"] != "tt9" || media.GetExternalIds()["tmdb"] != "99" || media.GetExternalIds()["tvdb"] != "77" || + len(media.GetSeriesExternalIds()) != 0 || media.GetSeasonNumber() != 0 || media.GetEpisodeNumber() != 0 { + t.Fatalf("series media = %#v", media) + } + + client := &fakeWatchSyncPluginClient{listResponses: []*pluginv1.WatchSyncListRemoteStateResponse{ + {CompleteSnapshot: true, Items: []*pluginv1.WatchSyncRemoteState{{ProviderItemKey: "s1", Media: media, Favorite: &pluginv1.WatchSyncRemoteListState{}}}}, + {Items: []*pluginv1.WatchSyncRemoteState{{ProviderItemKey: "s1", Media: media, Watched: &pluginv1.WatchSyncRemoteWatchedState{PlayCount: 1}}}}, + }} + provider := testPluginProviderWithDescriptor(t, client, &pluginv1.WatchSyncProviderDescriptor{ + AuthMethods: []pluginv1.WatchSyncAuthMethod{pluginv1.WatchSyncAuthMethod_WATCH_SYNC_AUTH_METHOD_API_KEY}, + ImportFavorites: true, ImportWatched: true, MaxBatchSize: 25, + SupportedMediaTypes: []pluginv1.WatchSyncMediaType{pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES}, + }) + favorites, err := provider.FetchFavoritesBatch(context.Background(), ServerConfig{}, Connection{}) + if err != nil { + t.Fatal(err) + } + if len(favorites.Rows) != 1 || favorites.Rows[0].Kind != historyimport.KindSeries || favorites.Rows[0].IMDbID != "tt9" || + favorites.Rows[0].TVDBID != "77" || favorites.Rows[0].SeriesIMDbID != "" { + t.Fatalf("favorites = %#v", favorites) + } + // SERIES is not defined for watched state; a series-level row would + // otherwise mark every local episode watched. + watched, err := provider.FetchWatchedBatch(context.Background(), ServerConfig{}, Connection{}) + if err != nil { + t.Fatal(err) + } + if len(watched.Rows) != 0 || len(watched.Warnings) != 1 || !strings.Contains(watched.Warnings[0], "series-level watched") { + t.Fatalf("watched = %#v", watched) + } +} + +func TestPluginProviderSendsSeriesListEventsOnlyWhenSupported(t *testing.T) { + series := LocalFavorite{MediaItemID: testSeriesMediaID, Kind: historyimport.KindSeries, IMDbID: "tt9", ProviderItemKey: "imdb:tt9"} + client := &fakeWatchSyncPluginClient{applyStatus: pluginv1.WatchSyncApplyStatus_WATCH_SYNC_APPLY_STATUS_APPLIED} + provider := testPluginProviderWithDescriptor(t, client, &pluginv1.WatchSyncProviderDescriptor{ + AuthMethods: []pluginv1.WatchSyncAuthMethod{pluginv1.WatchSyncAuthMethod_WATCH_SYNC_AUTH_METHOD_API_KEY}, + ExportFavorites: true, MaxBatchSize: 25, + SupportedMediaTypes: []pluginv1.WatchSyncMediaType{ + pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, + pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES, + }, + }) + result, err := provider.ExportFavorites(context.Background(), ServerConfig{}, Connection{}, []LocalFavorite{series}) + if err != nil { + t.Fatal(err) + } + if len(result.Sent) != 1 || client.applyRequest.GetEvents()[0].GetMedia().GetMediaType() != pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES { + t.Fatalf("result=%#v request=%#v", result, client.applyRequest) + } + + // A plugin that does not list SERIES still never receives series items. + client = &fakeWatchSyncPluginClient{} + result, err = testPluginProvider(t, client).ExportFavorites(context.Background(), ServerConfig{}, Connection{}, []LocalFavorite{series}) + if err != nil { + t.Fatal(err) + } + if client.applyRequest != nil || result.Failed[testSeriesMediaID] != watchSyncUnsupportedSeriesMediaMessage { + t.Fatalf("result=%#v request=%#v", result, client.applyRequest) + } +} + +func TestPluginProviderRatingEventsCarryValueAndDistinctIDs(t *testing.T) { + client := &fakeWatchSyncPluginClient{applyStatus: pluginv1.WatchSyncApplyStatus_WATCH_SYNC_APPLY_STATUS_APPLIED} + provider := testPluginProviderWithDescriptor(t, client, ratingTestDescriptor( + pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, + pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES, + )) + movie := LocalFavorite{MediaItemID: testMovieMediaID, Kind: historyimport.KindMovie, IMDbID: "tt1", ProviderItemKey: "imdb:tt1"} + ratedAt := time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC) + set := func(rating int, at time.Time) *pluginv1.WatchSyncEvent { + t.Helper() + result, err := provider.ExportRatings(context.Background(), ServerConfig{}, Connection{}, []LocalRating{{LocalFavorite: movie, Rating: rating, RatedAt: at}}) + if err != nil || len(result.Sent) != 1 || result.Sent[0] != testMovieMediaID { + t.Fatalf("result=%#v err=%v", result, err) + } + return client.applyRequest.GetEvents()[0] + } + remove := func() *pluginv1.WatchSyncEvent { + t.Helper() + result, err := provider.RemoveRatings(context.Background(), ServerConfig{}, Connection{}, []LocalFavorite{movie}) + if err != nil || len(result.Sent) != 1 || result.Sent[0] != testMovieMediaID { + t.Fatalf("result=%#v err=%v", result, err) + } + return client.applyRequest.GetEvents()[0] + } + + event := set(8, ratedAt) + wantID := fmt.Sprintf("%s:%s:8:%d", pluginv1.WatchSyncOperation_WATCH_SYNC_OPERATION_SET_RATING, testMovieMediaID, ratedAt.UnixNano()) + if event.GetEventId() != wantID || event.GetOperation() != pluginv1.WatchSyncOperation_WATCH_SYNC_OPERATION_SET_RATING || + event.GetRating() != 8 || !event.GetOccurredAt().AsTime().Equal(ratedAt) || event.GetProviderItemKey() != "imdb:tt1" || + event.GetMedia().GetMediaType() != pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE || event.GetMedia().GetExternalIds()["imdb"] != "tt1" { + t.Fatalf("set event = %#v", event) + } + if retry := set(8, ratedAt); retry.GetEventId() != event.GetEventId() { + t.Fatalf("retry event ID = %q, want %q", retry.GetEventId(), event.GetEventId()) + } + if rerated := set(8, ratedAt.Add(time.Hour)); rerated.GetEventId() == event.GetEventId() { + t.Fatal("a later re-rate to the same value reused the event ID") + } + if changed := set(6, ratedAt); changed.GetEventId() == event.GetEventId() { + t.Fatal("a different rating reused the event ID") + } + + removedAt := ratedAt.Add(2 * time.Hour) + provider.now = func() time.Time { return removedAt } + removal := remove() + wantID = fmt.Sprintf("%s:%s:%d", pluginv1.WatchSyncOperation_WATCH_SYNC_OPERATION_REMOVE_RATING, testMovieMediaID, removedAt.UnixNano()) + if removal.GetEventId() != wantID || removal.GetOperation() != pluginv1.WatchSyncOperation_WATCH_SYNC_OPERATION_REMOVE_RATING || + removal.GetRating() != 0 || removal.GetOccurredAt() != nil || removal.GetProviderItemKey() != "imdb:tt1" { + t.Fatalf("remove event = %#v", removal) + } + provider.now = func() time.Time { return removedAt.Add(time.Hour) } + if later := remove(); later.GetEventId() == removal.GetEventId() { + t.Fatal("a later removal reused the event ID") + } +} + +// Clearing an absent rating must answer APPLIED or NO_CHANGE, so a REJECTED +// removal is a failure the service retries. A rejected SET_RATING, like a +// rejected list or watched event, still means the plugin has no such title. +func TestPluginProviderRejectedRatingRemovalFails(t *testing.T) { + client := &fakeWatchSyncPluginClient{ + applyStatus: pluginv1.WatchSyncApplyStatus_WATCH_SYNC_APPLY_STATUS_REJECTED, + applyFault: &pluginv1.WatchSyncFault{SafeMessage: "rating removal refused for " + testSecretValue}, + } + provider := testPluginProviderWithDescriptor(t, client, ratingTestDescriptor(pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE)) + conn := Connection{AccessToken: testSecretValue} + movie := LocalFavorite{MediaItemID: testMovieMediaID, Kind: historyimport.KindMovie, IMDbID: "tt1", ProviderItemKey: "imdb:tt1"} + + removed, err := provider.RemoveRatings(context.Background(), ServerConfig{}, conn, []LocalFavorite{movie}) + if err != nil { + t.Fatal(err) + } + message := removed.Failed[testMovieMediaID] + if len(removed.Sent) != 0 || len(removed.NotFound) != 0 || !strings.HasPrefix(message, "rating removal refused for ") || + strings.Contains(message, testSecretValue) { + t.Fatalf("remove result = %#v", removed) + } + + set, err := provider.ExportRatings(context.Background(), ServerConfig{}, conn, []LocalRating{{LocalFavorite: movie, Rating: 8}}) + if err != nil { + t.Fatal(err) + } + if len(set.Failed) != 0 || !slices.Equal(set.NotFound, []string{testMovieMediaID}) { + t.Fatalf("set result = %#v", set) + } + + listRemoved, err := provider.RemoveFavorites(context.Background(), ServerConfig{}, conn, []LocalFavorite{movie}) + if err != nil { + t.Fatal(err) + } + if len(listRemoved.Failed) != 0 || !slices.Equal(listRemoved.NotFound, []string{testMovieMediaID}) { + t.Fatalf("favorite removal result = %#v", listRemoved) + } +} + +func TestPluginProviderRatingEventsFailUnsupportedMedia(t *testing.T) { + client := &fakeWatchSyncPluginClient{applyStatus: pluginv1.WatchSyncApplyStatus_WATCH_SYNC_APPLY_STATUS_APPLIED} + provider := testPluginProviderWithDescriptor(t, client, ratingTestDescriptor(pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE)) + series := LocalFavorite{MediaItemID: testSeriesMediaID, Kind: historyimport.KindSeries, IMDbID: "tt9"} + result, err := provider.ExportRatings(context.Background(), ServerConfig{}, Connection{}, []LocalRating{ + {LocalFavorite: LocalFavorite{MediaItemID: testMovieMediaID, Kind: historyimport.KindMovie, IMDbID: "tt1"}, Rating: 8}, + {LocalFavorite: series, Rating: 6}, + {LocalFavorite: LocalFavorite{MediaItemID: "movie-2", Kind: historyimport.KindMovie, IMDbID: "tt2"}, Rating: 0}, + }) + if err != nil { + t.Fatal(err) + } + if len(client.applyRequest.GetEvents()) != 1 || len(result.Sent) != 1 || result.Sent[0] != testMovieMediaID || + result.Failed[testSeriesMediaID] != watchSyncUnsupportedSeriesMediaMessage || result.Failed["movie-2"] == "" { + t.Fatalf("result=%#v request=%#v", result, client.applyRequest) + } + + client.applyRequest = nil + result, err = provider.RemoveRatings(context.Background(), ServerConfig{}, Connection{}, []LocalFavorite{series}) + if err != nil { + t.Fatal(err) + } + if client.applyRequest != nil || result.Failed[testSeriesMediaID] != watchSyncUnsupportedSeriesMediaMessage { + t.Fatalf("result=%#v request=%#v", result, client.applyRequest) + } +} + +func TestPluginProviderRatingCapabilitiesSurviveCapabilityStorage(t *testing.T) { + records, err := hostplugins.CapabilityRecordsFromManifest(&pluginv1.PluginManifest{Capabilities: []*pluginv1.CapabilityDescriptor{{ + Type: "watch_sync_provider.v1", Id: testPluginCapabilityID, DisplayName: "AniList", + WatchSyncProvider: ratingTestDescriptor( + pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, + pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES, + ), + }}}) + if err != nil || len(records) != 1 { + t.Fatalf("records=%#v err=%v", records, err) + } + // Capability metadata is stored as JSON and decoded on provider reload. + stored, err := json.Marshal(records[0].Metadata) + if err != nil { + t.Fatal(err) + } + records[0].Metadata = nil + if err := json.Unmarshal(stored, &records[0].Metadata); err != nil { + t.Fatal(err) + } + descriptor, err := hostplugins.DecodeCapability(&records[0]) + if err != nil { + t.Fatal(err) + } + provider := testPluginProviderWithDescriptor(t, &fakeWatchSyncPluginClient{}, descriptor.GetWatchSyncProvider()) + capabilities := provider.Capabilities() + if !capabilities.ImportRatings || !capabilities.ExportRatings || + !provider.supportsMedia(pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES) { + t.Fatalf("capabilities=%#v descriptor=%#v", capabilities, descriptor.GetWatchSyncProvider()) + } +} + +func TestSupportedWatchSyncMediaTypesIgnoresTypesFromNewerSDKs(t *testing.T) { + future := pluginv1.WatchSyncMediaType(99) + supported, err := supportedWatchSyncMediaTypes(&pluginv1.WatchSyncProviderDescriptor{ + SupportedMediaTypes: []pluginv1.WatchSyncMediaType{pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, future}, + }) + if err != nil { + t.Fatalf("a future media type must not reject the plugin: %v", err) + } + if _, ok := supported[future]; ok || len(supported) != 1 { + t.Fatalf("supported = %v, want only movie", supported) + } + if _, err := supportedWatchSyncMediaTypes(&pluginv1.WatchSyncProviderDescriptor{ + SupportedMediaTypes: []pluginv1.WatchSyncMediaType{future}, + }); err == nil { + t.Fatal("a plugin with no media type this server supports must be rejected") + } + if _, err := supportedWatchSyncMediaTypes(&pluginv1.WatchSyncProviderDescriptor{ + SupportedMediaTypes: []pluginv1.WatchSyncMediaType{pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_UNSPECIFIED}, + }); err == nil { + t.Fatal("an unspecified media type must still be rejected") + } +} + +func TestPluginProviderSyncsRatingKindFollowsSupportedMedia(t *testing.T) { + supported, err := supportedWatchSyncMediaTypes(&pluginv1.WatchSyncProviderDescriptor{ + SupportedMediaTypes: []pluginv1.WatchSyncMediaType{pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE}, + }) + if err != nil { + t.Fatal(err) + } + provider := &PluginProvider{supportedMedia: supported} + if !provider.SyncsRatingKind(historyimport.KindMovie) || provider.SyncsRatingKind(historyimport.KindSeries) { + t.Fatal("a movie-only plugin must rate movies and skip series") + } +} From 09c290c50a76a0eaa4bf39c1e0e2da3b5e22bed7 Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Thu, 24 Sep 2026 01:18:54 +0000 Subject: [PATCH 09/20] fix(apiv2): answer provider rate limits with 429 and Retry-After A provider 429 during a watch-provider call, such as Trakt's slow_down answer to a device-code poll, now reaches clients as a rate-limited problem carrying the provider's wait instead of an internal error. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/apiv2/request_lifecycle.go | 7 +++++++ internal/apiv2/request_lifecycle_test.go | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/internal/apiv2/request_lifecycle.go b/internal/apiv2/request_lifecycle.go index 774aff7da5..9f51199650 100644 --- a/internal/apiv2/request_lifecycle.go +++ b/internal/apiv2/request_lifecycle.go @@ -3,6 +3,7 @@ package apiv2 import ( "context" "errors" + "math" "net/http" "strconv" @@ -373,6 +374,12 @@ func watchProviderProblem(err error) *Problem { p := NewProblem(TypeRateLimited, "Watch provider sync recently ran. Try again later.") return p.WithHeader("Retry-After", strconv.Itoa(max(1, cooldown.RetryAfterSeconds))) } + if limited, ok := watchsync.AsRateLimited(err); ok { + // The provider throttled this call (for example Trakt's slow_down answer + // to a device-code poll); pass its wait on so the client backs off. + p := NewProblem(TypeRateLimited, "The watch provider is rate limiting requests. Try again later.") + return p.WithHeader("Retry-After", strconv.Itoa(max(1, int(math.Ceil(limited.RetryAfter.Seconds()))))) + } if watchsync.IsInvalidCredentialError(err) { return NewProblem(TypeValidationFailed, "The watch provider rejected the supplied credential.") } diff --git a/internal/apiv2/request_lifecycle_test.go b/internal/apiv2/request_lifecycle_test.go index e215c305b2..6b9f1c3857 100644 --- a/internal/apiv2/request_lifecycle_test.go +++ b/internal/apiv2/request_lifecycle_test.go @@ -122,6 +122,12 @@ func TestWatchProviderLifecycleErrors(t *testing.T) { if rec.Header().Get("Retry-After") != "42" { t.Fatal("cooldown header lost") } + w.err = watchsync.RateLimitedError{Provider: "trakt", RetryAfter: 1500 * time.Millisecond} + rec = do(t, h, http.MethodPost, Prefix+"/watch-providers/trakt/auth/poll", `{"auth_session_id":"00000000-0000-4000-8000-000000000001"}`, requestOwner) + requireProblem(t, rec, TypeRateLimited) + if rec.Header().Get("Retry-After") != "2" { + t.Fatalf("provider rate limit Retry-After = %q, want 2", rec.Header().Get("Retry-After")) + } } // A built-in provider that rejects the supplied API key is a client problem, From b96821adff8024457c5da3073fa975480b823fed Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Thu, 24 Sep 2026 01:20:09 +0000 Subject: [PATCH 10/20] fix(watchsync): read multi-page Trakt listings twice One title added and another removed between page requests keeps the item count unchanged while shifting page boundaries, so a row can be skipped without any header changing. A listing that spans several pages is now read twice, and the read fails unless both passes return the same rows. Single-page listings are read once. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../watchsync/providers/trakt/history_test.go | 2 +- .../providers/trakt/pagination_test.go | 51 ++++++++++++++++- .../watchsync/providers/trakt/provider.go | 55 ++++++++++++++++--- .../watchsync/providers/trakt/watched_test.go | 2 +- 4 files changed, 96 insertions(+), 14 deletions(-) diff --git a/internal/watchsync/providers/trakt/history_test.go b/internal/watchsync/providers/trakt/history_test.go index d4986a239c..9065fc2a46 100644 --- a/internal/watchsync/providers/trakt/history_test.go +++ b/internal/watchsync/providers/trakt/history_test.go @@ -51,7 +51,7 @@ func TestFetchHistoryImportsEveryPage(t *testing.T) { if err != nil { t.Fatal(err) } - if want := []string{"1", "2", "3"}; !reflect.DeepEqual(pages, want) { + if want := []string{"1", "2", "3", "1", "2", "3"}; !reflect.DeepEqual(pages, want) { t.Fatalf("pages = %v, want %v", pages, want) } want := []struct { diff --git a/internal/watchsync/providers/trakt/pagination_test.go b/internal/watchsync/providers/trakt/pagination_test.go index 6c6a1b79be..841b57d4b5 100644 --- a/internal/watchsync/providers/trakt/pagination_test.go +++ b/internal/watchsync/providers/trakt/pagination_test.go @@ -82,7 +82,7 @@ func TestFetchListsImportEveryPage(t *testing.T) { if err != nil { t.Fatal(err) } - wantRequests := []string{"movies:1", "movies:2", "movies:3", "shows:1", "shows:2", "shows:3"} + wantRequests := []string{"movies:1", "movies:2", "movies:3", "movies:1", "movies:2", "movies:3", "shows:1", "shows:2", "shows:3", "shows:1", "shows:2", "shows:3"} if !reflect.DeepEqual(requests, wantRequests) { t.Fatalf("requests = %v, want %v", requests, wantRequests) } @@ -158,7 +158,7 @@ func TestFetchTraktPagesStopsOnPaginationHeaders(t *testing.T) { name: "page count", headers: map[string]string{"X-Pagination-Limit": "250", "X-Pagination-Page-Count": "2", "X-Pagination-Item-Count": "2"}, bodies: map[string]string{"1": `[{"movie":{"ids":{"tmdb":1}}}]`, "2": `[{"movie":{"ids":{"tmdb":2}}}]`}, - wantPages: []string{"1", "2"}, + wantPages: []string{"1", "2", "1", "2"}, wantRows: 2, }, { @@ -166,7 +166,7 @@ func TestFetchTraktPagesStopsOnPaginationHeaders(t *testing.T) { name: "applied limit", headers: map[string]string{"X-Pagination-Limit": "2"}, bodies: map[string]string{"1": `[{"movie":{"ids":{"tmdb":1}}},{"movie":{"ids":{"tmdb":2}}}]`, "2": `[{"movie":{"ids":{"tmdb":3}}}]`}, - wantPages: []string{"1", "2"}, + wantPages: []string{"1", "2", "1", "2"}, wantRows: 3, }, } { @@ -255,3 +255,48 @@ func TestFetchTraktPagesFailsWhenTheListChangesMidRead(t *testing.T) { t.Fatalf("rows=%v err=%v, want an error and no rows", rows, err) } } + +func TestFetchTraktPagesFailsWhenAnEqualCountChangeShiftsPages(t *testing.T) { + // Between the two passes one title was removed and another added, so the + // item count is unchanged but page 2 now holds a different title. + pass := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + page := r.URL.Query().Get("page") + if page == "1" { + pass++ + } + w.Header().Set("X-Pagination-Item-Count", "251") + w.Header().Set("X-Pagination-Page-Count", "2") + switch { + case page == "1": + writeTraktFixture(t, w, `[{"listed_at":"2026-01-01T00:00:00Z","movie":{"title":"A","ids":{"trakt":1,"tmdb":1}}}]`) + case pass == 1: + writeTraktFixture(t, w, `[{"listed_at":"2026-01-01T00:00:00Z","movie":{"title":"B","ids":{"trakt":2,"tmdb":2}}}]`) + default: + writeTraktFixture(t, w, `[{"listed_at":"2026-01-01T00:00:00Z","movie":{"title":"C","ids":{"trakt":3,"tmdb":3}}}]`) + } + })) + defer server.Close() + + rows, err := fetchTraktPages[traktFavoriteMovie](context.Background(), NewProvider(server.Client(), server.URL), + watchsync.ServerConfig{}, watchsync.Connection{AccessToken: "t"}, "/sync/watchlist/movies", nil) + if err == nil || rows != nil { + t.Fatalf("rows=%v err=%v, want an error and no rows", rows, err) + } +} + +func TestFetchTraktPagesReadsASinglePageOnce(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + w.Header().Set("X-Pagination-Page-Count", "1") + writeTraktFixture(t, w, `[{"listed_at":"2026-01-01T00:00:00Z","movie":{"title":"A","ids":{"trakt":1,"tmdb":1}}}]`) + })) + defer server.Close() + + rows, err := fetchTraktPages[traktFavoriteMovie](context.Background(), NewProvider(server.Client(), server.URL), + watchsync.ServerConfig{}, watchsync.Connection{AccessToken: "t"}, "/sync/watchlist/movies", nil) + if err != nil || len(rows) != 1 || requests != 1 { + t.Fatalf("rows=%d requests=%d err=%v, want one row from one request", len(rows), requests, err) + } +} diff --git a/internal/watchsync/providers/trakt/provider.go b/internal/watchsync/providers/trakt/provider.go index 54b6a917c3..71c065a0f6 100644 --- a/internal/watchsync/providers/trakt/provider.go +++ b/internal/watchsync/providers/trakt/provider.go @@ -11,6 +11,7 @@ import ( "maps" "net/http" "net/url" + "slices" "strconv" "strings" "time" @@ -304,6 +305,11 @@ const ( // always sent; they replace any page or limit in query, and other parameters // such as extended are kept. A failure on any page returns an error and no // rows, so callers never import a partial listing. +// +// Offset pages shift when the list changes mid-read, which can skip or repeat +// a row, and callers treat a skipped row as removed. A listing that spans +// several pages is therefore read twice, and the read fails unless both +// passes return the same rows; the next sync retries it. func fetchTraktPages[T any]( ctx context.Context, p *Provider, @@ -312,33 +318,64 @@ func fetchTraktPages[T any]( path string, query url.Values, ) ([]T, error) { + raw, pages, err := fetchTraktPass(ctx, p, cfg, conn, path, query) + if err != nil { + return nil, err + } + if pages > 1 { + again, _, err := fetchTraktPass(ctx, p, cfg, conn, path, query) + if err != nil { + return nil, err + } + if !slices.EqualFunc(raw, again, func(a, b json.RawMessage) bool { return bytes.Equal(a, b) }) { + return nil, fmt.Errorf("trakt %s changed while it was read", path) + } + } + rows := make([]T, 0, len(raw)) + for _, item := range raw { + var row T + if err := json.Unmarshal(item, &row); err != nil { + return nil, fmt.Errorf("decode trakt response: %w", err) + } + rows = append(rows, row) + } + return rows, nil +} + +// fetchTraktPass reads every page of a listing once and reports how many pages +// it took. A changed X-Pagination-Item-Count between pages fails the pass. +func fetchTraktPass( + ctx context.Context, + p *Provider, + cfg watchsync.ServerConfig, + conn watchsync.Connection, + path string, + query url.Values, +) ([]json.RawMessage, int, error) { params := url.Values{} maps.Copy(params, query) params.Set("limit", strconv.Itoa(traktPageLimit)) - var rows []T + var rows []json.RawMessage itemCount := 0 for page := 1; page <= traktMaxPages; page++ { params.Set("page", strconv.Itoa(page)) - var batch []T + var batch []json.RawMessage header, err := p.doWithHeader(ctx, http.MethodGet, path+"?"+params.Encode(), cfg, conn.AccessToken, nil, &batch) if err != nil { - return nil, err + return nil, 0, err } - // Offset pages shift when the list changes mid-read, which can skip - // an item. A changed item count reveals that, and a skipped item - // would read as removed, so the read fails and is retried next sync. if count, ok := positiveHeaderInt(header, "X-Pagination-Item-Count"); ok { if itemCount != 0 && count != itemCount { - return nil, fmt.Errorf("trakt %s changed while it was read (%d items, then %d)", path, itemCount, count) + return nil, 0, fmt.Errorf("trakt %s changed while it was read (%d items, then %d)", path, itemCount, count) } itemCount = count } rows = append(rows, batch...) if lastTraktPage(header, page, len(batch)) { - return rows, nil + return rows, page, nil } } - return nil, fmt.Errorf("trakt %s did not reach its last page within %d pages", path, traktMaxPages) + return nil, 0, fmt.Errorf("trakt %s did not reach its last page within %d pages", path, traktMaxPages) } // lastTraktPage reports whether page, holding items rows, ends the listing. diff --git a/internal/watchsync/providers/trakt/watched_test.go b/internal/watchsync/providers/trakt/watched_test.go index a24bdd86c5..57bb9cf4c6 100644 --- a/internal/watchsync/providers/trakt/watched_test.go +++ b/internal/watchsync/providers/trakt/watched_test.go @@ -57,7 +57,7 @@ func TestFetchWatchedImportsEveryPageAndEpisodeProgress(t *testing.T) { if err != nil { t.Fatal(err) } - wantRequests := []string{"movies:1", "movies:2", "movies:3", "shows:1", "shows:2", "shows:3"} + wantRequests := []string{"movies:1", "movies:2", "movies:3", "movies:1", "movies:2", "movies:3", "shows:1", "shows:2", "shows:3", "shows:1", "shows:2", "shows:3"} if !reflect.DeepEqual(requests, wantRequests) { t.Fatalf("requests = %v, want %v", requests, wantRequests) } From c38658bb6eae94eef58983e09c49f9318273ad6f Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Thu, 24 Sep 2026 01:33:39 +0000 Subject: [PATCH 11/20] fix(watchsync): keep rating flags off v1 and fence rating imports Address review findings on the rating sync core: - The shared capabilities object is serialized by the frozen /api/v1 responses, so its rating flags were leaking into v1. v2 now projects its own WatchProviderCapabilities type (same schema name and fields, so the v2 contract is unchanged) and the shared type hides the flags. - A sync still in flight when the connection is re-bound to another provider account (possibly on another node) no longer applies the old account's ratings. - The import compare-and-set also compares rated_at, so re-saving the same stars after the sync read them still wins. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/architecture/api-contract.md | 8 +-- internal/api/handlers/watch_providers_test.go | 17 ++++++ internal/apiv2/request_lifecycle.go | 49 ++++++++++++++-- internal/apiv2/schema_names.go | 2 - internal/apiv2/schema_ownership_test.go | 2 +- internal/catalog/ratings_repo.go | 31 ++++++---- internal/catalog/ratings_repo_db_test.go | 29 ++++++++-- internal/watchsync/ratings.go | 23 ++++++-- internal/watchsync/ratings_test.go | 58 +++++++++++++++++-- internal/watchsync/types.go | 8 ++- 10 files changed, 188 insertions(+), 39 deletions(-) diff --git a/docs/architecture/api-contract.md b/docs/architecture/api-contract.md index 7ae7c745a3..f05dfe9c1f 100644 --- a/docs/architecture/api-contract.md +++ b/docs/architecture/api-contract.md @@ -1859,10 +1859,10 @@ web settings page displays the delay. No watch-provider consumers were found in the Apple and Android source inventory. Rating sync settings (`import_ratings_enabled`, `export_ratings_enabled`) and the -rating run counters exist only on v2. The frozen v1 connection and run responses omit -them, and a v1 settings update ignores them. The `import_ratings` and `export_ratings` -capability flags are the exception: v1 and v2 share the capabilities object, so v1 -responses gain those two additive fields. See +rating run counters exist only on v2, as do the `import_ratings` and `export_ratings` +capability flags, which v2 projects through its own `WatchProviderCapabilities` type. The +frozen v1 provider, connection, and run responses omit all of them, and a v1 settings +update ignores the toggles. See [watch-provider-rating-sync.md](watch-provider-rating-sync.md) for the sync rules. ### Webhook connection management diff --git a/internal/api/handlers/watch_providers_test.go b/internal/api/handlers/watch_providers_test.go index dd54c3cae6..d6a37b74e1 100644 --- a/internal/api/handlers/watch_providers_test.go +++ b/internal/api/handlers/watch_providers_test.go @@ -115,6 +115,23 @@ func TestWatchProviderHandlerListsProviders(t *testing.T) { } } +// The frozen v1 contract keeps its original capability fields; rating sync is +// served only by /api/v2. +func TestWatchProviderHandlerOmitsRatingCapabilitiesFromV1(t *testing.T) { + service := stubWatchProviderService{providers: []watchsync.ProviderSummary{{ + Key: "trakt", DisplayName: "Trakt", + Capabilities: watchsync.Capabilities{ImportWatched: true, ImportRatings: true, ExportRatings: true}, + }}} + rec := httptest.NewRecorder() + NewWatchProviderHandler(service).HandleListProviders(rec, httptest.NewRequest(http.MethodGet, "/watch-providers/", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d; body=%s", rec.Code, rec.Body.String()) + } + if body := rec.Body.String(); strings.Contains(body, "ratings") { + t.Fatalf("v1 provider list exposes rating fields: %s", body) + } +} + func TestWatchProviderHandlerReturnsEmptyProviderList(t *testing.T) { handler := NewWatchProviderHandler(stubWatchProviderService{}) diff --git a/internal/apiv2/request_lifecycle.go b/internal/apiv2/request_lifecycle.go index 3261b6abc9..cfa5a7405c 100644 --- a/internal/apiv2/request_lifecycle.go +++ b/internal/apiv2/request_lifecycle.go @@ -202,7 +202,7 @@ func registerRequestLifecycle(reg *Registry, requests RequestLifecycleService, p if err != nil { return nil, serviceProblem(err) } - out.Body.Items = append(out.Body.Items, WatchProviderSummary{Key: row.Key, DisplayName: row.DisplayName, Capabilities: row.Capabilities, ConnectionConfigSchema: schemas}) + out.Body.Items = append(out.Body.Items, WatchProviderSummary{Key: row.Key, DisplayName: row.DisplayName, Capabilities: watchProviderCapabilitiesOf(row.Capabilities), ConnectionConfigSchema: schemas}) } return out, nil }) @@ -386,10 +386,51 @@ func watchProviderProblem(err error) *Problem { return NewProblem(TypeInternalError, "Watch-provider operation failed.") } +// WatchProviderCapabilities is the v2 projection of a provider's sync +// capabilities. It is separate from watchsync.Capabilities so the frozen v1 +// responses, which serialize that type, keep their original fields. +type WatchProviderCapabilities struct { + ImportWatched bool `json:"import_watched"` + ImportProgress bool `json:"import_progress"` + ExportWatched bool `json:"export_watched"` + ExportUnwatched bool `json:"export_unwatched"` + ImportFavorites bool `json:"import_favorites"` + ExportFavorites bool `json:"export_favorites"` + RemoveFavorites bool `json:"remove_favorites"` + ImportWatchlist bool `json:"import_watchlist"` + ExportWatchlist bool `json:"export_watchlist"` + RemoveWatchlist bool `json:"remove_watchlist"` + // ProvidesWatchlistOrder is true when the provider returns its watchlist in a + // user-configurable order that Silo can mirror locally. + ProvidesWatchlistOrder bool `json:"provides_watchlist_order"` + ScrobblePlayback bool `json:"scrobble_playback"` + ImportRatings bool `json:"import_ratings"` + ExportRatings bool `json:"export_ratings"` +} + +func watchProviderCapabilitiesOf(c watchsync.Capabilities) WatchProviderCapabilities { + return WatchProviderCapabilities{ + ImportWatched: c.ImportWatched, + ImportProgress: c.ImportProgress, + ExportWatched: c.ExportWatched, + ExportUnwatched: c.ExportUnwatched, + ImportFavorites: c.ImportFavorites, + ExportFavorites: c.ExportFavorites, + RemoveFavorites: c.RemoveFavorites, + ImportWatchlist: c.ImportWatchlist, + ExportWatchlist: c.ExportWatchlist, + RemoveWatchlist: c.RemoveWatchlist, + ProvidesWatchlistOrder: c.ProvidesWatchlistOrder, + ScrobblePlayback: c.ScrobblePlayback, + ImportRatings: c.ImportRatings, + ExportRatings: c.ExportRatings, + } +} + type WatchProviderConnection struct { Provider string `json:"provider"` DisplayName string `json:"display_name"` - Capabilities watchsync.Capabilities `json:"capabilities"` + Capabilities WatchProviderCapabilities `json:"capabilities"` AuthMethod string `json:"auth_method"` Connected bool `json:"connected"` ProviderUsername string `json:"provider_username,omitempty"` @@ -426,7 +467,7 @@ func watchProviderConnectionOf(s watchsync.ConnectionStatus) (WatchProviderConne return WatchProviderConnection{ Provider: s.Provider, DisplayName: s.DisplayName, - Capabilities: s.Capabilities, + Capabilities: watchProviderCapabilitiesOf(s.Capabilities), AuthMethod: s.AuthMethod, Connected: s.Connected, ProviderUsername: s.ProviderUsername, @@ -530,6 +571,6 @@ var requestLifecycleOperationIDs = []string{"getRequestStatus", "cancelRequest", type WatchProviderSummary struct { Key string `json:"key"` DisplayName string `json:"display_name"` - Capabilities watchsync.Capabilities `json:"capabilities"` + Capabilities WatchProviderCapabilities `json:"capabilities"` ConnectionConfigSchema []AdminPluginConfigSchema `json:"connection_config_schema,omitempty"` } diff --git a/internal/apiv2/schema_names.go b/internal/apiv2/schema_names.go index 1420f98c41..be6353ef79 100644 --- a/internal/apiv2/schema_names.go +++ b/internal/apiv2/schema_names.go @@ -7,7 +7,6 @@ import ( "github.com/Silo-Server/silo-server/internal/collections/templates" "github.com/Silo-Server/silo-server/internal/diagnostics" "github.com/Silo-Server/silo-server/internal/onboarding" - "github.com/Silo-Server/silo-server/internal/watchsync" "github.com/danielgtaylor/huma/v2" ) @@ -19,7 +18,6 @@ var domainSchemaNames = map[reflect.Type]string{ reflect.TypeFor[templates.Template](): "CollectionTemplate", reflect.TypeFor[onboarding.Step](): "OnboardingStep", reflect.TypeFor[onboarding.Flow](): "OnboardingFlow", - reflect.TypeFor[watchsync.Capabilities](): "WatchProviderCapabilities", reflect.TypeFor[diagnostics.IngestResult](): "DiagnosticsIngestResult", reflect.TypeFor[catalogseed.PathRewrite](): "CatalogImportPathRewrite", } diff --git a/internal/apiv2/schema_ownership_test.go b/internal/apiv2/schema_ownership_test.go index 898d37d872..25e0552a2d 100644 --- a/internal/apiv2/schema_ownership_test.go +++ b/internal/apiv2/schema_ownership_test.go @@ -26,7 +26,7 @@ var approvedDomainSchemaTypes = map[string]string{ "github.com/Silo-Server/silo-server/internal/requests": "FeatureStatus RouterOption", "github.com/Silo-Server/silo-server/internal/streamtelemetry": "ParityMismatch ParityReport", "github.com/Silo-Server/silo-server/internal/subtitles/ai": "QuotaStatus", - "github.com/Silo-Server/silo-server/internal/watchsync": "Capabilities ConnectionUpdate", + "github.com/Silo-Server/silo-server/internal/watchsync": "ConnectionUpdate", "github.com/Silo-Server/silo-server/internal/webhooksync": "DiscoveredUser RotateWebhookResult UpdateConnectionInput", "github.com/danielgtaylor/huma/v2": "FormFile", } diff --git a/internal/catalog/ratings_repo.go b/internal/catalog/ratings_repo.go index 88255a1c47..bac505845d 100644 --- a/internal/catalog/ratings_repo.go +++ b/internal/catalog/ratings_repo.go @@ -53,14 +53,23 @@ func (r *RatingsRepo) Set(ctx context.Context, userID int, profileID, mediaItemI return nil } +// ObservedRating is the rating and rating time a caller read earlier, used to +// detect a change made since. A zero Rating means the item was unrated. +type ObservedRating struct { + Rating int + RatedAt time.Time +} + // SetIfUnchanged writes a rating observed elsewhere (a watch provider) only if -// the item's rating still equals expected, where 0 means unrated. ratedAt is -// the time the rating was made. It reports whether the write applied; false -// means a concurrent change won, and the caller should re-read before retrying. -func (r *RatingsRepo) SetIfUnchanged(ctx context.Context, userID int, profileID, mediaItemID string, expected, rating int, ratedAt time.Time) (bool, error) { +// the item's rating is still the observed one. Every local write stamps +// rated_at, so comparing it too catches a change back to the same value. +// ratedAt is the time the new rating was made. It reports whether the write +// applied; false means a concurrent change won, and the caller should re-read +// before retrying. +func (r *RatingsRepo) SetIfUnchanged(ctx context.Context, userID int, profileID, mediaItemID string, observed ObservedRating, rating int, ratedAt time.Time) (bool, error) { var query string args := []any{userID, profileID, mediaItemID, rating, ratedAt} - if expected == 0 { + if observed.Rating == 0 { query = ` INSERT INTO user_ratings (user_id, profile_id, media_item_id, rating, rated_at) VALUES ($1, $2, $3, $4, $5) @@ -68,8 +77,8 @@ func (r *RatingsRepo) SetIfUnchanged(ctx context.Context, userID int, profileID, } else { query = ` UPDATE user_ratings SET rating = $4, rated_at = $5 - WHERE user_id = $1 AND profile_id = $2 AND media_item_id = $3 AND rating = $6` - args = append(args, expected) + WHERE user_id = $1 AND profile_id = $2 AND media_item_id = $3 AND rating = $6 AND rated_at = $7` + args = append(args, observed.Rating, observed.RatedAt) } tag, err := r.pool.Exec(ctx, query, args...) if err != nil { @@ -78,13 +87,13 @@ func (r *RatingsRepo) SetIfUnchanged(ctx context.Context, userID int, profileID, return tag.RowsAffected() == 1, nil } -// DeleteIfUnchanged removes a rating only if it still equals expected. It +// DeleteIfUnchanged removes a rating only if it is still the observed one. It // reports whether a row was deleted. -func (r *RatingsRepo) DeleteIfUnchanged(ctx context.Context, userID int, profileID, mediaItemID string, expected int) (bool, error) { +func (r *RatingsRepo) DeleteIfUnchanged(ctx context.Context, userID int, profileID, mediaItemID string, observed ObservedRating) (bool, error) { tag, err := r.pool.Exec(ctx, ` DELETE FROM user_ratings - WHERE user_id = $1 AND profile_id = $2 AND media_item_id = $3 AND rating = $4`, - userID, profileID, mediaItemID, expected, + WHERE user_id = $1 AND profile_id = $2 AND media_item_id = $3 AND rating = $4 AND rated_at = $5`, + userID, profileID, mediaItemID, observed.Rating, observed.RatedAt, ) if err != nil { return false, fmt.Errorf("delete rating if unchanged: %w", err) diff --git a/internal/catalog/ratings_repo_db_test.go b/internal/catalog/ratings_repo_db_test.go index d83a9812cb..e277656045 100644 --- a/internal/catalog/ratings_repo_db_test.go +++ b/internal/catalog/ratings_repo_db_test.go @@ -33,7 +33,7 @@ func TestRatingsRepoCompareAndSetDB(t *testing.T) { const profile, item = "cas-profile", "cas-movie" ratedAt := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) - applied, err := repo.SetIfUnchanged(ctx, userID, profile, item, 0, 4, ratedAt) + applied, err := repo.SetIfUnchanged(ctx, userID, profile, item, ObservedRating{}, 4, ratedAt) if err != nil || !applied { t.Fatalf("insert when unrated: applied=%v err=%v", applied, err) } @@ -41,22 +41,39 @@ func TestRatingsRepoCompareAndSetDB(t *testing.T) { if err != nil || got == nil || got.Rating != 4 || !got.RatedAt.Equal(ratedAt) { t.Fatalf("stored rating = %#v err=%v, want 4 at the provider time", got, err) } - if applied, _ := repo.SetIfUnchanged(ctx, userID, profile, item, 0, 5, ratedAt); applied { + if applied, _ := repo.SetIfUnchanged(ctx, userID, profile, item, ObservedRating{}, 5, ratedAt); applied { t.Fatal("expecting unrated must not overwrite an existing rating") } - if applied, _ := repo.SetIfUnchanged(ctx, userID, profile, item, 3, 5, ratedAt); applied { + if applied, _ := repo.SetIfUnchanged(ctx, userID, profile, item, ObservedRating{Rating: 3, RatedAt: ratedAt}, 5, ratedAt); applied { t.Fatal("a stale expected value must not apply") } - if applied, _ := repo.DeleteIfUnchanged(ctx, userID, profile, item, 3); applied { + if applied, _ := repo.DeleteIfUnchanged(ctx, userID, profile, item, ObservedRating{Rating: 3, RatedAt: ratedAt}); applied { t.Fatal("a stale expected value must not delete") } + // A re-save of the same stars after the observation moves rated_at, so + // the stale observation no longer matches. + if err := repo.Set(ctx, userID, profile, item, 4); err != nil { + t.Fatal(err) + } + if applied, _ := repo.SetIfUnchanged(ctx, userID, profile, item, ObservedRating{Rating: 4, RatedAt: ratedAt}, 2, ratedAt); applied { + t.Fatal("a same-value re-save after the observation must win") + } + if applied, _ := repo.DeleteIfUnchanged(ctx, userID, profile, item, ObservedRating{Rating: 4, RatedAt: ratedAt}); applied { + t.Fatal("a same-value re-save after the observation must survive a stale delete") + } + resaved, err := repo.Get(ctx, userID, profile, item) + if err != nil || resaved == nil { + t.Fatalf("resaved = %#v err=%v", resaved, err) + } + ratedAt = resaved.RatedAt + // Two writers expecting the same value: exactly one wins. var wg sync.WaitGroup results := make(chan bool, 2) for _, rating := range []int{1, 2} { wg.Go(func() { - ok, err := repo.SetIfUnchanged(ctx, userID, profile, item, 4, rating, ratedAt) + ok, err := repo.SetIfUnchanged(ctx, userID, profile, item, ObservedRating{Rating: 4, RatedAt: ratedAt}, rating, ratedAt) if err != nil { t.Error(err) } @@ -79,7 +96,7 @@ func TestRatingsRepoCompareAndSetDB(t *testing.T) { if err != nil || current == nil { t.Fatalf("current = %#v err=%v", current, err) } - if applied, err := repo.DeleteIfUnchanged(ctx, userID, profile, item, current.Rating); err != nil || !applied { + if applied, err := repo.DeleteIfUnchanged(ctx, userID, profile, item, ObservedRating{Rating: current.Rating, RatedAt: current.RatedAt}); err != nil || !applied { t.Fatalf("delete with the current value: applied=%v err=%v", applied, err) } if gone, _ := repo.Get(ctx, userID, profile, item); gone != nil { diff --git a/internal/watchsync/ratings.go b/internal/watchsync/ratings.go index a2adabe01e..4351e9e650 100644 --- a/internal/watchsync/ratings.go +++ b/internal/watchsync/ratings.go @@ -49,8 +49,8 @@ const ( type ratingStore interface { ListAll(ctx context.Context, userID int, profileID string) ([]catalog.UserRating, error) Get(ctx context.Context, userID int, profileID, mediaItemID string) (*catalog.UserRating, error) - SetIfUnchanged(ctx context.Context, userID int, profileID, mediaItemID string, expected, rating int, ratedAt time.Time) (bool, error) - DeleteIfUnchanged(ctx context.Context, userID int, profileID, mediaItemID string, expected int) (bool, error) + SetIfUnchanged(ctx context.Context, userID int, profileID, mediaItemID string, observed catalog.ObservedRating, rating int, ratedAt time.Time) (bool, error) + DeleteIfUnchanged(ctx context.Context, userID int, profileID, mediaItemID string, observed catalog.ObservedRating) (bool, error) } // ratingProfileStaler marks a profile's recommendations stale after imports @@ -132,6 +132,11 @@ type ratingItem struct { remoteKey string } +// observedLocal is the local rating this sync read, for compare-and-set writes. +func (item *ratingItem) observedLocal() catalog.ObservedRating { + return catalog.ObservedRating{Rating: item.local, RatedAt: item.localAt} +} + // providerKey is the key recorded for an item and sent with its writes: the // provider's own key once a read returned one, so the provider's tombstones and // writes can name it, and otherwise Silo's key. @@ -237,6 +242,16 @@ func (s *Service) syncRatings(ctx context.Context, conn Connection, cfg ServerCo markRemoteUnknown(items) } + // The provider read can take a while, and the connection can be re-bound + // to another account meanwhile (on any node; the sync lock is local). Its + // ratings must not be applied to the profile once the account changed. + if current, err := s.reloadConnection(ctx, conn); err != nil { + return result, err + } else if current.ProviderAccountID != conn.ProviderAccountID { + result.Warnings = append(result.Warnings, "the connection moved to another provider account during the sync; ratings were not applied") + return result, nil + } + applied, err := s.reconcileRatings(ctx, conn, cfg, provider, items, importAllowed, exportAllowed, func() error { if !canImport { return nil @@ -736,13 +751,13 @@ func (s *Service) reconcileRatings( func (s *Service) importRating(ctx context.Context, conn Connection, item *ratingItem) (bool, error) { id := item.identity.MediaItemID if item.remote == 0 { - return s.ratings.DeleteIfUnchanged(ctx, conn.UserID, conn.ProfileID, id, item.local) + return s.ratings.DeleteIfUnchanged(ctx, conn.UserID, conn.ProfileID, id, item.observedLocal()) } ratedAt := item.remoteAt if ratedAt.IsZero() { ratedAt = s.now() } - return s.ratings.SetIfUnchanged(ctx, conn.UserID, conn.ProfileID, id, item.local, item.remote, ratedAt) + return s.ratings.SetIfUnchanged(ctx, conn.UserID, conn.ProfileID, id, item.observedLocal(), item.remote, ratedAt) } // gateRatingExports holds back new ratings that a provider would record as a diff --git a/internal/watchsync/ratings_test.go b/internal/watchsync/ratings_test.go index 629938fd65..848ef96d80 100644 --- a/internal/watchsync/ratings_test.go +++ b/internal/watchsync/ratings_test.go @@ -770,6 +770,14 @@ func (s *fakeRatingStore) remove(id string) { delete(s.ratings, id) } func (s *fakeRatingStore) stars(id string) int { return s.ratings[id].Rating } +func (s *fakeRatingStore) matches(id string, observed catalog.ObservedRating) bool { + current, ok := s.ratings[id] + if observed.Rating == 0 { + return !ok + } + return ok && current.Rating == observed.Rating && current.RatedAt.Equal(observed.RatedAt) +} + func (s *fakeRatingStore) ListAll(_ context.Context, _ int, _ string) ([]catalog.UserRating, error) { all := make([]catalog.UserRating, 0, len(s.ratings)) for _, rating := range s.ratings { @@ -787,16 +795,16 @@ func (s *fakeRatingStore) Get(_ context.Context, _ int, _ string, id string) (*c return &rating, nil } -func (s *fakeRatingStore) SetIfUnchanged(_ context.Context, _ int, _ string, id string, expected, rating int, ratedAt time.Time) (bool, error) { - if s.conflicts[id] || s.stars(id) != expected { +func (s *fakeRatingStore) SetIfUnchanged(_ context.Context, _ int, _ string, id string, observed catalog.ObservedRating, rating int, ratedAt time.Time) (bool, error) { + if s.conflicts[id] || !s.matches(id, observed) { return false, nil } s.ratings[id] = catalog.UserRating{UserID: ratingTestUserID, ProfileID: ratingTestProfileID, MediaItemID: id, Rating: rating, RatedAt: ratedAt} return true, nil } -func (s *fakeRatingStore) DeleteIfUnchanged(_ context.Context, _ int, _ string, id string, expected int) (bool, error) { - if s.conflicts[id] || s.stars(id) != expected { +func (s *fakeRatingStore) DeleteIfUnchanged(_ context.Context, _ int, _ string, id string, observed catalog.ObservedRating) (bool, error) { + if s.conflicts[id] || !s.matches(id, observed) { return false, nil } delete(s.ratings, id) @@ -861,6 +869,7 @@ type ratingProviderStub struct { exportErr error gateMovies bool onExport func() + onFetch func() // fetchedCursors are the cursors the last FetchRatings call received. fetchedCursors map[string]string // kinds limits the rated kinds when set; nil rates every kind. @@ -884,6 +893,9 @@ func (*ratingProviderStub) ConnectWithAPIKey(context.Context, string) (TokenSet, func (p *ratingProviderStub) FetchRatings(_ context.Context, _ ServerConfig, conn Connection) (RatingImportBatch, error) { p.fetches++ p.fetchedCursors = conn.SyncCursors + if p.onFetch != nil { + p.onFetch() + } return p.batch, nil } @@ -914,3 +926,41 @@ func (p *ratingProviderStub) RemoveRatings(_ context.Context, _ ServerConfig, _ func (p *ratingProviderStub) RatingExportRequiresWatched(kind string) bool { return p.gateMovies && kind == historyimport.KindMovie } + +func TestSyncRatingsSkipsApplyingAfterAnAccountRebind(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 3) + h.provider.batch = RatingImportBatch{Rows: []RemoteRating{h.remoteRow(ratingTestMovieB, 7)}, SnapshotKinds: []string{historyimport.KindMovie}} + // The connection is re-bound while the provider read is in flight. + h.provider.onFetch = func() { + key := connectionKey(h.conn.Provider, h.conn.UserID, h.conn.ProfileID) + rebound := h.repo.connections[key] + rebound.ProviderAccountID = "another-account" + h.repo.connections[key] = rebound + } + + result := h.sync() + + if h.store.stars(ratingTestMovieB) != 0 || len(h.provider.exported) != 0 || len(h.repo.ratingStates) != 0 { + t.Fatalf("a stale run must not apply ratings: movieB=%d exported=%#v states=%#v", + h.store.stars(ratingTestMovieB), h.provider.exported, h.repo.ratingStates) + } + if len(result.Warnings) == 0 { + t.Fatal("a skipped stale run should warn") + } +} + +func TestSyncRatingsImportLosesToASameValueResave(t *testing.T) { + h := newRatingHarness(t) + h.store.set(ratingTestMovieA, 4) + h.agree(ratingTestMovieA, 4, true) + h.provider.batch = RatingImportBatch{Rows: []RemoteRating{h.remoteRow(ratingTestMovieA, 2)}, SnapshotKinds: []string{historyimport.KindMovie}} + // The user re-saves the same stars after the sync read them. + h.provider.onFetch = func() { h.store.set(ratingTestMovieA, 4) } + + result := h.sync() + + if got := h.store.stars(ratingTestMovieA); got != 4 || result.Imported != 0 { + t.Fatalf("movie A = %d stars, imported %d; a newer local save must win", got, result.Imported) + } +} diff --git a/internal/watchsync/types.go b/internal/watchsync/types.go index aa0ac078a5..86623bee70 100644 --- a/internal/watchsync/types.go +++ b/internal/watchsync/types.go @@ -27,9 +27,11 @@ type Capabilities struct { ProvidesWatchlistOrder bool `json:"provides_watchlist_order"` ScrobblePlayback bool `json:"scrobble_playback"` // ImportRatings and ExportRatings cover movie and series ratings. - // ExportRatings means the provider can both set and clear a rating. - ImportRatings bool `json:"import_ratings"` - ExportRatings bool `json:"export_ratings"` + // ExportRatings means the provider can both set and clear a rating. They + // are served only by /api/v2, which projects them explicitly; the frozen + // v1 responses keep their original capability fields. + ImportRatings bool `json:"-"` + ExportRatings bool `json:"-"` } // ListKind identifies which personal list a sync operates on. The favorites and From 8f56199d44201e4da28ae379b1a7816b10170fa0 Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Thu, 24 Sep 2026 01:37:02 +0000 Subject: [PATCH 12/20] fix(watchsync): treat plugin rows without a rating as unreadable A complete RATING snapshot containing a nil item or a state without a rating payload silently dropped it and still claimed every supported kind, so a still-rated title could read as removed. Such rows now make the snapshot cover no kind, with a warning, like other unreadable rows. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/watchsync/plugin_provider_state.go | 5 +++++ internal/watchsync/plugin_provider_test.go | 14 +++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/internal/watchsync/plugin_provider_state.go b/internal/watchsync/plugin_provider_state.go index 73e7650ca1..62fbf11a1f 100644 --- a/internal/watchsync/plugin_provider_state.go +++ b/internal/watchsync/plugin_provider_state.go @@ -163,7 +163,12 @@ func (p *PluginProvider) FetchRatings( } droppedRating := false for _, state := range traversal.items { + // A RATING traversal must return rating state only. A missing item or + // rating payload may hide a title that is still rated, so it is + // unreadable like a malformed rating. if state.GetRating() == nil { + batch.Warnings = append(batch.Warnings, "watch sync plugin returned remote state without a rating") + droppedRating = true continue } row, err := remoteRatingFromProto(p.Key(), state) diff --git a/internal/watchsync/plugin_provider_test.go b/internal/watchsync/plugin_provider_test.go index 5cb9f7d05c..933acfb476 100644 --- a/internal/watchsync/plugin_provider_test.go +++ b/internal/watchsync/plugin_provider_test.go @@ -1455,10 +1455,8 @@ func TestPluginProviderDecodesRatingSnapshot(t *testing.T) { Items: []*pluginv1.WatchSyncRemoteState{ remoteRatingState("m1", pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, "tt1", 8, timestamppb.New(ratedAt)), remoteRatingState("s1", pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_SERIES, "tt2", 7, nil), - // Silo does not sync episode ratings, and a state without rating - // data is not a rating. + // Silo does not sync episode ratings. remoteRatingState("e1", pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_EPISODE, "tt3", 9, nil), - {ProviderItemKey: "f1", Favorite: &pluginv1.WatchSyncRemoteListState{}}, }, }} provider := testPluginProviderWithDescriptor(t, client, ratingTestDescriptor( @@ -1613,6 +1611,16 @@ func TestPluginProviderRatingSnapshotWithUnreadableRatingCoversNoKind(t *testing bad: remoteRatingState("x", pluginv1.WatchSyncMediaType_WATCH_SYNC_MEDIA_TYPE_MOVIE, "tt2", 11, nil), wantWarn: []string{"watch sync plugin returned an out-of-range rating 11", watchSyncIncompleteRatingSnapshotWarning}, }, + { + name: "missing rating payload", complete: true, + bad: &pluginv1.WatchSyncRemoteState{ProviderItemKey: "x"}, + wantWarn: []string{"watch sync plugin returned remote state without a rating", watchSyncIncompleteRatingSnapshotWarning}, + }, + { + name: "nil item", complete: true, + bad: nil, + wantWarn: []string{"watch sync plugin returned remote state without a rating", watchSyncIncompleteRatingSnapshotWarning}, + }, { name: "bad tombstone keeps the snapshot", complete: true, bad: &pluginv1.WatchSyncRemoteState{Rating: &pluginv1.WatchSyncRemoteRatingState{Removed: true}}, From 97edaf9b8e1d51db62abd6cf8458d139e4693aa1 Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Thu, 24 Sep 2026 02:01:26 +0000 Subject: [PATCH 13/20] fix(watchsync): pace paged Trakt reads within the GET budget Trakt allows 500 authenticated GETs per five minutes. Reading a large listing twice could exceed that, and the 429 discarded both passes, so a very large account retried from page 1 without ever finishing. Paged reads are now paced per token: a burst of 50 pages, then one every 675 ms, which keeps any five-minute window under the limit. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../providers/trakt/pagination_test.go | 29 +++++++++++++++++++ .../watchsync/providers/trakt/provider.go | 17 +++++++++-- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/internal/watchsync/providers/trakt/pagination_test.go b/internal/watchsync/providers/trakt/pagination_test.go index 841b57d4b5..60e3d8c8da 100644 --- a/internal/watchsync/providers/trakt/pagination_test.go +++ b/internal/watchsync/providers/trakt/pagination_test.go @@ -300,3 +300,32 @@ func TestFetchTraktPagesReadsASinglePageOnce(t *testing.T) { t.Fatalf("rows=%d requests=%d err=%v, want one row from one request", len(rows), requests, err) } } + +func TestFetchTraktPagesPacesReadsPerToken(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + w.Header().Set("X-Pagination-Page-Count", "3") + writeTraktFixture(t, w, `[{"listed_at":"2026-01-01T00:00:00Z","movie":{"title":"A","ids":{"trakt":1,"tmdb":1}}}]`) + })) + defer server.Close() + provider := NewProvider(server.Client(), server.URL) + // Two pages at once, then one per hour: the third page must wait, which + // the one-minute deadline refuses before the request is sent. + provider.pages = watchsync.NewCredentialLimiter(time.Hour, 2) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + rows, err := fetchTraktPages[traktFavoriteMovie](ctx, provider, watchsync.ServerConfig{}, + watchsync.Connection{AccessToken: "t"}, "/sync/watchlist/movies", nil) + if err == nil || rows != nil || requests != 2 { + t.Fatalf("rows=%v requests=%d err=%v, want the read limiter to stop the third page", rows, requests, err) + } +} + +func TestTraktPageBudgetStaysUnderTheGETLimit(t *testing.T) { + // Trakt allows 500 authenticated GETs per five minutes. + if perWindow := pageBurst + int((5*time.Minute)/pageInterval); perWindow >= 500 { + t.Fatalf("paged reads allow %d GETs in five minutes, want fewer than 500", perWindow) + } +} diff --git a/internal/watchsync/providers/trakt/provider.go b/internal/watchsync/providers/trakt/provider.go index 71c065a0f6..4086ca0279 100644 --- a/internal/watchsync/providers/trakt/provider.go +++ b/internal/watchsync/providers/trakt/provider.go @@ -28,11 +28,16 @@ const traktExtendedProgress = "progress" // Trakt rate limits, from its API rate-limiting guide: authenticated users get // one POST/PUT/DELETE per second (AUTHED_API_POST_LIMIT) and 500 GETs per -// five minutes (AUTHED_API_GET_LIMIT). A sync's sequential GETs stay well -// inside the GET budget, so only writes are paced. +// five minutes (AUTHED_API_GET_LIMIT). Writes are paced to one per second. +// Paged reads, which a large history can stretch to hundreds of pages (read +// twice for consistency), are paced so any five-minute window stays inside the +// GET budget: a burst of 50 covers ordinary accounts at full speed, and the +// refill keeps burst plus five minutes of refill under 500. const ( writeInterval = time.Second writeBurst = 1 + pageInterval = 675 * time.Millisecond + pageBurst = 50 // A 429 whose Retry-After is this short, which is typical of the // one-second write limit, is retried in place. Longer waits defer the @@ -52,6 +57,8 @@ type Provider struct { baseURL string // writes paces authenticated writes per access token. writes *watchsync.CredentialLimiter + // pages paces paginated reads per access token. + pages *watchsync.CredentialLimiter // sleep waits between in-place rate-limit retries; tests replace it. sleep func(context.Context, time.Duration) error } @@ -68,6 +75,7 @@ func NewProvider(client *http.Client, baseURL string) *Provider { client: client, baseURL: strings.TrimRight(baseURL, "/"), writes: watchsync.NewCredentialLimiter(writeInterval, writeBurst), + pages: watchsync.NewCredentialLimiter(pageInterval, pageBurst), sleep: watchsync.SleepContext, } } @@ -359,6 +367,11 @@ func fetchTraktPass( itemCount := 0 for page := 1; page <= traktMaxPages; page++ { params.Set("page", strconv.Itoa(page)) + if conn.AccessToken != "" { + if err := p.pages.Wait(ctx, conn.AccessToken); err != nil { + return nil, 0, fmt.Errorf("wait for trakt read limiter: %w", err) + } + } var batch []json.RawMessage header, err := p.doWithHeader(ctx, http.MethodGet, path+"?"+params.Encode(), cfg, conn.AccessToken, nil, &batch) if err != nil { From 9555767b46ac1cb17c4ec294af1540c62148db42 Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Thu, 24 Sep 2026 02:09:09 +0000 Subject: [PATCH 14/20] fix(watchsync): defer limiter refusals and Trakt device-code 429s A local limiter refuses at once when the next slot lies past the context deadline. That was an ordinary error, so watched exports that were never sent were marked failed. The refusal is now a RateLimitedError, which leaves the work pending for a later run; a cancelled context still returns its own error. MDBList's limiter gets the same treatment. Starting Trakt device authorization now reports a 429 as a rate limit with the provider's Retry-After, like polling and token refresh. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../watchsync/providers/mdblist/provider.go | 2 +- .../watchsync/providers/simkl/provider.go | 2 +- .../providers/simkl/ratelimit_test.go | 5 ++--- .../watchsync/providers/trakt/provider.go | 9 +++++++- .../providers/trakt/ratelimit_test.go | 18 ++++++++++++--- internal/watchsync/ratelimit.go | 15 +++++++++++++ internal/watchsync/ratelimit_test.go | 22 +++++++++++++++++++ 7 files changed, 64 insertions(+), 9 deletions(-) diff --git a/internal/watchsync/providers/mdblist/provider.go b/internal/watchsync/providers/mdblist/provider.go index b6fd7b89eb..5b85f693b6 100644 --- a/internal/watchsync/providers/mdblist/provider.go +++ b/internal/watchsync/providers/mdblist/provider.go @@ -554,7 +554,7 @@ func (p *Provider) do(ctx context.Context, method string, path string, apiKey st limiter := p.limiter(apiKey) for attempt := 0; ; attempt++ { if err := limiter.Wait(ctx); err != nil { - return fmt.Errorf("wait for mdblist rate limiter: %w", err) + return watchsync.LimiterWaitError(ctx, p.Key(), requestInterval, err) } retryAfter, err := p.doOnce(ctx, method, path, target, payload, out) if err == nil { diff --git a/internal/watchsync/providers/simkl/provider.go b/internal/watchsync/providers/simkl/provider.go index 594060cf7d..dbea238d12 100644 --- a/internal/watchsync/providers/simkl/provider.go +++ b/internal/watchsync/providers/simkl/provider.go @@ -521,7 +521,7 @@ func (p *Provider) do(ctx context.Context, method string, path string, cfg watch for attempt := 0; ; attempt++ { if paced { if err := p.writes.Wait(ctx, token); err != nil { - return fmt.Errorf("wait for simkl write limiter: %w", err) + return watchsync.LimiterWaitError(ctx, p.Key(), writeInterval, err) } } wait, limited, err := p.doOnce(ctx, method, path, cfg, token, payload, out) diff --git a/internal/watchsync/providers/simkl/ratelimit_test.go b/internal/watchsync/providers/simkl/ratelimit_test.go index 86674df060..83cc5f9583 100644 --- a/internal/watchsync/providers/simkl/ratelimit_test.go +++ b/internal/watchsync/providers/simkl/ratelimit_test.go @@ -6,7 +6,6 @@ import ( "io" "net/http" "net/http/httptest" - "strings" "sync/atomic" "testing" "time" @@ -255,8 +254,8 @@ func TestWriteLimiterPacesPerTokenAndLeavesReadsAlone(t *testing.T) { // so the request never reaches the server. Without pacing it would. deadline, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - if err := provider.Start(deadline, rateLimitTestConfig, tokenA, rateLimitTestEvent); err == nil || !strings.Contains(err.Error(), "write limiter") { - t.Fatalf("second write for token-a = %v, want the write limiter to refuse it", err) + if _, limited := watchsync.AsRateLimited(provider.Start(deadline, rateLimitTestConfig, tokenA, rateLimitTestEvent)); !limited { + t.Fatal("second write for token-a must be deferred as rate limited, unsent") } if err := provider.Start(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token-b"}, rateLimitTestEvent); err != nil { t.Fatalf("token-b must not wait behind token-a: %v", err) diff --git a/internal/watchsync/providers/trakt/provider.go b/internal/watchsync/providers/trakt/provider.go index 545937b184..3ac31f3b15 100644 --- a/internal/watchsync/providers/trakt/provider.go +++ b/internal/watchsync/providers/trakt/provider.go @@ -129,6 +129,13 @@ func (p *Provider) StartDeviceAuth( } defer resp.Body.Close() + if resp.StatusCode == http.StatusTooManyRequests { + wait, ok := watchsync.ParseRetryAfter(resp.Header.Get("Retry-After"), time.Now()) + if !ok { + wait = defaultRetryAfter + } + return watchsync.DeviceAuthSession{}, watchsync.RateLimitedError{Provider: p.Key(), RetryAfter: wait} + } if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { return watchsync.DeviceAuthSession{}, fmt.Errorf("trakt device auth request failed: status %d", resp.StatusCode) } @@ -647,7 +654,7 @@ func (p *Provider) do( for attempt := 0; ; attempt++ { if paced { if err := p.writes.Wait(ctx, token); err != nil { - return fmt.Errorf("wait for trakt write limiter: %w", err) + return watchsync.LimiterWaitError(ctx, p.Key(), writeInterval, err) } } wait, limited, err := p.doOnce(ctx, method, path, cfg, token, payload, out) diff --git a/internal/watchsync/providers/trakt/ratelimit_test.go b/internal/watchsync/providers/trakt/ratelimit_test.go index 9055814a06..520d9911e8 100644 --- a/internal/watchsync/providers/trakt/ratelimit_test.go +++ b/internal/watchsync/providers/trakt/ratelimit_test.go @@ -6,7 +6,6 @@ import ( "io" "net/http" "net/http/httptest" - "strings" "sync/atomic" "testing" "time" @@ -175,8 +174,8 @@ func TestWriteLimiterPacesPerTokenAndLeavesReadsAlone(t *testing.T) { // so the request never reaches the server. Without pacing it would. deadline, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - if err := provider.Start(deadline, rateLimitTestConfig, tokenA, event); err == nil || !strings.Contains(err.Error(), "write limiter") { - t.Fatalf("second write for token-a = %v, want the write limiter to refuse it", err) + if _, limited := watchsync.AsRateLimited(provider.Start(deadline, rateLimitTestConfig, tokenA, event)); !limited { + t.Fatal("second write for token-a must be deferred as rate limited, unsent") } if err := provider.Start(context.Background(), rateLimitTestConfig, watchsync.Connection{AccessToken: "token-b"}, event); err != nil { t.Fatalf("token-b must not wait behind token-a: %v", err) @@ -194,3 +193,16 @@ func TestWriteLimiterPacesPerTokenAndLeavesReadsAlone(t *testing.T) { t.Fatalf("server saw %d reads, want 3", reads.Load()) } } + +func TestStartDeviceAuthReportsRateLimits(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "30") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + _, err := NewProvider(server.Client(), server.URL).StartDeviceAuth(context.Background(), rateLimitTestConfig) + limited, ok := watchsync.AsRateLimited(err) + if !ok || limited.RetryAfter != 30*time.Second { + t.Fatalf("err = %v, want a 30s RateLimitedError", err) + } +} diff --git a/internal/watchsync/ratelimit.go b/internal/watchsync/ratelimit.go index db8d8015da..d158c1223a 100644 --- a/internal/watchsync/ratelimit.go +++ b/internal/watchsync/ratelimit.go @@ -51,6 +51,21 @@ func SleepContext(ctx context.Context, d time.Duration) error { } } +// LimiterWaitError classifies a failed local limiter wait. A canceled context +// is returned as is. Otherwise the limiter refused because the next slot lies +// past the context deadline: the request was never sent, so it is reported as +// a RateLimitedError, which leaves the caller's work pending for a later run +// instead of failing it. +func LimiterWaitError(ctx context.Context, provider string, retryAfter time.Duration, err error) error { + if err == nil { + return nil + } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return RateLimitedError{Provider: provider, RetryAfter: retryAfter} +} + // credentialLimiterIdleTTL is how long a credential's limiter may sit unused // before it becomes eligible for removal. const credentialLimiterIdleTTL = 10 * time.Minute diff --git a/internal/watchsync/ratelimit_test.go b/internal/watchsync/ratelimit_test.go index 074357597c..556aee47f2 100644 --- a/internal/watchsync/ratelimit_test.go +++ b/internal/watchsync/ratelimit_test.go @@ -93,3 +93,25 @@ func TestCredentialLimiterDropsIdleCredentials(t *testing.T) { t.Fatalf("got %d limiters after sweep, want 1", len(limiter.limiters)) } } + +func TestLimiterWaitErrorDefersRefusalsButKeepsCancellation(t *testing.T) { + limiter := NewCredentialLimiter(time.Hour, 1) + if err := limiter.Wait(context.Background(), "token"); err != nil { + t.Fatal(err) + } + deadline, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + err := LimiterWaitError(deadline, "trakt", time.Second, limiter.Wait(deadline, "token")) + if limited, ok := AsRateLimited(err); !ok || limited.RetryAfter != time.Second || limited.Provider != "trakt" { + t.Fatalf("refusal = %v, want a one-second RateLimitedError", err) + } + canceled, stop := context.WithCancel(context.Background()) + stop() + err = LimiterWaitError(canceled, "trakt", time.Second, limiter.Wait(canceled, "token")) + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled wait = %v, want context.Canceled", err) + } + if LimiterWaitError(context.Background(), "trakt", time.Second, nil) != nil { + t.Fatal("a successful wait must stay nil") + } +} From 09776ed5ddb78c24abadf9dbeb51974ebd0c0797 Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Thu, 24 Sep 2026 02:10:49 +0000 Subject: [PATCH 15/20] fix(watchsync): keep error classification when masking the MDBList key When a transport error quoted the keyed URL in its own message, the masking fallback replaced it with a plain error, so errors.Is and errors.As no longer matched cancellations, deadlines, or net.Error timeouts. The masked error now answers Is and As from the original, without an Unwrap that would expose its message. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../watchsync/providers/mdblist/provider.go | 15 ++++++++++- .../providers/mdblist/provider_test.go | 26 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/internal/watchsync/providers/mdblist/provider.go b/internal/watchsync/providers/mdblist/provider.go index 13499246ac..e4300b6d7b 100644 --- a/internal/watchsync/providers/mdblist/provider.go +++ b/internal/watchsync/providers/mdblist/provider.go @@ -653,11 +653,24 @@ func requestError(stage, apiKey string, err error) error { // redirect with an unparseable Location header reports that // server-supplied value verbatim. if msg := err.Error(); redactAPIKey(msg, apiKey) != msg { - err = errors.New(redactAPIKey(msg, apiKey)) + err = redactedError{message: redactAPIKey(msg, apiKey), cause: err} } return fmt.Errorf("%s mdblist request: %w", stage, err) } +// redactedError carries a message with the API key masked. It answers +// errors.Is and errors.As from the original error, so cancellations and +// timeouts stay classifiable, but has no Unwrap: walking the chain never +// reaches the original, key-bearing message. +type redactedError struct { + message string + cause error +} + +func (e redactedError) Error() string { return e.message } +func (e redactedError) Is(target error) bool { return errors.Is(e.cause, target) } +func (e redactedError) As(target any) bool { return errors.As(e.cause, target) } + // redactAPIKey masks every occurrence of the API key, raw or query-escaped. func redactAPIKey(text, apiKey string) string { for _, form := range apiKeyForms(apiKey) { diff --git a/internal/watchsync/providers/mdblist/provider_test.go b/internal/watchsync/providers/mdblist/provider_test.go index cd3109eb44..4f2e2a9388 100644 --- a/internal/watchsync/providers/mdblist/provider_test.go +++ b/internal/watchsync/providers/mdblist/provider_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "net" "net/http" @@ -1108,3 +1109,28 @@ func TestRedactAPIKeyMasksRawAndEscapedForms(t *testing.T) { t.Fatal("empty key must not alter text") } } + +func TestRequestErrorKeepsCauseClassificationWhenMaskingTheMessage(t *testing.T) { + const key = "SENTINEL-KEY-123" + cause := fmt.Errorf("GET https://api.mdblist.com/x?apikey=%s: %w", key, context.DeadlineExceeded) + err := requestError("send", key, cause) + for e := err; e != nil; e = errors.Unwrap(e) { + if strings.Contains(e.Error(), key) { + t.Fatalf("chain exposes the key: %q", e.Error()) + } + } + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err = %v, want it to still match context.DeadlineExceeded", err) + } + var netErr net.Error + timeout := &net.OpError{Op: "dial", Err: timeoutError{}} + if !errors.As(requestError("send", key, fmt.Errorf("%s: %w", key, timeout)), &netErr) || !netErr.Timeout() { + t.Fatal("a masked timeout must still be found as a net.Error") + } +} + +type timeoutError struct{} + +func (timeoutError) Error() string { return "i/o timeout" } +func (timeoutError) Timeout() bool { return true } +func (timeoutError) Temporary() bool { return true } From 066263c55786dc4d0db9f1de0d58380d3d3d498a Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Thu, 24 Sep 2026 02:14:53 +0000 Subject: [PATCH 16/20] fix(watchsync): update rating cursors in place and resend ratings changed mid-send Rating cursors now update with a jsonb patch conditioned on the bound account instead of rewriting the whole connection row, so a concurrent settings edit or rebind is not overwritten. A confirmed set or removal that finds the local rating changed during the send resends the current value once. An account rebind drops other accounts' agreed rows only after the new binding is saved. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/watchsync/ratings.go | 58 ++++++++++++++++++++------- internal/watchsync/ratings_db_test.go | 39 +++++++++++++++++- internal/watchsync/ratings_test.go | 34 ++++++++++++++-- internal/watchsync/repository.go | 37 ++++++++++++++--- internal/watchsync/service.go | 22 ++++++---- internal/watchsync/service_test.go | 22 +++++++++- 6 files changed, 177 insertions(+), 35 deletions(-) diff --git a/internal/watchsync/ratings.go b/internal/watchsync/ratings.go index 4351e9e650..487cb2f390 100644 --- a/internal/watchsync/ratings.go +++ b/internal/watchsync/ratings.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "log/slog" - "maps" "strings" "time" @@ -275,22 +274,24 @@ func (s *Service) saveRatingCursors(ctx context.Context, conn Connection, update if fresh.ProviderAccountID != conn.ProviderAccountID { return nil } - cursors := fresh.SyncCursors - if importAllowed && cursors[ratingImportCursorKey] == "" { - cursors = withoutRatingCursors(cursors) + var remove []string + if importAllowed && fresh.SyncCursors[ratingImportCursorKey] == "" { + for key := range fresh.SyncCursors { + if strings.Contains(key, ratingCursorSegment) { + remove = append(remove, key) + } + } } - cursors = mergeSyncCursors(cursors, updated) + set := mergeSyncCursors(nil, updated) if importAllowed { - cursors[ratingImportCursorKey] = "1" + set[ratingImportCursorKey] = "1" } else { - delete(cursors, ratingImportCursorKey) + remove = append(remove, ratingImportCursorKey) } - if maps.Equal(cursors, fresh.SyncCursors) { - return nil - } - fresh.SyncCursors = cursors - _, err = s.repo.UpsertConnection(ctx, fresh) - return err + // Only the rating cursor keys change, and only while the connection is + // still bound to this account: a full-row write from this snapshot could + // otherwise restore the old account over a rebind made meanwhile. + return s.repo.UpdateRatingCursors(ctx, conn.ID, conn.ProviderAccountID, remove, set) } // HandleLocalRatingEvent sends a profile's rating changes to the providers @@ -740,7 +741,7 @@ func (s *Service) reconcileRatings( return result, err } result.warnings = append(result.warnings, deferred...) - sent, warnings, err := s.sendRatings(ctx, conn, cfg, exporter, sets, removals) + sent, warnings, err := s.sendRatings(ctx, conn, cfg, exporter, sets, removals, true) result.sent = sent result.warnings = append(result.warnings, warnings...) return result, err @@ -817,9 +818,13 @@ func (s *Service) gateRatingExports(ctx context.Context, conn Connection, provid // its agreed row: the next read either finds the title unrated, which clears // the row, or finds another provider entry still rated, which is removed in // turn instead of being imported back. -func (s *Service) sendRatings(ctx context.Context, conn Connection, cfg ServerConfig, exporter RatingExporter, sets, removals []*ratingItem) (int, []string, error) { +func (s *Service) sendRatings(ctx context.Context, conn Connection, cfg ServerConfig, exporter RatingExporter, sets, removals []*ratingItem, followUp bool) (int, []string, error) { sent := 0 var warnings []string + // A rating changed while its write was in flight may have been sent by a + // newer event already, which this older write has just overwritten on the + // provider. The current value is sent once more so the provider ends on it. + var changedSets, changedRemovals []*ratingItem for start := 0; start < len(sets); start += ratingExportBatchSize { batch := sets[start:min(start+ratingExportBatchSize, len(sets))] payload := make([]LocalRating, 0, len(batch)) @@ -847,6 +852,16 @@ func (s *Service) sendRatings(ctx context.Context, conn Connection, cfg ServerCo return sent, warnings, err } if current == nil || current.Rating != item.local { + changed := *item + changed.local, changed.localAt = 0, time.Time{} + if current != nil { + changed.local, changed.localAt = current.Rating, current.RatedAt + } + if changed.local > 0 { + changedSets = append(changedSets, &changed) + } else { + changedRemovals = append(changedRemovals, &changed) + } continue } states = append(states, RatingSyncState{ @@ -877,11 +892,24 @@ func (s *Service) sendRatings(ctx context.Context, conn Connection, cfg ServerCo // A provider that no longer knows the title has no rating to clear. if confirmed, missing := exportItemOutcome(result, identity.MediaItemID, identity.ProviderItemKey); confirmed || missing { sent++ + current, err := s.ratings.Get(ctx, conn.UserID, conn.ProfileID, identity.MediaItemID) + if err != nil { + return sent, warnings, err + } + if current != nil { + changed := *item + changed.local, changed.localAt = current.Rating, current.RatedAt + changedSets = append(changedSets, &changed) + } continue } warnings = append(warnings, exportFailureReason(result, identity, "rating removal")+": "+identity.MediaItemID) } } + if followUp && (len(changedSets) > 0 || len(changedRemovals) > 0) { + more, moreWarnings, err := s.sendRatings(ctx, conn, cfg, exporter, changedSets, changedRemovals, false) + return sent + more, append(warnings, moreWarnings...), err + } return sent, warnings, nil } diff --git a/internal/watchsync/ratings_db_test.go b/internal/watchsync/ratings_db_test.go index e60a663446..207e09f744 100644 --- a/internal/watchsync/ratings_db_test.go +++ b/internal/watchsync/ratings_db_test.go @@ -149,7 +149,7 @@ func TestRatingSyncRepositoryDB(t *testing.T) { if states, _ := repo.ListRatingSyncStates(ctx, conn.ID, "", nil); len(states) != 1 { t.Fatalf("states after delete = %#v", states) } - if err := repo.ClearRatingSyncStates(ctx, conn.ID); err != nil { + if err := repo.ClearRatingSyncStates(ctx, conn.ID, "new-account"); err != nil { t.Fatal(err) } if states, _ := repo.ListRatingSyncStates(ctx, conn.ID, "", nil); len(states) != 0 { @@ -179,9 +179,44 @@ func TestRatingSyncRepositoryDB(t *testing.T) { if states, _ := repo.ListRatingSyncStates(ctx, conn.ID, "account-a", nil); len(states) != 0 { t.Fatalf("old account still sees %#v", states) } - if err := repo.ClearRatingSyncStates(ctx, conn.ID); err != nil { + // Clearing keeps the bound account's rows and drops the others. + if err := repo.UpsertRatingSyncStates(ctx, []RatingSyncState{ + {ConnectionID: conn.ID, ProviderAccountID: "account-a", MediaItemID: "m-4", Kind: "movie", SyncedRating: 2}, + }); err != nil { + t.Fatal(err) + } + if err := repo.ClearRatingSyncStates(ctx, conn.ID, "account-b"); err != nil { + t.Fatal(err) + } + if kept, _ := repo.ListRatingSyncStates(ctx, conn.ID, "account-b", nil); len(kept) != 1 { + t.Fatalf("bound account rows after clear = %#v, want m-3 kept", kept) + } + if old, _ := repo.ListRatingSyncStates(ctx, conn.ID, "account-a", nil); len(old) != 0 { + t.Fatalf("previous account rows after clear = %#v", old) + } + if err := repo.ClearRatingSyncStates(ctx, conn.ID, "none"); err != nil { + t.Fatal(err) + } + }) + + t.Run("rating cursors update in place for the bound account only", func(t *testing.T) { + if _, err := pool.Exec(ctx, `UPDATE watch_provider_connections SET provider_account_id='acct', sync_cursors='{"trakt.watched":"w","test.ratings.movies":"old"}' WHERE id=$1::uuid`, conn.ID); err != nil { + t.Fatal(err) + } + if err := repo.UpdateRatingCursors(ctx, conn.ID, "acct", []string{"test.ratings.movies"}, map[string]string{"test.ratings.shows": "s1"}); err != nil { t.Fatal(err) } + if err := repo.UpdateRatingCursors(ctx, conn.ID, "other-acct", nil, map[string]string{"stale": "x"}); err != nil { + t.Fatal(err) + } + fresh, ok, err := repo.GetConnectionByID(ctx, conn.ID) + if err != nil || !ok { + t.Fatalf("reload: %v", err) + } + want := map[string]string{"trakt.watched": "w", "test.ratings.shows": "s1"} + if len(fresh.SyncCursors) != len(want) || fresh.SyncCursors["trakt.watched"] != "w" || fresh.SyncCursors["test.ratings.shows"] != "s1" { + t.Fatalf("cursors = %#v, want %#v", fresh.SyncCursors, want) + } }) t.Run("connection delete cascades", func(t *testing.T) { diff --git a/internal/watchsync/ratings_test.go b/internal/watchsync/ratings_test.go index 848ef96d80..c18eb5c35a 100644 --- a/internal/watchsync/ratings_test.go +++ b/internal/watchsync/ratings_test.go @@ -539,14 +539,36 @@ func TestSyncRatingsIgnoresAgreedRatingsOfAnotherAccount(t *testing.T) { } } -func TestSyncRatingsDoesNotAgreeOnAValueChangedDuringTheSend(t *testing.T) { +func TestSyncRatingsResendsAValueChangedDuringTheSend(t *testing.T) { h := newRatingHarness(t) h.store.set(ratingTestMovieA, 3) - h.provider.onExport = func() { h.store.set(ratingTestMovieA, 5) } + // A newer edit lands while the first write is in flight, so the provider + // may now hold the older value: the current one is sent once more. + edited := false + h.provider.onExport = func() { + if !edited { + edited = true + h.store.set(ratingTestMovieA, 5) + } + } h.provider.batch = RatingImportBatch{SnapshotKinds: []string{historyimport.KindMovie}} h.sync() - if s := h.state(ratingTestMovieA); s != nil { - t.Fatalf("agreed row = %#v; a rating changed during the send must stay pending", s) + if len(h.provider.exported) != 2 || h.provider.exported[1].Rating != 10 { + t.Fatalf("exported = %#v, want the newer 5 stars sent last", h.provider.exported) + } + if s := h.state(ratingTestMovieA); s == nil || s.SyncedRating != 5 { + t.Fatalf("agreed row = %#v, want the resent 5 stars", s) + } +} + +func TestSyncRatingsResendsARatingSetDuringARemoval(t *testing.T) { + h := newRatingHarness(t) + h.agree(ratingTestMovieA, 4, true) + h.provider.onRemove = func() { h.store.set(ratingTestMovieA, 2) } + h.provider.batch = RatingImportBatch{Rows: []RemoteRating{h.remoteRow(ratingTestMovieA, 8)}, SnapshotKinds: []string{historyimport.KindMovie}} + h.sync() + if len(h.provider.removed) != 1 || len(h.provider.exported) != 1 || h.provider.exported[0].Rating != 4 { + t.Fatalf("removed = %#v exported = %#v, want the new rating sent after the removal", h.provider.removed, h.provider.exported) } } @@ -869,6 +891,7 @@ type ratingProviderStub struct { exportErr error gateMovies bool onExport func() + onRemove func() onFetch func() // fetchedCursors are the cursors the last FetchRatings call received. fetchedCursors map[string]string @@ -916,6 +939,9 @@ func (p *ratingProviderStub) ExportRatings(_ context.Context, _ ServerConfig, _ func (p *ratingProviderStub) RemoveRatings(_ context.Context, _ ServerConfig, _ Connection, items []LocalFavorite) (ExportResult, error) { p.removed = append(p.removed, items...) + if p.onRemove != nil { + p.onRemove() + } var result ExportResult for _, item := range items { result.Sent = append(result.Sent, item.MediaItemID, item.ProviderItemKey) diff --git a/internal/watchsync/repository.go b/internal/watchsync/repository.go index cce24aae9e..4fb70396e2 100644 --- a/internal/watchsync/repository.go +++ b/internal/watchsync/repository.go @@ -50,7 +50,8 @@ type Repository interface { ListRatingSyncStates(ctx context.Context, connectionID, providerAccountID string, mediaItemIDs []string) ([]RatingSyncState, error) UpsertRatingSyncStates(ctx context.Context, states []RatingSyncState) error DeleteRatingSyncStates(ctx context.Context, connectionID string, mediaItemIDs []string) error - ClearRatingSyncStates(ctx context.Context, connectionID string) error + ClearRatingSyncStates(ctx context.Context, connectionID, keepAccountID string) error + UpdateRatingCursors(ctx context.Context, connectionID, providerAccountID string, remove []string, set map[string]string) error ListScrobbleConnections(ctx context.Context, userID int, profileID string) ([]Connection, error) UpsertScrobbleSession(ctx context.Context, event ScrobbleEvent, connectionID string, action string) error PrepareConfirmedScrobbleStop(ctx context.Context, event ScrobbleEvent, connectionID string, staleBefore time.Time) (confirmedStopPreparation, time.Time, error) @@ -787,16 +788,42 @@ func (r *PostgresRepository) DeleteRatingSyncStates(ctx context.Context, connect return nil } -// ClearRatingSyncStates forgets every agreed rating of a connection, used when -// the connection is re-bound to a different provider account. -func (r *PostgresRepository) ClearRatingSyncStates(ctx context.Context, connectionID string) error { - _, err := r.pool.Exec(ctx, `DELETE FROM watch_provider_rating_items WHERE connection_id = $1::uuid`, connectionID) +// ClearRatingSyncStates forgets a connection's agreed ratings with every +// provider account other than keepAccountID, used after the connection is +// re-bound to that account. +func (r *PostgresRepository) ClearRatingSyncStates(ctx context.Context, connectionID, keepAccountID string) error { + _, err := r.pool.Exec(ctx, ` + DELETE FROM watch_provider_rating_items + WHERE connection_id = $1::uuid AND provider_account_id <> $2 + `, connectionID, keepAccountID) if err != nil { return fmt.Errorf("clear rating sync states: %w", err) } return nil } +// UpdateRatingCursors removes and sets sync cursor keys in place, only while the +// connection is still bound to providerAccountID. Other cursor keys and every +// other column are left alone, so a concurrent rebind or sync flow is never +// overwritten. +func (r *PostgresRepository) UpdateRatingCursors(ctx context.Context, connectionID, providerAccountID string, remove []string, set map[string]string) error { + if len(remove) == 0 && len(set) == 0 { + return nil + } + if remove == nil { + remove = []string{} + } + _, err := r.pool.Exec(ctx, ` + UPDATE watch_provider_connections + SET sync_cursors = (COALESCE(sync_cursors, '{}'::jsonb) - $3::text[]) || $4::jsonb + WHERE id = $1::uuid AND provider_account_id = $2 + `, connectionID, providerAccountID, remove, encodeSyncCursors(set)) + if err != nil { + return fmt.Errorf("update rating cursors: %w", err) + } + return nil +} + func (r *PostgresRepository) GetMediaDuration(ctx context.Context, mediaItemID string) (float64, error) { var duration float64 err := r.pool.QueryRow(ctx, mediaDurationQuery, mediaItemID).Scan(&duration) diff --git a/internal/watchsync/service.go b/internal/watchsync/service.go index dfe6b7cae5..3314f09103 100644 --- a/internal/watchsync/service.go +++ b/internal/watchsync/service.go @@ -608,12 +608,10 @@ func (s *Service) persistConnection( ImportRatingsEnabled: true, ExportRatingsEnabled: true, } - } else if conn.ProviderAccountID != "" && account.ID != "" && account.ID != conn.ProviderAccountID { - // The agreed ratings describe the previous account. Keeping them would - // read every rating missing from the new account as a removal. - if err := s.repo.ClearRatingSyncStates(ctx, conn.ID); err != nil { - return Connection{}, err - } + } + rebound := ok && conn.ProviderAccountID != "" && account.ID != "" && account.ID != conn.ProviderAccountID + if rebound { + // Rating read cursors belong to the previous account. conn.SyncCursors = withoutRatingCursors(conn.SyncCursors) } conn.Provider = providerKey @@ -624,7 +622,17 @@ func (s *Service) persistConnection( conn.ProviderUsername = account.Username conn.LastError = "" - return s.repo.UpsertConnection(ctx, conn) + saved, err := s.repo.UpsertConnection(ctx, conn) + if err != nil || !rebound { + return saved, err + } + // Agreed ratings are scoped to their account, so the previous account's + // rows are already ignored; dropping them only after the new binding is + // saved means a failed save never leaves the old account without them. + if err := s.repo.ClearRatingSyncStates(ctx, saved.ID, saved.ProviderAccountID); err != nil { + slog.WarnContext(ctx, "failed to clear agreed ratings of a previous provider account", "component", "watchsync", "provider", providerKey, "connection_id", saved.ID, "error", err) + } + return saved, nil } func (s *Service) SyncDueConnections(ctx context.Context) error { diff --git a/internal/watchsync/service_test.go b/internal/watchsync/service_test.go index 399d199602..587fea1797 100644 --- a/internal/watchsync/service_test.go +++ b/internal/watchsync/service_test.go @@ -575,10 +575,10 @@ func (r *serviceFakeRepo) DeleteRatingSyncStates(_ context.Context, connectionID return nil } -func (r *serviceFakeRepo) ClearRatingSyncStates(_ context.Context, connectionID string) error { +func (r *serviceFakeRepo) ClearRatingSyncStates(_ context.Context, connectionID, keepAccountID string) error { kept := r.ratingStates[:0] for _, state := range r.ratingStates { - if state.ConnectionID != connectionID { + if state.ConnectionID != connectionID || state.ProviderAccountID == keepAccountID { kept = append(kept, state) } } @@ -586,6 +586,24 @@ func (r *serviceFakeRepo) ClearRatingSyncStates(_ context.Context, connectionID return nil } +func (r *serviceFakeRepo) UpdateRatingCursors(_ context.Context, connectionID, providerAccountID string, remove []string, set map[string]string) error { + for key, conn := range r.connections { + if conn.ID != connectionID || conn.ProviderAccountID != providerAccountID { + continue + } + cursors := cloneStringMapForTest(conn.SyncCursors) + for _, k := range remove { + delete(cursors, k) + } + for k, v := range set { + cursors[k] = v + } + conn.SyncCursors = cursors + r.connections[key] = conn + } + return nil +} + func (r *serviceFakeRepo) GetListMediaItems(_ context.Context, mediaItemIDs []string) (map[string]LocalFavorite, error) { result := make(map[string]LocalFavorite, len(mediaItemIDs)) for _, id := range mediaItemIDs { From 21eb44c1fb6cbb475c96ddeb2918d6419d3c83c4 Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Thu, 24 Sep 2026 02:55:38 +0000 Subject: [PATCH 17/20] fix(watchsync): fence agreed ratings on the bound account and resend until settled Agreed-rating upserts and deletes now apply only while the connection is still bound to the row's provider account, so a run that outlived a rebind can't take rows back from the new account. Ratings that change while their write is in flight are resent up to three times; if they still haven't settled, the agreed row is dropped so the next merge keeps the newer rating instead of importing a stale provider value. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../watch-provider-rating-sync.md | 15 +++++-- internal/watchsync/ratings.go | 37 +++++++++++++---- internal/watchsync/ratings_db_test.go | 40 +++++++++++++++++-- internal/watchsync/ratings_test.go | 24 +++++++++++ internal/watchsync/repository.go | 15 +++++-- internal/watchsync/service_test.go | 4 +- 6 files changed, 115 insertions(+), 20 deletions(-) diff --git a/docs/architecture/watch-provider-rating-sync.md b/docs/architecture/watch-provider-rating-sync.md index 05b4cf5949..0577ff6d85 100644 --- a/docs/architecture/watch-provider-rating-sync.md +++ b/docs/architecture/watch-provider-rating-sync.md @@ -39,9 +39,17 @@ read confirms the title is unrated; if the provider still reports a rating (for a second entry for the same title), the removal is sent again instead of the rating being imported back. +A local rating that changed while its write was in flight may have been sent already by +a newer event, which the older write then overwrote. So the current value is sent again, +up to three times. If the rating is still changing after that, its agreed row is dropped, +so the next merge sees both sides changed and keeps the newer rating instead of importing +the provider's. + Rows are scoped to the provider account they were agreed with. A connection that moves to another account ignores the old rows, and so does a sync still running for the old -account, so no stale agreement can read as a removal. +account, so no stale agreement can read as a removal. Agreed rows and rating cursors are +written only while the connection is still bound to the account the run read, so a run +that outlives a re-bind cannot write for the old account. The row's `provider_item_key` is the provider's own key for the title once a read has returned one, and Silo's key (`imdb:`, `tmdb:`, or `tvdb:`) before that. Rating writes @@ -107,8 +115,9 @@ the provider already holds a rating for is not held back, and neither are remova ## Identity changes -Re-binding a connection to a different provider account clears its agreed ratings and -drops its rating read cursors, whose keys contain `.ratings`. Whenever ratings move to +Re-binding a connection to a different provider account drops its rating read cursors, +whose keys contain `.ratings`, and, once the new binding is saved, clears agreed ratings +of every other account. Whenever ratings move to another media item (a duplicate merge or a reattribution), the agreed rows move with them; the destination's row wins a collision. A moved row is unconfirmed and forgets its provider key: for the same title the next read confirms it again, and for a different diff --git a/internal/watchsync/ratings.go b/internal/watchsync/ratings.go index 487cb2f390..a6833a63d0 100644 --- a/internal/watchsync/ratings.go +++ b/internal/watchsync/ratings.go @@ -32,6 +32,9 @@ import ( const ( ratingExportBatchSize = 100 + // maxRatingResends bounds how many times sendRatings resends ratings that + // changed while their write was in flight. + maxRatingResends = 3 // ratingCursorSegment marks provider cursor keys that belong to rating // reads (for example "simkl.ratings.movies"), so they reset together with // the agreed ratings when a connection moves to another provider account. @@ -715,7 +718,7 @@ func (s *Service) reconcileRatings( if err := s.repo.UpsertRatingSyncStates(ctx, upserts); err != nil { return result, err } - if err := s.repo.DeleteRatingSyncStates(ctx, conn.ID, deletes); err != nil { + if err := s.repo.DeleteRatingSyncStates(ctx, conn.ID, conn.ProviderAccountID, deletes); err != nil { return result, err } if result.imported > 0 && s.ratingStaler != nil { @@ -741,7 +744,7 @@ func (s *Service) reconcileRatings( return result, err } result.warnings = append(result.warnings, deferred...) - sent, warnings, err := s.sendRatings(ctx, conn, cfg, exporter, sets, removals, true) + sent, warnings, err := s.sendRatings(ctx, conn, cfg, exporter, sets, removals, maxRatingResends) result.sent = sent result.warnings = append(result.warnings, warnings...) return result, err @@ -818,12 +821,14 @@ func (s *Service) gateRatingExports(ctx context.Context, conn Connection, provid // its agreed row: the next read either finds the title unrated, which clears // the row, or finds another provider entry still rated, which is removed in // turn instead of being imported back. -func (s *Service) sendRatings(ctx context.Context, conn Connection, cfg ServerConfig, exporter RatingExporter, sets, removals []*ratingItem, followUp bool) (int, []string, error) { +// +// A rating changed while its write was in flight may have been sent by a +// newer event already, which this older write has just overwritten on the +// provider. The current value is sent again, up to resends more times, so the +// provider ends on it. +func (s *Service) sendRatings(ctx context.Context, conn Connection, cfg ServerConfig, exporter RatingExporter, sets, removals []*ratingItem, resends int) (int, []string, error) { sent := 0 var warnings []string - // A rating changed while its write was in flight may have been sent by a - // newer event already, which this older write has just overwritten on the - // provider. The current value is sent once more so the provider ends on it. var changedSets, changedRemovals []*ratingItem for start := 0; start < len(sets); start += ratingExportBatchSize { batch := sets[start:min(start+ratingExportBatchSize, len(sets))] @@ -906,10 +911,26 @@ func (s *Service) sendRatings(ctx context.Context, conn Connection, cfg ServerCo warnings = append(warnings, exportFailureReason(result, identity, "rating removal")+": "+identity.MediaItemID) } } - if followUp && (len(changedSets) > 0 || len(changedRemovals) > 0) { - more, moreWarnings, err := s.sendRatings(ctx, conn, cfg, exporter, changedSets, changedRemovals, false) + if len(changedSets) == 0 && len(changedRemovals) == 0 { + return sent, warnings, nil + } + if resends > 0 { + more, moreWarnings, err := s.sendRatings(ctx, conn, cfg, exporter, changedSets, changedRemovals, resends-1) return sent + more, append(warnings, moreWarnings...), err } + // The rating kept changing through every resend, so the provider may hold + // an older value than the agreed row says. Forgetting the agreed rows of + // rated items makes the next merge see both sides changed and keep the + // newer rating instead of importing the provider's. A removal keeps its + // row: the provider holding the agreed value then reads as a local change. + stale := make([]string, 0, len(changedSets)) + for _, item := range changedSets { + stale = append(stale, item.identity.MediaItemID) + } + if err := s.repo.DeleteRatingSyncStates(ctx, conn.ID, conn.ProviderAccountID, stale); err != nil { + return sent, warnings, err + } + warnings = append(warnings, fmt.Sprintf("%d ratings changed while being sent and are left for the next sync", len(changedSets)+len(changedRemovals))) return sent, warnings, nil } diff --git a/internal/watchsync/ratings_db_test.go b/internal/watchsync/ratings_db_test.go index 207e09f744..268e45c3f6 100644 --- a/internal/watchsync/ratings_db_test.go +++ b/internal/watchsync/ratings_db_test.go @@ -143,7 +143,13 @@ func TestRatingSyncRepositoryDB(t *testing.T) { if len(only) != 1 || only[0].MediaItemID != "s-1" { t.Fatalf("filtered states = %#v", only) } - if err := repo.DeleteRatingSyncStates(ctx, conn.ID, []string{"m-1"}); err != nil { + if err := repo.DeleteRatingSyncStates(ctx, conn.ID, "other-account", []string{"m-1"}); err != nil { + t.Fatal(err) + } + if states, _ := repo.ListRatingSyncStates(ctx, conn.ID, "", nil); len(states) != 2 { + t.Fatalf("another account's delete removed rows: %#v", states) + } + if err := repo.DeleteRatingSyncStates(ctx, conn.ID, "", []string{"m-1"}); err != nil { t.Fatal(err) } if states, _ := repo.ListRatingSyncStates(ctx, conn.ID, "", nil); len(states) != 1 { @@ -157,7 +163,15 @@ func TestRatingSyncRepositoryDB(t *testing.T) { } }) + bind := func(t *testing.T, account string) { + t.Helper() + if _, err := pool.Exec(ctx, `UPDATE watch_provider_connections SET provider_account_id=$2 WHERE id=$1::uuid`, conn.ID, account); err != nil { + t.Fatal(err) + } + } + t.Run("agreed ratings are scoped to the provider account", func(t *testing.T) { + bind(t, "account-a") if err := repo.UpsertRatingSyncStates(ctx, []RatingSyncState{ {ConnectionID: conn.ID, ProviderAccountID: "account-a", MediaItemID: "m-3", Kind: "movie", SyncedRating: 3}, }); err != nil { @@ -171,6 +185,7 @@ func TestRatingSyncRepositoryDB(t *testing.T) { t.Fatalf("account rows = %#v (%v)", states, err) } // Re-agreeing under the new account takes the row over. + bind(t, "account-b") if err := repo.UpsertRatingSyncStates(ctx, []RatingSyncState{ {ConnectionID: conn.ID, ProviderAccountID: "account-b", MediaItemID: "m-3", Kind: "movie", SyncedRating: 5}, }); err != nil { @@ -179,12 +194,26 @@ func TestRatingSyncRepositoryDB(t *testing.T) { if states, _ := repo.ListRatingSyncStates(ctx, conn.ID, "account-a", nil); len(states) != 0 { t.Fatalf("old account still sees %#v", states) } - // Clearing keeps the bound account's rows and drops the others. + // A run still writing for the previous account changes nothing. if err := repo.UpsertRatingSyncStates(ctx, []RatingSyncState{ - {ConnectionID: conn.ID, ProviderAccountID: "account-a", MediaItemID: "m-4", Kind: "movie", SyncedRating: 2}, + {ConnectionID: conn.ID, ProviderAccountID: "account-a", MediaItemID: "m-3", Kind: "movie", SyncedRating: 1}, + {ConnectionID: conn.ID, ProviderAccountID: "account-a", MediaItemID: "m-5", Kind: "movie", SyncedRating: 1}, }); err != nil { t.Fatal(err) } + if err := repo.DeleteRatingSyncStates(ctx, conn.ID, "account-a", []string{"m-3"}); err != nil { + t.Fatal(err) + } + if states, _ := repo.ListRatingSyncStates(ctx, conn.ID, "account-b", nil); len(states) != 1 || states[0].SyncedRating != 5 { + t.Fatalf("bound account rows after a stale write = %#v, want m-3 at 5", states) + } + if states, _ := repo.ListRatingSyncStates(ctx, conn.ID, "account-a", nil); len(states) != 0 { + t.Fatalf("stale write recorded %#v", states) + } + // Clearing keeps the bound account's rows and drops the others. + if _, err := pool.Exec(ctx, `INSERT INTO watch_provider_rating_items (connection_id, provider_account_id, media_item_id, kind, synced_rating) VALUES ($1::uuid, 'account-a', 'm-4', 'movie', 2)`, conn.ID); err != nil { + t.Fatal(err) + } if err := repo.ClearRatingSyncStates(ctx, conn.ID, "account-b"); err != nil { t.Fatal(err) } @@ -197,6 +226,7 @@ func TestRatingSyncRepositoryDB(t *testing.T) { if err := repo.ClearRatingSyncStates(ctx, conn.ID, "none"); err != nil { t.Fatal(err) } + bind(t, "") }) t.Run("rating cursors update in place for the bound account only", func(t *testing.T) { @@ -220,9 +250,13 @@ func TestRatingSyncRepositoryDB(t *testing.T) { }) t.Run("connection delete cascades", func(t *testing.T) { + bind(t, "") if err := repo.UpsertRatingSyncStates(ctx, []RatingSyncState{{ConnectionID: conn.ID, MediaItemID: "m-2", Kind: "movie", SyncedRating: 3}}); err != nil { t.Fatal(err) } + if states, _ := repo.ListRatingSyncStates(ctx, conn.ID, "", nil); len(states) != 1 { + t.Fatalf("states before delete = %#v, want one", states) + } if err := repo.DeleteConnection(ctx, "ratings", userID, "ratings-p"); err != nil { t.Fatal(err) } diff --git a/internal/watchsync/ratings_test.go b/internal/watchsync/ratings_test.go index c18eb5c35a..299c624e24 100644 --- a/internal/watchsync/ratings_test.go +++ b/internal/watchsync/ratings_test.go @@ -561,6 +561,30 @@ func TestSyncRatingsResendsAValueChangedDuringTheSend(t *testing.T) { } } +func TestSyncRatingsForgetsTheAgreedRatingWhenResendsNeverSettle(t *testing.T) { + h := newRatingHarness(t) + h.agree(ratingTestMovieA, 1, true) + h.store.set(ratingTestMovieA, 2) + // Every write overlaps another edit, so no send ever confirms the current + // value. The agreed row must not be left claiming the provider holds it. + stars := 2 + h.provider.onExport = func() { + stars = stars%5 + 1 + h.store.set(ratingTestMovieA, stars) + } + h.provider.batch = RatingImportBatch{Rows: []RemoteRating{h.remoteRow(ratingTestMovieA, 2)}, SnapshotKinds: []string{historyimport.KindMovie}} + result := h.sync() + if len(h.provider.exported) != 1+maxRatingResends { + t.Fatalf("exports = %d, want the first send and %d resends", len(h.provider.exported), maxRatingResends) + } + if s := h.state(ratingTestMovieA); s != nil { + t.Fatalf("agreed row = %#v, want it forgotten", s) + } + if len(result.Warnings) == 0 { + t.Fatal("unsettled ratings must be reported") + } +} + func TestSyncRatingsResendsARatingSetDuringARemoval(t *testing.T) { h := newRatingHarness(t) h.agree(ratingTestMovieA, 4, true) diff --git a/internal/watchsync/repository.go b/internal/watchsync/repository.go index 4fb70396e2..448b1477a1 100644 --- a/internal/watchsync/repository.go +++ b/internal/watchsync/repository.go @@ -49,7 +49,7 @@ type Repository interface { ListRatingEventConnections(ctx context.Context, userID int, profileID string) ([]Connection, error) ListRatingSyncStates(ctx context.Context, connectionID, providerAccountID string, mediaItemIDs []string) ([]RatingSyncState, error) UpsertRatingSyncStates(ctx context.Context, states []RatingSyncState) error - DeleteRatingSyncStates(ctx context.Context, connectionID string, mediaItemIDs []string) error + DeleteRatingSyncStates(ctx context.Context, connectionID, providerAccountID string, mediaItemIDs []string) error ClearRatingSyncStates(ctx context.Context, connectionID, keepAccountID string) error UpdateRatingCursors(ctx context.Context, connectionID, providerAccountID string, remove []string, set map[string]string) error ListScrobbleConnections(ctx context.Context, userID int, profileID string) ([]Connection, error) @@ -729,6 +729,9 @@ func (r *PostgresRepository) ListRatingSyncStates(ctx context.Context, connectio return states, nil } +// UpsertRatingSyncStates records agreed ratings. A row is written only while +// its connection is still bound to the row's provider account, so a run that +// outlived a rebind cannot take a row back from the new account. func (r *PostgresRepository) UpsertRatingSyncStates(ctx context.Context, states []RatingSyncState) error { if len(states) == 0 { return nil @@ -757,6 +760,8 @@ func (r *PostgresRepository) UpsertRatingSyncStates(ctx context.Context, states input.provider_item_key, input.synced_rating, input.remote_seen FROM unnest($1::text[], $2::text[], $3::text[], $4::text[], $5::text[], $6::smallint[], $7::boolean[]) AS input(connection_id, provider_account_id, media_item_id, kind, provider_item_key, synced_rating, remote_seen) + JOIN watch_provider_connections conn + ON conn.id = input.connection_id::uuid AND conn.provider_account_id = input.provider_account_id ON CONFLICT (connection_id, media_item_id) DO UPDATE SET provider_account_id = EXCLUDED.provider_account_id, kind = CASE WHEN EXCLUDED.kind <> '' THEN EXCLUDED.kind ELSE watch_provider_rating_items.kind END, @@ -774,14 +779,16 @@ func (r *PostgresRepository) UpsertRatingSyncStates(ctx context.Context, states return nil } -func (r *PostgresRepository) DeleteRatingSyncStates(ctx context.Context, connectionID string, mediaItemIDs []string) error { +// DeleteRatingSyncStates forgets agreed ratings recorded for one provider +// account, leaving rows another account has since agreed on. +func (r *PostgresRepository) DeleteRatingSyncStates(ctx context.Context, connectionID, providerAccountID string, mediaItemIDs []string) error { if len(mediaItemIDs) == 0 { return nil } _, err := r.pool.Exec(ctx, ` DELETE FROM watch_provider_rating_items - WHERE connection_id = $1::uuid AND media_item_id = ANY($2) - `, connectionID, mediaItemIDs) + WHERE connection_id = $1::uuid AND provider_account_id = $2 AND media_item_id = ANY($3) + `, connectionID, providerAccountID, mediaItemIDs) if err != nil { return fmt.Errorf("delete rating sync states: %w", err) } diff --git a/internal/watchsync/service_test.go b/internal/watchsync/service_test.go index 587fea1797..2304991957 100644 --- a/internal/watchsync/service_test.go +++ b/internal/watchsync/service_test.go @@ -563,10 +563,10 @@ func (r *serviceFakeRepo) UpsertRatingSyncStates(_ context.Context, states []Rat return nil } -func (r *serviceFakeRepo) DeleteRatingSyncStates(_ context.Context, connectionID string, mediaItemIDs []string) error { +func (r *serviceFakeRepo) DeleteRatingSyncStates(_ context.Context, connectionID, providerAccountID string, mediaItemIDs []string) error { kept := r.ratingStates[:0] for _, state := range r.ratingStates { - if state.ConnectionID == connectionID && containsString(mediaItemIDs, state.MediaItemID) { + if state.ConnectionID == connectionID && state.ProviderAccountID == providerAccountID && containsString(mediaItemIDs, state.MediaItemID) { continue } kept = append(kept, state) From fc32cfcac5821f67d7218a01835b2fd56e414752 Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Thu, 24 Sep 2026 02:57:27 +0000 Subject: [PATCH 18/20] fix(watchsync): share the Trakt page budget across one account's tokens Co-Authored-By: Claude Opus 5.5 (1M context) --- .../providers/trakt/pagination_test.go | 29 +++++++++++++++++++ .../watchsync/providers/trakt/provider.go | 14 +++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/internal/watchsync/providers/trakt/pagination_test.go b/internal/watchsync/providers/trakt/pagination_test.go index 2aadd24397..0e4dc9ca5e 100644 --- a/internal/watchsync/providers/trakt/pagination_test.go +++ b/internal/watchsync/providers/trakt/pagination_test.go @@ -327,6 +327,35 @@ func TestFetchTraktPagesPacesReadsPerToken(t *testing.T) { } } +func TestFetchTraktPagesShareTheBudgetOfOneAccount(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + writeTraktFixture(t, w, `[]`) + })) + defer server.Close() + provider := NewProvider(server.Client(), server.URL) + provider.pages = watchsync.NewCredentialLimiter(time.Hour, 1) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + // Two profiles linked to one Trakt account hold different tokens. + first := watchsync.Connection{AccessToken: "token-a", ProviderAccountID: "trakt-user"} + second := watchsync.Connection{AccessToken: "token-b", ProviderAccountID: "trakt-user"} + if _, err := fetchTraktPages[traktFavoriteMovie](ctx, provider, watchsync.ServerConfig{}, first, "/sync/watchlist/movies", nil); err != nil { + t.Fatal(err) + } + _, err := fetchTraktPages[traktFavoriteMovie](ctx, provider, watchsync.ServerConfig{}, second, "/sync/watchlist/movies", nil) + if _, ok := watchsync.AsRateLimited(err); !ok || requests != 1 { + t.Fatalf("requests=%d err=%v, want the second token to wait on the account's budget", requests, err) + } + // Another account has its own budget. + other := watchsync.Connection{AccessToken: "token-c", ProviderAccountID: "other-user"} + if _, err := fetchTraktPages[traktFavoriteMovie](ctx, provider, watchsync.ServerConfig{}, other, "/sync/watchlist/movies", nil); err != nil { + t.Fatalf("another account: %v", err) + } +} + func TestTraktPageBudgetStaysUnderTheGETLimit(t *testing.T) { // Trakt allows 500 authenticated GETs per five minutes. if perWindow := pageBurst + int((5*time.Minute)/pageInterval); perWindow >= 500 { diff --git a/internal/watchsync/providers/trakt/provider.go b/internal/watchsync/providers/trakt/provider.go index 054dc3817f..4f8b40253a 100644 --- a/internal/watchsync/providers/trakt/provider.go +++ b/internal/watchsync/providers/trakt/provider.go @@ -57,7 +57,7 @@ type Provider struct { baseURL string // writes paces authenticated writes per access token. writes *watchsync.CredentialLimiter - // pages paces paginated reads per access token. + // pages paces paginated reads per Trakt account (see pageLimiterKey). pages *watchsync.CredentialLimiter // sleep waits between in-place rate-limit retries; tests replace it. sleep func(context.Context, time.Duration) error @@ -357,6 +357,16 @@ func fetchTraktPages[T any]( return rows, nil } +// pageLimiterKey identifies whose GET budget a paged read spends. Trakt counts +// requests per user, so profiles linked to one Trakt account with different +// tokens share a budget; the token is the fallback before the account is known. +func pageLimiterKey(conn watchsync.Connection) string { + if account := strings.TrimSpace(conn.ProviderAccountID); account != "" { + return "account:" + account + } + return "token:" + conn.AccessToken +} + // fetchTraktPass reads every page of a listing once and reports how many pages // it took. A changed X-Pagination-Item-Count between pages fails the pass. func fetchTraktPass( @@ -375,7 +385,7 @@ func fetchTraktPass( for page := 1; page <= traktMaxPages; page++ { params.Set("page", strconv.Itoa(page)) if conn.AccessToken != "" { - if err := p.pages.Wait(ctx, conn.AccessToken); err != nil { + if err := p.pages.Wait(ctx, pageLimiterKey(conn)); err != nil { return nil, 0, watchsync.LimiterWaitError(ctx, p.Key(), pageInterval, err) } } From 6dcf3a13dc77d7a7348946b6dfcdcdebeef8bca8 Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Thu, 24 Sep 2026 17:37:45 +0000 Subject: [PATCH 19/20] fix(watchsync): lock the binding for agreed-rating writes and keep late removals Agreed-rating upserts now share-lock the connection row and check the bound account under that lock, so a rebind either waits for the write and clears it or the write finds no binding. When resends run out, the agreed row records the value last confirmed on the provider, so a removal made during the last resend is sent next run instead of being imported back. The web settings page now refreshes rating, catalog, section and recommendation queries when the manual sync it started finishes. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../watch-provider-rating-sync.md | 9 ++-- internal/watchsync/ratings.go | 29 +++++++---- internal/watchsync/ratings_test.go | 45 ++++++++++++++-- internal/watchsync/repository.go | 52 +++++++++++++++++-- .../queries/ratingsSurfaceRefresh.test.ts | 24 ++++++++- .../hooks/queries/ratingsSurfaceRefresh.ts | 13 +++++ web/src/hooks/queries/watchProviders.test.ts | 16 ++++++ web/src/hooks/queries/watchProviders.ts | 51 ++++++++++++++++-- 8 files changed, 212 insertions(+), 27 deletions(-) diff --git a/docs/architecture/watch-provider-rating-sync.md b/docs/architecture/watch-provider-rating-sync.md index 0577ff6d85..8d2248fbe4 100644 --- a/docs/architecture/watch-provider-rating-sync.md +++ b/docs/architecture/watch-provider-rating-sync.md @@ -41,15 +41,16 @@ being imported back. A local rating that changed while its write was in flight may have been sent already by a newer event, which the older write then overwrote. So the current value is sent again, -up to three times. If the rating is still changing after that, its agreed row is dropped, -so the next merge sees both sides changed and keeps the newer rating instead of importing -the provider's. +up to three times. If the rating is still changing after that, its agreed row is set to +the value last confirmed on the provider, so the next merge reads the newer local value +(rating or removal) as a local change and sends it instead of importing the provider's. Rows are scoped to the provider account they were agreed with. A connection that moves to another account ignores the old rows, and so does a sync still running for the old account, so no stale agreement can read as a removal. Agreed rows and rating cursors are written only while the connection is still bound to the account the run read, so a run -that outlives a re-bind cannot write for the old account. +that outlives a re-bind cannot write for the old account. Agreed rows are written under a +share lock on the connection row, so a re-bind waits for them and then clears them. The row's `provider_item_key` is the provider's own key for the title once a read has returned one, and Silo's key (`imdb:`, `tmdb:`, or `tvdb:`) before that. Rating writes diff --git a/internal/watchsync/ratings.go b/internal/watchsync/ratings.go index a6833a63d0..827f769497 100644 --- a/internal/watchsync/ratings.go +++ b/internal/watchsync/ratings.go @@ -830,6 +830,10 @@ func (s *Service) sendRatings(ctx context.Context, conn Connection, cfg ServerCo sent := 0 var warnings []string var changedSets, changedRemovals []*ratingItem + // What this call last confirmed on the provider for each changed item, in + // case the resends run out. + var lastSent []RatingSyncState + var lastRemoved []string for start := 0; start < len(sets); start += ratingExportBatchSize { batch := sets[start:min(start+ratingExportBatchSize, len(sets))] payload := make([]LocalRating, 0, len(batch)) @@ -867,6 +871,14 @@ func (s *Service) sendRatings(ctx context.Context, conn Connection, cfg ServerCo } else { changedRemovals = append(changedRemovals, &changed) } + lastSent = append(lastSent, RatingSyncState{ + ConnectionID: conn.ID, + ProviderAccountID: conn.ProviderAccountID, + MediaItemID: identity.MediaItemID, + Kind: identity.Kind, + ProviderItemKey: identity.ProviderItemKey, + SyncedRating: item.local, + }) continue } states = append(states, RatingSyncState{ @@ -905,6 +917,7 @@ func (s *Service) sendRatings(ctx context.Context, conn Connection, cfg ServerCo changed := *item changed.local, changed.localAt = current.Rating, current.RatedAt changedSets = append(changedSets, &changed) + lastRemoved = append(lastRemoved, identity.MediaItemID) } continue } @@ -918,16 +931,14 @@ func (s *Service) sendRatings(ctx context.Context, conn Connection, cfg ServerCo more, moreWarnings, err := s.sendRatings(ctx, conn, cfg, exporter, changedSets, changedRemovals, resends-1) return sent + more, append(warnings, moreWarnings...), err } - // The rating kept changing through every resend, so the provider may hold - // an older value than the agreed row says. Forgetting the agreed rows of - // rated items makes the next merge see both sides changed and keep the - // newer rating instead of importing the provider's. A removal keeps its - // row: the provider holding the agreed value then reads as a local change. - stale := make([]string, 0, len(changedSets)) - for _, item := range changedSets { - stale = append(stale, item.identity.MediaItemID) + // The rating kept changing through every resend. The agreed row is set to + // what this call last confirmed on the provider, so the next merge reads + // the newer local value, rating or removal, as a local change and sends it, + // instead of importing a value the provider holds from an older write. + if err := s.repo.UpsertRatingSyncStates(ctx, lastSent); err != nil { + return sent, warnings, err } - if err := s.repo.DeleteRatingSyncStates(ctx, conn.ID, conn.ProviderAccountID, stale); err != nil { + if err := s.repo.DeleteRatingSyncStates(ctx, conn.ID, conn.ProviderAccountID, lastRemoved); err != nil { return sent, warnings, err } warnings = append(warnings, fmt.Sprintf("%d ratings changed while being sent and are left for the next sync", len(changedSets)+len(changedRemovals))) diff --git a/internal/watchsync/ratings_test.go b/internal/watchsync/ratings_test.go index 299c624e24..172c52f175 100644 --- a/internal/watchsync/ratings_test.go +++ b/internal/watchsync/ratings_test.go @@ -561,12 +561,13 @@ func TestSyncRatingsResendsAValueChangedDuringTheSend(t *testing.T) { } } -func TestSyncRatingsForgetsTheAgreedRatingWhenResendsNeverSettle(t *testing.T) { +func TestSyncRatingsAgreesOnTheLastSentValueWhenResendsNeverSettle(t *testing.T) { h := newRatingHarness(t) h.agree(ratingTestMovieA, 1, true) h.store.set(ratingTestMovieA, 2) // Every write overlaps another edit, so no send ever confirms the current - // value. The agreed row must not be left claiming the provider holds it. + // value. The agreed row must name what the provider was last sent, so the + // next merge sends the newer value instead of importing the sent one. stars := 2 h.provider.onExport = func() { stars = stars%5 + 1 @@ -577,14 +578,50 @@ func TestSyncRatingsForgetsTheAgreedRatingWhenResendsNeverSettle(t *testing.T) { if len(h.provider.exported) != 1+maxRatingResends { t.Fatalf("exports = %d, want the first send and %d resends", len(h.provider.exported), maxRatingResends) } - if s := h.state(ratingTestMovieA); s != nil { - t.Fatalf("agreed row = %#v, want it forgotten", s) + last := h.provider.exported[len(h.provider.exported)-1].Rating + if s := h.state(ratingTestMovieA); s == nil || providerRatingFromStars(s.SyncedRating) != last || s.RemoteSeen { + t.Fatalf("agreed row = %#v, want the last sent rating %d, unseen", s, last) } if len(result.Warnings) == 0 { t.Fatal("unsettled ratings must be reported") } } +func TestSyncRatingsSendsARemovalMadeDuringTheLastResend(t *testing.T) { + h := newRatingHarness(t) + h.agree(ratingTestMovieA, 1, true) + h.store.set(ratingTestMovieA, 2) + // The edits keep coming, and the last one removes the rating. + sends, stars := 0, 2 + h.provider.onExport = func() { + sends++ + if sends > maxRatingResends { + h.store.remove(ratingTestMovieA) + return + } + stars = stars%5 + 1 + h.store.set(ratingTestMovieA, stars) + } + h.provider.batch = RatingImportBatch{Rows: []RemoteRating{h.remoteRow(ratingTestMovieA, 2)}, SnapshotKinds: []string{historyimport.KindMovie}} + h.sync() + if len(h.provider.exported) != 1+maxRatingResends { + t.Fatalf("exports = %d, want the first send and %d resends", len(h.provider.exported), maxRatingResends) + } + last := h.provider.exported[len(h.provider.exported)-1].Rating + + // The provider now holds the last sent rating. The next run must send the + // removal rather than import that rating back. + h.provider.onExport = nil + h.provider.batch = RatingImportBatch{Rows: []RemoteRating{h.remoteRow(ratingTestMovieA, last)}, SnapshotKinds: []string{historyimport.KindMovie}} + h.sync() + if got := h.store.stars(ratingTestMovieA); got != 0 { + t.Fatalf("local rating = %d, want the removal kept", got) + } + if len(h.provider.removed) != 1 { + t.Fatalf("removed = %#v, want the removal sent", h.provider.removed) + } +} + func TestSyncRatingsResendsARatingSetDuringARemoval(t *testing.T) { h := newRatingHarness(t) h.agree(ratingTestMovieA, 4, true) diff --git a/internal/watchsync/repository.go b/internal/watchsync/repository.go index 448b1477a1..57b6ac79e8 100644 --- a/internal/watchsync/repository.go +++ b/internal/watchsync/repository.go @@ -731,8 +731,54 @@ func (r *PostgresRepository) ListRatingSyncStates(ctx context.Context, connectio // UpsertRatingSyncStates records agreed ratings. A row is written only while // its connection is still bound to the row's provider account, so a run that -// outlived a rebind cannot take a row back from the new account. +// outlived a rebind cannot take a row back from the new account. The binding +// is checked under a share lock on the connection row, held until the rows are +// written: a rebind waits for the write and then clears what it wrote, and a +// write that follows a rebind finds no match and writes nothing. func (r *PostgresRepository) UpsertRatingSyncStates(ctx context.Context, states []RatingSyncState) error { + if len(states) == 0 { + return nil + } + return pgx.BeginFunc(ctx, r.pool, func(tx pgx.Tx) error { + bound, err := lockBoundRatingAccounts(ctx, tx, states) + if err != nil { + return err + } + kept := states[:0:0] + for _, state := range states { + if bound[ratingBinding{state.ConnectionID, state.ProviderAccountID}] { + kept = append(kept, state) + } + } + return upsertRatingSyncStates(ctx, tx, kept) + }) +} + +type ratingBinding struct{ connectionID, providerAccountID string } + +// lockBoundRatingAccounts share-locks the connections the states belong to and +// reports which (connection, account) pairs are still bound. +func lockBoundRatingAccounts(ctx context.Context, tx pgx.Tx, states []RatingSyncState) (map[ratingBinding]bool, error) { + pairs := make(map[ratingBinding]bool) + for _, state := range states { + pairs[ratingBinding{state.ConnectionID, state.ProviderAccountID}] = false + } + for pair := range pairs { + var found bool + err := tx.QueryRow(ctx, ` + SELECT true FROM watch_provider_connections + WHERE id = $1::uuid AND provider_account_id = $2 + FOR SHARE + `, pair.connectionID, pair.providerAccountID).Scan(&found) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("lock rating connection: %w", err) + } + pairs[pair] = found + } + return pairs, nil +} + +func upsertRatingSyncStates(ctx context.Context, tx pgx.Tx, states []RatingSyncState) error { if len(states) == 0 { return nil } @@ -752,7 +798,7 @@ func (r *PostgresRepository) UpsertRatingSyncStates(ctx context.Context, states ratings[i] = int32(state.SyncedRating) seen[i] = state.RemoteSeen } - _, err := r.pool.Exec(ctx, ` + _, err := tx.Exec(ctx, ` INSERT INTO watch_provider_rating_items ( connection_id, provider_account_id, media_item_id, kind, provider_item_key, synced_rating, remote_seen ) @@ -760,8 +806,6 @@ func (r *PostgresRepository) UpsertRatingSyncStates(ctx context.Context, states input.provider_item_key, input.synced_rating, input.remote_seen FROM unnest($1::text[], $2::text[], $3::text[], $4::text[], $5::text[], $6::smallint[], $7::boolean[]) AS input(connection_id, provider_account_id, media_item_id, kind, provider_item_key, synced_rating, remote_seen) - JOIN watch_provider_connections conn - ON conn.id = input.connection_id::uuid AND conn.provider_account_id = input.provider_account_id ON CONFLICT (connection_id, media_item_id) DO UPDATE SET provider_account_id = EXCLUDED.provider_account_id, kind = CASE WHEN EXCLUDED.kind <> '' THEN EXCLUDED.kind ELSE watch_provider_rating_items.kind END, diff --git a/web/src/hooks/queries/ratingsSurfaceRefresh.test.ts b/web/src/hooks/queries/ratingsSurfaceRefresh.test.ts index c26b55a1da..1a74fa09b7 100644 --- a/web/src/hooks/queries/ratingsSurfaceRefresh.test.ts +++ b/web/src/hooks/queries/ratingsSurfaceRefresh.test.ts @@ -1,7 +1,10 @@ import { QueryClient } from "@tanstack/react-query"; import { describe, expect, it } from "vitest"; import { catalogKeys, ratingKeys, recKeys, sectionKeys } from "./keys"; -import { invalidateRatingSurfaceQueries } from "./ratingsSurfaceRefresh"; +import { + invalidateAllRatingSurfaceQueries, + invalidateRatingSurfaceQueries, +} from "./ratingsSurfaceRefresh"; describe("invalidateRatingSurfaceQueries", () => { it("marks rating and recommendation-derived surfaces stale", async () => { @@ -37,3 +40,22 @@ describe("invalidateRatingSurfaceQueries", () => { expect(queryClient.getQueryState(catalogKeys.itemDetail("item-1"))?.isInvalidated).toBe(false); }); }); + +describe("invalidateAllRatingSurfaceQueries", () => { + it("marks every rating-derived surface stale", async () => { + const queryClient = new QueryClient(); + queryClient.setQueryData(ratingKeys.list(), { items: [] }); + queryClient.setQueryData(catalogKeys.itemDetail("item-1"), { content_id: "item-1" }); + queryClient.setQueryData(recKeys.forYouMain(), { row: null }); + queryClient.setQueryData(sectionKeys.homeItems("for-you"), { section: { id: "for-you" } }); + queryClient.setQueryData(["unrelated"], {}); + + await invalidateAllRatingSurfaceQueries(queryClient); + + expect(queryClient.getQueryState(ratingKeys.list())?.isInvalidated).toBe(true); + expect(queryClient.getQueryState(catalogKeys.itemDetail("item-1"))?.isInvalidated).toBe(true); + expect(queryClient.getQueryState(recKeys.forYouMain())?.isInvalidated).toBe(true); + expect(queryClient.getQueryState(sectionKeys.homeItems("for-you"))?.isInvalidated).toBe(true); + expect(queryClient.getQueryState(["unrelated"])?.isInvalidated).toBe(false); + }); +}); diff --git a/web/src/hooks/queries/ratingsSurfaceRefresh.ts b/web/src/hooks/queries/ratingsSurfaceRefresh.ts index 7d45b4bbeb..1f1de58e05 100644 --- a/web/src/hooks/queries/ratingsSurfaceRefresh.ts +++ b/web/src/hooks/queries/ratingsSurfaceRefresh.ts @@ -24,6 +24,19 @@ export async function invalidateRatingSurfaceQueries(queryClient: QueryClient, i }); } +// invalidateAllRatingSurfaceQueries marks every rating-derived surface stale +// after ratings changed outside a single-item edit, such as a watch-provider +// import. +export async function invalidateAllRatingSurfaceQueries(queryClient: QueryClient) { + await queryClient.invalidateQueries({ + predicate: (query) => + startsWith(query.queryKey, ratingKeys.all) || + startsWith(query.queryKey, catalogKeys.all) || + startsWith(query.queryKey, recKeys.all) || + startsWith(query.queryKey, sectionKeys.all), + }); +} + function startsWith(queryKey: readonly unknown[], prefix: readonly unknown[]) { return ( prefix.length <= queryKey.length && prefix.every((part, index) => part === queryKey[index]) diff --git a/web/src/hooks/queries/watchProviders.test.ts b/web/src/hooks/queries/watchProviders.test.ts index 836644eb70..79fb794b81 100644 --- a/web/src/hooks/queries/watchProviders.test.ts +++ b/web/src/hooks/queries/watchProviders.test.ts @@ -10,6 +10,8 @@ import { triggerWatchProviderSync, updateWatchProviderConnection, deleteWatchProviderConnection, + syncRunFinished, + type WatchProviderSyncRun, } from "./watchProviders"; vi.mock("@/api/v2/request", () => ({ @@ -74,3 +76,17 @@ describe("watch provider v2 queries", () => { }); }); }); + +describe("syncRunFinished", () => { + const run = (id: string, status: WatchProviderSyncRun["status"]) => + ({ id, status }) as WatchProviderSyncRun; + + it("fires only when the watched run leaves queued or running", () => { + expect(syncRunFinished(run("a", "running"), run("a", "success"))).toBe(true); + expect(syncRunFinished(run("a", "queued"), run("a", "warning"))).toBe(true); + expect(syncRunFinished(run("a", "queued"), run("a", "running"))).toBe(false); + expect(syncRunFinished(run("a", "success"), run("a", "success"))).toBe(false); + expect(syncRunFinished(run("a", "running"), run("b", "success"))).toBe(false); + expect(syncRunFinished(undefined, run("a", "success"))).toBe(false); + }); +}); diff --git a/web/src/hooks/queries/watchProviders.ts b/web/src/hooks/queries/watchProviders.ts index ae2bbd5586..e420014ee7 100644 --- a/web/src/hooks/queries/watchProviders.ts +++ b/web/src/hooks/queries/watchProviders.ts @@ -1,7 +1,9 @@ -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { useEffect, useRef } from "react"; +import { useMutation, useQuery, useQueryClient, type QueryClient } from "@tanstack/react-query"; import { captureProfileRequestContext } from "@/api/client"; import { v2, V2ProblemError } from "@/api/v2/request"; -import { favoriteKeys, ratingKeys, watchlistKeys, watchProviderKeys } from "./keys"; +import { favoriteKeys, watchlistKeys, watchProviderKeys } from "./keys"; +import { invalidateAllRatingSurfaceQueries } from "./ratingsSurfaceRefresh"; import { toast } from "sonner"; import { storage } from "@/utils/storage"; import type { PluginConfigSchema } from "@/api/types"; @@ -246,15 +248,55 @@ export function useWatchProviderConnection(provider: string) { export function useWatchProviderSyncRuns(provider: string, enabled = true) { const profileId = getActiveProfileId(); - return useQuery({ + const queryClient = useQueryClient(); + const query = useQuery({ queryKey: watchProviderKeys.syncRuns(profileId, provider), queryFn: () => fetchWatchProviderSyncRuns(provider), enabled: enabled && Boolean(profileId), refetchInterval: (query) => { const latest = query.state.data?.runs?.[0]; - return latest?.status === "queued" || latest?.status === "running" ? 4_000 : false; + return latest && isActiveSyncRun(latest) ? 4_000 : false; }, }); + // A sync imports favorites, watchlist entries and ratings while it runs, so + // the surfaces showing them refresh once the run this page watched finishes. + const latest = query.data?.runs?.[0]; + const observed = useRef(undefined); + useEffect(() => { + if (!latest) return; + const previous = observed.current; + observed.current = latest; + if (syncRunFinished(previous, latest)) { + void invalidateSyncedSurfaces(queryClient); + } + }, [latest, queryClient]); + return query; +} + +function isActiveSyncRun(run: WatchProviderSyncRun) { + return run.status === "queued" || run.status === "running"; +} + +// syncRunFinished reports whether latest is the run previously seen as queued +// or running, now in a final state. +export function syncRunFinished( + previous: WatchProviderSyncRun | undefined, + latest: WatchProviderSyncRun, +) { + return ( + previous !== undefined && + previous.id === latest.id && + isActiveSyncRun(previous) && + !isActiveSyncRun(latest) + ); +} + +async function invalidateSyncedSurfaces(queryClient: QueryClient) { + await Promise.all([ + queryClient.invalidateQueries({ queryKey: favoriteKeys.list() }), + queryClient.invalidateQueries({ queryKey: watchlistKeys.list() }), + invalidateAllRatingSurfaceQueries(queryClient), + ]); } export function useStartWatchProviderDeviceAuth(provider: string) { @@ -369,7 +411,6 @@ export function useTriggerWatchProviderSync(provider: string) { }); queryClient.invalidateQueries({ queryKey: favoriteKeys.list() }); queryClient.invalidateQueries({ queryKey: watchlistKeys.list() }); - queryClient.invalidateQueries({ queryKey: ratingKeys.all }); toast.success("Watch provider sync started"); }, onError: (err) => { From 9238c036f8040eb3cdaa8ca2c67d36de7481d9b7 Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:33:15 +0000 Subject: [PATCH 20/20] chore(deps): build against plugin SDK v0.17.0 Co-Authored-By: Claude Opus 5.5 (1M context) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6a16fcc3ad..3b4baec977 100644 --- a/go.mod +++ b/go.mod @@ -128,7 +128,7 @@ require ( ) require ( - github.com/Silo-Server/silo-plugin-sdk v0.16.2-0.20260923231249-95b81a099ca7 + github.com/Silo-Server/silo-plugin-sdk v0.17.0 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 // indirect diff --git a/go.sum b/go.sum index ac5e206a67..afd10509fe 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0g github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/SherClockHolmes/webpush-go v1.4.0 h1:ocnzNKWN23T9nvHi6IfyrQjkIc0oJWv1B1pULsf9i3s= github.com/SherClockHolmes/webpush-go v1.4.0/go.mod h1:XSq8pKX11vNV8MJEMwjrlTkxhAj1zKfxmyhdV7Pd6UA= -github.com/Silo-Server/silo-plugin-sdk v0.16.2-0.20260923231249-95b81a099ca7 h1:7aVKMCncxENZ5QNXbDa99q/M12Y1KZSp/h/eQlFS3yU= -github.com/Silo-Server/silo-plugin-sdk v0.16.2-0.20260923231249-95b81a099ca7/go.mod h1:abwsCEKuPAAgeAqpNGbwoaut2eQlC/Kj97u89Vvg9qM= +github.com/Silo-Server/silo-plugin-sdk v0.17.0 h1:dHA4UbObMVIzAaLQC4clq1bSWuPaim88yptIRR5/f/M= +github.com/Silo-Server/silo-plugin-sdk v0.17.0/go.mod h1:abwsCEKuPAAgeAqpNGbwoaut2eQlC/Kj97u89Vvg9qM= github.com/TwiN/go-color v1.4.1 h1:mqG0P/KBgHKVqmtL5ye7K0/Gr4l6hTksPgTgMk3mUzc= github.com/TwiN/go-color v1.4.1/go.mod h1:WcPf/jtiW95WBIsEeY1Lc/b8aaWoiqQpu5cf8WFxu+s= github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4=