diff --git a/.env.example b/.env.example index ab9f127..a9c8881 100644 --- a/.env.example +++ b/.env.example @@ -9,7 +9,11 @@ # SYNAPS3_FILECOIN_PRIVATE_KEY=0x... # SYNAPS3_FILECOIN_RPC_URL=https://api.calibration.node.glif.io/rpc/v1 # SYNAPS3_CACHE_MAX_SIZE_GB=100 -# SYNAPS3_WORKER_PROVIDER_REPLACEMENT_CONCURRENCY=4 -# SYNAPS3_WORKER_PROVIDER_REPLACEMENT_POLL_INTERVAL=5s -# SYNAPS3_WORKER_PROVIDER_REPLACEMENT_MAX_RETRIES=5 +# SYNAPS3_WORKER_TASKS_CONCURRENCY=12 +# SYNAPS3_WORKER_TASKS_POLL_INTERVAL=5s +# SYNAPS3_WORKER_TASKS_LEASE_DURATION=5m +# SYNAPS3_WORKER_TASKS_MAX_RETRIES=5 +# SYNAPS3_WORKER_TASKS_RETENTION=168h +# SYNAPS3_WORKER_TASKS_PROVIDER_MUTATION_CONCURRENCY=4 +# SYNAPS3_WORKER_TASKS_DESTRUCTIVE_MUTATION_CONCURRENCY=2 # SYNAPS3_ADMIN_AUTH_USERNAME=admin diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4c1062..1d8520e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,39 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + postgres: + runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }} + timeout-minutes: 15 + services: + postgres: + image: postgres:17 + env: + POSTGRES_DB: synaps3_test + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d synaps3_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + GOTOOLCHAIN: auto + SYNAPS3_POSTGRES_TEST_DSN: postgres://postgres:postgres@localhost:5432/synaps3_test?sslmode=disable + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + + - name: Test PostgreSQL persistence contracts + run: go test ./internal/db/migrations ./internal/db/repository -count=1 + check: runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }} timeout-minutes: 30 diff --git a/cmd/synaps3/admin.go b/cmd/synaps3/admin.go index b6dda97..08c0682 100644 --- a/cmd/synaps3/admin.go +++ b/cmd/synaps3/admin.go @@ -355,24 +355,19 @@ func adminSettingsCommand() *cli.Command { func adminTaskCommand() *cli.Command { return &cli.Command{ Name: "task", - Usage: "inspect and retry background tasks", + Usage: "inspect and recover background tasks", Commands: []*cli.Command{ { Name: "list", Usage: "list background tasks", Flags: []cli.Flag{ &cli.StringFlag{Name: "type", Usage: "filter by task type"}, - &cli.StringFlag{Name: "stage", Usage: "filter by task stage; requires --type"}, - &cli.StringFlag{Name: "status", Usage: "filter by task status"}, + &cli.StringFlag{Name: "status", Usage: "filter by status (pending, running, completed, failed, cancelled, or dismissed)"}, &cli.IntFlag{Name: "limit", Value: 20, Usage: "maximum tasks to return"}, - &cli.IntFlag{Name: "offset", Usage: "task list offset"}, + &cli.Int64Flag{Name: "cursor", Usage: "continue before this task ID"}, }, Action: func(ctx context.Context, cmd *cli.Command) error { taskType := strings.TrimSpace(cmd.String("type")) - stage := strings.TrimSpace(cmd.String("stage")) - if stage != "" && taskType == "" { - return errors.New("--stage requires --type") - } client, opts, err := newAdminClientFromCommand(ctx, cmd) if err != nil { return err @@ -381,17 +376,14 @@ func adminTaskCommand() *cli.Command { if taskType != "" { query.Set("type", taskType) } - if stage != "" { - query.Set("stage", stage) - } if status := strings.TrimSpace(cmd.String("status")); status != "" { query.Set("status", status) } if cmd.IsSet("limit") { query.Set("limit", strconv.Itoa(cmd.Int("limit"))) } - if cmd.IsSet("offset") { - query.Set("offset", strconv.Itoa(cmd.Int("offset"))) + if cmd.IsSet("cursor") { + query.Set("cursor", strconv.FormatInt(cmd.Int64("cursor"), 10)) } path := "/api/v1/tasks" if encoded := query.Encode(); encoded != "" { @@ -427,7 +419,7 @@ func adminTaskCommand() *cli.Command { }, { Name: "retry", - Usage: "retry an exhausted task", + Usage: "recover a failed task", ArgsUsage: "", Action: func(ctx context.Context, cmd *cli.Command) error { taskID, err := requireSingleArg(cmd, "task id") @@ -452,6 +444,33 @@ func adminTaskCommand() *cli.Command { return err }, }, + { + Name: "acknowledge", + Usage: "dismiss a failed task", + ArgsUsage: "", + Action: func(ctx context.Context, cmd *cli.Command) error { + taskID, err := requireSingleArg(cmd, "task id") + if err != nil { + return err + } + if _, err := strconv.ParseInt(taskID, 10, 64); err != nil { + return fmt.Errorf("invalid task id %q", taskID) + } + client, opts, err := newAdminClientFromCommand(ctx, cmd) + if err != nil { + return err + } + var resp map[string]string + if err := client.postJSON(ctx, "/api/v1/tasks/"+url.PathEscape(taskID)+"/acknowledge", nil, &resp, false); err != nil { + return err + } + if opts.JSON { + return writeAdminJSON(cmd.Root().Writer, resp) + } + _, err = fmt.Fprintf(cmd.Root().Writer, "Task %s %s\n", taskID, resp["status"]) + return err + }, + }, }, } } @@ -917,16 +936,17 @@ type adminSettingsCacheConfig struct { } type adminSettingsWorkerConfig struct { - Upload adminSettingsWorkerPoolConfig `json:"upload"` - ProviderReplacement adminSettingsWorkerPoolConfig `json:"provider_replacement"` - Evictor adminSettingsWorkerPoolConfig `json:"evictor"` - StorageCleanup adminSettingsWorkerPoolConfig `json:"storage_cleanup"` + Tasks adminSettingsTaskWorkerConfig `json:"tasks"` } -type adminSettingsWorkerPoolConfig struct { - Concurrency int `json:"concurrency"` - PollInterval string `json:"poll_interval"` - MaxRetries int `json:"max_retries"` +type adminSettingsTaskWorkerConfig struct { + Concurrency int `json:"concurrency"` + PollInterval string `json:"poll_interval"` + LeaseDuration string `json:"lease_duration"` + MaxRetries int `json:"max_retries"` + Retention string `json:"retention"` + ProviderMutationConcurrency int `json:"provider_mutation_concurrency"` + DestructiveMutationConcurrency int `json:"destructive_mutation_concurrency"` } type adminSettingsLoggingConfig struct { @@ -941,26 +961,32 @@ type adminSettingsLoggingS3AccessConfig struct { } type adminTaskListResponse struct { - Tasks []adminTaskItem `json:"tasks"` - Total int `json:"total"` - Limit int `json:"limit"` - Offset int `json:"offset"` + Tasks []adminTaskItem `json:"tasks"` + NextCursor *int64 `json:"next_cursor,omitempty"` } type adminTaskItem struct { - ID int64 `json:"id"` - Type string `json:"type"` - Stage *string `json:"stage,omitempty"` - RefType string `json:"ref_type"` - RefID int64 `json:"ref_id"` - RefVersionID string `json:"ref_version_id"` - Status string `json:"status"` - RetryCount int `json:"retry_count"` - MaxRetries int `json:"max_retries"` - LastError *string `json:"last_error,omitempty"` - StatusMessage *string `json:"status_message,omitempty"` - WaitReason *string `json:"wait_reason,omitempty"` - ScheduledAt string `json:"scheduled_at"` + ID int64 `json:"id"` + Type string `json:"type"` + Operation string `json:"operation"` + Status string `json:"status"` + PresentationStatus string `json:"presentation_status"` + SubjectType *string `json:"subject_type,omitempty"` + SubjectKey *string `json:"subject_key,omitempty"` + RetryCount int `json:"retry_count"` + RetryLimit *int `json:"retry_limit,omitempty"` + Retryable bool `json:"retryable"` + Acknowledgeable bool `json:"acknowledgeable"` + LastError *string `json:"last_error,omitempty"` + StatusMessage *string `json:"status_message,omitempty"` + WaitReason *string `json:"wait_reason,omitempty"` + FailureReason *string `json:"failure_reason,omitempty"` + AvailableAt string `json:"available_at"` + StartedAt *string `json:"started_at,omitempty"` + FinishedAt *string `json:"finished_at,omitempty"` + AcknowledgedAt *string `json:"acknowledged_at,omitempty"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` } type adminTaskStatusCount struct { @@ -971,7 +997,7 @@ type adminTaskStatusCount struct { type adminStorageConfirmationAttention struct { CopyID int64 `json:"copy_id"` - UploadID int64 `json:"upload_id"` + ContentID int64 `json:"content_id"` CopyIndex int `json:"copy_index"` DataSetRowID int64 `json:"data_set_row_id"` ProviderID string `json:"provider_id"` @@ -1003,45 +1029,34 @@ type adminSettingSpec struct { } var adminEditableSettings = map[string]adminSettingSpec{ - "server.port": {path: []string{"server", "port"}, kind: adminSettingString}, - "server.max_connections": {path: []string{"server", "max_connections"}, kind: adminSettingInt}, - "server.max_requests": {path: []string{"server", "max_requests"}, kind: adminSettingInt}, - "server.tls.enabled": {path: []string{"server", "tls", "enabled"}, kind: adminSettingBool}, - "server.tls.cert_file": {path: []string{"server", "tls", "cert_file"}, kind: adminSettingString}, - "server.tls.key_file": {path: []string{"server", "tls", "key_file"}, kind: adminSettingString}, - "s3.region": {path: []string{"s3", "region"}, kind: adminSettingString}, - "filecoin.network": {path: []string{"filecoin", "network"}, kind: adminSettingString}, - "filecoin.rpc_url": {path: []string{"filecoin", "rpc_url"}, kind: adminSettingString}, - "filecoin.with_cdn": {path: []string{"filecoin", "with_cdn"}, kind: adminSettingBool}, - "filecoin.allow_private_networks": {path: []string{"filecoin", "allow_private_networks"}, kind: adminSettingBool}, - "filecoin.default_copies": {path: []string{"filecoin", "default_copies"}, kind: adminSettingInt}, - "cache.dir": {path: []string{"cache", "dir"}, kind: adminSettingString}, - "cache.max_size_gb": {path: []string{"cache", "max_size_gb"}, kind: adminSettingInt}, - "cache.eviction_policy": {path: []string{"cache", "eviction_policy"}, kind: adminSettingString}, - "cache.lru_high_watermark_percent": {path: []string{"cache", "lru_high_watermark_percent"}, kind: adminSettingInt}, - "cache.lru_low_watermark_percent": {path: []string{"cache", "lru_low_watermark_percent"}, kind: adminSettingInt}, - "worker.upload.concurrency": {path: []string{"worker", "upload", "concurrency"}, kind: adminSettingInt}, - "worker.upload.poll_interval": {path: []string{"worker", "upload", "poll_interval"}, kind: adminSettingString}, - "worker.upload.max_retries": {path: []string{"worker", "upload", "max_retries"}, kind: adminSettingInt}, - "worker.provider_replacement.concurrency": { - path: []string{"worker", "provider_replacement", "concurrency"}, kind: adminSettingInt, - }, - "worker.provider_replacement.poll_interval": { - path: []string{"worker", "provider_replacement", "poll_interval"}, kind: adminSettingString, - }, - "worker.provider_replacement.max_retries": { - path: []string{"worker", "provider_replacement", "max_retries"}, kind: adminSettingInt, - }, - "worker.evictor.concurrency": {path: []string{"worker", "evictor", "concurrency"}, kind: adminSettingInt}, - "worker.evictor.poll_interval": {path: []string{"worker", "evictor", "poll_interval"}, kind: adminSettingString}, - "worker.evictor.max_retries": {path: []string{"worker", "evictor", "max_retries"}, kind: adminSettingInt}, - "worker.storage_cleanup.concurrency": {path: []string{"worker", "storage_cleanup", "concurrency"}, kind: adminSettingInt}, - "worker.storage_cleanup.poll_interval": {path: []string{"worker", "storage_cleanup", "poll_interval"}, kind: adminSettingString}, - "worker.storage_cleanup.max_retries": {path: []string{"worker", "storage_cleanup", "max_retries"}, kind: adminSettingInt}, - "logging.level": {path: []string{"logging", "level"}, kind: adminSettingString}, - "logging.format": {path: []string{"logging", "format"}, kind: adminSettingString}, - "logging.s3_access.enabled": {path: []string{"logging", "s3_access", "enabled"}, kind: adminSettingBool}, - "logging.s3_access.level": {path: []string{"logging", "s3_access", "level"}, kind: adminSettingString}, + "server.port": {path: []string{"server", "port"}, kind: adminSettingString}, + "server.max_connections": {path: []string{"server", "max_connections"}, kind: adminSettingInt}, + "server.max_requests": {path: []string{"server", "max_requests"}, kind: adminSettingInt}, + "server.tls.enabled": {path: []string{"server", "tls", "enabled"}, kind: adminSettingBool}, + "server.tls.cert_file": {path: []string{"server", "tls", "cert_file"}, kind: adminSettingString}, + "server.tls.key_file": {path: []string{"server", "tls", "key_file"}, kind: adminSettingString}, + "s3.region": {path: []string{"s3", "region"}, kind: adminSettingString}, + "filecoin.network": {path: []string{"filecoin", "network"}, kind: adminSettingString}, + "filecoin.rpc_url": {path: []string{"filecoin", "rpc_url"}, kind: adminSettingString}, + "filecoin.with_cdn": {path: []string{"filecoin", "with_cdn"}, kind: adminSettingBool}, + "filecoin.allow_private_networks": {path: []string{"filecoin", "allow_private_networks"}, kind: adminSettingBool}, + "filecoin.default_copies": {path: []string{"filecoin", "default_copies"}, kind: adminSettingInt}, + "cache.dir": {path: []string{"cache", "dir"}, kind: adminSettingString}, + "cache.max_size_gb": {path: []string{"cache", "max_size_gb"}, kind: adminSettingInt}, + "cache.eviction_policy": {path: []string{"cache", "eviction_policy"}, kind: adminSettingString}, + "cache.lru_high_watermark_percent": {path: []string{"cache", "lru_high_watermark_percent"}, kind: adminSettingInt}, + "cache.lru_low_watermark_percent": {path: []string{"cache", "lru_low_watermark_percent"}, kind: adminSettingInt}, + "worker.tasks.concurrency": {path: []string{"worker", "tasks", "concurrency"}, kind: adminSettingInt}, + "worker.tasks.poll_interval": {path: []string{"worker", "tasks", "poll_interval"}, kind: adminSettingString}, + "worker.tasks.lease_duration": {path: []string{"worker", "tasks", "lease_duration"}, kind: adminSettingString}, + "worker.tasks.max_retries": {path: []string{"worker", "tasks", "max_retries"}, kind: adminSettingInt}, + "worker.tasks.retention": {path: []string{"worker", "tasks", "retention"}, kind: adminSettingString}, + "worker.tasks.provider_mutation_concurrency": {path: []string{"worker", "tasks", "provider_mutation_concurrency"}, kind: adminSettingInt}, + "worker.tasks.destructive_mutation_concurrency": {path: []string{"worker", "tasks", "destructive_mutation_concurrency"}, kind: adminSettingInt}, + "logging.level": {path: []string{"logging", "level"}, kind: adminSettingString}, + "logging.format": {path: []string{"logging", "format"}, kind: adminSettingString}, + "logging.s3_access.enabled": {path: []string{"logging", "s3_access", "enabled"}, kind: adminSettingBool}, + "logging.s3_access.level": {path: []string{"logging", "s3_access", "level"}, kind: adminSettingString}, } type adminSettingsUpdates struct { @@ -1173,7 +1188,7 @@ func adminStructJSONFieldValue(value reflect.Value, name string) (reflect.Value, valueType := value.Type() for i := range value.NumField() { field := valueType.Field(i) - jsonName := strings.Split(field.Tag.Get("json"), ",")[0] + jsonName, _, _ := strings.Cut(field.Tag.Get("json"), ",") if jsonName == name { return value.Field(i), true } @@ -1383,20 +1398,15 @@ func writeAdminSettingsSummary(w io.Writer, settings adminSettingsResponse) erro }, }, { - title: "Worker", + title: "Task Engine", rows: []adminOutputRow{ - {Name: "worker.upload.concurrency", Value: strconv.Itoa(settings.Config.Worker.Upload.Concurrency)}, - {Name: "worker.upload.poll_interval", Value: settings.Config.Worker.Upload.PollInterval}, - {Name: "worker.upload.max_retries", Value: strconv.Itoa(settings.Config.Worker.Upload.MaxRetries)}, - {Name: "worker.provider_replacement.concurrency", Value: strconv.Itoa(settings.Config.Worker.ProviderReplacement.Concurrency)}, - {Name: "worker.provider_replacement.poll_interval", Value: settings.Config.Worker.ProviderReplacement.PollInterval}, - {Name: "worker.provider_replacement.max_retries", Value: strconv.Itoa(settings.Config.Worker.ProviderReplacement.MaxRetries)}, - {Name: "worker.evictor.concurrency", Value: strconv.Itoa(settings.Config.Worker.Evictor.Concurrency)}, - {Name: "worker.evictor.poll_interval", Value: settings.Config.Worker.Evictor.PollInterval}, - {Name: "worker.evictor.max_retries", Value: strconv.Itoa(settings.Config.Worker.Evictor.MaxRetries)}, - {Name: "worker.storage_cleanup.concurrency", Value: strconv.Itoa(settings.Config.Worker.StorageCleanup.Concurrency)}, - {Name: "worker.storage_cleanup.poll_interval", Value: settings.Config.Worker.StorageCleanup.PollInterval}, - {Name: "worker.storage_cleanup.max_retries", Value: strconv.Itoa(settings.Config.Worker.StorageCleanup.MaxRetries)}, + {Name: "worker.tasks.concurrency", Value: strconv.Itoa(settings.Config.Worker.Tasks.Concurrency)}, + {Name: "worker.tasks.poll_interval", Value: settings.Config.Worker.Tasks.PollInterval}, + {Name: "worker.tasks.lease_duration", Value: settings.Config.Worker.Tasks.LeaseDuration}, + {Name: "worker.tasks.max_retries", Value: strconv.Itoa(settings.Config.Worker.Tasks.MaxRetries)}, + {Name: "worker.tasks.retention", Value: settings.Config.Worker.Tasks.Retention}, + {Name: "worker.tasks.provider_mutation_concurrency", Value: strconv.Itoa(settings.Config.Worker.Tasks.ProviderMutationConcurrency)}, + {Name: "worker.tasks.destructive_mutation_concurrency", Value: strconv.Itoa(settings.Config.Worker.Tasks.DestructiveMutationConcurrency)}, }, }, { @@ -1429,19 +1439,18 @@ func writeAdminTasksTable(w io.Writer, tasks []adminTaskItem) error { return err } tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) - _, _ = fmt.Fprintln(tw, "ID\tTYPE\tSTAGE\tSTATUS\tRETRIES\tREF\tSCHEDULED\tDETAILS") + _, _ = fmt.Fprintln(tw, "ID\tOPERATION\tSTATUS\tRETRIES\tSUBJECT\tAVAILABLE\tDETAILS") for _, task := range tasks { - stage := "" - if task.Stage != nil { - stage = *task.Stage - } details := adminTaskDetails(task) - ref := task.RefType + ":" + strconv.FormatInt(task.RefID, 10) - if task.RefVersionID != "" { - ref += ":" + task.RefVersionID + subject := "" + if task.SubjectType != nil && task.SubjectKey != nil { + subject = *task.SubjectType + ":" + *task.SubjectKey + } + retries := strconv.Itoa(task.RetryCount) + if task.RetryLimit != nil { + retries = fmt.Sprintf("%d/%d", task.RetryCount, *task.RetryLimit) } - retries := fmt.Sprintf("%d/%d", task.RetryCount, task.MaxRetries) - _, _ = fmt.Fprintf(tw, "%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", task.ID, task.Type, stage, task.Status, retries, ref, task.ScheduledAt, details) + _, _ = fmt.Fprintf(tw, "%d\t%s\t%s\t%s\t%s\t%s\t%s\n", task.ID, task.Operation, task.PresentationStatus, retries, subject, task.AvailableAt, details) } return tw.Flush() } @@ -1449,9 +1458,6 @@ func writeAdminTasksTable(w io.Writer, tasks []adminTaskItem) error { func adminTaskDetails(task adminTaskItem) string { switch { case task.StatusMessage != nil && *task.StatusMessage != "": - if task.WaitReason != nil && *task.WaitReason != "" { - return *task.WaitReason + ": " + *task.StatusMessage - } return *task.StatusMessage case task.LastError != nil: return *task.LastError @@ -1478,7 +1484,7 @@ func writeAdminTaskStatsTable(w io.Writer, stats []adminTaskStatusCount) error { func writeAdminStorageConfirmationsTable(w io.Writer, confirmations []adminStorageConfirmationAttention) error { tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) - if _, err := fmt.Fprintln(tw, "COPY ID\tUPLOAD ID\tCOPY\tPROVIDER\tDATA SET\tPIECE CID\tATTEMPT\tATTEMPTED AT\tTRANSACTION\tREASON\tATTENTION AT"); err != nil { + if _, err := fmt.Fprintln(tw, "COPY ID\tCONTENT ID\tCOPY\tPROVIDER\tDATA SET\tPIECE CID\tATTEMPT\tATTEMPTED AT\tTRANSACTION\tREASON\tATTENTION AT"); err != nil { return err } for _, confirmation := range confirmations { @@ -1486,7 +1492,7 @@ func writeAdminStorageConfirmationsTable(w io.Writer, confirmations []adminStora tw, "%d\t%d\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", confirmation.CopyID, - confirmation.UploadID, + confirmation.ContentID, confirmation.CopyIndex, confirmation.ProviderID, confirmation.DataSetID, diff --git a/cmd/synaps3/admin_test.go b/cmd/synaps3/admin_test.go index fc89f52..0dd5c8e 100644 --- a/cmd/synaps3/admin_test.go +++ b/cmd/synaps3/admin_test.go @@ -9,6 +9,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "slices" "strings" "testing" "time" @@ -581,7 +582,7 @@ func TestAdminSettingsSetValidationAndPayload(t *testing.T) { "100.00 GiB", "cache.lru_high_watermark_percent", "cache.lru_low_watermark_percent", - "worker.provider_replacement.concurrency", + "worker.tasks.concurrency", "Logging", } { if !strings.Contains(out, want) { @@ -726,9 +727,9 @@ func TestAdminSettingsSetValidationAndPayload(t *testing.T) { t.Fatalf("cache.lru_low_watermark_percent = %#v, want 70", cache["lru_low_watermark_percent"]) } worker := body["worker"].(map[string]any) - providerReplacement := worker["provider_replacement"].(map[string]any) - if providerReplacement["poll_interval"] != "9s" { - t.Fatalf("worker.provider_replacement.poll_interval = %#v, want 9s", providerReplacement["poll_interval"]) + tasks := worker["tasks"].(map[string]any) + if tasks["poll_interval"] != "9s" { + t.Fatalf("worker.tasks.poll_interval = %#v, want 9s", tasks["poll_interval"]) } filecoin := body["filecoin"].(map[string]any) if filecoin["with_cdn"] != true { @@ -755,7 +756,7 @@ func TestAdminSettingsSetValidationAndPayload(t *testing.T) { out, err := runAdminCommand(t, []string{ "synaps3", "admin", "--admin-url", ts.URL, "settings", "set", "cache.max_size_gb=8", "cache.lru_high_watermark_percent=85", - "cache.lru_low_watermark_percent=70", "worker.provider_replacement.poll_interval=9s", + "cache.lru_low_watermark_percent=70", "worker.tasks.poll_interval=9s", "filecoin.with_cdn=true", "logging.level=debug", "logging.s3_access.enabled=false", "logging.s3_access.level=debug", }) @@ -788,19 +789,29 @@ func TestAdminSettingsSetValidationAndPayload(t *testing.T) { func TestAdminTaskCommandsAndAPIErrorFields(t *testing.T) { t.Setenv(configEnvVar, "") + t.Run("task list help documents dismissed status", func(t *testing.T) { + out, err := runAdminCommand(t, []string{"synaps3", "admin", "task", "list", "--help"}) + if err != nil { + t.Fatalf("task list help: %v\n%s", err, out) + } + if !strings.Contains(out, "pending, running, completed, failed, cancelled, or dismissed") { + t.Fatalf("task list help missing status filters:\n%s", out) + } + }) + t.Run("task list query and retry path", func(t *testing.T) { var sawList, sawRetry bool ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodGet && r.URL.Path == "/api/v1/tasks": sawList = true - if got := r.URL.Query().Get("status"); got != "exhausted" { - t.Fatalf("status query = %q, want exhausted", got) + if got := r.URL.Query().Get("status"); got != "failed" { + t.Fatalf("status query = %q, want failed", got) } if got := r.URL.Query().Get("limit"); got != "50" { t.Fatalf("limit query = %q, want 50", got) } - writeAdminTestJSON(t, w, http.StatusOK, map[string]any{"tasks": []any{}, "total": 0, "limit": 50, "offset": 0}) + writeAdminTestJSON(t, w, http.StatusOK, map[string]any{"tasks": []any{}}) case r.Method == http.MethodPost && r.URL.Path == "/api/v1/tasks/42/retry": sawRetry = true if got := r.Header.Get("X-SynapS3-Settings-Write"); got != "" { @@ -813,7 +824,7 @@ func TestAdminTaskCommandsAndAPIErrorFields(t *testing.T) { })) defer ts.Close() - if out, err := runAdminCommand(t, []string{"synaps3", "admin", "--admin-url", ts.URL, "task", "list", "--status", "exhausted", "--limit", "50"}); err != nil { + if out, err := runAdminCommand(t, []string{"synaps3", "admin", "--admin-url", ts.URL, "task", "list", "--status", "failed", "--limit", "50"}); err != nil { t.Fatalf("task list: %v\n%s", err, out) } if out, err := runAdminCommand(t, []string{"synaps3", "admin", "--admin-url", ts.URL, "task", "retry", "42"}); err != nil { @@ -859,44 +870,24 @@ func TestAdminTaskCommandsAndAPIErrorFields(t *testing.T) { } }) - t.Run("stage filter requires type before request", func(t *testing.T) { - var called bool - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - called = true - writeAdminTestJSON(t, w, http.StatusOK, map[string]any{"tasks": []any{}, "total": 0, "limit": 20, "offset": 0}) - })) - defer ts.Close() - - out, err := runAdminCommand(t, []string{"synaps3", "admin", "--admin-url", ts.URL, "task", "list", "--stage", "prepare_upload"}) - if err == nil { - t.Fatalf("expected error, output:\n%s", out) - } - if called { - t.Fatal("request was sent") - } - }) - - t.Run("task list ref includes version id", func(t *testing.T) { + t.Run("task list includes subject key", func(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet || r.URL.Path != "/api/v1/tasks" { t.Fatalf("request = %s %s", r.Method, r.URL.Path) } writeAdminTestJSON(t, w, http.StatusOK, map[string]any{ "tasks": []map[string]any{{ - "id": 7, - "type": "upload", - "stage": "primary_commit", - "ref_type": "object", - "ref_id": 11, - "ref_version_id": "version-1", - "status": "exhausted", - "retry_count": 5, - "max_retries": 5, - "scheduled_at": "2026-05-05T10:00:00Z", + "id": 7, + "type": "upload_plan", + "operation": "Prepare storage", + "status": "failed", + "presentation_status": "Failed", + "subject_type": "object_version", + "subject_key": "version-1", + "retry_count": 5, + "retry_limit": 5, + "available_at": "2026-05-05T10:00:00Z", }}, - "total": 1, - "limit": 20, - "offset": 0, }) })) defer ts.Close() @@ -905,33 +896,56 @@ func TestAdminTaskCommandsAndAPIErrorFields(t *testing.T) { if err != nil { t.Fatalf("task list: %v\n%s", err, out) } - if !strings.Contains(out, "object:11:version-1") { - t.Fatalf("task output missing version id:\n%s", out) + if !strings.Contains(out, "object_version:version-1") { + t.Fatalf("task output missing subject key:\n%s", out) + } + }) + + t.Run("task list json preserves diagnostic and lifecycle fields", func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + writeAdminTestJSON(t, w, http.StatusOK, map[string]any{"tasks": []map[string]any{{ + "id": 9, "type": "storage_store", "operation": "Store content", + "status": "failed", "presentation_status": "dismissed", + "retry_count": 5, "retry_limit": 5, "retryable": false, "acknowledgeable": false, + "failure_reason": "provider_error", "last_error": "provider unavailable", + "available_at": "2026-05-05T10:00:00Z", "started_at": "2026-05-05T10:00:01Z", + "finished_at": "2026-05-05T10:00:02Z", "acknowledged_at": "2026-05-05T10:00:03Z", + "created_at": "2026-05-05T09:59:00Z", "updated_at": "2026-05-05T10:00:03Z", + }}}) + })) + defer ts.Close() + + out, err := runAdminCommand(t, []string{"synaps3", "admin", "--admin-url", ts.URL, "--json", "task", "list"}) + if err != nil { + t.Fatalf("task list json: %v\n%s", err, out) + } + for _, field := range []string{"failure_reason", "started_at", "finished_at", "acknowledged_at", "created_at", "updated_at"} { + if !strings.Contains(out, `"`+field+`"`) { + t.Fatalf("task list json dropped %s: %s", field, out) + } } }) - t.Run("task list shows waiting status details", func(t *testing.T) { + t.Run("task list shows presentation status and message", func(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet || r.URL.Path != "/api/v1/tasks" { t.Fatalf("request = %s %s", r.Method, r.URL.Path) } writeAdminTestJSON(t, w, http.StatusOK, map[string]any{ "tasks": []map[string]any{{ - "id": 8, - "type": "evict_cache", - "ref_type": "object", - "ref_id": 12, - "ref_version_id": "version-2", - "status": "waiting", - "retry_count": 0, - "max_retries": 5, - "wait_reason": "dependency", - "status_message": "waiting for all copies to commit", - "scheduled_at": "2026-05-05T10:00:00Z", + "id": 8, + "type": "cache_evict", + "operation": "Free local cache space", + "subject_type": "object_version", + "subject_key": "version-2", + "status": "pending", + "presentation_status": "Waiting", + "retry_count": 0, + "retry_limit": 5, + "wait_reason": "durability_pending", + "status_message": "Waiting for durable storage", + "available_at": "2026-05-05T10:00:00Z", }}, - "total": 1, - "limit": 20, - "offset": 0, }) })) defer ts.Close() @@ -943,7 +957,7 @@ func TestAdminTaskCommandsAndAPIErrorFields(t *testing.T) { if !strings.Contains(out, "DETAILS") || strings.Contains(out, "LAST_ERROR") { t.Fatalf("task output did not use details column:\n%s", out) } - if !strings.Contains(out, "dependency: waiting for all copies to commit") { + if !strings.Contains(out, "Waiting for durable storage") || strings.Contains(out, "durability_pending") { t.Fatalf("task output missing waiting details:\n%s", out) } }) @@ -965,7 +979,7 @@ func TestAdminStorageConfirmationCommands(t *testing.T) { t.Fatalf("limit = %q, want 25", got) } writeAdminTestJSON(t, w, http.StatusOK, []map[string]any{{ - "copy_id": 42, "upload_id": 7, "copy_index": 1, + "copy_id": 42, "content_id": 7, "copy_index": 1, "data_set_row_id": 9, "provider_id": "provider-1", "data_set_id": "dataset-1", "piece_cid": "bafy-piece-1", "attempt_id": "attempt-1", "transaction_id": "0xcommit", "reason_code": "attempt_only_ambiguous", @@ -991,7 +1005,8 @@ func TestAdminStorageConfirmationCommands(t *testing.T) { if err != nil { t.Fatalf("storage-confirmation list: %v\n%s", err, out) } - if !strings.Contains(out, "PIECE CID") || !strings.Contains(out, "ATTEMPTED AT") || + if !strings.Contains(out, "CONTENT ID") || strings.Contains(out, "UPLOAD ID") || + !strings.Contains(out, "PIECE CID") || !strings.Contains(out, "ATTEMPTED AT") || !strings.Contains(out, "bafy-piece-1") || !strings.Contains(out, "2026-08-30T01:00:00Z") || !strings.Contains(out, "attempt_only_ambiguous") || !strings.Contains(out, "provider-1") || !strings.Contains(out, "attempt-1") || !strings.Contains(out, "0xcommit") { @@ -1077,10 +1092,15 @@ func adminTestSettings(network string, allowPrivate bool) map[string]any { "lru_low_watermark_percent": 80, }, "worker": map[string]any{ - "upload": map[string]any{"concurrency": 4, "poll_interval": "5s", "max_retries": 5}, - "provider_replacement": map[string]any{"concurrency": 4, "poll_interval": "5s", "max_retries": 5}, - "evictor": map[string]any{"concurrency": 2, "poll_interval": "1m0s", "max_retries": 3}, - "storage_cleanup": map[string]any{"concurrency": 2, "poll_interval": "1m0s", "max_retries": 5}, + "tasks": map[string]any{ + "concurrency": 12, + "poll_interval": "5s", + "lease_duration": "5m0s", + "max_retries": 5, + "retention": "168h0m0s", + "provider_mutation_concurrency": 4, + "destructive_mutation_concurrency": 2, + }, }, "logging": map[string]any{ "level": "info", @@ -1092,10 +1112,5 @@ func adminTestSettings(network string, allowPrivate bool) map[string]any { } func containsString(values []string, want string) bool { - for _, value := range values { - if value == want { - return true - } - } - return false + return slices.Contains(values, want) } diff --git a/cmd/synaps3/main.go b/cmd/synaps3/main.go index 76858de..c85ed0f 100644 --- a/cmd/synaps3/main.go +++ b/cmd/synaps3/main.go @@ -429,18 +429,13 @@ func setupModeAllowedField(field string) bool { "filecoin.observability.interval", "filecoin.observability.timeout", "filecoin.observability.concurrency", - "worker.upload.concurrency", - "worker.upload.poll_interval", - "worker.upload.max_retries", - "worker.provider_replacement.concurrency", - "worker.provider_replacement.poll_interval", - "worker.provider_replacement.max_retries", - "worker.evictor.concurrency", - "worker.evictor.poll_interval", - "worker.evictor.max_retries", - "worker.storage_cleanup.concurrency", - "worker.storage_cleanup.poll_interval", - "worker.storage_cleanup.max_retries", + "worker.tasks.concurrency", + "worker.tasks.poll_interval", + "worker.tasks.lease_duration", + "worker.tasks.max_retries", + "worker.tasks.retention", + "worker.tasks.provider_mutation_concurrency", + "worker.tasks.destructive_mutation_concurrency", "logging.level", "logging.format", "logging.s3_access.enabled", diff --git a/cmd/synaps3/setup_mode_test.go b/cmd/synaps3/setup_mode_test.go index 7cde8d2..0247c87 100644 --- a/cmd/synaps3/setup_mode_test.go +++ b/cmd/synaps3/setup_mode_test.go @@ -36,11 +36,10 @@ func TestShouldStartSetupModeAllowsEditableConfigErrors(t *testing.T) { cfg.S3.Region = "" cfg.Filecoin.RPCURL = "ftp://example.invalid/rpc" cfg.Cache.MaxSizeGB = 0 - cfg.Worker.Upload.PollInterval = 0 - cfg.Worker.Upload.MaxRetries = -1 - cfg.Worker.ProviderReplacement.Concurrency = 0 - cfg.Worker.ProviderReplacement.PollInterval = 0 - cfg.Worker.ProviderReplacement.MaxRetries = -1 + cfg.Worker.Tasks.Concurrency = 0 + cfg.Worker.Tasks.PollInterval = 0 + cfg.Worker.Tasks.LeaseDuration = 0 + cfg.Worker.Tasks.MaxRetries = -1 cfg.Logging.Level = "verbose" cfg.Logging.S3Access.Level = "verbose" diff --git a/docs/en/concepts/architecture.md b/docs/en/concepts/architecture.md index c068fce..a4ff55c 100644 --- a/docs/en/concepts/architecture.md +++ b/docs/en/concepts/architecture.md @@ -44,7 +44,7 @@ The key boundary is between the S3 response and Filecoin upload. When a write is | Behavior | Operator impact | | --- | --- | | S3 writes land locally first | While local runtime data is intact, accepted writes remain available from local storage until eligible cache eviction. After eviction, reads require an available remote copy. | -| Background tasks handle Filecoin upload | Watch task queues and exhausted tasks. | +| Background tasks handle Filecoin storage | Watch pending, running, and failed tasks. | | Cache is part of durability | Treat cache disk as runtime data, not disposable scratch space. | | Admin API controls operations | Use Admin auth; keep it on loopback or behind HTTPS and access control. | diff --git a/docs/en/concepts/filecoin-storage-flow.md b/docs/en/concepts/filecoin-storage-flow.md index 4bc32ae..a91c52f 100644 --- a/docs/en/concepts/filecoin-storage-flow.md +++ b/docs/en/concepts/filecoin-storage-flow.md @@ -17,10 +17,9 @@ flowchart TD committing --> replicating["replicating"] replicating --> stored["stored"] stored --> policy{"cache eviction policy"} - policy -->|"after_upload"| evict["queue after-upload eviction"] + policy -->|"after_upload"| evict["remove local cache"] policy -->|"lru at high watermark"| evict policy -->|"none"| retain["retain local cache"] - evict --> evicted["cache_evicted"] ``` ## Object States @@ -33,20 +32,21 @@ flowchart TD | `replicating` | At least one readable committed copy exists, but the bucket's minimum durable copies are not yet met. | | `stored` | The bucket's minimum durable copies are readable and committed; remaining target copies may still be syncing. | | `failed` | The active lifecycle step failed and may be retried. | -| `cache_evicted` | Local cache has been removed after remote durability. | + +Cache presence is tracked separately from storage state. A `stored` object may remain in the local cache or be available only from remote storage. ## Retries and Recovery If SynapS3 is interrupted, unfinished tasks become eligible to continue after the service restarts. -Retries are bounded by background task settings. Tasks that exhaust retries need operator action: +Retries are bounded by background task settings. A task that reaches its limit enters `failed` and needs operator action: ```bash -synaps3 admin task list --status exhausted --limit 100 +synaps3 admin task list --status failed --limit 100 synaps3 admin task retry 42 ``` -Retry after restoring RPC connectivity, storage provider reachability, wallet funds, FWSS approval, or cache capacity. +Retry after restoring RPC connectivity, storage provider reachability, wallet funds, FWSS approval, or cache capacity, and only when the task is marked retryable. ## Provider Health @@ -56,7 +56,7 @@ If an established provider becomes temporarily unavailable while the initial cop ## Target and Minimum Replicas -The target replica count is frozen when an upload starts. By default, **Release cache after** is **All replicas (strict)**: every target replica frozen for that upload must be readable and committed. A bucket can instead set an explicit count from 1 through the current target. An explicit count stays if Replicas later increases; lowering Replicas below that count is rejected until Release cache after is also lowered. **All replicas (strict)** follows each upload's frozen target. Once that threshold is met, the version becomes stored and its cache follows the configured eviction policy, while remaining replicas continue until the upload's target is reached. The dashboard keeps showing replica sync progress until every frozen target replica is done. +The target replica count is frozen when an upload starts. A new bucket sets **Release cache after** to its full replica count, so every target replica frozen for that upload must be readable and committed before the cache is released. A bucket can instead set an explicit count from 1 through the current target, and that count stays if Replicas later increases. **Replicas** can only be raised; lowering it is not supported. Once that threshold is met, the version becomes stored and its cache follows the configured eviction policy, while remaining replicas continue until the upload's target is reached. The dashboard keeps showing replica sync progress until every frozen target replica is done. Changing the target affects new uploads. Changing the minimum also re-evaluates retained cache for current uploads. Increasing the minimum does not move versions that are already stored back to an earlier state and cannot restore cache that has already been deleted. @@ -66,7 +66,7 @@ When a provider becomes permanently unavailable, or you plan to move away from o One confirmation covers the whole move. SynapS3 creates the new storage service, switches new uploads to it once it is ready, copies existing data across, and only then shuts down the old provider. Objects copy from another replica or from local cache. An object with neither cannot be copied, and the old provider is not shut down. Data that can still be read from the old provider stays readable until every retained version is readable on the new one. -While replacements run, the replica shows every move that still needs progress or operator attention, including parallel moves on other replicas. Discovery uses an indeterminate progress bar because the final count is not known yet. Once discovery finishes, progress uses the processed share of the final total. Progress counts unique stored content, so content shared by several versions is copied once, and separates content transferred from content deleted before it needed to move. The same compact progress appears in Tasks; the Data Sets card shows the full breakdown. +While replacements run, the Data Sets card shows every move that still needs progress or operator attention, including parallel moves on other replicas. Discovery uses an indeterminate progress bar because the final count is not known yet. Once discovery finishes, progress uses the processed share of the final total. Progress counts unique stored content, so content shared by several versions is copied once, and separates content transferred from content deleted before it needed to move. Some steps wait rather than fail. The dashboard distinguishes creating the new service, waiting for it to become writable, an unreachable provider, wallet funds, and missing readable content. Most waits resume on their own. If an object has no other replica and no local cache, replacement waits until a source is available; this does not count toward the retry limit. Temporary copy failures resume after a restart and do not stop other content or another replacement. Use **Retry replacement** from the Data Sets list when the dashboard shows that content needs attention, or when shutting down the old provider needs a payment settled first. A target already in use cannot be retried; choose another provider. diff --git a/docs/en/concepts/write-path-cache.md b/docs/en/concepts/write-path-cache.md index 1bdbc3a..dbd9c11 100644 --- a/docs/en/concepts/write-path-cache.md +++ b/docs/en/concepts/write-path-cache.md @@ -42,7 +42,7 @@ Repeated reads of the same version coalesce access-time updates to at most one d | `after_upload` | Queue each version for removal after its bucket's minimum durable copies commit. | | `none` | Do not create or run automatic cache eviction work. | -Each bucket defaults to strict cache release, so the minimum equals the target replicas frozen for each upload. In the dashboard, set this on the bucket under Settings → Replica policy. An operator can set an explicit count from 1 through the current target. That count stays if Replicas later increases. Lowering Replicas below the stored count is rejected until Release cache after is also lowered. Lowering the threshold makes retained cache eligible for removal while remaining replicas continue syncing. Actual removal still follows `after_upload`, `lru`, or `none`. The minimum is clamped to each upload's target. Raising it affects cache that still exists; it cannot recreate cache that has already been deleted. +A bucket is created with the minimum equal to its replica target, so cache is released only once every replica of an upload is stored. In the dashboard, set this on the bucket under Settings → Replica policy. An operator can set an explicit count from 1 through the current target. That count stays if Replicas later increases. Replicas itself can only be raised: lowering it is not supported, so those options are shown but not selectable. Lowering the threshold makes retained cache eligible for removal while remaining replicas continue syncing. Actual removal still follows `after_upload`, `lru`, or `none`. The minimum is clamped to each upload's target. Raising it affects cache that still exists; it cannot recreate cache that has already been deleted. Only versions that currently meet their minimum and have a readable committed remote copy are eligible. Eviction checks the current minimum again before authorizing deletion and waits for active reads of the same version to close. Because cleanup is asynchronous, writes can still return `507 Insufficient Storage` when cleanup cannot keep pace or no safe candidate exists. diff --git a/docs/en/configuration/environment.md b/docs/en/configuration/environment.md index 6d790d8..0778e51 100644 --- a/docs/en/configuration/environment.md +++ b/docs/en/configuration/environment.md @@ -36,18 +36,13 @@ Configuration environment variables use the `SYNAPS3_` prefix and map underscore | `SYNAPS3_CACHE_EVICTION_POLICY` | `cache.eviction_policy` | | `SYNAPS3_CACHE_LRU_HIGH_WATERMARK_PERCENT` | `cache.lru_high_watermark_percent` | | `SYNAPS3_CACHE_LRU_LOW_WATERMARK_PERCENT` | `cache.lru_low_watermark_percent` | -| `SYNAPS3_WORKER_UPLOAD_CONCURRENCY` | `worker.upload.concurrency` | -| `SYNAPS3_WORKER_UPLOAD_POLL_INTERVAL` | `worker.upload.poll_interval` | -| `SYNAPS3_WORKER_UPLOAD_MAX_RETRIES` | `worker.upload.max_retries` | -| `SYNAPS3_WORKER_PROVIDER_REPLACEMENT_CONCURRENCY` | `worker.provider_replacement.concurrency` | -| `SYNAPS3_WORKER_PROVIDER_REPLACEMENT_POLL_INTERVAL` | `worker.provider_replacement.poll_interval` | -| `SYNAPS3_WORKER_PROVIDER_REPLACEMENT_MAX_RETRIES` | `worker.provider_replacement.max_retries` | -| `SYNAPS3_WORKER_EVICTOR_CONCURRENCY` | `worker.evictor.concurrency` | -| `SYNAPS3_WORKER_EVICTOR_POLL_INTERVAL` | `worker.evictor.poll_interval` | -| `SYNAPS3_WORKER_EVICTOR_MAX_RETRIES` | `worker.evictor.max_retries` | -| `SYNAPS3_WORKER_STORAGE_CLEANUP_CONCURRENCY` | `worker.storage_cleanup.concurrency` | -| `SYNAPS3_WORKER_STORAGE_CLEANUP_POLL_INTERVAL` | `worker.storage_cleanup.poll_interval` | -| `SYNAPS3_WORKER_STORAGE_CLEANUP_MAX_RETRIES` | `worker.storage_cleanup.max_retries` | +| `SYNAPS3_WORKER_TASKS_CONCURRENCY` | `worker.tasks.concurrency` | +| `SYNAPS3_WORKER_TASKS_POLL_INTERVAL` | `worker.tasks.poll_interval` | +| `SYNAPS3_WORKER_TASKS_LEASE_DURATION` | `worker.tasks.lease_duration` | +| `SYNAPS3_WORKER_TASKS_MAX_RETRIES` | `worker.tasks.max_retries` | +| `SYNAPS3_WORKER_TASKS_RETENTION` | `worker.tasks.retention` | +| `SYNAPS3_WORKER_TASKS_PROVIDER_MUTATION_CONCURRENCY` | `worker.tasks.provider_mutation_concurrency` | +| `SYNAPS3_WORKER_TASKS_DESTRUCTIVE_MUTATION_CONCURRENCY` | `worker.tasks.destructive_mutation_concurrency` | | `SYNAPS3_LOGGING_LEVEL` | `logging.level` | | `SYNAPS3_LOGGING_FORMAT` | `logging.format` | | `SYNAPS3_LOGGING_S3_ACCESS_ENABLED` | `logging.s3_access.enabled` | diff --git a/docs/en/configuration/model.md b/docs/en/configuration/model.md index 1d90b1a..7aef8a9 100644 --- a/docs/en/configuration/model.md +++ b/docs/en/configuration/model.md @@ -71,10 +71,7 @@ SQLite is the default and recommended database for SynapS3 single-node deploymen | `filecoin.observability` | Provider and local data set health checks. | | `database` | SQLite or PostgreSQL metadata database. | | `cache` | Local object cache directory, capacity, and eviction policy. | -| `worker.upload` | Background Filecoin storage concurrency, polling, and retries. | -| `worker.provider_replacement` | Provider-replacement transfer concurrency, polling, and copy retries. | -| `worker.evictor` | Local cache eviction tasks. | -| `worker.storage_cleanup` | Remote copy cleanup tasks. | +| `worker.tasks` | Shared background task execution, recovery, retention, and provider mutation limits. | | `logging` | Runtime log level, format, and S3 access logs. | | `admin` | Dashboard, Admin API listener, and Admin auth settings. | @@ -95,18 +92,20 @@ SQLite is the default and recommended database for SynapS3 single-node deploymen | `cache.eviction_policy` | `lru` | | `cache.lru_high_watermark_percent` | `90` | | `cache.lru_low_watermark_percent` | `80` | -| `worker.upload.concurrency` | `4` | -| `worker.upload.max_retries` | `5` | -| `worker.provider_replacement.concurrency` | `4` | -| `worker.provider_replacement.poll_interval` | `5s` | -| `worker.provider_replacement.max_retries` | `5` | +| `worker.tasks.concurrency` | `12` | +| `worker.tasks.poll_interval` | `5s` | +| `worker.tasks.lease_duration` | `5m` | +| `worker.tasks.max_retries` | `5` | +| `worker.tasks.retention` | `168h` | +| `worker.tasks.provider_mutation_concurrency` | `4` | +| `worker.tasks.destructive_mutation_concurrency` | `2` | | `admin.addr` | `127.0.0.1:9090` | | `admin.trusted_proxies` | `[]` | | `admin.auth.enabled` | `true` | | `admin.auth.username` | `admin` | | `admin.auth.session_ttl` | `12h` | -`worker.provider_replacement` settings do not affect ordinary uploads. Changing `worker.provider_replacement.max_retries` applies to replacement work discovered afterward and content retried with **Retry replacement**; work already in progress keeps its current limit. Changes to provider replacement concurrency, polling, or retries require a SynapS3 restart. +`worker.tasks.concurrency` limits all background operations. Remote storage creation, Store, Pull, and commit submission additionally share `provider_mutation_concurrency`; remote cleanup and service retirement share `destructive_mutation_concurrency`. Status and confirmation checks do not consume either mutation limit. Wallet mutations are serialized. Task settings require a SynapS3 restart, and existing tasks retain the retry limit recorded when they were created. ## Admin Session Lifetime @@ -127,7 +126,7 @@ The login page uses a browser-session cookie by default. Selecting **Keep me sig Cache eviction policies have these user-visible results: - `lru`: when cache usage reaches the high watermark, SynapS3 removes the least recently accessed remotely safe entries until usage reaches the low watermark. -- `after_upload`: after a version meets its bucket's minimum durable copies, SynapS3 queues it for removal at the next Evictor poll. A later remote read can restore the cache, and that restored entry is not immediately removed again. +- `after_upload`: after a version meets its bucket's minimum durable copies, SynapS3 queues it for asynchronous removal. A later remote read can restore the cache, and that restored entry is not immediately removed again. - `none`: SynapS3 does not automatically remove local cache data. The LRU watermarks must always satisfy `0 <= low < high <= 100`. They remain saved but have no effect under `after_upload` or `none`. diff --git a/docs/en/configuration/runtime-data.md b/docs/en/configuration/runtime-data.md index 22fa726..c27497b 100644 --- a/docs/en/configuration/runtime-data.md +++ b/docs/en/configuration/runtime-data.md @@ -51,7 +51,7 @@ Keep `config.toml`, `.env`, credential files, and exported secrets at permission ## Before a Backup 1. Check `curl http://127.0.0.1:9090/healthz` and record any non-`ok` result. -2. Review active and exhausted work with `synaps3 admin task stats` and `synaps3 admin task list --status exhausted`. +2. Review active and failed work with `synaps3 admin task stats` and `synaps3 admin task list --status failed`. 3. Stop SynapS3 with the service manager used by your deployment so object data, metadata, and task state cannot change during the backup. Do not create a filesystem archive while SynapS3 is still running. @@ -103,6 +103,8 @@ sha256sum -c synaps3-data.tgz.sha256 2. Verify the archive checksum and confirm the database and cache have the same recovery-point label. 3. Restore the runtime volume into an empty replacement location. For PostgreSQL, restore the database-native backup before attaching the matching configuration and cache data. 4. Confirm the restored configuration and credential files are `0600` and readable by the SynapS3 account. -5. Start SynapS3, check `/healthz`, review task statistics and exhausted tasks, then read a known object through the S3 API. +5. Start SynapS3, check `/healthz`, review task statistics and failed tasks, then read a known object through the S3 API. Do not combine a database backup with cache data from another point in time. + +Restore a backup only with a compatible SynapS3 version. If startup reports that the database is incompatible, leave the backup unchanged and follow [Upgrade and Recovery](../operations/upgrade-recovery.md). diff --git a/docs/en/getting-started/s3-clients.md b/docs/en/getting-started/s3-clients.md index d92edb6..c04e278 100644 --- a/docs/en/getting-started/s3-clients.md +++ b/docs/en/getting-started/s3-clients.md @@ -134,6 +134,7 @@ The HTTP endpoints in these examples are for local evaluation. For production, u | --- | --- | | `AccessDenied` | Confirm the access key and secret key came from `synaps3 admin s3-user create`. | | Client tries virtual-hosted buckets | Enable path-style addressing or equivalent client setting. | -| Upload succeeds but Filecoin storage is pending | Check the dashboard task view or `synaps3 admin task list --status queued`. | +| The first upload after creating a bucket returns `SlowDown` | The storage provider may still be preparing the bucket. Wait briefly, retry the upload, then confirm it with the read command. | +| Upload succeeds but Filecoin storage is pending | Check the dashboard task view or `synaps3 admin task list --status pending`. | | Object size is rejected | Keep the object between `127` and `1,065,353,216` bytes. | | Remote host cannot reach the admin dashboard | Keep admin on loopback and use `ssh -L 9090:127.0.0.1:9090 user@server`. | diff --git a/docs/en/operations/health-metrics.md b/docs/en/operations/health-metrics.md index 083b2e7..1631125 100644 --- a/docs/en/operations/health-metrics.md +++ b/docs/en/operations/health-metrics.md @@ -31,7 +31,7 @@ Missing required configuration: Failed check: ```json -{"status":"unhealthy","errors":["worker/uploader: not responding"]} +{"status":"unhealthy","errors":["worker/tasks: not responding"]} ``` `setup` means required configuration is missing. `unhealthy` means a database, cache, or background task check failed; check the returned errors first. @@ -47,7 +47,7 @@ synaps3 admin status synaps3 admin task stats ``` -Status should show background task processing as healthy. Task stats show whether work is queued, running, failed, or exhausted. +Status should show background task processing as healthy. Task stats report acknowledged failures separately as dismissed, and the dashboard presents pending work as queued, scheduled, or waiting. ## Prometheus Metrics @@ -74,9 +74,6 @@ Key metrics: | `synaps3_cache_used_bytes` | Current cache disk usage. | | `synaps3_cache_hits_total` / `synaps3_cache_misses_total` | Cache read behavior. | | `synaps3_cache_lru_eviction_paused` | `1` when LRU eviction is paused because recent cache access could not be retained safely. Resolve the persistence error and restart SynapS3. | -| `synaps3_worker_tasks_processed_total` | Background task throughput by result. | -| `synaps3_worker_tasks_exhausted_total` | Tasks that exhausted retries. | -| `synaps3_worker_task_duration_seconds` | Background task processing duration. | | `synaps3_task_queue_depth` | Active tasks by type and status. | | `synaps3_object_state_distribution` | Object counts by lifecycle state. | @@ -87,7 +84,7 @@ Key metrics: | `/healthz` returns `setup` | Run `synaps3 admin status` or `synaps3 admin settings get`, set the reported missing configuration, restart, and check again. | | `/healthz` returns `unhealthy` | Check database, cache directory, and background task error messages. | | Cache usage approaches capacity | Increase capacity or restore upload and eviction progress. | -| Exhausted task count increases | Fix the dependency, then retry tasks. | +| Failed task count increases | Fix the dependency, then retry only tasks marked retryable. | | Provider health is degraded | Check RPC, provider URLs, and network reachability. | See [Troubleshooting](./troubleshooting.md) for recovery steps. diff --git a/docs/en/operations/production-checklist.md b/docs/en/operations/production-checklist.md index f854b01..cffc08d 100644 --- a/docs/en/operations/production-checklist.md +++ b/docs/en/operations/production-checklist.md @@ -77,7 +77,7 @@ At minimum, monitor: - `GET /metrics` - cache usage - task queue depth -- exhausted task count +- failed task count - background task activity - provider and data set health @@ -85,15 +85,17 @@ Treat `{"status":"unhealthy"}` as a problem to investigate. It means database, c ## Upgrade Readiness +Back up the database and cache before changing versions, then follow [Upgrade and Recovery](./upgrade-recovery.md). If startup reports that the database is incompatible, leave it unchanged and use a new database and cache directory. + Before upgrading: ```bash curl http://127.0.0.1:9090/healthz synaps3 admin task stats -synaps3 admin task list --status exhausted --limit 50 +synaps3 admin task list --status failed --limit 50 ``` -Expected result: health is `ok`, task queues are understood, and every exhausted task has a clear handling decision before the process is replaced. +Expected result: health is `ok`, task queues are understood, and every failed task has a clear handling decision before the running process is replaced. ## Recovery Entry Points diff --git a/docs/en/operations/troubleshooting.md b/docs/en/operations/troubleshooting.md index 361ec24..9ed72ec 100644 --- a/docs/en/operations/troubleshooting.md +++ b/docs/en/operations/troubleshooting.md @@ -55,7 +55,7 @@ Expected result: health changes from `setup` to `ok` after restart. Example: ```json -{"status":"unhealthy","errors":["worker/uploader: not responding"]} +{"status":"unhealthy","errors":["worker/tasks: not responding"]} ``` Check task pressure: @@ -109,21 +109,21 @@ Recovery options: - Confirm the host has free disk space, then increase `cache.max_size_gb` if capacity allows. - Restore storage provider connectivity and background task progress so queued uploads can complete and cache eviction can run. - Use the default `lru` policy for capacity-based cleanup. Lower the high watermark to leave more write headroom, and keep `0 <= low < high <= 100`. -- Use `after_upload` only when each version should be removed on the next Evictor poll after its bucket's minimum durable copies commit. +- Use `after_upload` only when each version should be removed asynchronously after its bucket's minimum durable copies commit. - Use `none` when automatic removal must be disabled. -LRU cannot remove multipart staging data, versions below their bucket's minimum durable copies, or versions without a readable committed remote copy. A write does not synchronously run eviction, so `507 Insufficient Storage` can continue until the Evictor catches up or safe candidates become available. +LRU cannot remove multipart staging data, versions below their bucket's minimum durable copies, or versions without a readable committed remote copy. A write does not synchronously run cleanup, so `507 Insufficient Storage` can continue until background cleanup catches up or safe candidates become available. -Failed LRU deletion tasks remain visible as exhausted work and become eligible again after a one-hour cooldown. Fix the reported filesystem or database problem first; use `synaps3 admin task retry ` to retry sooner. +Failed LRU deletion tasks remain visible as failed work. Fix the reported filesystem or database problem first; use `synaps3 admin task retry ` when the task is marked retryable. After changing a cache setting, restart SynapS3, check `/healthz`, and verify the effective cache values with `synaps3 admin settings get`. -## Exhausted Tasks +## Failed Tasks -List exhausted work: +List failed work: ```bash -synaps3 admin task list --status exhausted --limit 100 +synaps3 admin task list --status failed --limit 100 ``` Retry only after RPC connectivity, storage provider availability, wallet funds, FWSS approval, and cache disk capacity are ready. @@ -132,7 +132,7 @@ Retry only after RPC connectivity, storage provider availability, wallet funds, synaps3 admin task retry 42 ``` -Provider replacement work is the exception: do not retry it from Tasks. Copy retries and waits for readable content resume automatically, including after a restart. Finished or stopped replacement tasks provide **Open Data Sets**, which opens the affected bucket directly at **Details** → **Storage** → **Data Sets**. Use **Retry replacement** only when that action is shown. It retries content that needs attention with the current `worker.provider_replacement.max_retries` setting and keeps completed work. If the selected provider already stores this bucket, choose a different provider instead. +The API decides whether each failed task can be retried safely. Provider replacement work is recovered from **Details** → **Storage** → **Data Sets**. A wallet operation can be recovered from Tasks only when no broadcast started; an uncertain broadcast remains non-retryable. An uncertain Store offers **Check again**, which observes the provider without uploading again. Use **Dismiss** or `synaps3 admin task acknowledge ` only after reviewing the failure; acknowledged tasks remain available for the configured retention period before cleanup. ## Provider or RPC Issues @@ -159,4 +159,4 @@ Check these in order: 2. Access key and secret came from `synaps3 admin s3-user create`. 3. Endpoint is `http://localhost:8080` for local evaluation or the correct HTTPS address for production. 4. Object size is between `127` and `1,065,353,216` bytes, and the object key meets the [S3 compatibility limits](../reference/s3-compatibility.md#stable-limits). -5. Dashboard task view shows whether Filecoin storage is queued, running, or exhausted. +5. Dashboard task view shows whether Filecoin storage is queued, running, waiting, or failed. diff --git a/docs/en/operations/upgrade-recovery.md b/docs/en/operations/upgrade-recovery.md index 279c7b8..0713d6f 100644 --- a/docs/en/operations/upgrade-recovery.md +++ b/docs/en/operations/upgrade-recovery.md @@ -1,11 +1,11 @@ --- title: Upgrade and Recovery -description: Upgrade SynapS3 safely and recover from common single-node failure scenarios. +description: Upgrade SynapS3 safely and recover background work. --- # Upgrade and Recovery -SynapS3 is a single-node gateway. During an upgrade or recovery, protect locally durable object data and metadata as one consistent set, then resume background tasks. Restore failed dependencies before retrying work. +Before changing versions, protect the database and cache as one recovery point. Restore failed dependencies before retrying background work. ## Before Upgrading @@ -14,27 +14,20 @@ Run: ```bash curl http://127.0.0.1:9090/healthz synaps3 admin task stats -synaps3 admin task list --status exhausted --limit 50 +synaps3 admin task list --status failed --limit 50 ``` -Expected result: health is `ok`, and every exhausted task has a clear handling decision before the process is replaced. +Expected result: health is `ok`, and every failed task has a clear handling decision before the process is replaced. -Before upgrading, stop incoming S3 writes but leave the current SynapS3 process running until uploads and provider replacements have finished. If the new version refuses to start because storage work is still in progress, run the previous version against the unchanged database, restore the affected provider if necessary, and let that work finish before retrying the upgrade. Do not alter the database to bypass this check. - -The upgrade is also refused when a stored copy still records a storage confirmation whose outcome was never resolved, which an interrupted earlier version could leave behind. The message reports how many copies are affected and includes the query that lists them, so you can confirm with the provider whether each piece was stored before clearing that record. A development environment can start from a fresh database instead. - -Stop S3 traffic and SynapS3 with the service manager used by your deployment before creating a backup. - -- SQLite deployments: archive the complete runtime data volume, then verify the archive and its checksum. -- PostgreSQL deployments: create a database-native backup and archive the matching configuration and cache data. - -Keep every backup artifact at the same recovery point. Follow [Runtime Data](../configuration/runtime-data.md) for exact backup, verification, and restart steps. +Stop incoming S3 traffic and SynapS3 before creating a backup. Keep the database, cache, configuration, and credentials at the same recovery point. Follow [Runtime Data](../configuration/runtime-data.md) for backup and verification steps. ## Upgrade SynapS3 -Replace the executable, package, or container image through the same installation method used for the current deployment. Docker-specific commands are documented on the [Docker Deployment](../getting-started/docker.md) page. +Replace the executable, package, or container image through the same installation method used for the deployment. Docker-specific commands are documented on the [Docker Deployment](../getting-started/docker.md) page. -Start SynapS3 with the service manager used by your deployment, then run: +Start SynapS3 with the intended database and cache. If startup reports that the database is incompatible, stop the process, leave the database unchanged, and follow [If the Database Is Incompatible](#if-the-database-is-incompatible). + +After startup, run: ```bash curl http://127.0.0.1:9090/healthz @@ -42,53 +35,73 @@ synaps3 admin settings get synaps3 admin task stats ``` -Expected result: health is `ok`, effective settings match the deployment, and task queues resume without unexpected exhausted work. Read a known object through the S3 API before restoring normal traffic. +Expected result: health is `ok`, effective settings match the deployment, and background work resumes without unexpected failures. Read a known object through the S3 API before restoring normal traffic. + +## If the Database Is Incompatible + +1. Stop incoming S3 traffic and every SynapS3 process using the deployment. +2. Back up the reported database and verify the backup. +3. Keep the database and matching cache read-only. Do not edit either location to bypass the compatibility check. +4. Configure an empty database and cache directory for the replacement installation. +5. Start SynapS3 and verify health, effective settings, and background task processing before restoring traffic. -## Runtime Flow +For SQLite, create a consistent backup after the process stops and verify that it opens: -```text -Receive write -> save object -> record metadata -> return success -> continue background storage +```bash +sqlite3 /old/path/synaps3.db ".backup '/backup/path/synaps3-pre-upgrade.db'" +sqlite3 -readonly /backup/path/synaps3-pre-upgrade.db "PRAGMA integrity_check;" ``` -- Writes commit to local cache and metadata before provider upload. -- Failed storage tasks retry and move to `exhausted` after the configured retry limit. -- `GetObject` reads from cache first and can retrieve from the provider when metadata is available. -- Bucket deletion is not supported and returns `501`; object deletion removes the object from S3 visibility while cleanup continues safely. +The integrity check must print `ok`. Protect the backup, its WAL/SHM files when retained, the matching cache, and the configuration as one recovery set. PostgreSQL deployments should use `pg_dump` or the deployment's approved database snapshot and verify that artifact separately. -## Recovery Matrix +SynapS3 leaves an incompatible database unchanged. Deprecated `worker.upload`, `worker.provider_replacement`, `worker.evictor`, and `worker.storage_cleanup` configuration sections are also rejected; replace them with `worker.tasks` settings. -| Scenario | Recovery | -| --- | --- | -| Established provider is temporarily unavailable during initial storage | Restore the original provider. Other assigned writable copies continue, while the unfinished copy waits without consuming retries and resumes automatically. SynapS3 does not select a replacement provider. | -| Background storage task cannot reach a provider | Restore connectivity, then retry exhausted storage tasks. | -| RPC node down | Restore RPC connectivity, then retry exhausted tasks. | -| Private provider URL blocked | Keep blocked by default; enable `filecoin.allow_private_networks` only for trusted private deployments. | -| Database full | Free space or scale the database. | -| Cache disk full | Increase disk, raise `cache.max_size_gb`, or restore upload and eviction progress. | -| Provider is permanently unavailable, or must be evacuated | Open the bucket, choose **Details**, then **Storage** → **Data Sets**, and replace the provider. New uploads move to the new provider once it is ready. Existing objects copy from another replica or from local cache; an object with neither cannot be copied, and the old provider is not shut down. If the selected target is already in use, choose another provider rather than retrying it. | -| Process crash | Restart the service, then verify health and task statistics. Most unfinished storage work resumes automatically without submitting the same piece again. Work whose outcome cannot be determined safely appears in `synaps3 admin storage-confirmation list`; inspect the listed provider and transaction details before taking action. Recorded service-shutdown transactions are checked before another shutdown is submitted. | +Starting with an empty database does not import existing buckets, objects, users, storage data sets, wallet operations, provider replacements, or tasks. Existing paid remote storage services remain active. Keep the verified backup so those services and records can be reviewed and handled separately. -A provider becoming unavailable after a copy has already been stored does not necessarily create a retryable task. Use storage-health views to identify affected copies. Restoring the target copy count is part of [Planned Replica Repair](../concepts/filecoin-storage-flow.md#planned-replica-repair). +Do not run the preserved installation and its replacement against the same database, cache, wallet workflow, or S3 traffic. After starting the replacement, create an S3 user and test bucket, then write and read a test object before restoring normal traffic. -## Restore or Roll Back +## Recover Background Work -1. Stop S3 traffic and SynapS3. -2. Verify archive checksums and select database and cache artifacts from the same recovery point. -3. For SQLite, restore the complete runtime data volume. For PostgreSQL, restore the database-native backup first, then the matching configuration and cache data. -4. If rolling back the application, start the previous release only with data that is compatible with that version. When compatibility is uncertain, restore the pre-upgrade recovery point. -5. Start SynapS3, then check `/healthz`, effective settings, task statistics, exhausted tasks, wallet readiness, and a known S3 object. +After a restart, unfinished work becomes eligible to continue automatically. -Do not resume normal traffic until these checks pass. +- Retry a failed task only when the dashboard or API marks it retryable. +- Recover provider replacements from **Details** → **Storage** → **Data Sets**. +- A wallet operation can be retried from Tasks only when no broadcast started. An uncertain broadcast remains non-retryable. +- For an uncertain Store, **Check again** checks the provider without uploading the object again. +- `status=failed` lists unacknowledged failures. Use `status=dismissed` to list acknowledged failures. +- Review unresolved storage confirmations with `synaps3 admin storage-confirmation list`. Useful commands: ```bash -synaps3 admin task list --status exhausted --limit 100 +synaps3 admin task list --status failed --limit 100 +synaps3 admin task list --status dismissed --limit 100 synaps3 admin task stats synaps3 admin task retry 42 +synaps3 admin task acknowledge 42 synaps3 admin storage-confirmation list -synaps3 admin s3-user list synaps3 admin settings get ``` -After changing a recovery-related setting, restart SynapS3 and verify both `/healthz` and `synaps3 admin settings get`. +Restore failed dependencies before retrying work. Use the dashboard, Admin API, or CLI instead of editing the application database. + +## Recovery Matrix + +| Scenario | Recovery | +| --- | --- | +| Storage provider or RPC is temporarily unavailable | Restore connectivity. Waiting work resumes automatically; retry only failed tasks marked retryable. | +| Database full | Stop traffic, free space or scale the database, then verify health. | +| Cache disk full | Increase disk or `cache.max_size_gb`, or restore remote storage and cache-cleanup progress. | +| Provider must be evacuated | Open the bucket and use **Details** → **Storage** → **Data Sets**. Do not retry the replacement from Tasks. | +| Process crash | Restart SynapS3, verify health and task statistics, then review any unresolved storage confirmation or wallet outcome. | +| Startup reports an incompatible database | Stop the process, verify that the configured database is the intended one, and preserve it unchanged before using an empty replacement database. | + +## Restore or Roll Back + +1. Stop S3 traffic and SynapS3. +2. Verify backup checksums and select database and cache artifacts from the same recovery point. +3. For SQLite, restore the complete runtime data volume. For PostgreSQL, restore the database-native backup first, then the matching configuration and cache data. +4. If rolling back the application, use only data compatible with the selected version. When compatibility is uncertain, restore the pre-upgrade recovery point. +5. Start SynapS3, then check `/healthz`, effective settings, task statistics, failed tasks, wallet readiness, and a known S3 object. + +Do not resume normal traffic until these checks pass. diff --git a/docs/en/reference/admin-api.md b/docs/en/reference/admin-api.md index 26de359..9967fba 100644 --- a/docs/en/reference/admin-api.md +++ b/docs/en/reference/admin-api.md @@ -35,7 +35,6 @@ Runtime metrics, buckets, objects, tasks, wallet operations, storage health, and | `/api/v1/auth/refresh`, `/api/v1/auth/logout` | Require a valid browser session and CSRF header; HTTP Basic auth is not accepted. | | `/api/v1/*` | Browser session cookie with CSRF for unsafe methods, or HTTP Basic auth. | | `/metrics` | Browser session cookie or HTTP Basic auth. | -| `/admin/exhausted-tasks*` | Browser session cookie with CSRF for unsafe methods, or HTTP Basic auth. | ### Browser Sessions @@ -92,7 +91,7 @@ Treat these endpoints as change-window operations. They can change data, credent | Wallet | `POST /api/v1/wallet/fund`, `POST /api/v1/wallet/withdraw`, `POST /api/v1/wallet/approve` | Creates on-chain payment operations. | | S3 users | `POST /api/v1/s3-users`, `PUT /api/v1/s3-users/{accessKey}`, `POST /api/v1/s3-users/{accessKey}/secret`, `DELETE /api/v1/s3-users/{accessKey}` | Changes client access or invalidates credentials. | | Buckets and objects | bucket create, owner/copy-policy updates, object upload/download/delete/restore/permanent-delete | Changes or exposes user-visible S3 data and metadata. | -| Tasks and storage health | task retry, diagnostic refresh, storage provider and data set refresh | Requeues work or refreshes operational status. | +| Tasks and storage health | task retry and acknowledgement, storage provider and data set refresh | Requeues work, dismisses a reviewed failure and starts its retention period, or refreshes operational status. | | Provider replacement | `POST /api/v1/buckets/{name}/data-sets/{id}/replacement`, `POST /api/v1/storage-replacements/{id}/retry` | Creates a new paid storage service, moves a replica to it, and ends the old service. | | Storage confirmation | `POST /api/v1/storage-confirmations/{copy-id}/release` | May permit the provider to store the same piece again. Verify the current attempt before releasing it. | @@ -142,12 +141,14 @@ For object upload, the HTTP `Content-Type` is the uploaded object's content type ### Bucket Copy Policy -`POST /api/v1/buckets` accepts optional `default_copies` and `minimum_durable_copies` fields. Bucket list, detail, create, and policy-update responses include: +`POST /api/v1/buckets` accepts optional `default_copies` and `minimum_durable_copies` fields. Either one left out is taken from the server configuration. A bucket stores the policy it was created with, so later configuration changes leave existing buckets alone. Bucket list, detail, create, and policy-update responses include both values as plain integers: -- `minimum_durable_copies`: the explicit bucket value, or `null` for strict per-upload behavior; -- `effective_minimum_durable_copies`: the current display value after clamping the bucket minimum to the current target. +- `default_copies`: the bucket's replica target; +- `minimum_durable_copies`: how many replicas must be stored before the cache may be released. It never exceeds the target. -`PUT /api/v1/buckets/{name}/copy-policy` accepts `default_copies` and `minimum_durable_copies` independently. An omitted field is unchanged. `default_copies: null` inherits the current runtime target for new uploads. `minimum_durable_copies: null` requires every replica frozen for each upload before releasing its cache. An explicit minimum must be between `1` and `8` and cannot exceed the target produced by the same request. An empty request or an invalid final combination returns `400 Bad Request`. +`PUT /api/v1/buckets/{name}/copy-policy` accepts `default_copies` and `minimum_durable_copies` independently. An omitted field is unchanged. `null` resets that field: `default_copies: null` restores the configured default, and `minimum_durable_copies: null` sets the minimum equal to the replica target. An explicit value must be between `1` and `8`, and the minimum cannot exceed the target produced by the same request. An empty request or an invalid final combination returns `400 Bad Request`. + +**Lowering `default_copies` is not supported and returns `400 Bad Request`.** The replicas above a lower target would keep running and keep costing, and nothing retires them, so the target can only be raised. A `null` reset that would land below the bucket's current target is refused for the same reason. Target changes affect new uploads. Minimum changes also re-evaluate retained cache for current uploads. Increasing the minimum cannot restore cache that has already been deleted. @@ -279,18 +280,18 @@ If the attempt changed after it was inspected, the API returns `409 Conflict`. I | Method | Path | Purpose | | --- | --- | --- | -| `GET` | `/api/v1/tasks` | List background tasks. Supports filters such as `type`, `stage`, `status`, `limit`, and `offset`. | +| `GET` | `/api/v1/tasks` | List background tasks. Supports `type`, `status`, `limit`, and ID-based `cursor`. | | `GET` | `/api/v1/tasks/stats` | Count tasks by status. | -| `GET` | `/api/v1/tasks/{id}/ref-detail` | Resolve the object or storage operation related to a task. | -| `GET` | `/api/v1/tasks/{id}/diagnostic` | Read task diagnostics. | -| `POST` | `/api/v1/tasks/{id}/diagnostic/refresh` | Refresh diagnostics. | -| `POST` | `/api/v1/tasks/{id}/retry` | Retry an exhausted task. | -| `GET` | `/admin/exhausted-tasks` | List exhausted tasks. Supports `limit` up to `1000`. | -| `POST` | `/admin/exhausted-tasks/{id}/retry` | Retry an exhausted task (legacy path). | +| `POST` | `/api/v1/tasks/{id}/retry` | Recover a failed task when `retryable` is true. | +| `POST` | `/api/v1/tasks/{id}/acknowledge` | Dismiss a failed task when `acknowledgeable` is true. Acknowledgement starts its retention period, after which it may be cleaned up. | + +`status` is `pending`, `running`, `completed`, `failed`, or `cancelled`. `presentation_status` renders pending work as `queued`, `scheduled`, or `waiting`, and acknowledged failures as `dismissed`. Responses also include `operation`, optional subject identity, and server-computed `retryable` and `acknowledgeable` flags. + +The `status` filter also accepts `dismissed`. `status=failed` returns only unacknowledged failures, while `status=dismissed` returns acknowledged failures. `/api/v1/tasks/stats` reports those groups separately as `failed` and `dismissed`. -Replacement and retirement task responses that refer to a bucket include `bucket_name` in list and reference-detail responses. Retrying replacement work from the task queue returns `409 Conflict` with `"code": "replacement_task_retry_unsupported"`. When a replacement task has completed or stopped, use **Open Data Sets**, or open the bucket and go to Details → Storage → Data Sets. A `target_in_use` failure has no Retry action because it requires a different provider. +`/api/v1/overview` groups `tasks.by_status` by `status`, so its `failed` count includes acknowledged failures. Use `tasks.attention.failed` for unacknowledged failures or `/api/v1/tasks/stats` for counts split between `failed` and `dismissed`. -Task list `progress` is a scope-discriminated object. With `scope: "ingress_store"`, it contains `attempt`, `uploaded_bytes`, `total_bytes`, optional `percent`, `done`, and `updated_at`. With `scope: "provider_replacement"`, it contains the same replacement progress returned by the bucket response. Clients must branch on `scope`. +Pagination is newest-first. When `next_cursor` is present, pass it as `cursor` to fetch the next page. Provider replacement recovery remains in the Data Sets API. Wallet operations are retryable only before a broadcast starts. Retrying an uncertain Store checks the provider and does not upload the bytes again. ## Wallet and Filecoin diff --git a/docs/en/reference/cli-api.md b/docs/en/reference/cli-api.md index a418bf3..b762d46 100644 --- a/docs/en/reference/cli-api.md +++ b/docs/en/reference/cli-api.md @@ -81,8 +81,9 @@ synaps3 admin settings get synaps3 admin settings set cache.max_size_gb=200 synaps3 admin settings set cache.eviction_policy=lru cache.lru_high_watermark_percent=90 cache.lru_low_watermark_percent=80 synaps3 admin task stats -synaps3 admin task list --status exhausted --limit 100 +synaps3 admin task list --status failed --limit 100 synaps3 admin task retry 42 +synaps3 admin task acknowledge 42 synaps3 admin storage-confirmation list synaps3 admin storage-confirmation release 42 --attempt-id current-attempt-id --yes ``` @@ -97,9 +98,9 @@ Admin global flags must appear after `admin` and before the subcommand: | `--json` | Return successful responses as JSON. | | `--timeout ` | Set the Admin API request timeout. | -Task listing supports `--type`, `--stage`, `--status`, `--limit`, and `--offset`. `--stage` requires `--type`. +Task listing supports `--type`, `--status`, `--limit`, and ID-based `--cursor`. Valid status filters are `pending`, `running`, `completed`, `failed`, `cancelled`, and `dismissed`. Pending work is presented as queued, scheduled, or waiting; `failed` returns unacknowledged failures and `dismissed` returns acknowledged failures. -`synaps3 admin task retry` does not retry provider replacement work. Use **Open Data Sets** from a finished or stopped replacement task, or open the bucket and go to **Details** → **Storage** → **Data Sets**. If the selected provider already stores this bucket, choose a different provider instead of retrying. +`synaps3 admin task retry` recovers only tasks whose response says they are retryable. Provider replacement recovery remains under **Details** → **Storage** → **Data Sets**. A wallet operation can be retried only when its broadcast never started; an operation with an uncertain broadcast remains non-retryable. For an uncertain Store, the dashboard labels Retry as **Check again**: this checks the provider without uploading again. Use `synaps3 admin task acknowledge ` to dismiss a failed task after reviewing its outcome; acknowledgement starts its retention period, after which it may be cleaned up. `synaps3 admin storage-confirmation list` shows storage confirmations that need review. Verify the piece CID, provider, current attempt ID, attempted time, and any available transaction evidence before running `storage-confirmation release --attempt-id --yes`; release only if you accept that the provider may already store the piece and resubmission may create duplicate paid storage. A stale attempt ID is refused. diff --git a/docs/zh/concepts/architecture.md b/docs/zh/concepts/architecture.md index df387c8..624813d 100644 --- a/docs/zh/concepts/architecture.md +++ b/docs/zh/concepts/architecture.md @@ -44,7 +44,7 @@ flowchart LR | 行为 | 运维影响 | | --- | --- | | S3 写入先落本地 | 本地运行数据完好且缓存尚未淘汰时,已接受写入可从本地存储读取;淘汰后,读取需要可用的远端副本。 | -| 后台任务处理 Filecoin 上传 | 需要关注任务队列和 exhausted 任务。 | +| 后台任务处理 Filecoin 存储 | 需要关注 pending、running 和 failed 任务。 | | 缓存是持久性的一部分 | 缓存磁盘不是可随意丢弃的临时目录。 | | Admin API 控制运维操作 | 使用 Admin 认证;保持本机回环地址或放在 HTTPS 和访问控制之后。 | diff --git a/docs/zh/concepts/filecoin-storage-flow.md b/docs/zh/concepts/filecoin-storage-flow.md index cda81a0..59b410d 100644 --- a/docs/zh/concepts/filecoin-storage-flow.md +++ b/docs/zh/concepts/filecoin-storage-flow.md @@ -17,10 +17,9 @@ flowchart TD committing --> replicating["replicating"] replicating --> stored["stored"] stored --> policy{"缓存淘汰策略"} - policy -->|"after_upload"| evict["排队执行上传后淘汰"] + policy -->|"after_upload"| evict["清理本地缓存"] policy -->|"lru 达到高水位"| evict policy -->|"none"| retain["保留本地缓存"] - evict --> evicted["cache_evicted"] ``` ## 对象状态 @@ -33,20 +32,21 @@ flowchart TD | `replicating` | 至少已有一个可读已提交副本,但尚未达到存储桶的最低耐久副本门槛。 | | `stored` | 存储桶要求的最低耐久副本已可读并提交;其余目标副本可能仍在补齐。 | | `failed` | 正在执行的生命周期步骤失败,可重试。 | -| `cache_evicted` | 远端持久化后,本地缓存已清理。 | + +缓存是否存在与存储状态分开记录。`stored` 对象可能仍保留在本地缓存,也可能仅能从远端存储读取。 ## 重试与恢复 如果 SynapS3 运行中断,未完成的后台任务会在服务重启后重新进入可继续处理的状态。 -重试次数由后台任务设置限制。耗尽重试次数的任务需要运维处理: +重试次数由后台任务设置限制。达到上限的任务会进入 `failed`,需要运维处理: ```bash -synaps3 admin task list --status exhausted --limit 100 +synaps3 admin task list --status failed --limit 100 synaps3 admin task retry 42 ``` -重试前先恢复 RPC 连接、存储提供方可达性、钱包余额、FWSS approval 或缓存容量。 +重试前先恢复 RPC 连接、存储提供方可达性、钱包余额、FWSS approval 或缓存容量,并且只重试标记为可重试的任务。 ## 存储提供方健康状态 @@ -56,7 +56,7 @@ synaps3 admin task retry 42 ## 目标副本与最低耐久副本 -上传开始时会冻结目标副本数。默认情况下,**Release cache after** 为 **All replicas (strict)**:该次上传冻结的全部目标副本都必须可读并完成提交。存储桶也可以设置 1 到当前目标之间的显式数量。显式数量在目标随后提高时保持不变;如果把 Replicas 降到低于该数量,请求会被拒绝,必须同时降低 Release cache after。**All replicas (strict)** 会跟随每次上传冻结的目标。达到该门槛后,版本进入已存储状态,缓存按已配置的淘汰策略处理;其余副本会继续补齐,直到达到该次上传的目标副本数。在冻结的目标副本全部完成前,仪表盘会继续显示副本同步进度。 +上传开始时会冻结目标副本数。新建存储桶时 **Release cache after** 等于其目标副本数,因此该次上传冻结的全部目标副本都必须可读并完成提交后才释放缓存。存储桶也可以设置 1 到当前目标之间的显式数量,该数量在目标随后提高时保持不变。**Replicas** 只能调高,不支持调低。达到该门槛后,版本进入已存储状态,缓存按已配置的淘汰策略处理;其余副本会继续补齐,直到达到该次上传的目标副本数。在冻结的目标副本全部完成前,仪表盘会继续显示副本同步进度。 修改目标副本数只影响新上传。修改最低耐久副本数也会重新评估当前上传仍保留的缓存。提高门槛不会让已经进入已存储状态的版本回退,也无法恢复已经删除的缓存。 @@ -66,7 +66,7 @@ synaps3 admin task retry 42 一次确认覆盖整个迁移过程。SynapS3 会创建新的存储服务,新存储提供方就绪后把新上传切换过去,复制已有数据,然后才关闭旧存储提供方。对象从其他副本或本地缓存复制。两者都没有的对象无法复制,旧存储提供方也不会被关闭。仍能从旧存储提供方读取的数据会保持可读,直到每个保留版本都能从新存储提供方读取。 -替换进行期间,副本会显示仍需推进或需要操作的所有迁移,包括其他副本上的并行迁移。发现内容时还不知道最终总数,因此使用不确定进度条;发现完成后,按最终总数中的已处理比例显示确定进度。进度按唯一存储内容计数,因此被多个版本共享的内容只复制一次,并分别统计已迁移内容与复制前已删除、无需再迁移的内容。Tasks 显示同一进度的紧凑形态,Data Sets 卡片显示完整明细。 +替换进行期间,Data Sets 卡片会显示仍需推进或需要操作的所有迁移,包括其他副本上的并行迁移。发现内容时还不知道最终总数,因此使用不确定进度条;发现完成后,按最终总数中的已处理比例显示确定进度。进度按唯一存储内容计数,因此被多个版本共享的内容只复制一次,并分别统计已迁移内容与复制前已删除、无需再迁移的内容。 有些步骤是等待而不是失败。仪表盘会区分创建新服务、等待服务可写、存储提供方不可达、钱包资金和缺少可读内容。多数等待会自行继续。如果某个对象既没有其他副本、也没有本地缓存,替换会等待到来源可用,并且这段等待不计入重试次数。暂时的复制失败会在重启后继续,也不会阻断其他内容或另一条替换。当仪表盘提示内容需要处理,或关闭旧存储提供方前需要结清欠费时,在 Data Sets 列表中使用 **Retry replacement**。目标已被占用时不能重试,请改选存储提供方。 diff --git a/docs/zh/concepts/write-path-cache.md b/docs/zh/concepts/write-path-cache.md index e9cf4bb..93f9c53 100644 --- a/docs/zh/concepts/write-path-cache.md +++ b/docs/zh/concepts/write-path-cache.md @@ -42,7 +42,7 @@ SynapS3 会校验请求,保存对象及其元数据,再返回 S3 兼容的 E | `after_upload` | 存储桶要求的最低耐久副本提交后,为该版本排队清理。 | | `none` | 不创建或执行自动缓存淘汰任务。 | -每个存储桶默认采用严格缓存释放策略,因此最低耐久副本数等于每次上传冻结的目标副本数。在仪表盘中,到该存储桶的 Settings → Replica policy 设置。运维人员可以设置 1 到当前目标之间的显式数量;目标随后提高时该数字保持不变。如果把 Replicas 降到低于已保存的显式数量,请求会被拒绝,必须同时降低 Release cache after。降低门槛只会让仍保留的缓存有资格被删除,其余副本会继续补齐。是否真正删除仍取决于 `after_upload`、`lru` 或 `none`。该门槛不会超过单次上传的目标副本数。提高门槛只影响尚未删除的缓存,无法恢复已经删除的缓存。 +存储桶创建时最低耐久副本数等于其目标副本数,因此只有该次上传的全部副本都已存储后才释放缓存。在仪表盘中,到该存储桶的 Settings → Replica policy 设置。运维人员可以设置 1 到当前目标之间的显式数量;目标随后提高时该数字保持不变。Replicas 本身只能调高:不支持调低,界面上更小的选项会显示但不可选。降低门槛只会让仍保留的缓存有资格被删除,其余副本会继续补齐。是否真正删除仍取决于 `after_upload`、`lru` 或 `none`。该门槛不会超过单次上传的目标副本数。提高门槛只影响尚未删除的缓存,无法恢复已经删除的缓存。 只有当前满足最低耐久副本数且存在可读已提交远端副本的版本才可淘汰。系统会在授权删除前再次检查当前门槛,并等待同一版本正在进行的读取关闭。清理是异步流程,因此清理追赶不及时或没有安全候选时,写入仍可能返回 `507 Insufficient Storage`。 diff --git a/docs/zh/configuration/environment.md b/docs/zh/configuration/environment.md index c53ccff..3e87c92 100644 --- a/docs/zh/configuration/environment.md +++ b/docs/zh/configuration/environment.md @@ -36,18 +36,13 @@ description: 使用 SYNAPS3 环境变量覆盖配置,并理解适用场景。 | `SYNAPS3_CACHE_EVICTION_POLICY` | `cache.eviction_policy` | | `SYNAPS3_CACHE_LRU_HIGH_WATERMARK_PERCENT` | `cache.lru_high_watermark_percent` | | `SYNAPS3_CACHE_LRU_LOW_WATERMARK_PERCENT` | `cache.lru_low_watermark_percent` | -| `SYNAPS3_WORKER_UPLOAD_CONCURRENCY` | `worker.upload.concurrency` | -| `SYNAPS3_WORKER_UPLOAD_POLL_INTERVAL` | `worker.upload.poll_interval` | -| `SYNAPS3_WORKER_UPLOAD_MAX_RETRIES` | `worker.upload.max_retries` | -| `SYNAPS3_WORKER_PROVIDER_REPLACEMENT_CONCURRENCY` | `worker.provider_replacement.concurrency` | -| `SYNAPS3_WORKER_PROVIDER_REPLACEMENT_POLL_INTERVAL` | `worker.provider_replacement.poll_interval` | -| `SYNAPS3_WORKER_PROVIDER_REPLACEMENT_MAX_RETRIES` | `worker.provider_replacement.max_retries` | -| `SYNAPS3_WORKER_EVICTOR_CONCURRENCY` | `worker.evictor.concurrency` | -| `SYNAPS3_WORKER_EVICTOR_POLL_INTERVAL` | `worker.evictor.poll_interval` | -| `SYNAPS3_WORKER_EVICTOR_MAX_RETRIES` | `worker.evictor.max_retries` | -| `SYNAPS3_WORKER_STORAGE_CLEANUP_CONCURRENCY` | `worker.storage_cleanup.concurrency` | -| `SYNAPS3_WORKER_STORAGE_CLEANUP_POLL_INTERVAL` | `worker.storage_cleanup.poll_interval` | -| `SYNAPS3_WORKER_STORAGE_CLEANUP_MAX_RETRIES` | `worker.storage_cleanup.max_retries` | +| `SYNAPS3_WORKER_TASKS_CONCURRENCY` | `worker.tasks.concurrency` | +| `SYNAPS3_WORKER_TASKS_POLL_INTERVAL` | `worker.tasks.poll_interval` | +| `SYNAPS3_WORKER_TASKS_LEASE_DURATION` | `worker.tasks.lease_duration` | +| `SYNAPS3_WORKER_TASKS_MAX_RETRIES` | `worker.tasks.max_retries` | +| `SYNAPS3_WORKER_TASKS_RETENTION` | `worker.tasks.retention` | +| `SYNAPS3_WORKER_TASKS_PROVIDER_MUTATION_CONCURRENCY` | `worker.tasks.provider_mutation_concurrency` | +| `SYNAPS3_WORKER_TASKS_DESTRUCTIVE_MUTATION_CONCURRENCY` | `worker.tasks.destructive_mutation_concurrency` | | `SYNAPS3_LOGGING_LEVEL` | `logging.level` | | `SYNAPS3_LOGGING_FORMAT` | `logging.format` | | `SYNAPS3_LOGGING_S3_ACCESS_ENABLED` | `logging.s3_access.enabled` | diff --git a/docs/zh/configuration/model.md b/docs/zh/configuration/model.md index 3d69817..7b3fe73 100644 --- a/docs/zh/configuration/model.md +++ b/docs/zh/configuration/model.md @@ -71,10 +71,7 @@ SQLite 是 SynapS3 单机部署的默认且推荐数据库。已有 PostgreSQL | `filecoin.observability` | 存储提供方和本地数据集健康检查。 | | `database` | SQLite 或 PostgreSQL 元数据数据库。 | | `cache` | 本地对象缓存目录、容量和淘汰策略。 | -| `worker.upload` | 后台 Filecoin 存储并发、轮询和重试。 | -| `worker.provider_replacement` | 存储提供方替换传输的并发、轮询与复制重试。 | -| `worker.evictor` | 本地缓存淘汰任务。 | -| `worker.storage_cleanup` | 远端副本清理任务。 | +| `worker.tasks` | 统一后台任务执行、恢复、保留时间和存储变更并发限制。 | | `logging` | 运行时日志等级、格式和 S3 access log。 | | `admin` | 仪表盘、Admin API 监听地址和 Admin 认证设置。 | @@ -95,18 +92,20 @@ SQLite 是 SynapS3 单机部署的默认且推荐数据库。已有 PostgreSQL | `cache.eviction_policy` | `lru` | | `cache.lru_high_watermark_percent` | `90` | | `cache.lru_low_watermark_percent` | `80` | -| `worker.upload.concurrency` | `4` | -| `worker.upload.max_retries` | `5` | -| `worker.provider_replacement.concurrency` | `4` | -| `worker.provider_replacement.poll_interval` | `5s` | -| `worker.provider_replacement.max_retries` | `5` | +| `worker.tasks.concurrency` | `12` | +| `worker.tasks.poll_interval` | `5s` | +| `worker.tasks.lease_duration` | `5m` | +| `worker.tasks.max_retries` | `5` | +| `worker.tasks.retention` | `168h` | +| `worker.tasks.provider_mutation_concurrency` | `4` | +| `worker.tasks.destructive_mutation_concurrency` | `2` | | `admin.addr` | `127.0.0.1:9090` | | `admin.trusted_proxies` | `[]` | | `admin.auth.enabled` | `true` | | `admin.auth.username` | `admin` | | `admin.auth.session_ttl` | `12h` | -`worker.provider_replacement` 配置不会影响普通上传。修改 `worker.provider_replacement.max_retries` 后,新发现的替换内容和通过 **Retry replacement** 重试的内容会使用新值;已经进行中的内容保留原有上限。修改存储提供方替换的并发、轮询或重试配置后必须重启 SynapS3。 +`worker.tasks.concurrency` 限制全部后台操作。创建远端存储、Store、Pull 和提交存储承诺共同受 `provider_mutation_concurrency` 限制;远端清理与服务退休共同受 `destructive_mutation_concurrency` 限制。状态和确认查询不占用这些变更并发额度。钱包变更始终串行执行。任务设置修改后必须重启 SynapS3,已经创建的任务保留创建时记录的重试上限。 ## Admin 会话时长 @@ -127,7 +126,7 @@ SQLite 是 SynapS3 单机部署的默认且推荐数据库。已有 PostgreSQL 缓存淘汰策略会产生以下用户可见结果: - `lru`:缓存使用量达到高水位后,SynapS3 按最近访问时间淘汰最久未使用且远端安全的条目,直到降至低水位。 -- `after_upload`:版本达到其存储桶要求的最低耐久副本数后,会在下一次 Evictor 轮询时加入清理。之后从远端读取并回填的缓存不会再次被立即删除。 +- `after_upload`:版本达到其存储桶要求的最低耐久副本数后,会加入异步清理。之后从远端读取并回填的缓存不会再次被立即删除。 - `none`:SynapS3 不会自动清理本地缓存。 LRU 水位始终必须满足 `0 <= low < high <= 100`。在 `after_upload` 或 `none` 下仍会保存这些值,但不会生效。 diff --git a/docs/zh/configuration/runtime-data.md b/docs/zh/configuration/runtime-data.md index c4d5ce3..0df3c16 100644 --- a/docs/zh/configuration/runtime-data.md +++ b/docs/zh/configuration/runtime-data.md @@ -51,7 +51,7 @@ Docker 部署通过 `synaps3-data` volume 挂载该路径。Docker 专用的生 ## 备份前检查 1. 检查 `curl http://127.0.0.1:9090/healthz`,并记录任何非 `ok` 结果。 -2. 运行 `synaps3 admin task stats` 和 `synaps3 admin task list --status exhausted`,检查活动任务和耗尽重试的任务。 +2. 运行 `synaps3 admin task stats` 和 `synaps3 admin task list --status failed`,检查活动任务和失败任务。 3. 使用当前部署方式的服务管理器停止 SynapS3,避免备份过程中对象数据、元数据和任务状态继续变化。 不要在 SynapS3 仍在运行时创建文件系统归档。 @@ -103,6 +103,8 @@ sha256sum -c synaps3-data.tgz.sha256 2. 验证归档校验和,确认数据库与缓存带有相同的恢复时间点标记。 3. 把运行数据卷恢复到空的替换位置。PostgreSQL 部署先恢复数据库原生备份,再连接匹配的配置和缓存数据。 4. 确认恢复后的配置和凭据文件权限为 `0600`,并允许 SynapS3 运行账户读取。 -5. 启动 SynapS3,检查 `/healthz`、任务统计和耗尽重试的任务,再通过 S3 API 读取一个已知对象。 +5. 启动 SynapS3,检查 `/healthz`、任务统计和失败任务,再通过 S3 API 读取一个已知对象。 不要把一个时间点的数据库备份与另一个时间点的缓存数据混用。 + +只使用兼容的 SynapS3 版本恢复备份。如果启动时报告数据库不兼容,请保持备份不变,并按[升级与恢复](../operations/upgrade-recovery.md)处理。 diff --git a/docs/zh/getting-started/s3-clients.md b/docs/zh/getting-started/s3-clients.md index 1c0c545..c4a894d 100644 --- a/docs/zh/getting-started/s3-clients.md +++ b/docs/zh/getting-started/s3-clients.md @@ -134,6 +134,7 @@ alias 会把凭据保存在 `~/.mc/config.json`。无回显提示可以避免 se | --- | --- | | `AccessDenied` | 确认 access key 和 secret key 来自 `synaps3 admin s3-user create`。 | | 客户端使用 virtual-hosted 存储桶访问 | 开启 path-style addressing 或客户端中的等价设置。 | -| 上传成功但 Filecoin 存储仍在等待 | 查看仪表盘任务页,或运行 `synaps3 admin task list --status queued`。 | +| 创建存储桶后的首次上传返回 `SlowDown` | 存储提供方可能仍在准备该存储桶。短暂等待后重试上传,再使用读取命令确认结果。 | +| 上传成功但 Filecoin 存储仍在等待 | 查看仪表盘任务页,或运行 `synaps3 admin task list --status pending`。 | | 对象大小被拒绝 | 确保对象大小在 `127` 到 `1,065,353,216` 字节之间。 | | 远程主机无法访问 Admin 仪表盘 | 保持 Admin 监听本机回环地址,并使用 `ssh -L 9090:127.0.0.1:9090 user@server`。 | diff --git a/docs/zh/operations/health-metrics.md b/docs/zh/operations/health-metrics.md index 771f748..50e171c 100644 --- a/docs/zh/operations/health-metrics.md +++ b/docs/zh/operations/health-metrics.md @@ -31,7 +31,7 @@ curl http://127.0.0.1:9090/healthz 检查失败时: ```json -{"status":"unhealthy","errors":["worker/uploader: not responding"]} +{"status":"unhealthy","errors":["worker/tasks: not responding"]} ``` `setup` 表示需要补齐缺失配置。`unhealthy` 表示数据库、缓存或后台任务检查失败,优先查看返回的错误列表。 @@ -47,7 +47,7 @@ synaps3 admin status synaps3 admin task stats ``` -status 应显示后台任务处理正常。task stats 会显示 `queued`、`running`、`failed` 或 `exhausted` 的任务数量。 +status 应显示后台任务处理正常。task stats 会把已确认失败单独统计为 dismissed,仪表盘会把 pending 工作显示为 queued、scheduled 或 waiting。 ## Prometheus Metrics @@ -74,9 +74,6 @@ scrape_configs: | `synaps3_cache_used_bytes` | 当前缓存磁盘使用量。 | | `synaps3_cache_hits_total` / `synaps3_cache_misses_total` | 缓存读取行为。 | | `synaps3_cache_lru_eviction_paused` | 因近期缓存访问记录无法安全保留而暂停 LRU 淘汰时为 `1`;请先排除持久化错误,再重启 SynapS3。 | -| `synaps3_worker_tasks_processed_total` | 按结果统计的后台任务吞吐。 | -| `synaps3_worker_tasks_exhausted_total` | 已耗尽重试次数的任务。 | -| `synaps3_worker_task_duration_seconds` | 后台任务处理耗时。 | | `synaps3_task_queue_depth` | 按类型和状态统计的活跃任务。 | | `synaps3_object_state_distribution` | 按生命周期状态统计的对象数量。 | @@ -87,7 +84,7 @@ scrape_configs: | `/healthz` 返回 `setup` | 运行 `synaps3 admin status` 或 `synaps3 admin settings get`,按报告补齐必要配置,重启后再次检查。 | | `/healthz` 返回 `unhealthy` | 检查数据库、缓存目录和后台任务错误信息。 | | 缓存使用量接近容量 | 增大容量,或恢复上传和淘汰进度。 | -| exhausted 任务增加 | 修复依赖后重试任务。 | +| failed 任务增加 | 修复依赖后,只重试标记为可重试的任务。 | | 存储提供方健康状态下降 | 检查 RPC、存储提供方 URL 和网络可达性。 | 恢复步骤见[故障排查](./troubleshooting.md)。 diff --git a/docs/zh/operations/production-checklist.md b/docs/zh/operations/production-checklist.md index f143651..9accb38 100644 --- a/docs/zh/operations/production-checklist.md +++ b/docs/zh/operations/production-checklist.md @@ -77,7 +77,7 @@ synaps3 admin settings set filecoin.network=mainnet --yes - `GET /metrics` - 缓存使用量 - 任务队列深度 -- exhausted 任务数量 +- failed 任务数量 - 后台任务活动 - 存储提供方和数据集健康状态 @@ -85,15 +85,17 @@ synaps3 admin settings set filecoin.network=mainnet --yes ## 升级准备 +更改版本前先备份数据库和缓存,再按[升级与恢复](./upgrade-recovery.md)操作。如果启动时报告数据库不兼容,请保持该数据库不变,并改用新的数据库和缓存目录。 + 升级前运行: ```bash curl http://127.0.0.1:9090/healthz synaps3 admin task stats -synaps3 admin task list --status exhausted --limit 50 +synaps3 admin task list --status failed --limit 50 ``` -预期结果:`/healthz` 返回 `ok`,任务队列状态已确认,所有 exhausted 任务都有明确处理方式。 +预期结果:`/healthz` 返回 `ok`,任务队列状态已确认,所有 failed 任务都有明确处理方式,然后再替换正在运行的进程。 ## 恢复入口 diff --git a/docs/zh/operations/troubleshooting.md b/docs/zh/operations/troubleshooting.md index 945c893..1635512 100644 --- a/docs/zh/operations/troubleshooting.md +++ b/docs/zh/operations/troubleshooting.md @@ -55,7 +55,7 @@ synaps3 wallet generate 示例: ```json -{"status":"unhealthy","errors":["worker/uploader: not responding"]} +{"status":"unhealthy","errors":["worker/tasks: not responding"]} ``` 检查任务状态: @@ -109,21 +109,21 @@ synaps3 admin settings get cache.lru_low_watermark_percent - 先确认主机仍有可用磁盘空间,再按容量增大 `cache.max_size_gb`。 - 恢复存储提供方连接和后台任务进度,让排队上传完成并触发缓存淘汰。 - 默认的 `lru` 适合按容量自动清理。降低高水位可以为新写入保留更多余量,并始终满足 `0 <= low < high <= 100`。 -- 只有希望版本达到存储桶要求的最低耐久副本数后,在下一次 Evictor 轮询中删除对应版本时,才使用 `after_upload`。 +- 只有希望版本达到存储桶要求的最低耐久副本数后异步删除对应版本时,才使用 `after_upload`。 - 需要完全禁用自动清理时使用 `none`。 -LRU 无法清理 multipart 暂存数据、未达到存储桶最低耐久副本数的版本,或没有可读已提交远端副本的版本。写入不会同步触发淘汰,因此在 Evictor 追赶完成或出现安全候选前,仍可能继续返回 `507 Insufficient Storage`。 +LRU 无法清理 multipart 暂存数据、未达到存储桶最低耐久副本数的版本,或没有可读已提交远端副本的版本。写入不会同步触发清理,因此在后台清理追赶完成或出现安全候选前,仍可能继续返回 `507 Insufficient Storage`。 -LRU 删除失败后,任务仍会作为 exhausted 工作保留,并在一小时冷却后重新具备执行资格。先修复任务中报告的文件系统或数据库问题;需要提前重试时,运行 `synaps3 admin task retry `。 +LRU 删除失败后,任务仍会作为 failed 工作保留。先修复任务中报告的文件系统或数据库问题;任务标记为可重试时,可运行 `synaps3 admin task retry `。 修改缓存设置后,重启 SynapS3,检查 `/healthz`,再运行 `synaps3 admin settings get` 验证实际生效的缓存设置。 -## Exhausted 任务 +## Failed 任务 -列出 exhausted 任务: +列出 failed 任务: ```bash -synaps3 admin task list --status exhausted --limit 100 +synaps3 admin task list --status failed --limit 100 ``` 确认 RPC 连接、存储提供方可用性、钱包余额、FWSS approval 和缓存磁盘容量都已就绪后,再重试。 @@ -132,7 +132,7 @@ synaps3 admin task list --status exhausted --limit 100 synaps3 admin task retry 42 ``` -存储提供方替换工作是例外:不要从 Tasks 重试。复制重试和等待可读内容会自动继续,重启后也一样。已完成或已停止的替换任务会提供 **Open Data Sets**,直接打开受影响存储桶的 **Details** → **Storage** → **Data Sets**。仅在页面显示 **Retry replacement** 时使用它;需要处理的内容会按当前 `worker.provider_replacement.max_retries` 设置重试,已经完成的内容会保留。如果所选存储提供方已经存储该桶,请改选其他存储提供方。 +API 会判断每个失败任务能否安全重试。存储提供方替换从 **Details** → **Storage** → **Data Sets** 恢复。只有尚未发出广播的钱包操作可以从 Tasks 恢复;广播结果不确定时仍不可重试。Store 结果不确定时会提供 **Check again**,它只观察存储提供方,不会再次上传。只有在核对失败结果后才使用 **Dismiss** 或 `synaps3 admin task acknowledge `;确认后的任务会继续保留配置的时长,再由后台清理。 ## 存储提供方或 RPC 问题 @@ -159,4 +159,4 @@ curl -u admin http://127.0.0.1:9090/api/v1/observability/providers 2. Access key 和 secret key 来自 `synaps3 admin s3-user create`。 3. 本地评估使用 `http://localhost:8080`,生产环境使用正确的 HTTPS S3 地址。 4. 对象大小在 `127` 到 `1,065,353,216` 字节之间,并且对象键符合 [S3 兼容性限制](../reference/s3-compatibility.md#稳定限制)。 -5. 仪表盘任务页显示 Filecoin 存储处于 `queued`、`running` 还是 `exhausted`。 +5. 仪表盘任务页显示 Filecoin 存储处于 `queued`、`running`、`waiting` 还是 `failed`。 diff --git a/docs/zh/operations/upgrade-recovery.md b/docs/zh/operations/upgrade-recovery.md index b0077ea..5a8154a 100644 --- a/docs/zh/operations/upgrade-recovery.md +++ b/docs/zh/operations/upgrade-recovery.md @@ -1,11 +1,11 @@ --- title: 升级与恢复 -description: 安全升级 SynapS3,并从常见单机故障场景恢复。 +description: 安全升级 SynapS3 并恢复后台工作。 --- # 升级与恢复 -SynapS3 是单机网关。升级或恢复时,先把本地持久化对象和元数据作为一个一致的数据集保护起来,再恢复后台任务。依赖失败时,先恢复依赖,再重试任务。 +更改版本前,把数据库和缓存作为同一恢复点保护。重试后台工作前,先恢复失效的依赖。 ## 升级前 @@ -14,27 +14,20 @@ SynapS3 是单机网关。升级或恢复时,先把本地持久化对象和元 ```bash curl http://127.0.0.1:9090/healthz synaps3 admin task stats -synaps3 admin task list --status exhausted --limit 50 +synaps3 admin task list --status failed --limit 50 ``` -预期结果:`/healthz` 返回 `ok`,并且所有 exhausted 任务在升级前都有明确处理方式。 +预期结果:健康状态为 `ok`,替换进程前每个 failed 任务都有明确处理方式。 -升级前,先停止新的 S3 写入,但保持当前 SynapS3 进程运行,直到上传和存储提供方替换都已完成。如果新版本因仍有存储工作进行中而拒绝启动,请让旧版本继续使用未改动的数据库;必要时恢复对应存储提供方,等待工作完成后再重试升级。不要通过修改数据库绕过检查。 - -如果某个副本仍保留着结果从未确认的存储确认记录(可能由此前中断的旧版本遗留),升级同样会被拒绝。提示信息会给出受影响的副本数量,并附带列出这些副本的查询语句,便于逐条向存储提供方确认该数据是否已经存储,再清除该记录。开发环境也可以直接使用全新数据库重新开始。 - -创建备份前,停止 S3 流量,并使用当前部署方式的服务管理器停止 SynapS3。 - -- SQLite 部署:归档完整运行数据卷,并验证归档和校验和。 -- PostgreSQL 部署:创建数据库原生备份,并归档同一时间点的配置和缓存数据。 - -所有备份产物必须处于同一恢复时间点。准确的备份、校验和重启步骤见[运行数据](../configuration/runtime-data.md)。 +创建备份前,停止新的 S3 流量和 SynapS3。数据库、缓存、配置和凭据必须位于同一恢复点。备份和验证步骤见[运行数据](../configuration/runtime-data.md)。 ## 升级 SynapS3 -使用当前部署对应的安装方式替换可执行文件、软件包或容器镜像。Docker 专用命令见 [Docker 部署](../getting-started/docker.md)。 +使用部署环境原有的安装方式替换可执行文件、软件包或容器镜像。Docker 命令见 [Docker 部署](../getting-started/docker.md)。 + +使用预期的数据库和缓存启动 SynapS3。如果启动时报告数据库不兼容,请停止进程、保持数据库不变,然后按[数据库不兼容时](#数据库不兼容时)处理。 -使用当前部署方式的服务管理器启动 SynapS3,然后运行: +启动后运行: ```bash curl http://127.0.0.1:9090/healthz @@ -42,53 +35,73 @@ synaps3 admin settings get synaps3 admin task stats ``` -预期结果:`/healthz` 返回 `ok`,生效设置与部署一致,任务队列恢复推进,并且没有意外出现耗尽重试的任务。恢复正常流量前,通过 S3 API 读取一个已知对象。 +预期结果:健康状态为 `ok`,生效设置与部署一致,后台工作继续且没有意外失败。恢复正常流量前,通过 S3 API 读取一个已知对象。 + +## 数据库不兼容时 -## 运行流程 +1. 停止新的 S3 流量,并停止使用该部署的所有 SynapS3 进程。 +2. 备份报告的数据库,并验证该备份。 +3. 将数据库和匹配的缓存保留为只读数据。不要修改这两个位置来绕过兼容性检查。 +4. 为替换安装配置空数据库和空缓存目录。 +5. 启动 SynapS3;验证健康状态、生效设置和后台任务处理后,再恢复流量。 -```text -接收写入 -> 保存对象 -> 记录元数据 -> 返回成功 -> 继续后台存储 +SQLite 在停止进程后创建一致性备份,并验证能够打开: + +```bash +sqlite3 /old/path/synaps3.db ".backup '/backup/path/synaps3-pre-upgrade.db'" +sqlite3 -readonly /backup/path/synaps3-pre-upgrade.db "PRAGMA integrity_check;" ``` -- 写入会先提交到本地缓存和元数据,再上传到存储提供方。 -- 存储任务失败后会重试,达到配置的重试上限后进入 `exhausted`。 -- `GetObject` 优先从缓存读取;已有远端元数据时,可以从存储提供方取回对象。 -- 删除存储桶不受支持并返回 `501`;删除对象会让对象从 S3 视图中消失,后续清理会安全继续。 +完整性检查必须输出 `ok`。把备份、需要保留的 WAL/SHM 文件、匹配的缓存和配置作为同一恢复集保护。PostgreSQL 部署应使用 `pg_dump` 或部署批准的数据库快照,并单独验证该备份产物。 -## 恢复矩阵 +SynapS3 不会修改不兼容的数据库。废弃的 `worker.upload`、`worker.provider_replacement`、`worker.evictor` 和 `worker.storage_cleanup` 配置段也会被拒绝;请替换为 `worker.tasks` 设置。 -| 场景 | 恢复方式 | -| --- | --- | -| 已建立的存储提供方在首次存储期间暂时不可用 | 恢复原存储提供方。其他已分配且可写的副本会继续处理;未完成副本会等待且不消耗重试次数,并在恢复后自动继续。SynapS3 不会选择替代存储提供方。 | -| 后台存储任务无法连接存储提供方 | 恢复连接,然后重试 exhausted 存储任务。 | -| RPC 节点故障 | 恢复 RPC 连接,然后重试 exhausted 任务。 | -| 私有存储提供方 URL 被阻止 | 默认保持阻止;只在可信私有部署中开启 `filecoin.allow_private_networks`。 | -| 数据库空间不足 | 释放空间或扩容数据库。 | -| 缓存磁盘空间不足 | 扩容磁盘、提高 `cache.max_size_gb`,或恢复上传和淘汰进度。 | -| 存储提供方永久不可用,或需要计划性迁离 | 打开该存储桶,选择 **Details**,然后到 **Storage** → **Data Sets** 替换存储提供方。新存储提供方就绪后,新上传会切过去。已有对象从其他副本或本地缓存复制;两者都没有的对象无法复制,旧存储提供方也不会被关闭。所选目标已被占用时,请改选存储提供方,而不是重试。 | -| 进程崩溃 | 重启服务,再检查健康状态和任务统计。大多数未完成的存储工作会自动恢复,不会重复提交同一个 piece。无法安全判定结果的工作会出现在 `synaps3 admin storage-confirmation list` 中;采取操作前先核对列出的存储提供方和 transaction 信息。对于已记录的服务关闭交易,SynapS3 会先检查交易状态,再决定是否再次提交。 | +使用空数据库启动时,不会导入原有的存储桶、对象、用户、存储数据集、钱包操作、存储提供方替换或任务。已创建的远端付费存储服务仍会运行。请保留经过验证的备份,以便单独核对和处理这些服务与记录。 -副本完成存储后,如果存储提供方变为不可用,不一定会产生可重试任务。使用存储健康视图识别受影响副本;恢复目标副本数属于[计划支持的副本修复](../concepts/filecoin-storage-flow.md#计划支持的副本修复)。 +不要让保留的安装和替换安装共用同一数据库、缓存、钱包工作流或 S3 流量。启动替换安装后,创建 S3 用户和测试存储桶,再写入并读取测试对象,然后恢复正常流量。 -## 恢复或回退 +## 恢复后台工作 -1. 停止 S3 流量和 SynapS3。 -2. 验证归档校验和,并选择同一恢复时间点的数据库和缓存产物。 -3. SQLite 恢复完整运行数据卷;PostgreSQL 先恢复数据库原生备份,再恢复匹配的配置和缓存数据。 -4. 回退应用版本时,只让旧版本使用与其兼容的数据。如果无法确认兼容性,恢复升级前的完整恢复时间点。 -5. 启动 SynapS3,再检查 `/healthz`、生效设置、任务统计、耗尽重试的任务、钱包就绪状态,并通过 S3 读取一个已知对象。 +重启后,未完成的工作会自动恢复处理。 -这些检查通过前,不要恢复正常流量。 +- 只重试仪表盘或 API 标记为可重试的失败任务。 +- 从 **Details** → **Storage** → **Data Sets** 恢复存储提供方替换。 +- 钱包操作只有在尚未发出广播时才能从 Tasks 重试;广播结果不确定时仍不可重试。 +- Store 结果不确定时,**Check again** 只查询存储提供方,不会重新上传对象。 +- `status=failed` 只列出尚未确认的失败;使用 `status=dismissed` 查看已确认的失败。 +- 使用 `synaps3 admin storage-confirmation list` 核对尚未解决的存储确认。 常用命令: ```bash -synaps3 admin task list --status exhausted --limit 100 +synaps3 admin task list --status failed --limit 100 +synaps3 admin task list --status dismissed --limit 100 synaps3 admin task stats synaps3 admin task retry 42 +synaps3 admin task acknowledge 42 synaps3 admin storage-confirmation list -synaps3 admin s3-user list synaps3 admin settings get ``` -修改恢复相关设置后,重启 SynapS3,并同时检查 `/healthz` 和 `synaps3 admin settings get`。 +重试前先恢复失效的依赖。使用仪表盘、Admin API 或 CLI 操作,不要直接修改应用数据库。 + +## 恢复矩阵 + +| 场景 | 恢复方式 | +| --- | --- | +| 存储提供方或 RPC 暂时不可用 | 恢复连接。等待中的工作会自动继续;只重试标记为可重试的失败任务。 | +| 数据库空间不足 | 停止流量,释放空间或扩容数据库,再检查健康状态。 | +| 缓存磁盘空间不足 | 扩容磁盘、提高 `cache.max_size_gb`,或恢复远端存储与缓存清理进度。 | +| 需要迁离存储提供方 | 打开存储桶并使用 **Details** → **Storage** → **Data Sets**。不要从 Tasks 重试替换。 | +| 进程崩溃 | 重启 SynapS3,验证健康状态和任务统计,再核对任何未解决的存储确认或钱包结果。 | +| 启动时报告数据库不兼容 | 停止进程,确认配置的数据库是预期目标,将其原样保留,然后改用空数据库。 | + +## 恢复或回滚 + +1. 停止 S3 流量和 SynapS3。 +2. 验证备份校验和,选择同一恢复点的数据库和缓存产物。 +3. SQLite 恢复完整运行数据卷。PostgreSQL 先恢复数据库原生备份,再恢复匹配的配置和缓存数据。 +4. 回滚应用时,只使用与所选版本兼容的数据。无法确认兼容性时,恢复升级前的恢复点。 +5. 启动 SynapS3,然后检查 `/healthz`、生效设置、任务统计、failed 任务、钱包准备状态和已知 S3 对象。 + +这些检查全部通过前,不要恢复正常流量。 diff --git a/docs/zh/reference/admin-api.md b/docs/zh/reference/admin-api.md index 5959429..f06e273 100644 --- a/docs/zh/reference/admin-api.md +++ b/docs/zh/reference/admin-api.md @@ -35,7 +35,6 @@ Setup 模式不提供运行时指标、存储桶、对象、后台任务、钱 | `/api/v1/auth/refresh`、`/api/v1/auth/logout` | 需要有效浏览器会话和 CSRF header;不接受 HTTP Basic auth。 | | `/api/v1/*` | 浏览器 session cookie;写请求方法需要 CSRF。也可用 HTTP Basic auth。 | | `/metrics` | 浏览器 session cookie 或 HTTP Basic auth。 | -| `/admin/exhausted-tasks*` | 浏览器 session cookie;写请求方法需要 CSRF。也可用 HTTP Basic auth。 | ### 浏览器会话 @@ -92,7 +91,7 @@ Admin 响应包含 `Content-Security-Policy`、`X-Content-Type-Options: nosniff` | 钱包 | `POST /api/v1/wallet/fund`、`POST /api/v1/wallet/withdraw`、`POST /api/v1/wallet/approve` | 创建链上支付操作。 | | S3 用户 | `POST /api/v1/s3-users`、`PUT /api/v1/s3-users/{accessKey}`、`POST /api/v1/s3-users/{accessKey}/secret`、`DELETE /api/v1/s3-users/{accessKey}` | 改变客户端访问权限,或让已有凭据失效。 | | 存储桶和对象 | 创建存储桶、更新 owner/copy-policy,以及上传、下载、删除、恢复或永久删除对象 | 改变或暴露用户可见的 S3 数据和元数据。 | -| 后台任务和存储健康 | 任务重试、诊断刷新、存储提供方和数据集刷新 | 重新入队任务,或刷新运维状态。 | +| 后台任务和存储健康 | 任务重试与确认、存储提供方和数据集刷新 | 重新入队任务、将已核对的失败标记为已处理并开始保留期,或刷新运维状态。 | | 存储提供方替换 | `POST /api/v1/buckets/{name}/data-sets/{id}/replacement`、`POST /api/v1/storage-replacements/{id}/retry` | 创建新的付费存储服务,把副本迁移过去,并终止旧服务。 | | 存储确认 | `POST /api/v1/storage-confirmations/{copy-id}/release` | 可能允许存储提供方再次存储同一个 piece。释放前必须核对当前 attempt。 | @@ -142,12 +141,14 @@ Admin 响应包含 `Content-Security-Policy`、`X-Content-Type-Options: nosniff` ### 存储桶副本策略 -`POST /api/v1/buckets` 接受可选的 `default_copies` 和 `minimum_durable_copies` 字段。存储桶列表、详情、创建和策略更新响应包含: +`POST /api/v1/buckets` 接受可选的 `default_copies` 和 `minimum_durable_copies` 字段,缺省时取服务端配置值。存储桶会保存创建时的策略,之后修改配置不影响已有存储桶。存储桶列表、详情、创建和策略更新响应中,这两个字段都是整数: -- `minimum_durable_copies`:存储桶显式设置的值;`null` 表示按每次上传采用严格策略; -- `effective_minimum_durable_copies`:将当前存储桶门槛限制在当前目标副本数以内后,用于展示的值。 +- `default_copies`:该存储桶的目标副本数; +- `minimum_durable_copies`:释放缓存前必须完成的副本数,不会超过目标副本数。 -`PUT /api/v1/buckets/{name}/copy-policy` 可以独立接收 `default_copies` 和 `minimum_durable_copies`。字段缺省时保持不变。`default_copies: null` 表示新上传继承当前运行时目标副本数。`minimum_durable_copies: null` 表示必须完成单次上传冻结的所有副本后才能释放缓存。显式门槛必须在 `1` 到 `8` 之间,且不能超过同一请求产生的最终目标副本数。空请求或无效的最终组合返回 `400 Bad Request`。 +`PUT /api/v1/buckets/{name}/copy-policy` 可以独立接收 `default_copies` 和 `minimum_durable_copies`。字段缺省时保持不变。传 `null` 表示重置该字段:`default_copies: null` 恢复为配置的默认值,`minimum_durable_copies: null` 将门槛设为与目标副本数相同。显式值必须在 `1` 到 `8` 之间,且门槛不能超过同一请求产生的最终目标副本数。空请求或无效的最终组合返回 `400 Bad Request`。 + +**不支持调低 `default_copies`,调低会返回 `400 Bad Request`。** 超出新目标的那些副本仍会继续运行、继续计费,而且没有任何机制会退役它们,所以目标副本数只能调高。如果 `null` 重置后的值低于存储桶当前的目标副本数,同样会被拒绝。 目标副本数变更只影响新上传。最低耐久副本数变更还会重新评估当前上传仍保留的缓存。提高门槛无法恢复已经删除的缓存。 @@ -279,18 +280,18 @@ Admin 响应包含 `Content-Security-Policy`、`X-Content-Type-Options: nosniff` | Method | Path | 用途 | | --- | --- | --- | -| `GET` | `/api/v1/tasks` | 列出后台任务。支持 `type`、`stage`、`status`、`limit`、`offset` 等过滤。 | +| `GET` | `/api/v1/tasks` | 列出后台任务。支持 `type`、`status`、`limit` 和基于 ID 的 `cursor`。 | | `GET` | `/api/v1/tasks/stats` | 按状态统计任务。 | -| `GET` | `/api/v1/tasks/{id}/ref-detail` | 解析后台任务关联的对象或存储操作。 | -| `GET` | `/api/v1/tasks/{id}/diagnostic` | 读取任务诊断。 | -| `POST` | `/api/v1/tasks/{id}/diagnostic/refresh` | 刷新诊断。 | -| `POST` | `/api/v1/tasks/{id}/retry` | 重试 exhausted 任务。 | -| `GET` | `/admin/exhausted-tasks` | 列出 exhausted 任务。支持最大为 `1000` 的 `limit`。 | -| `POST` | `/admin/exhausted-tasks/{id}/retry` | 重试 exhausted 任务(遗留路径)。 | +| `POST` | `/api/v1/tasks/{id}/retry` | 当 `retryable` 为 true 时恢复失败任务。 | +| `POST` | `/api/v1/tasks/{id}/acknowledge` | 当 `acknowledgeable` 为 true 时把失败任务标记为已处理。确认后开始计算保留期,到期后可能被清理。 | + +`status` 为 `pending`、`running`、`completed`、`failed` 或 `cancelled`。`presentation_status` 会把 pending 工作显示为 `queued`、`scheduled` 或 `waiting`,并把已确认的失败任务显示为 `dismissed`。响应还包含 `operation`、可选的 subject 身份,以及服务端计算的 `retryable` 和 `acknowledgeable`。 + +`status` 过滤还接受 `dismissed`。`status=failed` 只返回尚未确认的失败,`status=dismissed` 返回已确认的失败;`/api/v1/tasks/stats` 也分别以 `failed` 和 `dismissed` 统计两组任务。 -引用存储桶的替换和退休任务会在列表与引用详情响应中包含 `bucket_name`。从任务队列重试存储提供方替换工作会返回 `409 Conflict` 和 `"code": "replacement_task_retry_unsupported"`。替换任务完成或停止后,可以使用 **Open Data Sets**,或打开存储桶并前往 Details → Storage → Data Sets。`target_in_use` 失败不会显示 Retry,因为它需要改选存储提供方。 +`/api/v1/overview` 的 `tasks.by_status` 按 `status` 聚合,因此其中的 `failed` 会包含已确认的失败。需要尚未确认的失败数时使用 `tasks.attention.failed`;需要分别统计 `failed` 和 `dismissed` 时使用 `/api/v1/tasks/stats`。 -任务列表中的 `progress` 是按 `scope` 区分的联合对象。`scope: "ingress_store"` 返回 `attempt`、`uploaded_bytes`、`total_bytes`、可选 `percent`、`done` 与 `updated_at`。`scope: "provider_replacement"` 返回与存储桶响应相同的替换进度。客户端必须先按 `scope` 分支。 +分页按任务 ID 从新到旧。响应存在 `next_cursor` 时,把它作为下一次请求的 `cursor`。存储提供方替换仍通过 Data Sets API 恢复。钱包操作只有在广播开始前才可重试。重试结果不确定的 Store 只会查询存储提供方,不会重新上传字节。 ## 钱包和 Filecoin diff --git a/docs/zh/reference/cli-api.md b/docs/zh/reference/cli-api.md index ff14921..c4a1126 100644 --- a/docs/zh/reference/cli-api.md +++ b/docs/zh/reference/cli-api.md @@ -81,8 +81,9 @@ synaps3 admin settings get synaps3 admin settings set cache.max_size_gb=200 synaps3 admin settings set cache.eviction_policy=lru cache.lru_high_watermark_percent=90 cache.lru_low_watermark_percent=80 synaps3 admin task stats -synaps3 admin task list --status exhausted --limit 100 +synaps3 admin task list --status failed --limit 100 synaps3 admin task retry 42 +synaps3 admin task acknowledge 42 synaps3 admin storage-confirmation list synaps3 admin storage-confirmation release 42 --attempt-id current-attempt-id --yes ``` @@ -97,9 +98,9 @@ Admin 全局 flags 必须放在 `admin` 之后、子命令之前: | `--json` | 以 JSON 返回成功响应。 | | `--timeout ` | 设置 Admin API 请求超时。 | -列出后台任务时支持 `--type`、`--stage`、`--status`、`--limit` 和 `--offset`。`--stage` 必须与 `--type` 一起使用。 +列出后台任务时支持 `--type`、`--status`、`--limit` 和基于任务 ID 的 `--cursor`。有效的状态过滤值为 `pending`、`running`、`completed`、`failed`、`cancelled` 和 `dismissed`。pending 工作会显示为 queued、scheduled 或 waiting;`failed` 返回尚未确认的失败,`dismissed` 返回已确认的失败。 -`synaps3 admin task retry` 不会重试存储提供方替换工作。请在已完成或已停止的替换任务上使用 **Open Data Sets**,或打开存储桶并前往 **Details** → **Storage** → **Data Sets**。如果所选存储提供方已经存储该桶,请改选其他存储提供方,而不是重试。 +`synaps3 admin task retry` 只恢复响应中标记为可重试的失败任务。存储提供方替换仍在 **Details** → **Storage** → **Data Sets** 中恢复。只有尚未发出广播的钱包操作可以重试;广播结果不确定时仍不可重试。Store 结果不确定时,dashboard 会把 Retry 显示为 **Check again**:该操作只查询存储提供方,不会重新上传。核对失败结果后,可用 `synaps3 admin task acknowledge ` 将任务标记为已处理;确认后开始计算保留期,到期后可能被清理。 `synaps3 admin storage-confirmation list` 会显示需要核对的存储确认。核对存储提供方、transaction 和当前 attempt 后,使用 `storage-confirmation release --attempt-id --yes` 表示确认存储提供方可能已经接受该 piece,并允许正常恢复流程再次提交。过期的 attempt ID 会被拒绝。 diff --git a/internal/admin/api_auth_test.go b/internal/admin/api_auth_test.go index ded62e1..8f0204a 100644 --- a/internal/admin/api_auth_test.go +++ b/internal/admin/api_auth_test.go @@ -678,16 +678,13 @@ func TestAdminAuthBasicAuthRejectsCrossSiteBrowserUnsafeRequests(t *testing.T) { } } -func TestAdminAuthFailureLimitLogoutExpiryAndLegacyProtection(t *testing.T) { +func TestAdminAuthFailureLimitLogoutAndExpiry(t *testing.T) { srv := newTestAuthServer(t, "admin-password") mux := http.NewServeMux() srv.registerAuthRoutes(mux) mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusOK, map[string]string{"status": "metrics"}) }) - mux.HandleFunc("GET /admin/exhausted-tasks", func(w http.ResponseWriter, _ *http.Request) { - writeJSON(w, http.StatusOK, map[string]string{"status": "legacy"}) - }) handler := srv.withAdminAuth(mux) for i := 0; i < 5; i++ { @@ -707,7 +704,7 @@ func TestAdminAuthFailureLimitLogoutExpiryAndLegacyProtection(t *testing.T) { t.Fatalf("limited login status = %d, want 429", limitedRR.Code) } - for _, path := range []string{"/metrics", "/admin/exhausted-tasks"} { + for _, path := range []string{"/metrics"} { rr := httptest.NewRecorder() handler.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, path, nil)) if rr.Code != http.StatusUnauthorized { diff --git a/internal/admin/api_bucket_storage_health.go b/internal/admin/api_bucket_storage_health.go index 68c027a..6a02f0d 100644 --- a/internal/admin/api_bucket_storage_health.go +++ b/internal/admin/api_bucket_storage_health.go @@ -121,12 +121,12 @@ func (s *Server) handleAPIBucketStorageHealthAffectedVersions(w http.ResponseWri writeJSON(w, http.StatusNotFound, map[string]string{"error": "bucket not found"}) return } - if s.repos.Uploads == nil { + if s.repos.Contents == nil { writeJSON(w, http.StatusOK, bucketStorageHealthAffectedVersionsResponse{Versions: []bucketStorageHealthAffectedVersionResponse{}}) return } - page, err := s.repos.Uploads.ListBucketStorageHealthAffectedVersions(ctx, repository.BucketStorageHealthAffectedVersionsInput{ + page, err := s.repos.Contents.ListBucketStorageHealthAffectedVersions(ctx, repository.BucketStorageHealthAffectedVersionsInput{ BucketID: bucket.ID, LocalDataSetID: localDataSetID, Prefix: prefix, @@ -146,11 +146,11 @@ func (s *Server) handleAPIBucketStorageHealthAffectedVersions(w http.ResponseWri } func (s *Server) bucketStorageHealthSummaries(ctx context.Context, bucketID int64) (map[int64]bucketStorageHealthSummaryResponse, bool) { - if s.repos.Uploads == nil { + if s.repos.Contents == nil { return nil, false } staleBefore := s.bucketStorageHealthStaleBefore() - summaries, err := s.repos.Uploads.ListBucketStorageHealthSummaries(ctx, bucketID, staleBefore, bucketStorageHealthAffectedVersionsCap) + summaries, err := s.repos.Contents.ListBucketStorageHealthSummaries(ctx, bucketID, staleBefore, bucketStorageHealthAffectedVersionsCap) if err != nil { if s.logger != nil { s.logger.Warn("api: failed to load bucket storage health facts", "error", err, "bucketID", bucketID) diff --git a/internal/admin/api_buckets.go b/internal/admin/api_buckets.go index 6844833..6f48668 100644 --- a/internal/admin/api_buckets.go +++ b/internal/admin/api_buckets.go @@ -17,12 +17,16 @@ import ( "time" s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/strahe/synaps3/internal/bucketlifecycle" + "github.com/strahe/synaps3/internal/cacheeviction" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/objectdeletion" "github.com/strahe/synaps3/internal/objectkey" "github.com/strahe/synaps3/internal/objectreader" "github.com/strahe/synaps3/internal/observability" + "github.com/strahe/synaps3/internal/storagecleanup" + taskengine "github.com/strahe/synaps3/internal/task" idtypes "github.com/strahe/synaps3/internal/types" "github.com/versity/versitygw/auth" "golang.org/x/sync/errgroup" @@ -42,18 +46,16 @@ const ( ) type bucketListItem struct { - ID int64 `json:"id"` - Name string `json:"name"` - OwnerAccessKey *string `json:"owner_access_key"` - DefaultCopies *int `json:"default_copies"` - EffectiveCopies int `json:"effective_copies"` - MinimumDurableCopies *int `json:"minimum_durable_copies"` - EffectiveMinimumDurableCopies int `json:"effective_minimum_durable_copies"` - Status string `json:"status"` - ObjectCount int64 `json:"object_count"` - TotalSizeBytes int64 `json:"total_size_bytes"` - StorageHealth bucketStorageHealthSummaryResponse `json:"storage_health"` - CreatedAt string `json:"created_at"` + ID int64 `json:"id"` + Name string `json:"name"` + OwnerAccessKey *string `json:"owner_access_key"` + DefaultCopies int `json:"default_copies"` + MinimumDurableCopies int `json:"minimum_durable_copies"` + Status string `json:"status"` + ObjectCount int64 `json:"object_count"` + TotalSizeBytes int64 `json:"total_size_bytes"` + StorageHealth bucketStorageHealthSummaryResponse `json:"storage_health"` + CreatedAt string `json:"created_at"` } type bucketCreateRequest struct { @@ -64,33 +66,29 @@ type bucketCreateRequest struct { } type bucketMutationResponse struct { - ID int64 `json:"id"` - Name string `json:"name"` - OwnerAccessKey *string `json:"owner_access_key"` - DefaultCopies *int `json:"default_copies"` - EffectiveCopies int `json:"effective_copies"` - MinimumDurableCopies *int `json:"minimum_durable_copies"` - EffectiveMinimumDurableCopies int `json:"effective_minimum_durable_copies"` - Status string `json:"status"` + ID int64 `json:"id"` + Name string `json:"name"` + OwnerAccessKey *string `json:"owner_access_key"` + DefaultCopies int `json:"default_copies"` + MinimumDurableCopies int `json:"minimum_durable_copies"` + Status string `json:"status"` } type bucketDetailResponse struct { - ID int64 `json:"id"` - Name string `json:"name"` - OwnerAccessKey *string `json:"owner_access_key"` - DefaultCopies *int `json:"default_copies"` - EffectiveCopies int `json:"effective_copies"` - MinimumDurableCopies *int `json:"minimum_durable_copies"` - EffectiveMinimumDurableCopies int `json:"effective_minimum_durable_copies"` - Status string `json:"status"` - ObjectCount int64 `json:"object_count"` - TotalSizeBytes int64 `json:"total_size_bytes"` - StorageHealth bucketStorageHealthSummaryResponse `json:"storage_health"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` - VersioningStatus string `json:"versioning_status"` - VersioningEnforced bool `json:"versioning_enforced"` - DataSets []storageDataSetSummaryResponse `json:"data_sets"` + ID int64 `json:"id"` + Name string `json:"name"` + OwnerAccessKey *string `json:"owner_access_key"` + DefaultCopies int `json:"default_copies"` + MinimumDurableCopies int `json:"minimum_durable_copies"` + Status string `json:"status"` + ObjectCount int64 `json:"object_count"` + TotalSizeBytes int64 `json:"total_size_bytes"` + StorageHealth bucketStorageHealthSummaryResponse `json:"storage_health"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + VersioningStatus string `json:"versioning_status"` + VersioningEnforced bool `json:"versioning_enforced"` + DataSets []storageDataSetSummaryResponse `json:"data_sets"` // Replacements is the bucket's full history, newest first. Replacements []providerReplacementResponse `json:"replacements"` } @@ -100,7 +98,7 @@ type storageDataSetSummaryResponse struct { BucketID int64 `json:"bucket_id"` BucketName string `json:"bucket_name,omitempty"` CopyIndex int `json:"copy_index"` - Generation int `json:"generation"` + Generation int64 `json:"generation"` IsCurrent bool `json:"is_current"` Replaceable bool `json:"replaceable"` ProviderID string `json:"provider_id"` @@ -108,8 +106,8 @@ type storageDataSetSummaryResponse struct { DataSetID *string `json:"data_set_id,omitempty"` ClientDataSetID *string `json:"client_data_set_id,omitempty"` Status string `json:"status"` - CreatedByUploadID *int64 `json:"created_by_upload_id,omitempty"` - LastUsedUploadID *int64 `json:"last_used_upload_id,omitempty"` + CreatedByContentID *int64 `json:"created_by_content_id,omitempty"` + LastUsedContentID *int64 `json:"last_used_content_id,omitempty"` CommittedCopies int64 `json:"committed_copies"` ReadableCopies int64 `json:"readable_copies"` PhysicalBytes int64 `json:"physical_bytes"` @@ -139,21 +137,6 @@ type bucketCopyPolicyUpdateRequest struct { MinimumDurableCopies json.RawMessage `json:"minimum_durable_copies"` } -func (s *Server) effectiveBucketCopies(bucket *model.Bucket) int { - if bucket != nil && bucket.DefaultCopies != nil { - return boundedBucketCopies(*bucket.DefaultCopies) - } - return boundedBucketCopies(s.filecoinDefaultCopies) -} - -func (s *Server) effectiveBucketMinimumDurableCopies(bucket *model.Bucket) int { - target := s.effectiveBucketCopies(bucket) - if bucket == nil || bucket.MinimumDurableCopies == nil || *bucket.MinimumDurableCopies > target { - return target - } - return *bucket.MinimumDurableCopies -} - func boundedBucketCopies(copies int) int { return model.ClampStorageCopies(copies) } @@ -220,18 +203,16 @@ func (s *Server) handleAPIListBuckets(w http.ResponseWriter, r *http.Request) { } stats := statsMap[b.ID] items = append(items, bucketListItem{ - ID: b.ID, - Name: b.Name, - OwnerAccessKey: s.adminOwnerAccessKey(b.OwnerAccessKey), - DefaultCopies: b.DefaultCopies, - EffectiveCopies: s.effectiveBucketCopies(&b), - MinimumDurableCopies: b.MinimumDurableCopies, - EffectiveMinimumDurableCopies: s.effectiveBucketMinimumDurableCopies(&b), - Status: string(b.Status), - ObjectCount: stats.Count, - TotalSizeBytes: stats.TotalSize, - StorageHealth: bucketStorageHealthSummaryForBucket(storageHealthMap, b.ID, storageHealthFailed), - CreatedAt: b.CreatedAt.Format(time.RFC3339), + ID: b.ID, + Name: b.Name, + OwnerAccessKey: s.adminOwnerAccessKey(b.OwnerAccessKey), + DefaultCopies: b.DefaultCopies, + MinimumDurableCopies: b.MinimumDurableCopies, + Status: string(b.Status), + ObjectCount: stats.Count, + TotalSizeBytes: stats.TotalSize, + StorageHealth: bucketStorageHealthSummaryForBucket(storageHealthMap, b.ID, storageHealthFailed), + CreatedAt: b.CreatedAt.Format(time.RFC3339), }) } @@ -285,31 +266,19 @@ func (s *Server) handleAPICreateBucket(w http.ResponseWriter, r *http.Request) { return } - var bucket *model.Bucket - err = s.repos.WithTx(r.Context(), func(txRepos *repository.Repositories) error { - owner, err := txRepos.S3Accounts.LockByAccessKey(r.Context(), actualOwnerAccessKey) - if err != nil { - return err - } - if owner == nil { - return auth.ErrNoSuchUser - } - bucket = &model.Bucket{ - Name: name, - ACL: acl, - OwnerAccessKey: &actualOwnerAccessKey, - DefaultCopies: req.DefaultCopies, - MinimumDurableCopies: req.MinimumDurableCopies, - Status: model.BucketStatusActive, - } - return txRepos.Buckets.Create(r.Context(), bucket) + bucket, err := s.bucketLifecycle.CreateWithOptions(r.Context(), bucketlifecycle.CreateOptions{ + Name: name, + ACL: acl, + OwnerAccessKey: &actualOwnerAccessKey, + DefaultCopies: req.DefaultCopies, + MinimumDurableCopies: req.MinimumDurableCopies, }) if err != nil { if errors.Is(err, repository.ErrAlreadyExists) { writeJSON(w, http.StatusConflict, map[string]string{"error": "bucket already exists"}) return } - if errors.Is(err, auth.ErrNoSuchUser) { + if errors.Is(err, bucketlifecycle.ErrOwnerNotFound) { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "S3 owner not found"}) return } @@ -317,17 +286,13 @@ func (s *Server) handleAPICreateBucket(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) return } - s.bucketLifecycle.EnsureCacheBucketDir(r.Context(), name) - writeJSON(w, http.StatusCreated, bucketMutationResponse{ - ID: bucket.ID, - Name: bucket.Name, - OwnerAccessKey: s.adminOwnerAccessKey(bucket.OwnerAccessKey), - DefaultCopies: bucket.DefaultCopies, - EffectiveCopies: s.effectiveBucketCopies(bucket), - MinimumDurableCopies: bucket.MinimumDurableCopies, - EffectiveMinimumDurableCopies: s.effectiveBucketMinimumDurableCopies(bucket), - Status: string(bucket.Status), + ID: bucket.ID, + Name: bucket.Name, + OwnerAccessKey: s.adminOwnerAccessKey(bucket.OwnerAccessKey), + DefaultCopies: bucket.DefaultCopies, + MinimumDurableCopies: bucket.MinimumDurableCopies, + Status: string(bucket.Status), }) } @@ -357,8 +322,8 @@ func (s *Server) handleAPIGetBucket(w http.ResponseWriter, r *http.Request) { return } dataSets := make([]storageDataSetSummaryResponse, 0) - if s.repos.Uploads != nil { - summaries, err := s.repos.Uploads.ListDataSetSummaries(ctx, bucket.ID) + if s.repos.Contents != nil { + summaries, err := s.repos.Contents.ListDataSetSummaries(ctx, bucket.ID) if err != nil { s.logger.Error("api: failed to list bucket storage data sets", "error", err, "name", bucketName) writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) @@ -369,23 +334,21 @@ func (s *Server) handleAPIGetBucket(w http.ResponseWriter, r *http.Request) { storageHealthMap, storageHealthFailed := s.bucketStorageHealthSummaries(ctx, bucket.ID) writeJSON(w, http.StatusOK, bucketDetailResponse{ - ID: bucket.ID, - Name: bucket.Name, - OwnerAccessKey: s.adminOwnerAccessKey(bucket.OwnerAccessKey), - DefaultCopies: bucket.DefaultCopies, - EffectiveCopies: s.effectiveBucketCopies(bucket), - MinimumDurableCopies: bucket.MinimumDurableCopies, - EffectiveMinimumDurableCopies: s.effectiveBucketMinimumDurableCopies(bucket), - Status: string(bucket.Status), - ObjectCount: stats.Count, - TotalSizeBytes: stats.TotalSize, - StorageHealth: bucketStorageHealthSummaryForBucket(storageHealthMap, bucket.ID, storageHealthFailed), - CreatedAt: bucket.CreatedAt.Format(time.RFC3339), - UpdatedAt: bucket.UpdatedAt.Format(time.RFC3339), - VersioningStatus: "Enabled", - VersioningEnforced: true, - DataSets: dataSets, - Replacements: s.bucketReplacementResponses(ctx, bucket.Name, bucket.ID), + ID: bucket.ID, + Name: bucket.Name, + OwnerAccessKey: s.adminOwnerAccessKey(bucket.OwnerAccessKey), + DefaultCopies: bucket.DefaultCopies, + MinimumDurableCopies: bucket.MinimumDurableCopies, + Status: string(bucket.Status), + ObjectCount: stats.Count, + TotalSizeBytes: stats.TotalSize, + StorageHealth: bucketStorageHealthSummaryForBucket(storageHealthMap, bucket.ID, storageHealthFailed), + CreatedAt: bucket.CreatedAt.Format(time.RFC3339), + UpdatedAt: bucket.UpdatedAt.Format(time.RFC3339), + VersioningStatus: "Enabled", + VersioningEnforced: true, + DataSets: dataSets, + Replacements: s.bucketReplacementResponses(ctx, bucket.Name, bucket.ID), }) } @@ -445,14 +408,12 @@ func (s *Server) handleAPIUpdateBucketOwner(w http.ResponseWriter, r *http.Reque } writeJSON(w, http.StatusOK, bucketMutationResponse{ - ID: bucket.ID, - Name: bucket.Name, - OwnerAccessKey: s.adminOwnerAccessKey(&actualOwnerAccessKey), - DefaultCopies: bucket.DefaultCopies, - EffectiveCopies: s.effectiveBucketCopies(bucket), - MinimumDurableCopies: bucket.MinimumDurableCopies, - EffectiveMinimumDurableCopies: s.effectiveBucketMinimumDurableCopies(bucket), - Status: string(bucket.Status), + ID: bucket.ID, + Name: bucket.Name, + OwnerAccessKey: s.adminOwnerAccessKey(&actualOwnerAccessKey), + DefaultCopies: bucket.DefaultCopies, + MinimumDurableCopies: bucket.MinimumDurableCopies, + Status: string(bucket.Status), }) } @@ -478,12 +439,22 @@ func (s *Server) handleAPIUpdateBucketCopyPolicy(w http.ResponseWriter, r *http. writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) return } + // A bucket stores its policy, so an explicit null is a reset to the current + // configured target rather than a standing inheritance. + if setDefaultCopies && defaultCopies == nil { + configured := s.filecoinDefaultCopies + defaultCopies = &configured + } if !setDefaultCopies && !setMinimumCopies { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "copy policy update requires at least one field"}) return } var bucket *model.Bucket + if s.taskService == nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "task service unavailable"}) + return + } err = s.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { updated, err := txRepos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ Name: bucketName, @@ -498,10 +469,23 @@ func (s *Server) handleAPIUpdateBucketCopyPolicy(w http.ResponseWriter, r *http. if updated == nil || !updated.Status.IsAdminVisible() { return repository.ErrNotFound } - if updated.MinimumDurableCopies != nil && *updated.MinimumDurableCopies > s.effectiveBucketCopies(updated) { - return fmt.Errorf("minimum_durable_copies cannot exceed the effective replica target: %w", repository.ErrInvalidInput) + if err := s.bucketLifecycle.ScheduleProvision(ctx, txRepos, updated); err != nil { + return err + } + generation, err := txRepos.CacheEvictions.NextDurabilityGeneration(ctx, updated.ID) + if err != nil { + return err + } + taskRow, _, err := s.taskService.EnqueueInTransaction(ctx, txRepos, taskengine.EnqueueRequest{ + Type: model.TaskTypeCacheReconcileDurability, + IdempotencyKey: cacheeviction.DurabilityTaskKey(updated.ID, generation), + Input: cacheeviction.DurabilityInput{BucketID: updated.ID, Generation: generation}, + SubjectType: "bucket", SubjectKey: strconv.FormatInt(updated.ID, 10), + }) + if err != nil { + return err } - if _, err := txRepos.CacheEvictions.EnsureBucketDurabilityReconciliation(ctx, updated.ID, s.evictMaxRetries); err != nil { + if err := txRepos.CacheEvictions.BindDurabilityTask(ctx, updated.ID, generation, taskRow.ID); err != nil { return err } bucket = updated @@ -511,6 +495,12 @@ func (s *Server) handleAPIUpdateBucketCopyPolicy(w http.ResponseWriter, r *http. writeJSON(w, http.StatusNotFound, map[string]string{"error": "bucket not found"}) return } + if errors.Is(err, repository.ErrReplicaTargetLowered) { + writeJSON(w, http.StatusBadRequest, map[string]string{ + "error": "lowering the replica target is not supported yet; existing replicas above the new target would keep running", + }) + return + } if errors.Is(err, repository.ErrInvalidInput) { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "minimum_durable_copies cannot exceed the effective replica target"}) return @@ -522,14 +512,12 @@ func (s *Server) handleAPIUpdateBucketCopyPolicy(w http.ResponseWriter, r *http. } writeJSON(w, http.StatusOK, bucketMutationResponse{ - ID: bucket.ID, - Name: bucket.Name, - OwnerAccessKey: s.adminOwnerAccessKey(bucket.OwnerAccessKey), - DefaultCopies: bucket.DefaultCopies, - EffectiveCopies: s.effectiveBucketCopies(bucket), - MinimumDurableCopies: bucket.MinimumDurableCopies, - EffectiveMinimumDurableCopies: s.effectiveBucketMinimumDurableCopies(bucket), - Status: string(bucket.Status), + ID: bucket.ID, + Name: bucket.Name, + OwnerAccessKey: s.adminOwnerAccessKey(bucket.OwnerAccessKey), + DefaultCopies: bucket.DefaultCopies, + MinimumDurableCopies: bucket.MinimumDurableCopies, + Status: string(bucket.Status), }) } @@ -634,8 +622,8 @@ func (s *Server) storageDataSetSummaryResponses(ctx context.Context, summaries [ DataSetID: onChainIDStringPtr(summary.DataSetID), ClientDataSetID: onChainIDStringPtr(summary.ClientDataSetID), Status: string(summary.Status), - CreatedByUploadID: summary.CreatedByUploadID, - LastUsedUploadID: summary.LastUsedUploadID, + CreatedByContentID: summary.CreatedByContentID, + LastUsedContentID: summary.LastUsedContentID, CommittedCopies: summary.CommittedCopies, ReadableCopies: summary.ReadableCopies, PhysicalBytes: summary.PhysicalBytes, @@ -729,7 +717,6 @@ type objectListItem struct { Size int64 `json:"size"` State string `json:"state"` Status string `json:"status"` - UploadStatus *string `json:"upload_status,omitempty"` Progress *uploadProgressResponse `json:"progress,omitempty"` Location objectLocation `json:"location"` ContentType string `json:"content_type"` @@ -745,29 +732,25 @@ type objectLocation struct { } type objectStatusDetailResponse struct { - VersionID string `json:"version_id"` - State string `json:"state"` - Status string `json:"status"` - UploadStatus *string `json:"upload_status,omitempty"` - Progress *uploadProgressResponse `json:"progress,omitempty"` - FailedAtState *string `json:"failed_at_state,omitempty"` - Message *string `json:"message,omitempty"` - UpdatedAt string `json:"updated_at"` + VersionID string `json:"version_id"` + State string `json:"state"` + Status string `json:"status"` + Progress *uploadProgressResponse `json:"progress,omitempty"` + Message *string `json:"message,omitempty"` + UpdatedAt string `json:"updated_at"` } type objectProvenanceResponse struct { - VersionID string `json:"version_id"` - State string `json:"state"` - Status string `json:"status"` - UploadStatus *string `json:"upload_status,omitempty"` - Progress *uploadProgressResponse `json:"progress,omitempty"` - PieceCID *string `json:"piece_cid,omitempty"` - RequestedCopies int `json:"requested_copies"` - SuccessCopies int `json:"success_copies"` - CopyHealth copyHealthSummaryResponse `json:"copy_health"` - Copies []objectProvenanceCopyResponse `json:"copies"` - Failures []objectProvenanceFailureResponse `json:"failures"` - UpdatedAt string `json:"updated_at"` + VersionID string `json:"version_id"` + State string `json:"state"` + Status string `json:"status"` + Progress *uploadProgressResponse `json:"progress,omitempty"` + PieceCID *string `json:"piece_cid,omitempty"` + RequestedCopies int `json:"requested_copies"` + SuccessCopies int `json:"success_copies"` + CopyHealth copyHealthSummaryResponse `json:"copy_health"` + Copies []objectProvenanceCopyResponse `json:"copies"` + UpdatedAt string `json:"updated_at"` } type objectProvenanceCopyResponse struct { @@ -785,15 +768,6 @@ type objectProvenanceCopyResponse struct { AttentionAt *string `json:"attention_at,omitempty"` } -type objectProvenanceFailureResponse struct { - AttemptIndex int `json:"attempt_index"` - ProviderID *string `json:"provider_id,omitempty"` - ProviderIdentity *providerIdentityResponse `json:"provider_identity,omitempty"` - TransferMethod string `json:"transfer_method"` - Stage *string `json:"stage,omitempty"` - Error *string `json:"error,omitempty"` -} - type uploadProgressResponse struct { Scope string `json:"scope"` Attempt int `json:"attempt"` @@ -804,23 +778,14 @@ type uploadProgressResponse struct { UpdatedAt string `json:"updated_at"` } -func objectAdminStatusWithUpload(state model.ObjectState, inCache, inFilecoin bool, uploadStatus *model.StorageUploadStatus) string { +// objectAdminStatusWithUpload derives the operator-facing status. It no longer +// takes a separate upload status: that value described the same pipeline as +// state, which is itself derived from the copy rows, so the two could only ever +// agree or be wrong. +func objectAdminStatusWithUpload(state model.ObjectState, inCache, inFilecoin bool) string { if state == model.ObjectStateFailed { return objectAdminStatusWarning } - if uploadStatus != nil { - switch *uploadStatus { - case model.StorageUploadStatusFailed, - model.StorageUploadStatusRejected: - return objectAdminStatusWarning - case model.StorageUploadStatusIngressReady: - return objectAdminStatusUploading - case model.StorageUploadStatusReadable: - return objectAdminStatusSyncing - case model.StorageUploadStatusComplete: - return objectAdminStatusSuccess - } - } if !inCache && !inFilecoin { return objectAdminStatusUnavailable } @@ -829,7 +794,7 @@ func objectAdminStatusWithUpload(state model.ObjectState, inCache, inFilecoin bo return objectAdminStatusUploading case model.ObjectStateCommitting, model.ObjectStateReplicating: return objectAdminStatusSyncing - case model.ObjectStateStored, model.ObjectStateCacheEvicted: + case model.ObjectStateStored: return objectAdminStatusSuccess default: return objectAdminStatusUnavailable @@ -837,120 +802,95 @@ func objectAdminStatusWithUpload(state model.ObjectState, inCache, inFilecoin bo } type objectAdminUploadInfo struct { - Status *model.StorageUploadStatus Message *string Progress *uploadProgressResponse } -func (s *Server) objectAdminStorageUpload(ctx context.Context, version model.ObjectVersion) (*model.StorageUpload, error) { - if s.repos.Uploads == nil { +func (s *Server) objectAdminStorageContent(ctx context.Context, version model.ObjectVersion) (*model.StorageContent, error) { + if s.repos.Contents == nil { return nil, nil } - if version.StorageUploadID != nil { - return s.repos.Uploads.GetByID(ctx, *version.StorageUploadID) + if version.ContentID != nil { + return s.repos.Contents.GetByID(ctx, *version.ContentID) } - return s.repos.Uploads.FindLatestUploadBySourceVersion(ctx, version.VersionID) + return nil, nil } func (s *Server) objectAdminUploadInfo(ctx context.Context, version model.ObjectVersion) (objectAdminUploadInfo, error) { - upload, err := s.objectAdminStorageUpload(ctx, version) + upload, err := s.objectAdminStorageContent(ctx, version) if err != nil || upload == nil { return objectAdminUploadInfo{}, err } + ingress, err := s.repos.Contents.GetIngressCopy(ctx, upload.ID) + if err != nil { + return objectAdminUploadInfo{}, err + } return objectAdminUploadInfo{ - Status: &upload.Status, Message: uploadStatusMessage(upload), - Progress: uploadProgressResponseFromUpload(upload), + Progress: uploadProgressResponseFromUpload(ingress), }, nil } func (s *Server) objectAdminUploadInfos(ctx context.Context, versions []model.ObjectVersion) (map[string]objectAdminUploadInfo, error) { infos := make(map[string]objectAdminUploadInfo, len(versions)) - if s.repos.Uploads == nil || len(versions) == 0 { + if s.repos.Contents == nil || len(versions) == 0 { return infos, nil } - uploadIDSet := make(map[int64]struct{}) - versionIDSet := make(map[string]struct{}) + contentIDSet := make(map[int64]struct{}) for _, version := range versions { if version.IsDeleteMarker { continue } - if version.StorageUploadID != nil { - uploadIDSet[*version.StorageUploadID] = struct{}{} - } else { - versionIDSet[version.VersionID] = struct{}{} + if version.ContentID != nil { + contentIDSet[*version.ContentID] = struct{}{} } } - uploadIDs := make([]int64, 0, len(uploadIDSet)) - for uploadID := range uploadIDSet { - uploadIDs = append(uploadIDs, uploadID) + contentIDs := make([]int64, 0, len(contentIDSet)) + for contentID := range contentIDSet { + contentIDs = append(contentIDs, contentID) } - uploadsByID, err := s.repos.Uploads.GetByIDs(ctx, uploadIDs) - if err != nil { - return nil, err - } - versionIDs := make([]string, 0, len(versionIDSet)) - for versionID := range versionIDSet { - versionIDs = append(versionIDs, versionID) - } - uploadsByVersionID, err := s.repos.Uploads.FindLatestUploadsBySourceVersions(ctx, versionIDs) + uploadsByID, err := s.repos.Contents.GetByIDs(ctx, contentIDs) if err != nil { return nil, err } for _, version := range versions { - var upload model.StorageUpload - var ok bool - if version.StorageUploadID != nil { - upload, ok = uploadsByID[*version.StorageUploadID] - } else { - upload, ok = uploadsByVersionID[version.VersionID] + if version.ContentID == nil { + continue } + upload, ok := uploadsByID[*version.ContentID] if !ok { continue } - status := upload.Status + ingress, err := s.repos.Contents.GetIngressCopy(ctx, upload.ID) + if err != nil { + return nil, err + } infos[version.VersionID] = objectAdminUploadInfo{ - Status: &status, Message: uploadStatusMessage(&upload), - Progress: uploadProgressResponseFromUpload(&upload), + Progress: uploadProgressResponseFromUpload(ingress), } } return infos, nil } -func uploadStatusString(status *model.StorageUploadStatus) *string { - if status == nil { - return nil - } - value := string(*status) - return &value -} - -func uploadStatusMessage(upload *model.StorageUpload) *string { +func uploadStatusMessage(upload *model.StorageContent) *string { if upload == nil { return nil } if upload.ErrorMessage != nil && *upload.ErrorMessage != "" { return upload.ErrorMessage } - if upload.AcceptError != nil && *upload.AcceptError != "" { - return upload.AcceptError - } return nil } -func uploadProgressResponseFromUpload(upload *model.StorageUpload) *uploadProgressResponse { +// uploadProgressResponseFromUpload reads the ingress copy, not the content: +// progress belongs to the transfer that produced it. +func uploadProgressResponseFromUpload(upload *model.StorageCopy) *uploadProgressResponse { if upload == nil || upload.ProgressUpdatedAt == nil || upload.IngressStoreAttempt <= 0 { return nil } - uploaded := upload.IngressBytesTransferred - if uploaded < 0 { - uploaded = 0 - } - total := upload.ContentSize - if total < 0 { - total = 0 - } + uploaded := max(upload.IngressBytesTransferred, 0) + total := max(upload.ContentSize, 0) if uploaded > total { uploaded = total } @@ -1016,9 +956,12 @@ type permanentDeleteObjectRequest struct { } type permanentDeleteObjectResponse struct { - Key string `json:"key"` - VersionID string `json:"version_id"` - CacheCleanupStatus string `json:"cache_cleanup_status"` + Key string `json:"key"` + VersionID string `json:"version_id"` + // CacheRelease reports what happened to the cached bytes: released when this + // deletion removed the last reference, retained when another version still + // names them, failed when the local file could not be removed. + CacheRelease string `json:"cache_release"` StorageCleanupTaskID *int64 `json:"storage_cleanup_task_id,omitempty"` } @@ -1037,12 +980,9 @@ type permanentDeleteDeletedObjectResponse struct { } type objectDeletionListItem struct { - Key string `json:"key"` - VersionID string `json:"version_id"` - CacheCleanupStatus string `json:"cache_cleanup_status"` - CacheError *string `json:"cache_error,omitempty"` - CreatedAt string `json:"created_at"` - DeletedAt string `json:"deleted_at"` + Key string `json:"key"` + VersionID string `json:"version_id"` + DeletedAt string `json:"deleted_at"` } type objectDeletionListResponse struct { @@ -1126,8 +1066,7 @@ func (s *Server) handleAPIBucketObjects(w http.ResponseWriter, r *http.Request) CurrentVersionID: o.VersionID, Size: o.Size, State: string(o.State), - Status: objectAdminStatusWithUpload(o.State, o.InCache, o.InFilecoin, uploadInfo.Status), - UploadStatus: uploadStatusString(uploadInfo.Status), + Status: objectAdminStatusWithUpload(o.State, o.InCache, o.InFilecoin), Progress: uploadInfo.Progress, Location: objectLocation{Cache: o.InCache, Filecoin: o.InFilecoin}, ContentType: o.ContentType, @@ -1281,11 +1220,20 @@ func (s *Server) handleAPIPermanentDeleteBucketObject(w http.ResponseWriter, r * return } - result, err := s.repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: bucket.ID, - Key: key, - VersionID: versionID, - StorageCleanupMaxRetries: &s.storageCleanupMaxRetries, + if s.taskService == nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "task service unavailable"}) + return + } + var result repository.DeleteObjectVersionResult + err = s.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { + var deleteErr error + result, deleteErr = txRepos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ + BucketID: bucket.ID, Key: key, VersionID: versionID, + }) + if deleteErr != nil { + return deleteErr + } + return s.bindStorageCleanupTask(ctx, txRepos, result.StorageCleanup) }) if err != nil { switch { @@ -1302,12 +1250,17 @@ func (s *Server) handleAPIPermanentDeleteBucketObject(w http.ResponseWriter, r * return } - status := s.recordPermanentDeleteCacheCleanupWithTimeout(ctx, bucket.Name, versionID, result.CacheKey) + outcome, releaseErr := s.releaseContentCache(ctx, bucket.Name, result.ContentID) + status := string(outcome) + if releaseErr != nil { + s.logger.Warn("api: failed to release permanently deleted object cache", "bucket", bucketName, "key", key, "versionID", versionID, "error", releaseErr) + status = "failed" + } writeJSON(w, http.StatusOK, permanentDeleteObjectResponse{ Key: key, VersionID: versionID, - CacheCleanupStatus: string(status), - StorageCleanupTaskID: result.StorageCleanupTaskID, + CacheRelease: status, + StorageCleanupTaskID: cleanupReservationTaskID(result.StorageCleanup), }) } @@ -1341,11 +1294,25 @@ func (s *Server) handleAPIPermanentDeleteDeletedBucketObject(w http.ResponseWrit return } - result, err := s.repos.Objects.DeleteDeletedObjectPermanently(ctx, repository.DeleteDeletedObjectInput{ - BucketID: bucket.ID, - Key: key, - DeleteMarkerVersionID: deleteMarkerVersionID, - StorageCleanupMaxRetries: &s.storageCleanupMaxRetries, + if s.taskService == nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "task service unavailable"}) + return + } + var result repository.DeleteDeletedObjectResult + err = s.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { + var deleteErr error + result, deleteErr = txRepos.Objects.DeleteDeletedObjectPermanently(ctx, repository.DeleteDeletedObjectInput{ + BucketID: bucket.ID, Key: key, DeleteMarkerVersionID: deleteMarkerVersionID, + }) + if deleteErr != nil { + return deleteErr + } + for i := range result.StorageCleanups { + if err := s.bindStorageCleanupTask(ctx, txRepos, &result.StorageCleanups[i]); err != nil { + return err + } + } + return nil }) if err != nil { switch { @@ -1363,9 +1330,11 @@ func (s *Server) handleAPIPermanentDeleteDeletedBucketObject(w http.ResponseWrit } cacheCleanupFailedCount := s.recordDeletedObjectPermanentDeleteCacheCleanup(ctx, bucket.Name, result.DeletedVersions) - storageCleanupTaskIDs := result.StorageCleanupTaskIDs - if storageCleanupTaskIDs == nil { - storageCleanupTaskIDs = []int64{} + storageCleanupTaskIDs := make([]int64, 0, len(result.StorageCleanups)) + for i := range result.StorageCleanups { + if result.StorageCleanups[i].TaskID != nil { + storageCleanupTaskIDs = append(storageCleanupTaskIDs, *result.StorageCleanups[i].TaskID) + } } writeJSON(w, http.StatusOK, permanentDeleteDeletedObjectResponse{ Key: result.Key, @@ -1377,6 +1346,34 @@ func (s *Server) handleAPIPermanentDeleteDeletedBucketObject(w http.ResponseWrit }) } +func (s *Server) bindStorageCleanupTask(ctx context.Context, repos *repository.Repositories, cleanup *repository.StorageCleanupReservation) error { + if cleanup == nil || cleanup.TaskID != nil { + return nil + } + taskRow, _, err := s.taskService.EnqueueInTransaction(ctx, repos, taskengine.EnqueueRequest{ + Type: model.TaskTypeStorageCleanup, + IdempotencyKey: storagecleanup.TaskKey(cleanup.ContentID, cleanup.Generation), + Input: storagecleanup.Input{ContentID: cleanup.ContentID, Generation: cleanup.Generation}, + SubjectType: "storage_content", + SubjectKey: strconv.FormatInt(cleanup.ContentID, 10), + }) + if err != nil { + return err + } + if err := repos.StorageCleanup.BindTask(ctx, cleanup.ContentID, cleanup.Generation, taskRow.ID); err != nil { + return err + } + cleanup.TaskID = &taskRow.ID + return nil +} + +func cleanupReservationTaskID(cleanup *repository.StorageCleanupReservation) *int64 { + if cleanup == nil { + return nil + } + return cleanup.TaskID +} + func (s *Server) handleAPIBucketObjectDeletions(w http.ResponseWriter, r *http.Request) { ctx := r.Context() bucketName := r.PathValue("name") @@ -1411,7 +1408,7 @@ func (s *Server) handleAPIBucketObjectDeletions(w http.ResponseWriter, r *http.R q := s.db.NewSelect(). Model(&deletions). Where("bucket_id = ?", bucket.ID). - OrderExpr("created_at DESC, id DESC"). + OrderExpr("deleted_at DESC, id DESC"). Limit(limit). Offset(offset) if key := r.URL.Query().Get("key"); key != "" { @@ -1425,45 +1422,46 @@ func (s *Server) handleAPIBucketObjectDeletions(w http.ResponseWriter, r *http.R items := make([]objectDeletionListItem, 0, len(deletions)) for _, deletion := range deletions { items = append(items, objectDeletionListItem{ - Key: deletion.Key, - VersionID: deletion.VersionID, - CacheCleanupStatus: string(deletion.CacheCleanupStatus), - CacheError: deletion.CacheError, - CreatedAt: deletion.CreatedAt.Format(time.RFC3339), - DeletedAt: deletion.DeletedAt.Format(time.RFC3339), + Key: deletion.Key, + VersionID: deletion.VersionID, + DeletedAt: deletion.DeletedAt.Format(time.RFC3339), }) } writeJSON(w, http.StatusOK, objectDeletionListResponse{Deletions: items}) } -func (s *Server) recordPermanentDeleteCacheCleanup(ctx context.Context, bucketName string, versionID string, cacheKey string) model.CacheCleanupStatus { - return objectdeletion.RecordCacheCleanup( - ctx, +// releaseContentCache frees the cached bytes of a content payload, but only +// after the deletion that removed its last live reference. Residency is +// content-addressed, so bytes still named by another version must survive. +func (s *Server) releaseContentCache( + ctx context.Context, + bucketName string, + contentID *int64, +) (objectdeletion.CacheReleaseOutcome, error) { + if contentID == nil { + return objectdeletion.CacheReleaseRetained, nil + } + cleanupCtx, cancelCleanup := context.WithTimeout(context.WithoutCancel(ctx), permanentDeleteCacheCleanupTimeout) + defer cancelCleanup() + return objectdeletion.ReleaseContentCache( + cleanupCtx, s.cache, s.cacheGate, s.cacheAccessTracker, s.repos.Objects, - s.logger, bucketName, - versionID, - cacheKey, + *contentID, ) } -func (s *Server) recordPermanentDeleteCacheCleanupWithTimeout(ctx context.Context, bucketName string, versionID string, cacheKey string) model.CacheCleanupStatus { - cleanupCtx, cancelCleanup := context.WithTimeout(context.WithoutCancel(ctx), permanentDeleteCacheCleanupTimeout) - defer cancelCleanup() - return s.recordPermanentDeleteCacheCleanup(cleanupCtx, bucketName, versionID, cacheKey) -} - func (s *Server) recordDeletedObjectPermanentDeleteCacheCleanup(ctx context.Context, bucketName string, versions []repository.DeletedObjectVersionSnapshot) int { var failed atomic.Int32 var group errgroup.Group group.SetLimit(permanentDeleteCacheCleanupConcurrency) for _, version := range versions { - version := version group.Go(func() error { - if s.recordPermanentDeleteCacheCleanupWithTimeout(ctx, bucketName, version.VersionID, version.CacheKey) == model.CacheCleanupStatusFailed { + if _, err := s.releaseContentCache(ctx, bucketName, version.ContentID); err != nil { + s.logger.Warn("api: failed to release permanently deleted object cache", "bucket", bucketName, "versionID", version.VersionID, "error", err) failed.Add(1) } return nil @@ -1664,7 +1662,6 @@ type objectVersionListItem struct { State string `json:"state"` Status string `json:"status"` IsDeleteMarker bool `json:"is_delete_marker"` - UploadStatus *string `json:"upload_status,omitempty"` Progress *uploadProgressResponse `json:"progress,omitempty"` Location objectLocation `json:"location"` ContentType string `json:"content_type"` @@ -1723,24 +1720,15 @@ func (s *Server) handleAPIBucketObjectStatusDetail(w http.ResponseWriter, r *htt return } - var failedAtState *string - if version.FailedAtState != nil { - state := string(*version.FailedAtState) - failedAtState = &state - } - message := version.LastError - if message == nil { - message = uploadInfo.Message - } + // The stage a failure happened at is no longer recorded: position is derived + // from the copies, and the copy's own error is what an operator can act on. writeJSON(w, http.StatusOK, objectStatusDetailResponse{ - VersionID: version.VersionID, - State: string(version.State), - Status: objectAdminStatusWithUpload(version.State, version.InCache, version.InFilecoin, uploadInfo.Status), - UploadStatus: uploadStatusString(uploadInfo.Status), - Progress: uploadInfo.Progress, - FailedAtState: failedAtState, - Message: message, - UpdatedAt: version.UpdatedAt.Format(time.RFC3339), + VersionID: version.VersionID, + State: string(version.State), + Status: objectAdminStatusWithUpload(version.State, version.InCache, version.InFilecoin), + Progress: uploadInfo.Progress, + Message: uploadInfo.Message, + UpdatedAt: version.UpdatedAt.Format(time.RFC3339), }) } @@ -1782,14 +1770,13 @@ func (s *Server) handleAPIBucketObjectProvenance(w http.ResponseWriter, r *http. resp := objectProvenanceResponse{ VersionID: version.VersionID, State: string(version.State), - Status: objectAdminStatusWithUpload(version.State, version.InCache, version.InFilecoin, nil), + Status: objectAdminStatusWithUpload(version.State, version.InCache, version.InFilecoin), CopyHealth: emptyCopyHealthSummary(), Copies: make([]objectProvenanceCopyResponse, 0), - Failures: make([]objectProvenanceFailureResponse, 0), UpdatedAt: version.UpdatedAt.Format(time.RFC3339), } - upload, err := s.objectAdminStorageUpload(ctx, *version) + upload, err := s.objectAdminStorageContent(ctx, *version) if err != nil { s.logger.Error("api: failed to load object provenance upload", "error", err, "bucket", bucketName, "versionID", versionID) writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) @@ -1801,9 +1788,9 @@ func (s *Server) handleAPIBucketObjectProvenance(w http.ResponseWriter, r *http. return } - provenance, err := s.repos.Uploads.GetUploadProvenance(ctx, upload.ID) + provenance, err := s.repos.Contents.GetUploadProvenance(ctx, upload.ID) if err != nil { - s.logger.Error("api: failed to load object provenance", "error", err, "bucket", bucketName, "versionID", versionID, "uploadID", upload.ID) + s.logger.Error("api: failed to load object provenance", "error", err, "bucket", bucketName, "versionID", versionID, "contentID", upload.ID) writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) return } @@ -1811,30 +1798,21 @@ func (s *Server) handleAPIBucketObjectProvenance(w http.ResponseWriter, r *http. writeJSON(w, http.StatusOK, resp) return } - readableCopies, err := s.repos.Uploads.ListReadableCommittedCopies(ctx, upload.ID) + readableCopies, err := s.repos.Contents.ListReadableCommittedCopies(ctx, upload.ID) if err != nil { - s.logger.Error("api: failed to count readable provenance copies", "error", err, "bucket", bucketName, "versionID", versionID, "uploadID", upload.ID) + s.logger.Error("api: failed to count readable provenance copies", "error", err, "bucket", bucketName, "versionID", versionID, "contentID", upload.ID) writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) return } - - resp.UploadStatus = uploadStatusString(&provenance.Upload.Status) - resp.Progress = uploadProgressResponseFromUpload(&provenance.Upload) - resp.Status = objectAdminStatusWithUpload(version.State, version.InCache, version.InFilecoin, &provenance.Upload.Status) + resp.Progress = uploadProgressResponseFromUpload(provenance.IngressCopy) + resp.Status = objectAdminStatusWithUpload(version.State, version.InCache, version.InFilecoin) resp.PieceCID = provenance.Upload.PieceCID resp.RequestedCopies = provenance.Upload.RequestedCopies resp.SuccessCopies = len(readableCopies) resp.UpdatedAt = provenance.Upload.UpdatedAt.Format(time.RFC3339) - providerIDs := make([]idtypes.OnChainID, 0, len(provenance.Copies)+len(provenance.Failures)) + providerIDs := make([]idtypes.OnChainID, 0, len(provenance.Copies)) for _, copyRow := range provenance.Copies { - if copyRow.ProviderID != nil { - providerIDs = append(providerIDs, *copyRow.ProviderID) - } - } - for _, failure := range provenance.Failures { - if failure.ProviderID != nil { - providerIDs = append(providerIDs, *failure.ProviderID) - } + providerIDs = append(providerIDs, copyRow.ProviderID) } providerIdentities := s.providerIdentities(providerIDs) copyFacts := provenanceCopyHealthFacts(bucket.ID, version.VersionID, provenance.Upload, provenance.Copies) @@ -1850,6 +1828,7 @@ func (s *Server) handleAPIBucketObjectProvenance(w http.ResponseWriter, r *http. copyHealthByIndex[*fact.CopyIndex] = copyHealthInfoFromSignal(copyHealthSignalFromFact(fact, copyObservations, copyHealthFailed, copyHealthInterval, copyHealthNow)) } for _, copyRow := range provenance.Copies { + providerID := copyRow.ProviderID var attentionAt *string if copyRow.CommitAttentionAt != nil { value := copyRow.CommitAttentionAt.Format(time.RFC3339) @@ -1859,8 +1838,8 @@ func (s *Server) handleAPIBucketObjectProvenance(w http.ResponseWriter, r *http. CopyIndex: copyRow.CopyIndex, Status: string(copyRow.Status), Health: copyHealthByIndex[copyRow.CopyIndex], - ProviderID: onChainIDStringPtr(copyRow.ProviderID), - ProviderIdentity: providerIdentityFromSnapshotPtr(providerIdentities, copyRow.ProviderID), + ProviderID: onChainIDStringPtr(&providerID), + ProviderIdentity: providerIdentityFromSnapshotPtr(providerIdentities, &providerID), DataSetID: onChainIDStringPtr(copyRow.DataSetID), PieceID: onChainIDStringPtr(copyRow.PieceID), TransferMethod: string(copyRow.TransferMethod), @@ -1870,16 +1849,6 @@ func (s *Server) handleAPIBucketObjectProvenance(w http.ResponseWriter, r *http. AttentionAt: attentionAt, }) } - for _, failure := range provenance.Failures { - resp.Failures = append(resp.Failures, objectProvenanceFailureResponse{ - AttemptIndex: failure.AttemptIndex, - ProviderID: onChainIDStringPtr(failure.ProviderID), - ProviderIdentity: providerIdentityFromSnapshotPtr(providerIdentities, failure.ProviderID), - TransferMethod: failure.TransferMethod, - Stage: failure.Stage, - Error: failure.ErrorMessage, - }) - } writeJSON(w, http.StatusOK, resp) } @@ -1965,9 +1934,8 @@ func (s *Server) handleAPIBucketObjectVersions(w http.ResponseWriter, r *http.Re Key: v.Key, Size: v.Size, State: string(v.State), - Status: objectAdminStatusWithUpload(v.State, v.InCache, v.InFilecoin, uploadInfo.Status), + Status: objectAdminStatusWithUpload(v.State, v.InCache, v.InFilecoin), IsDeleteMarker: v.IsDeleteMarker, - UploadStatus: uploadStatusString(uploadInfo.Status), Progress: uploadInfo.Progress, Location: objectLocation{Cache: v.InCache, Filecoin: v.InFilecoin}, ContentType: v.ContentType, diff --git a/internal/admin/api_buckets_test.go b/internal/admin/api_buckets_test.go index 5e65b9d..b122485 100644 --- a/internal/admin/api_buckets_test.go +++ b/internal/admin/api_buckets_test.go @@ -1,7 +1,10 @@ package admin import ( + "bytes" "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -10,6 +13,7 @@ import ( "net/http/httptest" "net/url" "reflect" + "slices" "strconv" "strings" "sync" @@ -17,14 +21,16 @@ import ( "testing" "time" + "github.com/strahe/synaps3/internal/bucketlifecycle" + "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/cacheeviction" "github.com/strahe/synaps3/internal/config" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/objectreader" "github.com/strahe/synaps3/internal/observability" "github.com/strahe/synaps3/internal/s3iam" + taskengine "github.com/strahe/synaps3/internal/task" "github.com/strahe/synaps3/internal/testutil" idtypes "github.com/strahe/synaps3/internal/types" "github.com/strahe/synapse-go/chain" @@ -38,6 +44,28 @@ func newBucketAPITestServer(t *testing.T) (*Server, *repository.Repositories) { return newBucketAPITestServerWithRuntimeCopies(t, config.DefaultFilecoinCopies) } +func insertAdminStorageContentSnapshot( + t *testing.T, + db bun.IDB, + bucketID int64, + originVersionID string, + contentSize int64, + checksum string, + requestedCopies int, +) *model.StorageContent { + t.Helper() + upload := &model.StorageContent{ + BucketID: bucketID, + ContentSize: contentSize, + Checksum: testutil.StorageChecksum(checksum), + RequestedCopies: requestedCopies, + } + if _, err := db.NewInsert().Model(upload).Exec(t.Context()); err != nil { + t.Fatalf("insert historical storage upload snapshot: %v", err) + } + return upload +} + func newBucketAPITestServerWithRuntimeCopies(t *testing.T, filecoinDefaultCopies int) (*Server, *repository.Repositories) { t.Helper() @@ -49,6 +77,7 @@ func newBucketAPITestServerWithRuntimeCopies(t *testing.T, filecoinDefaultCopies repos := repository.NewRepositories(db) srv := newTestServer("127.0.0.1:0", db, localCache, 1<<20, repos, nil, nil, filecoinDefaultCopies, testLogger()) + srv.WithTaskService(newAdminTestTaskService(t, repos)) return srv, repos } @@ -112,7 +141,7 @@ type writeDeadlineRecorder struct { } type failingBucketStorageHealthUploadRepo struct { - repository.StorageUploadRepository + repository.StorageContentRepository } func (r failingBucketStorageHealthUploadRepo) ListBucketStorageHealthSummaries(context.Context, int64, time.Time, int) ([]repository.BucketStorageHealthSummary, error) { @@ -207,8 +236,21 @@ func (u *cacheBackedObjectUploader) PutObject(ctx context.Context, input s3respo return s3response.PutObjectOutput{}, err } versionID := model.NewVersionID() - cacheKey := ".versions/" + versionID - info, err := u.cache.Put(ctx, *input.Bucket, cacheKey, input.Body) + body, err := io.ReadAll(input.Body) + if err != nil { + return s3response.PutObjectOutput{}, err + } + sum := sha256.Sum256(body) + content, err := u.repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, + ContentSize: int64(len(body)), + Checksum: hex.EncodeToString(sum[:]), + RequestedCopies: 1, + }) + if err != nil { + return s3response.PutObjectOutput{}, err + } + info, err := u.cache.Put(ctx, *input.Bucket, model.ContentCacheKey(content.ID), bytes.NewReader(body)) if err != nil { return s3response.PutObjectOutput{}, err } @@ -220,12 +262,10 @@ func (u *cacheBackedObjectUploader) PutObject(ctx context.Context, input s3respo VersionID: versionID, BucketID: bucket.ID, Key: *input.Key, + ContentID: &content.ID, Size: info.Size, ETag: info.ETag, - Checksum: info.Checksum, ContentType: contentType, - CacheKey: cacheKey, - State: model.ObjectStateCached, }); err != nil { return s3response.PutObjectOutput{}, err } @@ -267,58 +307,27 @@ func (c *contextRecordingDeleteCache) recordedContexts() []context.Context { return append([]context.Context(nil), c.contexts...) } -type blockingDeleteCache struct { +type selectiveFailingDeleteCache struct { cache.Cache - started chan struct{} - release chan struct{} - failKey string - deletes atomic.Int32 - active atomic.Int32 - maxActive atomic.Int32 + failKey string + deletes atomic.Int32 } -func (c *blockingDeleteCache) Delete(ctx context.Context, _, key string) error { - currentActive := c.active.Add(1) - defer c.active.Add(-1) - for { - maxActive := c.maxActive.Load() - if currentActive <= maxActive || c.maxActive.CompareAndSwap(maxActive, currentActive) { - break - } - } +func (c *selectiveFailingDeleteCache) Delete(_ context.Context, _, key string) error { c.deletes.Add(1) - c.started <- struct{}{} - select { - case <-c.release: - case <-ctx.Done(): - return ctx.Err() - } if key == c.failKey { return errors.New("cache error") } return nil } -func waitDeleteStarts(t *testing.T, started <-chan struct{}, want int) { - t.Helper() - timer := time.NewTimer(2 * time.Second) - defer timer.Stop() - for i := 0; i < want; i++ { - select { - case <-started: - case <-timer.C: - t.Fatalf("cache cleanup starts = %d, want %d", i, want) - } - } -} - type storageUploadSelectCounter struct { selects atomic.Int32 } func (c *storageUploadSelectCounter) BeforeQuery(ctx context.Context, event *bun.QueryEvent) context.Context { query := strings.ToLower(strings.TrimSpace(event.Query)) - if strings.HasPrefix(query, "select") && (strings.Contains(query, `from "storage_uploads"`) || strings.Contains(query, "from storage_uploads")) { + if strings.HasPrefix(query, "select") && (strings.Contains(query, `from "storage_contents"`) || strings.Contains(query, "from storage_contents")) { c.selects.Add(1) } return ctx @@ -345,38 +354,37 @@ func (f *fakeAPIProviderIdentityResolver) ProviderIdentities(providerIDs []idtyp return out } -func seedAdminObjectVersion(t *testing.T, repos *repository.Repositories, bucket *model.Bucket, key string, size int64, etag, checksum, contentType, cacheKey string, state model.ObjectState) (int64, string) { +func seedAdminObjectVersion(t *testing.T, db *bun.DB, repos *repository.Repositories, bucket *model.Bucket, key string, size int64, etag, checksum, contentType string, state model.ObjectState) (int64, string) { t.Helper() versionID := model.NewVersionID() - if cacheKey == "" { - cacheKey = ".versions/" + versionID + if checksum == "" { + checksum = "checksum-" + versionID } - createState := state - if state == model.ObjectStateStored || state == model.ObjectStateCacheEvicted { - createState = model.ObjectStateUploading + requestedCopies := bucket.DefaultCopies + content, err := repos.Contents.EnsureContent(context.Background(), repository.EnsureContentInput{ + BucketID: bucket.ID, + ContentSize: size, + Checksum: testutil.StorageChecksum(checksum), + RequestedCopies: requestedCopies, + }) + if err != nil { + t.Fatalf("Contents.EnsureContent: %v", err) } version := &model.ObjectVersion{ VersionID: versionID, BucketID: bucket.ID, Key: key, + ContentID: &content.ID, Size: size, ETag: etag, - Checksum: checksum, ContentType: contentType, - CacheKey: cacheKey, - State: createState, } objID, err := repos.Objects.CreateVersionAndSetCurrent(context.Background(), version) if err != nil { t.Fatalf("Objects.CreateVersionAndSetCurrent: %v", err) } - if state == model.ObjectStateStored || state == model.ObjectStateCacheEvicted { - acceptAdminVersionUpload(t, repos, versionID, "piece-"+versionID, "https://provider.example/piece/"+versionID) - if state == model.ObjectStateCacheEvicted { - if err := repos.Objects.UpdateVersionState(context.Background(), versionID, model.ObjectStateStored, model.ObjectStateCacheEvicted); err != nil { - t.Fatalf("Objects.UpdateVersionState cache_evicted: %v", err) - } - } + if state == model.ObjectStateStored { + acceptAdminVersionUpload(t, db, repos, versionID, "piece-"+versionID, "https://provider.example/piece/"+versionID) } return objID, versionID } @@ -390,7 +398,7 @@ type adminStorageCopySeed struct { RetrievalURL string } -func seedAdminCommittedCopies(t *testing.T, repos *repository.Repositories, bucketID int64, uploadID int64, pieceCID string, copies []adminStorageCopySeed) { +func seedAdminCommittedCopies(t *testing.T, db *bun.DB, repos *repository.Repositories, bucketID int64, contentID int64, pieceCID string, copies []adminStorageCopySeed) { t.Helper() ctx := context.Background() inputs := make([]repository.UploadCopyBindingInput, 0, len(copies)) @@ -399,16 +407,16 @@ func seedAdminCommittedCopies(t *testing.T, repos *repository.Repositories, buck if i > 0 && copyIndex == 0 { copyIndex = i } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucketID, - ProviderID: copySeed.ProviderID, - CopyIndex: copyIndex, - CreatedByUploadID: uploadID, + binding, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucketID, + ProviderID: copySeed.ProviderID, + CopyIndex: copyIndex, + CreatedByContentID: contentID, }) if err != nil { t.Fatalf("ensure dataset binding: %v", err) } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, UploadID: uploadID, DataSetID: copySeed.DataSetID}); err != nil { + if err := repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, ContentID: contentID, DataSetID: copySeed.DataSetID}); err != nil { t.Fatalf("mark dataset ready: %v", err) } transferMethod := copySeed.TransferMethod @@ -426,80 +434,113 @@ func seedAdminCommittedCopies(t *testing.T, repos *repository.Repositories, buck }) copies[i].CopyIndex = copyIndex } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, uploadID, inputs); err != nil { + if err := repos.Contents.CreateUploadCopiesForBindings(ctx, contentID, inputs); err != nil { t.Fatalf("create upload copies: %v", err) } for _, copySeed := range copies { - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: uploadID, + testutil.CommitStorageCopy(t, db, repos, repository.MarkUploadCopyCommittedInput{ + ContentID: contentID, CopyIndex: copySeed.CopyIndex, PieceCID: pieceCID, PieceID: copySeed.PieceID, RetrievalURL: copySeed.RetrievalURL, - }); err != nil { - t.Fatalf("mark copy committed: %v", err) - } + }) } } -func acceptAdminVersionUpload(t *testing.T, repos *repository.Repositories, versionID string, pieceCID string, retrievalURL string) *model.StorageUpload { +// adminVersionCacheKey is the content-addressed cache key backing one version. +func adminVersionCacheKey(t *testing.T, repos *repository.Repositories, versionID string) string { t.Helper() - ctx := context.Background() - version, err := repos.Objects.GetVersionByID(ctx, versionID) + version, err := repos.Objects.GetVersionByID(context.Background(), versionID) if err != nil || version == nil { - t.Fatalf("get version for upload accept: version=%v err=%v", version, err) + t.Fatalf("get version %s: version=%v err=%v", versionID, version, err) } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: version.BucketID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("start upload attempt: %v", err) + return version.CacheKey() +} + +// adminContentForVersion returns the content a seeded version points at. The +// bytes own their row now, so upload seeding attaches copies to it rather than +// minting a second identity for the same version. +func adminContentForVersion(t *testing.T, repos *repository.Repositories, version *model.ObjectVersion) *model.StorageContent { + t.Helper() + if version.ContentID == nil { + t.Fatalf("version %s has no content", version.VersionID) } - providerID := onChainID(t, "101") - dataSetID := onChainID(t, fmt.Sprintf("1001%d", upload.ID)) - pieceID := onChainIDPtr(t, "1") - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: version.BucketID, - ProviderID: providerID, - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) + content, err := repos.Contents.GetByID(context.Background(), *version.ContentID) + if err != nil || content == nil { + t.Fatalf("get content %d: content=%v err=%v", *version.ContentID, content, err) + } + return content +} + +// adminSlotBinding returns the data set bound to one of the bucket's replica +// slots, creating it on the given provider only when the slot is still free. +// Slots belong to the bucket, so several contents share the same data sets. +func adminSlotBinding(t *testing.T, repos *repository.Repositories, bucketID int64, copyIndex int, provider string, contentID int64) *model.StorageDataSet { + t.Helper() + ctx := context.Background() + binding, err := repos.Contents.GetDataSetBindingByCopyIndex(ctx, bucketID, copyIndex) if err != nil { - t.Fatalf("ensure dataset binding: %v", err) + t.Fatalf("load dataset binding %d: %v", copyIndex, err) } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, UploadID: upload.ID, DataSetID: dataSetID}); err != nil { - t.Fatalf("mark dataset ready: %v", err) + if binding == nil { + binding, err = repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucketID, ProviderID: onChainID(t, provider), CopyIndex: copyIndex, CreatedByContentID: contentID, + }) + if err != nil { + t.Fatalf("ensure dataset binding %d: %v", copyIndex, err) + } } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: providerID, - }}); err != nil { - t.Fatalf("create upload copy: %v", err) + return binding +} + +func acceptAdminVersionUpload(t *testing.T, db *bun.DB, repos *repository.Repositories, versionID string, pieceCID string, retrievalURL string) *model.StorageContent { + t.Helper() + ctx := context.Background() + version, err := repos.Objects.GetVersionByID(ctx, versionID) + if err != nil || version == nil { + t.Fatalf("get version for upload accept: version=%v err=%v", version, err) } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: pieceCID, - PieceID: pieceID, - RetrievalURL: retrievalURL, - }); err != nil { - t.Fatalf("mark upload copy committed: %v", err) + upload := adminContentForVersion(t, repos, version) + // Accepting content means every requested replica is committed, so the seed + // follows the content's own durability target rather than assuming one. + for copyIndex := range max(upload.RequestedCopies, 1) { + pieceID := onChainIDPtr(t, fmt.Sprintf("1%d", copyIndex)) + binding := adminSlotBinding(t, repos, version.BucketID, copyIndex, fmt.Sprintf("10%d", copyIndex+1), upload.ID) + providerID := binding.ProviderID + if binding.DataSetID == nil { + dataSetID := onChainID(t, fmt.Sprintf("1001%d%d", upload.ID, copyIndex)) + if err := repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, ContentID: upload.ID, DataSetID: dataSetID}); err != nil { + t.Fatalf("mark dataset %d ready: %v", copyIndex, err) + } + } + transferMethod := model.StorageCopyTransferMethodPeerPull + if copyIndex == 0 { + transferMethod = model.StorageCopyTransferMethodIngress + } + if err := repos.Contents.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: binding.ID, + CopyIndex: copyIndex, + TransferMethod: transferMethod, + ProviderID: providerID, + }}); err != nil { + t.Fatalf("create upload copy %d: %v", copyIndex, err) + } + testutil.CommitStorageCopy(t, db, repos, repository.MarkUploadCopyCommittedInput{ + ContentID: upload.ID, + CopyIndex: copyIndex, + PieceCID: pieceCID, + PieceID: pieceID, + RetrievalURL: retrievalURL, + }) } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: version.BucketID, - ContentSize: version.Size, - Checksum: version.Checksum, + if _, err := repos.Contents.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ + ContentID: upload.ID, + BucketID: version.BucketID, }); err != nil { t.Fatalf("bind readable upload: %v", err) } - if finalized, _, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}); err != nil { + if finalized, _, err := repos.Contents.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{ContentID: upload.ID}); err != nil { t.Fatalf("finalize upload: %v", err) } else if !finalized { t.Fatal("finalize upload = false, want true") @@ -507,110 +548,82 @@ func acceptAdminVersionUpload(t *testing.T, repos *repository.Repositories, vers return upload } -func bindAdminPartialUpload(t *testing.T, repos *repository.Repositories, versionID string) *model.StorageUpload { +func bindAdminPartialUpload(t *testing.T, db *bun.DB, repos *repository.Repositories, versionID string) *model.StorageContent { t.Helper() ctx := context.Background() version, err := repos.Objects.GetVersionByID(ctx, versionID) if err != nil || version == nil { t.Fatalf("get version for partial upload: version=%v err=%v", version, err) } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("partial uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("partial committing: %v", err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: version.BucketID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("start partial upload attempt: %v", err) - } - primary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: version.BucketID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("primary binding: %v", err) - } - secondary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: version.BucketID, ProviderID: onChainID(t, "202"), CopyIndex: 1, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("secondary binding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: primary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}); err != nil { - t.Fatalf("primary dataset ready: %v", err) + upload := adminContentForVersion(t, repos, version) + primary := adminSlotBinding(t, repos, version.BucketID, 0, "101", upload.ID) + secondary := adminSlotBinding(t, repos, version.BucketID, 1, "202", upload.ID) + if primary.DataSetID == nil { + if err := repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: primary.ID, ContentID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}); err != nil { + t.Fatalf("primary dataset ready: %v", err) + } } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: secondary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1002"), ClientDataSetID: onChainIDPtr(t, "9002")}); err != nil { - t.Fatalf("secondary dataset ready: %v", err) + if secondary.DataSetID == nil { + if err := repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: secondary.ID, ContentID: upload.ID, DataSetID: onChainID(t, "1002"), ClientDataSetID: onChainIDPtr(t, "9002")}); err != nil { + t.Fatalf("secondary dataset ready: %v", err) + } } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - {StorageDataSetID: secondary.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202")}, + if err := repos.Contents.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ + {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: primary.ProviderID}, + {StorageDataSetID: secondary.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: secondary.ProviderID}, }); err != nil { t.Fatalf("create upload copies: %v", err) } pieceCID := "piece-partial-" + versionID - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, + testutil.CommitStorageCopy(t, db, repos, repository.MarkUploadCopyCommittedInput{ + ContentID: upload.ID, CopyIndex: 0, PieceCID: pieceCID, PieceID: onChainIDPtr(t, "301"), RetrievalURL: "https://primary.example/piece/" + versionID, - }); err != nil { - t.Fatalf("primary committed: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: version.BucketID, - ContentSize: version.Size, - Checksum: version.Checksum, + }) + if _, err := repos.Contents.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ + ContentID: upload.ID, + BucketID: version.BucketID, }); err != nil { t.Fatalf("bind primary committed upload: %v", err) } - if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: upload.ID, CopyIndex: 1, LastError: "secondary pull: timeout"}); err != nil { + if err := repos.Contents.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{ContentID: upload.ID, CopyIndex: 1, LastError: "secondary pull: timeout"}); err != nil { t.Fatalf("mark secondary failed: %v", err) } return upload } -func markAdminStoredOnPrimaryUpload(t *testing.T, repos *repository.Repositories, versionID string) *model.StorageUpload { +func markAdminStoredOnPrimaryUpload(t *testing.T, db *bun.DB, repos *repository.Repositories, versionID string) *model.StorageContent { t.Helper() ctx := context.Background() version, err := repos.Objects.GetVersionByID(ctx, versionID) if err != nil || version == nil { t.Fatalf("get version for stored-on-primary upload: version=%v err=%v", version, err) } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("stored-on-primary uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("stored-on-primary committing: %v", err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ + upload, err := repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ BucketID: version.BucketID, - SourceVersionID: version.VersionID, ContentSize: version.Size, - Checksum: version.Checksum, + Checksum: testutil.StorageChecksum("checksum-" + version.VersionID), RequestedCopies: 3, }) if err != nil { t.Fatalf("start stored-on-primary upload attempt: %v", err) } - primary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: version.BucketID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID}) + primary, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: version.BucketID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByContentID: upload.ID}) if err != nil { t.Fatalf("primary binding: %v", err) } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: primary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}); err != nil { + if err := repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: primary.ID, ContentID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}); err != nil { t.Fatalf("primary dataset ready: %v", err) } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ + if err := repos.Contents.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, }); err != nil { t.Fatalf("create upload copy: %v", err) } - if err := repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: upload.ID, + if err := repos.Contents.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ + ContentID: upload.ID, CopyIndex: 0, PieceCID: "piece-primary-" + versionID, RetrievalURL: "https://primary.example/piece/" + versionID, @@ -620,36 +633,24 @@ func markAdminStoredOnPrimaryUpload(t *testing.T, repos *repository.Repositories return upload } -func markAdminFailedUpload(t *testing.T, repos *repository.Repositories, versionID string, message string) *model.StorageUpload { +func markAdminFailedUpload(t *testing.T, db *bun.DB, repos *repository.Repositories, versionID string, message string) *model.StorageContent { t.Helper() ctx := context.Background() version, err := repos.Objects.GetVersionByID(ctx, versionID) if err != nil || version == nil { t.Fatalf("get version for failed upload: version=%v err=%v", version, err) } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("failed upload state: %v", err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: version.BucketID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("start failed upload attempt: %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: version.BucketID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, + upload := adminContentForVersion(t, repos, version) + binding, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: version.BucketID, + ProviderID: onChainID(t, "101"), + CopyIndex: 0, + CreatedByContentID: upload.ID, }) if err != nil { t.Fatalf("ensure failed upload dataset: %v", err) } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ + if err := repos.Contents.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, @@ -657,22 +658,36 @@ func markAdminFailedUpload(t *testing.T, repos *repository.Repositories, version }}); err != nil { t.Fatalf("create failed upload copy: %v", err) } - if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: upload.ID, CopyIndex: 0, LastError: message}); err != nil { + if err := repos.Contents.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{ContentID: upload.ID, CopyIndex: 0, LastError: message}); err != nil { t.Fatalf("mark failed upload copy: %v", err) } + // Ingest failure is reported to operators from the content, so the copy + // error is recorded there too, the way the pipeline does it. + if err := repos.Contents.RecordContentFailure(ctx, upload.ID, message); err != nil { + t.Fatalf("record content failure: %v", err) + } return upload } func seedCachedDownloadObject(t *testing.T, srv *Server, repos *repository.Repositories, bucketName, key, body string) *cache.ObjectInfo { t.Helper() ctx := context.Background() - bucket := &model.Bucket{Name: bucketName, Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: bucketName, Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } versionID := model.NewVersionID() - cacheKey := ".versions/" + versionID - info, err := srv.cache.Put(ctx, bucket.Name, cacheKey, strings.NewReader(body)) + sum := sha256.Sum256([]byte(body)) + content, err := repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, + ContentSize: int64(len(body)), + Checksum: hex.EncodeToString(sum[:]), + RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("Contents.EnsureContent: %v", err) + } + info, err := srv.cache.Put(ctx, bucket.Name, model.ContentCacheKey(content.ID), strings.NewReader(body)) if err != nil { t.Fatalf("cache.Put: %v", err) } @@ -680,12 +695,10 @@ func seedCachedDownloadObject(t *testing.T, srv *Server, repos *repository.Repos VersionID: versionID, BucketID: bucket.ID, Key: key, + ContentID: &content.ID, Size: info.Size, ETag: info.ETag, - Checksum: info.Checksum, ContentType: "text/plain", - CacheKey: cacheKey, - State: model.ObjectStateCached, }); err != nil { t.Fatalf("Objects.CreateVersionAndSetCurrent: %v", err) } @@ -757,8 +770,12 @@ func TestHandleAPIBuckets_CreateBucket(t *testing.T) { if bucket == nil { t.Fatal("expected bucket to be created") } - if bucket.Status != model.BucketStatusActive { - t.Fatalf("bucket status = %s, want %s", bucket.Status, model.BucketStatusActive) + if bucket.Status != model.BucketStatusProvisioning { + t.Fatalf("bucket status = %s, want %s", bucket.Status, model.BucketStatusProvisioning) + } + provisionTask, err := repos.Tasks.GetByIdentity(ctx, model.TaskTypeBucketProvision, bucketlifecycle.ProvisionKey(bucket.ID, bucket.DefaultCopies)) + if err != nil || provisionTask == nil || provisionTask.Status != model.TaskStatusPending { + t.Fatalf("bucket provision task = %#v, err=%v", provisionTask, err) } acl, err := auth.ParseACL(bucket.ACL) if err != nil { @@ -767,20 +784,18 @@ func TestHandleAPIBuckets_CreateBucket(t *testing.T) { if acl.Owner != "owner-access" { t.Fatalf("owner = %q, want owner-access", acl.Owner) } - if bucket.DefaultCopies == nil || *bucket.DefaultCopies != 4 { + if bucket.DefaultCopies != 4 { t.Fatalf("bucket default_copies = %v, want 4", bucket.DefaultCopies) } - if bucket.MinimumDurableCopies == nil || *bucket.MinimumDurableCopies != 2 { + if bucket.MinimumDurableCopies != 2 { t.Fatalf("bucket minimum_durable_copies = %v, want 2", bucket.MinimumDurableCopies) } var body struct { - Name string `json:"name"` - OwnerAccessKey *string `json:"owner_access_key"` - DefaultCopies *int `json:"default_copies"` - EffectiveCopies int `json:"effective_copies"` - MinimumCopies *int `json:"minimum_durable_copies"` - EffectiveMin int `json:"effective_minimum_durable_copies"` + Name string `json:"name"` + OwnerAccessKey *string `json:"owner_access_key"` + DefaultCopies int `json:"default_copies"` + MinimumCopies int `json:"minimum_durable_copies"` } if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { t.Fatalf("Decode response: %v", err) @@ -788,11 +803,11 @@ func TestHandleAPIBuckets_CreateBucket(t *testing.T) { if body.OwnerAccessKey == nil || *body.OwnerAccessKey != "owner-access" { t.Fatalf("owner_access_key = %v, want owner-access", body.OwnerAccessKey) } - if body.DefaultCopies == nil || *body.DefaultCopies != 4 || body.EffectiveCopies != 4 { - t.Fatalf("copy policy response = default:%v effective:%d, want 4/4", body.DefaultCopies, body.EffectiveCopies) + if body.DefaultCopies != 4 { + t.Fatalf("copy policy response = %d, want 4", body.DefaultCopies) } - if body.MinimumCopies == nil || *body.MinimumCopies != 2 || body.EffectiveMin != 2 { - t.Fatalf("minimum copy policy response = minimum:%v effective:%d, want 2/2", body.MinimumCopies, body.EffectiveMin) + if body.MinimumCopies != 2 { + t.Fatalf("minimum copy policy response = %d, want 2", body.MinimumCopies) } } @@ -913,11 +928,95 @@ func TestHandleAPIBuckets_CreateBucketRejectsMinimumAboveTarget(t *testing.T) { } } +func TestAPIBucketObjects_ActiveBucket(t *testing.T) { + srv, repos := newBucketAPITestServer(t) + ctx := context.Background() + + bucket := &model.Bucket{Name: "objects-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) + } + _, versionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "kept.txt", 4, "etag-kept", "checksum-kept", "text/plain", model.ObjectStateCached) + acceptAdminVersionUpload(t, srv.db, repos, versionID, "piece-kept", "https://provider.example/kept") + + ts := httptest.NewServer(newBucketAPIMux(srv)) + defer ts.Close() + + resp, err := http.Get(ts.URL + "/api/v1/buckets/objects-bucket/objects") + if err != nil { + t.Fatalf("GET bucket objects: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + + var body struct { + Objects []struct { + Key string `json:"key"` + CurrentVersionID string `json:"current_version_id"` + State string `json:"state"` + Status string `json:"status"` + Location objectLocation `json:"location"` + } `json:"objects"` + Folders []objectFolderItem `json:"folders"` + } + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("Decode: %v", err) + } + if len(body.Objects) != 1 { + t.Fatalf("objects len = %d, want 1", len(body.Objects)) + } + if body.Objects[0].Key != "kept.txt" { + t.Fatalf("key = %q, want %q", body.Objects[0].Key, "kept.txt") + } + if body.Objects[0].CurrentVersionID == "" { + t.Fatal("expected current version id") + } + if body.Objects[0].Status != "success" { + t.Fatalf("status = %q, want success", body.Objects[0].Status) + } + if body.Objects[0].State != string(model.ObjectStateStored) { + t.Fatalf("state = %q, want stored", body.Objects[0].State) + } + if !body.Objects[0].Location.Cache || !body.Objects[0].Location.Filecoin { + t.Fatalf("location = %#v, want cache and filecoin", body.Objects[0].Location) + } + if len(body.Folders) != 0 { + t.Fatalf("folders len = %d, want 0 for flat object list", len(body.Folders)) + } + if body.Folders == nil { + t.Fatal("folders should be an empty array, not null") + } + + resp, err = http.Get(ts.URL + "/api/v1/buckets/objects-bucket/objects") + if err != nil { + t.Fatalf("GET bucket objects raw: %v", err) + } + defer func() { _ = resp.Body.Close() }() + var raw struct { + Objects []map[string]any `json:"objects"` + } + if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil { + t.Fatalf("Decode raw: %v", err) + } + if raw.Objects[0]["state"] != string(model.ObjectStateStored) { + t.Fatalf("object list state = %#v, want stored", raw.Objects[0]["state"]) + } + if _, ok := raw.Objects[0]["storage"]; ok { + t.Fatal("object list exposed storage instead of location") + } + if _, ok := raw.Objects[0]["attention"]; ok { + t.Fatal("object list exposed attention") + } +} + func TestAPIBucketDetail(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "detail-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "detail-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -929,7 +1028,7 @@ func TestAPIBucketDetail(t *testing.T) { {key: "a.txt", size: 5}, {key: "b.txt", size: 7}, } { - seedAdminObjectVersion(t, repos, bucket, tc.key, tc.size, tc.key, tc.key, "text/plain", "", model.ObjectStateCached) + seedAdminObjectVersion(t, srv.db, repos, bucket, tc.key, tc.size, tc.key, tc.key, "text/plain", model.ObjectStateCached) } ts := httptest.NewServer(newBucketAPIMux(srv)) @@ -991,26 +1090,17 @@ func TestAPIBucketDetail_IncludesProviderDataSets(t *testing.T) { srv.WithProviderIdentityResolver(identityResolver) ctx := context.Background() - bucket := &model.Bucket{Name: "detail-datasets-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "detail-datasets-bucket", Status: model.BucketStatusActive, DefaultCopies: 2, MinimumDurableCopies: 2} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J000000000000000DATASET1", - ContentSize: 1, - Checksum: "checksum-dataset-detail", - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } + upload := insertAdminStorageContentSnapshot(t, srv.db, bucket.ID, "01J000000000000000DATASET1", 1, "checksum-dataset-detail", 3) pieceCID := "bafk2bzacedatasetdetail" - seedAdminCommittedCopies(t, repos, bucket.ID, upload.ID, pieceCID, []adminStorageCopySeed{ + seedAdminCommittedCopies(t, srv.db, repos, bucket.ID, upload.ID, pieceCID, []adminStorageCopySeed{ {ProviderID: onChainID(t, "101"), DataSetID: onChainID(t, "1001"), PieceID: onChainIDPtr(t, "2001"), TransferMethod: model.StorageCopyTransferMethodIngress, RetrievalURL: "https://provider.example/1"}, {ProviderID: onChainID(t, "202"), DataSetID: onChainID(t, "2002"), PieceID: onChainIDPtr(t, "3001"), TransferMethod: model.StorageCopyTransferMethodPeerPull, RetrievalURL: "https://provider.example/2"}, }) - summaries, err := repos.Uploads.ListDataSetSummaries(ctx, bucket.ID) + summaries, err := repos.Contents.ListDataSetSummaries(ctx, bucket.ID) if err != nil { t.Fatalf("ListDataSetSummaries: %v", err) } @@ -1112,21 +1202,12 @@ func TestAPIBucketDetail_DataSetStorageHealthQueryFailureReturnsUnknownPlacehold srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "detail-datasets-storage-health-error", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "detail-datasets-storage-health-error", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J000000000000000AVAILERR", - ContentSize: 1, - Checksum: "checksum-dataset-storage-health-error", - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - seedAdminCommittedCopies(t, repos, bucket.ID, upload.ID, "bafk2bzacestoragehealtherror", []adminStorageCopySeed{ + upload := insertAdminStorageContentSnapshot(t, srv.db, bucket.ID, "01J000000000000000AVAILERR", 1, "checksum-dataset-storage-health-error", 1) + seedAdminCommittedCopies(t, srv.db, repos, bucket.ID, upload.ID, "bafk2bzacestoragehealtherror", []adminStorageCopySeed{ {ProviderID: onChainID(t, "101"), DataSetID: onChainID(t, "1001"), PieceID: onChainIDPtr(t, "2001"), TransferMethod: model.StorageCopyTransferMethodIngress, RetrievalURL: "https://provider.example/1"}, }) srv.WithObservability(observability.NewService(observability.ServiceOptions{ @@ -1207,26 +1288,18 @@ func TestAPIBucketsStorageHealthSummaryUsesRetainedVersionRisk(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "storage-health-summary-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "storage-health-summary-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - seedAdminObjectVersion(t, repos, bucket, "stored.txt", 4, "etag-storage-health", "checksum-storage-health", "text/plain", "", model.ObjectStateStored) - unreferencedUpload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J0000000000000000APISH0", - ContentSize: 1, - Checksum: "checksum-unreferenced-storage-health", - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt unreferenced: %v", err) - } - if _, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "202"), - CopyIndex: 1, - CreatedByUploadID: unreferencedUpload.ID, + testutil.OpenBucketReplicaSlots(t, srv.db, bucket.ID, 2) + seedAdminObjectVersion(t, srv.db, repos, bucket, "stored.txt", 4, "etag-storage-health", "checksum-storage-health", "text/plain", model.ObjectStateStored) + unreferencedUpload := insertAdminStorageContentSnapshot(t, srv.db, bucket.ID, "01J0000000000000000APISH0", 1, "checksum-unreferenced-storage-health", 1) + if _, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, + ProviderID: onChainID(t, "202"), + CopyIndex: 1, + CreatedByContentID: unreferencedUpload.ID, }); err != nil { t.Fatalf("EnsureDataSetBinding unreferenced: %v", err) } @@ -1329,7 +1402,7 @@ func TestCopyHealthSummaryIgnoresExtraCopyRowsAfterRequestedCopiesMet(t *testing checkedAt := time.Now().UTC() staleCheckedAt := checkedAt.Add(-3 * time.Hour) bucketID := int64(1) - uploadID := int64(10) + contentID := int64(10) requestedCopies := 2 versionID := "01J0000000000000000COPYOK" firstLocalDataSetID := int64(101) @@ -1347,8 +1420,8 @@ func TestCopyHealthSummaryIgnoresExtraCopyRowsAfterRequestedCopiesMet(t *testing firstRetrievalURL := "https://provider.example/one" secondRetrievalURL := "https://provider.example/two" staleRetrievalURL := "https://provider.example/stale" - failedStatus := model.StorageUploadCopyStatusFailed - committedStatus := model.StorageUploadCopyStatusCommitted + failedStatus := model.StorageCopyStatusFailed + committedStatus := model.StorageCopyStatusCommitted failedCopyIndex := 2 staleCopyIndex := 3 firstCopyIndex := 0 @@ -1358,7 +1431,7 @@ func TestCopyHealthSummaryIgnoresExtraCopyRowsAfterRequestedCopiesMet(t *testing { BucketID: bucketID, VersionID: versionID, - UploadID: &uploadID, + ContentID: &contentID, RequestedCopies: requestedCopies, CopyIndex: &firstCopyIndex, CopyStatus: &committedStatus, @@ -1371,7 +1444,7 @@ func TestCopyHealthSummaryIgnoresExtraCopyRowsAfterRequestedCopiesMet(t *testing { BucketID: bucketID, VersionID: versionID, - UploadID: &uploadID, + ContentID: &contentID, RequestedCopies: requestedCopies, CopyIndex: &secondCopyIndex, CopyStatus: &committedStatus, @@ -1384,7 +1457,7 @@ func TestCopyHealthSummaryIgnoresExtraCopyRowsAfterRequestedCopiesMet(t *testing { BucketID: bucketID, VersionID: versionID, - UploadID: &uploadID, + ContentID: &contentID, RequestedCopies: requestedCopies, CopyIndex: &failedCopyIndex, CopyStatus: &failedStatus, @@ -1392,7 +1465,7 @@ func TestCopyHealthSummaryIgnoresExtraCopyRowsAfterRequestedCopiesMet(t *testing { BucketID: bucketID, VersionID: versionID, - UploadID: &uploadID, + ContentID: &contentID, RequestedCopies: requestedCopies, CopyIndex: &staleCopyIndex, CopyStatus: &committedStatus, @@ -1426,7 +1499,7 @@ func TestCopyHealthSummaryIgnoresExtraCopyRowsAfterRequestedCopiesMet(t *testing func TestCopyHealthSummaryTreatsUnobservedCommittedCopyAsNotVerified(t *testing.T) { bucketID := int64(1) - uploadID := int64(10) + contentID := int64(10) requestedCopies := 1 versionID := "01J0000000000000000COPYNV" localDataSetID := int64(101) @@ -1435,12 +1508,12 @@ func TestCopyHealthSummaryTreatsUnobservedCommittedCopyAsNotVerified(t *testing. pieceID := idtypes.NewOnChainID(3001) retrievalURL := "https://provider.example/unverified" copyIndex := 0 - committedStatus := model.StorageUploadCopyStatusCommitted + committedStatus := model.StorageCopyStatusCommitted summaries := copyHealthSummariesByBucket([]copyHealthFact{{ BucketID: bucketID, VersionID: versionID, - UploadID: &uploadID, + ContentID: &contentID, RequestedCopies: requestedCopies, CopyIndex: ©Index, CopyStatus: &committedStatus, @@ -1464,79 +1537,15 @@ func TestCopyHealthSummaryTreatsUnobservedCommittedCopyAsNotVerified(t *testing. } } -func TestCopyHealthSummaryClassifiesUploadGapsBySeverity(t *testing.T) { - bucketID := int64(1) - baseUploadID := int64(10) - requestedCopies := 3 - - tests := []struct { - name string - uploadStatus model.StorageUploadStatus - wantStatus observability.Status - wantPending int - wantFailed int - wantUnknown int - wantReason observability.ReasonCode - }{ - { - name: "running upload gaps are pending", - uploadStatus: model.StorageUploadStatusRunning, - wantStatus: observability.StatusDegraded, - wantPending: requestedCopies, - wantReason: observability.ReasonCopyPending, - }, - { - name: "failed upload gaps are unavailable", - uploadStatus: model.StorageUploadStatusFailed, - wantStatus: observability.StatusUnavailable, - wantFailed: requestedCopies, - wantReason: observability.ReasonCopyFailed, - }, - { - name: "complete upload gaps are unknown", - uploadStatus: model.StorageUploadStatusComplete, - wantStatus: observability.StatusUnknown, - wantUnknown: requestedCopies, - wantReason: observability.ReasonCopyObservationMissing, - }, - } - - for i, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - uploadID := baseUploadID + int64(i) - summaries := copyHealthSummariesByBucket([]copyHealthFact{{ - BucketID: bucketID, - VersionID: fmt.Sprintf("01J0000000000000000GAP%03d", i), - UploadID: &uploadID, - UploadStatus: &tt.uploadStatus, - RequestedCopies: requestedCopies, - }}, nil, false, time.Hour) - - health := summaries[bucketID] - if health.Status != string(tt.wantStatus) || - health.TotalObjects != 1 || - health.UnhealthyObjects != 1 || - health.RequestedCopies != requestedCopies || - health.PendingCopies != tt.wantPending || - health.FailedCopies != tt.wantFailed || - health.UnknownCopies != tt.wantUnknown || - !hasReason(health.ReasonCodes, observability.ReasonCopyUnderReplicated) || - !hasReason(health.ReasonCodes, tt.wantReason) { - t.Fatalf("copy_health = %#v, want %s gaps classified by upload status", health, tt.wantStatus) - } - }) - } -} - func TestAPIBucketsCopyHealthObservabilityFailureReturnsUnknown(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "copy-health-observability-error", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "copy-health-observability-error", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - _, versionID := seedAdminObjectVersion(t, repos, bucket, "file.txt", 1, "etag-copy-health-error", "checksum-copy-health-error", "text/plain", "", model.ObjectStateStored) + _, versionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "file.txt", 1, "etag-copy-health-error", "checksum-copy-health-error", "text/plain", model.ObjectStateStored) srv.WithObservability(observability.NewService(observability.ServiceOptions{ Store: &bucketStorageHealthErrorStore{err: errors.New("database password leaked copy health")}, })) @@ -1576,11 +1585,11 @@ func TestAPIBucketObjectProvenanceUsesStoredCopyHealthObservations(t *testing.T) srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "stored-copy-health-observation", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "stored-copy-health-observation", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - _, versionID := seedAdminObjectVersion(t, repos, bucket, "file.txt", 1, "etag-stored-copy-health", "checksum-stored-copy-health", "text/plain", "", model.ObjectStateStored) + _, versionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "file.txt", 1, "etag-stored-copy-health", "checksum-stored-copy-health", "text/plain", model.ObjectStateStored) replaceBucketDataSetObservability(t, repos, bucket.ID, observability.StatusAvailable, nil, time.Now().UTC()) ts := httptest.NewServer(newBucketAPIMux(srv)) @@ -1620,11 +1629,11 @@ func TestAPIBucketsStorageHealthSummaryFailureReturnsUnknown(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "storage-health-summary-error", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "storage-health-summary-error", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - repos.Uploads = failingBucketStorageHealthUploadRepo{StorageUploadRepository: repos.Uploads} + repos.Contents = failingBucketStorageHealthUploadRepo{StorageContentRepository: repos.Contents} ts := httptest.NewServer(newBucketAPIMux(srv)) defer ts.Close() @@ -1683,11 +1692,11 @@ func TestAPIBucketsStorageHealthNoUploadObjectHasNoDataRisk(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "storage-health-no-upload", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "storage-health-no-upload", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - _, versionID := seedAdminObjectVersion(t, repos, bucket, "cached.txt", 1, "etag-no-upload-health", "checksum-no-upload-health", "text/plain", "", model.ObjectStateCached) + _, versionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "cached.txt", 1, "etag-no-upload-health", "checksum-no-upload-health", "text/plain", model.ObjectStateCached) ts := httptest.NewServer(newBucketAPIMux(srv)) defer ts.Close() @@ -1740,687 +1749,799 @@ func TestAPIBucketsStorageHealthNoUploadObjectHasNoDataRisk(t *testing.T) { if err := json.NewDecoder(resp.Body).Decode(&provenanceBody); err != nil { t.Fatalf("Decode provenance: %v", err) } + // A data version always has content now, so the object is under-replicated + // against that content's target rather than having no durability target. if provenanceBody.CopyHealth.Status != string(observability.StatusUnknown) || provenanceBody.CopyHealth.TotalObjects != 1 || provenanceBody.CopyHealth.UnhealthyObjects != 1 || - provenanceBody.CopyHealth.RequestedCopies != 0 || - provenanceBody.CopyHealth.UnknownCopies != 0 || + provenanceBody.CopyHealth.RequestedCopies != 1 || + provenanceBody.CopyHealth.ReadableCopies != 0 || !hasReason(provenanceBody.CopyHealth.ReasonCodes, observability.ReasonCopyObservationMissing) { - t.Fatalf("provenance copy_health = %#v, want no-upload object marked unknown without virtual copies", provenanceBody.CopyHealth) + t.Fatalf("provenance copy_health = %#v, want an un-ingested object marked unknown against its content target", provenanceBody.CopyHealth) } } -func TestAPIBucketStorageHealthAffectedVersionsRequiresStableCutoff(t *testing.T) { - srv, repos := newBucketAPITestServer(t) - ctx := context.Background() - bucket := &model.Bucket{Name: "storage-risk-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create: %v", err) - } - otherBucket := &model.Bucket{Name: "storage-risk-other-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, otherBucket); err != nil { - t.Fatalf("Other Buckets.Create: %v", err) - } +func TestBucketStorageHealthAffectedVersionsResponseDeduplicatesProviderLookups(t *testing.T) { + providerID := onChainID(t, "501") + identityResolver := &fakeAPIProviderIdentityResolver{ + identities: map[string]*providerIdentityResponse{ + "501": {RegistryProviderID: "501", Name: "risk-pdp"}, + }, + } + srv := &Server{providerIdentity: identityResolver} + createdAt := time.Date(2026, 5, 23, 12, 30, 0, 123456789, time.UTC) + status := observability.StatusUnavailable - currentVersion := &model.ObjectVersion{ - VersionID: "01J000000000000000APIR01", - BucketID: bucket.ID, - Key: "docs/current.txt", - Size: 4, - ETag: "etag-current-risk", - Checksum: "checksum-current-risk", - ContentType: "text/plain", - CacheKey: ".versions/current-risk", - State: model.ObjectStateCached, + resp := srv.bucketStorageHealthAffectedVersionsResponse(repository.BucketStorageHealthAffectedVersionPage{ + Versions: []repository.BucketStorageHealthAffectedVersion{{ + Version: model.ObjectVersion{ + VersionID: "01J000000000000000APIR99", + Key: "docs/current.txt", + CreatedAt: createdAt, + UpdatedAt: createdAt, + }, + RiskDataSets: []repository.BucketStorageHealthRiskDataSet{ + {LocalDataSetID: 1, ProviderID: providerID, LocalStatus: model.StorageDataSetStatusReady, ObservationStatus: &status}, + {LocalDataSetID: 2, ProviderID: providerID, LocalStatus: model.StorageDataSetStatusReady, ObservationStatus: &status}, + }, + }}, + }, createdAt.Add(-time.Hour)) + + if got, want := identityResolver.requests, [][]string{{"501"}}; !reflect.DeepEqual(got, want) { + t.Fatalf("provider identity requests = %#v, want %#v", got, want) } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, currentVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent current: %v", err) + if len(resp.Versions) != 1 || len(resp.Versions[0].RiskDataSets) != 2 { + t.Fatalf("response risk datasets = %#v, want two datasets", resp.Versions) } - currentUpload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: currentVersion.VersionID, - ContentSize: currentVersion.Size, - Checksum: currentVersion.Checksum, - RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt current: %v", err) + for _, dataSet := range resp.Versions[0].RiskDataSets { + if dataSet.ProviderIdentity == nil || dataSet.ProviderIdentity.Name != "risk-pdp" { + t.Fatalf("provider identity = %#v, want reused identity", dataSet.ProviderIdentity) + } } - seedAdminCommittedCopies(t, repos, bucket.ID, currentUpload.ID, "bafk2bzacestorageriskcurrent", []adminStorageCopySeed{ - {CopyIndex: 0, ProviderID: onChainID(t, "501"), DataSetID: onChainID(t, "9501"), PieceID: onChainIDPtr(t, "9901"), TransferMethod: model.StorageCopyTransferMethodIngress, RetrievalURL: "https://provider.example/current-risk"}, - {CopyIndex: 1, ProviderID: onChainID(t, "502"), DataSetID: onChainID(t, "9502"), PieceID: onChainIDPtr(t, "9902"), TransferMethod: model.StorageCopyTransferMethodPeerPull, RetrievalURL: "https://provider.example/current-readable"}, +} + +func TestBucketStorageHealthSummaryFromRepositoryFormatsLastCheckedAtUTC(t *testing.T) { + checkedAt := time.Date(2026, 5, 23, 20, 0, 0, 0, time.FixedZone("UTC+8", 8*60*60)) + got := bucketStorageHealthSummaryFromRepository(repository.BucketStorageHealthSummary{ + BucketID: 1, + ReasonCodes: []observability.ReasonCode{}, + AffectedVersionsCap: 200, + AffectedVersionsCapped: 0, + LastCheckedAt: &checkedAt, }) - if err := repos.Objects.UpdateVersionState(ctx, currentVersion.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("UpdateVersionState current uploading: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForVersion(ctx, repository.BindReadableUploadForVersionInput{ - UploadID: currentUpload.ID, - BucketID: bucket.ID, - ContentSize: currentVersion.Size, - Checksum: currentVersion.Checksum, - VersionID: currentVersion.VersionID, - }); err != nil { - t.Fatalf("BindReadableUploadForVersion current: %v", err) + if got.LastCheckedAt != "2026-05-23T12:00:00Z" { + t.Fatalf("last_checked_at = %q, want UTC RFC3339", got.LastCheckedAt) } +} - oldVersion := &model.ObjectVersion{ - VersionID: "01J000000000000000APIR02", - BucketID: bucket.ID, - Key: "docs/old.txt", - Size: 5, - ETag: "etag-old-risk", - Checksum: "checksum-old-risk", - ContentType: "text/plain", - CacheKey: ".versions/old-risk", - State: model.ObjectStateCached, - } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent old: %v", err) +func TestAPIBucketsCopyHealthObservabilityFailurePreservesLocalCopyStatus(t *testing.T) { + srv, repos := newBucketAPITestServer(t) + ctx := context.Background() + + partialCopies := 2 + bucket := &model.Bucket{Name: "copy-health-observability-local-status", Status: model.BucketStatusActive, DefaultCopies: partialCopies, MinimumDurableCopies: partialCopies} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) } - oldUpload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: oldVersion.VersionID, - ContentSize: oldVersion.Size, - Checksum: oldVersion.Checksum, - RequestedCopies: 1, - }) + _, versionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "partial.txt", 1, "etag-copy-health-local-status", "checksum-copy-health-local-status", "text/plain", model.ObjectStateCached) + bindAdminPartialUpload(t, srv.db, repos, versionID) + srv.WithObservability(observability.NewService(observability.ServiceOptions{ + Store: &bucketStorageHealthErrorStore{err: errors.New("database password leaked copy health")}, + })) + + ts := httptest.NewServer(newBucketAPIMux(srv)) + defer ts.Close() + + resp, err := http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/objects/provenance?version_id=" + url.QueryEscape(versionID)) if err != nil { - t.Fatalf("StartObjectUploadAttempt old: %v", err) - } - seedAdminCommittedCopies(t, repos, bucket.ID, oldUpload.ID, "bafk2bzacestorageriskold", []adminStorageCopySeed{ - {CopyIndex: 2, ProviderID: onChainID(t, "503"), DataSetID: onChainID(t, "9503"), PieceID: onChainIDPtr(t, "9903"), TransferMethod: model.StorageCopyTransferMethodIngress, RetrievalURL: "https://provider.example/old-risk"}, - }) - if err := repos.Objects.UpdateVersionState(ctx, oldVersion.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("UpdateVersionState old uploading: %v", err) + t.Fatalf("GET provenance: %v", err) } - if _, err := repos.Uploads.BindReadableUploadForVersion(ctx, repository.BindReadableUploadForVersionInput{ - UploadID: oldUpload.ID, - BucketID: bucket.ID, - ContentSize: oldVersion.Size, - Checksum: oldVersion.Checksum, - VersionID: oldVersion.VersionID, - }); err != nil { - t.Fatalf("BindReadableUploadForVersion old: %v", err) + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) } - if err := repos.Objects.SetVersionCachePresence(ctx, oldVersion.VersionID, false); err != nil { - t.Fatalf("SetVersionCachePresence old: %v", err) + var body struct { + CopyHealth apiCopyHealthBody `json:"copy_health"` } - replacement := &model.ObjectVersion{ - VersionID: "01J000000000000000APIR03", - BucketID: bucket.ID, - Key: oldVersion.Key, - Size: 6, - ETag: "etag-old-replacement", - Checksum: "checksum-old-replacement", - ContentType: "text/plain", - CacheKey: ".versions/old-replacement", - State: model.ObjectStateCached, + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("Decode: %v", err) } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, replacement); err != nil { - t.Fatalf("CreateVersionAndSetCurrent replacement: %v", err) + if body.CopyHealth.Status != string(observability.StatusUnavailable) || + body.CopyHealth.FailedCopies != 1 || + body.CopyHealth.UnknownCopies != 1 || + body.CopyHealth.PendingCopies != 0 || + body.CopyHealth.LastError != "copy health query failed" || + !hasReason(body.CopyHealth.ReasonCodes, observability.ReasonCopyFailed) { + t.Fatalf("copy_health = %#v, want failed local copy preserved with committed copy unknown", body.CopyHealth) } +} - summaries, err := repos.Uploads.ListDataSetSummaries(ctx, bucket.ID) +func replaceBucketDataSetObservability(t *testing.T, repos *repository.Repositories, bucketID int64, status observability.Status, reasons []observability.ReasonCode, checkedAt time.Time) { + t.Helper() + summaries, err := repos.Contents.ListDataSetSummaries(context.Background(), bucketID) if err != nil { t.Fatalf("ListDataSetSummaries: %v", err) } - byCopyIndex := make(map[int]repository.StorageDataSetSummary) + states := make([]observability.DataSetState, 0, len(summaries)) for _, summary := range summaries { - byCopyIndex[summary.CopyIndex] = summary - } - checkedAt := time.Now().UTC().Truncate(time.Microsecond) - if checkedAt.Nanosecond() == 0 { - checkedAt = checkedAt.Add(time.Microsecond) + states = append(states, observability.DataSetState{ + LocalDataSetID: summary.ID, + BucketID: summary.BucketID, + BucketName: summary.BucketName, + CopyIndex: summary.CopyIndex, + ProviderID: summary.ProviderID, + ChainDataSetID: summary.DataSetID, + ClientDataSetID: summary.ClientDataSetID, + LocalStatus: summary.Status, + Status: status, + ReasonCodes: reasons, + LastCheckedAt: checkedAt, + Evidence: map[string]any{}, + }) } - if err := repos.Observability.ReplaceDataSetStates(ctx, checkedAt, []observability.DataSetState{ - {LocalDataSetID: byCopyIndex[0].ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: 0, ProviderID: byCopyIndex[0].ProviderID, ChainDataSetID: byCopyIndex[0].DataSetID, Status: observability.StatusUnavailable, ReasonCodes: []observability.ReasonCode{observability.ReasonChainDataSetMissing}, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: byCopyIndex[1].ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: 1, ProviderID: byCopyIndex[1].ProviderID, ChainDataSetID: byCopyIndex[1].DataSetID, Status: observability.StatusAvailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: byCopyIndex[2].ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: 2, ProviderID: byCopyIndex[2].ProviderID, ChainDataSetID: byCopyIndex[2].DataSetID, Status: observability.StatusDegraded, ReasonCodes: []observability.ReasonCode{observability.ReasonChainDataSetUnmanaged}, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - }); err != nil { + if err := repos.Observability.ReplaceDataSetStates(context.Background(), checkedAt, states); err != nil { t.Fatalf("ReplaceDataSetStates: %v", err) } +} + +func copyHealthDataSetObservationForTest(localID int64, checkedAt time.Time) observability.DataSetObservation { + return observability.DataSetObservationFromState(observability.DataSetState{ + LocalDataSetID: localID, + Status: observability.StatusAvailable, + LastCheckedAt: checkedAt, + }, time.Hour, checkedAt) +} + +func assertUnavailableBucketStorageHealth(t *testing.T, health apiBucketStorageHealthBody, wantReason observability.ReasonCode) { + t.Helper() + if health.Status != string(observability.StatusUnavailable) || + health.AbnormalDataSets != 2 || + health.AffectedVersionsCapped != 1 || + health.AffectedVersionsCap != 200 || + health.AffectedVersionsExceedsCap || + !hasReason(health.ReasonCodes, wantReason) || + hasReason(health.ReasonCodes, observability.ReasonChainDataSetInactive) { + t.Fatalf("storage_health = %#v, want unavailable storage source risk from one retained version", health) + } +} + +func assertNoLegacyBucketStorageHealthFields(t *testing.T, raw map[string]any) { + t.Helper() + health, ok := raw["storage_health"].(map[string]any) + if !ok { + t.Fatalf("storage_health missing or wrong shape: %#v", raw["storage_health"]) + } + for _, field := range []string{ + "affected_data_sets", + "affected_objects", + "objects_with_readable_copy", + "objects_without_readable_copy", + "unavailable_objects", + "unknown_objects", + } { + if _, ok := health[field]; ok { + t.Fatalf("storage_health includes legacy field %q: %#v", field, health) + } + } +} + +func hasReason(reasons []observability.ReasonCode, want observability.ReasonCode) bool { + return slices.Contains(reasons, want) +} + +type bucketStorageHealthErrorStore struct { + err error +} + +func (s *bucketStorageHealthErrorStore) ReplaceProviderStates(context.Context, time.Time, []observability.ProviderState) error { + return nil +} + +func (s *bucketStorageHealthErrorStore) ListProviderStates(context.Context, observability.ListOptions) (observability.ProviderStatePage, error) { + return observability.ProviderStatePage{}, nil +} + +func (s *bucketStorageHealthErrorStore) ReplaceDataSetStates(context.Context, time.Time, []observability.DataSetState) error { + return nil +} + +func (s *bucketStorageHealthErrorStore) ListDataSetStates(context.Context, observability.ListOptions) (observability.DataSetStatePage, error) { + return observability.DataSetStatePage{}, nil +} + +func (s *bucketStorageHealthErrorStore) GetDataSetStatesByLocalIDs(context.Context, []int64) (map[int64]observability.DataSetState, error) { + return nil, s.err +} + +func TestAPIBuckets_ListAllBuckets(t *testing.T) { + srv, repos := newBucketAPITestServer(t) + ctx := context.Background() + + for _, bucket := range []*model.Bucket{ + {Name: "alpha-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1}, + {Name: "beta-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1}, + {Name: "gamma-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1}, + } { + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create(%s): %v", bucket.Name, err) + } + } ts := httptest.NewServer(newBucketAPIMux(srv)) defer ts.Close() - resp, err := http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?prefix=" + url.QueryEscape("docs/") + "&limit=1") + resp, err := http.Get(ts.URL + "/api/v1/buckets") if err != nil { - t.Fatalf("GET affected versions: %v", err) + t.Fatalf("GET buckets: %v", err) } defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) } - var body struct { - CopyHealth json.RawMessage `json:"copy_health"` - Versions []struct { - Key string `json:"key"` - VersionID string `json:"version_id"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` - IsCurrent bool `json:"is_current"` - InCache bool `json:"in_cache"` - ReadableAlternativeCount int `json:"readable_alternative_count"` - HasReadableAlternative bool `json:"has_readable_alternative"` - CopyHealth json.RawMessage `json:"copy_health"` - RiskDataSets []struct { - LocalDataSetID int64 `json:"local_data_set_id"` - CopyIndex int `json:"copy_index"` - ProviderID string `json:"provider_id"` - DataSetID string `json:"data_set_id"` - LocalStatus string `json:"local_status"` - StorageHealth struct { - Status string `json:"status"` - ReasonCodes []observability.ReasonCode `json:"reason_codes"` - LastChecked string `json:"last_checked_at"` - } `json:"storage_health"` - } `json:"risk_data_sets"` - } `json:"versions"` - HasMore bool `json:"has_more"` - NextKeyMarker string `json:"next_key_marker"` - NextVersionMarker string `json:"next_version_marker"` - NextCreatedAt string `json:"next_created_at_marker"` - StaleBefore string `json:"stale_before"` + + var body []struct { + Name string `json:"name"` + Status string `json:"status"` } if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { t.Fatalf("Decode: %v", err) } - if body.CopyHealth != nil { - t.Fatalf("response copy_health = %s, want omitted", body.CopyHealth) + + got := make(map[string]string, len(body)) + for _, item := range body { + got[item.Name] = item.Status } - if len(body.Versions) != 1 || body.Versions[0].VersionID != currentVersion.VersionID || !body.Versions[0].IsCurrent { - t.Fatalf("first page versions = %#v, want current risk version first", body.Versions) + + for _, name := range []string{"alpha-bucket", "beta-bucket", "gamma-bucket"} { + if got[name] != string(model.BucketStatusActive) { + t.Fatalf("%s status = %q, want %q", name, got[name], model.BucketStatusActive) + } } - if !body.Versions[0].InCache { - t.Fatalf("current in_cache = false, want true") +} + +func TestAPIBuckets_ListAndDetailIncludeOwnerAccessKey(t *testing.T) { + srv, repos := newBucketAPITestServerWithRuntimeCopies(t, 5) + ctx := context.Background() + + ownedACL, err := json.Marshal(auth.ACL{Owner: "owner-access"}) + if err != nil { + t.Fatalf("Marshal ACL: %v", err) } - if _, err := time.Parse(time.RFC3339Nano, body.Versions[0].CreatedAt); err != nil { - t.Fatalf("created_at = %q, want RFC3339Nano: %v", body.Versions[0].CreatedAt, err) + ownerAccess := "owner-access" + if err := repos.S3Accounts.Create(ctx, &model.S3Account{AccessKey: ownerAccess, SecretKey: "owner-secret", Role: auth.RoleUserPlus}); err != nil { + t.Fatalf("S3Accounts.Create: %v", err) } - if _, err := time.Parse(time.RFC3339Nano, body.Versions[0].UpdatedAt); err != nil { - t.Fatalf("updated_at = %q, want RFC3339Nano: %v", body.Versions[0].UpdatedAt, err) + overrideCopies := 3 + for _, bucket := range []*model.Bucket{ + {Name: "owned-bucket", Status: model.BucketStatusActive, ACL: ownedACL, OwnerAccessKey: &ownerAccess, DefaultCopies: overrideCopies, MinimumDurableCopies: overrideCopies}, + {Name: "unassigned-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1}, + } { + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create(%s): %v", bucket.Name, err) + } } - if body.Versions[0].CopyHealth != nil { - t.Fatalf("version copy_health = %s, want omitted", body.Versions[0].CopyHealth) + + ts := httptest.NewServer(newBucketAPIMux(srv)) + defer ts.Close() + + resp, err := http.Get(ts.URL + "/api/v1/buckets") + if err != nil { + t.Fatalf("GET buckets: %v", err) } - if !body.Versions[0].HasReadableAlternative || body.Versions[0].ReadableAlternativeCount != 1 { - t.Fatalf("current alternative = %#v, want one recorded readable alternative", body.Versions[0]) + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Fatalf("list status = %d, want %d", resp.StatusCode, http.StatusOK) } - if len(body.Versions[0].RiskDataSets) != 1 || body.Versions[0].RiskDataSets[0].LocalDataSetID != byCopyIndex[0].ID || body.Versions[0].RiskDataSets[0].StorageHealth.Status != string(observability.StatusUnavailable) { - t.Fatalf("current risk datasets = %#v, want unavailable dataset", body.Versions[0].RiskDataSets) + var listBody []struct { + Name string `json:"name"` + OwnerAccessKey *string `json:"owner_access_key"` + DefaultCopies int `json:"default_copies"` } - if !body.HasMore || body.NextKeyMarker != currentVersion.Key || body.NextVersionMarker != currentVersion.VersionID || body.NextCreatedAt == "" { - t.Fatalf("pagination = %#v, want next marker after current", body) + if err := json.NewDecoder(resp.Body).Decode(&listBody); err != nil { + t.Fatalf("Decode list: %v", err) } - if _, err := time.Parse(time.RFC3339Nano, body.NextCreatedAt); err != nil { - t.Fatalf("next_created_at_marker = %q, want RFC3339Nano: %v", body.NextCreatedAt, err) + owners := make(map[string]*string, len(listBody)) + copyPolicies := make(map[string]int, len(listBody)) + for _, item := range listBody { + owners[item.Name] = item.OwnerAccessKey + copyPolicies[item.Name] = item.DefaultCopies } - if _, err := time.Parse(time.RFC3339Nano, body.StaleBefore); err != nil { - t.Fatalf("stale_before = %q, want RFC3339Nano: %v", body.StaleBefore, err) + if owners["owned-bucket"] == nil || *owners["owned-bucket"] != "owner-access" { + t.Fatalf("owned-bucket owner = %v, want owner-access", owners["owned-bucket"]) } - if got, want := body.Versions[0].RiskDataSets[0].StorageHealth.LastChecked, checkedAt.Format(time.RFC3339Nano); got != want { - t.Fatalf("risk dataset last_checked_at = %q, want %q", got, want) + if owners["unassigned-bucket"] != nil { + t.Fatalf("unassigned-bucket owner = %v, want nil", *owners["unassigned-bucket"]) + } + if copyPolicies["owned-bucket"] != 3 { + t.Fatalf("owned bucket copy policy = %d, want 3", copyPolicies["owned-bucket"]) + } + if copyPolicies["unassigned-bucket"] != 1 { + t.Fatalf("unassigned bucket copy policy = %d, want 1", copyPolicies["unassigned-bucket"]) } - secondPageURL := ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?key_marker=" + url.QueryEscape(body.NextKeyMarker) + "&version_marker=" + url.QueryEscape(body.NextVersionMarker) + "&created_at_marker=" + url.QueryEscape(body.NextCreatedAt) - resp, err = http.Get(secondPageURL) + detailResp, err := http.Get(ts.URL + "/api/v1/buckets/owned-bucket") if err != nil { - t.Fatalf("GET affected versions second page: %v", err) + t.Fatalf("GET bucket detail: %v", err) + } + defer func() { _ = detailResp.Body.Close() }() + if detailResp.StatusCode != http.StatusOK { + t.Fatalf("detail status = %d, want %d", detailResp.StatusCode, http.StatusOK) + } + var detailBody struct { + OwnerAccessKey *string `json:"owner_access_key"` + DefaultCopies int `json:"default_copies"` + } + if err := json.NewDecoder(detailResp.Body).Decode(&detailBody); err != nil { + t.Fatalf("Decode detail: %v", err) + } + if detailBody.OwnerAccessKey == nil || *detailBody.OwnerAccessKey != "owner-access" { + t.Fatalf("detail owner = %v, want owner-access", detailBody.OwnerAccessKey) + } + if detailBody.DefaultCopies != 3 { + t.Fatalf("detail copy policy = %d, want 3", detailBody.DefaultCopies) + } +} + +func TestAPIBucketCopyPolicy_UpdateAndClear(t *testing.T) { + srv, repos := newBucketAPITestServerWithRuntimeCopies(t, 5) + ctx := context.Background() + bucket := &model.Bucket{Name: "copy-policy-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) + } + + ts := httptest.NewServer(newBucketAPIMux(srv)) + defer ts.Close() + + setReq, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/buckets/copy-policy-bucket/copy-policy", strings.NewReader(`{"default_copies":5}`)) + if err != nil { + t.Fatalf("NewRequest set: %v", err) + } + setReq.Header.Set("Content-Type", "application/json") + setBucketWriteHeaders(setReq) + setResp, err := http.DefaultClient.Do(setReq) + if err != nil { + t.Fatalf("PUT copy policy set: %v", err) + } + defer func() { _ = setResp.Body.Close() }() + if setResp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(setResp.Body) + t.Fatalf("set status = %d, want %d body=%s", setResp.StatusCode, http.StatusOK, body) + } + var setBody struct { + Name string `json:"name"` + DefaultCopies int `json:"default_copies"` + } + if err := json.NewDecoder(setResp.Body).Decode(&setBody); err != nil { + t.Fatalf("Decode set response: %v", err) + } + if setBody.Name != bucket.Name || setBody.DefaultCopies != 5 { + t.Fatalf("set copy policy response = %#v, want target 5", setBody) + } + + clearReq, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/buckets/copy-policy-bucket/copy-policy", strings.NewReader("{\"default_copies\": \n null \t}")) + if err != nil { + t.Fatalf("NewRequest clear: %v", err) + } + clearReq.Header.Set("Content-Type", "application/json") + setBucketWriteHeaders(clearReq) + clearResp, err := http.DefaultClient.Do(clearReq) + if err != nil { + t.Fatalf("PUT copy policy clear: %v", err) + } + defer func() { _ = clearResp.Body.Close() }() + if clearResp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(clearResp.Body) + t.Fatalf("clear status = %d, want %d body=%s", clearResp.StatusCode, http.StatusOK, body) } - defer func() { _ = resp.Body.Close() }() - var missingCutoffBody struct { - Error string `json:"error"` + var clearBody struct { + DefaultCopies int `json:"default_copies"` } - if err := json.NewDecoder(resp.Body).Decode(&missingCutoffBody); err != nil { - t.Fatalf("Decode missing cutoff: %v", err) + if err := json.NewDecoder(clearResp.Body).Decode(&clearBody); err != nil { + t.Fatalf("Decode clear response: %v", err) } - if resp.StatusCode != http.StatusBadRequest || missingCutoffBody.Error != "key_marker, version_marker, created_at_marker, and stale_before must be provided together" { - t.Fatalf("second page without cutoff status/body = %d/%#v, want stable cutoff error", resp.StatusCode, missingCutoffBody) + // An explicit null resets the stored target to the configured default + // rather than parking the bucket on a null that has to be resolved later. + if clearBody.DefaultCopies != 5 { + t.Fatalf("clear copy policy response = %#v, want reset to the configured 5", clearBody) } - secondPageURL += "&stale_before=" + url.QueryEscape(body.StaleBefore) - resp, err = http.Get(secondPageURL) + // Reset resolves to a concrete number before it is stored, so a reset that + // would land below the bucket's target is a lowering like any other. + raiseReq, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/buckets/copy-policy-bucket/copy-policy", strings.NewReader(`{"default_copies":7}`)) if err != nil { - t.Fatalf("GET affected versions second page with cutoff: %v", err) - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - t.Fatalf("second page with cutoff status = %d, want %d", resp.StatusCode, http.StatusOK) + t.Fatalf("NewRequest raise: %v", err) } - - resp, err = http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?local_data_set_id=" + strconv.FormatInt(byCopyIndex[2].ID, 10)) + raiseReq.Header.Set("Content-Type", "application/json") + setBucketWriteHeaders(raiseReq) + raiseResp, err := http.DefaultClient.Do(raiseReq) if err != nil { - t.Fatalf("GET affected versions dataset: %v", err) + t.Fatalf("PUT copy policy raise: %v", err) } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - t.Fatalf("dataset status = %d, want %d", resp.StatusCode, http.StatusOK) + defer func() { _ = raiseResp.Body.Close() }() + if raiseResp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(raiseResp.Body) + t.Fatalf("raise status = %d, want %d body=%s", raiseResp.StatusCode, http.StatusOK, body) } - var datasetBody struct { - Versions []struct { - VersionID string `json:"version_id"` - IsCurrent bool `json:"is_current"` - InCache bool `json:"in_cache"` - ReadableAlternativeCount int `json:"readable_alternative_count"` - } `json:"versions"` + + resetReq, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/buckets/copy-policy-bucket/copy-policy", strings.NewReader(`{"default_copies":null}`)) + if err != nil { + t.Fatalf("NewRequest reset: %v", err) } - if err := json.NewDecoder(resp.Body).Decode(&datasetBody); err != nil { - t.Fatalf("Decode dataset: %v", err) + resetReq.Header.Set("Content-Type", "application/json") + setBucketWriteHeaders(resetReq) + resetResp, err := http.DefaultClient.Do(resetReq) + if err != nil { + t.Fatalf("PUT copy policy reset: %v", err) } - if len(datasetBody.Versions) != 1 || datasetBody.Versions[0].VersionID != oldVersion.VersionID || datasetBody.Versions[0].IsCurrent || datasetBody.Versions[0].ReadableAlternativeCount != 0 { - t.Fatalf("dataset filtered versions = %#v, want retained old version without alternative", datasetBody.Versions) + defer func() { _ = resetResp.Body.Close() }() + if resetResp.StatusCode != http.StatusBadRequest { + body, _ := io.ReadAll(resetResp.Body) + t.Fatalf("reset-below-target status = %d, want %d body=%s", resetResp.StatusCode, http.StatusBadRequest, body) } - if datasetBody.Versions[0].InCache { - t.Fatalf("dataset filtered in_cache = true, want false") + stored, err := repos.Buckets.GetByName(ctx, bucket.Name) + if err != nil || stored == nil || stored.DefaultCopies != 7 { + t.Fatalf("stored bucket after refused reset = %#v err=%v, want target still 7", stored, err) } +} - resp, err = http.Get(ts.URL + "/api/v1/buckets/" + otherBucket.Name + "/storage-health/affected-versions?local_data_set_id=" + strconv.FormatInt(byCopyIndex[0].ID, 10)) - if err != nil { - t.Fatalf("GET affected versions other bucket dataset: %v", err) - } - defer func() { _ = resp.Body.Close() }() - var emptyBody struct { - Versions []struct{} `json:"versions"` +func TestAPIBucketCopyPolicy_IndependentFieldsValidateFinalPolicyAndUseOneCoordinator(t *testing.T) { + srv, repos := newBucketAPITestServerWithRuntimeCopies(t, 5) + ctx := context.Background() + bucket := &model.Bucket{Name: "independent-copy-policy-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) } - if err := json.NewDecoder(resp.Body).Decode(&emptyBody); err != nil { - t.Fatalf("Decode other bucket dataset: %v", err) + + update := func(body string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodPut, "/api/v1/buckets/independent-copy-policy-bucket/copy-policy", strings.NewReader(body)) + req.SetPathValue("name", bucket.Name) + req.Header.Set("Content-Type", "application/json") + setBucketWriteHeaders(req) + rr := httptest.NewRecorder() + srv.handleAPIUpdateBucketCopyPolicy(rr, req) + return rr } - if resp.StatusCode != http.StatusOK || len(emptyBody.Versions) != 0 { - t.Fatalf("other bucket dataset status/body = %d/%#v, want empty OK", resp.StatusCode, emptyBody) + assertStored := func(wantTarget, wantMinimum int) { + t.Helper() + got, err := repos.Buckets.GetByName(ctx, bucket.Name) + if err != nil || got == nil { + t.Fatalf("GetByName: bucket=%#v err=%v", got, err) + } + if got.DefaultCopies != wantTarget || got.MinimumDurableCopies != wantMinimum { + t.Fatalf("stored policy = target:%d minimum:%d, want target:%d minimum:%d", got.DefaultCopies, got.MinimumDurableCopies, wantTarget, wantMinimum) + } + assertActiveBucketReplicaSlots(t, ctx, repos, bucket.ID, wantTarget) } - resp, err = http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?prefix=docs/&key=docs/current.txt") - if err != nil { - t.Fatalf("GET affected versions conflicting filters: %v", err) + if rr := update(`{"default_copies":4,"minimum_durable_copies":2}`); rr.Code != http.StatusOK { + t.Fatalf("joint update status = %d body=%s", rr.Code, rr.Body.String()) } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusBadRequest { - t.Fatalf("conflicting filter status = %d, want %d", resp.StatusCode, http.StatusBadRequest) + assertStored(4, 2) + + if rr := update(`{"minimum_durable_copies":3}`); rr.Code != http.StatusOK { + t.Fatalf("minimum-only update status = %d body=%s", rr.Code, rr.Body.String()) } + assertStored(4, 3) - resp, err = http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?local_data_set_id=bad") - if err != nil { - t.Fatalf("GET affected versions invalid dataset: %v", err) + if rr := update(`{"default_copies":6}`); rr.Code != http.StatusOK { + t.Fatalf("target-only update status = %d body=%s", rr.Code, rr.Body.String()) } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusBadRequest { - t.Fatalf("invalid dataset status = %d, want %d", resp.StatusCode, http.StatusBadRequest) + assertStored(6, 3) + + // Lowering is refused whole: neither the target nor the slots move. + if rr := update(`{"default_copies":3}`); rr.Code != http.StatusBadRequest { + t.Fatalf("target reduction status = %d body=%s", rr.Code, rr.Body.String()) } + assertStored(6, 3) - for _, query := range []string{ - "key_marker=" + url.QueryEscape(currentVersion.Key), - "key_marker=" + url.QueryEscape(currentVersion.Key) + "&version_marker=" + url.QueryEscape(currentVersion.VersionID), - } { - resp, err = http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?" + query) - if err != nil { - t.Fatalf("GET affected versions invalid marker %q: %v", query, err) + if rr := update(`{"minimum_durable_copies":null}`); rr.Code != http.StatusOK { + t.Fatalf("clear minimum status = %d body=%s", rr.Code, rr.Body.String()) + } else { + var response struct { + Minimum int `json:"minimum_durable_copies"` + Target int `json:"default_copies"` } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusBadRequest { - t.Fatalf("invalid marker %q status = %d, want %d", query, resp.StatusCode, http.StatusBadRequest) + if err := json.NewDecoder(rr.Body).Decode(&response); err != nil { + t.Fatalf("Decode clear minimum response: %v", err) + } + if response.Minimum != 6 || response.Target != 6 { + t.Fatalf("clear minimum response = %#v, want the target materialised as 6/6", response) } } - resp, err = http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?key_marker=" + url.QueryEscape(currentVersion.Key) + "&version_marker=" + url.QueryEscape(currentVersion.VersionID) + "&created_at_marker=not-a-time&stale_before=" + url.QueryEscape(body.StaleBefore)) + assertStored(6, 6) + + page, err := repos.Tasks.List(ctx, repository.TaskListFilter{ + Type: model.TaskTypeCacheReconcileDurability, + Limit: 10, + }) if err != nil { - t.Fatalf("GET affected versions invalid marker time: %v", err) + t.Fatalf("List coordinator tasks: %v", err) } - defer func() { _ = resp.Body.Close() }() - var invalidTimeBody struct { - Error string `json:"error"` + latestBucket, err := repos.Buckets.GetByID(ctx, bucket.ID) + if err != nil || latestBucket == nil { + t.Fatalf("GetByID: bucket=%#v err=%v", latestBucket, err) } - if err := json.NewDecoder(resp.Body).Decode(&invalidTimeBody); err != nil { - t.Fatalf("Decode invalid marker time: %v", err) + if len(page.Tasks) != 4 || page.Tasks[0].SubjectType == nil || *page.Tasks[0].SubjectType != "bucket" || + page.Tasks[0].SubjectKey == nil || *page.Tasks[0].SubjectKey != strconv.FormatInt(bucket.ID, 10) { + t.Fatalf("coordinator tasks=%#v, want four immutable policy generations", page.Tasks) } - if resp.StatusCode != http.StatusBadRequest || invalidTimeBody.Error != "created_at_marker must be RFC3339Nano" { - t.Fatalf("invalid marker time status/body = %d/%#v, want RFC3339Nano error", resp.StatusCode, invalidTimeBody) + if latestBucket.DurabilityGeneration != 4 || latestBucket.DurabilityTaskID == nil || *latestBucket.DurabilityTaskID != page.Tasks[0].ID { + t.Fatalf("latest bucket fence = generation:%d task:%v, want generation 4 task %d", latestBucket.DurabilityGeneration, latestBucket.DurabilityTaskID, page.Tasks[0].ID) } - resp, err = http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?stale_before=not-a-time") - if err != nil { - t.Fatalf("GET affected versions invalid stale_before: %v", err) +} + +func TestAPIBucketCopyPolicy_RejectsMinimumAboveReplicaTarget(t *testing.T) { + _, repos := newBucketAPITestServerWithRuntimeCopies(t, 2) + ctx := context.Background() + bucket := &model.Bucket{ + Name: "clamped-copy-policy-bucket", Status: model.BucketStatusActive, + DefaultCopies: 2, MinimumDurableCopies: 2, } - defer func() { _ = resp.Body.Close() }() - var invalidCutoffBody struct { - Error string `json:"error"` + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) } - if err := json.NewDecoder(resp.Body).Decode(&invalidCutoffBody); err != nil { - t.Fatalf("Decode invalid stale_before: %v", err) + // The policy is stored, so a minimum above the replica target is refused + // instead of being written and clamped on every read. + minimum := 5 + if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, SetMinimumDurableCopies: true, MinimumDurableCopies: &minimum, + }); !errors.Is(err, repository.ErrInvalidInput) { + t.Fatalf("UpdateCopyPolicy minimum above target error = %v, want ErrInvalidInput", err) } - if resp.StatusCode != http.StatusBadRequest || invalidCutoffBody.Error != "stale_before must be RFC3339Nano" { - t.Fatalf("invalid stale_before status/body = %d/%#v, want RFC3339Nano error", resp.StatusCode, invalidCutoffBody) + stored, err := repos.Buckets.GetByName(ctx, bucket.Name) + if err != nil || stored == nil || stored.MinimumDurableCopies != 2 { + t.Fatalf("stored bucket = %#v err=%v, want unchanged minimum 2", stored, err) } +} - resp, err = http.Get(ts.URL + "/api/v1/buckets/BadBucket/storage-health/affected-versions") - if err != nil { - t.Fatalf("GET affected versions invalid bucket: %v", err) +func TestAPIBucketCopyPolicy_RejectsInvalidPayloads(t *testing.T) { + srv, repos := newBucketAPITestServer(t) + ctx := context.Background() + bucket := &model.Bucket{Name: "invalid-copy-policy-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusBadRequest { - t.Fatalf("invalid bucket status = %d, want %d", resp.StatusCode, http.StatusBadRequest) + + for _, tc := range []struct { + name string + body string + }{ + {name: "missing field", body: `{}`}, + {name: "unknown field", body: `{"default_copies":3,"extra":true}`}, + {name: "string copies", body: `{"default_copies":"3"}`}, + {name: "fractional copies", body: `{"default_copies":3.5}`}, + {name: "zero copies", body: `{"default_copies":0}`}, + {name: "too many copies", body: `{"default_copies":9}`}, + {name: "zero minimum", body: `{"minimum_durable_copies":0}`}, + {name: "too many minimum", body: `{"minimum_durable_copies":9}`}, + {name: "minimum exceeds target", body: `{"default_copies":2,"minimum_durable_copies":3}`}, + } { + t.Run(tc.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPut, "/api/v1/buckets/invalid-copy-policy-bucket/copy-policy", strings.NewReader(tc.body)) + req.SetPathValue("name", bucket.Name) + req.Header.Set("Content-Type", "application/json") + setBucketWriteHeaders(req) + rr := httptest.NewRecorder() + + srv.handleAPIUpdateBucketCopyPolicy(rr, req) + + if rr.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d body=%s", rr.Code, http.StatusBadRequest, rr.Body.String()) + } + }) } +} - resp, err = http.Get(ts.URL + "/api/v1/buckets/missing-storage-risk-bucket/storage-health/affected-versions") +func TestAPIBucketOwner_UpdateAssignsExistingS3User(t *testing.T) { + srv, repos := newBucketAPITestServerWithS3UsersAndRuntimeCopies(t, 4, "owner-access") + ctx := context.Background() + bucket := &model.Bucket{Name: "assign-owner-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) + } + + req := httptest.NewRequest(http.MethodPut, "/api/v1/buckets/assign-owner-bucket/owner", strings.NewReader(`{"owner_access_key":"owner-access"}`)) + req.SetPathValue("name", bucket.Name) + req.Header.Set("Content-Type", "application/json") + setBucketWriteHeaders(req) + rr := httptest.NewRecorder() + + srv.handleAPIUpdateBucketOwner(rr, req) + + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusOK, rr.Body.String()) + } + updated, err := repos.Buckets.GetByName(ctx, bucket.Name) if err != nil { - t.Fatalf("GET affected versions missing bucket: %v", err) + t.Fatalf("GetByName: %v", err) } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusNotFound { - t.Fatalf("missing bucket status = %d, want %d", resp.StatusCode, http.StatusNotFound) + acl, err := auth.ParseACL(updated.ACL) + if err != nil { + t.Fatalf("ParseACL: %v", err) + } + if acl.Owner != "owner-access" { + t.Fatalf("owner = %q, want owner-access", acl.Owner) + } + var body struct { + OwnerAccessKey *string `json:"owner_access_key"` + DefaultCopies int `json:"default_copies"` + } + if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { + t.Fatalf("Decode response: %v", err) + } + if body.OwnerAccessKey == nil || *body.OwnerAccessKey != "owner-access" { + t.Fatalf("response owner = %v, want owner-access", body.OwnerAccessKey) + } + if body.DefaultCopies != 1 { + t.Fatalf("owner update copy policy = %d, want the bucket's stored 1", body.DefaultCopies) } } -func TestAPIBucketStorageHealthAffectedVersionsClampsLimit(t *testing.T) { - srv, repos := newBucketAPITestServer(t) +func TestAPIBucketOwner_UpdateAllowsInternalRootOwner(t *testing.T) { + srv, repos := newBucketAPITestServerWithS3Users(t, "owner-access") ctx := context.Background() - bucket := &model.Bucket{Name: "storage-risk-limit-bucket", Status: model.BucketStatusActive} + owner := "owner-access" + acl, err := json.Marshal(auth.ACL{Owner: owner}) + if err != nil { + t.Fatalf("Marshal ACL: %v", err) + } + bucket := &model.Bucket{Name: "root-transfer-bucket", Status: model.BucketStatusActive, OwnerAccessKey: &owner, ACL: acl, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - for i := range 51 { - version := &model.ObjectVersion{ - VersionID: fmt.Sprintf("01J000000000000LIMIT%03d", i), - BucketID: bucket.ID, - Key: fmt.Sprintf("docs/limit-%03d.txt", i), - Size: int64(i + 1), - ETag: fmt.Sprintf("etag-limit-%03d", i), - Checksum: fmt.Sprintf("checksum-limit-%03d", i), - ContentType: "text/plain", - CacheKey: fmt.Sprintf(".versions/limit-%03d", i), - State: model.ObjectStateCached, - } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent %d: %v", i, err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt %d: %v", i, err) - } - seedAdminCommittedCopies(t, repos, bucket.ID, upload.ID, fmt.Sprintf("bafk2bzacelimit%03d", i), []adminStorageCopySeed{{ - CopyIndex: 0, - ProviderID: onChainID(t, "901"), - DataSetID: onChainID(t, "9901"), - PieceID: onChainIDPtr(t, fmt.Sprintf("99010%d", i)), - TransferMethod: model.StorageCopyTransferMethodIngress, - RetrievalURL: fmt.Sprintf("https://provider.example/limit-%03d", i), - }}) - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("UpdateVersionState %d: %v", i, err) - } - if _, err := repos.Uploads.BindReadableUploadForVersion(ctx, repository.BindReadableUploadForVersionInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: version.Size, - Checksum: version.Checksum, - VersionID: version.VersionID, - }); err != nil { - t.Fatalf("BindReadableUploadForVersion %d: %v", i, err) - } + req := httptest.NewRequest(http.MethodPut, "/api/v1/buckets/root-transfer-bucket/owner", strings.NewReader(`{"owner_access_key":"`+internalRootOwnerAccessKey+`"}`)) + req.SetPathValue("name", bucket.Name) + req.Header.Set("Content-Type", "application/json") + setBucketWriteHeaders(req) + rr := httptest.NewRecorder() + + srv.handleAPIUpdateBucketOwner(rr, req) + + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusOK, rr.Body.String()) } - - summaries, err := repos.Uploads.ListDataSetSummaries(ctx, bucket.ID) + updated, err := repos.Buckets.GetByName(ctx, bucket.Name) if err != nil { - t.Fatalf("ListDataSetSummaries: %v", err) - } - if len(summaries) != 1 { - t.Fatalf("data set summaries = %d, want 1", len(summaries)) + t.Fatalf("GetByName: %v", err) } - checkedAt := time.Now().UTC() - if err := repos.Observability.ReplaceDataSetStates(ctx, checkedAt, []observability.DataSetState{{ - LocalDataSetID: summaries[0].ID, - BucketID: bucket.ID, - BucketName: bucket.Name, - CopyIndex: summaries[0].CopyIndex, - ProviderID: summaries[0].ProviderID, - ChainDataSetID: summaries[0].DataSetID, - Status: observability.StatusUnavailable, - ReasonCodes: []observability.ReasonCode{observability.ReasonChainDataSetMissing}, - LastCheckedAt: checkedAt, - Evidence: map[string]any{}, - }}); err != nil { - t.Fatalf("ReplaceDataSetStates: %v", err) + if updated.OwnerAccessKey == nil || *updated.OwnerAccessKey != srv.s3RootAccess { + t.Fatalf("stored owner = %v, want root access", updated.OwnerAccessKey) } - - ts := httptest.NewServer(newBucketAPIMux(srv)) - defer ts.Close() - resp, err := http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?limit=1001") + updatedACL, err := auth.ParseACL(updated.ACL) if err != nil { - t.Fatalf("GET affected versions: %v", err) + t.Fatalf("ParseACL: %v", err) + } + if updatedACL.Owner != srv.s3RootAccess { + t.Fatalf("ACL owner = %q, want root access", updatedACL.Owner) } - defer func() { _ = resp.Body.Close() }() var body struct { - Versions []struct{} `json:"versions"` - HasMore bool `json:"has_more"` + OwnerAccessKey *string `json:"owner_access_key"` } - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("Decode: %v", err) + if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { + t.Fatalf("Decode response: %v", err) } - if resp.StatusCode != http.StatusOK || body.HasMore || len(body.Versions) != 51 { - t.Fatalf("status/body = %d/%#v, want all 51 versions with clamped limit", resp.StatusCode, body) + if body.OwnerAccessKey == nil || *body.OwnerAccessKey != internalRootOwnerAccessKey { + t.Fatalf("response owner = %v, want internal root token", body.OwnerAccessKey) } } -func TestBucketStorageHealthAffectedVersionsResponseDeduplicatesProviderLookups(t *testing.T) { - providerID := onChainID(t, "501") - identityResolver := &fakeAPIProviderIdentityResolver{ - identities: map[string]*providerIdentityResponse{ - "501": {RegistryProviderID: "501", Name: "risk-pdp"}, - }, +func TestAPIBucketOwner_UpdateRejectsUnknownS3User(t *testing.T) { + srv, repos := newBucketAPITestServerWithS3Users(t, "owner-access") + ctx := context.Background() + bucket := &model.Bucket{Name: "unknown-owner-target", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) } - srv := &Server{providerIdentity: identityResolver} - createdAt := time.Date(2026, 5, 23, 12, 30, 0, 123456789, time.UTC) - status := observability.StatusUnavailable - resp := srv.bucketStorageHealthAffectedVersionsResponse(repository.BucketStorageHealthAffectedVersionPage{ - Versions: []repository.BucketStorageHealthAffectedVersion{{ - Version: model.ObjectVersion{ - VersionID: "01J000000000000000APIR99", - Key: "docs/current.txt", - CreatedAt: createdAt, - UpdatedAt: createdAt, - }, - RiskDataSets: []repository.BucketStorageHealthRiskDataSet{ - {LocalDataSetID: 1, ProviderID: providerID, LocalStatus: model.StorageDataSetStatusReady, ObservationStatus: &status}, - {LocalDataSetID: 2, ProviderID: providerID, LocalStatus: model.StorageDataSetStatusReady, ObservationStatus: &status}, - }, - }}, - }, createdAt.Add(-time.Hour)) + req := httptest.NewRequest(http.MethodPut, "/api/v1/buckets/unknown-owner-target/owner", strings.NewReader(`{"owner_access_key":"missing-owner"}`)) + req.SetPathValue("name", bucket.Name) + req.Header.Set("Content-Type", "application/json") + setBucketWriteHeaders(req) + rr := httptest.NewRecorder() - if got, want := identityResolver.requests, [][]string{{"501"}}; !reflect.DeepEqual(got, want) { - t.Fatalf("provider identity requests = %#v, want %#v", got, want) - } - if len(resp.Versions) != 1 || len(resp.Versions[0].RiskDataSets) != 2 { - t.Fatalf("response risk datasets = %#v, want two datasets", resp.Versions) - } - for _, dataSet := range resp.Versions[0].RiskDataSets { - if dataSet.ProviderIdentity == nil || dataSet.ProviderIdentity.Name != "risk-pdp" { - t.Fatalf("provider identity = %#v, want reused identity", dataSet.ProviderIdentity) - } + srv.handleAPIUpdateBucketOwner(rr, req) + + if rr.Code != http.StatusNotFound { + t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusNotFound, rr.Body.String()) } } -func TestBucketStorageHealthSummaryFromRepositoryFormatsLastCheckedAtUTC(t *testing.T) { - checkedAt := time.Date(2026, 5, 23, 20, 0, 0, 0, time.FixedZone("UTC+8", 8*60*60)) - got := bucketStorageHealthSummaryFromRepository(repository.BucketStorageHealthSummary{ - BucketID: 1, - ReasonCodes: []observability.ReasonCode{}, - AffectedVersionsCap: 200, - AffectedVersionsCapped: 0, - LastCheckedAt: &checkedAt, - }) - if got.LastCheckedAt != "2026-05-23T12:00:00Z" { - t.Fatalf("last_checked_at = %q, want UTC RFC3339", got.LastCheckedAt) +func TestAPIBucketOwner_UpdateRejectsMalformedStrictJSON(t *testing.T) { + srv, repos := newBucketAPITestServerWithS3Users(t, "owner-access") + ctx := context.Background() + bucket := &model.Bucket{Name: "strict-owner-target", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) + } + + for _, tc := range []struct { + name string + body string + }{ + {name: "unknown field", body: `{"owner_access_key":"owner-access","extra":true}`}, + {name: "trailing object", body: `{"owner_access_key":"owner-access"} {}`}, + } { + t.Run(tc.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPut, "/api/v1/buckets/strict-owner-target/owner", strings.NewReader(tc.body)) + req.SetPathValue("name", bucket.Name) + req.Header.Set("Content-Type", "application/json") + setBucketWriteHeaders(req) + rr := httptest.NewRecorder() + + srv.handleAPIUpdateBucketOwner(rr, req) + + if rr.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusBadRequest, rr.Body.String()) + } + }) } } -func TestAPIBucketsCopyHealthObservabilityFailurePreservesLocalCopyStatus(t *testing.T) { +func TestAPIBucketDetail_ActiveBucket(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "copy-health-observability-local-status", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "active-detail-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - _, versionID := seedAdminObjectVersion(t, repos, bucket, "partial.txt", 1, "etag-copy-health-local-status", "checksum-copy-health-local-status", "text/plain", "", model.ObjectStateCached) - bindAdminPartialUpload(t, repos, versionID) - srv.WithObservability(observability.NewService(observability.ServiceOptions{ - Store: &bucketStorageHealthErrorStore{err: errors.New("database password leaked copy health")}, - })) ts := httptest.NewServer(newBucketAPIMux(srv)) defer ts.Close() - resp, err := http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/objects/provenance?version_id=" + url.QueryEscape(versionID)) + resp, err := http.Get(ts.URL + "/api/v1/buckets/active-detail-bucket") if err != nil { - t.Fatalf("GET provenance: %v", err) + t.Fatalf("GET bucket detail: %v", err) } defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) } + var body struct { - CopyHealth apiCopyHealthBody `json:"copy_health"` + Name string `json:"name"` + Status string `json:"status"` + VersioningStatus string `json:"versioning_status"` + VersioningEnforced bool `json:"versioning_enforced"` } if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { t.Fatalf("Decode: %v", err) } - if body.CopyHealth.Status != string(observability.StatusUnavailable) || - body.CopyHealth.FailedCopies != 1 || - body.CopyHealth.UnknownCopies != 1 || - body.CopyHealth.PendingCopies != 0 || - body.CopyHealth.LastError != "copy health query failed" || - !hasReason(body.CopyHealth.ReasonCodes, observability.ReasonCopyFailed) { - t.Fatalf("copy_health = %#v, want failed local copy preserved with committed copy unknown", body.CopyHealth) - } -} - -func replaceBucketDataSetObservability(t *testing.T, repos *repository.Repositories, bucketID int64, status observability.Status, reasons []observability.ReasonCode, checkedAt time.Time) { - t.Helper() - summaries, err := repos.Uploads.ListDataSetSummaries(context.Background(), bucketID) - if err != nil { - t.Fatalf("ListDataSetSummaries: %v", err) - } - states := make([]observability.DataSetState, 0, len(summaries)) - for _, summary := range summaries { - states = append(states, observability.DataSetState{ - LocalDataSetID: summary.ID, - BucketID: summary.BucketID, - BucketName: summary.BucketName, - CopyIndex: summary.CopyIndex, - ProviderID: summary.ProviderID, - ChainDataSetID: summary.DataSetID, - ClientDataSetID: summary.ClientDataSetID, - LocalStatus: summary.Status, - Status: status, - ReasonCodes: reasons, - LastCheckedAt: checkedAt, - Evidence: map[string]any{}, - }) - } - if err := repos.Observability.ReplaceDataSetStates(context.Background(), checkedAt, states); err != nil { - t.Fatalf("ReplaceDataSetStates: %v", err) - } -} - -func copyHealthDataSetObservationForTest(localID int64, checkedAt time.Time) observability.DataSetObservation { - return observability.DataSetObservationFromState(observability.DataSetState{ - LocalDataSetID: localID, - Status: observability.StatusAvailable, - LastCheckedAt: checkedAt, - }, time.Hour, checkedAt) -} - -func assertUnavailableBucketStorageHealth(t *testing.T, health apiBucketStorageHealthBody, wantReason observability.ReasonCode) { - t.Helper() - if health.Status != string(observability.StatusUnavailable) || - health.AbnormalDataSets != 2 || - health.AffectedVersionsCapped != 1 || - health.AffectedVersionsCap != 200 || - health.AffectedVersionsExceedsCap || - !hasReason(health.ReasonCodes, wantReason) || - hasReason(health.ReasonCodes, observability.ReasonChainDataSetInactive) { - t.Fatalf("storage_health = %#v, want unavailable storage source risk from one retained version", health) - } -} - -func assertNoLegacyBucketStorageHealthFields(t *testing.T, raw map[string]any) { - t.Helper() - health, ok := raw["storage_health"].(map[string]any) - if !ok { - t.Fatalf("storage_health missing or wrong shape: %#v", raw["storage_health"]) + if body.Name != bucket.Name { + t.Fatalf("name = %q, want %q", body.Name, bucket.Name) } - for _, field := range []string{ - "affected_data_sets", - "affected_objects", - "objects_with_readable_copy", - "objects_without_readable_copy", - "unavailable_objects", - "unknown_objects", - } { - if _, ok := health[field]; ok { - t.Fatalf("storage_health includes legacy field %q: %#v", field, health) - } + if body.Status != string(model.BucketStatusActive) { + t.Fatalf("status = %q, want %q", body.Status, model.BucketStatusActive) } -} - -func hasReason(reasons []observability.ReasonCode, want observability.ReasonCode) bool { - for _, reason := range reasons { - if reason == want { - return true - } + if body.VersioningStatus != "Enabled" || !body.VersioningEnforced { + t.Fatalf("versioning = %q enforced=%v, want Enabled/enforced", body.VersioningStatus, body.VersioningEnforced) } - return false -} - -type bucketStorageHealthErrorStore struct { - err error -} - -func (s *bucketStorageHealthErrorStore) ReplaceProviderStates(context.Context, time.Time, []observability.ProviderState) error { - return nil -} - -func (s *bucketStorageHealthErrorStore) ListProviderStates(context.Context, observability.ListOptions) (observability.ProviderStatePage, error) { - return observability.ProviderStatePage{}, nil -} - -func (s *bucketStorageHealthErrorStore) ReplaceDataSetStates(context.Context, time.Time, []observability.DataSetState) error { - return nil -} - -func (s *bucketStorageHealthErrorStore) ListDataSetStates(context.Context, observability.ListOptions) (observability.DataSetStatePage, error) { - return observability.DataSetStatePage{}, nil -} - -func (s *bucketStorageHealthErrorStore) GetDataSetStatesByLocalIDs(context.Context, []int64) (map[int64]observability.DataSetState, error) { - return nil, s.err } -func TestAPIBuckets_ListAllBuckets(t *testing.T) { +func TestAPIBucketObjectsDelimiterReturnsCurrentLevelFoldersAndFiles(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - for _, bucket := range []*model.Bucket{ - {Name: "alpha-bucket", Status: model.BucketStatusActive}, - {Name: "beta-bucket", Status: model.BucketStatusActive}, - {Name: "gamma-bucket", Status: model.BucketStatusActive}, - } { - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create(%s): %v", bucket.Name, err) - } + bucket := &model.Bucket{Name: "folder-list-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) } + seedAdminObjectVersion(t, srv.db, repos, bucket, "docs/guide.md", 4, "etag-doc", "checksum-doc", "text/markdown", model.ObjectStateStored) + seedAdminObjectVersion(t, srv.db, repos, bucket, "photos/", 0, "etag-marker", "checksum-marker", "application/x-directory", model.ObjectStateCached) + seedAdminObjectVersion(t, srv.db, repos, bucket, "photos/2026/a.jpg", 7, "etag-photo", "checksum-photo", "image/jpeg", model.ObjectStateStored) + seedAdminObjectVersion(t, srv.db, repos, bucket, "root.txt", 5, "etag-root", "checksum-root", "text/plain", model.ObjectStateStored) ts := httptest.NewServer(newBucketAPIMux(srv)) defer ts.Close() - resp, err := http.Get(ts.URL + "/api/v1/buckets") + resp, err := http.Get(ts.URL + "/api/v1/buckets/folder-list-bucket/objects?delimiter=/") if err != nil { - t.Fatalf("GET buckets: %v", err) + t.Fatalf("GET bucket objects: %v", err) } defer func() { _ = resp.Body.Close() }() @@ -2428,1462 +2549,1493 @@ func TestAPIBuckets_ListAllBuckets(t *testing.T) { t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) } - var body []struct { - Name string `json:"name"` - Status string `json:"status"` - } + var body objectListResponse if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { t.Fatalf("Decode: %v", err) } - - got := make(map[string]string, len(body)) - for _, item := range body { - got[item.Name] = item.Status + if len(body.Folders) != 2 { + t.Fatalf("folders len = %d, want 2: %#v", len(body.Folders), body.Folders) } - - for _, name := range []string{"alpha-bucket", "beta-bucket", "gamma-bucket"} { - if got[name] != string(model.BucketStatusActive) { - t.Fatalf("%s status = %q, want %q", name, got[name], model.BucketStatusActive) - } + if body.Folders[0].Name != "docs" || body.Folders[0].Prefix != "docs/" { + t.Fatalf("first folder = %#v, want docs/", body.Folders[0]) + } + if body.Folders[1].Name != "photos" || body.Folders[1].Prefix != "photos/" { + t.Fatalf("second folder = %#v, want photos/", body.Folders[1]) + } + if len(body.Objects) != 1 || body.Objects[0].Key != "root.txt" { + t.Fatalf("objects = %#v, want root.txt only", body.Objects) } } -func TestAPIBuckets_ListAndDetailIncludeOwnerAccessKey(t *testing.T) { - srv, repos := newBucketAPITestServerWithRuntimeCopies(t, 5) +func TestAPIBucketObjectsDelimiterPrefixReturnsNestedLevel(t *testing.T) { + srv, repos := newBucketAPITestServer(t) ctx := context.Background() - ownedACL, err := json.Marshal(auth.ACL{Owner: "owner-access"}) - if err != nil { - t.Fatalf("Marshal ACL: %v", err) - } - ownerAccess := "owner-access" - if err := repos.S3Accounts.Create(ctx, &model.S3Account{AccessKey: ownerAccess, SecretKey: "owner-secret", Role: auth.RoleUserPlus}); err != nil { - t.Fatalf("S3Accounts.Create: %v", err) - } - overrideCopies := 3 - for _, bucket := range []*model.Bucket{ - {Name: "owned-bucket", Status: model.BucketStatusActive, ACL: ownedACL, OwnerAccessKey: &ownerAccess, DefaultCopies: &overrideCopies}, - {Name: "unassigned-bucket", Status: model.BucketStatusActive}, - } { - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create(%s): %v", bucket.Name, err) - } + bucket := &model.Bucket{Name: "nested-folder-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) } + seedAdminObjectVersion(t, srv.db, repos, bucket, "photos/", 0, "etag-marker", "checksum-marker", "application/x-directory", model.ObjectStateCached) + seedAdminObjectVersion(t, srv.db, repos, bucket, "photos/cover.jpg", 5, "etag-cover", "checksum-cover", "image/jpeg", model.ObjectStateStored) + seedAdminObjectVersion(t, srv.db, repos, bucket, "photos/2026/a.jpg", 7, "etag-photo", "checksum-photo", "image/jpeg", model.ObjectStateStored) ts := httptest.NewServer(newBucketAPIMux(srv)) defer ts.Close() - resp, err := http.Get(ts.URL + "/api/v1/buckets") + resp, err := http.Get(ts.URL + "/api/v1/buckets/nested-folder-bucket/objects?prefix=" + url.QueryEscape("photos/") + "&delimiter=/") if err != nil { - t.Fatalf("GET buckets: %v", err) + t.Fatalf("GET bucket objects: %v", err) } defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { - t.Fatalf("list status = %d, want %d", resp.StatusCode, http.StatusOK) - } - var listBody []struct { - Name string `json:"name"` - OwnerAccessKey *string `json:"owner_access_key"` - DefaultCopies *int `json:"default_copies"` - EffectiveCopies int `json:"effective_copies"` - } - if err := json.NewDecoder(resp.Body).Decode(&listBody); err != nil { - t.Fatalf("Decode list: %v", err) - } - owners := make(map[string]*string, len(listBody)) - copyPolicies := make(map[string]struct { - defaultCopies *int - effectiveCopies int - }, len(listBody)) - for _, item := range listBody { - owners[item.Name] = item.OwnerAccessKey - copyPolicies[item.Name] = struct { - defaultCopies *int - effectiveCopies int - }{defaultCopies: item.DefaultCopies, effectiveCopies: item.EffectiveCopies} - } - if owners["owned-bucket"] == nil || *owners["owned-bucket"] != "owner-access" { - t.Fatalf("owned-bucket owner = %v, want owner-access", owners["owned-bucket"]) - } - if owners["unassigned-bucket"] != nil { - t.Fatalf("unassigned-bucket owner = %v, want nil", *owners["unassigned-bucket"]) - } - if copyPolicies["owned-bucket"].defaultCopies == nil || *copyPolicies["owned-bucket"].defaultCopies != 3 || copyPolicies["owned-bucket"].effectiveCopies != 3 { - t.Fatalf("owned bucket copy policy = %#v, want override 3", copyPolicies["owned-bucket"]) - } - if copyPolicies["unassigned-bucket"].defaultCopies != nil || copyPolicies["unassigned-bucket"].effectiveCopies != 5 { - t.Fatalf("unassigned bucket copy policy = %#v, want inherited 5", copyPolicies["unassigned-bucket"]) + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) } - detailResp, err := http.Get(ts.URL + "/api/v1/buckets/owned-bucket") - if err != nil { - t.Fatalf("GET bucket detail: %v", err) - } - defer func() { _ = detailResp.Body.Close() }() - if detailResp.StatusCode != http.StatusOK { - t.Fatalf("detail status = %d, want %d", detailResp.StatusCode, http.StatusOK) - } - var detailBody struct { - OwnerAccessKey *string `json:"owner_access_key"` - DefaultCopies *int `json:"default_copies"` - EffectiveCopies int `json:"effective_copies"` - } - if err := json.NewDecoder(detailResp.Body).Decode(&detailBody); err != nil { - t.Fatalf("Decode detail: %v", err) + var body objectListResponse + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("Decode: %v", err) } - if detailBody.OwnerAccessKey == nil || *detailBody.OwnerAccessKey != "owner-access" { - t.Fatalf("detail owner = %v, want owner-access", detailBody.OwnerAccessKey) + if len(body.Folders) != 1 || body.Folders[0].Name != "2026" || body.Folders[0].Prefix != "photos/2026/" { + t.Fatalf("folders = %#v, want photos/2026/", body.Folders) } - if detailBody.DefaultCopies == nil || *detailBody.DefaultCopies != 3 || detailBody.EffectiveCopies != 3 { - t.Fatalf("detail copy policy = default:%v effective:%d, want 3/3", detailBody.DefaultCopies, detailBody.EffectiveCopies) + if len(body.Objects) != 1 || body.Objects[0].Key != "photos/cover.jpg" { + t.Fatalf("objects = %#v, want photos/cover.jpg only", body.Objects) } } -func TestAPIBucketCopyPolicy_UpdateAndClear(t *testing.T) { - srv, repos := newBucketAPITestServerWithRuntimeCopies(t, 5) +func TestAPIBucketObjectsDelimiterPreservesSlashOnlyFolderName(t *testing.T) { + srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "copy-policy-bucket", Status: model.BucketStatusActive} + + bucket := &model.Bucket{Name: "slash-folder-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } + seedAdminObjectVersion(t, srv.db, repos, bucket, "a//child.txt", 1, "etag-slash", "checksum-slash", "text/plain", model.ObjectStateStored) ts := httptest.NewServer(newBucketAPIMux(srv)) defer ts.Close() - setReq, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/buckets/copy-policy-bucket/copy-policy", strings.NewReader(`{"default_copies":6}`)) - if err != nil { - t.Fatalf("NewRequest set: %v", err) - } - setReq.Header.Set("Content-Type", "application/json") - setBucketWriteHeaders(setReq) - setResp, err := http.DefaultClient.Do(setReq) + resp, err := http.Get(ts.URL + "/api/v1/buckets/slash-folder-bucket/objects?prefix=" + url.QueryEscape("a/") + "&delimiter=/") if err != nil { - t.Fatalf("PUT copy policy set: %v", err) - } - defer func() { _ = setResp.Body.Close() }() - if setResp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(setResp.Body) - t.Fatalf("set status = %d, want %d body=%s", setResp.StatusCode, http.StatusOK, body) + t.Fatalf("GET bucket objects: %v", err) } - var setBody struct { - Name string `json:"name"` - DefaultCopies *int `json:"default_copies"` - EffectiveCopies int `json:"effective_copies"` + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) } - if err := json.NewDecoder(setResp.Body).Decode(&setBody); err != nil { - t.Fatalf("Decode set response: %v", err) + + var body objectListResponse + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("Decode: %v", err) } - if setBody.Name != bucket.Name || setBody.DefaultCopies == nil || *setBody.DefaultCopies != 6 || setBody.EffectiveCopies != 6 { - t.Fatalf("set copy policy response = %#v, want override 6", setBody) + if len(body.Folders) != 1 || body.Folders[0].Name != "/" || body.Folders[0].Prefix != "a//" { + t.Fatalf("folders = %#v, want slash-only folder a//", body.Folders) } +} - clearReq, err := http.NewRequest(http.MethodPut, ts.URL+"/api/v1/buckets/copy-policy-bucket/copy-policy", strings.NewReader("{\"default_copies\": \n null \t}")) - if err != nil { - t.Fatalf("NewRequest clear: %v", err) +func TestAPIBucketObjectsRejectsUnsupportedDelimiter(t *testing.T) { + srv, repos := newBucketAPITestServer(t) + ctx := context.Background() + + bucket := &model.Bucket{Name: "unsupported-delimiter-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) } - clearReq.Header.Set("Content-Type", "application/json") - setBucketWriteHeaders(clearReq) - clearResp, err := http.DefaultClient.Do(clearReq) + + ts := httptest.NewServer(newBucketAPIMux(srv)) + defer ts.Close() + + resp, err := http.Get(ts.URL + "/api/v1/buckets/unsupported-delimiter-bucket/objects?delimiter=:") if err != nil { - t.Fatalf("PUT copy policy clear: %v", err) - } - defer func() { _ = clearResp.Body.Close() }() - if clearResp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(clearResp.Body) - t.Fatalf("clear status = %d, want %d body=%s", clearResp.StatusCode, http.StatusOK, body) - } - var clearBody struct { - DefaultCopies *int `json:"default_copies"` - EffectiveCopies int `json:"effective_copies"` - } - if err := json.NewDecoder(clearResp.Body).Decode(&clearBody); err != nil { - t.Fatalf("Decode clear response: %v", err) + t.Fatalf("GET bucket objects: %v", err) } - if clearBody.DefaultCopies != nil || clearBody.EffectiveCopies != 5 { - t.Fatalf("clear copy policy response = %#v, want inherited 5", clearBody) + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusBadRequest) } } -func TestAPIBucketCopyPolicy_IndependentFieldsValidateFinalPolicyAndUseOneCoordinator(t *testing.T) { - srv, repos := newBucketAPITestServerWithRuntimeCopies(t, 5) +func TestAPIBucketObjectsDelimiterPaginationSkipsDuplicateFolders(t *testing.T) { + srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "independent-copy-policy-bucket", Status: model.BucketStatusActive} + + bucket := &model.Bucket{Name: "folder-page-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } + seedAdminObjectVersion(t, srv.db, repos, bucket, "a/1.txt", 1, "etag-a1", "checksum-a1", "text/plain", model.ObjectStateStored) + seedAdminObjectVersion(t, srv.db, repos, bucket, "a/2.txt", 1, "etag-a2", "checksum-a2", "text/plain", model.ObjectStateStored) + seedAdminObjectVersion(t, srv.db, repos, bucket, "b/1.txt", 1, "etag-b1", "checksum-b1", "text/plain", model.ObjectStateStored) - update := func(body string) *httptest.ResponseRecorder { - t.Helper() - req := httptest.NewRequest(http.MethodPut, "/api/v1/buckets/independent-copy-policy-bucket/copy-policy", strings.NewReader(body)) - req.SetPathValue("name", bucket.Name) - req.Header.Set("Content-Type", "application/json") - setBucketWriteHeaders(req) - rr := httptest.NewRecorder() - srv.handleAPIUpdateBucketCopyPolicy(rr, req) - return rr - } - assertStored := func(wantTarget, wantMinimum *int) { - t.Helper() - got, err := repos.Buckets.GetByName(ctx, bucket.Name) - if err != nil || got == nil { - t.Fatalf("GetByName: bucket=%#v err=%v", got, err) - } - if !reflect.DeepEqual(got.DefaultCopies, wantTarget) || !reflect.DeepEqual(got.MinimumDurableCopies, wantMinimum) { - t.Fatalf("stored policy = target:%v minimum:%v, want target:%v minimum:%v", got.DefaultCopies, got.MinimumDurableCopies, wantTarget, wantMinimum) - } - } + ts := httptest.NewServer(newBucketAPIMux(srv)) + defer ts.Close() - targetFour, minimumTwo := 4, 2 - if rr := update(`{"default_copies":4,"minimum_durable_copies":2}`); rr.Code != http.StatusOK { - t.Fatalf("joint update status = %d body=%s", rr.Code, rr.Body.String()) + resp, err := http.Get(ts.URL + "/api/v1/buckets/folder-page-bucket/objects?delimiter=/&limit=1") + if err != nil { + t.Fatalf("GET page 1: %v", err) } - assertStored(&targetFour, &minimumTwo) + defer func() { _ = resp.Body.Close() }() - minimumThree := 3 - if rr := update(`{"minimum_durable_copies":3}`); rr.Code != http.StatusOK { - t.Fatalf("minimum-only update status = %d body=%s", rr.Code, rr.Body.String()) + var page1 objectListResponse + if err := json.NewDecoder(resp.Body).Decode(&page1); err != nil { + t.Fatalf("Decode page 1: %v", err) + } + if len(page1.Folders) != 1 || page1.Folders[0].Prefix != "a/" || len(page1.Objects) != 0 || !page1.HasMore || page1.NextMarker == "" { + t.Fatalf("page 1 = %#v, want a/ folder and next marker", page1) } - assertStored(&targetFour, &minimumThree) - targetThree := 3 - if rr := update(`{"default_copies":3}`); rr.Code != http.StatusOK { - t.Fatalf("target-only update status = %d body=%s", rr.Code, rr.Body.String()) + resp2, err := http.Get(ts.URL + "/api/v1/buckets/folder-page-bucket/objects?delimiter=/&limit=1&after=" + url.QueryEscape(page1.NextMarker)) + if err != nil { + t.Fatalf("GET page 2: %v", err) } - assertStored(&targetThree, &minimumThree) + defer func() { _ = resp2.Body.Close() }() - if rr := update(`{"default_copies":2}`); rr.Code != http.StatusBadRequest { - t.Fatalf("invalid target reduction status = %d body=%s", rr.Code, rr.Body.String()) + var page2 objectListResponse + if err := json.NewDecoder(resp2.Body).Decode(&page2); err != nil { + t.Fatalf("Decode page 2: %v", err) + } + if len(page2.Folders) != 1 || page2.Folders[0].Prefix != "b/" || len(page2.Objects) != 0 { + t.Fatalf("page 2 = %#v, want b/ folder only", page2) } - assertStored(&targetThree, &minimumThree) +} - if rr := update(`{"minimum_durable_copies":null}`); rr.Code != http.StatusOK { - t.Fatalf("clear minimum status = %d body=%s", rr.Code, rr.Body.String()) - } else { - var response struct { - Minimum *int `json:"minimum_durable_copies"` - Effective int `json:"effective_minimum_durable_copies"` - Target *int `json:"default_copies"` - TargetValue int `json:"effective_copies"` - } - if err := json.NewDecoder(rr.Body).Decode(&response); err != nil { - t.Fatalf("Decode clear minimum response: %v", err) - } - if response.Minimum != nil || response.Effective != 3 || response.Target == nil || *response.Target != 3 || response.TargetValue != 3 { - t.Fatalf("clear minimum response = %#v, want strict 3/3", response) - } +func TestListBucketObjectEntriesSkipsEmittedFolderSubtree(t *testing.T) { + keys := make([]string, adminObjectListingBatchSize*2+1) + for i := range keys { + keys[i] = fmt.Sprintf("a/%04d.txt", i) } - assertStored(&targetThree, nil) + objects := &recordingObjectListRepo{keys: keys} + srv := &Server{repos: &repository.Repositories{Objects: objects}} - tasks, total, err := repos.Tasks.List(ctx, string(model.TaskTypeEvictCache), cacheeviction.StageReconcileBucketDurability, "", 10, 0) + folders, files, hasMore, nextMarker, err := srv.listBucketObjectEntries(t.Context(), 1, "", "/", "", 50) if err != nil { - t.Fatalf("List coordinator tasks: %v", err) - } - if total != 1 || len(tasks) != 1 || tasks[0].RefType != "bucket" || tasks[0].RefID != bucket.ID { - t.Fatalf("coordinator tasks total=%d tasks=%#v, want one bucket task", total, tasks) + t.Fatalf("listBucketObjectEntries: %v", err) } -} -func TestAPIBucketCopyPolicy_EffectiveMinimumClampsWithoutRewritingStoredValue(t *testing.T) { - srv, repos := newBucketAPITestServerWithRuntimeCopies(t, 2) - minimum := 5 - bucket := &model.Bucket{ - Name: "clamped-copy-policy-bucket", - MinimumDurableCopies: &minimum, - Status: model.BucketStatusActive, - } - if err := repos.Buckets.Create(context.Background(), bucket); err != nil { - t.Fatalf("Buckets.Create: %v", err) + if len(folders) != 1 || folders[0].Prefix != "a/" || len(files) != 0 || hasMore || nextMarker != "" { + t.Fatalf("listing = folders:%#v files:%#v hasMore:%v nextMarker:%q, want a/ folder only", folders, files, hasMore, nextMarker) } - req := httptest.NewRequest(http.MethodGet, "/api/v1/buckets/clamped-copy-policy-bucket", nil) - req.SetPathValue("name", bucket.Name) - rr := httptest.NewRecorder() - srv.handleAPIGetBucket(rr, req) - if rr.Code != http.StatusOK { - t.Fatalf("GET status = %d body=%s", rr.Code, rr.Body.String()) + if objects.scanCalls() > 2 { + t.Fatalf("object list scans = %d, want at most 2 without walking every child batch", objects.scanCalls()) } - var response struct { - Minimum *int `json:"minimum_durable_copies"` - Effective int `json:"effective_minimum_durable_copies"` +} + +func TestListBucketObjectEntriesKeepsCurrentBatchAcrossSiblingFolders(t *testing.T) { + keys := make([]string, 50) + for i := range keys { + keys[i] = fmt.Sprintf("dir-%02d/file.txt", i) } - if err := json.NewDecoder(rr.Body).Decode(&response); err != nil { - t.Fatalf("Decode response: %v", err) + objects := &recordingObjectListRepo{keys: keys} + srv := &Server{repos: &repository.Repositories{Objects: objects}} + + folders, files, hasMore, nextMarker, err := srv.listBucketObjectEntries(t.Context(), 1, "", "/", "", 50) + if err != nil { + t.Fatalf("listBucketObjectEntries: %v", err) } - if response.Minimum == nil || *response.Minimum != 5 || response.Effective != 2 { - t.Fatalf("minimum response = stored:%v effective:%d, want 5 clamped to 2", response.Minimum, response.Effective) + + if len(folders) != 50 || len(files) != 0 || hasMore || nextMarker != "" { + t.Fatalf("listing = folders:%d files:%#v hasMore:%v nextMarker:%q, want 50 folders only", len(folders), files, hasMore, nextMarker) } - stored, err := repos.Buckets.GetByName(context.Background(), bucket.Name) - if err != nil || stored == nil || stored.MinimumDurableCopies == nil || *stored.MinimumDurableCopies != 5 { - t.Fatalf("stored bucket after GET = %#v err=%v, want unchanged minimum 5", stored, err) + if objects.scanCalls() != 1 { + t.Fatalf("object list scans = %d, want 1 for sibling folders in one batch", objects.scanCalls()) } } -func TestAPIBucketCopyPolicy_RejectsInvalidPayloads(t *testing.T) { - srv, repos := newBucketAPITestServer(t) - ctx := context.Background() - bucket := &model.Bucket{Name: "invalid-copy-policy-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create: %v", err) +func TestListBucketObjectEntriesSkipsDuplicateRowsBeforeSiblingFolders(t *testing.T) { + keys := make([]string, 0, 100) + for i := range 50 { + keys = append(keys, fmt.Sprintf("dir-%02d/a.txt", i), fmt.Sprintf("dir-%02d/b.txt", i)) } + objects := &recordingObjectListRepo{keys: keys} + srv := &Server{repos: &repository.Repositories{Objects: objects}} - for _, tc := range []struct { - name string - body string - }{ - {name: "missing field", body: `{}`}, - {name: "unknown field", body: `{"default_copies":3,"extra":true}`}, - {name: "string copies", body: `{"default_copies":"3"}`}, - {name: "fractional copies", body: `{"default_copies":3.5}`}, - {name: "zero copies", body: `{"default_copies":0}`}, - {name: "too many copies", body: `{"default_copies":9}`}, - {name: "zero minimum", body: `{"minimum_durable_copies":0}`}, - {name: "too many minimum", body: `{"minimum_durable_copies":9}`}, - {name: "minimum exceeds target", body: `{"default_copies":2,"minimum_durable_copies":3}`}, - } { - t.Run(tc.name, func(t *testing.T) { - req := httptest.NewRequest(http.MethodPut, "/api/v1/buckets/invalid-copy-policy-bucket/copy-policy", strings.NewReader(tc.body)) - req.SetPathValue("name", bucket.Name) - req.Header.Set("Content-Type", "application/json") - setBucketWriteHeaders(req) - rr := httptest.NewRecorder() - - srv.handleAPIUpdateBucketCopyPolicy(rr, req) + folders, files, hasMore, nextMarker, err := srv.listBucketObjectEntries(t.Context(), 1, "", "/", "", 50) + if err != nil { + t.Fatalf("listBucketObjectEntries: %v", err) + } - if rr.Code != http.StatusBadRequest { - t.Fatalf("status = %d, want %d body=%s", rr.Code, http.StatusBadRequest, rr.Body.String()) - } - }) + if len(folders) != 50 || len(files) != 0 || hasMore || nextMarker != "" { + t.Fatalf("listing = folders:%d files:%#v hasMore:%v nextMarker:%q, want 50 folders only", len(folders), files, hasMore, nextMarker) + } + if objects.scanCalls() != 1 { + t.Fatalf("object list scans = %d, want 1 while duplicate folder rows fit in one batch", objects.scanCalls()) } } -func TestAPIBucketOwner_UpdateAssignsExistingS3User(t *testing.T) { - srv, repos := newBucketAPITestServerWithS3UsersAndRuntimeCopies(t, 4, "owner-access") +func TestAPIBucketObjects_DeleteListAndRestore(t *testing.T) { + srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "assign-owner-bucket", Status: model.BucketStatusActive} + + bucket := &model.Bucket{Name: "trash-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } + _, versionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/file.txt", 7, "etag-file", "checksum-file", "text/plain", model.ObjectStateCached) - req := httptest.NewRequest(http.MethodPut, "/api/v1/buckets/assign-owner-bucket/owner", strings.NewReader(`{"owner_access_key":"owner-access"}`)) - req.SetPathValue("name", bucket.Name) - req.Header.Set("Content-Type", "application/json") - setBucketWriteHeaders(req) - rr := httptest.NewRecorder() - - srv.handleAPIUpdateBucketOwner(rr, req) + ts := httptest.NewServer(newBucketAPIMux(srv)) + defer ts.Close() - if rr.Code != http.StatusOK { - t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusOK, rr.Body.String()) - } - updated, err := repos.Buckets.GetByName(ctx, bucket.Name) + deleteReq, err := http.NewRequest(http.MethodDelete, ts.URL+"/api/v1/buckets/trash-bucket/objects?key="+url.QueryEscape("folder/file.txt"), nil) if err != nil { - t.Fatalf("GetByName: %v", err) + t.Fatalf("NewRequest delete: %v", err) } - acl, err := auth.ParseACL(updated.ACL) + setBucketWriteHeaders(deleteReq) + deleteResp, err := ts.Client().Do(deleteReq) if err != nil { - t.Fatalf("ParseACL: %v", err) - } - if acl.Owner != "owner-access" { - t.Fatalf("owner = %q, want owner-access", acl.Owner) + t.Fatalf("DELETE bucket object: %v", err) } - var body struct { - OwnerAccessKey *string `json:"owner_access_key"` - DefaultCopies *int `json:"default_copies"` - EffectiveCopies int `json:"effective_copies"` + defer func() { _ = deleteResp.Body.Close() }() + if deleteResp.StatusCode != http.StatusOK { + t.Fatalf("delete status = %d, want %d, body=%s", deleteResp.StatusCode, http.StatusOK, readBody(t, deleteResp.Body)) } - if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { - t.Fatalf("Decode response: %v", err) + var deleteBody struct { + Key string `json:"key"` + DeleteMarkerVersionID string `json:"delete_marker_version_id"` + DeletedAt string `json:"deleted_at"` } - if body.OwnerAccessKey == nil || *body.OwnerAccessKey != "owner-access" { - t.Fatalf("response owner = %v, want owner-access", body.OwnerAccessKey) + if err := json.NewDecoder(deleteResp.Body).Decode(&deleteBody); err != nil { + t.Fatalf("Decode delete: %v", err) } - if body.DefaultCopies != nil || body.EffectiveCopies != 4 { - t.Fatalf("owner update copy policy = default:%v effective:%d, want inherited 4", body.DefaultCopies, body.EffectiveCopies) + if deleteBody.Key != "folder/file.txt" || deleteBody.DeleteMarkerVersionID == "" || deleteBody.DeletedAt == "" { + t.Fatalf("delete response = %#v, want marker metadata", deleteBody) } -} -func TestAPIBucketOwner_UpdateAllowsInternalRootOwner(t *testing.T) { - srv, repos := newBucketAPITestServerWithS3Users(t, "owner-access") - ctx := context.Background() - owner := "owner-access" - acl, err := json.Marshal(auth.ACL{Owner: owner}) + listResp, err := http.Get(ts.URL + "/api/v1/buckets/trash-bucket/objects?prefix=" + url.QueryEscape("folder/")) if err != nil { - t.Fatalf("Marshal ACL: %v", err) + t.Fatalf("GET live objects: %v", err) } - bucket := &model.Bucket{Name: "root-transfer-bucket", Status: model.BucketStatusActive, OwnerAccessKey: &owner, ACL: acl} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create: %v", err) + defer func() { _ = listResp.Body.Close() }() + var live objectListResponse + if err := json.NewDecoder(listResp.Body).Decode(&live); err != nil { + t.Fatalf("Decode live objects: %v", err) } - - req := httptest.NewRequest(http.MethodPut, "/api/v1/buckets/root-transfer-bucket/owner", strings.NewReader(`{"owner_access_key":"`+internalRootOwnerAccessKey+`"}`)) - req.SetPathValue("name", bucket.Name) - req.Header.Set("Content-Type", "application/json") - setBucketWriteHeaders(req) - rr := httptest.NewRecorder() - - srv.handleAPIUpdateBucketOwner(rr, req) - - if rr.Code != http.StatusOK { - t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusOK, rr.Body.String()) + if len(live.Objects) != 0 { + t.Fatalf("live objects = %#v, want deleted object hidden", live.Objects) } - updated, err := repos.Buckets.GetByName(ctx, bucket.Name) + + versionsResp, err := http.Get(ts.URL + "/api/v1/buckets/trash-bucket/objects/versions?key=" + url.QueryEscape("folder/file.txt")) if err != nil { - t.Fatalf("GetByName: %v", err) + t.Fatalf("GET object versions: %v", err) } - if updated.OwnerAccessKey == nil || *updated.OwnerAccessKey != srv.s3RootAccess { - t.Fatalf("stored owner = %v, want root access", updated.OwnerAccessKey) + defer func() { _ = versionsResp.Body.Close() }() + var versionsBody struct { + Versions []struct { + VersionID string `json:"version_id"` + IsCurrent bool `json:"is_current"` + IsDeleteMarker bool `json:"is_delete_marker"` + UploadStatus string `json:"upload_status"` + DownloadVisible bool `json:"download_visible"` + } `json:"versions"` } - updatedACL, err := auth.ParseACL(updated.ACL) + if err := json.NewDecoder(versionsResp.Body).Decode(&versionsBody); err != nil { + t.Fatalf("Decode versions: %v", err) + } + if len(versionsBody.Versions) != 2 { + t.Fatalf("versions len = %d, want 2", len(versionsBody.Versions)) + } + if versionsBody.Versions[0].VersionID != deleteBody.DeleteMarkerVersionID || !versionsBody.Versions[0].IsCurrent || !versionsBody.Versions[0].IsDeleteMarker { + t.Fatalf("first version = %#v, want current delete marker", versionsBody.Versions[0]) + } + if versionsBody.Versions[1].VersionID != versionID || versionsBody.Versions[1].IsDeleteMarker { + t.Fatalf("second version = %#v, want data version %s", versionsBody.Versions[1], versionID) + } + + deletedResp, err := http.Get(ts.URL + "/api/v1/buckets/trash-bucket/objects/deleted?prefix=" + url.QueryEscape("folder/")) if err != nil { - t.Fatalf("ParseACL: %v", err) + t.Fatalf("GET deleted objects: %v", err) } - if updatedACL.Owner != srv.s3RootAccess { - t.Fatalf("ACL owner = %q, want root access", updatedACL.Owner) + defer func() { _ = deletedResp.Body.Close() }() + var deletedBody struct { + Objects []struct { + Key string `json:"key"` + DeleteMarkerVersionID string `json:"delete_marker_version_id"` + DeletedAt string `json:"deleted_at"` + RestoreVersionID string `json:"restore_version_id"` + RestoreSize int64 `json:"restore_size"` + RestoreContentType string `json:"restore_content_type"` + RestoreETag string `json:"restore_etag"` + } `json:"objects"` + HasMore bool `json:"has_more"` } - var body struct { - OwnerAccessKey *string `json:"owner_access_key"` + if err := json.NewDecoder(deletedResp.Body).Decode(&deletedBody); err != nil { + t.Fatalf("Decode deleted objects: %v", err) } - if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { - t.Fatalf("Decode response: %v", err) + if len(deletedBody.Objects) != 1 { + t.Fatalf("deleted objects len = %d, want 1", len(deletedBody.Objects)) } - if body.OwnerAccessKey == nil || *body.OwnerAccessKey != internalRootOwnerAccessKey { - t.Fatalf("response owner = %v, want internal root token", body.OwnerAccessKey) + deleted := deletedBody.Objects[0] + if deleted.Key != "folder/file.txt" || deleted.DeleteMarkerVersionID != deleteBody.DeleteMarkerVersionID || deleted.RestoreVersionID != versionID || deleted.RestoreSize != 7 || deleted.RestoreContentType != "text/plain" || deleted.RestoreETag != "etag-file" || deleted.DeletedAt == "" { + t.Fatalf("deleted object = %#v, want recoverable file", deleted) } -} -func TestAPIBucketOwner_UpdateRejectsUnknownS3User(t *testing.T) { - srv, repos := newBucketAPITestServerWithS3Users(t, "owner-access") - ctx := context.Background() - bucket := &model.Bucket{Name: "unknown-owner-target", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create: %v", err) + restoreReq, err := http.NewRequest(http.MethodPost, ts.URL+"/api/v1/buckets/trash-bucket/objects/restore", strings.NewReader(fmt.Sprintf(`{"key":%q,"delete_marker_version_id":%q}`, "folder/file.txt", deleteBody.DeleteMarkerVersionID))) + if err != nil { + t.Fatalf("NewRequest restore: %v", err) } - - req := httptest.NewRequest(http.MethodPut, "/api/v1/buckets/unknown-owner-target/owner", strings.NewReader(`{"owner_access_key":"missing-owner"}`)) - req.SetPathValue("name", bucket.Name) - req.Header.Set("Content-Type", "application/json") - setBucketWriteHeaders(req) - rr := httptest.NewRecorder() - - srv.handleAPIUpdateBucketOwner(rr, req) - - if rr.Code != http.StatusNotFound { - t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusNotFound, rr.Body.String()) + setBucketWriteHeaders(restoreReq) + restoreResp, err := ts.Client().Do(restoreReq) + if err != nil { + t.Fatalf("POST restore object: %v", err) } -} - -func TestAPIBucketOwner_UpdateRejectsMalformedStrictJSON(t *testing.T) { - srv, repos := newBucketAPITestServerWithS3Users(t, "owner-access") - ctx := context.Background() - bucket := &model.Bucket{Name: "strict-owner-target", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create: %v", err) + defer func() { _ = restoreResp.Body.Close() }() + if restoreResp.StatusCode != http.StatusOK { + t.Fatalf("restore status = %d, want %d, body=%s", restoreResp.StatusCode, http.StatusOK, readBody(t, restoreResp.Body)) + } + var restoreBody struct { + Key string `json:"key"` + RestoredVersionID string `json:"restored_version_id"` + } + if err := json.NewDecoder(restoreResp.Body).Decode(&restoreBody); err != nil { + t.Fatalf("Decode restore: %v", err) + } + if restoreBody.Key != "folder/file.txt" || restoreBody.RestoredVersionID != versionID { + t.Fatalf("restore response = %#v, want data version restored", restoreBody) } - for _, tc := range []struct { - name string - body string - }{ - {name: "unknown field", body: `{"owner_access_key":"owner-access","extra":true}`}, - {name: "trailing object", body: `{"owner_access_key":"owner-access"} {}`}, - } { - t.Run(tc.name, func(t *testing.T) { - req := httptest.NewRequest(http.MethodPut, "/api/v1/buckets/strict-owner-target/owner", strings.NewReader(tc.body)) - req.SetPathValue("name", bucket.Name) - req.Header.Set("Content-Type", "application/json") - setBucketWriteHeaders(req) - rr := httptest.NewRecorder() - - srv.handleAPIUpdateBucketOwner(rr, req) + restoredListResp, err := http.Get(ts.URL + "/api/v1/buckets/trash-bucket/objects?prefix=" + url.QueryEscape("folder/")) + if err != nil { + t.Fatalf("GET restored objects: %v", err) + } + defer func() { _ = restoredListResp.Body.Close() }() + var restoredLive objectListResponse + if err := json.NewDecoder(restoredListResp.Body).Decode(&restoredLive); err != nil { + t.Fatalf("Decode restored objects: %v", err) + } + if len(restoredLive.Objects) != 1 || restoredLive.Objects[0].Key != "folder/file.txt" || restoredLive.Objects[0].CurrentVersionID != versionID { + t.Fatalf("restored live objects = %#v, want restored file", restoredLive.Objects) + } - if rr.Code != http.StatusBadRequest { - t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusBadRequest, rr.Body.String()) - } - }) + emptyDeletedResp, err := http.Get(ts.URL + "/api/v1/buckets/trash-bucket/objects/deleted?prefix=" + url.QueryEscape("folder/")) + if err != nil { + t.Fatalf("GET deleted objects after restore: %v", err) + } + defer func() { _ = emptyDeletedResp.Body.Close() }() + var emptyDeleted struct { + Objects []struct{} `json:"objects"` + } + if err := json.NewDecoder(emptyDeletedResp.Body).Decode(&emptyDeleted); err != nil { + t.Fatalf("Decode empty deleted: %v", err) + } + if len(emptyDeleted.Objects) != 0 { + t.Fatalf("deleted objects after restore len = %d, want 0", len(emptyDeleted.Objects)) } } -func TestAPIBucketDetail_ActiveBucket(t *testing.T) { +func TestAPIBucketObjectDeleteRejectsInvalidObjectKey(t *testing.T) { srv, repos := newBucketAPITestServer(t) - ctx := context.Background() - - bucket := &model.Bucket{Name: "active-detail-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { + bucket := &model.Bucket{Name: "delete-key-validation-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(context.Background(), bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } + key := strings.Repeat("你", 342) + req := httptest.NewRequest(http.MethodDelete, "/api/v1/buckets/delete-key-validation-bucket/objects?key="+url.QueryEscape(key), nil) + req.SetPathValue("name", bucket.Name) + setBucketWriteHeaders(req) + rr := httptest.NewRecorder() - ts := httptest.NewServer(newBucketAPIMux(srv)) - defer ts.Close() - - resp, err := http.Get(ts.URL + "/api/v1/buckets/active-detail-bucket") - if err != nil { - t.Fatalf("GET bucket detail: %v", err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) - } + srv.handleAPIDeleteBucketObject(rr, req) - var body struct { - Name string `json:"name"` - Status string `json:"status"` - VersioningStatus string `json:"versioning_status"` - VersioningEnforced bool `json:"versioning_enforced"` - } - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("Decode: %v", err) + if rr.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusBadRequest, rr.Body.String()) } - if body.Name != bucket.Name { - t.Fatalf("name = %q, want %q", body.Name, bucket.Name) + if !strings.Contains(rr.Body.String(), "1024 UTF-8 bytes") { + t.Fatalf("body = %s, want object key byte-limit error", rr.Body.String()) } - if body.Status != string(model.BucketStatusActive) { - t.Fatalf("status = %q, want %q", body.Status, model.BucketStatusActive) + version, err := repos.Objects.GetCurrentVersionByBucketAndKey(context.Background(), bucket.ID, key) + if err != nil { + t.Fatalf("GetCurrentVersionByBucketAndKey: %v", err) } - if body.VersioningStatus != "Enabled" || !body.VersioningEnforced { - t.Fatalf("versioning = %q enforced=%v, want Enabled/enforced", body.VersioningStatus, body.VersioningEnforced) + if version != nil { + t.Fatalf("current version = %#v, want no persisted delete marker", version) } } -func TestAPIBucketObjects_ActiveBucket(t *testing.T) { +func TestAPIBucketDeletedObjectPermanentDeleteRemovesDeletedObject(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "objects-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "trash-permanent-delete-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - _, versionID := seedAdminObjectVersion(t, repos, bucket, "kept.txt", 4, "etag-kept", "checksum-kept", "text/plain", "", model.ObjectStateCached) - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark uploading: %v", err) - } - acceptAdminVersionUpload(t, repos, versionID, "piece-kept", "https://provider.example/kept") + _, versionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/file.txt", 7, "etag-file", "checksum-file", "text/plain", model.ObjectStateCached) ts := httptest.NewServer(newBucketAPIMux(srv)) defer ts.Close() - resp, err := http.Get(ts.URL + "/api/v1/buckets/objects-bucket/objects") + deleteReq, err := http.NewRequest(http.MethodDelete, ts.URL+"/api/v1/buckets/trash-permanent-delete-bucket/objects?key="+url.QueryEscape("folder/file.txt"), nil) if err != nil { - t.Fatalf("GET bucket objects: %v", err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + t.Fatalf("NewRequest delete: %v", err) } - - var body struct { - Objects []struct { - Key string `json:"key"` - CurrentVersionID string `json:"current_version_id"` - State string `json:"state"` - Status string `json:"status"` - Location objectLocation `json:"location"` - } `json:"objects"` - Folders []objectFolderItem `json:"folders"` + setBucketWriteHeaders(deleteReq) + deleteResp, err := ts.Client().Do(deleteReq) + if err != nil { + t.Fatalf("DELETE bucket object: %v", err) } - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("Decode: %v", err) + defer func() { _ = deleteResp.Body.Close() }() + if deleteResp.StatusCode != http.StatusOK { + t.Fatalf("delete status = %d, want %d, body=%s", deleteResp.StatusCode, http.StatusOK, readBody(t, deleteResp.Body)) } - if len(body.Objects) != 1 { - t.Fatalf("objects len = %d, want 1", len(body.Objects)) + var deleteBody struct { + DeleteMarkerVersionID string `json:"delete_marker_version_id"` } - if body.Objects[0].Key != "kept.txt" { - t.Fatalf("key = %q, want %q", body.Objects[0].Key, "kept.txt") + if err := json.NewDecoder(deleteResp.Body).Decode(&deleteBody); err != nil { + t.Fatalf("Decode delete: %v", err) } - if body.Objects[0].CurrentVersionID == "" { - t.Fatal("expected current version id") + + req, err := http.NewRequest(http.MethodPost, ts.URL+"/api/v1/buckets/trash-permanent-delete-bucket/objects/deleted/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"delete_marker_version_id":%q}`, "folder/file.txt", deleteBody.DeleteMarkerVersionID))) + if err != nil { + t.Fatalf("NewRequest permanent delete: %v", err) } - if body.Objects[0].Status != "success" { - t.Fatalf("status = %q, want success", body.Objects[0].Status) + setBucketWriteHeaders(req) + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatalf("POST permanent delete deleted object: %v", err) } - if body.Objects[0].State != string(model.ObjectStateStored) { - t.Fatalf("state = %q, want stored", body.Objects[0].State) + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Fatalf("permanent delete status = %d, want %d, body=%s", resp.StatusCode, http.StatusOK, readBody(t, resp.Body)) } - if !body.Objects[0].Location.Cache || !body.Objects[0].Location.Filecoin { - t.Fatalf("location = %#v, want cache and filecoin", body.Objects[0].Location) + var body struct { + Key string `json:"key"` + DeleteMarkerVersionID string `json:"delete_marker_version_id"` + DataVersionsDeleted int `json:"data_versions_deleted"` + DeleteMarkersDeleted int `json:"delete_markers_deleted"` + StorageCleanupTaskIDs []int64 `json:"storage_cleanup_task_ids"` } - if len(body.Folders) != 0 { - t.Fatalf("folders len = %d, want 0 for flat object list", len(body.Folders)) + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("Decode permanent delete: %v", err) } - if body.Folders == nil { - t.Fatal("folders should be an empty array, not null") + if body.Key != "folder/file.txt" || body.DeleteMarkerVersionID != deleteBody.DeleteMarkerVersionID || body.DataVersionsDeleted != 1 || body.DeleteMarkersDeleted != 1 { + t.Fatalf("permanent delete response = %#v, want deleted object counts", body) } - resp, err = http.Get(ts.URL + "/api/v1/buckets/objects-bucket/objects") + deletedResp, err := http.Get(ts.URL + "/api/v1/buckets/trash-permanent-delete-bucket/objects/deleted?prefix=" + url.QueryEscape("folder/")) if err != nil { - t.Fatalf("GET bucket objects raw: %v", err) + t.Fatalf("GET deleted objects: %v", err) } - defer func() { _ = resp.Body.Close() }() - var raw struct { - Objects []map[string]any `json:"objects"` + defer func() { _ = deletedResp.Body.Close() }() + var deletedBody struct { + Objects []struct{} `json:"objects"` } - if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil { - t.Fatalf("Decode raw: %v", err) + if err := json.NewDecoder(deletedResp.Body).Decode(&deletedBody); err != nil { + t.Fatalf("Decode deleted objects: %v", err) } - if raw.Objects[0]["state"] != string(model.ObjectStateStored) { - t.Fatalf("object list state = %#v, want stored", raw.Objects[0]["state"]) + if len(deletedBody.Objects) != 0 { + t.Fatalf("deleted objects after permanent delete len = %d, want 0", len(deletedBody.Objects)) } - if _, ok := raw.Objects[0]["storage"]; ok { - t.Fatal("object list exposed storage instead of location") + + gotVersion, err := repos.Objects.GetVersionByID(ctx, versionID) + if err != nil { + t.Fatalf("GetVersionByID: %v", err) } - if _, ok := raw.Objects[0]["attention"]; ok { - t.Fatal("object list exposed attention") + if gotVersion != nil { + t.Fatalf("data version still exists after deleted object permanent delete: %#v", gotVersion) } } -func TestAPIBucketObjectsDelimiterReturnsCurrentLevelFoldersAndFiles(t *testing.T) { +func TestAPIBucketDeletedObjectPermanentDeleteReportsActiveStorageWork(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - - bucket := &model.Bucket{Name: "folder-list-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "trash-permanent-delete-busy-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - seedAdminObjectVersion(t, repos, bucket, "docs/guide.md", 4, "etag-doc", "checksum-doc", "text/markdown", "", model.ObjectStateStored) - seedAdminObjectVersion(t, repos, bucket, "photos/", 0, "etag-marker", "checksum-marker", "application/x-directory", "", model.ObjectStateCached) - seedAdminObjectVersion(t, repos, bucket, "photos/2026/a.jpg", 7, "etag-photo", "checksum-photo", "image/jpeg", "", model.ObjectStateStored) - seedAdminObjectVersion(t, repos, bucket, "root.txt", 5, "etag-root", "checksum-root", "text/plain", "", model.ObjectStateStored) - - ts := httptest.NewServer(newBucketAPIMux(srv)) - defer ts.Close() - - resp, err := http.Get(ts.URL + "/api/v1/buckets/folder-list-bucket/objects?delimiter=/") + _, versionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/file.txt", 7, "etag-file", "checksum-file", "text/plain", model.ObjectStateUploading) + if _, _, err := srv.taskService.Enqueue(ctx, taskengine.EnqueueRequest{ + Type: model.TaskTypeUploadPlan, IdempotencyKey: "upload:" + versionID, + Input: map[string]any{"version_id": versionID}, SubjectType: "object_version", SubjectKey: versionID, + }); err != nil { + t.Fatalf("Enqueue upload task: %v", err) + } + marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "folder/file.txt", model.NewVersionID()) if err != nil { - t.Fatalf("GET bucket objects: %v", err) + t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + req := httptest.NewRequest(http.MethodPost, "/api/v1/buckets/trash-permanent-delete-busy-bucket/objects/deleted/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"delete_marker_version_id":%q}`, "folder/file.txt", marker.VersionID))) + setBucketWriteHeaders(req) + rr := httptest.NewRecorder() + newBucketAPIMux(srv).ServeHTTP(rr, req) + if rr.Code != http.StatusConflict { + t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusConflict, rr.Body.String()) } - - var body objectListResponse - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + var body map[string]string + if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { t.Fatalf("Decode: %v", err) } - if len(body.Folders) != 2 { - t.Fatalf("folders len = %d, want 2: %#v", len(body.Folders), body.Folders) - } - if body.Folders[0].Name != "docs" || body.Folders[0].Prefix != "docs/" { - t.Fatalf("first folder = %#v, want docs/", body.Folders[0]) - } - if body.Folders[1].Name != "photos" || body.Folders[1].Prefix != "photos/" { - t.Fatalf("second folder = %#v, want photos/", body.Folders[1]) + want := "Storage work for one or more versions is still in progress or awaiting Filecoin confirmation. Check the related tasks, then try again." + if body["error"] != want { + t.Fatalf("error = %q, want %q", body["error"], want) } - if len(body.Objects) != 1 || body.Objects[0].Key != "root.txt" { - t.Fatalf("objects = %#v, want root.txt only", body.Objects) + for _, id := range []string{versionID, marker.VersionID} { + got, loadErr := repos.Objects.GetVersionByID(ctx, id) + if loadErr != nil || got == nil { + t.Fatalf("version %s after rejected delete = %#v err=%v, want retained", id, got, loadErr) + } } } -func TestAPIBucketObjectsDelimiterPrefixReturnsNestedLevel(t *testing.T) { - srv, repos := newBucketAPITestServer(t) - ctx := context.Background() - - bucket := &model.Bucket{Name: "nested-folder-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create: %v", err) - } - seedAdminObjectVersion(t, repos, bucket, "photos/", 0, "etag-marker", "checksum-marker", "application/x-directory", "", model.ObjectStateCached) - seedAdminObjectVersion(t, repos, bucket, "photos/cover.jpg", 5, "etag-cover", "checksum-cover", "image/jpeg", "", model.ObjectStateStored) - seedAdminObjectVersion(t, repos, bucket, "photos/2026/a.jpg", 7, "etag-photo", "checksum-photo", "image/jpeg", "", model.ObjectStateStored) - - ts := httptest.NewServer(newBucketAPIMux(srv)) - defer ts.Close() +// TestCopyHealthSummaryClassifiesCopyGapsBySeverity pins how missing replicas +// are reported. Severity comes from the copies themselves now, so a content +// whose copies all failed reads unavailable while one still in flight reads +// degraded. +func TestCopyHealthSummaryClassifiesCopyGapsBySeverity(t *testing.T) { + bucketID := int64(1) + baseContentID := int64(10) + requestedCopies := 3 - resp, err := http.Get(ts.URL + "/api/v1/buckets/nested-folder-bucket/objects?prefix=" + url.QueryEscape("photos/") + "&delimiter=/") - if err != nil { - t.Fatalf("GET bucket objects: %v", err) + tests := []struct { + name string + copyStatus *model.StorageCopyStatus + wantStatus observability.Status + wantPending int + wantFailed int + wantUnknown int + wantReason observability.ReasonCode + }{ + { + name: "in-flight copies leave the rest pending", + copyStatus: new(model.StorageCopyStatusPending), + wantStatus: observability.StatusDegraded, + wantPending: requestedCopies, + wantReason: observability.ReasonCopyPending, + }, + { + name: "failed copies make the gaps unavailable", + copyStatus: new(model.StorageCopyStatusFailed), + wantStatus: observability.StatusUnavailable, + wantFailed: requestedCopies, + wantReason: observability.ReasonCopyFailed, + }, + { + name: "content with no copies at all is unknown", + copyStatus: nil, + wantStatus: observability.StatusUnknown, + wantUnknown: requestedCopies, + wantReason: observability.ReasonCopyObservationMissing, + }, } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) - } + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + contentID := baseContentID + int64(i) + copyIndex := 0 + fact := copyHealthFact{ + BucketID: bucketID, + VersionID: fmt.Sprintf("01J0000000000000000GAP%03d", i), + ContentID: &contentID, + RequestedCopies: requestedCopies, + } + if tt.copyStatus != nil { + fact.CopyIndex = ©Index + fact.CopyStatus = tt.copyStatus + } + summaries := copyHealthSummariesByBucket([]copyHealthFact{fact}, nil, false, time.Hour) - var body objectListResponse - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("Decode: %v", err) - } - if len(body.Folders) != 1 || body.Folders[0].Name != "2026" || body.Folders[0].Prefix != "photos/2026/" { - t.Fatalf("folders = %#v, want photos/2026/", body.Folders) - } - if len(body.Objects) != 1 || body.Objects[0].Key != "photos/cover.jpg" { - t.Fatalf("objects = %#v, want photos/cover.jpg only", body.Objects) + health := summaries[bucketID] + if health.Status != string(tt.wantStatus) || + health.TotalObjects != 1 || + health.UnhealthyObjects != 1 || + health.RequestedCopies != requestedCopies || + health.PendingCopies != tt.wantPending || + health.FailedCopies != tt.wantFailed || + health.UnknownCopies != tt.wantUnknown || + !hasReason(health.ReasonCodes, observability.ReasonCopyUnderReplicated) || + !hasReason(health.ReasonCodes, tt.wantReason) { + t.Fatalf("copy_health = %#v, want %s gaps classified by copy status", health, tt.wantStatus) + } + }) } } -func TestAPIBucketObjectsDelimiterPreservesSlashOnlyFolderName(t *testing.T) { +func TestAPIBucketStorageHealthAffectedVersionsRequiresStableCutoff(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - - bucket := &model.Bucket{Name: "slash-folder-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "storage-risk-bucket", Status: model.BucketStatusActive, DefaultCopies: 2, MinimumDurableCopies: 2} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - seedAdminObjectVersion(t, repos, bucket, "a//child.txt", 1, "etag-slash", "checksum-slash", "text/plain", "", model.ObjectStateStored) - - ts := httptest.NewServer(newBucketAPIMux(srv)) - defer ts.Close() - - resp, err := http.Get(ts.URL + "/api/v1/buckets/slash-folder-bucket/objects?prefix=" + url.QueryEscape("a/") + "&delimiter=/") - if err != nil { - t.Fatalf("GET bucket objects: %v", err) + testutil.OpenBucketReplicaSlots(t, srv.db, bucket.ID, 3) + otherBucket := &model.Bucket{Name: "storage-risk-other-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, otherBucket); err != nil { + t.Fatalf("Other Buckets.Create: %v", err) } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + currentContent, err := repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, ContentSize: 4, Checksum: testutil.StorageChecksum("checksum-current-risk"), RequestedCopies: 2, + }) + if err != nil { + t.Fatalf("EnsureContent current: %v", err) } - - var body objectListResponse - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("Decode: %v", err) + currentVersion := &model.ObjectVersion{ + VersionID: "01J000000000000000APIR01", + BucketID: bucket.ID, + Key: "docs/current.txt", + ContentID: ¤tContent.ID, + Size: 4, + ETag: "etag-current-risk", + ContentType: "text/plain", } - if len(body.Folders) != 1 || body.Folders[0].Name != "/" || body.Folders[0].Prefix != "a//" { - t.Fatalf("folders = %#v, want slash-only folder a//", body.Folders) + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, currentVersion); err != nil { + t.Fatalf("CreateVersionAndSetCurrent current: %v", err) } -} - -func TestAPIBucketObjectsRejectsUnsupportedDelimiter(t *testing.T) { - srv, repos := newBucketAPITestServer(t) - ctx := context.Background() - - bucket := &model.Bucket{Name: "unsupported-delimiter-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create: %v", err) + seedAdminCommittedCopies(t, srv.db, repos, bucket.ID, currentContent.ID, "bafk2bzacestorageriskcurrent", []adminStorageCopySeed{ + {CopyIndex: 0, ProviderID: onChainID(t, "501"), DataSetID: onChainID(t, "9501"), PieceID: onChainIDPtr(t, "9901"), TransferMethod: model.StorageCopyTransferMethodIngress, RetrievalURL: "https://provider.example/current-risk"}, + {CopyIndex: 1, ProviderID: onChainID(t, "502"), DataSetID: onChainID(t, "9502"), PieceID: onChainIDPtr(t, "9902"), TransferMethod: model.StorageCopyTransferMethodPeerPull, RetrievalURL: "https://provider.example/current-readable"}, + }) + if _, err := repos.Contents.BindReadableUploadForVersion(ctx, repository.BindReadableUploadForVersionInput{ + ContentID: currentContent.ID, + BucketID: bucket.ID, + VersionID: currentVersion.VersionID, + }); err != nil { + t.Fatalf("BindReadableUploadForVersion current: %v", err) } - ts := httptest.NewServer(newBucketAPIMux(srv)) - defer ts.Close() - - resp, err := http.Get(ts.URL + "/api/v1/buckets/unsupported-delimiter-bucket/objects?delimiter=:") + oldContent, err := repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, ContentSize: 5, Checksum: testutil.StorageChecksum("checksum-old-risk"), RequestedCopies: 1, + }) if err != nil { - t.Fatalf("GET bucket objects: %v", err) + t.Fatalf("EnsureContent old: %v", err) } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusBadRequest { - t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusBadRequest) + oldVersion := &model.ObjectVersion{ + VersionID: "01J000000000000000APIR02", + BucketID: bucket.ID, + Key: "docs/old.txt", + ContentID: &oldContent.ID, + Size: 5, + ETag: "etag-old-risk", + ContentType: "text/plain", } -} - -func TestAPIBucketObjectsDelimiterPaginationSkipsDuplicateFolders(t *testing.T) { - srv, repos := newBucketAPITestServer(t) - ctx := context.Background() - - bucket := &model.Bucket{Name: "folder-page-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create: %v", err) + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion); err != nil { + t.Fatalf("CreateVersionAndSetCurrent old: %v", err) } - seedAdminObjectVersion(t, repos, bucket, "a/1.txt", 1, "etag-a1", "checksum-a1", "text/plain", "", model.ObjectStateStored) - seedAdminObjectVersion(t, repos, bucket, "a/2.txt", 1, "etag-a2", "checksum-a2", "text/plain", "", model.ObjectStateStored) - seedAdminObjectVersion(t, repos, bucket, "b/1.txt", 1, "etag-b1", "checksum-b1", "text/plain", "", model.ObjectStateStored) - - ts := httptest.NewServer(newBucketAPIMux(srv)) - defer ts.Close() - - resp, err := http.Get(ts.URL + "/api/v1/buckets/folder-page-bucket/objects?delimiter=/&limit=1") + seedAdminCommittedCopies(t, srv.db, repos, bucket.ID, oldContent.ID, "bafk2bzacestorageriskold", []adminStorageCopySeed{ + {CopyIndex: 2, ProviderID: onChainID(t, "503"), DataSetID: onChainID(t, "9503"), PieceID: onChainIDPtr(t, "9903"), TransferMethod: model.StorageCopyTransferMethodIngress, RetrievalURL: "https://provider.example/old-risk"}, + }) + if _, err := repos.Contents.BindReadableUploadForVersion(ctx, repository.BindReadableUploadForVersionInput{ + ContentID: oldContent.ID, + BucketID: bucket.ID, + VersionID: oldVersion.VersionID, + }); err != nil { + t.Fatalf("BindReadableUploadForVersion old: %v", err) + } + if err := repos.Objects.ClearContentCachePresence(ctx, oldContent.ID); err != nil { + t.Fatalf("ClearContentCachePresence old: %v", err) + } + replacementContent, err := repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, ContentSize: 6, Checksum: testutil.StorageChecksum("checksum-old-replacement"), RequestedCopies: 1, + }) if err != nil { - t.Fatalf("GET page 1: %v", err) + t.Fatalf("EnsureContent replacement: %v", err) } - defer func() { _ = resp.Body.Close() }() - - var page1 objectListResponse - if err := json.NewDecoder(resp.Body).Decode(&page1); err != nil { - t.Fatalf("Decode page 1: %v", err) + replacement := &model.ObjectVersion{ + VersionID: "01J000000000000000APIR03", + BucketID: bucket.ID, + Key: oldVersion.Key, + ContentID: &replacementContent.ID, + Size: 6, + ETag: "etag-old-replacement", + ContentType: "text/plain", } - if len(page1.Folders) != 1 || page1.Folders[0].Prefix != "a/" || len(page1.Objects) != 0 || !page1.HasMore || page1.NextMarker == "" { - t.Fatalf("page 1 = %#v, want a/ folder and next marker", page1) + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, replacement); err != nil { + t.Fatalf("CreateVersionAndSetCurrent replacement: %v", err) } - resp2, err := http.Get(ts.URL + "/api/v1/buckets/folder-page-bucket/objects?delimiter=/&limit=1&after=" + url.QueryEscape(page1.NextMarker)) + summaries, err := repos.Contents.ListDataSetSummaries(ctx, bucket.ID) if err != nil { - t.Fatalf("GET page 2: %v", err) + t.Fatalf("ListDataSetSummaries: %v", err) } - defer func() { _ = resp2.Body.Close() }() - - var page2 objectListResponse - if err := json.NewDecoder(resp2.Body).Decode(&page2); err != nil { - t.Fatalf("Decode page 2: %v", err) + byCopyIndex := make(map[int]repository.StorageDataSetSummary) + for _, summary := range summaries { + byCopyIndex[summary.CopyIndex] = summary } - if len(page2.Folders) != 1 || page2.Folders[0].Prefix != "b/" || len(page2.Objects) != 0 { - t.Fatalf("page 2 = %#v, want b/ folder only", page2) + checkedAt := time.Now().UTC().Truncate(time.Microsecond) + if checkedAt.Nanosecond() == 0 { + checkedAt = checkedAt.Add(time.Microsecond) } -} - -func TestListBucketObjectEntriesSkipsEmittedFolderSubtree(t *testing.T) { - keys := make([]string, adminObjectListingBatchSize*2+1) - for i := range keys { - keys[i] = fmt.Sprintf("a/%04d.txt", i) + if err := repos.Observability.ReplaceDataSetStates(ctx, checkedAt, []observability.DataSetState{ + {LocalDataSetID: byCopyIndex[0].ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: 0, ProviderID: byCopyIndex[0].ProviderID, ChainDataSetID: byCopyIndex[0].DataSetID, LocalStatus: byCopyIndex[0].Status, Status: observability.StatusUnavailable, ReasonCodes: []observability.ReasonCode{observability.ReasonChainDataSetMissing}, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, + {LocalDataSetID: byCopyIndex[1].ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: 1, ProviderID: byCopyIndex[1].ProviderID, ChainDataSetID: byCopyIndex[1].DataSetID, LocalStatus: byCopyIndex[1].Status, Status: observability.StatusAvailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, + {LocalDataSetID: byCopyIndex[2].ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: 2, ProviderID: byCopyIndex[2].ProviderID, ChainDataSetID: byCopyIndex[2].DataSetID, LocalStatus: byCopyIndex[2].Status, Status: observability.StatusDegraded, ReasonCodes: []observability.ReasonCode{observability.ReasonChainDataSetUnmanaged}, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, + }); err != nil { + t.Fatalf("ReplaceDataSetStates: %v", err) } - objects := &recordingObjectListRepo{keys: keys} - srv := &Server{repos: &repository.Repositories{Objects: objects}} - folders, files, hasMore, nextMarker, err := srv.listBucketObjectEntries(t.Context(), 1, "", "/", "", 50) + ts := httptest.NewServer(newBucketAPIMux(srv)) + defer ts.Close() + + resp, err := http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?prefix=" + url.QueryEscape("docs/") + "&limit=1") if err != nil { - t.Fatalf("listBucketObjectEntries: %v", err) + t.Fatalf("GET affected versions: %v", err) } - - if len(folders) != 1 || folders[0].Prefix != "a/" || len(files) != 0 || hasMore || nextMarker != "" { - t.Fatalf("listing = folders:%#v files:%#v hasMore:%v nextMarker:%q, want a/ folder only", folders, files, hasMore, nextMarker) + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) } - if objects.scanCalls() > 2 { - t.Fatalf("object list scans = %d, want at most 2 without walking every child batch", objects.scanCalls()) + var body struct { + CopyHealth json.RawMessage `json:"copy_health"` + Versions []struct { + Key string `json:"key"` + VersionID string `json:"version_id"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + IsCurrent bool `json:"is_current"` + InCache bool `json:"in_cache"` + ReadableAlternativeCount int `json:"readable_alternative_count"` + HasReadableAlternative bool `json:"has_readable_alternative"` + CopyHealth json.RawMessage `json:"copy_health"` + RiskDataSets []struct { + LocalDataSetID int64 `json:"local_data_set_id"` + CopyIndex int `json:"copy_index"` + ProviderID string `json:"provider_id"` + DataSetID string `json:"data_set_id"` + LocalStatus string `json:"local_status"` + StorageHealth struct { + Status string `json:"status"` + ReasonCodes []observability.ReasonCode `json:"reason_codes"` + LastChecked string `json:"last_checked_at"` + } `json:"storage_health"` + } `json:"risk_data_sets"` + } `json:"versions"` + HasMore bool `json:"has_more"` + NextKeyMarker string `json:"next_key_marker"` + NextVersionMarker string `json:"next_version_marker"` + NextCreatedAt string `json:"next_created_at_marker"` + StaleBefore string `json:"stale_before"` } -} - -func TestListBucketObjectEntriesKeepsCurrentBatchAcrossSiblingFolders(t *testing.T) { - keys := make([]string, 50) - for i := range keys { - keys[i] = fmt.Sprintf("dir-%02d/file.txt", i) + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("Decode: %v", err) } - objects := &recordingObjectListRepo{keys: keys} - srv := &Server{repos: &repository.Repositories{Objects: objects}} - - folders, files, hasMore, nextMarker, err := srv.listBucketObjectEntries(t.Context(), 1, "", "/", "", 50) - if err != nil { - t.Fatalf("listBucketObjectEntries: %v", err) + if body.CopyHealth != nil { + t.Fatalf("response copy_health = %s, want omitted", body.CopyHealth) } - - if len(folders) != 50 || len(files) != 0 || hasMore || nextMarker != "" { - t.Fatalf("listing = folders:%d files:%#v hasMore:%v nextMarker:%q, want 50 folders only", len(folders), files, hasMore, nextMarker) + if len(body.Versions) != 1 || body.Versions[0].VersionID != currentVersion.VersionID || !body.Versions[0].IsCurrent { + t.Fatalf("first page versions = %#v, want current risk version first", body.Versions) } - if objects.scanCalls() != 1 { - t.Fatalf("object list scans = %d, want 1 for sibling folders in one batch", objects.scanCalls()) + if !body.Versions[0].InCache { + t.Fatalf("current in_cache = false, want true") } -} - -func TestListBucketObjectEntriesSkipsDuplicateRowsBeforeSiblingFolders(t *testing.T) { - keys := make([]string, 0, 100) - for i := 0; i < 50; i++ { - keys = append(keys, fmt.Sprintf("dir-%02d/a.txt", i), fmt.Sprintf("dir-%02d/b.txt", i)) + if _, err := time.Parse(time.RFC3339Nano, body.Versions[0].CreatedAt); err != nil { + t.Fatalf("created_at = %q, want RFC3339Nano: %v", body.Versions[0].CreatedAt, err) } - objects := &recordingObjectListRepo{keys: keys} - srv := &Server{repos: &repository.Repositories{Objects: objects}} - - folders, files, hasMore, nextMarker, err := srv.listBucketObjectEntries(t.Context(), 1, "", "/", "", 50) - if err != nil { - t.Fatalf("listBucketObjectEntries: %v", err) + if _, err := time.Parse(time.RFC3339Nano, body.Versions[0].UpdatedAt); err != nil { + t.Fatalf("updated_at = %q, want RFC3339Nano: %v", body.Versions[0].UpdatedAt, err) } - - if len(folders) != 50 || len(files) != 0 || hasMore || nextMarker != "" { - t.Fatalf("listing = folders:%d files:%#v hasMore:%v nextMarker:%q, want 50 folders only", len(folders), files, hasMore, nextMarker) + if body.Versions[0].CopyHealth != nil { + t.Fatalf("version copy_health = %s, want omitted", body.Versions[0].CopyHealth) } - if objects.scanCalls() != 1 { - t.Fatalf("object list scans = %d, want 1 while duplicate folder rows fit in one batch", objects.scanCalls()) + if !body.Versions[0].HasReadableAlternative || body.Versions[0].ReadableAlternativeCount != 1 { + t.Fatalf("current alternative = %#v, want one recorded readable alternative", body.Versions[0]) } -} - -func TestAPIBucketObjects_DeleteListAndRestore(t *testing.T) { - srv, repos := newBucketAPITestServer(t) - ctx := context.Background() - - bucket := &model.Bucket{Name: "trash-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create: %v", err) + if len(body.Versions[0].RiskDataSets) != 1 || body.Versions[0].RiskDataSets[0].LocalDataSetID != byCopyIndex[0].ID || body.Versions[0].RiskDataSets[0].StorageHealth.Status != string(observability.StatusUnavailable) { + t.Fatalf("current risk datasets = %#v, want unavailable dataset", body.Versions[0].RiskDataSets) + } + if !body.HasMore || body.NextKeyMarker != currentVersion.Key || body.NextVersionMarker != currentVersion.VersionID || body.NextCreatedAt == "" { + t.Fatalf("pagination = %#v, want next marker after current", body) + } + if _, err := time.Parse(time.RFC3339Nano, body.NextCreatedAt); err != nil { + t.Fatalf("next_created_at_marker = %q, want RFC3339Nano: %v", body.NextCreatedAt, err) + } + if _, err := time.Parse(time.RFC3339Nano, body.StaleBefore); err != nil { + t.Fatalf("stale_before = %q, want RFC3339Nano: %v", body.StaleBefore, err) + } + if got, want := body.Versions[0].RiskDataSets[0].StorageHealth.LastChecked, checkedAt.Format(time.RFC3339Nano); got != want { + t.Fatalf("risk dataset last_checked_at = %q, want %q", got, want) } - _, versionID := seedAdminObjectVersion(t, repos, bucket, "folder/file.txt", 7, "etag-file", "checksum-file", "text/plain", "", model.ObjectStateCached) - ts := httptest.NewServer(newBucketAPIMux(srv)) - defer ts.Close() - - deleteReq, err := http.NewRequest(http.MethodDelete, ts.URL+"/api/v1/buckets/trash-bucket/objects?key="+url.QueryEscape("folder/file.txt"), nil) - if err != nil { - t.Fatalf("NewRequest delete: %v", err) - } - setBucketWriteHeaders(deleteReq) - deleteResp, err := ts.Client().Do(deleteReq) + secondPageURL := ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?key_marker=" + url.QueryEscape(body.NextKeyMarker) + "&version_marker=" + url.QueryEscape(body.NextVersionMarker) + "&created_at_marker=" + url.QueryEscape(body.NextCreatedAt) + resp, err = http.Get(secondPageURL) if err != nil { - t.Fatalf("DELETE bucket object: %v", err) - } - defer func() { _ = deleteResp.Body.Close() }() - if deleteResp.StatusCode != http.StatusOK { - t.Fatalf("delete status = %d, want %d, body=%s", deleteResp.StatusCode, http.StatusOK, readBody(t, deleteResp.Body)) + t.Fatalf("GET affected versions second page: %v", err) } - var deleteBody struct { - Key string `json:"key"` - DeleteMarkerVersionID string `json:"delete_marker_version_id"` - DeletedAt string `json:"deleted_at"` + defer func() { _ = resp.Body.Close() }() + var missingCutoffBody struct { + Error string `json:"error"` } - if err := json.NewDecoder(deleteResp.Body).Decode(&deleteBody); err != nil { - t.Fatalf("Decode delete: %v", err) + if err := json.NewDecoder(resp.Body).Decode(&missingCutoffBody); err != nil { + t.Fatalf("Decode missing cutoff: %v", err) } - if deleteBody.Key != "folder/file.txt" || deleteBody.DeleteMarkerVersionID == "" || deleteBody.DeletedAt == "" { - t.Fatalf("delete response = %#v, want marker metadata", deleteBody) + if resp.StatusCode != http.StatusBadRequest || missingCutoffBody.Error != "key_marker, version_marker, created_at_marker, and stale_before must be provided together" { + t.Fatalf("second page without cutoff status/body = %d/%#v, want stable cutoff error", resp.StatusCode, missingCutoffBody) } - listResp, err := http.Get(ts.URL + "/api/v1/buckets/trash-bucket/objects?prefix=" + url.QueryEscape("folder/")) + secondPageURL += "&stale_before=" + url.QueryEscape(body.StaleBefore) + resp, err = http.Get(secondPageURL) if err != nil { - t.Fatalf("GET live objects: %v", err) - } - defer func() { _ = listResp.Body.Close() }() - var live objectListResponse - if err := json.NewDecoder(listResp.Body).Decode(&live); err != nil { - t.Fatalf("Decode live objects: %v", err) + t.Fatalf("GET affected versions second page with cutoff: %v", err) } - if len(live.Objects) != 0 { - t.Fatalf("live objects = %#v, want deleted object hidden", live.Objects) + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Fatalf("second page with cutoff status = %d, want %d", resp.StatusCode, http.StatusOK) } - versionsResp, err := http.Get(ts.URL + "/api/v1/buckets/trash-bucket/objects/versions?key=" + url.QueryEscape("folder/file.txt")) + resp, err = http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?local_data_set_id=" + strconv.FormatInt(byCopyIndex[2].ID, 10)) if err != nil { - t.Fatalf("GET object versions: %v", err) + t.Fatalf("GET affected versions dataset: %v", err) } - defer func() { _ = versionsResp.Body.Close() }() - var versionsBody struct { + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Fatalf("dataset status = %d, want %d", resp.StatusCode, http.StatusOK) + } + var datasetBody struct { Versions []struct { - VersionID string `json:"version_id"` - IsCurrent bool `json:"is_current"` - IsDeleteMarker bool `json:"is_delete_marker"` - UploadStatus string `json:"upload_status"` - DownloadVisible bool `json:"download_visible"` + VersionID string `json:"version_id"` + IsCurrent bool `json:"is_current"` + InCache bool `json:"in_cache"` + ReadableAlternativeCount int `json:"readable_alternative_count"` } `json:"versions"` } - if err := json.NewDecoder(versionsResp.Body).Decode(&versionsBody); err != nil { - t.Fatalf("Decode versions: %v", err) - } - if len(versionsBody.Versions) != 2 { - t.Fatalf("versions len = %d, want 2", len(versionsBody.Versions)) + if err := json.NewDecoder(resp.Body).Decode(&datasetBody); err != nil { + t.Fatalf("Decode dataset: %v", err) } - if versionsBody.Versions[0].VersionID != deleteBody.DeleteMarkerVersionID || !versionsBody.Versions[0].IsCurrent || !versionsBody.Versions[0].IsDeleteMarker { - t.Fatalf("first version = %#v, want current delete marker", versionsBody.Versions[0]) + if len(datasetBody.Versions) != 1 || datasetBody.Versions[0].VersionID != oldVersion.VersionID || datasetBody.Versions[0].IsCurrent || datasetBody.Versions[0].ReadableAlternativeCount != 0 { + t.Fatalf("dataset filtered versions = %#v, want retained old version without alternative", datasetBody.Versions) } - if versionsBody.Versions[1].VersionID != versionID || versionsBody.Versions[1].IsDeleteMarker { - t.Fatalf("second version = %#v, want data version %s", versionsBody.Versions[1], versionID) + if datasetBody.Versions[0].InCache { + t.Fatalf("dataset filtered in_cache = true, want false") } - deletedResp, err := http.Get(ts.URL + "/api/v1/buckets/trash-bucket/objects/deleted?prefix=" + url.QueryEscape("folder/")) + resp, err = http.Get(ts.URL + "/api/v1/buckets/" + otherBucket.Name + "/storage-health/affected-versions?local_data_set_id=" + strconv.FormatInt(byCopyIndex[0].ID, 10)) if err != nil { - t.Fatalf("GET deleted objects: %v", err) - } - defer func() { _ = deletedResp.Body.Close() }() - var deletedBody struct { - Objects []struct { - Key string `json:"key"` - DeleteMarkerVersionID string `json:"delete_marker_version_id"` - DeletedAt string `json:"deleted_at"` - RestoreVersionID string `json:"restore_version_id"` - RestoreSize int64 `json:"restore_size"` - RestoreContentType string `json:"restore_content_type"` - RestoreETag string `json:"restore_etag"` - } `json:"objects"` - HasMore bool `json:"has_more"` + t.Fatalf("GET affected versions other bucket dataset: %v", err) } - if err := json.NewDecoder(deletedResp.Body).Decode(&deletedBody); err != nil { - t.Fatalf("Decode deleted objects: %v", err) + defer func() { _ = resp.Body.Close() }() + var emptyBody struct { + Versions []struct{} `json:"versions"` } - if len(deletedBody.Objects) != 1 { - t.Fatalf("deleted objects len = %d, want 1", len(deletedBody.Objects)) + if err := json.NewDecoder(resp.Body).Decode(&emptyBody); err != nil { + t.Fatalf("Decode other bucket dataset: %v", err) } - deleted := deletedBody.Objects[0] - if deleted.Key != "folder/file.txt" || deleted.DeleteMarkerVersionID != deleteBody.DeleteMarkerVersionID || deleted.RestoreVersionID != versionID || deleted.RestoreSize != 7 || deleted.RestoreContentType != "text/plain" || deleted.RestoreETag != "etag-file" || deleted.DeletedAt == "" { - t.Fatalf("deleted object = %#v, want recoverable file", deleted) + if resp.StatusCode != http.StatusOK || len(emptyBody.Versions) != 0 { + t.Fatalf("other bucket dataset status/body = %d/%#v, want empty OK", resp.StatusCode, emptyBody) } - restoreReq, err := http.NewRequest(http.MethodPost, ts.URL+"/api/v1/buckets/trash-bucket/objects/restore", strings.NewReader(fmt.Sprintf(`{"key":%q,"delete_marker_version_id":%q}`, "folder/file.txt", deleteBody.DeleteMarkerVersionID))) + resp, err = http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?prefix=docs/&key=docs/current.txt") if err != nil { - t.Fatalf("NewRequest restore: %v", err) + t.Fatalf("GET affected versions conflicting filters: %v", err) } - setBucketWriteHeaders(restoreReq) - restoreResp, err := ts.Client().Do(restoreReq) + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("conflicting filter status = %d, want %d", resp.StatusCode, http.StatusBadRequest) + } + + resp, err = http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?local_data_set_id=bad") if err != nil { - t.Fatalf("POST restore object: %v", err) + t.Fatalf("GET affected versions invalid dataset: %v", err) } - defer func() { _ = restoreResp.Body.Close() }() - if restoreResp.StatusCode != http.StatusOK { - t.Fatalf("restore status = %d, want %d, body=%s", restoreResp.StatusCode, http.StatusOK, readBody(t, restoreResp.Body)) + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("invalid dataset status = %d, want %d", resp.StatusCode, http.StatusBadRequest) } - var restoreBody struct { - Key string `json:"key"` - RestoredVersionID string `json:"restored_version_id"` + + for _, query := range []string{ + "key_marker=" + url.QueryEscape(currentVersion.Key), + "key_marker=" + url.QueryEscape(currentVersion.Key) + "&version_marker=" + url.QueryEscape(currentVersion.VersionID), + } { + resp, err = http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?" + query) + if err != nil { + t.Fatalf("GET affected versions invalid marker %q: %v", query, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("invalid marker %q status = %d, want %d", query, resp.StatusCode, http.StatusBadRequest) + } } - if err := json.NewDecoder(restoreResp.Body).Decode(&restoreBody); err != nil { - t.Fatalf("Decode restore: %v", err) + resp, err = http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?key_marker=" + url.QueryEscape(currentVersion.Key) + "&version_marker=" + url.QueryEscape(currentVersion.VersionID) + "&created_at_marker=not-a-time&stale_before=" + url.QueryEscape(body.StaleBefore)) + if err != nil { + t.Fatalf("GET affected versions invalid marker time: %v", err) } - if restoreBody.Key != "folder/file.txt" || restoreBody.RestoredVersionID != versionID { - t.Fatalf("restore response = %#v, want data version restored", restoreBody) + defer func() { _ = resp.Body.Close() }() + var invalidTimeBody struct { + Error string `json:"error"` } - - restoredListResp, err := http.Get(ts.URL + "/api/v1/buckets/trash-bucket/objects?prefix=" + url.QueryEscape("folder/")) + if err := json.NewDecoder(resp.Body).Decode(&invalidTimeBody); err != nil { + t.Fatalf("Decode invalid marker time: %v", err) + } + if resp.StatusCode != http.StatusBadRequest || invalidTimeBody.Error != "created_at_marker must be RFC3339Nano" { + t.Fatalf("invalid marker time status/body = %d/%#v, want RFC3339Nano error", resp.StatusCode, invalidTimeBody) + } + resp, err = http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?stale_before=not-a-time") if err != nil { - t.Fatalf("GET restored objects: %v", err) + t.Fatalf("GET affected versions invalid stale_before: %v", err) } - defer func() { _ = restoredListResp.Body.Close() }() - var restoredLive objectListResponse - if err := json.NewDecoder(restoredListResp.Body).Decode(&restoredLive); err != nil { - t.Fatalf("Decode restored objects: %v", err) + defer func() { _ = resp.Body.Close() }() + var invalidCutoffBody struct { + Error string `json:"error"` } - if len(restoredLive.Objects) != 1 || restoredLive.Objects[0].Key != "folder/file.txt" || restoredLive.Objects[0].CurrentVersionID != versionID { - t.Fatalf("restored live objects = %#v, want restored file", restoredLive.Objects) + if err := json.NewDecoder(resp.Body).Decode(&invalidCutoffBody); err != nil { + t.Fatalf("Decode invalid stale_before: %v", err) + } + if resp.StatusCode != http.StatusBadRequest || invalidCutoffBody.Error != "stale_before must be RFC3339Nano" { + t.Fatalf("invalid stale_before status/body = %d/%#v, want RFC3339Nano error", resp.StatusCode, invalidCutoffBody) } - emptyDeletedResp, err := http.Get(ts.URL + "/api/v1/buckets/trash-bucket/objects/deleted?prefix=" + url.QueryEscape("folder/")) + resp, err = http.Get(ts.URL + "/api/v1/buckets/BadBucket/storage-health/affected-versions") if err != nil { - t.Fatalf("GET deleted objects after restore: %v", err) + t.Fatalf("GET affected versions invalid bucket: %v", err) } - defer func() { _ = emptyDeletedResp.Body.Close() }() - var emptyDeleted struct { - Objects []struct{} `json:"objects"` + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("invalid bucket status = %d, want %d", resp.StatusCode, http.StatusBadRequest) } - if err := json.NewDecoder(emptyDeletedResp.Body).Decode(&emptyDeleted); err != nil { - t.Fatalf("Decode empty deleted: %v", err) + + resp, err = http.Get(ts.URL + "/api/v1/buckets/missing-storage-risk-bucket/storage-health/affected-versions") + if err != nil { + t.Fatalf("GET affected versions missing bucket: %v", err) } - if len(emptyDeleted.Objects) != 0 { - t.Fatalf("deleted objects after restore len = %d, want 0", len(emptyDeleted.Objects)) + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("missing bucket status = %d, want %d", resp.StatusCode, http.StatusNotFound) } } -func TestAPIBucketObjectDeleteRejectsInvalidObjectKey(t *testing.T) { +func TestAPIBucketStorageHealthAffectedVersionsClampsLimit(t *testing.T) { srv, repos := newBucketAPITestServer(t) - bucket := &model.Bucket{Name: "delete-key-validation-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(context.Background(), bucket); err != nil { + ctx := context.Background() + bucket := &model.Bucket{Name: "storage-risk-limit-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - key := strings.Repeat("你", 342) - req := httptest.NewRequest(http.MethodDelete, "/api/v1/buckets/delete-key-validation-bucket/objects?key="+url.QueryEscape(key), nil) - req.SetPathValue("name", bucket.Name) - setBucketWriteHeaders(req) - rr := httptest.NewRecorder() - srv.handleAPIDeleteBucketObject(rr, req) + for i := range 51 { + _, versionID := seedAdminObjectVersion(t, srv.db, repos, bucket, fmt.Sprintf("docs/limit-%03d.txt", i), int64(i+1), + fmt.Sprintf("etag-limit-%03d", i), fmt.Sprintf("checksum-limit-%03d", i), "text/plain", model.ObjectStateCached) + version, err := repos.Objects.GetVersionByID(ctx, versionID) + if err != nil || version == nil { + t.Fatalf("GetVersionByID %d: version=%v err=%v", i, version, err) + } + content := adminContentForVersion(t, repos, version) + seedAdminCommittedCopies(t, srv.db, repos, bucket.ID, content.ID, fmt.Sprintf("bafk2bzacelimit%03d", i), []adminStorageCopySeed{{ + CopyIndex: 0, + ProviderID: onChainID(t, "901"), + DataSetID: onChainID(t, "9901"), + PieceID: onChainIDPtr(t, fmt.Sprintf("99010%d", i)), + TransferMethod: model.StorageCopyTransferMethodIngress, + RetrievalURL: fmt.Sprintf("https://provider.example/limit-%03d", i), + }}) + if _, err := repos.Contents.BindReadableUploadForVersion(ctx, repository.BindReadableUploadForVersionInput{ + ContentID: content.ID, + BucketID: bucket.ID, + VersionID: version.VersionID, + }); err != nil { + t.Fatalf("BindReadableUploadForVersion %d: %v", i, err) + } + } - if rr.Code != http.StatusBadRequest { - t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusBadRequest, rr.Body.String()) + summaries, err := repos.Contents.ListDataSetSummaries(ctx, bucket.ID) + if err != nil { + t.Fatalf("ListDataSetSummaries: %v", err) + } + if len(summaries) != 1 { + t.Fatalf("data set summaries = %d, want 1", len(summaries)) + } + checkedAt := time.Now().UTC() + if err := repos.Observability.ReplaceDataSetStates(ctx, checkedAt, []observability.DataSetState{{ + LocalDataSetID: summaries[0].ID, + BucketID: bucket.ID, + BucketName: bucket.Name, + CopyIndex: summaries[0].CopyIndex, + ProviderID: summaries[0].ProviderID, + ChainDataSetID: summaries[0].DataSetID, + LocalStatus: summaries[0].Status, + Status: observability.StatusUnavailable, + ReasonCodes: []observability.ReasonCode{observability.ReasonChainDataSetMissing}, + LastCheckedAt: checkedAt, + Evidence: map[string]any{}, + }}); err != nil { + t.Fatalf("ReplaceDataSetStates: %v", err) + } + + ts := httptest.NewServer(newBucketAPIMux(srv)) + defer ts.Close() + resp, err := http.Get(ts.URL + "/api/v1/buckets/" + bucket.Name + "/storage-health/affected-versions?limit=1001") + if err != nil { + t.Fatalf("GET affected versions: %v", err) } - if !strings.Contains(rr.Body.String(), "1024 UTF-8 bytes") { - t.Fatalf("body = %s, want object key byte-limit error", rr.Body.String()) + defer func() { _ = resp.Body.Close() }() + var body struct { + Versions []struct{} `json:"versions"` + HasMore bool `json:"has_more"` } - version, err := repos.Objects.GetCurrentVersionByBucketAndKey(context.Background(), bucket.ID, key) - if err != nil { - t.Fatalf("GetCurrentVersionByBucketAndKey: %v", err) + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("Decode: %v", err) } - if version != nil { - t.Fatalf("current version = %#v, want no persisted delete marker", version) + if resp.StatusCode != http.StatusOK || body.HasMore || len(body.Versions) != 51 { + t.Fatalf("status/body = %d/%#v, want all 51 versions with clamped limit", resp.StatusCode, body) } } -func TestAPIBucketDeletedObjectPermanentDeleteRemovesDeletedObject(t *testing.T) { +// TestAPIBucketObjects_StatusMappingAndDetail pins the operator-facing status a +// version maps to. Every input is now a fact about the content's copies or its +// cache entry, so the mapping is exercised through those rather than through a +// state column. +func TestAPIBucketObjects_StatusMappingAndDetail(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "trash-permanent-delete-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "status-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - _, versionID := seedAdminObjectVersion(t, repos, bucket, "folder/file.txt", 7, "etag-file", "checksum-file", "text/plain", "", model.ObjectStateCached) + _, warningVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "warning.txt", 4, "etag-warning", "checksum-warning", "text/plain", model.ObjectStateCached) + markAdminFailedUpload(t, srv.db, repos, warningVersionID, "provider rejected piece") + + _, unavailableVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "unavailable.txt", 1, "etag-unavailable", "checksum-unavailable", "text/plain", model.ObjectStateCached) + unavailableVersion, err := repos.Objects.GetVersionByID(ctx, unavailableVersionID) + if err != nil || unavailableVersion == nil { + t.Fatalf("GetVersionByID(unavailable): version=%v err=%v", unavailableVersion, err) + } + if err := repos.Objects.ClearContentCachePresence(ctx, *unavailableVersion.ContentID); err != nil { + t.Fatalf("unavailable cache presence: %v", err) + } + + _, storedVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "stored.txt", 2, "etag-stored", "checksum-stored", "text/plain", model.ObjectStateCached) + acceptAdminVersionUpload(t, srv.db, repos, storedVersionID, "piece-stored", "https://provider.example/stored") + + partialCopies := 2 + partialBucket := &model.Bucket{Name: "status-partial-bucket", Status: model.BucketStatusActive, DefaultCopies: partialCopies, MinimumDurableCopies: partialCopies} + if err := repos.Buckets.Create(ctx, partialBucket); err != nil { + t.Fatalf("Buckets.Create(partial): %v", err) + } + _, partialVersionID := seedAdminObjectVersion(t, srv.db, repos, partialBucket, "partial.txt", 3, "etag-partial", "checksum-partial", "text/plain", model.ObjectStateCached) + bindAdminPartialUpload(t, srv.db, repos, partialVersionID) ts := httptest.NewServer(newBucketAPIMux(srv)) defer ts.Close() - deleteReq, err := http.NewRequest(http.MethodDelete, ts.URL+"/api/v1/buckets/trash-permanent-delete-bucket/objects?key="+url.QueryEscape("folder/file.txt"), nil) - if err != nil { - t.Fatalf("NewRequest delete: %v", err) + statusByKey := func(bucketName string) map[string]string { + t.Helper() + resp, err := http.Get(ts.URL + "/api/v1/buckets/" + bucketName + "/objects") + if err != nil { + t.Fatalf("GET bucket objects: %v", err) + } + defer func() { _ = resp.Body.Close() }() + var list struct { + Objects []struct { + Key string `json:"key"` + Status string `json:"status"` + } `json:"objects"` + } + if err := json.NewDecoder(resp.Body).Decode(&list); err != nil { + t.Fatalf("Decode list: %v", err) + } + byKey := map[string]string{} + for _, object := range list.Objects { + byKey[object.Key] = object.Status + } + return byKey } - setBucketWriteHeaders(deleteReq) - deleteResp, err := ts.Client().Do(deleteReq) - if err != nil { - t.Fatalf("DELETE bucket object: %v", err) + + main := statusByKey("status-bucket") + if main["warning.txt"] != "warning" { + t.Fatalf("warning status = %q, want warning", main["warning.txt"]) } - defer func() { _ = deleteResp.Body.Close() }() - if deleteResp.StatusCode != http.StatusOK { - t.Fatalf("delete status = %d, want %d, body=%s", deleteResp.StatusCode, http.StatusOK, readBody(t, deleteResp.Body)) + if main["unavailable.txt"] != "unavailable" { + t.Fatalf("unavailable status = %q, want unavailable", main["unavailable.txt"]) } - var deleteBody struct { - DeleteMarkerVersionID string `json:"delete_marker_version_id"` + if main["stored.txt"] != "success" { + t.Fatalf("stored status = %q, want success", main["stored.txt"]) } - if err := json.NewDecoder(deleteResp.Body).Decode(&deleteBody); err != nil { - t.Fatalf("Decode delete: %v", err) + if partial := statusByKey("status-partial-bucket"); partial["partial.txt"] != "syncing" { + t.Fatalf("partial status = %q, want syncing", partial["partial.txt"]) } - req, err := http.NewRequest(http.MethodPost, ts.URL+"/api/v1/buckets/trash-permanent-delete-bucket/objects/deleted/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"delete_marker_version_id":%q}`, "folder/file.txt", deleteBody.DeleteMarkerVersionID))) - if err != nil { - t.Fatalf("NewRequest permanent delete: %v", err) - } - setBucketWriteHeaders(req) - resp, err := ts.Client().Do(req) + resp, err := http.Get(ts.URL + "/api/v1/buckets/status-bucket/objects/status-detail?version_id=" + url.QueryEscape(warningVersionID)) if err != nil { - t.Fatalf("POST permanent delete deleted object: %v", err) + t.Fatalf("GET status detail: %v", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - t.Fatalf("permanent delete status = %d, want %d, body=%s", resp.StatusCode, http.StatusOK, readBody(t, resp.Body)) - } - var body struct { - Key string `json:"key"` - DeleteMarkerVersionID string `json:"delete_marker_version_id"` - DataVersionsDeleted int `json:"data_versions_deleted"` - DeleteMarkersDeleted int `json:"delete_markers_deleted"` - StorageCleanupTaskIDs []int64 `json:"storage_cleanup_task_ids"` - } - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("Decode permanent delete: %v", err) - } - if body.Key != "folder/file.txt" || body.DeleteMarkerVersionID != deleteBody.DeleteMarkerVersionID || body.DataVersionsDeleted != 1 || body.DeleteMarkersDeleted != 1 { - t.Fatalf("permanent delete response = %#v, want deleted object counts", body) + t.Fatalf("status detail code = %d, want %d", resp.StatusCode, http.StatusOK) } - - deletedResp, err := http.Get(ts.URL + "/api/v1/buckets/trash-permanent-delete-bucket/objects/deleted?prefix=" + url.QueryEscape("folder/")) - if err != nil { - t.Fatalf("GET deleted objects: %v", err) + var detail objectStatusDetailResponse + if err := json.NewDecoder(resp.Body).Decode(&detail); err != nil { + t.Fatalf("Decode detail: %v", err) } - defer func() { _ = deletedResp.Body.Close() }() - var deletedBody struct { - Objects []struct{} `json:"objects"` + if detail.VersionID != warningVersionID || detail.Status != "warning" { + t.Fatalf("detail = %#v, want version %s warning", detail, warningVersionID) } - if err := json.NewDecoder(deletedResp.Body).Decode(&deletedBody); err != nil { - t.Fatalf("Decode deleted objects: %v", err) + if detail.State != string(model.ObjectStateFailed) { + t.Fatalf("detail state = %q, want failed", detail.State) } - if len(deletedBody.Objects) != 0 { - t.Fatalf("deleted objects after permanent delete len = %d, want 0", len(deletedBody.Objects)) + if detail.Message == nil || *detail.Message != "provider rejected piece" { + t.Fatalf("message = %#v, want provider rejected piece", detail.Message) } - gotVersion, err := repos.Objects.GetVersionByID(ctx, versionID) + otherBucket := &model.Bucket{Name: "other-status-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, otherBucket); err != nil { + t.Fatalf("other bucket: %v", err) + } + resp, err = http.Get(ts.URL + "/api/v1/buckets/other-status-bucket/objects/status-detail?version_id=" + url.QueryEscape(warningVersionID)) if err != nil { - t.Fatalf("GetVersionByID: %v", err) + t.Fatalf("GET status detail wrong bucket: %v", err) } - if gotVersion != nil { - t.Fatalf("data version still exists after deleted object permanent delete: %#v", gotVersion) + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("wrong bucket status detail code = %d, want %d", resp.StatusCode, http.StatusNotFound) } } -func TestAPIBucketDeletedObjectPermanentDeleteReportsActiveStorageWork(t *testing.T) { +func TestAPIBucketObjectProvenance(t *testing.T) { srv, repos := newBucketAPITestServer(t) + identityResolver := &fakeAPIProviderIdentityResolver{ + identities: map[string]*providerIdentityResponse{ + "101": { + RegistryProviderID: "101", + Name: "alpha-pdp", + ServiceProviderAddress: "0x1111111111111111111111111111111111111111", + FilecoinActorID: "f01234", + ServiceURL: "https://alpha.example", + }, + }, + } + srv.WithProviderIdentityResolver(identityResolver) ctx := context.Background() - bucket := &model.Bucket{Name: "trash-permanent-delete-busy-bucket", Status: model.BucketStatusActive} + + provenanceCopies := 2 + bucket := &model.Bucket{Name: "provenance-bucket", Status: model.BucketStatusActive, DefaultCopies: provenanceCopies, MinimumDurableCopies: provenanceCopies} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - objectID, versionID := seedAdminObjectVersion(t, repos, bucket, "folder/file.txt", 7, "etag-file", "checksum-file", "text/plain", "", model.ObjectStateUploading) - stage := "prepare_upload" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "object", RefID: objectID, RefVersionID: versionID, - IdempotencyKey: "upload:" + versionID, Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), + _, oldVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "file.txt", 7, "etag-old-provenance", "checksum-old-provenance", "text/plain", model.ObjectStateStored) + _, versionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "file.txt", 8, "etag-provenance", "checksum-provenance", "text/plain", model.ObjectStateCached) + version, err := repos.Objects.GetVersionByID(ctx, versionID) + if err != nil || version == nil { + t.Fatalf("GetVersionByID: version=%v err=%v", version, err) } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Tasks.Create: %v", err) + content := adminContentForVersion(t, repos, version) + pieceCID := "bafk2bzaceadminprovenance" + seedAdminCommittedCopies(t, srv.db, repos, bucket.ID, content.ID, pieceCID, []adminStorageCopySeed{ + {ProviderID: onChainID(t, "101"), DataSetID: onChainID(t, "1001"), PieceID: onChainIDPtr(t, "2001"), TransferMethod: model.StorageCopyTransferMethodIngress, RetrievalURL: "https://ingress.example/piece"}, + {ProviderID: onChainID(t, "102"), DataSetID: onChainID(t, "2002"), PieceID: onChainIDPtr(t, "3001"), TransferMethod: model.StorageCopyTransferMethodPeerPull, RetrievalURL: "https://peer.example/piece"}, + }) + if _, err := repos.Contents.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ + ContentID: content.ID, + BucketID: bucket.ID, + }); err != nil { + t.Fatalf("BindReadableUploadForContent: %v", err) } - marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "folder/file.txt", model.NewVersionID()) - if err != nil { - t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) + if _, _, err := repos.Contents.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{ContentID: content.ID}); err != nil { + t.Fatalf("FinalizeUploadIfTargetCopiesMet: %v", err) } - req := httptest.NewRequest(http.MethodPost, "/api/v1/buckets/trash-permanent-delete-busy-bucket/objects/deleted/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"delete_marker_version_id":%q}`, "folder/file.txt", marker.VersionID))) - setBucketWriteHeaders(req) - rr := httptest.NewRecorder() - newBucketAPIMux(srv).ServeHTTP(rr, req) - if rr.Code != http.StatusConflict { - t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusConflict, rr.Body.String()) - } - var body map[string]string - if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { - t.Fatalf("Decode: %v", err) - } - want := "Storage work for one or more versions is still in progress or awaiting Filecoin confirmation. Check the related tasks, then try again." - if body["error"] != want { - t.Fatalf("error = %q, want %q", body["error"], want) + _, noCopyVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "cached.txt", 3, "etag-cached", "checksum-cached", "text/plain", model.ObjectStateCached) + _, partialVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "partial-provenance.txt", 4, "etag-partial-provenance", "checksum-partial-provenance", "text/plain", model.ObjectStateCached) + bindAdminPartialUpload(t, srv.db, repos, partialVersionID) + _, failedVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "failed-provenance.txt", 5, "etag-failed-provenance", "checksum-failed-provenance", "text/plain", model.ObjectStateCached) + markAdminFailedUpload(t, srv.db, repos, failedVersionID, "provider rejected piece") + replaceBucketDataSetObservability(t, repos, bucket.ID, observability.StatusAvailable, nil, time.Now().UTC()) + srv.WithObservability(observability.NewService(observability.ServiceOptions{ + Store: repos.Observability, + RefreshInterval: time.Hour, + })) + + ts := httptest.NewServer(newBucketAPIMux(srv)) + defer ts.Close() + + type provenanceBody struct { + VersionID string `json:"version_id"` + Status string `json:"status"` + PieceCID string `json:"piece_cid"` + RequestedCopies int `json:"requested_copies"` + SuccessCopies int `json:"success_copies"` + CopyHealth apiCopyHealthBody `json:"copy_health"` + Copies []struct { + CopyIndex int `json:"copy_index"` + Status string `json:"status"` + Health apiCopyHealthBody `json:"health"` + ProviderID string `json:"provider_id"` + ProviderIdentity *providerIdentityResponse `json:"provider_identity"` + DataSetID string `json:"data_set_id"` + PieceID string `json:"piece_id"` + TransferMethod string `json:"transfer_method"` + RetrievalURL string `json:"retrieval_url"` + IsNewDataSet bool `json:"is_new_data_set"` + } `json:"copies"` } - for _, id := range []string{versionID, marker.VersionID} { - got, loadErr := repos.Objects.GetVersionByID(ctx, id) - if loadErr != nil || got == nil { - t.Fatalf("version %s after rejected delete = %#v err=%v, want retained", id, got, loadErr) + getProvenance := func(bucketName, targetVersionID string) (provenanceBody, int) { + t.Helper() + resp, err := http.Get(ts.URL + "/api/v1/buckets/" + bucketName + "/objects/provenance?version_id=" + url.QueryEscape(targetVersionID)) + if err != nil { + t.Fatalf("GET provenance: %v", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return provenanceBody{}, resp.StatusCode + } + var body provenanceBody + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("Decode provenance: %v", err) } + return body, resp.StatusCode } -} - -func TestAPIBucketDeletedObjectPermanentDeleteCompletesCacheCleanupWhenRequestIsCanceled(t *testing.T) { - srv, repos := newBucketAPITestServer(t) - ctx := context.Background() - bucket := &model.Bucket{Name: "trash-permanent-delete-cancel-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create: %v", err) + detail, statusCode := getProvenance("provenance-bucket", versionID) + if statusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", statusCode, http.StatusOK) } - seedAdminObjectVersion(t, repos, bucket, "folder/file.txt", 7, "etag-first", "checksum-first", "text/plain", ".versions/cancel-first", model.ObjectStateCached) - seedAdminObjectVersion(t, repos, bucket, "folder/file.txt", 8, "etag-second", "checksum-second", "text/plain", ".versions/cancel-second", model.ObjectStateCached) - marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "folder/file.txt", model.NewVersionID()) - if err != nil { - t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) + if detail.VersionID != versionID || detail.Status != "success" { + t.Fatalf("detail status = %#v, want stored provenance", detail) } - - reqCtx, cancel := context.WithCancel(context.Background()) - defer cancel() - testCache := &cancelingDeleteCache{Cache: srv.cache, cancel: cancel} - srv.cache = testCache - - req := httptest.NewRequest(http.MethodPost, "/api/v1/buckets/trash-permanent-delete-cancel-bucket/objects/deleted/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"delete_marker_version_id":%q}`, "folder/file.txt", marker.VersionID))).WithContext(reqCtx) - setBucketWriteHeaders(req) - rr := httptest.NewRecorder() - newBucketAPIMux(srv).ServeHTTP(rr, req) - - if rr.Code != http.StatusOK { - t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusOK, rr.Body.String()) + if detail.PieceCID != pieceCID || detail.RequestedCopies != 2 || detail.SuccessCopies != 2 { + t.Fatalf("detail counts = %#v, want piece and 2/2 copies", detail) } - var body struct { - DataVersionsDeleted int `json:"data_versions_deleted"` + if detail.CopyHealth.Status != string(observability.StatusAvailable) || detail.CopyHealth.RequestedCopies != 2 || detail.CopyHealth.ReadableCopies != 2 || len(detail.CopyHealth.ReasonCodes) != 0 { + t.Fatalf("copy_health = %#v, want healthy 2/2 provenance", detail.CopyHealth) } - if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { - t.Fatalf("Decode: %v", err) + if len(detail.Copies) != 2 || detail.Copies[0].ProviderID != "101" || detail.Copies[0].TransferMethod != string(model.StorageCopyTransferMethodIngress) || detail.Copies[1].TransferMethod != string(model.StorageCopyTransferMethodPeerPull) { + t.Fatalf("copies = %#v, want provider scoped copy provenance", detail.Copies) } - if body.DataVersionsDeleted != 2 { - t.Fatalf("data_versions_deleted = %d, want 2", body.DataVersionsDeleted) + if detail.Copies[0].IsNewDataSet || detail.Copies[1].IsNewDataSet { + t.Fatalf("copies = %#v, want reused data sets derived from the earlier content", detail.Copies) } - if got := testCache.deletes.Load(); got != 2 { - t.Fatalf("cache cleanup deletes = %d, want 2 after request cancellation", got) + if detail.Copies[0].Health.Status != string(observability.StatusAvailable) || len(detail.Copies[0].Health.ReasonCodes) != 0 { + t.Fatalf("copy health = %#v, want healthy copy", detail.Copies[0].Health) } -} - -func TestAPIBucketDeletedObjectPermanentDeleteUsesIsolatedCacheCleanupContexts(t *testing.T) { - srv, repos := newBucketAPITestServer(t) - ctx := context.Background() - - bucket := &model.Bucket{Name: "trash-permanent-delete-context-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create: %v", err) + if detail.Copies[0].ProviderIdentity == nil || detail.Copies[0].ProviderIdentity.Name != "alpha-pdp" || detail.Copies[0].ProviderIdentity.FilecoinActorID != "f01234" { + t.Fatalf("copy provider_identity = %#v, want enriched copy identity", detail.Copies[0].ProviderIdentity) + } + if detail.Copies[1].ProviderID != "102" || detail.Copies[1].ProviderIdentity != nil { + t.Fatalf("secondary copy = %#v, want provider_id compatibility without identity when lookup fails", detail.Copies[1]) + } + peerBinding, err := repos.Contents.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 1) + if err != nil || peerBinding == nil { + t.Fatalf("GetDataSetBindingByCopyIndex peer: binding=%v err=%v", peerBinding, err) + } + // A draining generation still serves reads, so it keeps counting. + if err := repos.Contents.MarkDataSetDraining(ctx, peerBinding.ID, "provider service ended"); err != nil { + t.Fatalf("MarkDataSetDraining peer: %v", err) + } + drainingDetail, statusCode := getProvenance("provenance-bucket", versionID) + if statusCode != http.StatusOK { + t.Fatalf("draining status = %d, want %d", statusCode, http.StatusOK) + } + if drainingDetail.SuccessCopies != 2 { + t.Fatalf("draining provenance = %#v, want draining dataset counted as readable", drainingDetail) } - seedAdminObjectVersion(t, repos, bucket, "folder/file.txt", 7, "etag-first", "checksum-first", "text/plain", ".versions/context-first", model.ObjectStateCached) - seedAdminObjectVersion(t, repos, bucket, "folder/file.txt", 8, "etag-second", "checksum-second", "text/plain", ".versions/context-second", model.ObjectStateCached) - marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "folder/file.txt", model.NewVersionID()) - if err != nil { - t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) + if !reflect.DeepEqual(identityResolver.requests[0], []string{"101", "102"}) { + t.Fatalf("provider identity request = %#v, want one provenance snapshot request", identityResolver.requests) } - testCache := &contextRecordingDeleteCache{Cache: srv.cache} - srv.cache = testCache - - req := httptest.NewRequest(http.MethodPost, "/api/v1/buckets/trash-permanent-delete-context-bucket/objects/deleted/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"delete_marker_version_id":%q}`, "folder/file.txt", marker.VersionID))) - setBucketWriteHeaders(req) - rr := httptest.NewRecorder() - newBucketAPIMux(srv).ServeHTTP(rr, req) - - if rr.Code != http.StatusOK { - t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusOK, rr.Body.String()) - } - contexts := testCache.recordedContexts() - if got := len(contexts); got != 2 { - t.Fatalf("cache cleanup contexts = %d, want 2", got) + oldDetail, statusCode := getProvenance("provenance-bucket", oldVersionID) + if statusCode != http.StatusOK { + t.Fatalf("old version status = %d, want %d", statusCode, http.StatusOK) } - if contexts[0] == contexts[1] { - t.Fatal("cache cleanup reused one context across deleted versions") + // The historical version was accepted against the same two-slot policy, so + // it reports both replicas too. + if oldDetail.VersionID != oldVersionID || oldDetail.Status != "success" || oldDetail.SuccessCopies != 2 || + len(oldDetail.Copies) != 2 || !oldDetail.Copies[0].IsNewDataSet || !oldDetail.Copies[1].IsNewDataSet { + t.Fatalf("old version provenance = %#v, want historical stored version", oldDetail) } -} - -func TestAPIBucketDeletedObjectPermanentDeleteRunsCacheCleanupInParallel(t *testing.T) { - srv, repos := newBucketAPITestServer(t) - ctx := context.Background() - bucket := &model.Bucket{Name: "trash-permanent-delete-parallel-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create: %v", err) + partialDetail, statusCode := getProvenance("provenance-bucket", partialVersionID) + if statusCode != http.StatusOK { + t.Fatalf("partial status = %d, want %d", statusCode, http.StatusOK) } - seedAdminObjectVersion(t, repos, bucket, "folder/file.txt", 7, "etag-first", "checksum-first", "text/plain", ".versions/parallel-first", model.ObjectStateCached) - seedAdminObjectVersion(t, repos, bucket, "folder/file.txt", 8, "etag-second", "checksum-second", "text/plain", ".versions/parallel-second", model.ObjectStateCached) - seedAdminObjectVersion(t, repos, bucket, "folder/file.txt", 9, "etag-third", "checksum-third", "text/plain", ".versions/parallel-third", model.ObjectStateCached) - marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "folder/file.txt", model.NewVersionID()) - if err != nil { - t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) + if partialDetail.Status != "syncing" || partialDetail.SuccessCopies != 1 { + t.Fatalf("partial provenance = %#v, want partially replicated detail", partialDetail) } - testCache := &blockingDeleteCache{ - Cache: srv.cache, - started: make(chan struct{}, 3), - release: make(chan struct{}), - failKey: ".versions/parallel-second", + failedDetail, statusCode := getProvenance("provenance-bucket", failedVersionID) + if statusCode != http.StatusOK { + t.Fatalf("failed status = %d, want %d", statusCode, http.StatusOK) + } + if failedDetail.Status != "warning" || failedDetail.SuccessCopies != 0 { + t.Fatalf("failed provenance = %#v, want failed ingest detail", failedDetail) } - srv.cache = testCache - - req := httptest.NewRequest(http.MethodPost, "/api/v1/buckets/trash-permanent-delete-parallel-bucket/objects/deleted/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"delete_marker_version_id":%q}`, "folder/file.txt", marker.VersionID))) - setBucketWriteHeaders(req) - rr := httptest.NewRecorder() - done := make(chan struct{}) - go func() { - newBucketAPIMux(srv).ServeHTTP(rr, req) - close(done) - }() - waitDeleteStarts(t, testCache.started, 3) - if got := testCache.maxActive.Load(); got < 2 { - t.Fatalf("max concurrent cache cleanups = %d, want at least 2", got) + // Once it is retired it serves nothing, and drops out of readable copies. + if _, err := srv.db.NewUpdate().Model((*model.StorageDataSet)(nil)). + Set("status = ?", model.StorageDataSetStatusRetired). + Where("id = ?", peerBinding.ID). + Exec(ctx); err != nil { + t.Fatalf("retire peer binding: %v", err) } - select { - case <-done: - t.Fatal("handler returned before cache cleanup completed") - default: + retiredDetail, statusCode := getProvenance("provenance-bucket", versionID) + if statusCode != http.StatusOK { + t.Fatalf("retired status = %d, want %d", statusCode, http.StatusOK) } - close(testCache.release) - select { - case <-done: - case <-time.After(2 * time.Second): - t.Fatal("handler did not return after cache cleanup completed") + if retiredDetail.SuccessCopies != 1 { + t.Fatalf("retired provenance = %#v, want the retired dataset excluded from readable copies", retiredDetail) } - if rr.Code != http.StatusOK { - t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusOK, rr.Body.String()) - } - var body struct { - DataVersionsDeleted int `json:"data_versions_deleted"` - CacheCleanupFailedCount int `json:"cache_cleanup_failed_count"` - } - if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { - t.Fatalf("Decode: %v", err) + // A version whose content has no copies yet still answers, with nothing to + // show for provenance. + noCopies, statusCode := getProvenance("provenance-bucket", noCopyVersionID) + if statusCode != http.StatusOK { + t.Fatalf("no-copy status = %d, want %d", statusCode, http.StatusOK) } - if body.DataVersionsDeleted != 3 { - t.Fatalf("data_versions_deleted = %d, want 3", body.DataVersionsDeleted) + if noCopies.VersionID != noCopyVersionID || len(noCopies.Copies) != 0 { + t.Fatalf("no-copy provenance = %#v, want empty copy detail", noCopies) } - if body.CacheCleanupFailedCount != 1 { - t.Fatalf("cache_cleanup_failed_count = %d, want 1", body.CacheCleanupFailedCount) + + otherBucket := &model.Bucket{Name: "other-provenance-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, otherBucket); err != nil { + t.Fatalf("other bucket: %v", err) } - if got := testCache.deletes.Load(); got != 3 { - t.Fatalf("cache cleanup deletes = %d, want 3", got) + _, statusCode = getProvenance("other-provenance-bucket", versionID) + if statusCode != http.StatusNotFound { + t.Fatalf("wrong bucket provenance code = %d, want %d", statusCode, http.StatusNotFound) } } -func TestAPIBucketObjectPermanentDeleteRemovesVersion(t *testing.T) { +func TestAPIBucketObjectVersions(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "version-permanent-delete-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "versions-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - _, oldVersionID := seedAdminObjectVersion(t, repos, bucket, "folder/file.txt", 7, "etag-old", "checksum-old", "text/plain", "", model.ObjectStateCached) - _, currentVersionID := seedAdminObjectVersion(t, repos, bucket, "folder/file.txt", 8, "etag-current", "checksum-current", "text/plain", "", model.ObjectStateCached) + _, oldVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "file.txt", 4, "etag-old", "checksum-old", "text/plain", model.ObjectStateCached) + acceptAdminVersionUpload(t, srv.db, repos, oldVersionID, "piece-old", "https://provider.example/old") + _, currentVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "file.txt", 7, "etag-current", "checksum-current", "text/plain", model.ObjectStateCached) ts := httptest.NewServer(newBucketAPIMux(srv)) defer ts.Close() - req, err := http.NewRequest(http.MethodPost, ts.URL+"/api/v1/buckets/version-permanent-delete-bucket/objects/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"version_id":%q}`, "folder/file.txt", currentVersionID))) - if err != nil { - t.Fatalf("NewRequest permanent delete version: %v", err) - } - setBucketWriteHeaders(req) - resp, err := ts.Client().Do(req) + resp, err := http.Get(ts.URL + "/api/v1/buckets/versions-bucket/objects/versions?key=" + url.QueryEscape("file.txt")) if err != nil { - t.Fatalf("POST permanent delete version: %v", err) + t.Fatalf("GET object versions: %v", err) } defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { - t.Fatalf("permanent delete version status = %d, want %d, body=%s", resp.StatusCode, http.StatusOK, readBody(t, resp.Body)) + t.Fatalf("status = %d, want %d, body=%s", resp.StatusCode, http.StatusOK, readBody(t, resp.Body)) } + var body struct { - Key string `json:"key"` - VersionID string `json:"version_id"` - CacheCleanupStatus string `json:"cache_cleanup_status"` + Versions []struct { + VersionID string `json:"version_id"` + State string `json:"state"` + Status string `json:"status"` + Location objectLocation `json:"location"` + IsCurrent bool `json:"is_current"` + } `json:"versions"` } if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("Decode permanent delete version: %v", err) + t.Fatalf("Decode: %v", err) } - if body.Key != "folder/file.txt" || body.VersionID != currentVersionID || body.CacheCleanupStatus == "" { - t.Fatalf("permanent delete version response = %#v, want deleted version metadata", body) + if len(body.Versions) != 2 { + t.Fatalf("versions len = %d, want 2", len(body.Versions)) } - gotVersion, err := repos.Objects.GetVersionByID(ctx, currentVersionID) - if err != nil { - t.Fatalf("GetVersionByID(deleted): %v", err) + if body.Versions[0].VersionID != currentVersionID || !body.Versions[0].IsCurrent { + t.Fatalf("first version = %#v, want current %s", body.Versions[0], currentVersionID) } - if gotVersion != nil { - t.Fatalf("data version still exists after version permanent delete: %#v", gotVersion) + if body.Versions[1].VersionID != oldVersionID || body.Versions[1].IsCurrent { + t.Fatalf("second version = %#v, want old %s", body.Versions[1], oldVersionID) } - gotOldVersion, err := repos.Objects.GetVersionByID(ctx, oldVersionID) - if err != nil || gotOldVersion == nil { - t.Fatalf("GetVersionByID(promoted): version=%v err=%v", gotOldVersion, err) + // Each version carries its own bytes here, so their derived positions differ. + if body.Versions[0].Status != "uploading" { + t.Fatalf("current version status = %q, want uploading", body.Versions[0].Status) } - if !gotOldVersion.IsCurrent { - t.Fatalf("old version is_current = false, want true after current version permanent delete") + if body.Versions[0].State != string(model.ObjectStateCached) { + t.Fatalf("current version state = %q, want cached", body.Versions[0].State) } -} - -func TestAPIBucketObjectPermanentDeleteReportsActiveStorageWork(t *testing.T) { - srv, repos := newBucketAPITestServer(t) - ctx := context.Background() - bucket := &model.Bucket{Name: "version-permanent-delete-busy-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create: %v", err) + if !body.Versions[0].Location.Cache || body.Versions[0].Location.Filecoin { + t.Fatalf("current version location = %#v, want cache only", body.Versions[0].Location) } - objectID, versionID := seedAdminObjectVersion(t, repos, bucket, "folder/file.txt", 8, "etag-current", "checksum-current", "text/plain", "", model.ObjectStateCached) - stage := "prepare_upload" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "object", RefID: objectID, RefVersionID: versionID, - IdempotencyKey: "upload:" + versionID, Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), + if body.Versions[1].Status != "success" { + t.Fatalf("old version status = %q, want success", body.Versions[1].Status) + } + if body.Versions[1].State != string(model.ObjectStateStored) { + t.Fatalf("old version state = %q, want stored", body.Versions[1].State) } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Tasks.Create: %v", err) + if !body.Versions[1].Location.Cache || !body.Versions[1].Location.Filecoin { + t.Fatalf("old version location = %#v, want cache and filecoin", body.Versions[1].Location) } - req := httptest.NewRequest(http.MethodPost, "/api/v1/buckets/version-permanent-delete-busy-bucket/objects/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"version_id":%q}`, "folder/file.txt", versionID))) - setBucketWriteHeaders(req) - rr := httptest.NewRecorder() - newBucketAPIMux(srv).ServeHTTP(rr, req) - if rr.Code != http.StatusConflict { - t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusConflict, rr.Body.String()) + resp, err = http.Get(ts.URL + "/api/v1/buckets/versions-bucket/objects/versions?key=" + url.QueryEscape("file.txt")) + if err != nil { + t.Fatalf("GET object versions raw: %v", err) } - var body map[string]string - if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { - t.Fatalf("Decode: %v", err) + defer func() { _ = resp.Body.Close() }() + var raw struct { + Versions []map[string]any `json:"versions"` } - want := "Storage work for this version is still in progress or awaiting Filecoin confirmation. Check the related task, then try again." - if body["error"] != want { - t.Fatalf("error = %q, want %q", body["error"], want) + if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil { + t.Fatalf("Decode raw: %v", err) + } + if raw.Versions[0]["state"] != string(model.ObjectStateCached) { + t.Fatalf("version list state = %#v, want cached", raw.Versions[0]["state"]) + } + if _, ok := raw.Versions[0]["storage"]; ok { + t.Fatal("version list exposed storage instead of location") } } -func TestAPIBucketObjectPermanentDeleteRecordsCacheCleanupWhenRequestIsCanceled(t *testing.T) { +func TestAPIBucketObjectDeletionsSupportsOffset(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "version-permanent-delete-cancel-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "deletion-history-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - _, currentVersionID := seedAdminObjectVersion(t, repos, bucket, "folder/file.txt", 8, "etag-current", "checksum-current", "text/plain", "cache-key", model.ObjectStateCached) - - reqCtx, cancel := context.WithCancel(ctx) - defer cancel() - testCache := &cancelingDeleteCache{Cache: srv.cache, cancel: cancel} - srv.cache = testCache + // object_deletions is a pure tombstone now: it records that a version was + // removed and nothing about the cache. + for i := 1; i <= 3; i++ { + deletedAt := time.Date(2026, time.May, 10, 12, i, 0, 0, time.UTC) + if _, err := srv.db.NewRaw(`INSERT INTO object_deletions + (bucket_id, object_id, key, version_id, size, deleted_at) + VALUES (?, ?, ?, ?, ?, ?)`, + bucket.ID, int64(i), fmt.Sprintf("file-%d.txt", i), fmt.Sprintf("version-%d", i), int64(i), deletedAt, + ).Exec(ctx); err != nil { + t.Fatalf("insert object deletion %d: %v", i, err) + } + } - req := httptest.NewRequest(http.MethodPost, "/api/v1/buckets/version-permanent-delete-cancel-bucket/objects/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"version_id":%q}`, "folder/file.txt", currentVersionID))).WithContext(reqCtx) - setBucketWriteHeaders(req) - rr := httptest.NewRecorder() - newBucketAPIMux(srv).ServeHTTP(rr, req) + ts := httptest.NewServer(newBucketAPIMux(srv)) + defer ts.Close() - if rr.Code != http.StatusOK { - t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusOK, rr.Body.String()) - } - var body struct { - CacheCleanupStatus string `json:"cache_cleanup_status"` + req, err := http.NewRequest(http.MethodGet, ts.URL+"/api/v1/buckets/deletion-history-bucket/objects/deletions?limit=1&offset=1", nil) + if err != nil { + t.Fatalf("NewRequest object deletions: %v", err) } - if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { - t.Fatalf("Decode: %v", err) + setBucketWriteHeaders(req) + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatalf("GET object deletions: %v", err) } - if body.CacheCleanupStatus != string(model.CacheCleanupStatusDeleted) { - t.Fatalf("cache cleanup status = %q, want %q", body.CacheCleanupStatus, model.CacheCleanupStatusDeleted) + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Fatalf("object deletions status = %d, want %d, body=%s", resp.StatusCode, http.StatusOK, readBody(t, resp.Body)) } - if got := testCache.deletes.Load(); got != 1 { - t.Fatalf("cache cleanup deletes = %d, want 1 after request cancellation", got) + var body struct { + Deletions []struct { + Key string `json:"key"` + VersionID string `json:"version_id"` + DeletedAt string `json:"deleted_at"` + } `json:"deletions"` } - var gotStatus model.CacheCleanupStatus - if err := srv.db.NewRaw(`SELECT cache_cleanup_status FROM object_deletions WHERE version_id = ?`, currentVersionID).Scan(ctx, &gotStatus); err != nil { - t.Fatalf("select object deletion cache cleanup status: %v", err) + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("Decode object deletions: %v", err) } - if gotStatus != model.CacheCleanupStatusDeleted { - t.Fatalf("recorded cache cleanup status = %q, want %q", gotStatus, model.CacheCleanupStatusDeleted) + if len(body.Deletions) != 1 || body.Deletions[0].Key != "file-2.txt" || body.Deletions[0].VersionID != "version-2" || body.Deletions[0].DeletedAt == "" { + t.Fatalf("object deletions page = %#v, want second newest deletion", body.Deletions) } } -func TestAPIBucketObjectPermanentDeleteReportsCacheCleanupFailedOnCacheError(t *testing.T) { +func TestAPIBucketObjectPermanentDeleteRemovesVersion(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "version-permanent-delete-cache-error-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "version-permanent-delete-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - _, currentVersionID := seedAdminObjectVersion(t, repos, bucket, "folder/file.txt", 8, "etag-current", "checksum-current", "text/plain", "cache-key", model.ObjectStateCached) + _, oldVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/file.txt", 7, "etag-old", "checksum-old", "text/plain", model.ObjectStateCached) + _, currentVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/file.txt", 8, "etag-current", "checksum-current", "text/plain", model.ObjectStateCached) - srv.cache = &cancelingDeleteCache{Cache: srv.cache, deleteErr: errors.New("cache error")} ts := httptest.NewServer(newBucketAPIMux(srv)) defer ts.Close() - req, err := http.NewRequest(http.MethodPost, ts.URL+"/api/v1/buckets/version-permanent-delete-cache-error-bucket/objects/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"version_id":%q}`, "folder/file.txt", currentVersionID))) + req, err := http.NewRequest(http.MethodPost, ts.URL+"/api/v1/buckets/version-permanent-delete-bucket/objects/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"version_id":%q}`, "folder/file.txt", currentVersionID))) if err != nil { t.Fatalf("NewRequest permanent delete version: %v", err) } @@ -3897,453 +4049,447 @@ func TestAPIBucketObjectPermanentDeleteReportsCacheCleanupFailedOnCacheError(t * t.Fatalf("permanent delete version status = %d, want %d, body=%s", resp.StatusCode, http.StatusOK, readBody(t, resp.Body)) } var body struct { - CacheCleanupStatus string `json:"cache_cleanup_status"` + Key string `json:"key"` + VersionID string `json:"version_id"` + CacheRelease string `json:"cache_release"` } if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { t.Fatalf("Decode permanent delete version: %v", err) } - if body.CacheCleanupStatus != string(model.CacheCleanupStatusFailed) { - t.Fatalf("cache cleanup status = %q, want %q", body.CacheCleanupStatus, model.CacheCleanupStatusFailed) + // The two versions carry different bytes, so removing one releases its own + // content's cache file rather than retaining shared bytes. + if body.Key != "folder/file.txt" || body.VersionID != currentVersionID || body.CacheRelease != "released" { + t.Fatalf("permanent delete version response = %#v, want deleted version metadata", body) } gotVersion, err := repos.Objects.GetVersionByID(ctx, currentVersionID) if err != nil { t.Fatalf("GetVersionByID(deleted): %v", err) } if gotVersion != nil { - t.Fatalf("data version still exists after cache cleanup failed: %#v", gotVersion) - } - var deletion model.ObjectDeletion - if err := srv.db.NewSelect().Model(&deletion).Where("version_id = ?", currentVersionID).Scan(ctx); err != nil { - t.Fatalf("select object deletion: %v", err) + t.Fatalf("data version still exists after version permanent delete: %#v", gotVersion) } - if deletion.CacheCleanupStatus != model.CacheCleanupStatusFailed { - t.Fatalf("recorded cache cleanup status = %q, want %q", deletion.CacheCleanupStatus, model.CacheCleanupStatusFailed) + gotOldVersion, err := repos.Objects.GetVersionByID(ctx, oldVersionID) + if err != nil || gotOldVersion == nil { + t.Fatalf("GetVersionByID(promoted): version=%v err=%v", gotOldVersion, err) } - if deletion.CacheError == nil || !strings.Contains(*deletion.CacheError, "cache error") { - t.Fatalf("recorded cache error = %v, want cache error", deletion.CacheError) + if !gotOldVersion.IsCurrent { + t.Fatalf("old version is_current = false, want true after current version permanent delete") } } -func TestAPIBucketObjectDeletionsSupportsOffset(t *testing.T) { +func TestAPIBucketObjectPermanentDeleteReleasesCacheWhenRequestIsCanceled(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "deletion-history-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "version-permanent-delete-cancel-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - for i := 1; i <= 3; i++ { - ts := time.Date(2026, time.May, 10, 12, i, 0, 0, time.UTC) - if _, err := srv.db.NewRaw(`INSERT INTO object_deletions - (bucket_id, object_id, key, version_id, cache_key, size, checksum, cache_cleanup_status, created_at, updated_at, deleted_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - bucket.ID, int64(i), fmt.Sprintf("file-%d.txt", i), fmt.Sprintf("version-%d", i), fmt.Sprintf("cache-%d", i), int64(i), fmt.Sprintf("checksum-%d", i), model.CacheCleanupStatusDeleted, ts, ts, ts, - ).Exec(ctx); err != nil { - t.Fatalf("insert object deletion %d: %v", i, err) - } - } + _, currentVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/file.txt", 8, "etag-current", "checksum-current", "text/plain", model.ObjectStateCached) - ts := httptest.NewServer(newBucketAPIMux(srv)) - defer ts.Close() + reqCtx, cancel := context.WithCancel(ctx) + defer cancel() + testCache := &cancelingDeleteCache{Cache: srv.cache, cancel: cancel} + srv.cache = testCache - req, err := http.NewRequest(http.MethodGet, ts.URL+"/api/v1/buckets/deletion-history-bucket/objects/deletions?limit=1&offset=1", nil) - if err != nil { - t.Fatalf("NewRequest object deletions: %v", err) - } + req := httptest.NewRequest(http.MethodPost, "/api/v1/buckets/version-permanent-delete-cancel-bucket/objects/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"version_id":%q}`, "folder/file.txt", currentVersionID))).WithContext(reqCtx) setBucketWriteHeaders(req) - resp, err := ts.Client().Do(req) - if err != nil { - t.Fatalf("GET object deletions: %v", err) - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - t.Fatalf("object deletions status = %d, want %d, body=%s", resp.StatusCode, http.StatusOK, readBody(t, resp.Body)) + rr := httptest.NewRecorder() + newBucketAPIMux(srv).ServeHTTP(rr, req) + + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusOK, rr.Body.String()) } var body struct { - Deletions []struct { - Key string `json:"key"` - VersionID string `json:"version_id"` - DeletedAt string `json:"deleted_at"` - } `json:"deletions"` + CacheRelease string `json:"cache_release"` } - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("Decode object deletions: %v", err) + if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { + t.Fatalf("Decode: %v", err) } - if len(body.Deletions) != 1 || body.Deletions[0].Key != "file-2.txt" || body.Deletions[0].VersionID != "version-2" || body.Deletions[0].DeletedAt == "" { - t.Fatalf("object deletions page = %#v, want second newest deletion", body.Deletions) + // Cache release runs on a context detached from the request, so cancelling + // the caller must not leave the bytes behind. + if body.CacheRelease != "released" { + t.Fatalf("cache_release = %q, want released", body.CacheRelease) + } + if got := testCache.deletes.Load(); got != 1 { + t.Fatalf("cache cleanup deletes = %d, want 1 after request cancellation", got) } } -func TestAPIBucketObjectDeletionsAllowsReadWithoutWriteHeader(t *testing.T) { +func TestAPIBucketObjectPermanentDeleteReportsCacheReleaseFailedOnCacheError(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "deletion-history-auth-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "version-permanent-delete-cache-error-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } + _, currentVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/file.txt", 8, "etag-current", "checksum-current", "text/plain", model.ObjectStateCached) + srv.cache = &cancelingDeleteCache{Cache: srv.cache, deleteErr: errors.New("cache error")} ts := httptest.NewServer(newBucketAPIMux(srv)) defer ts.Close() - resp, err := http.Get(ts.URL + "/api/v1/buckets/deletion-history-auth-bucket/objects/deletions") + req, err := http.NewRequest(http.MethodPost, ts.URL+"/api/v1/buckets/version-permanent-delete-cache-error-bucket/objects/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"version_id":%q}`, "folder/file.txt", currentVersionID))) if err != nil { - t.Fatalf("GET object deletions: %v", err) + t.Fatalf("NewRequest permanent delete version: %v", err) + } + setBucketWriteHeaders(req) + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatalf("POST permanent delete version: %v", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - t.Fatalf("object deletions status = %d, want %d, body=%s", resp.StatusCode, http.StatusOK, readBody(t, resp.Body)) + t.Fatalf("permanent delete version status = %d, want %d, body=%s", resp.StatusCode, http.StatusOK, readBody(t, resp.Body)) + } + var body struct { + CacheRelease string `json:"cache_release"` + } + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("Decode permanent delete version: %v", err) + } + // A failed release is reported, but the deletion itself still stands. + if body.CacheRelease != "failed" { + t.Fatalf("cache_release = %q, want failed", body.CacheRelease) + } + gotVersion, err := repos.Objects.GetVersionByID(ctx, currentVersionID) + if err != nil { + t.Fatalf("GetVersionByID(deleted): %v", err) + } + if gotVersion != nil { + t.Fatalf("data version still exists after cache cleanup failed: %#v", gotVersion) + } + deletions, err := srv.db.NewSelect(). + Model((*model.ObjectDeletion)(nil)). + Where("version_id = ?", currentVersionID). + Count(ctx) + if err != nil { + t.Fatalf("count object deletions: %v", err) + } + if deletions != 1 { + t.Fatalf("object deletion tombstones = %d, want 1", deletions) } } -func TestAPIBucketObjects_RestoreRejectsStaleMarker(t *testing.T) { +func TestAPIBucketDeletedObjectPermanentDeleteCompletesCacheCleanupWhenRequestIsCanceled(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "stale-restore-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "trash-permanent-delete-cancel-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - seedAdminObjectVersion(t, repos, bucket, "file.txt", 4, "etag-old", "checksum-old", "text/plain", "", model.ObjectStateCached) - marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "file.txt", "01J0000000000000000000ADM1") + seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/file.txt", 7, "etag-first", "checksum-first", "text/plain", model.ObjectStateCached) + seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/file.txt", 8, "etag-second", "checksum-second", "text/plain", model.ObjectStateCached) + marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "folder/file.txt", model.NewVersionID()) if err != nil { t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) } - seedAdminObjectVersion(t, repos, bucket, "file.txt", 6, "etag-new", "checksum-new", "text/plain", "", model.ObjectStateCached) - ts := httptest.NewServer(newBucketAPIMux(srv)) - defer ts.Close() + reqCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + testCache := &cancelingDeleteCache{Cache: srv.cache, cancel: cancel} + srv.cache = testCache - req, err := http.NewRequest(http.MethodPost, ts.URL+"/api/v1/buckets/stale-restore-bucket/objects/restore", strings.NewReader(fmt.Sprintf(`{"key":%q,"delete_marker_version_id":%q}`, "file.txt", marker.VersionID))) - if err != nil { - t.Fatalf("NewRequest restore: %v", err) - } + req := httptest.NewRequest(http.MethodPost, "/api/v1/buckets/trash-permanent-delete-cancel-bucket/objects/deleted/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"delete_marker_version_id":%q}`, "folder/file.txt", marker.VersionID))).WithContext(reqCtx) setBucketWriteHeaders(req) - resp, err := ts.Client().Do(req) - if err != nil { - t.Fatalf("POST stale restore: %v", err) + rr := httptest.NewRecorder() + newBucketAPIMux(srv).ServeHTTP(rr, req) + + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusOK, rr.Body.String()) } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusConflict { - t.Fatalf("status = %d, want %d, body=%s", resp.StatusCode, http.StatusConflict, readBody(t, resp.Body)) + var body struct { + DataVersionsDeleted int `json:"data_versions_deleted"` + } + if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { + t.Fatalf("Decode: %v", err) + } + if body.DataVersionsDeleted != 2 { + t.Fatalf("data_versions_deleted = %d, want 2", body.DataVersionsDeleted) + } + if got := testCache.deletes.Load(); got != 2 { + t.Fatalf("cache cleanup deletes = %d, want 2 after request cancellation", got) } } -func TestAPIBucketDeletedObjects_ExcludesMarkerWithoutDataVersion(t *testing.T) { +func TestAPIBucketDeletedObjectPermanentDeleteUsesIsolatedCacheCleanupContexts(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "empty-trash-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "trash-permanent-delete-context-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "missing.txt", "01J0000000000000000000ADM2") + seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/file.txt", 7, "etag-first", "checksum-first", "text/plain", model.ObjectStateCached) + seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/file.txt", 8, "etag-second", "checksum-second", "text/plain", model.ObjectStateCached) + marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "folder/file.txt", model.NewVersionID()) if err != nil { t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) } - ts := httptest.NewServer(newBucketAPIMux(srv)) - defer ts.Close() + testCache := &contextRecordingDeleteCache{Cache: srv.cache} + srv.cache = testCache - resp, err := http.Get(ts.URL + "/api/v1/buckets/empty-trash-bucket/objects/deleted") - if err != nil { - t.Fatalf("GET deleted objects: %v", err) - } - defer func() { _ = resp.Body.Close() }() - var body struct { - Objects []struct{} `json:"objects"` - } - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("Decode deleted objects: %v", err) - } - if len(body.Objects) != 0 { - t.Fatalf("deleted objects len = %d, want marker without data hidden", len(body.Objects)) - } + req := httptest.NewRequest(http.MethodPost, "/api/v1/buckets/trash-permanent-delete-context-bucket/objects/deleted/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"delete_marker_version_id":%q}`, "folder/file.txt", marker.VersionID))) + setBucketWriteHeaders(req) + rr := httptest.NewRecorder() + newBucketAPIMux(srv).ServeHTTP(rr, req) - req, err := http.NewRequest(http.MethodPost, ts.URL+"/api/v1/buckets/empty-trash-bucket/objects/restore", strings.NewReader(fmt.Sprintf(`{"key":%q,"delete_marker_version_id":%q}`, "missing.txt", marker.VersionID))) - if err != nil { - t.Fatalf("NewRequest restore: %v", err) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusOK, rr.Body.String()) } - setBucketWriteHeaders(req) - restoreResp, err := ts.Client().Do(req) - if err != nil { - t.Fatalf("POST restore without data: %v", err) + contexts := testCache.recordedContexts() + if got := len(contexts); got != 2 { + t.Fatalf("cache cleanup contexts = %d, want 2", got) } - defer func() { _ = restoreResp.Body.Close() }() - if restoreResp.StatusCode != http.StatusNotFound { - t.Fatalf("restore status = %d, want %d, body=%s", restoreResp.StatusCode, http.StatusNotFound, readBody(t, restoreResp.Body)) + if contexts[0] == contexts[1] { + t.Fatal("cache cleanup reused one context across deleted versions") } } -func TestAPIBucketObjectVersions(t *testing.T) { +func TestAPIBucketDeletedObjectPermanentDeleteReportsEveryCacheCleanupOutcome(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "versions-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "trash-permanent-delete-parallel-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - _, oldVersionID := seedAdminObjectVersion(t, repos, bucket, "file.txt", 4, "etag-old", "checksum-old", "text/plain", "", model.ObjectStateCached) - if err := repos.Objects.UpdateVersionState(ctx, oldVersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("old uploading: %v", err) - } - acceptAdminVersionUpload(t, repos, oldVersionID, "piece-old", "https://provider.example/old") - _, currentVersionID := seedAdminObjectVersion(t, repos, bucket, "file.txt", 7, "etag-current", "checksum-current", "text/plain", "", model.ObjectStateCached) - - ts := httptest.NewServer(newBucketAPIMux(srv)) - defer ts.Close() - - resp, err := http.Get(ts.URL + "/api/v1/buckets/versions-bucket/objects/versions?key=" + url.QueryEscape("file.txt")) + seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/file.txt", 7, "etag-first", "checksum-first", "text/plain", model.ObjectStateCached) + _, secondVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/file.txt", 8, "etag-second", "checksum-second", "text/plain", model.ObjectStateCached) + seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/file.txt", 9, "etag-third", "checksum-third", "text/plain", model.ObjectStateCached) + marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "folder/file.txt", model.NewVersionID()) if err != nil { - t.Fatalf("GET object versions: %v", err) + t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d, want %d, body=%s", resp.StatusCode, http.StatusOK, readBody(t, resp.Body)) + testCache := &selectiveFailingDeleteCache{ + Cache: srv.cache, + failKey: adminVersionCacheKey(t, repos, secondVersionID), } + srv.cache = testCache + + req := httptest.NewRequest(http.MethodPost, "/api/v1/buckets/trash-permanent-delete-parallel-bucket/objects/deleted/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"delete_marker_version_id":%q}`, "folder/file.txt", marker.VersionID))) + setBucketWriteHeaders(req) + rr := httptest.NewRecorder() + newBucketAPIMux(srv).ServeHTTP(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusOK, rr.Body.String()) + } var body struct { - Versions []struct { - VersionID string `json:"version_id"` - State string `json:"state"` - Status string `json:"status"` - Location objectLocation `json:"location"` - IsCurrent bool `json:"is_current"` - } `json:"versions"` + DataVersionsDeleted int `json:"data_versions_deleted"` + CacheCleanupFailedCount int `json:"cache_cleanup_failed_count"` } - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { t.Fatalf("Decode: %v", err) } - if len(body.Versions) != 2 { - t.Fatalf("versions len = %d, want 2", len(body.Versions)) - } - if body.Versions[0].VersionID != currentVersionID || !body.Versions[0].IsCurrent { - t.Fatalf("first version = %#v, want current %s", body.Versions[0], currentVersionID) - } - if body.Versions[1].VersionID != oldVersionID || body.Versions[1].IsCurrent { - t.Fatalf("second version = %#v, want old %s", body.Versions[1], oldVersionID) - } - if body.Versions[0].Status != "uploading" { - t.Fatalf("current version status = %q, want uploading", body.Versions[0].Status) - } - if body.Versions[0].State != string(model.ObjectStateCached) { - t.Fatalf("current version state = %q, want cached", body.Versions[0].State) - } - if !body.Versions[0].Location.Cache || body.Versions[0].Location.Filecoin { - t.Fatalf("current version location = %#v, want cache only", body.Versions[0].Location) - } - if body.Versions[1].Status != "success" { - t.Fatalf("old version status = %q, want success", body.Versions[1].Status) + if body.DataVersionsDeleted != 3 { + t.Fatalf("data_versions_deleted = %d, want 3", body.DataVersionsDeleted) } - if body.Versions[1].State != string(model.ObjectStateStored) { - t.Fatalf("old version state = %q, want stored", body.Versions[1].State) + if body.CacheCleanupFailedCount != 1 { + t.Fatalf("cache_cleanup_failed_count = %d, want 1", body.CacheCleanupFailedCount) } - if !body.Versions[1].Location.Cache || !body.Versions[1].Location.Filecoin { - t.Fatalf("old version location = %#v, want cache and filecoin", body.Versions[1].Location) + if got := testCache.deletes.Load(); got != 3 { + t.Fatalf("cache cleanup deletes = %d, want 3", got) } +} - resp, err = http.Get(ts.URL + "/api/v1/buckets/versions-bucket/objects/versions?key=" + url.QueryEscape("file.txt")) - if err != nil { - t.Fatalf("GET object versions raw: %v", err) +func TestAPIBucketObjectPermanentDeleteReportsActiveStorageWork(t *testing.T) { + srv, repos := newBucketAPITestServer(t) + ctx := context.Background() + bucket := &model.Bucket{Name: "version-permanent-delete-busy-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) } - defer func() { _ = resp.Body.Close() }() - var raw struct { - Versions []map[string]any `json:"versions"` + _, versionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/file.txt", 8, "etag-current", "checksum-current", "text/plain", model.ObjectStateCached) + if _, _, err := srv.taskService.Enqueue(ctx, taskengine.EnqueueRequest{ + Type: model.TaskTypeUploadPlan, IdempotencyKey: "upload:" + versionID, + Input: map[string]any{"version_id": versionID}, SubjectType: "object_version", SubjectKey: versionID, + }); err != nil { + t.Fatalf("Enqueue upload task: %v", err) } - if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil { - t.Fatalf("Decode raw: %v", err) + + req := httptest.NewRequest(http.MethodPost, "/api/v1/buckets/version-permanent-delete-busy-bucket/objects/permanent-delete", strings.NewReader(fmt.Sprintf(`{"key":%q,"version_id":%q}`, "folder/file.txt", versionID))) + setBucketWriteHeaders(req) + rr := httptest.NewRecorder() + newBucketAPIMux(srv).ServeHTTP(rr, req) + if rr.Code != http.StatusConflict { + t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusConflict, rr.Body.String()) } - if raw.Versions[0]["state"] != string(model.ObjectStateCached) { - t.Fatalf("version list state = %#v, want cached", raw.Versions[0]["state"]) + var body map[string]string + if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { + t.Fatalf("Decode: %v", err) } - if _, ok := raw.Versions[0]["storage"]; ok { - t.Fatal("version list exposed storage instead of location") + want := "Storage work for this version is still in progress or awaiting Filecoin confirmation. Check the related task, then try again." + if body["error"] != want { + t.Fatalf("error = %q, want %q", body["error"], want) } } -func TestAPIBucketObjects_StatusMappingAndDetail(t *testing.T) { +func TestAPIBucketObjectDeletionsAllowsReadWithoutWriteHeader(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "status-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "deletion-history-auth-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - _, warningVersionID := seedAdminObjectVersion(t, repos, bucket, "warning.txt", 4, "etag-warning", "checksum-warning", "text/plain", "", model.ObjectStateCached) - if err := repos.Objects.UpdateVersionState(ctx, warningVersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("warning uploading: %v", err) - } - if err := repos.Objects.UpdateVersionStateToFailed(ctx, warningVersionID, model.ObjectStateUploading, "provider rejected piece"); err != nil { - t.Fatalf("warning failed: %v", err) - } - - unavailable := &model.ObjectVersion{ - VersionID: "01J000000000000000UNAVAIL", - BucketID: bucket.ID, - Key: "unavailable.txt", - Size: 1, - ETag: "etag-unavailable", - Checksum: "checksum-unavailable", - ContentType: "text/plain", - CacheKey: "cache-unavailable", - State: model.ObjectStateCached, - InCache: false, - } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, unavailable); err != nil { - t.Fatalf("unavailable version: %v", err) - } - if err := repos.Objects.SetVersionCachePresence(ctx, unavailable.VersionID, false); err != nil { - t.Fatalf("unavailable cache presence: %v", err) - } - _, storedOnPrimaryVersionID := seedAdminObjectVersion(t, repos, bucket, "stored-primary.txt", 2, "etag-primary", "checksum-primary", "text/plain", "", model.ObjectStateCached) - markAdminStoredOnPrimaryUpload(t, repos, storedOnPrimaryVersionID) - _, partialVersionID := seedAdminObjectVersion(t, repos, bucket, "partial.txt", 3, "etag-partial", "checksum-partial", "text/plain", "", model.ObjectStateCached) - bindAdminPartialUpload(t, repos, partialVersionID) - _, failedUploadVersionID := seedAdminObjectVersion(t, repos, bucket, "failed-upload.txt", 5, "etag-failed-upload", "checksum-failed-upload", "text/plain", "", model.ObjectStateCached) - markAdminFailedUpload(t, repos, failedUploadVersionID, "provider rejected piece") ts := httptest.NewServer(newBucketAPIMux(srv)) defer ts.Close() - resp, err := http.Get(ts.URL + "/api/v1/buckets/status-bucket/objects") - if err != nil { - t.Fatalf("GET bucket objects: %v", err) - } - defer func() { _ = resp.Body.Close() }() - - var list struct { - Objects []struct { - Key string `json:"key"` - Status string `json:"status"` - UploadStatus string `json:"upload_status"` - } `json:"objects"` - } - if err := json.NewDecoder(resp.Body).Decode(&list); err != nil { - t.Fatalf("Decode list: %v", err) - } - - statusByKey := map[string]string{} - uploadStatusByKey := map[string]string{} - for _, object := range list.Objects { - statusByKey[object.Key] = object.Status - uploadStatusByKey[object.Key] = object.UploadStatus - } - if statusByKey["warning.txt"] != "warning" { - t.Fatalf("warning status = %q, want warning", statusByKey["warning.txt"]) - } - if statusByKey["unavailable.txt"] != "unavailable" { - t.Fatalf("unavailable status = %q, want unavailable", statusByKey["unavailable.txt"]) - } - if uploadStatusByKey["stored-primary.txt"] != string(model.StorageUploadStatusIngressReady) { - t.Fatalf("stored-primary upload_status = %q, want ingress_ready", uploadStatusByKey["stored-primary.txt"]) - } - if statusByKey["partial.txt"] != "syncing" || uploadStatusByKey["partial.txt"] != string(model.StorageUploadStatusReadable) { - t.Fatalf("partial status/upload_status = %q/%q, want syncing/readable", statusByKey["partial.txt"], uploadStatusByKey["partial.txt"]) - } - if statusByKey["failed-upload.txt"] != "warning" || uploadStatusByKey["failed-upload.txt"] != string(model.StorageUploadStatusFailed) { - t.Fatalf("failed upload status/upload_status = %q/%q, want warning/failed", statusByKey["failed-upload.txt"], uploadStatusByKey["failed-upload.txt"]) - } - - resp, err = http.Get(ts.URL + "/api/v1/buckets/status-bucket/objects/status-detail?version_id=" + url.QueryEscape(warningVersionID)) + resp, err := http.Get(ts.URL + "/api/v1/buckets/deletion-history-auth-bucket/objects/deletions") if err != nil { - t.Fatalf("GET status detail: %v", err) + t.Fatalf("GET object deletions: %v", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - t.Fatalf("status detail code = %d, want %d", resp.StatusCode, http.StatusOK) - } - var detail objectStatusDetailResponse - if err := json.NewDecoder(resp.Body).Decode(&detail); err != nil { - t.Fatalf("Decode detail: %v", err) - } - if detail.VersionID != warningVersionID || detail.Status != "warning" { - t.Fatalf("detail = %#v, want version %s warning", detail, warningVersionID) - } - if detail.State != string(model.ObjectStateFailed) { - t.Fatalf("detail state = %q, want failed", detail.State) + t.Fatalf("object deletions status = %d, want %d, body=%s", resp.StatusCode, http.StatusOK, readBody(t, resp.Body)) } - if detail.FailedAtState == nil || *detail.FailedAtState != string(model.ObjectStateUploading) { - t.Fatalf("failed_at_state = %#v, want uploading", detail.FailedAtState) +} + +func TestAPIBucketObjects_RestoreRejectsStaleMarker(t *testing.T) { + srv, repos := newBucketAPITestServer(t) + ctx := context.Background() + + bucket := &model.Bucket{Name: "stale-restore-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) } - if detail.Message == nil || *detail.Message != "provider rejected piece" { - t.Fatalf("message = %#v, want provider rejected piece", detail.Message) + seedAdminObjectVersion(t, srv.db, repos, bucket, "file.txt", 4, "etag-old", "checksum-old", "text/plain", model.ObjectStateCached) + marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "file.txt", "01J0000000000000000000ADM1") + if err != nil { + t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) } + seedAdminObjectVersion(t, srv.db, repos, bucket, "file.txt", 6, "etag-new", "checksum-new", "text/plain", model.ObjectStateCached) - resp, err = http.Get(ts.URL + "/api/v1/buckets/status-bucket/objects/status-detail?version_id=" + url.QueryEscape(partialVersionID)) + ts := httptest.NewServer(newBucketAPIMux(srv)) + defer ts.Close() + + req, err := http.NewRequest(http.MethodPost, ts.URL+"/api/v1/buckets/stale-restore-bucket/objects/restore", strings.NewReader(fmt.Sprintf(`{"key":%q,"delete_marker_version_id":%q}`, "file.txt", marker.VersionID))) if err != nil { - t.Fatalf("GET partial status detail: %v", err) + t.Fatalf("NewRequest restore: %v", err) } - defer func() { _ = resp.Body.Close() }() - var partialDetail struct { - VersionID string `json:"version_id"` - Status string `json:"status"` - UploadStatus string `json:"upload_status"` - Message string `json:"message"` + setBucketWriteHeaders(req) + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatalf("POST stale restore: %v", err) } - if err := json.NewDecoder(resp.Body).Decode(&partialDetail); err != nil { - t.Fatalf("Decode partial detail: %v", err) + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want %d, body=%s", resp.StatusCode, http.StatusConflict, readBody(t, resp.Body)) } - if partialDetail.VersionID != partialVersionID || partialDetail.Status != "syncing" || partialDetail.UploadStatus != string(model.StorageUploadStatusReadable) { - t.Fatalf("partial detail = %#v, want readable syncing detail", partialDetail) +} + +func TestAPIBucketDeletedObjects_ExcludesMarkerWithoutDataVersion(t *testing.T) { + srv, repos := newBucketAPITestServer(t) + ctx := context.Background() + + bucket := &model.Bucket{Name: "empty-trash-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) } - if partialDetail.Message != "secondary pull: timeout" { - t.Fatalf("partial message = %q, want secondary pull: timeout", partialDetail.Message) + marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "missing.txt", "01J0000000000000000000ADM2") + if err != nil { + t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) } - resp, err = http.Get(ts.URL + "/api/v1/buckets/status-bucket/objects/status-detail?version_id=" + url.QueryEscape(failedUploadVersionID)) + ts := httptest.NewServer(newBucketAPIMux(srv)) + defer ts.Close() + + resp, err := http.Get(ts.URL + "/api/v1/buckets/empty-trash-bucket/objects/deleted") if err != nil { - t.Fatalf("GET failed upload status detail: %v", err) + t.Fatalf("GET deleted objects: %v", err) } defer func() { _ = resp.Body.Close() }() - var failedUploadDetail struct { - UploadStatus string `json:"upload_status"` - Message string `json:"message"` + var body struct { + Objects []struct{} `json:"objects"` } - if err := json.NewDecoder(resp.Body).Decode(&failedUploadDetail); err != nil { - t.Fatalf("Decode failed upload detail: %v", err) + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("Decode deleted objects: %v", err) } - if failedUploadDetail.UploadStatus != string(model.StorageUploadStatusFailed) || failedUploadDetail.Message != "provider rejected piece" { - t.Fatalf("failed upload detail = %#v, want failed/provider rejected piece", failedUploadDetail) + if len(body.Objects) != 0 { + t.Fatalf("deleted objects len = %d, want marker without data hidden", len(body.Objects)) } - otherBucket := &model.Bucket{Name: "other-status-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, otherBucket); err != nil { - t.Fatalf("other bucket: %v", err) + req, err := http.NewRequest(http.MethodPost, ts.URL+"/api/v1/buckets/empty-trash-bucket/objects/restore", strings.NewReader(fmt.Sprintf(`{"key":%q,"delete_marker_version_id":%q}`, "missing.txt", marker.VersionID))) + if err != nil { + t.Fatalf("NewRequest restore: %v", err) } - resp, err = http.Get(ts.URL + "/api/v1/buckets/other-status-bucket/objects/status-detail?version_id=" + url.QueryEscape(warningVersionID)) + setBucketWriteHeaders(req) + restoreResp, err := ts.Client().Do(req) if err != nil { - t.Fatalf("GET status detail wrong bucket: %v", err) + t.Fatalf("POST restore without data: %v", err) } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusNotFound { - t.Fatalf("wrong bucket status detail code = %d, want %d", resp.StatusCode, http.StatusNotFound) + defer func() { _ = restoreResp.Body.Close() }() + if restoreResp.StatusCode != http.StatusNotFound { + t.Fatalf("restore status = %d, want %d, body=%s", restoreResp.StatusCode, http.StatusNotFound, readBody(t, restoreResp.Body)) } } func TestAPIBucketObjectsIncludesPrimaryTransferProgress(t *testing.T) { srv, repos := newBucketAPITestServer(t) - bucket := &model.Bucket{Name: "object-progress-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "object-progress-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(context.Background(), bucket); err != nil { t.Fatalf("Create bucket: %v", err) } - _, versionID := seedAdminObjectVersion(t, repos, bucket, "uploading.txt", 10, "etag-progress", "checksum-progress", "text/plain", "", model.ObjectStateUploading) - upload, err := repos.Uploads.StartObjectUploadAttempt(context.Background(), repository.StartObjectUploadAttemptInput{ + seedAdminObjectVersion(t, srv.db, repos, bucket, "uploading.txt", 10, "etag-progress", "checksum-progress", "text/plain", model.ObjectStateUploading) + upload, err := repos.Contents.EnsureContent(context.Background(), repository.EnsureContentInput{ BucketID: bucket.ID, - SourceVersionID: versionID, ContentSize: 10, - Checksum: "checksum-progress", + Checksum: testutil.StorageChecksum("checksum-progress"), RequestedCopies: 3, }) if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) + t.Fatalf("EnsureContent: %v", err) } - progressUpload, err := repos.Uploads.BeginIngressStoreProgress(context.Background(), upload.ID) + // Ingress progress belongs to the transfer that produces it, so the content + // needs its ingress copy before progress can be recorded. + binding, err := repos.Contents.EnsureDataSetBinding(context.Background(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, + ProviderID: onChainID(t, "101"), + CopyIndex: 0, + CreatedByContentID: upload.ID, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + if err := repos.Contents.CreateUploadCopiesForBindings(context.Background(), upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: binding.ID, + CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodIngress, + ProviderID: binding.ProviderID, + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings: %v", err) + } + copies, err := repos.Contents.ListCopies(context.Background(), upload.ID) + if err != nil || len(copies) != 1 { + t.Fatalf("ListCopies = %#v, err=%v", copies, err) + } + taskRow, _, err := repos.Tasks.Enqueue(context.Background(), &model.Task{ + Type: model.TaskTypeStorageStore, IdempotencyKey: "progress-display", InputVersion: 1, + Input: []byte(`{}`), InputHash: "progress-display", Status: model.TaskStatusPending, + ResumeMode: model.TaskResumeModeExecute, AvailableAt: time.Now(), + }) + if err != nil { + t.Fatalf("Enqueue store task: %v", err) + } + if err := repos.Contents.BindCopyTask(context.Background(), copies[0].ID, 1, taskRow.ID); err != nil { + t.Fatalf("BindCopyTask: %v", err) + } + progressUpload, err := repos.Contents.BeginIngressStoreProgress(context.Background(), repository.BeginIngressStoreProgressInput{ + CopyID: copies[0].ID, Generation: 1, TaskID: taskRow.ID, Attempt: 1, + }) if err != nil { t.Fatalf("BeginIngressStoreProgress: %v", err) } - if _, err := repos.Uploads.RecordIngressStoreProgress(context.Background(), repository.RecordIngressStoreProgressInput{ - UploadID: upload.ID, + if _, err := repos.Contents.RecordIngressStoreProgress(context.Background(), repository.RecordIngressStoreProgressInput{ + CopyID: copies[0].ID, + Generation: 1, + TaskID: taskRow.ID, Attempt: progressUpload.IngressStoreAttempt, BytesUploaded: 4, }); err != nil { @@ -4381,233 +4527,6 @@ func TestAPIBucketObjectsIncludesPrimaryTransferProgress(t *testing.T) { } } -func TestAPIBucketObjectProvenance(t *testing.T) { - srv, repos := newBucketAPITestServer(t) - identityResolver := &fakeAPIProviderIdentityResolver{ - identities: map[string]*providerIdentityResponse{ - "101": { - RegistryProviderID: "101", - Name: "alpha-pdp", - ServiceProviderAddress: "0x1111111111111111111111111111111111111111", - FilecoinActorID: "f01234", - ServiceURL: "https://alpha.example", - }, - "303": { - RegistryProviderID: "303", - Name: "gamma-pdp", - FilecoinActorID: "f05678", - }, - }, - } - srv.WithProviderIdentityResolver(identityResolver) - ctx := context.Background() - - bucket := &model.Bucket{Name: "provenance-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create: %v", err) - } - _, oldVersionID := seedAdminObjectVersion(t, repos, bucket, "file.txt", 7, "etag-old-provenance", "checksum-old-provenance", "text/plain", "", model.ObjectStateStored) - _, versionID := seedAdminObjectVersion(t, repos, bucket, "file.txt", 8, "etag-provenance", "checksum-provenance", "text/plain", "", model.ObjectStateCached) - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark uploading: %v", err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: 8, - Checksum: "checksum-provenance", - RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - pieceCID := "bafk2bzaceadminprovenance" - seedAdminCommittedCopies(t, repos, bucket.ID, upload.ID, pieceCID, []adminStorageCopySeed{ - {ProviderID: onChainID(t, "101"), DataSetID: onChainID(t, "1001"), PieceID: onChainIDPtr(t, "2001"), TransferMethod: model.StorageCopyTransferMethodIngress, RetrievalURL: "https://ingress.example/piece"}, - {ProviderID: onChainID(t, "202"), DataSetID: onChainID(t, "2002"), PieceID: onChainIDPtr(t, "3001"), TransferMethod: model.StorageCopyTransferMethodPeerPull, RetrievalURL: "https://peer.example/piece"}, - }) - if err := repos.Uploads.AppendUploadFailure(ctx, repository.AppendUploadFailureInput{ - UploadID: upload.ID, - ProviderID: onChainIDPtr(t, "303"), - TransferMethod: string(model.StorageCopyTransferMethodPeerPull), - Stage: "peer_pull", - ErrorMessage: "provider timed out", - }); err != nil { - t.Fatalf("AppendUploadFailure: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: 8, - Checksum: "checksum-provenance", - }); err != nil { - t.Fatalf("BindReadableUploadForContent: %v", err) - } - if _, _, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}); err != nil { - t.Fatalf("FinalizeUploadIfTargetCopiesMet: %v", err) - } - - _, noUploadVersionID := seedAdminObjectVersion(t, repos, bucket, "cached.txt", 3, "etag-cached", "checksum-cached", "text/plain", "", model.ObjectStateCached) - _, partialVersionID := seedAdminObjectVersion(t, repos, bucket, "partial-provenance.txt", 4, "etag-partial-provenance", "checksum-partial-provenance", "text/plain", "", model.ObjectStateCached) - bindAdminPartialUpload(t, repos, partialVersionID) - _, failedVersionID := seedAdminObjectVersion(t, repos, bucket, "failed-provenance.txt", 5, "etag-failed-provenance", "checksum-failed-provenance", "text/plain", "", model.ObjectStateCached) - markAdminFailedUpload(t, repos, failedVersionID, "provider rejected piece") - replaceBucketDataSetObservability(t, repos, bucket.ID, observability.StatusAvailable, nil, time.Now().UTC()) - srv.WithObservability(observability.NewService(observability.ServiceOptions{ - Store: repos.Observability, - RefreshInterval: time.Hour, - })) - - ts := httptest.NewServer(newBucketAPIMux(srv)) - defer ts.Close() - - type provenanceBody struct { - VersionID string `json:"version_id"` - Status string `json:"status"` - UploadStatus string `json:"upload_status"` - PieceCID string `json:"piece_cid"` - RequestedCopies int `json:"requested_copies"` - SuccessCopies int `json:"success_copies"` - CopyHealth apiCopyHealthBody `json:"copy_health"` - Copies []struct { - CopyIndex int `json:"copy_index"` - Status string `json:"status"` - Health apiCopyHealthBody `json:"health"` - ProviderID string `json:"provider_id"` - ProviderIdentity *providerIdentityResponse `json:"provider_identity"` - DataSetID string `json:"data_set_id"` - PieceID string `json:"piece_id"` - TransferMethod string `json:"transfer_method"` - RetrievalURL string `json:"retrieval_url"` - IsNewDataSet bool `json:"is_new_data_set"` - } `json:"copies"` - Failures []struct { - AttemptIndex int `json:"attempt_index"` - ProviderID string `json:"provider_id"` - ProviderIdentity *providerIdentityResponse `json:"provider_identity"` - TransferMethod string `json:"transfer_method"` - Stage string `json:"stage"` - Error string `json:"error"` - } `json:"failures"` - } - getProvenance := func(bucketName, targetVersionID string) (provenanceBody, int) { - t.Helper() - resp, err := http.Get(ts.URL + "/api/v1/buckets/" + bucketName + "/objects/provenance?version_id=" + url.QueryEscape(targetVersionID)) - if err != nil { - t.Fatalf("GET provenance: %v", err) - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - return provenanceBody{}, resp.StatusCode - } - var body provenanceBody - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("Decode provenance: %v", err) - } - return body, resp.StatusCode - } - - detail, statusCode := getProvenance("provenance-bucket", versionID) - if statusCode != http.StatusOK { - t.Fatalf("status = %d, want %d", statusCode, http.StatusOK) - } - if detail.VersionID != versionID || detail.Status != "success" || detail.UploadStatus != string(model.StorageUploadStatusComplete) { - t.Fatalf("detail status = %#v, want stored provenance", detail) - } - if detail.PieceCID != pieceCID || detail.RequestedCopies != 2 || detail.SuccessCopies != 2 { - t.Fatalf("detail counts = %#v, want piece and 2/2 copies", detail) - } - if detail.CopyHealth.Status != string(observability.StatusAvailable) || detail.CopyHealth.RequestedCopies != 2 || detail.CopyHealth.ReadableCopies != 2 || len(detail.CopyHealth.ReasonCodes) != 0 { - t.Fatalf("copy_health = %#v, want healthy 2/2 provenance", detail.CopyHealth) - } - if len(detail.Copies) != 2 || detail.Copies[0].ProviderID != "101" || detail.Copies[0].TransferMethod != string(model.StorageCopyTransferMethodIngress) || detail.Copies[1].TransferMethod != string(model.StorageCopyTransferMethodPeerPull) { - t.Fatalf("copies = %#v, want provider scoped copy provenance", detail.Copies) - } - if detail.Copies[0].Health.Status != string(observability.StatusAvailable) || len(detail.Copies[0].Health.ReasonCodes) != 0 { - t.Fatalf("copy health = %#v, want healthy copy", detail.Copies[0].Health) - } - if detail.Copies[0].ProviderIdentity == nil || detail.Copies[0].ProviderIdentity.Name != "alpha-pdp" || detail.Copies[0].ProviderIdentity.FilecoinActorID != "f01234" { - t.Fatalf("copy provider_identity = %#v, want enriched copy identity", detail.Copies[0].ProviderIdentity) - } - if detail.Copies[1].ProviderID != "202" || detail.Copies[1].ProviderIdentity != nil { - t.Fatalf("secondary copy = %#v, want provider_id compatibility without identity when lookup fails", detail.Copies[1]) - } - if len(detail.Failures) != 1 || detail.Failures[0].ProviderID != "303" || detail.Failures[0].Stage != "peer_pull" || detail.Failures[0].Error != "provider timed out" { - t.Fatalf("failures = %#v, want recorded provider attempt", detail.Failures) - } - if detail.Failures[0].ProviderIdentity == nil || detail.Failures[0].ProviderIdentity.Name != "gamma-pdp" || detail.Failures[0].ProviderIdentity.FilecoinActorID != "f05678" { - t.Fatalf("failure provider_identity = %#v, want enriched failure identity", detail.Failures[0].ProviderIdentity) - } - peerBinding, err := repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 1) - if err != nil || peerBinding == nil { - t.Fatalf("GetDataSetBindingByCopyIndex peer: binding=%v err=%v", peerBinding, err) - } - if err := repos.Uploads.MarkDataSetUnavailable(ctx, peerBinding.ID, "provider dataset retired"); err != nil { - t.Fatalf("MarkDataSetUnavailable peer: %v", err) - } - unavailableDetail, statusCode := getProvenance("provenance-bucket", versionID) - if statusCode != http.StatusOK { - t.Fatalf("unavailable status = %d, want %d", statusCode, http.StatusOK) - } - if unavailableDetail.SuccessCopies != 1 { - t.Fatalf("unavailable provenance = %#v, want unavailable dataset excluded from readable copies", unavailableDetail) - } - if err := repos.Uploads.MarkDataSetDraining(ctx, peerBinding.ID, "provider service ended"); err != nil { - t.Fatalf("MarkDataSetDraining peer: %v", err) - } - drainingDetail, statusCode := getProvenance("provenance-bucket", versionID) - if statusCode != http.StatusOK { - t.Fatalf("draining status = %d, want %d", statusCode, http.StatusOK) - } - if drainingDetail.SuccessCopies != 2 { - t.Fatalf("draining provenance = %#v, want draining dataset counted as readable", drainingDetail) - } - if !reflect.DeepEqual(identityResolver.requests[0], []string{"101", "202", "303"}) { - t.Fatalf("provider identity request = %#v, want one provenance snapshot request", identityResolver.requests) - } - - oldDetail, statusCode := getProvenance("provenance-bucket", oldVersionID) - if statusCode != http.StatusOK { - t.Fatalf("old version status = %d, want %d", statusCode, http.StatusOK) - } - if oldDetail.VersionID != oldVersionID || oldDetail.Status != "success" || oldDetail.SuccessCopies != 1 { - t.Fatalf("old version provenance = %#v, want historical stored version", oldDetail) - } - - partialDetail, statusCode := getProvenance("provenance-bucket", partialVersionID) - if statusCode != http.StatusOK { - t.Fatalf("partial status = %d, want %d", statusCode, http.StatusOK) - } - if partialDetail.Status != "syncing" || partialDetail.UploadStatus != string(model.StorageUploadStatusReadable) || partialDetail.SuccessCopies != 1 { - t.Fatalf("partial provenance = %#v, want readable upload detail", partialDetail) - } - - failedDetail, statusCode := getProvenance("provenance-bucket", failedVersionID) - if statusCode != http.StatusOK { - t.Fatalf("failed status = %d, want %d", statusCode, http.StatusOK) - } - if failedDetail.Status != "warning" || failedDetail.UploadStatus != string(model.StorageUploadStatusFailed) || failedDetail.SuccessCopies != 0 { - t.Fatalf("failed provenance = %#v, want failed upload detail", failedDetail) - } - - noUpload, statusCode := getProvenance("provenance-bucket", noUploadVersionID) - if statusCode != http.StatusOK { - t.Fatalf("no-upload status = %d, want %d", statusCode, http.StatusOK) - } - if noUpload.VersionID != noUploadVersionID || len(noUpload.Copies) != 0 || len(noUpload.Failures) != 0 || noUpload.UploadStatus != "" { - t.Fatalf("no-upload provenance = %#v, want empty upload detail", noUpload) - } - - otherBucket := &model.Bucket{Name: "other-provenance-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, otherBucket); err != nil { - t.Fatalf("other bucket: %v", err) - } - _, statusCode = getProvenance("other-provenance-bucket", versionID) - if statusCode != http.StatusNotFound { - t.Fatalf("wrong bucket provenance code = %d, want %d", statusCode, http.StatusNotFound) - } -} - func TestAPIBucketObjects_LoadsUploadStatusInBatches(t *testing.T) { db := testutil.NewTestDB(t) counter := &storageUploadSelectCounter{} @@ -4619,17 +4538,17 @@ func TestAPIBucketObjects_LoadsUploadStatusInBatches(t *testing.T) { repos := repository.NewRepositories(db) srv := newTestServer("127.0.0.1:0", db, localCache, 1<<20, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()) ctx := context.Background() - bucket := &model.Bucket{Name: "batched-upload-status-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "batched-upload-status-bucket", Status: model.BucketStatusActive, DefaultCopies: 2, MinimumDurableCopies: 2} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - seedAdminObjectVersion(t, repos, bucket, "stored.txt", 2, "etag-stored", "checksum-stored", "text/plain", "", model.ObjectStateStored) - _, primaryVersionID := seedAdminObjectVersion(t, repos, bucket, "primary.txt", 3, "etag-primary", "checksum-primary", "text/plain", "", model.ObjectStateCached) - markAdminStoredOnPrimaryUpload(t, repos, primaryVersionID) - _, partialVersionID := seedAdminObjectVersion(t, repos, bucket, "partial.txt", 4, "etag-partial", "checksum-partial", "text/plain", "", model.ObjectStateCached) - bindAdminPartialUpload(t, repos, partialVersionID) - _, failedVersionID := seedAdminObjectVersion(t, repos, bucket, "failed.txt", 5, "etag-failed", "checksum-failed", "text/plain", "", model.ObjectStateCached) - markAdminFailedUpload(t, repos, failedVersionID, "provider rejected piece") + seedAdminObjectVersion(t, srv.db, repos, bucket, "stored.txt", 2, "etag-stored", "checksum-stored", "text/plain", model.ObjectStateStored) + _, primaryVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "primary.txt", 3, "etag-primary", "checksum-primary", "text/plain", model.ObjectStateCached) + markAdminStoredOnPrimaryUpload(t, srv.db, repos, primaryVersionID) + _, partialVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "partial.txt", 4, "etag-partial", "checksum-partial", "text/plain", model.ObjectStateCached) + bindAdminPartialUpload(t, srv.db, repos, partialVersionID) + _, failedVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "failed.txt", 5, "etag-failed", "checksum-failed", "text/plain", model.ObjectStateCached) + markAdminFailedUpload(t, srv.db, repos, failedVersionID, "provider rejected piece") counter.selects.Store(0) ts := httptest.NewServer(newBucketAPIMux(srv)) @@ -4658,17 +4577,17 @@ func TestAPIBucketObjectVersions_LoadsUploadStatusInBatches(t *testing.T) { repos := repository.NewRepositories(db) srv := newTestServer("127.0.0.1:0", db, localCache, 1<<20, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()) ctx := context.Background() - bucket := &model.Bucket{Name: "batched-version-status-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "batched-version-status-bucket", Status: model.BucketStatusActive, DefaultCopies: 2, MinimumDurableCopies: 2} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - seedAdminObjectVersion(t, repos, bucket, "file.txt", 2, "etag-stored", "checksum-stored", "text/plain", "", model.ObjectStateStored) - _, primaryVersionID := seedAdminObjectVersion(t, repos, bucket, "file.txt", 3, "etag-primary", "checksum-primary", "text/plain", "", model.ObjectStateCached) - markAdminStoredOnPrimaryUpload(t, repos, primaryVersionID) - _, partialVersionID := seedAdminObjectVersion(t, repos, bucket, "file.txt", 4, "etag-partial", "checksum-partial", "text/plain", "", model.ObjectStateCached) - bindAdminPartialUpload(t, repos, partialVersionID) - _, failedVersionID := seedAdminObjectVersion(t, repos, bucket, "file.txt", 5, "etag-failed", "checksum-failed", "text/plain", "", model.ObjectStateCached) - markAdminFailedUpload(t, repos, failedVersionID, "provider rejected piece") + seedAdminObjectVersion(t, srv.db, repos, bucket, "file.txt", 2, "etag-stored", "checksum-stored", "text/plain", model.ObjectStateStored) + _, primaryVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "file.txt", 3, "etag-primary", "checksum-primary", "text/plain", model.ObjectStateCached) + markAdminStoredOnPrimaryUpload(t, srv.db, repos, primaryVersionID) + _, partialVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "file.txt", 4, "etag-partial", "checksum-partial", "text/plain", model.ObjectStateCached) + bindAdminPartialUpload(t, srv.db, repos, partialVersionID) + _, failedVersionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "file.txt", 5, "etag-failed", "checksum-failed", "text/plain", model.ObjectStateCached) + markAdminFailedUpload(t, srv.db, repos, failedVersionID, "provider rejected piece") counter.selects.Store(0) ts := httptest.NewServer(newBucketAPIMux(srv)) @@ -4689,7 +4608,7 @@ func TestAPIBucketObjectVersions_LoadsUploadStatusInBatches(t *testing.T) { func TestAPIBucketObjectUpload_PassesRequestToUploader(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "upload-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "upload-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -4760,7 +4679,7 @@ func TestAPIBucketObjectUpload_PassesRequestToUploader(t *testing.T) { func TestAPIBucketObjectUpload_RejectsMissingUploaderSize(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "upload-missing-size-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "upload-missing-size-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -4790,7 +4709,7 @@ func TestAPIBucketObjectUpload_RejectsMissingUploaderSize(t *testing.T) { func TestAPIBucketObjectUpload_RejectsEmptyObject(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "upload-empty-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "upload-empty-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -4843,7 +4762,7 @@ func TestAPIBucketObjectUpload_RejectsFOCSizeLimits(t *testing.T) { t.Run(tt.name, func(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "upload-size-limit-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "upload-size-limit-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -4972,7 +4891,7 @@ func TestAPIBucketObjectUpload_MapsUploaderErrors(t *testing.T) { func TestAPIBucketObjectUpload_AppearsInObjectList(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "upload-list-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "upload-list-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -5054,14 +4973,23 @@ func TestAPIBucketObjectDownload_FromCache(t *testing.T) { func TestAPIBucketObjectDownload_WithVersionID(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "download-version-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "download-version-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } oldVersionID := model.NewVersionID() - oldCacheKey := ".versions/" + oldVersionID - oldInfo, err := srv.cache.Put(ctx, bucket.Name, oldCacheKey, strings.NewReader("old admin")) + oldSum := sha256.Sum256([]byte("old admin")) + oldContent, err := repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, + ContentSize: int64(len("old admin")), + Checksum: hex.EncodeToString(oldSum[:]), + RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("EnsureContent old: %v", err) + } + oldInfo, err := srv.cache.Put(ctx, bucket.Name, model.ContentCacheKey(oldContent.ID), strings.NewReader("old admin")) if err != nil { t.Fatalf("cache.Put old: %v", err) } @@ -5069,18 +4997,25 @@ func TestAPIBucketObjectDownload_WithVersionID(t *testing.T) { VersionID: oldVersionID, BucketID: bucket.ID, Key: "folder/report.txt", + ContentID: &oldContent.ID, Size: oldInfo.Size, ETag: oldInfo.ETag, - Checksum: oldInfo.Checksum, ContentType: "text/plain", - CacheKey: oldCacheKey, - State: model.ObjectStateCached, }); err != nil { t.Fatalf("create old version: %v", err) } newVersionID := model.NewVersionID() - newCacheKey := ".versions/" + newVersionID - newInfo, err := srv.cache.Put(ctx, bucket.Name, newCacheKey, strings.NewReader("new admin")) + newSum := sha256.Sum256([]byte("new admin")) + newContent, err := repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, + ContentSize: int64(len("new admin")), + Checksum: hex.EncodeToString(newSum[:]), + RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("EnsureContent new: %v", err) + } + newInfo, err := srv.cache.Put(ctx, bucket.Name, model.ContentCacheKey(newContent.ID), strings.NewReader("new admin")) if err != nil { t.Fatalf("cache.Put new: %v", err) } @@ -5088,12 +5023,10 @@ func TestAPIBucketObjectDownload_WithVersionID(t *testing.T) { VersionID: newVersionID, BucketID: bucket.ID, Key: "folder/report.txt", + ContentID: &newContent.ID, Size: newInfo.Size, ETag: newInfo.ETag, - Checksum: newInfo.Checksum, ContentType: "text/plain", - CacheKey: newCacheKey, - State: model.ObjectStateCached, }); err != nil { t.Fatalf("create new version: %v", err) } @@ -5119,7 +5052,7 @@ func TestAPIBucketObjectDownload_WithVersionID(t *testing.T) { func TestAPIBucketObjectDownload_DeleteMarkerVersionIsMethodNotAllowed(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "download-marker-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "download-marker-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -5142,11 +5075,11 @@ func TestAPIBucketObjectDownload_DeleteMarkerVersionIsMethodNotAllowed(t *testin func TestAPIBucketObjectDownload_ClearsWriteDeadline(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "deadline-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "deadline-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - seedAdminObjectVersion(t, repos, bucket, "folder/report.txt", 11, "etag", "checksum", "text/plain", "", model.ObjectStateCached) + seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/report.txt", 11, "etag", "checksum", "text/plain", model.ObjectStateCached) var rr *writeDeadlineRecorder mockCache := &testutil.MockCache{ GetFunc: func(_ context.Context, _, _ string) (io.ReadCloser, *cache.ObjectInfo, error) { @@ -5189,7 +5122,7 @@ func TestAPIBucketObjectDownload_NotFound(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "download-missing-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "download-missing-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -5243,7 +5176,7 @@ func TestAPIBucket_DeleteReturnsNotImplemented(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "delete-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "delete-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -5283,12 +5216,12 @@ func TestAPIBucket_DeleteRecursiveReturnsNotImplemented(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "recursive-delete-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "recursive-delete-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - seedAdminObjectVersion(t, repos, bucket, "file.txt", 5, "etag-file", "checksum-file", "text/plain", "", model.ObjectStateStored) + seedAdminObjectVersion(t, srv.db, repos, bucket, "file.txt", 5, "etag-file", "checksum-file", "text/plain", model.ObjectStateStored) req := httptest.NewRequest(http.MethodDelete, "/api/v1/buckets/recursive-delete-bucket?recursive=true", nil) req.SetPathValue("name", bucket.Name) @@ -5320,3 +5253,14 @@ func TestAPIBucket_DeleteRecursiveReturnsNotImplemented(t *testing.T) { t.Fatalf("visible objects len = %d, want 1", len(objects)) } } + +func assertActiveBucketReplicaSlots(t *testing.T, ctx context.Context, repos *repository.Repositories, bucketID int64, want int) { + t.Helper() + slots, err := repos.Buckets.ActiveReplicaSlots(ctx, bucketID) + if err != nil { + t.Fatalf("ActiveReplicaSlots: %v", err) + } + if len(slots) != want { + t.Fatalf("active replica slots = %v, want %d", slots, want) + } +} diff --git a/internal/admin/api_copy_health.go b/internal/admin/api_copy_health.go index 1c110a3..476fe1e 100644 --- a/internal/admin/api_copy_health.go +++ b/internal/admin/api_copy_health.go @@ -2,6 +2,7 @@ package admin import ( "context" + "slices" "sort" "time" @@ -38,11 +39,10 @@ type copyHealthSummaryResponse struct { type copyHealthFact struct { BucketID int64 VersionID string - UploadID *int64 - UploadStatus *model.StorageUploadStatus + ContentID *int64 RequestedCopies int CopyIndex *int - CopyStatus *model.StorageUploadCopyStatus + CopyStatus *model.StorageCopyStatus ProviderID *types.OnChainID LocalDataSetID *int64 ChainDataSetID *types.OnChainID @@ -130,15 +130,17 @@ func copyHealthSummariesByBucket(facts []copyHealthFact, observations map[int64] objects[key] = object order = append(order, key) } - if fact.UploadID != nil { + if fact.ContentID != nil { object.hasUpload = true } - if fact.UploadStatus != nil { - object.uploadStatus = fact.UploadStatus - } if fact.CopyStatus == nil { continue } + if *fact.CopyStatus == model.StorageCopyStatusFailed { + object.sawFailedCopy = true + } else { + object.sawActiveCopy = true + } signal := copyHealthSignalFromFact(fact, observations, observationFailed, interval, now) object.addCandidate(fact, signal) } @@ -195,18 +197,21 @@ type copyHealthObjectAccumulator struct { bucketID int64 requestedCopies int hasUpload bool - uploadStatus *model.StorageUploadStatus - candidates []copyHealthCandidate - status observability.Status - reasons []observability.ReasonCode - reasonSet map[observability.ReasonCode]struct{} - stale bool - lastCheckedAt *time.Time - lastError *string - readableCopies int - pendingCopies int - failedCopies int - unknownCopies int + // Gap classification reads the copy rows directly. It used to read a + // stored upload status, which described the same pipeline one level away. + sawFailedCopy bool + sawActiveCopy bool + candidates []copyHealthCandidate + status observability.Status + reasons []observability.ReasonCode + reasonSet map[observability.ReasonCode]struct{} + stale bool + lastCheckedAt *time.Time + lastError *string + readableCopies int + pendingCopies int + failedCopies int + unknownCopies int } func (a *copyHealthObjectAccumulator) addCandidate(fact copyHealthFact, signal observability.Signal) { @@ -253,7 +258,7 @@ func (a *copyHealthObjectAccumulator) summarizePolicyCandidates() { }) selected := min(a.requestedCopies, len(a.candidates)) - for i := 0; i < selected; i++ { + for i := range selected { candidate := a.candidates[i] a.addSelectedCandidate(candidate) underReplicated = underReplicated || candidate.underReplicated @@ -308,10 +313,10 @@ func (a *copyHealthObjectAccumulator) classifyGapCopies(count int) { return } switch { - case a.uploadStatus != nil && (*a.uploadStatus == model.StorageUploadStatusFailed || *a.uploadStatus == model.StorageUploadStatusRejected): + case a.sawFailedCopy && !a.sawActiveCopy: a.failedCopies += count a.addReason(observability.ReasonCopyFailed) - case a.uploadStatus != nil && (*a.uploadStatus == model.StorageUploadStatusRunning || *a.uploadStatus == model.StorageUploadStatusIngressReady || *a.uploadStatus == model.StorageUploadStatusReadable): + case a.sawActiveCopy: a.pendingCopies += count a.addReason(observability.ReasonCopyPending) default: @@ -337,11 +342,11 @@ func (s *Server) copyHealthRefreshInterval() time.Duration { func copyHealthCandidateKindForFact(fact copyHealthFact, signal observability.Signal) copyHealthCandidateKind { switch derefCopyStatus(fact.CopyStatus) { - case model.StorageUploadCopyStatusPending, model.StorageUploadCopyStatusPieceReady, model.StorageUploadCopyStatusCommitting: + case model.StorageCopyStatusPending, model.StorageCopyStatusPieceReady, model.StorageCopyStatusCommitting: return copyHealthCandidatePending - case model.StorageUploadCopyStatusFailed: + case model.StorageCopyStatusFailed: return copyHealthCandidateFailed - case model.StorageUploadCopyStatusCommitted: + case model.StorageCopyStatusCommitted: if signal.Status == observability.StatusAvailable { return copyHealthCandidateReadable } @@ -407,40 +412,38 @@ func copyHealthSignalFromFact(fact copyHealthFact, observations map[int64]observ RetrievalURL: fact.RetrievalURL, LastError: fact.LastError, }, observation, interval, now) - if observationFailed && derefCopyStatus(fact.CopyStatus) == model.StorageUploadCopyStatusCommitted && hasAnyReason(signal.ReasonCodes, observability.ReasonCopyObservationMissing) { + if observationFailed && derefCopyStatus(fact.CopyStatus) == model.StorageCopyStatusCommitted && hasAnyReason(signal.ReasonCodes, observability.ReasonCopyObservationMissing) { return copyHealthQueryFailureSignal(interval, now) } return signal } -func provenanceCopyHealthFacts(bucketID int64, versionID string, upload model.StorageUpload, copies []model.StorageUploadCopy) []copyHealthFact { +func provenanceCopyHealthFacts(bucketID int64, versionID string, upload model.StorageContent, copies []model.StorageCopy) []copyHealthFact { if len(copies) == 0 { - uploadID := upload.ID - uploadStatus := upload.Status + contentID := upload.ID return []copyHealthFact{{ BucketID: bucketID, VersionID: versionID, - UploadID: &uploadID, - UploadStatus: &uploadStatus, + ContentID: &contentID, RequestedCopies: upload.RequestedCopies, }} } facts := make([]copyHealthFact, 0, len(copies)) for _, copyRow := range copies { - uploadID := upload.ID - uploadStatus := upload.Status + contentID := upload.ID copyIndex := copyRow.CopyIndex copyStatus := copyRow.Status + providerID := copyRow.ProviderID + localDataSetID := copyRow.StorageDataSetID facts = append(facts, copyHealthFact{ BucketID: bucketID, VersionID: versionID, - UploadID: &uploadID, - UploadStatus: &uploadStatus, + ContentID: &contentID, RequestedCopies: upload.RequestedCopies, CopyIndex: ©Index, CopyStatus: ©Status, - ProviderID: copyRow.ProviderID, - LocalDataSetID: copyRow.StorageDataSetID, + ProviderID: &providerID, + LocalDataSetID: &localDataSetID, ChainDataSetID: copyRow.DataSetID, PieceID: copyRow.PieceID, RetrievalURL: copyRow.RetrievalURL, @@ -510,7 +513,7 @@ func oldestLastCheckedAtString(current string, candidate *time.Time) string { return current } -func derefCopyStatus(status *model.StorageUploadCopyStatus) model.StorageUploadCopyStatus { +func derefCopyStatus(status *model.StorageCopyStatus) model.StorageCopyStatus { if status == nil { return "" } @@ -539,10 +542,8 @@ func observabilityStatusRank(status observability.Status) int { func hasAnyReason(reasons []observability.ReasonCode, want ...observability.ReasonCode) bool { for _, reason := range reasons { - for _, target := range want { - if reason == target { - return true - } + if slices.Contains(want, reason) { + return true } } return false diff --git a/internal/admin/api_object_version_restore_test.go b/internal/admin/api_object_version_restore_test.go index 544b6d5..e2be41d 100644 --- a/internal/admin/api_object_version_restore_test.go +++ b/internal/admin/api_object_version_restore_test.go @@ -168,13 +168,13 @@ func TestAPIObjectVersionRestoreMapsErrors(t *testing.T) { func TestAPIObjectVersionsReturnsCurrentVersionIDOnEveryPage(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "version-token-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "version-token-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - _, firstID := seedAdminObjectVersion(t, repos, bucket, "file.txt", 1, "etag-1", "checksum-1", "text/plain", "", model.ObjectStateCached) - _, secondID := seedAdminObjectVersion(t, repos, bucket, "file.txt", 2, "etag-2", "checksum-2", "text/plain", "", model.ObjectStateCached) - _, currentID := seedAdminObjectVersion(t, repos, bucket, "file.txt", 3, "etag-3", "checksum-3", "text/plain", "", model.ObjectStateCached) + _, firstID := seedAdminObjectVersion(t, srv.db, repos, bucket, "file.txt", 1, "etag-1", "checksum-1", "text/plain", model.ObjectStateCached) + _, secondID := seedAdminObjectVersion(t, srv.db, repos, bucket, "file.txt", 2, "etag-2", "checksum-2", "text/plain", model.ObjectStateCached) + _, currentID := seedAdminObjectVersion(t, srv.db, repos, bucket, "file.txt", 3, "etag-3", "checksum-3", "text/plain", model.ObjectStateCached) ts := httptest.NewServer(newBucketAPIMux(srv)) defer ts.Close() @@ -226,11 +226,11 @@ func TestAPIObjectVersionsReturnsCurrentVersionIDOnEveryPage(t *testing.T) { func TestAPIObjectVersionsFallsBackToCurrentVersionInListedPage(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() - bucket := &model.Bucket{Name: "version-token-race-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "version-token-race-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } - _, currentID := seedAdminObjectVersion(t, repos, bucket, "file.txt", 1, "etag", "checksum", "text/plain", "", model.ObjectStateCached) + _, currentID := seedAdminObjectVersion(t, srv.db, repos, bucket, "file.txt", 1, "etag", "checksum", "text/plain", model.ObjectStateCached) repos.Objects = missingCurrentObjectRepository{ObjectRepository: repos.Objects} req := httptest.NewRequest(http.MethodGet, "/api/v1/buckets/"+bucket.Name+"/objects/versions?key=file.txt", nil) req.SetPathValue("name", bucket.Name) diff --git a/internal/admin/api_overview.go b/internal/admin/api_overview.go index 9e5beca..dc88840 100644 --- a/internal/admin/api_overview.go +++ b/internal/admin/api_overview.go @@ -47,14 +47,13 @@ type objectAttentionOverview struct { } type taskAttentionOverview struct { - Failed int64 `json:"failed"` - Exhausted int64 `json:"exhausted"` + Failed int64 `json:"failed"` } type taskPipelineOverview struct { - Pipeline string `json:"pipeline"` - ByStatus map[string]int64 `json:"by_status"` - Total int64 `json:"total"` + Operation string `json:"operation"` + ByStatus map[string]int64 `json:"by_status"` + Total int64 `json:"total"` } type cacheOverview struct { @@ -129,12 +128,17 @@ func (s *Server) handleAPIOverview(w http.ResponseWriter, r *http.Request) { s.logger.Warn("overview: failed to count tasks", "error", err) } else { for _, tc := range taskCounts { + if model.TaskType(tc.Type).IsRecurringSystem() && tc.Status != string(model.TaskStatusFailed) { + continue + } resp.Tasks.ByStatus[tc.Status] += tc.Count } - resp.Tasks.Attention = taskAttentionOverview{ - Failed: resp.Tasks.ByStatus[string(model.TaskStatusFailed)], - Exhausted: resp.Tasks.ByStatus[string(model.TaskStatusExhausted)], - } + } + unacknowledgedFailed, err := s.repos.Tasks.CountUnacknowledgedFailed(ctx) + if err != nil { + s.logger.Warn("overview: failed to count task attention", "error", err) + } else { + resp.Tasks.Attention = taskAttentionOverview{Failed: unacknowledgedFailed} } taskPipelineCounts, err := s.repos.Tasks.CountOverviewActivePipeline(ctx) if err != nil { @@ -213,25 +217,20 @@ func (s *Server) filecoinStorageHealthOverview(ctx context.Context) filecoinStor } func taskPipelineOverviewRows(counts []repository.TaskPipelineCount) []taskPipelineOverview { - pipelines := []string{"prepare", "upload", "commit", "sync", "evict", "cleanup"} - rows := make([]taskPipelineOverview, 0, len(pipelines)) - index := make(map[string]int, len(pipelines)) - for _, pipeline := range pipelines { - index[pipeline] = len(rows) - rows = append(rows, taskPipelineOverview{ - Pipeline: pipeline, - ByStatus: map[string]int64{ - string(model.TaskStatusQueued): 0, - string(model.TaskStatusScheduled): 0, - string(model.TaskStatusWaiting): 0, - string(model.TaskStatusRunning): 0, - }, - }) - } + rows := make([]taskPipelineOverview, 0) + index := make(map[string]int) for _, count := range counts { i, ok := index[count.Pipeline] if !ok { - continue + i = len(rows) + index[count.Pipeline] = i + rows = append(rows, taskPipelineOverview{ + Operation: count.Pipeline, + ByStatus: map[string]int64{ + string(model.TaskStatusPending): 0, + string(model.TaskStatusRunning): 0, + }, + }) } rows[i].ByStatus[count.Status] += count.Count rows[i].Total += count.Count diff --git a/internal/admin/api_overview_test.go b/internal/admin/api_overview_test.go index 877b049..23c8d8c 100644 --- a/internal/admin/api_overview_test.go +++ b/internal/admin/api_overview_test.go @@ -13,126 +13,11 @@ import ( "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/observability" + taskengine "github.com/strahe/synaps3/internal/task" "github.com/strahe/synaps3/internal/testutil" "github.com/uptrace/bun" ) -func TestAPIOverviewIncludesAttentionAndActivePipeline(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := overviewSeedBucket(t, db, "overview-bucket") - - healthy := overviewObjectVersion(bucket.ID, "healthy.txt", "01J00000000000000000000B01") - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, healthy); err != nil { - t.Fatalf("seed healthy object: %v", err) - } - failed := overviewObjectVersion(bucket.ID, "failed.txt", "01J00000000000000000000B02") - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, failed); err != nil { - t.Fatalf("seed failed object: %v", err) - } - if err := repos.Objects.UpdateVersionStateToFailed(ctx, failed.VersionID, model.ObjectStateCached, "upload failed"); err != nil { - t.Fatalf("mark failed object: %v", err) - } - unavailable := overviewObjectVersion(bucket.ID, "unavailable.txt", "01J00000000000000000000B03") - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, unavailable); err != nil { - t.Fatalf("seed unavailable object: %v", err) - } - overviewMustExec(t, db, `INSERT INTO storage_uploads (bucket_id, source_version_id, content_size, checksum, status, requested_copies) VALUES (?, ?, ?, ?, ?, ?)`, - bucket.ID, unavailable.VersionID, unavailable.Size, unavailable.Checksum, model.StorageUploadStatusComplete, 3) - overviewMustExec(t, db, `UPDATE object_versions SET state = ?, storage_upload_id = (SELECT MAX(id) FROM storage_uploads), in_cache = FALSE WHERE version_id = ?`, - model.ObjectStateStored, unavailable.VersionID) - - overviewSeedTask(t, repos, model.TaskTypeUpload, "prepare_upload", model.TaskStatusQueued) - overviewSeedTask(t, repos, model.TaskTypeUpload, "ingress_store", model.TaskStatusRunning) - overviewSeedTask(t, repos, model.TaskTypeUpload, "peer_commit", model.TaskStatusWaiting) - overviewSeedTask(t, repos, model.TaskTypeEvictCache, "", model.TaskStatusScheduled) - overviewSeedTask(t, repos, model.TaskTypeStorageCleanup, "", model.TaskStatusRunning) - overviewSeedTask(t, repos, model.TaskTypeUpload, "ingress_store", model.TaskStatusCompleted) - overviewSeedTask(t, repos, model.TaskTypeUpload, "ingress_store", model.TaskStatusFailed) - overviewSeedTask(t, repos, model.TaskTypeEvictCache, "", model.TaskStatusExhausted) - - srv := newTestServer(":0", db, &stubCache{rootDir: t.TempDir(), usedByte: 42}, 100, repos, &stubWorkerHealth{health: map[string]bool{"uploader": true}}, nil, config.DefaultFilecoinCopies, testLogger()) - rr := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/api/v1/overview", nil) - srv.handleAPIOverview(rr, req) - if rr.Code != http.StatusOK { - t.Fatalf("status = %d, want 200, body=%s", rr.Code, rr.Body.String()) - } - - var body struct { - Objects struct { - ByState map[string]int64 `json:"by_state"` - TotalSizeBytes int64 `json:"total_size_bytes"` - Attention struct { - NeedsAttention int64 `json:"needs_attention"` - Unavailable int64 `json:"unavailable"` - } `json:"attention"` - } `json:"objects"` - Tasks struct { - ByStatus map[string]int64 `json:"by_status"` - Attention struct { - Failed int64 `json:"failed"` - Exhausted int64 `json:"exhausted"` - } `json:"attention"` - ActivePipeline []struct { - Pipeline string `json:"pipeline"` - ByStatus map[string]int64 `json:"by_status"` - Total int64 `json:"total"` - } `json:"active_pipeline"` - } `json:"tasks"` - } - if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { - t.Fatalf("decode overview: %v", err) - } - if body.Objects.ByState[string(model.ObjectStateCached)] == 0 { - t.Fatal("overview should keep legacy object state counts") - } - if body.Objects.TotalSizeBytes != 30 { - t.Fatalf("total_size_bytes = %d, want 30", body.Objects.TotalSizeBytes) - } - if body.Tasks.ByStatus[string(model.TaskStatusCompleted)] == 0 { - t.Fatal("overview should keep legacy task status counts") - } - if body.Objects.Attention.NeedsAttention != 1 { - t.Fatalf("needs_attention = %d, want 1", body.Objects.Attention.NeedsAttention) - } - if body.Objects.Attention.Unavailable != 1 { - t.Fatalf("unavailable = %d, want 1", body.Objects.Attention.Unavailable) - } - if body.Tasks.Attention.Failed != 1 || body.Tasks.Attention.Exhausted != 1 { - t.Fatalf("task attention = failed:%d exhausted:%d, want 1/1", body.Tasks.Attention.Failed, body.Tasks.Attention.Exhausted) - } - pipeline := make(map[string]struct { - total int64 - byStatus map[string]int64 - }) - for _, row := range body.Tasks.ActivePipeline { - pipeline[row.Pipeline] = struct { - total int64 - byStatus map[string]int64 - }{total: row.Total, byStatus: row.ByStatus} - } - if pipeline["prepare"].byStatus[string(model.TaskStatusQueued)] != 1 { - t.Fatalf("prepare queued = %d, want 1", pipeline["prepare"].byStatus[string(model.TaskStatusQueued)]) - } - if pipeline["upload"].byStatus[string(model.TaskStatusRunning)] != 1 { - t.Fatalf("upload running = %d, want 1", pipeline["upload"].byStatus[string(model.TaskStatusRunning)]) - } - if pipeline["sync"].byStatus[string(model.TaskStatusWaiting)] != 1 { - t.Fatalf("sync waiting = %d, want 1", pipeline["sync"].byStatus[string(model.TaskStatusWaiting)]) - } - if pipeline["evict"].byStatus[string(model.TaskStatusScheduled)] != 1 { - t.Fatalf("evict scheduled = %d, want 1", pipeline["evict"].byStatus[string(model.TaskStatusScheduled)]) - } - if pipeline["cleanup"].byStatus[string(model.TaskStatusRunning)] != 1 { - t.Fatalf("cleanup running = %d, want 1", pipeline["cleanup"].byStatus[string(model.TaskStatusRunning)]) - } - if pipeline["upload"].total != 1 { - t.Fatalf("upload total = %d, want only active tasks", pipeline["upload"].total) - } -} - func TestAPIOverviewFilecoinStorageHealthUsesObservabilitySummaries(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) @@ -264,10 +149,10 @@ func TestAPIOverviewFilecoinStorageHealthHandlesObservabilityQueryFailures(t *te func TestAPIOverviewFilecoinStorageHealthIgnoresTaskPressure(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) + taskService := newAdminTestTaskService(t, repos) checkedAt := time.Date(2026, 5, 18, 12, 0, 0, 0, time.UTC) - overviewSeedTask(t, repos, model.TaskTypeUpload, "ingress_store", model.TaskStatusRunning) - overviewSeedTask(t, repos, model.TaskTypeUpload, "ingress_store", model.TaskStatusFailed) - overviewSeedTask(t, repos, model.TaskTypeEvictCache, "", model.TaskStatusExhausted) + overviewSeedTask(t, taskService, repos, model.TaskTypeStorageStore, "running", model.TaskStatusRunning) + overviewSeedTask(t, taskService, repos, model.TaskTypeStorageStore, "failed", model.TaskStatusFailed) srv := newTestServer(":0", db, &stubCache{rootDir: t.TempDir()}, 100, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()). WithObservability(observability.NewService(observability.ServiceOptions{ Store: &observabilityStateStore{ @@ -313,56 +198,221 @@ func TestAPIOverviewDoesNotCallFilecoinReadiness(t *testing.T) { } } +func TestAPIOverviewIncludesAttentionAndActivePipeline(t *testing.T) { + db := testutil.NewTestDB(t) + repos := repository.NewRepositories(db) + taskService := newAdminTestTaskService(t, repos) + ctx := context.Background() + bucket := overviewSeedBucket(t, db, "overview-bucket") + + healthy := overviewObjectVersion(t, repos, bucket.ID, "healthy.txt", "01J00000000000000000000B01") + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, healthy); err != nil { + t.Fatalf("seed healthy object: %v", err) + } + // A version needs attention when its content's ingest failed, which is a + // fact about the copies rather than a state written on the version. + failed := overviewObjectVersion(t, repos, bucket.ID, "failed.txt", "01J00000000000000000000B02") + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, failed); err != nil { + t.Fatalf("seed failed object: %v", err) + } + overviewSeedFailedCopy(t, repos, bucket.ID, *failed.ContentID) + + // A version is unavailable when nothing can serve it: no cached bytes and + // no readable committed copy. + unavailable := overviewObjectVersion(t, repos, bucket.ID, "unavailable.txt", "01J00000000000000000000B03") + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, unavailable); err != nil { + t.Fatalf("seed unavailable object: %v", err) + } + if err := repos.Objects.ClearContentCachePresence(ctx, *unavailable.ContentID); err != nil { + t.Fatalf("clear unavailable cache presence: %v", err) + } + + overviewSeedTask(t, taskService, repos, model.TaskTypeStorageStore, "store-running", model.TaskStatusRunning) + overviewSeedTask(t, taskService, repos, model.TaskTypeStorageCleanup, "cleanup-running", model.TaskStatusRunning) + overviewSeedTask(t, taskService, repos, model.TaskTypeUploadPlan, "plan-completed", model.TaskStatusCompleted) + overviewSeedTask(t, taskService, repos, model.TaskTypeStorageStore, "store-failed", model.TaskStatusFailed) + dismissed := overviewSeedTask(t, taskService, repos, model.TaskTypeStoragePull, "pull-dismissed", model.TaskStatusFailed) + if err := repos.Tasks.AcknowledgeFailed(ctx, dismissed.ID, time.Hour); err != nil { + t.Fatalf("acknowledge failed task: %v", err) + } + overviewSeedTask(t, taskService, repos, model.TaskTypeUploadPlan, "plan-pending", model.TaskStatusPending) + overviewSeedTask(t, taskService, repos, model.TaskTypeStorageCommit, "commit-pending", model.TaskStatusPending) + overviewSeedTask(t, taskService, repos, model.TaskTypeCacheEvict, "evict-pending", model.TaskStatusPending) + + srv := newTestServer(":0", db, &stubCache{rootDir: t.TempDir(), usedByte: 42}, 100, repos, &stubWorkerHealth{health: map[string]bool{"uploader": true}}, nil, config.DefaultFilecoinCopies, testLogger()) + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/overview", nil) + srv.handleAPIOverview(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, want 200, body=%s", rr.Code, rr.Body.String()) + } + + var body struct { + Objects struct { + ByState map[string]int64 `json:"by_state"` + TotalSizeBytes int64 `json:"total_size_bytes"` + Attention struct { + NeedsAttention int64 `json:"needs_attention"` + Unavailable int64 `json:"unavailable"` + } `json:"attention"` + } `json:"objects"` + Tasks struct { + ByStatus map[string]int64 `json:"by_status"` + Attention struct { + Failed int64 `json:"failed"` + } `json:"attention"` + ActivePipeline []struct { + Operation string `json:"operation"` + ByStatus map[string]int64 `json:"by_status"` + Total int64 `json:"total"` + } `json:"active_pipeline"` + } `json:"tasks"` + } + if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { + t.Fatalf("decode overview: %v", err) + } + if body.Objects.ByState[string(model.ObjectStateCached)] == 0 { + t.Fatal("overview should report derived object state counts") + } + if body.Objects.TotalSizeBytes != 30 { + t.Fatalf("total_size_bytes = %d, want 30", body.Objects.TotalSizeBytes) + } + if body.Tasks.ByStatus[string(model.TaskStatusCompleted)] == 0 { + t.Fatal("overview should keep legacy task status counts") + } + if body.Objects.Attention.NeedsAttention != 1 { + t.Fatalf("needs_attention = %d, want 1", body.Objects.Attention.NeedsAttention) + } + if body.Objects.Attention.Unavailable != 1 { + t.Fatalf("unavailable = %d, want 1", body.Objects.Attention.Unavailable) + } + if body.Tasks.Attention.Failed != 1 { + t.Fatalf("task attention failed = %d, want 1", body.Tasks.Attention.Failed) + } + pipeline := make(map[string]struct { + total int64 + byStatus map[string]int64 + }) + for _, row := range body.Tasks.ActivePipeline { + pipeline[row.Operation] = struct { + total int64 + byStatus map[string]int64 + }{total: row.Total, byStatus: row.ByStatus} + } + if pipeline[string(model.TaskTypeUploadPlan)].byStatus[string(model.TaskStatusPending)] != 1 { + t.Fatalf("upload plan pending = %d, want 1", pipeline[string(model.TaskTypeUploadPlan)].byStatus[string(model.TaskStatusPending)]) + } + if pipeline[string(model.TaskTypeStorageStore)].byStatus[string(model.TaskStatusRunning)] != 1 { + t.Fatalf("storage store running = %d, want 1", pipeline[string(model.TaskTypeStorageStore)].byStatus[string(model.TaskStatusRunning)]) + } + if pipeline[string(model.TaskTypeStorageCommit)].byStatus[string(model.TaskStatusPending)] != 1 { + t.Fatalf("storage commit pending = %d, want 1", pipeline[string(model.TaskTypeStorageCommit)].byStatus[string(model.TaskStatusPending)]) + } + if pipeline[string(model.TaskTypeCacheEvict)].byStatus[string(model.TaskStatusPending)] != 1 { + t.Fatalf("cache evict pending = %d, want 1", pipeline[string(model.TaskTypeCacheEvict)].byStatus[string(model.TaskStatusPending)]) + } + if pipeline[string(model.TaskTypeStorageCleanup)].byStatus[string(model.TaskStatusRunning)] != 1 { + t.Fatalf("storage cleanup running = %d, want 1", pipeline[string(model.TaskTypeStorageCleanup)].byStatus[string(model.TaskStatusRunning)]) + } + if pipeline[string(model.TaskTypeStorageStore)].total != 1 { + t.Fatalf("storage store total = %d, want only active tasks", pipeline[string(model.TaskTypeStorageStore)].total) + } +} + +// overviewSeedFailedCopy binds one copy for a content and fails it, which is +// how a content's ingest failure is now expressed. +func overviewSeedFailedCopy(t *testing.T, repos *repository.Repositories, bucketID, contentID int64) { + t.Helper() + ctx := context.Background() + binding, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucketID, ProviderID: onChainID(t, "909"), CopyIndex: 0, CreatedByContentID: contentID, + }) + if err != nil { + t.Fatalf("seed failed copy binding: %v", err) + } + if err := repos.Contents.CreateUploadCopiesForBindings(ctx, contentID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: binding.ID, CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: binding.ProviderID, + }}); err != nil { + t.Fatalf("seed failed copy: %v", err) + } + if err := repos.Contents.RecordContentFailure(ctx, contentID, "ingest failed"); err != nil { + t.Fatalf("record content failure: %v", err) + } +} + func overviewSeedBucket(t *testing.T, db *bun.DB, name string) *model.Bucket { t.Helper() - bucket := &model.Bucket{Name: name, Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: name, Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if _, err := db.NewInsert().Model(bucket).Exec(context.Background()); err != nil { t.Fatalf("seed bucket: %v", err) } + testutil.OpenBucketReplicaSlots(t, db, bucket.ID, bucket.DefaultCopies) return bucket } -func overviewObjectVersion(bucketID int64, key, versionID string) *model.ObjectVersion { +// overviewObjectVersion seeds the content for one version and returns the +// version pointing at it. A data version cannot exist without its bytes. +func overviewObjectVersion(t *testing.T, repos *repository.Repositories, bucketID int64, key, versionID string) *model.ObjectVersion { + t.Helper() + content, err := repos.Contents.EnsureContent(context.Background(), repository.EnsureContentInput{ + BucketID: bucketID, + ContentSize: 10, + Checksum: testutil.StorageChecksum("checksum-" + versionID), + RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("seed overview content: %v", err) + } return &model.ObjectVersion{ VersionID: versionID, BucketID: bucketID, Key: key, + ContentID: &content.ID, Size: 10, ETag: "etag-" + versionID, - Checksum: "checksum-" + versionID, ContentType: "text/plain", - CacheKey: ".versions/" + versionID, - State: model.ObjectStateCached, } } -func overviewSeedTask(t *testing.T, repos *repository.Repositories, taskType model.TaskType, stage string, status model.TaskStatus) { +func overviewSeedTask( + t *testing.T, + service *taskengine.Service, + repos *repository.Repositories, + taskType model.TaskType, + key string, + status model.TaskStatus, +) *model.Task { t.Helper() - task := &model.Task{ - Type: taskType, - RefType: "object", - RefID: 1, - RefVersionID: "01J0000000000000000000TASK", - IdempotencyKey: string(taskType) + ":" + stage + ":" + string(status), - Status: status, + availableAt := time.Now() + if status == model.TaskStatusPending { + availableAt = availableAt.Add(time.Hour) + } + taskRow, _, err := service.Enqueue(t.Context(), taskengine.EnqueueRequest{ + Type: taskType, IdempotencyKey: key, Input: map[string]any{"key": key}, AvailableAt: availableAt, + }) + if err != nil { + t.Fatalf("Enqueue(%s): %v", key, err) } - if stage != "" { - task.Stage = &stage + if status == model.TaskStatusPending { + return taskRow } - if taskType == model.TaskTypeStorageCleanup { - task.RefType = "storage_upload" - task.RefVersionID = "" + claimed, err := repos.Tasks.ClaimNext(t.Context(), time.Minute) + if err != nil || claimed == nil || claimed.ID != taskRow.ID { + t.Fatalf("ClaimNext(%s) = %#v, err=%v", key, claimed, err) } - if err := repos.Tasks.Create(context.Background(), task); err != nil { - t.Fatalf("seed task: %v", err) + if status == model.TaskStatusRunning { + return taskRow } -} - -func overviewMustExec(t *testing.T, db *bun.DB, query string, args ...interface{}) { - t.Helper() - if _, err := db.NewRaw(query, args...).Exec(context.Background()); err != nil { - t.Fatalf("exec %q: %v", query, err) + transition := repository.TaskTransition{Status: status, ResumeMode: model.TaskResumeModeRecover} + if status == model.TaskStatusCompleted || status == model.TaskStatusCancelled { + retentionUntil := time.Now().Add(7 * 24 * time.Hour) + transition.RetentionUntil = &retentionUntil + } + if err := repos.Tasks.Settle(t.Context(), claimed.ID, claimed.ClaimGeneration, transition); err != nil { + t.Fatalf("Settle(%s): %v", key, err) } + return taskRow } func decodeOverviewResponse(t *testing.T, srv *Server) overviewResponse { diff --git a/internal/admin/api_replacement.go b/internal/admin/api_replacement.go index 1ad5aa3..1c119a4 100644 --- a/internal/admin/api_replacement.go +++ b/internal/admin/api_replacement.go @@ -12,6 +12,7 @@ import ( "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/storagereplacement" + taskengine "github.com/strahe/synaps3/internal/task" idtypes "github.com/strahe/synaps3/internal/types" ) @@ -54,19 +55,37 @@ type providerReplacementResponse struct { Source replacementDataSetResponse `json:"source"` Target replacementDataSetResponse `json:"target"` - ItemsTotal int `json:"items_total"` - ItemsCopied int `json:"items_copied"` - Progress *taskProgressResponse `json:"progress,omitempty"` - LastError *string `json:"last_error"` - TerminationEpoch *int64 `json:"termination_epoch"` + ItemsTotal int `json:"items_total"` + ItemsCopied int `json:"items_copied"` + Progress *replacementProgressResponse `json:"progress,omitempty"` + LastError *string `json:"last_error"` + TerminationEpoch *int64 `json:"termination_epoch"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` } +type replacementProgressResponse struct { + Scope string `json:"scope"` + Phase string `json:"phase"` + SeedingComplete bool `json:"seeding_complete"` + ItemsTotal int `json:"items_total"` + ItemsProcessed int `json:"items_processed"` + ItemsCopied int `json:"items_copied"` + ItemsNoLongerNeeded int `json:"items_no_longer_needed"` + ItemsPending int `json:"items_pending"` + ItemsActive int `json:"items_active"` + ItemsAttention int `json:"items_attention"` + ItemsRetrying int `json:"items_retrying"` + ItemsWaitingSource int `json:"items_waiting_source"` + ItemsFailed int `json:"items_failed"` + Percent *int `json:"percent,omitempty"` + NextRetryAt *string `json:"next_retry_at,omitempty"` +} + type replacementDataSetResponse struct { ID int64 `json:"id"` - Generation int `json:"generation"` + Generation int64 `json:"generation"` IsCurrent bool `json:"is_current"` Status string `json:"status"` ProviderID string `json:"provider_id"` @@ -135,13 +154,35 @@ func (s *Server) handleAPIStartDataSetReplacement(w http.ResponseWriter, r *http return } - row, created, err := s.repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ - BucketID: bucket.ID, - SourceDataSetID: source.ID, - SelectionMode: mode, - TargetProviderID: targetProvider, - ClientRequestID: req.ClientRequestID, - MaxRetries: s.uploadMaxRetries, + if s.taskService == nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "task service unavailable"}) + return + } + var row *storagereplacement.Replacement + created := false + err = s.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { + var authorizeErr error + row, created, authorizeErr = txRepos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, SelectionMode: mode, + TargetProviderID: targetProvider, ClientRequestID: req.ClientRequestID, + }) + if authorizeErr != nil || !created { + return authorizeErr + } + taskRow, _, enqueueErr := s.taskService.EnqueueInTransaction(ctx, txRepos, taskengine.EnqueueRequest{ + Type: model.TaskTypeProviderReplacementCoordinate, + IdempotencyKey: storagereplacement.CoordinateTaskKey(row.ID, row.TaskGeneration), + Input: storagereplacement.CoordinateInput{ReplacementID: row.ID, Generation: row.TaskGeneration}, + SubjectType: "storage_replacement", SubjectKey: strconv.FormatInt(row.ID, 10), + }) + if enqueueErr != nil { + return enqueueErr + } + if bindErr := txRepos.Replacements.BindTask(ctx, row.ID, row.TaskGeneration, taskRow.ID); bindErr != nil { + return bindErr + } + row.TaskID = &taskRow.ID + return nil }) if err != nil { s.writeReplacementError(w, err, name) @@ -246,7 +287,7 @@ func (s *Server) replacementProviderCandidates( if err != nil { return nil, err } - bindings, err := s.repos.Uploads.ListDataSetBindings(ctx, bucket.ID) + bindings, err := s.repos.Contents.ListDataSetBindings(ctx, bucket.ID) if err != nil { return nil, err } @@ -265,10 +306,31 @@ func (s *Server) handleAPIRetryStorageReplacement(w http.ResponseWriter, r *http return } ctx := r.Context() - row, err := s.repos.Replacements.Retry(ctx, repository.RetryReplacementInput{ - ReplacementID: id, - MaxRetries: s.uploadMaxRetries, - ItemMaxRetries: s.providerReplacementMaxRetries, + if s.taskService == nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "task service unavailable"}) + return + } + var row *storagereplacement.Replacement + err = s.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { + var retryErr error + row, retryErr = txRepos.Replacements.Retry(ctx, repository.RetryReplacementInput{ReplacementID: id}) + if retryErr != nil { + return retryErr + } + taskRow, _, enqueueErr := s.taskService.EnqueueInTransaction(ctx, txRepos, taskengine.EnqueueRequest{ + Type: model.TaskTypeProviderReplacementCoordinate, + IdempotencyKey: storagereplacement.CoordinateTaskKey(row.ID, row.TaskGeneration), + Input: storagereplacement.CoordinateInput{ReplacementID: row.ID, Generation: row.TaskGeneration}, + SubjectType: "storage_replacement", SubjectKey: strconv.FormatInt(row.ID, 10), + }) + if enqueueErr != nil { + return enqueueErr + } + if bindErr := txRepos.Replacements.BindTask(ctx, row.ID, row.TaskGeneration, taskRow.ID); bindErr != nil { + return bindErr + } + row.TaskID = &taskRow.ID + return nil }) if err != nil { s.writeReplacementError(w, err, "") @@ -374,31 +436,43 @@ func (s *Server) providerReplacementResponseWithProgress( row *storagereplacement.Replacement, progress storagereplacement.ProgressSnapshot, ) (providerReplacementResponse, error) { - source, err := s.repos.Uploads.GetDataSetBindingByID(ctx, row.SourceDataSetID) + source, err := s.repos.Contents.GetDataSetBindingByID(ctx, row.SourceDataSetID) if err != nil { return providerReplacementResponse{}, err } - target, err := s.repos.Uploads.GetDataSetBindingByID(ctx, row.TargetDataSetID) + target, err := s.repos.Contents.GetDataSetBindingByID(ctx, row.TargetDataSetID) if err != nil { return providerReplacementResponse{}, err } identities := s.providerIdentities(replacementProviderIDs(source, target)) response := providerReplacementResponse{ - ID: row.ID, - BucketName: bucketName, - CopyIndex: row.CopyIndex, - Status: string(row.Status), - SelectionMode: string(row.SelectionMode), - Source: replacementDataSetView(source, identities), - Target: replacementDataSetView(target, identities), - ItemsTotal: row.ItemsTotal, - ItemsCopied: row.ItemsCopied, - Progress: taskProgressFromReplacement(progress), + ID: row.ID, + BucketName: bucketName, + CopyIndex: row.CopyIndex, + Status: string(row.Status), + SelectionMode: string(row.SelectionMode), + Source: replacementDataSetView(source, identities), + Target: replacementDataSetView(target, identities), + ItemsTotal: row.ItemsTotal, + ItemsCopied: row.ItemsCopied, + Progress: &replacementProgressResponse{ + Scope: "provider_replacement", Phase: string(progress.Phase), SeedingComplete: progress.SeedingComplete, + ItemsTotal: progress.ItemsTotal, ItemsProcessed: progress.ItemsProcessed, + ItemsCopied: progress.ItemsCopied, ItemsNoLongerNeeded: progress.ItemsNoLongerNeeded, + ItemsPending: progress.ItemsPending, ItemsActive: progress.ItemsActive, + ItemsAttention: progress.ItemsAttention, ItemsRetrying: progress.ItemsRetrying, + ItemsWaitingSource: progress.ItemsWaitingSource, ItemsFailed: progress.ItemsFailed, + Percent: progress.Percent, + }, LastError: row.LastError, TerminationEpoch: row.TerminationEpoch, CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt, } + if progress.NextRetryAt != nil { + value := progress.NextRetryAt.Format(time.RFC3339) + response.Progress.NextRetryAt = &value + } if row.WaitReason != nil { response.WaitReason = string(*row.WaitReason) response.WaitMessage = row.WaitReason.Message() @@ -489,7 +563,7 @@ func (s *Server) replacementSubject( writeJSON(w, http.StatusNotFound, map[string]string{"error": "bucket not found"}) return nil, nil, false } - source, err := s.repos.Uploads.GetDataSetBindingByID(ctx, dataSetID) + source, err := s.repos.Contents.GetDataSetBindingByID(ctx, dataSetID) if err != nil { s.logger.Error("api: failed to load data set for replacement", "error", err, "dataSetID", dataSetID) writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) diff --git a/internal/admin/api_replacement_test.go b/internal/admin/api_replacement_test.go index 39c8f16..d9a0084 100644 --- a/internal/admin/api_replacement_test.go +++ b/internal/admin/api_replacement_test.go @@ -3,7 +3,6 @@ package admin import ( "context" "encoding/json" - "errors" "net/http" "net/http/httptest" "strconv" @@ -57,11 +56,11 @@ func newReplacementAPIFixture(t *testing.T, selector providerReplacementSelector srv.WithProviderReplacement(selector) } ctx := context.Background() - bucket := &model.Bucket{Name: "replacement-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "replacement-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := srv.repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Create bucket: %v", err) } - binding, err := srv.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + binding, err := srv.repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ BucketID: bucket.ID, ProviderID: onChainIDValue("101"), CopyIndex: 0, @@ -69,13 +68,13 @@ func newReplacementAPIFixture(t *testing.T, selector providerReplacementSelector if err != nil { t.Fatalf("EnsureDataSetBinding: %v", err) } - if err := srv.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + if err := srv.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ ID: binding.ID, DataSetID: onChainIDValue("1001"), }); err != nil { t.Fatalf("MarkDataSetReady: %v", err) } - source, err := srv.repos.Uploads.GetDataSetBindingByID(ctx, binding.ID) + source, err := srv.repos.Contents.GetDataSetBindingByID(ctx, binding.ID) if err != nil || source == nil { t.Fatalf("GetDataSetBindingByID: %#v err=%v", source, err) } @@ -131,6 +130,7 @@ func TestAPIStartDataSetReplacementManualMode(t *testing.T) { if rec.Code != http.StatusCreated { t.Fatalf("status = %d body=%s, want 201", rec.Code, rec.Body.String()) } + rawBody := rec.Body.String() body := decodeReplacement(t, rec) if body.Status != string(storagereplacement.StatusPreparingTarget) { t.Fatalf("status = %s, want preparing_target", body.Status) @@ -145,9 +145,16 @@ func TestAPIStartDataSetReplacementManualMode(t *testing.T) { if body.ItemsTotal != 0 || body.ItemsCopied != 0 { t.Fatalf("progress = %d/%d, want no migration work yet", body.ItemsCopied, body.ItemsTotal) } - if body.Progress == nil || body.Progress.Scope != "provider_replacement" || body.Progress.SeedingComplete || body.Progress.Percent != nil { + if body.Progress == nil || body.Progress.Phase != string(storagereplacement.PhasePrepare) || body.Progress.SeedingComplete || body.Progress.Percent != nil { t.Fatalf("structured progress = %#v, want indeterminate provider replacement progress", body.Progress) } + if body.Progress.Scope != "provider_replacement" || + !strings.Contains(rawBody, `"items_active":0`) || + !strings.Contains(rawBody, `"items_retrying":0`) || + !strings.Contains(rawBody, `"items_waiting_source":0`) || + !strings.Contains(rawBody, `"items_failed":0`) { + t.Fatalf("replacement progress wire contract = %s", rawBody) + } } // Automatic selection must exclude every provider the bucket has ever used. @@ -177,7 +184,7 @@ func TestAPIStartDataSetReplacementIsIdempotent(t *testing.T) { } firstReplacement := decodeReplacement(t, first) ctx := context.Background() - if err := fixture.srv.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + if err := fixture.srv.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ ID: firstReplacement.Target.ID, DataSetID: onChainIDValue("2002"), }); err != nil { t.Fatalf("MarkDataSetReady: %v", err) @@ -364,55 +371,6 @@ func TestAPIRetryStorageReplacementRejectsPermanentTargetConflict(t *testing.T) } } -// Replacement work carries state the task queue knows nothing about, so the -// generic retry must refuse it and point the operator at the Data Sets surface. -func TestRetryExhaustedRejectsReplacementCoordinator(t *testing.T) { - srv, _ := newBucketAPITestServer(t) - ctx := context.Background() - stage := storagereplacement.StageMigrate - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "bucket", - RefID: 1, - IdempotencyKey: storagereplacement.MigrateTaskKey(42), - Payload: storagereplacement.NewMigratePayload(42), - Status: model.TaskStatusQueued, - MaxRetries: 1, - } - if err := srv.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task: %v", err) - } - if _, err := srv.db.NewUpdate(). - Model((*model.Task)(nil)). - Set("status = ?", model.TaskStatusExhausted). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - t.Fatalf("mark exhausted: %v", err) - } - - err := srv.repos.Tasks.RetryExhausted(ctx, task.ID) - if !errors.Is(err, repository.ErrReplacementRetryUnsupported) { - t.Fatalf("RetryExhausted = %v, want the replacement rejection", err) - } - - mux := http.NewServeMux() - mux.HandleFunc("POST /api/v1/tasks/{id}/retry", srv.handleRetryExhausted) - req := httptest.NewRequest(http.MethodPost, "/api/v1/tasks/"+strconv.FormatInt(task.ID, 10)+"/retry", nil) - rec := httptest.NewRecorder() - mux.ServeHTTP(rec, req) - if rec.Code != http.StatusConflict { - t.Fatalf("status = %d body=%s, want 409", rec.Code, rec.Body.String()) - } - body := decodeAPIError(t, rec) - if body["code"] != storagereplacement.CodeTaskRetryUnsupported { - t.Fatalf("code = %q, want %q", body["code"], storagereplacement.CodeTaskRetryUnsupported) - } - if !strings.Contains(body["error"], "Data Sets") { - t.Fatalf("error = %q, want it to point at the Data Sets surface", body["error"]) - } -} - func (f *replacementAPIFixture) listProviders(t *testing.T) *httptest.ResponseRecorder { t.Helper() path := "/api/v1/buckets/" + f.bucket.Name + "/data-sets/" + diff --git a/internal/admin/api_s3_users_test.go b/internal/admin/api_s3_users_test.go index 1398771..f9a97a3 100644 --- a/internal/admin/api_s3_users_test.go +++ b/internal/admin/api_s3_users_test.go @@ -199,7 +199,7 @@ func TestS3UsersListIncludesOwnedBucketCount(t *testing.T) { {name: "owner-b-one", owner: "owner-b"}, {name: "unassigned"}, } { - bucket := &model.Bucket{Name: seed.name, Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: seed.name, Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if seed.owner != "" { data, err := json.Marshal(auth.ACL{Owner: seed.owner}) if err != nil { @@ -282,7 +282,7 @@ func TestS3UsersUpdateIncludesOwnedBucketCount(t *testing.T) { t.Fatalf("Marshal ACL: %v", err) } owner := "owner-access" - if err := repos.Buckets.Create(ctx, &model.Bucket{Name: name, Status: model.BucketStatusActive, OwnerAccessKey: &owner, ACL: data}); err != nil { + if err := repos.Buckets.Create(ctx, &model.Bucket{Name: name, Status: model.BucketStatusActive, OwnerAccessKey: &owner, ACL: data, DefaultCopies: 1, MinimumDurableCopies: 1}); err != nil { t.Fatalf("Buckets.Create(%s): %v", name, err) } } @@ -315,7 +315,7 @@ func TestS3UsersDeleteOwnedUserReturnsConflict(t *testing.T) { t.Fatalf("Marshal ACL: %v", err) } owner := "owner-access" - if err := repos.Buckets.Create(context.Background(), &model.Bucket{Name: "owned-bucket", Status: model.BucketStatusActive, OwnerAccessKey: &owner, ACL: data}); err != nil { + if err := repos.Buckets.Create(context.Background(), &model.Bucket{Name: "owned-bucket", Status: model.BucketStatusActive, OwnerAccessKey: &owner, ACL: data, DefaultCopies: 1, MinimumDurableCopies: 1}); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -360,6 +360,7 @@ func TestS3UsersDeleteSucceedsAfterBucketsTransferredToRoot(t *testing.T) { Status: model.BucketStatusActive, OwnerAccessKey: &owner, ACL: acl, + DefaultCopies: 8, MinimumDurableCopies: 8, }); err != nil { t.Fatalf("Buckets.Create: %v", err) } diff --git a/internal/admin/api_settings.go b/internal/admin/api_settings.go index 8e1449c..c22fa8a 100644 --- a/internal/admin/api_settings.go +++ b/internal/admin/api_settings.go @@ -175,10 +175,7 @@ func (s *SettingsService) settingsDraft(req settingsUpdateRequest) (*config.Conf setInt("cache.lru_low_watermark_percent", &next.Cache.LRULowWatermarkPercent, req.Cache.LRULowWatermarkPercent) } if req.Worker != nil { - applyWorkerPoolUpdate(req.Worker.Upload, &next.Worker.Upload, "worker.upload", setInt, setDuration) - applyWorkerPoolUpdate(req.Worker.ProviderReplacement, &next.Worker.ProviderReplacement, "worker.provider_replacement", setInt, setDuration) - applyWorkerPoolUpdate(req.Worker.Evictor, &next.Worker.Evictor, "worker.evictor", setInt, setDuration) - applyWorkerPoolUpdate(req.Worker.StorageCleanup, &next.Worker.StorageCleanup, "worker.storage_cleanup", setInt, setDuration) + applyTaskWorkerUpdate(req.Worker.Tasks, &next.Worker.Tasks, setInt, setDuration) } if req.Logging != nil { setString("logging.level", &next.Logging.Level, req.Logging.Level) @@ -268,19 +265,22 @@ func (s *SettingsService) FilecoinDraftConfig(req *settingsFilecoinUpdate) (*con return next, nil } -func applyWorkerPoolUpdate( - req *settingsWorkerPoolUpdate, - target *config.WorkerPoolConfig, - prefix string, +func applyTaskWorkerUpdate( + req *settingsTaskWorkerUpdate, + target *config.TaskWorkerConfig, setInt func(string, *int, *int), setDuration func(string, *time.Duration, *string), ) { if req == nil { return } - setInt(prefix+".concurrency", &target.Concurrency, req.Concurrency) - setDuration(prefix+".poll_interval", &target.PollInterval, req.PollInterval) - setInt(prefix+".max_retries", &target.MaxRetries, req.MaxRetries) + setInt("worker.tasks.concurrency", &target.Concurrency, req.Concurrency) + setDuration("worker.tasks.poll_interval", &target.PollInterval, req.PollInterval) + setDuration("worker.tasks.lease_duration", &target.LeaseDuration, req.LeaseDuration) + setInt("worker.tasks.max_retries", &target.MaxRetries, req.MaxRetries) + setDuration("worker.tasks.retention", &target.Retention, req.Retention) + setInt("worker.tasks.provider_mutation_concurrency", &target.ProviderMutationConcurrency, req.ProviderMutationConcurrency) + setInt("worker.tasks.destructive_mutation_concurrency", &target.DestructiveMutationConcurrency, req.DestructiveMutationConcurrency) } func (s *SettingsService) snapshotLocked(writable bool) settingsResponse { @@ -402,16 +402,17 @@ type settingsCacheConfig struct { } type settingsWorkerConfig struct { - Upload settingsWorkerPoolConfig `json:"upload"` - ProviderReplacement settingsWorkerPoolConfig `json:"provider_replacement"` - Evictor settingsWorkerPoolConfig `json:"evictor"` - StorageCleanup settingsWorkerPoolConfig `json:"storage_cleanup"` + Tasks settingsTaskWorkerConfig `json:"tasks"` } -type settingsWorkerPoolConfig struct { - Concurrency int `json:"concurrency"` - PollInterval string `json:"poll_interval"` - MaxRetries int `json:"max_retries"` +type settingsTaskWorkerConfig struct { + Concurrency int `json:"concurrency"` + PollInterval string `json:"poll_interval"` + LeaseDuration string `json:"lease_duration"` + MaxRetries int `json:"max_retries"` + Retention string `json:"retention"` + ProviderMutationConcurrency int `json:"provider_mutation_concurrency"` + DestructiveMutationConcurrency int `json:"destructive_mutation_concurrency"` } type settingsLoggingConfig struct { @@ -509,16 +510,17 @@ type settingsCacheUpdate struct { } type settingsWorkerUpdate struct { - Upload *settingsWorkerPoolUpdate `json:"upload,omitempty"` - ProviderReplacement *settingsWorkerPoolUpdate `json:"provider_replacement,omitempty"` - Evictor *settingsWorkerPoolUpdate `json:"evictor,omitempty"` - StorageCleanup *settingsWorkerPoolUpdate `json:"storage_cleanup,omitempty"` + Tasks *settingsTaskWorkerUpdate `json:"tasks,omitempty"` } -type settingsWorkerPoolUpdate struct { - Concurrency *int `json:"concurrency,omitempty"` - PollInterval *string `json:"poll_interval,omitempty"` - MaxRetries *int `json:"max_retries,omitempty"` +type settingsTaskWorkerUpdate struct { + Concurrency *int `json:"concurrency,omitempty"` + PollInterval *string `json:"poll_interval,omitempty"` + LeaseDuration *string `json:"lease_duration,omitempty"` + MaxRetries *int `json:"max_retries,omitempty"` + Retention *string `json:"retention,omitempty"` + ProviderMutationConcurrency *int `json:"provider_mutation_concurrency,omitempty"` + DestructiveMutationConcurrency *int `json:"destructive_mutation_concurrency,omitempty"` } type settingsLoggingUpdate struct { @@ -567,10 +569,7 @@ func toSettingsEditableConfig(cfg *config.Config) settingsEditableConfig { LRULowWatermarkPercent: cfg.Cache.LRULowWatermarkPercent, }, Worker: settingsWorkerConfig{ - Upload: toSettingsWorkerPoolConfig(cfg.Worker.Upload), - ProviderReplacement: toSettingsWorkerPoolConfig(cfg.Worker.ProviderReplacement), - Evictor: toSettingsWorkerPoolConfig(cfg.Worker.Evictor), - StorageCleanup: toSettingsWorkerPoolConfig(cfg.Worker.StorageCleanup), + Tasks: toSettingsTaskWorkerConfig(cfg.Worker.Tasks), }, Logging: settingsLoggingConfig{ Level: cfg.Logging.Level, @@ -583,11 +582,15 @@ func toSettingsEditableConfig(cfg *config.Config) settingsEditableConfig { } } -func toSettingsWorkerPoolConfig(cfg config.WorkerPoolConfig) settingsWorkerPoolConfig { - return settingsWorkerPoolConfig{ - Concurrency: cfg.Concurrency, - PollInterval: cfg.PollInterval.String(), - MaxRetries: cfg.MaxRetries, +func toSettingsTaskWorkerConfig(cfg config.TaskWorkerConfig) settingsTaskWorkerConfig { + return settingsTaskWorkerConfig{ + Concurrency: cfg.Concurrency, + PollInterval: cfg.PollInterval.String(), + LeaseDuration: cfg.LeaseDuration.String(), + MaxRetries: cfg.MaxRetries, + Retention: cfg.Retention.String(), + ProviderMutationConcurrency: cfg.ProviderMutationConcurrency, + DestructiveMutationConcurrency: cfg.DestructiveMutationConcurrency, } } @@ -619,39 +622,34 @@ func toSettingsSecretStatus(cfg *config.Config) settingsSecretStatus { func editableValidationErrors(cfg *config.Config) []config.FieldError { editable := map[string]struct{}{ - "server.port": {}, - "server.max_connections": {}, - "server.max_requests": {}, - "server.tls.cert_file": {}, - "server.tls.key_file": {}, - "s3.region": {}, - "cache.dir": {}, - "cache.max_size_gb": {}, - "cache.eviction_policy": {}, - "cache.lru_high_watermark_percent": {}, - "cache.lru_low_watermark_percent": {}, - "filecoin.network": {}, - "filecoin.rpc_url": {}, - "filecoin.default_copies": {}, - "filecoin.observability.interval": {}, - "filecoin.observability.timeout": {}, - "filecoin.observability.concurrency": {}, - "worker.upload.concurrency": {}, - "worker.upload.poll_interval": {}, - "worker.upload.max_retries": {}, - "worker.provider_replacement.concurrency": {}, - "worker.provider_replacement.poll_interval": {}, - "worker.provider_replacement.max_retries": {}, - "worker.evictor.concurrency": {}, - "worker.evictor.poll_interval": {}, - "worker.evictor.max_retries": {}, - "worker.storage_cleanup.concurrency": {}, - "worker.storage_cleanup.poll_interval": {}, - "worker.storage_cleanup.max_retries": {}, - "logging.level": {}, - "logging.format": {}, - "logging.s3_access.enabled": {}, - "logging.s3_access.level": {}, + "server.port": {}, + "server.max_connections": {}, + "server.max_requests": {}, + "server.tls.cert_file": {}, + "server.tls.key_file": {}, + "s3.region": {}, + "cache.dir": {}, + "cache.max_size_gb": {}, + "cache.eviction_policy": {}, + "cache.lru_high_watermark_percent": {}, + "cache.lru_low_watermark_percent": {}, + "filecoin.network": {}, + "filecoin.rpc_url": {}, + "filecoin.default_copies": {}, + "filecoin.observability.interval": {}, + "filecoin.observability.timeout": {}, + "filecoin.observability.concurrency": {}, + "worker.tasks.concurrency": {}, + "worker.tasks.poll_interval": {}, + "worker.tasks.lease_duration": {}, + "worker.tasks.max_retries": {}, + "worker.tasks.retention": {}, + "worker.tasks.provider_mutation_concurrency": {}, + "worker.tasks.destructive_mutation_concurrency": {}, + "logging.level": {}, + "logging.format": {}, + "logging.s3_access.enabled": {}, + "logging.s3_access.level": {}, } var out []config.FieldError diff --git a/internal/admin/api_settings_test.go b/internal/admin/api_settings_test.go index ce3c33d..9c20954 100644 --- a/internal/admin/api_settings_test.go +++ b/internal/admin/api_settings_test.go @@ -342,8 +342,8 @@ func TestSettingsValidateReportsDraftValidationRules(t *testing.T) { }, { name: "duration parse", - payload: `{"worker":{"upload":{"poll_interval":"not-a-duration"}}}`, - field: "worker.upload.poll_interval", + payload: `{"worker":{"tasks":{"poll_interval":"not-a-duration"}}}`, + field: "worker.tasks.poll_interval", }, { name: "env managed field", @@ -448,7 +448,7 @@ func TestSettingsPUTPersistsNonSecretFieldsAndReturnsRestartRequired(t *testing. "lru_high_watermark_percent":86, "lru_low_watermark_percent":71 }, - "worker":{"upload":{"poll_interval":"9s"}}, + "worker":{"tasks":{"poll_interval":"9s"}}, "logging":{"format":"text","s3_access":{"enabled":false,"level":"debug"}} }`)) req.Header.Set("Content-Type", "application/json") @@ -482,8 +482,8 @@ func TestSettingsPUTPersistsNonSecretFieldsAndReturnsRestartRequired(t *testing. if loaded.Server.Port != ":8088" { t.Fatalf("saved server.port = %q, want :8088", loaded.Server.Port) } - if loaded.Worker.Upload.PollInterval.String() != "9s" { - t.Fatalf("saved worker.upload.poll_interval = %s, want 9s", loaded.Worker.Upload.PollInterval) + if loaded.Worker.Tasks.PollInterval.String() != "9s" { + t.Fatalf("saved worker.tasks.poll_interval = %s, want 9s", loaded.Worker.Tasks.PollInterval) } if loaded.Filecoin.DefaultCopies != 3 { t.Fatalf("saved filecoin.default_copies = %d, want 3", loaded.Filecoin.DefaultCopies) @@ -746,15 +746,13 @@ func TestSettingsPUTRejectsEnvManagedFieldChanges(t *testing.T) { {name: "cache eviction policy", envName: "SYNAPS3_CACHE_EVICTION_POLICY", payload: `{"cache":{"eviction_policy":"after_upload"}}`, field: "cache.eviction_policy"}, {name: "cache LRU high watermark", envName: "SYNAPS3_CACHE_LRU_HIGH_WATERMARK_PERCENT", payload: `{"cache":{"lru_high_watermark_percent":85}}`, field: "cache.lru_high_watermark_percent"}, {name: "cache LRU low watermark", envName: "SYNAPS3_CACHE_LRU_LOW_WATERMARK_PERCENT", payload: `{"cache":{"lru_low_watermark_percent":70}}`, field: "cache.lru_low_watermark_percent"}, - {name: "upload concurrency", envName: "SYNAPS3_WORKER_UPLOAD_CONCURRENCY", payload: `{"worker":{"upload":{"concurrency":2}}}`, field: "worker.upload.concurrency"}, - {name: "upload poll interval", envName: "SYNAPS3_WORKER_UPLOAD_POLL_INTERVAL", payload: `{"worker":{"upload":{"poll_interval":"9s"}}}`, field: "worker.upload.poll_interval"}, - {name: "upload max retries", envName: "SYNAPS3_WORKER_UPLOAD_MAX_RETRIES", payload: `{"worker":{"upload":{"max_retries":9}}}`, field: "worker.upload.max_retries"}, - {name: "provider replacement concurrency", envName: "SYNAPS3_WORKER_PROVIDER_REPLACEMENT_CONCURRENCY", payload: `{"worker":{"provider_replacement":{"concurrency":2}}}`, field: "worker.provider_replacement.concurrency"}, - {name: "provider replacement poll interval", envName: "SYNAPS3_WORKER_PROVIDER_REPLACEMENT_POLL_INTERVAL", payload: `{"worker":{"provider_replacement":{"poll_interval":"9s"}}}`, field: "worker.provider_replacement.poll_interval"}, - {name: "provider replacement max retries", envName: "SYNAPS3_WORKER_PROVIDER_REPLACEMENT_MAX_RETRIES", payload: `{"worker":{"provider_replacement":{"max_retries":9}}}`, field: "worker.provider_replacement.max_retries"}, - {name: "evictor concurrency", envName: "SYNAPS3_WORKER_EVICTOR_CONCURRENCY", payload: `{"worker":{"evictor":{"concurrency":2}}}`, field: "worker.evictor.concurrency"}, - {name: "evictor poll interval", envName: "SYNAPS3_WORKER_EVICTOR_POLL_INTERVAL", payload: `{"worker":{"evictor":{"poll_interval":"2m"}}}`, field: "worker.evictor.poll_interval"}, - {name: "evictor max retries", envName: "SYNAPS3_WORKER_EVICTOR_MAX_RETRIES", payload: `{"worker":{"evictor":{"max_retries":4}}}`, field: "worker.evictor.max_retries"}, + {name: "task concurrency", envName: "SYNAPS3_WORKER_TASKS_CONCURRENCY", payload: `{"worker":{"tasks":{"concurrency":2}}}`, field: "worker.tasks.concurrency"}, + {name: "task poll interval", envName: "SYNAPS3_WORKER_TASKS_POLL_INTERVAL", payload: `{"worker":{"tasks":{"poll_interval":"9s"}}}`, field: "worker.tasks.poll_interval"}, + {name: "task lease duration", envName: "SYNAPS3_WORKER_TASKS_LEASE_DURATION", payload: `{"worker":{"tasks":{"lease_duration":"9m"}}}`, field: "worker.tasks.lease_duration"}, + {name: "task max retries", envName: "SYNAPS3_WORKER_TASKS_MAX_RETRIES", payload: `{"worker":{"tasks":{"max_retries":9}}}`, field: "worker.tasks.max_retries"}, + {name: "task retention", envName: "SYNAPS3_WORKER_TASKS_RETENTION", payload: `{"worker":{"tasks":{"retention":"72h"}}}`, field: "worker.tasks.retention"}, + {name: "provider mutation concurrency", envName: "SYNAPS3_WORKER_TASKS_PROVIDER_MUTATION_CONCURRENCY", payload: `{"worker":{"tasks":{"provider_mutation_concurrency":2}}}`, field: "worker.tasks.provider_mutation_concurrency"}, + {name: "destructive mutation concurrency", envName: "SYNAPS3_WORKER_TASKS_DESTRUCTIVE_MUTATION_CONCURRENCY", payload: `{"worker":{"tasks":{"destructive_mutation_concurrency":2}}}`, field: "worker.tasks.destructive_mutation_concurrency"}, {name: "logging level", envName: "SYNAPS3_LOGGING_LEVEL", payload: `{"logging":{"level":"debug"}}`, field: "logging.level"}, {name: "logging format", envName: "SYNAPS3_LOGGING_FORMAT", payload: `{"logging":{"format":"text"}}`, field: "logging.format"}, {name: "s3 access logging enabled", envName: "SYNAPS3_LOGGING_S3_ACCESS_ENABLED", payload: `{"logging":{"s3_access":{"enabled":false}}}`, field: "logging.s3_access.enabled", envValue: "true"}, @@ -799,7 +797,7 @@ func TestSettingsPUTRejectsInvalidEditableFields(t *testing.T) { "server":{"port":"not-a-port"}, "s3":{"region":""}, "filecoin":{"rpc_url":"ftp://example.invalid/rpc","default_copies":0}, - "worker":{"upload":{"max_retries":-1}}, + "worker":{"tasks":{"max_retries":-1}}, "logging":{"level":"verbose","format":"xml","s3_access":{"level":"verbose"}} }`)) req.Header.Set("Content-Type", "application/json") @@ -816,7 +814,7 @@ func TestSettingsPUTRejectsInvalidEditableFields(t *testing.T) { "s3.region", "filecoin.rpc_url", "filecoin.default_copies", - "worker.upload.max_retries", + "worker.tasks.max_retries", "logging.level", "logging.format", "logging.s3_access.level", diff --git a/internal/admin/api_storage_confirmations.go b/internal/admin/api_storage_confirmations.go index bd55ece..d02a79f 100644 --- a/internal/admin/api_storage_confirmations.go +++ b/internal/admin/api_storage_confirmations.go @@ -12,7 +12,7 @@ import ( type storageConfirmationAttentionResponse struct { CopyID int64 `json:"copy_id"` - UploadID int64 `json:"upload_id"` + ContentID int64 `json:"content_id"` CopyIndex int `json:"copy_index"` DataSetRowID int64 `json:"data_set_row_id"` ProviderID string `json:"provider_id"` @@ -44,7 +44,7 @@ func (s *Server) handleAPIListStorageConfirmations(w http.ResponseWriter, r *htt } limit = parsed } - records, err := s.repos.Uploads.ListCommitAttention(r.Context(), limit) + records, err := s.repos.Contents.ListCommitAttention(r.Context(), limit) if err != nil { s.logger.Error("api: failed to list storage confirmations", "error", err) writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) @@ -53,7 +53,7 @@ func (s *Server) handleAPIListStorageConfirmations(w http.ResponseWriter, r *htt response := make([]storageConfirmationAttentionResponse, 0, len(records)) for _, record := range records { response = append(response, storageConfirmationAttentionResponse{ - CopyID: record.CopyID, UploadID: record.UploadID, CopyIndex: record.CopyIndex, + CopyID: record.CopyID, ContentID: record.ContentID, CopyIndex: record.CopyIndex, DataSetRowID: record.DataSetRowID, ProviderID: record.ProviderID, DataSetID: record.DataSetID, PieceCID: record.PieceCID, AttemptID: record.AttemptID, TransactionID: record.TransactionID, ReasonCode: string(record.Code), AttemptedAt: record.AttemptedAt, AttentionAt: record.AttentionAt, @@ -78,7 +78,7 @@ func (s *Server) handleAPIReleaseStorageConfirmation(w http.ResponseWriter, r *h }) return } - err = s.repos.Uploads.ReleaseCommitAttention(r.Context(), storagecommit.ManualReleaseInput{ + err = s.repos.Contents.ReleaseCommitAttention(r.Context(), storagecommit.ManualReleaseInput{ CopyID: copyID, ExpectedAttemptID: req.ExpectedAttemptID, AcknowledgePossibleDuplicate: true, }) switch { diff --git a/internal/admin/api_storage_confirmations_test.go b/internal/admin/api_storage_confirmations_test.go index 07e5f8b..fc7e6ec 100644 --- a/internal/admin/api_storage_confirmations_test.go +++ b/internal/admin/api_storage_confirmations_test.go @@ -14,7 +14,7 @@ import ( ) type storageConfirmationAPIRepo struct { - repository.StorageUploadRepository + repository.StorageContentRepository records []storagecommit.AttentionRecord releaseInput storagecommit.ManualReleaseInput releaseErr error @@ -32,12 +32,12 @@ func (r *storageConfirmationAPIRepo) ReleaseCommitAttention(_ context.Context, i func TestAPIStorageConfirmationsListAndRelease(t *testing.T) { now := time.Date(2026, 8, 30, 1, 2, 3, 0, time.UTC) repo := &storageConfirmationAPIRepo{records: []storagecommit.AttentionRecord{{ - CopyID: 12, UploadID: 7, CopyIndex: 1, DataSetRowID: 9, + CopyID: 12, ContentID: 7, CopyIndex: 1, DataSetRowID: 9, ProviderID: "provider-1", DataSetID: "dataset-1", PieceCID: "piece-1", AttemptID: "attempt-1", Code: storagecommit.AttentionAttemptOnlyAmbiguous, AttemptedAt: now.Add(-time.Second), AttentionAt: now, }}} - srv := &Server{repos: &repository.Repositories{Uploads: repo}, logger: testLogger()} + srv := &Server{repos: &repository.Repositories{Contents: repo}, logger: testLogger()} mux := http.NewServeMux() mux.HandleFunc("GET /api/v1/storage-confirmations", srv.handleAPIListStorageConfirmations) mux.HandleFunc("POST /api/v1/storage-confirmations/{id}/release", srv.handleAPIReleaseStorageConfirmation) @@ -69,7 +69,7 @@ func TestAPIStorageConfirmationsListAndRelease(t *testing.T) { func TestAPIStorageConfirmationReleaseRequiresAcknowledgement(t *testing.T) { repo := &storageConfirmationAPIRepo{} - srv := &Server{repos: &repository.Repositories{Uploads: repo}, logger: testLogger()} + srv := &Server{repos: &repository.Repositories{Contents: repo}, logger: testLogger()} req := httptest.NewRequest(http.MethodPost, "/api/v1/storage-confirmations/12/release", strings.NewReader(`{}`)) req.SetPathValue("id", "12") req.Header.Set("Content-Type", "application/json") @@ -86,7 +86,7 @@ func TestAPIStorageConfirmationReleaseRequiresAcknowledgement(t *testing.T) { func TestAPIStorageConfirmationReleaseRejectsStaleAttempt(t *testing.T) { repo := &storageConfirmationAPIRepo{releaseErr: repository.ErrConflict} - srv := &Server{repos: &repository.Repositories{Uploads: repo}, logger: testLogger()} + srv := &Server{repos: &repository.Repositories{Contents: repo}, logger: testLogger()} req := httptest.NewRequest(http.MethodPost, "/api/v1/storage-confirmations/12/release", strings.NewReader(`{"acknowledge_possible_duplicate":true,"expected_attempt_id":"stale-attempt"}`)) req.SetPathValue("id", "12") req.Header.Set("Content-Type", "application/json") diff --git a/internal/admin/api_task_diagnostics.go b/internal/admin/api_task_diagnostics.go deleted file mode 100644 index 08babee..0000000 --- a/internal/admin/api_task_diagnostics.go +++ /dev/null @@ -1,346 +0,0 @@ -package admin - -import ( - "context" - "net/http" - "strconv" - "time" - - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/observability" - "github.com/strahe/synaps3/internal/synapse" - "github.com/strahe/synaps3/internal/types" -) - -type taskDiagnosticStatusChecker interface { - CheckDataSetCreationStatus(context.Context, synapse.DataSetCreationStatusInput) synapse.PDPStatusResult - CheckAddPiecesStatus(context.Context, synapse.AddPiecesStatusInput) synapse.PDPStatusResult -} - -// SynapS3 currently commits one PieceInput per upload copy. -const taskDiagnosticAddPiecesExpectedPieceCount = 1 - -func (s *Server) WithTaskDiagnosticStatusChecker(checker taskDiagnosticStatusChecker) *Server { - if checker != nil { - s.taskDiagnosticChecker = checker - } - return s -} - -func (s *Server) handleAPITaskDiagnostic(w http.ResponseWriter, r *http.Request) { - s.handleAPITaskDiagnosticCommon(w, r, false) -} - -func (s *Server) handleAPITaskDiagnosticRefresh(w http.ResponseWriter, r *http.Request) { - s.handleAPITaskDiagnosticCommon(w, r, true) -} - -func (s *Server) handleAPITaskDiagnosticCommon(w http.ResponseWriter, r *http.Request, refresh bool) { - ctx := r.Context() - id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) - if err != nil { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid id"}) - return - } - if id <= 0 { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid id"}) - return - } - task, err := s.repos.Tasks.GetByID(ctx, id) - if err != nil { - s.logger.Error("api: failed to get task diagnostic", "error", err, "taskID", id) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) - return - } - if task == nil { - writeJSON(w, http.StatusNotFound, map[string]string{"error": "task not found"}) - return - } - facts, err := s.taskDiagnosticFacts(ctx, task) - if err != nil { - s.logger.Error("api: failed to build task diagnostic facts", "error", err, "taskID", id) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) - return - } - var live *observability.TaskDiagnosticLiveCheck - if refresh { - live = s.taskDiagnosticLiveCheck(ctx, facts) - } - writeJSON(w, http.StatusOK, observability.TaskDiagnosticFromFacts(facts, live, time.Now().UTC())) -} - -func (s *Server) taskDiagnosticFacts(ctx context.Context, task *model.Task) (observability.TaskDiagnosticFacts, error) { - stage := "" - if ptr := taskStage(task); ptr != nil { - stage = *ptr - } - scheduledAt := task.ScheduledAt - facts := observability.TaskDiagnosticFacts{ - Task: observability.TaskDiagnosticTaskFacts{ - ID: task.ID, - Type: task.Type, - Stage: stage, - Status: task.Status, - RetryCount: task.RetryCount, - MaxRetries: task.MaxRetries, - LastError: task.LastError, - StatusMessage: task.StatusMessage, - WaitReason: task.WaitReason, - ScheduledAt: &scheduledAt, - }, - } - if task.Type != model.TaskTypeUpload || s.repos == nil || s.repos.Uploads == nil { - return facts, nil - } - - upload, err := s.taskDiagnosticUpload(ctx, task) - if err != nil { - return facts, err - } - if upload != nil { - facts.Upload = &observability.TaskDiagnosticUploadFacts{ - ID: upload.ID, - Status: upload.Status, - RequestedCopies: upload.RequestedCopies, - ErrorMessage: upload.ErrorMessage, - AcceptError: upload.AcceptError, - } - } - - copyIndex := taskPayloadInt(task.Payload, "copy_index") - var copyRow *model.StorageUploadCopy - if upload != nil && copyIndex != nil { - copyRow, err = s.repos.Uploads.GetUploadCopy(ctx, upload.ID, *copyIndex) - if err != nil { - return facts, err - } - if copyRow != nil { - facts.Copy = taskDiagnosticCopyFacts(copyRow) - } - } - - binding, err := s.taskDiagnosticDataSet(ctx, upload, copyRow, copyIndex) - if err != nil { - return facts, err - } - if binding != nil { - facts.DataSet = taskDiagnosticDataSetFacts(binding) - } - - providerID := taskDiagnosticProviderID(copyRow, binding) - if providerID != nil { - facts.Provider = s.taskDiagnosticProviderFacts(ctx, *providerID) - } - if facts.Provider == nil && providerID != nil { - facts.Provider = &observability.TaskDiagnosticProviderFacts{ProviderID: *providerID} - } - facts.Transaction = taskDiagnosticTransactionFacts(facts) - return facts, nil -} - -func (s *Server) taskDiagnosticUpload(ctx context.Context, task *model.Task) (*model.StorageUpload, error) { - if uploadID := taskPayloadInt64(task.Payload, "upload_id"); uploadID != nil { - return s.repos.Uploads.GetByID(ctx, *uploadID) - } - return nil, nil -} - -func (s *Server) taskDiagnosticDataSet(ctx context.Context, upload *model.StorageUpload, copyRow *model.StorageUploadCopy, copyIndex *int) (*model.StorageDataSet, error) { - if copyRow != nil && copyRow.StorageDataSetID != nil { - return s.repos.Uploads.GetDataSetBindingByID(ctx, *copyRow.StorageDataSetID) - } - if upload != nil && copyIndex != nil { - return s.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, upload.BucketID, *copyIndex) - } - return nil, nil -} - -func (s *Server) taskDiagnosticProviderFacts(ctx context.Context, providerID types.OnChainID) *observability.TaskDiagnosticProviderFacts { - if s.observability == nil { - return nil - } - page, err := s.observability.ListProviderObservations(ctx, observability.ListOptions{ProviderID: &providerID, Limit: 1}) - if err != nil || len(page.Items) == 0 { - return nil - } - observation := page.Items[0] - return &observability.TaskDiagnosticProviderFacts{ - ProviderID: observation.Facts.ProviderID, - Status: observation.Signal.Status, - ReasonCodes: observation.Signal.ReasonCodes, - ServiceURL: observation.Facts.ServiceURL, - HealthStatus: observation.Facts.HealthStatus, - LastError: observation.Signal.LastError, - } -} - -func (s *Server) taskDiagnosticLiveCheck(ctx context.Context, facts observability.TaskDiagnosticFacts) *observability.TaskDiagnosticLiveCheck { - if facts.Transaction == nil { - return &observability.TaskDiagnosticLiveCheck{State: observability.TaskDiagnosticLiveSkipped} - } - checker := s.taskDiagnosticChecker - if checker == nil { - return &observability.TaskDiagnosticLiveCheck{State: observability.TaskDiagnosticLiveUnavailable, Error: "task diagnostic checker not configured"} - } - switch facts.Transaction.Kind { - case observability.TaskDiagnosticOperationCreateDataSet: - if facts.Transaction.StatusURL == "" { - return &observability.TaskDiagnosticLiveCheck{State: observability.TaskDiagnosticLiveUnavailable, Error: "missing data set creation status URL"} - } - result := checker.CheckDataSetCreationStatus(ctx, synapse.DataSetCreationStatusInput{ - StatusURL: facts.Transaction.StatusURL, - TransactionID: facts.Transaction.TransactionID, - ExpectedDataSetID: facts.Transaction.DataSetID, - }) - return taskDiagnosticLiveCheckFromSynapse(result, observability.TaskDiagnosticOperationCreateDataSet) - case observability.TaskDiagnosticOperationAddPieces: - result := checker.CheckAddPiecesStatus(ctx, synapse.AddPiecesStatusInput{ - ServiceURL: facts.Transaction.ServiceURL, - StatusURL: facts.Transaction.StatusURL, - DataSetID: facts.Transaction.DataSetID, - TransactionID: facts.Transaction.TransactionID, - ExpectedPieceCount: facts.Transaction.PieceCount, - }) - return taskDiagnosticLiveCheckFromSynapse(result, observability.TaskDiagnosticOperationAddPieces) - default: - return &observability.TaskDiagnosticLiveCheck{State: observability.TaskDiagnosticLiveSkipped} - } -} - -func taskDiagnosticCopyFacts(copyRow *model.StorageUploadCopy) *observability.TaskDiagnosticCopyFacts { - return &observability.TaskDiagnosticCopyFacts{ - UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, - Status: copyRow.Status, - ProviderID: copyRow.ProviderID, - StorageDataSetID: copyRow.StorageDataSetID, - ChainDataSetID: copyRow.DataSetID, - PieceID: copyRow.PieceID, - TransferMethod: copyRow.TransferMethod, - CommitTransactionID: copyRow.CommitTransactionID, - LastError: copyRow.LastError, - } -} - -func taskDiagnosticDataSetFacts(binding *model.StorageDataSet) *observability.TaskDiagnosticDataSetFacts { - return &observability.TaskDiagnosticDataSetFacts{ - ID: binding.ID, - Status: binding.Status, - ProviderID: binding.ProviderID, - CopyIndex: binding.CopyIndex, - ChainDataSetID: binding.DataSetID, - ClientDataSetID: binding.ClientDataSetID, - CreateTransactionID: binding.CreateTransactionID, - CreateStatusURL: binding.CreateStatusURL, - LastError: binding.LastError, - } -} - -func taskDiagnosticProviderID(copyRow *model.StorageUploadCopy, binding *model.StorageDataSet) *types.OnChainID { - if copyRow != nil && copyRow.ProviderID != nil { - return copyRow.ProviderID - } - if binding != nil { - return &binding.ProviderID - } - return nil -} - -func taskDiagnosticTransactionFacts(facts observability.TaskDiagnosticFacts) *observability.TaskDiagnosticTransactionFacts { - switch observability.TaskDiagnosticOperationForTask(facts.Task.Type, facts.Task.Stage) { - case observability.TaskDiagnosticOperationCreateDataSet: - if facts.DataSet == nil { - return nil - } - tx := &observability.TaskDiagnosticTransactionFacts{ - Kind: observability.TaskDiagnosticOperationCreateDataSet, - StatusURL: derefString(facts.DataSet.CreateStatusURL), - } - if facts.DataSet.CreateTransactionID != nil { - tx.TransactionID = *facts.DataSet.CreateTransactionID - } - if facts.DataSet.ChainDataSetID != nil { - tx.DataSetID = facts.DataSet.ChainDataSetID.String() - } - if facts.Provider != nil && facts.Provider.ServiceURL != nil { - tx.ServiceURL = *facts.Provider.ServiceURL - } - return tx - case observability.TaskDiagnosticOperationAddPieces: - if facts.Copy == nil { - return nil - } - transactionID := derefString(facts.Copy.CommitTransactionID) - if transactionID == "" { - return nil - } - tx := &observability.TaskDiagnosticTransactionFacts{ - Kind: observability.TaskDiagnosticOperationAddPieces, - TransactionID: transactionID, - PieceCount: taskDiagnosticAddPiecesExpectedPieceCount, - } - if facts.DataSet != nil && facts.DataSet.ChainDataSetID != nil { - tx.DataSetID = facts.DataSet.ChainDataSetID.String() - } - if facts.Provider != nil && facts.Provider.ServiceURL != nil { - tx.ServiceURL = *facts.Provider.ServiceURL - } - return tx - default: - return nil - } -} - -func taskDiagnosticLiveCheckFromSynapse(result synapse.PDPStatusResult, operation observability.TaskDiagnosticOperation) *observability.TaskDiagnosticLiveCheck { - live := &observability.TaskDiagnosticLiveCheck{ - State: taskDiagnosticLiveStateFromSynapse(result.State), - StatusURL: result.StatusURL, - TxStatus: result.TxStatus, - DataSetID: result.DataSetID, - Error: result.Error, - } - if result.State == synapse.PDPStatusUnavailable || result.State == synapse.PDPStatusUnknown { - return live - } - switch operation { - case observability.TaskDiagnosticOperationCreateDataSet: - live.DataSetCreated = boolPtr(result.DataSetCreated) - case observability.TaskDiagnosticOperationAddPieces: - live.PiecesAdded = boolPtr(result.PiecesAdded) - live.PieceCount = intPtr(result.PieceCount) - live.SetConfirmedPieceIDs(result.ConfirmedPieceIDs) - } - return live -} - -func taskDiagnosticLiveStateFromSynapse(state synapse.PDPStatusState) observability.TaskDiagnosticLiveState { - switch state { - case synapse.PDPStatusPending: - return observability.TaskDiagnosticLivePending - case synapse.PDPStatusConfirmed: - return observability.TaskDiagnosticLiveConfirmed - case synapse.PDPStatusRejected: - return observability.TaskDiagnosticLiveRejected - case synapse.PDPStatusMismatch: - return observability.TaskDiagnosticLiveMismatch - case synapse.PDPStatusUnavailable: - return observability.TaskDiagnosticLiveUnavailable - default: - return observability.TaskDiagnosticLiveUnknown - } -} - -func boolPtr(value bool) *bool { - return &value -} - -func intPtr(value int) *int { - return &value -} - -func derefString(value *string) string { - if value == nil { - return "" - } - return *value -} diff --git a/internal/admin/api_task_diagnostics_test.go b/internal/admin/api_task_diagnostics_test.go deleted file mode 100644 index 2f0d8a1..0000000 --- a/internal/admin/api_task_diagnostics_test.go +++ /dev/null @@ -1,698 +0,0 @@ -package admin - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "strconv" - "testing" - "time" - - "github.com/strahe/synaps3/internal/config" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/observability" - "github.com/strahe/synaps3/internal/storagecommit" - "github.com/strahe/synaps3/internal/synapse" - "github.com/strahe/synaps3/internal/testutil" - "github.com/uptrace/bun" -) - -type fakeTaskDiagnosticStatusChecker struct { - creationResult synapse.PDPStatusResult - addPiecesResult synapse.PDPStatusResult - creationInput synapse.DataSetCreationStatusInput - addPiecesInput synapse.AddPiecesStatusInput - creationCalls int - addPiecesCalls int -} - -func (c *fakeTaskDiagnosticStatusChecker) CheckDataSetCreationStatus(_ context.Context, input synapse.DataSetCreationStatusInput) synapse.PDPStatusResult { - c.creationCalls++ - c.creationInput = input - c.creationResult.StatusURL = input.StatusURL - return c.creationResult -} - -func (c *fakeTaskDiagnosticStatusChecker) CheckAddPiecesStatus(_ context.Context, input synapse.AddPiecesStatusInput) synapse.PDPStatusResult { - c.addPiecesCalls++ - c.addPiecesInput = input - return c.addPiecesResult -} - -func TestAPITaskDiagnosticGETReturnsEvidence(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - task := &model.Task{ - Type: model.TaskTypeEvictCache, - RefType: "object", - RefID: 1, - RefVersionID: "01J000000000000000DIAG001", - IdempotencyKey: "task-diagnostic-non-upload", - Status: model.TaskStatusQueued, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task: %v", err) - } - checker := &fakeTaskDiagnosticStatusChecker{} - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()). - WithTaskDiagnosticStatusChecker(checker) - body, status := getTaskDiagnostic(t, srv, task.ID) - if status != http.StatusOK { - t.Fatalf("status = %d, want 200", status) - } - if body.CurrentState != observability.TaskDiagnosticStateNotApplicable || body.Evidence.Task.Type != model.TaskTypeEvictCache || body.Evidence.LiveCheck != nil { - t.Fatalf("diagnostic = %#v, want non-upload evidence without live check", body) - } - if checker.creationCalls != 0 || checker.addPiecesCalls != 0 { - t.Fatalf("checker calls = creation:%d add:%d, want none for GET", checker.creationCalls, checker.addPiecesCalls) - } -} - -func TestAPITaskDiagnosticRefreshChecksLiveStatusWithoutMutatingTask(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - task, _ := seedTaskDiagnosticCommitTask(t, db, repos, "task-diagnostic-refresh", "01J000000000000000DIAG002") - before, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil { - t.Fatalf("GetByID before: %v", err) - } - checker := &fakeTaskDiagnosticStatusChecker{ - addPiecesResult: synapse.PDPStatusResult{ - State: synapse.PDPStatusConfirmed, - StatusURL: "https://provider.example/pdp/data-sets/1001/pieces/added/0xcommit", - TxStatus: "confirmed", - DataSetID: "1001", - PiecesAdded: true, - PieceCount: 1, - ConfirmedPieceIDs: []string{"2001"}, - }, - } - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()). - WithObservability(taskDiagnosticObservabilityService(t, repos)). - WithTaskDiagnosticStatusChecker(checker) - - body, status := refreshTaskDiagnostic(t, srv, task.ID) - if status != http.StatusOK { - t.Fatalf("status = %d, want 200", status) - } - if body.CurrentState != observability.TaskDiagnosticStateConfirmed || body.Evidence.LiveCheck == nil || body.Evidence.LiveCheck.State != observability.TaskDiagnosticLiveConfirmed { - t.Fatalf("diagnostic = %#v, want confirmed live check", body) - } - if checker.addPiecesInput.DataSetID != "1001" || checker.addPiecesInput.TransactionID != "0xcommit" || checker.addPiecesInput.ExpectedPieceCount != 1 { - t.Fatalf("checker input = %#v, want add-pieces status facts", checker.addPiecesInput) - } - after, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil { - t.Fatalf("GetByID after: %v", err) - } - if after.Status != before.Status || after.RetryCount != before.RetryCount || !after.ScheduledAt.Equal(before.ScheduledAt) { - t.Fatalf("task mutated = before:%#v after:%#v", before, after) - } -} - -func TestAPITaskDiagnosticRefreshPassesDataSetCreationEvidence(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - task, _, _ := seedTaskDiagnosticCreateDataSetTask(t, db, repos, "task-diagnostic-create-status", "01J000000000000000DIAGCRE") - checker := &fakeTaskDiagnosticStatusChecker{ - creationResult: synapse.PDPStatusResult{ - State: synapse.PDPStatusPending, - StatusURL: "https://provider.example/pdp/data-sets/created/0xcreate", - TxStatus: "pending", - DataSetCreated: false, - }, - } - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()). - WithObservability(taskDiagnosticObservabilityService(t, repos)). - WithTaskDiagnosticStatusChecker(checker) - - body, status := refreshTaskDiagnostic(t, srv, task.ID) - if status != http.StatusOK { - t.Fatalf("status = %d, want 200", status) - } - if body.Evidence.LiveCheck == nil || body.Evidence.LiveCheck.State != observability.TaskDiagnosticLivePending { - t.Fatalf("diagnostic = %#v, want pending creation live check", body) - } - if checker.creationInput.StatusURL != "https://provider.example/pdp/data-sets/created/0xcreate" || - checker.creationInput.TransactionID != "0xcreate" || - checker.creationInput.ExpectedDataSetID != "" { - t.Fatalf("creation input = %#v, want creation status evidence", checker.creationInput) - } -} - -func TestAPITaskDiagnosticRefreshUnavailableStillReturnsDiagnostic(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - task, _ := seedTaskDiagnosticCommitTask(t, db, repos, "task-diagnostic-refresh-unavailable", "01J000000000000000DIAG003") - checker := &fakeTaskDiagnosticStatusChecker{ - addPiecesResult: synapse.PDPStatusResult{ - State: synapse.PDPStatusUnavailable, - StatusURL: "https://provider.example/pdp/data-sets/1001/pieces/added/0xcommit", - Error: "context deadline exceeded", - }, - } - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()). - WithObservability(taskDiagnosticObservabilityService(t, repos)). - WithTaskDiagnosticStatusChecker(checker) - - body, status := refreshTaskDiagnostic(t, srv, task.ID) - if status != http.StatusOK { - t.Fatalf("status = %d, want 200", status) - } - if body.CurrentState != observability.TaskDiagnosticStateUnknown || body.Signal.Status != observability.StatusUnknown || body.Evidence.LiveCheck == nil || body.Evidence.LiveCheck.State != observability.TaskDiagnosticLiveUnavailable { - t.Fatalf("diagnostic = %#v, want unknown unavailable live check", body) - } -} - -func TestAPITaskDiagnosticRefreshOmitsUncertainLiveEvidence(t *testing.T) { - cases := []struct { - name string - result synapse.PDPStatusResult - }{ - { - name: "unavailable", - result: synapse.PDPStatusResult{ - State: synapse.PDPStatusUnavailable, - StatusURL: "https://provider.example/pdp/data-sets/1001/pieces/added/0xcommit", - Error: "context deadline exceeded", - }, - }, - { - name: "unknown", - result: synapse.PDPStatusResult{ - State: synapse.PDPStatusUnknown, - StatusURL: "https://provider.example/pdp/data-sets/1001/pieces/added/0xcommit", - TxStatus: "queued", - DataSetID: "1001", - PiecesAdded: false, - PieceCount: 0, - ConfirmedPieceIDs: []string{}, - }, - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - task, _ := seedTaskDiagnosticCommitTask(t, db, repos, "task-diagnostic-refresh-uncertain-"+tc.name, "01J000000000000000DIAG004") - checker := &fakeTaskDiagnosticStatusChecker{addPiecesResult: tc.result} - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()). - WithObservability(taskDiagnosticObservabilityService(t, repos)). - WithTaskDiagnosticStatusChecker(checker) - - raw, status := refreshTaskDiagnosticRaw(t, srv, task.ID) - if status != http.StatusOK { - t.Fatalf("status = %d, want 200", status) - } - var payload map[string]any - if err := json.Unmarshal(raw, &payload); err != nil { - t.Fatalf("Unmarshal raw diagnostic: %v", err) - } - live := payload["evidence"].(map[string]any)["live_check"].(map[string]any) - if _, ok := live["pieces_added"]; ok { - t.Fatalf("pieces_added encoded for %s live check: %#v", tc.name, live) - } - if _, ok := live["data_set_created"]; ok { - t.Fatalf("data_set_created encoded for %s live check: %#v", tc.name, live) - } - if _, ok := live["piece_count"]; ok { - t.Fatalf("piece_count encoded for %s live check: %#v", tc.name, live) - } - if _, ok := live["confirmed_piece_ids"]; ok { - t.Fatalf("confirmed_piece_ids encoded for %s live check: %#v", tc.name, live) - } - }) - } -} - -func TestAPITaskDiagnosticRefreshEncodesFalseLiveCheckFields(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - task, _ := seedTaskDiagnosticCommitTask(t, db, repos, "task-diagnostic-refresh-mismatch", "01J000000000000000DIAG004") - checker := &fakeTaskDiagnosticStatusChecker{ - addPiecesResult: synapse.PDPStatusResult{ - State: synapse.PDPStatusMismatch, - StatusURL: "https://provider.example/pdp/data-sets/1001/pieces/added/0xcommit", - TxStatus: "confirmed", - DataSetID: "1001", - PiecesAdded: false, - PieceCount: 1, - }, - } - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()). - WithObservability(taskDiagnosticObservabilityService(t, repos)). - WithTaskDiagnosticStatusChecker(checker) - - raw, status := refreshTaskDiagnosticRaw(t, srv, task.ID) - if status != http.StatusOK { - t.Fatalf("status = %d, want 200", status) - } - var payload map[string]any - if err := json.Unmarshal(raw, &payload); err != nil { - t.Fatalf("Unmarshal raw diagnostic: %v", err) - } - live := payload["evidence"].(map[string]any)["live_check"].(map[string]any) - if got, ok := live["pieces_added"].(bool); !ok || got { - t.Fatalf("pieces_added = %#v, want encoded false", live["pieces_added"]) - } -} - -func TestAPITaskDiagnosticRefreshEncodesZeroAndEmptyAddPiecesEvidence(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - task, _ := seedTaskDiagnosticCommitTask(t, db, repos, "task-diagnostic-refresh-zero-evidence", "01J000000000000000DIAGZRO") - checker := &fakeTaskDiagnosticStatusChecker{ - addPiecesResult: synapse.PDPStatusResult{ - State: synapse.PDPStatusMismatch, - StatusURL: "https://provider.example/pdp/data-sets/1001/pieces/added/0xcommit", - TxStatus: "confirmed", - DataSetID: "1001", - PiecesAdded: true, - PieceCount: 0, - ConfirmedPieceIDs: []string{}, - }, - } - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()). - WithObservability(taskDiagnosticObservabilityService(t, repos)). - WithTaskDiagnosticStatusChecker(checker) - - raw, status := refreshTaskDiagnosticRaw(t, srv, task.ID) - if status != http.StatusOK { - t.Fatalf("status = %d, want 200", status) - } - var payload map[string]any - if err := json.Unmarshal(raw, &payload); err != nil { - t.Fatalf("Unmarshal raw diagnostic: %v", err) - } - live := payload["evidence"].(map[string]any)["live_check"].(map[string]any) - if got, ok := live["piece_count"].(float64); !ok || got != 0 { - t.Fatalf("piece_count = %#v, want encoded zero", live["piece_count"]) - } - confirmed, ok := live["confirmed_piece_ids"].([]any) - if !ok || len(confirmed) != 0 { - t.Fatalf("confirmed_piece_ids = %#v, want encoded empty array", live["confirmed_piece_ids"]) - } -} - -func TestAPITaskDiagnosticRefreshMissingProviderServiceURLReturnsDiagnosticUnavailable(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - task, _ := seedTaskDiagnosticCommitTask(t, db, repos, "task-diagnostic-missing-service-url", "01J000000000000000DIAGURL") - before, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil { - t.Fatalf("GetByID before: %v", err) - } - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()) - - body, status := refreshTaskDiagnostic(t, srv, task.ID) - if status != http.StatusOK { - t.Fatalf("status = %d, want 200", status) - } - if body.Evidence.LiveCheck == nil || body.Evidence.LiveCheck.State != observability.TaskDiagnosticLiveUnavailable { - t.Fatalf("diagnostic = %#v, want unavailable live check", body) - } - if body.Signal.Status != observability.StatusUnavailable || len(body.ReasonCodes) != 1 || body.ReasonCodes[0] != observability.ReasonTaskDiagnosticUnavailable { - t.Fatalf("diagnostic signal = status:%s reasons:%v, want diagnostic unavailable", body.Signal.Status, body.ReasonCodes) - } - after, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil { - t.Fatalf("GetByID after: %v", err) - } - if after.Status != before.Status || after.RetryCount != before.RetryCount || !after.ScheduledAt.Equal(before.ScheduledAt) { - t.Fatalf("task mutated = before:%#v after:%#v", before, after) - } -} - -func TestAPITaskDiagnosticRefreshSkipsLiveCheckWithoutTransactionEvidence(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - stage := "prepare_upload" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: 1, - RefVersionID: "01J000000000000000DIAG005", - IdempotencyKey: "task-diagnostic-refresh-prepare", - Status: model.TaskStatusQueued, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task: %v", err) - } - checker := &fakeTaskDiagnosticStatusChecker{} - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()). - WithTaskDiagnosticStatusChecker(checker) - - body, status := refreshTaskDiagnostic(t, srv, task.ID) - if status != http.StatusOK { - t.Fatalf("status = %d, want 200", status) - } - if body.CurrentState != observability.TaskDiagnosticStatePreparing || body.Evidence.LiveCheck == nil || body.Evidence.LiveCheck.State != observability.TaskDiagnosticLiveSkipped { - t.Fatalf("diagnostic = %#v, want preparing diagnosis with skipped live check", body) - } - if checker.creationCalls != 0 || checker.addPiecesCalls != 0 { - t.Fatalf("checker calls = creation:%d add:%d, want none", checker.creationCalls, checker.addPiecesCalls) - } -} - -func TestAPITaskDiagnosticRefreshSkipsAddPiecesWithoutCommitTransactionID(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - task, upload := seedTaskDiagnosticCommitTask(t, db, repos, "task-diagnostic-refresh-missing-tx", "01J000000000000000DIAGMTX") - if _, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("commit_transaction_id = NULL"). - Where("upload_id = ? AND copy_index = ?", upload.ID, 0). - Exec(ctx); err != nil { - t.Fatalf("clear commit transaction: %v", err) - } - checker := &fakeTaskDiagnosticStatusChecker{} - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()). - WithObservability(taskDiagnosticObservabilityService(t, repos)). - WithTaskDiagnosticStatusChecker(checker) - - body, status := refreshTaskDiagnostic(t, srv, task.ID) - if status != http.StatusOK { - t.Fatalf("status = %d, want 200", status) - } - if body.CurrentState != observability.TaskDiagnosticStateWaitingForChain || body.Evidence.LiveCheck == nil || body.Evidence.LiveCheck.State != observability.TaskDiagnosticLiveSkipped { - t.Fatalf("diagnostic = %#v, want waiting diagnosis with skipped live check", body) - } - if body.Evidence.Transaction != nil { - t.Fatalf("transaction evidence = %#v, want none without commit transaction ID", body.Evidence.Transaction) - } - if checker.creationCalls != 0 || checker.addPiecesCalls != 0 { - t.Fatalf("checker calls = creation:%d add:%d, want none", checker.creationCalls, checker.addPiecesCalls) - } -} - -func TestAPITaskDiagnosticMissingCopyIndexDoesNotReadReplicaZero(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - seeded, upload := seedTaskDiagnosticCommitTask(t, db, repos, "task-diagnostic-missing-copy-index", "01J000000000000000DIAG006") - stage := "ingress_commit" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: seeded.RefType, - RefID: seeded.RefID, - RefVersionID: seeded.RefVersionID, - IdempotencyKey: "task-diagnostic-missing-copy-index-no-copy", - Payload: map[string]interface{}{"upload_id": upload.ID}, - Status: model.TaskStatusWaiting, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task: %v", err) - } - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()). - WithObservability(taskDiagnosticObservabilityService(t, repos)) - - body, status := getTaskDiagnostic(t, srv, task.ID) - if status != http.StatusOK { - t.Fatalf("status = %d, want 200", status) - } - if body.Evidence.Copy != nil || body.Evidence.Transaction != nil { - t.Fatalf("diagnostic = %#v, want no inferred replica zero copy or transaction", body) - } - if body.CurrentState != observability.TaskDiagnosticStateUnknown { - t.Fatalf("state = %s, want unknown missing evidence", body.CurrentState) - } -} - -func TestAPITaskDiagnosticMissingUploadIDDoesNotFallbackToLatestUpload(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - seeded, _ := seedTaskDiagnosticCommitTask(t, db, repos, "task-diagnostic-missing-upload-id", "01J000000000000000DIAG007") - stage := "ingress_commit" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: seeded.RefType, - RefID: seeded.RefID, - RefVersionID: seeded.RefVersionID, - IdempotencyKey: "task-diagnostic-missing-upload-id-no-fallback", - Payload: map[string]interface{}{"copy_index": 0}, - Status: model.TaskStatusWaiting, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task: %v", err) - } - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()). - WithObservability(taskDiagnosticObservabilityService(t, repos)) - - body, status := getTaskDiagnostic(t, srv, task.ID) - if status != http.StatusOK { - t.Fatalf("status = %d, want 200", status) - } - if body.Evidence.Upload != nil || body.Evidence.Copy != nil || body.Evidence.Transaction != nil { - t.Fatalf("diagnostic = %#v, want no upload fallback evidence", body) - } -} - -func TestAPITaskDiagnosticInvalidIDReturns400(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()) - - _, status := getTaskDiagnostic(t, srv, 0) - if status != http.StatusBadRequest { - t.Fatalf("status = %d, want 400", status) - } -} - -func TestAPITaskDiagnosticMissingTaskReturns404(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()) - _, status := getTaskDiagnostic(t, srv, 404) - if status != http.StatusNotFound { - t.Fatalf("status = %d, want 404", status) - } -} - -func seedTaskDiagnosticCommitTask(t *testing.T, db *bun.DB, repos *repository.Repositories, key string, versionID string) (*model.Task, *model.StorageUpload) { - t.Helper() - ctx := context.Background() - bucket := testutil.SeedBucket(t, db, key+"-bucket") - objectID, seededVersionID := seedAdminObjectVersion(t, repos, bucket, key+".txt", 20, key+"-etag", key+"-checksum", "text/plain", "", model.ObjectStateUploading) - if seededVersionID != versionID { - versionID = seededVersionID - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: 20, - Checksum: key + "-checksum", - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: upload.ID, - DataSetID: onChainID(t, "1001"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: onChainID(t, "101"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "piece-" + key, - PieceID: onChainIDPtr(t, "2001"), - RetrievalURL: "https://provider.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - copyRow, err := repos.Uploads.GetUploadCopy(ctx, upload.ID, 0) - if err != nil || copyRow == nil || copyRow.StorageDataSetID == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - identity := storagecommit.CopyIdentity{ - StorageUploadCopyID: copyRow.ID, UploadID: upload.ID, CopyIndex: 0, - StorageDataSetID: *copyRow.StorageDataSetID, - } - if _, err := repos.Uploads.ReserveCommitAttempt(ctx, storagecommit.ReserveInput{ - Copy: identity, AttemptID: "diagnostic-attempt", - }); err != nil { - t.Fatalf("ReserveCommitAttempt: %v", err) - } - if _, err := repos.Uploads.MarkCommitAttempted(ctx, storagecommit.AttemptInput{ - Copy: identity, AttemptID: "diagnostic-attempt", ExtraDataHex: "abcd", - }); err != nil { - t.Fatalf("MarkCommitAttempted: %v", err) - } - if err := repos.Uploads.RecordCommitTransaction(ctx, storagecommit.EvidenceInput{ - Copy: identity, AttemptID: "diagnostic-attempt", TransactionID: "0xcommit", - }); err != nil { - t.Fatalf("RecordCommitTransaction: %v", err) - } - stage := "ingress_commit" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objectID, - RefVersionID: versionID, - IdempotencyKey: key, - Payload: map[string]interface{}{"upload_id": upload.ID, "copy_index": 0}, - Status: model.TaskStatusWaiting, - RetryCount: 2, - MaxRetries: 5, - ScheduledAt: time.Now().Add(time.Minute).UTC().Truncate(time.Microsecond), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task: %v", err) - } - return task, upload -} - -func seedTaskDiagnosticCreateDataSetTask(t *testing.T, db *bun.DB, repos *repository.Repositories, key string, versionID string) (*model.Task, *model.StorageUpload, *model.StorageDataSet) { - t.Helper() - ctx := context.Background() - bucket := testutil.SeedBucket(t, db, key+"-bucket") - objectID, seededVersionID := seedAdminObjectVersion(t, repos, bucket, key+".txt", 20, key+"-etag", key+"-checksum", "text/plain", "", model.ObjectStateUploading) - if seededVersionID != versionID { - versionID = seededVersionID - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: 20, - Checksum: key + "-checksum", - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ - ID: binding.ID, - UploadID: upload.ID, - TransactionID: "0xcreate", - StatusURL: "https://provider.example/pdp/data-sets/created/0xcreate", - }); err != nil { - t.Fatalf("MarkDataSetCreating: %v", err) - } - binding, err = repos.Uploads.GetDataSetBindingByID(ctx, binding.ID) - if err != nil { - t.Fatalf("GetDataSetBindingByID: %v", err) - } - stage := "ensure_dataset" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objectID, - RefVersionID: versionID, - IdempotencyKey: key, - Payload: map[string]interface{}{"upload_id": upload.ID, "copy_index": 0}, - Status: model.TaskStatusWaiting, - MaxRetries: 5, - ScheduledAt: time.Now().Add(time.Minute).UTC().Truncate(time.Microsecond), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task: %v", err) - } - return task, upload, binding -} - -func taskDiagnosticObservabilityService(t *testing.T, repos *repository.Repositories) *observability.Service { - t.Helper() - now := time.Date(2026, 5, 22, 10, 0, 0, 0, time.UTC) - serviceURL := "https://provider.example" - if err := repos.Observability.ReplaceProviderStates(context.Background(), now, []observability.ProviderState{{ - ProviderID: onChainID(t, "101"), - Status: observability.StatusAvailable, - ReasonCodes: []observability.ReasonCode{}, - ServiceURL: &serviceURL, - LastCheckedAt: now, - Evidence: map[string]any{}, - }}); err != nil { - t.Fatalf("ReplaceProviderStates: %v", err) - } - return observability.NewService(observability.ServiceOptions{ - Store: repos.Observability, - Now: func() time.Time { return now }, - }) -} - -func getTaskDiagnostic(t *testing.T, srv *Server, taskID int64) (observability.TaskDiagnostic, int) { - t.Helper() - return callTaskDiagnostic(t, srv, http.MethodGet, taskID) -} - -func refreshTaskDiagnostic(t *testing.T, srv *Server, taskID int64) (observability.TaskDiagnostic, int) { - t.Helper() - return callTaskDiagnostic(t, srv, http.MethodPost, taskID) -} - -func refreshTaskDiagnosticRaw(t *testing.T, srv *Server, taskID int64) ([]byte, int) { - t.Helper() - return callTaskDiagnosticRaw(t, srv, http.MethodPost, taskID) -} - -func callTaskDiagnostic(t *testing.T, srv *Server, method string, taskID int64) (observability.TaskDiagnostic, int) { - t.Helper() - raw, status := callTaskDiagnosticRaw(t, srv, method, taskID) - var body observability.TaskDiagnostic - if status != http.StatusNotFound { - if err := json.Unmarshal(raw, &body); err != nil { - t.Fatalf("Decode: %v", err) - } - } - return body, status -} - -func callTaskDiagnosticRaw(t *testing.T, srv *Server, method string, taskID int64) ([]byte, int) { - t.Helper() - mux := http.NewServeMux() - mux.HandleFunc("GET /api/v1/tasks/{id}/diagnostic", srv.handleAPITaskDiagnostic) - mux.HandleFunc("POST /api/v1/tasks/{id}/diagnostic/refresh", srv.handleAPITaskDiagnosticRefresh) - path := "/api/v1/tasks/" + strconv.FormatInt(taskID, 10) + "/diagnostic" - if method == http.MethodPost { - path += "/refresh" - } - req := httptest.NewRequest(method, path, nil) - rr := httptest.NewRecorder() - mux.ServeHTTP(rr, req) - return rr.Body.Bytes(), rr.Code -} diff --git a/internal/admin/api_tasks.go b/internal/admin/api_tasks.go index 06be569..b22e3cd 100644 --- a/internal/admin/api_tasks.go +++ b/internal/admin/api_tasks.go @@ -1,688 +1,258 @@ package admin import ( - "context" - "encoding/json" "net/http" "strconv" - "strings" "time" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/uptrace/bun" ) type taskListItem struct { - ID int64 `json:"id"` - Type string `json:"type"` - Stage *string `json:"stage,omitempty"` - UploadID *int64 `json:"upload_id,omitempty"` - CopyIndex *int `json:"copy_index,omitempty"` - RefType string `json:"ref_type"` - RefID int64 `json:"ref_id"` - BucketName string `json:"bucket_name,omitempty"` - RefVersionID string `json:"ref_version_id"` - Status string `json:"status"` - Progress *taskProgressResponse `json:"progress,omitempty"` - RetryCount int `json:"retry_count"` - MaxRetries int `json:"max_retries"` - LastError *string `json:"last_error,omitempty"` - StatusMessage *string `json:"status_message,omitempty"` - WaitReason *string `json:"wait_reason,omitempty"` - ScheduledAt string `json:"scheduled_at"` - ClaimedAt *string `json:"claimed_at,omitempty"` - CompletedAt *string `json:"completed_at,omitempty"` + ID int64 `json:"id"` + Type string `json:"type"` + Operation string `json:"operation"` + Status string `json:"status"` + Presentation string `json:"presentation_status"` + SubjectType *string `json:"subject_type,omitempty"` + SubjectKey *string `json:"subject_key,omitempty"` + RetryCount int `json:"retry_count"` + RetryLimit *int `json:"retry_limit,omitempty"` + Retryable bool `json:"retryable"` + Acknowledgeable bool `json:"acknowledgeable"` + WaitReason *string `json:"wait_reason,omitempty"` + FailureReason *string `json:"failure_reason,omitempty"` + LastError *string `json:"last_error,omitempty"` + StatusMessage *string `json:"status_message,omitempty"` + AvailableAt string `json:"available_at"` + StartedAt *string `json:"started_at,omitempty"` + FinishedAt *string `json:"finished_at,omitempty"` + AcknowledgedAt *string `json:"acknowledged_at,omitempty"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` } type taskListResponse struct { - Tasks []taskListItem `json:"tasks"` - Total int `json:"total"` - Limit int `json:"limit"` - Offset int `json:"offset"` -} - -type taskRefDetailResponse struct { - RefType string `json:"ref_type"` - RefID int64 `json:"ref_id"` - RefVersionID string `json:"ref_version_id"` - BucketName string `json:"bucket_name,omitempty"` - Object *taskRefObjectDetail `json:"object"` - StorageCleanup *taskRefStorageCleanupDetail `json:"storage_cleanup,omitempty"` -} - -type taskRefObjectDetail struct { - BucketName string `json:"bucket_name"` - Key string `json:"key"` - VersionID string `json:"version_id"` - Size int64 `json:"size"` - State string `json:"state"` - Status string `json:"status"` - UploadStatus *string `json:"upload_status,omitempty"` - Progress *uploadProgressResponse `json:"progress,omitempty"` - Location objectLocation `json:"location"` - ContentType string `json:"content_type"` - UpdatedAt string `json:"updated_at"` -} - -type taskRefStorageCleanupDetail struct { - UploadID int64 `json:"upload_id"` - DeletedVersions []taskRefStorageCleanupDeletedVersion `json:"deleted_versions"` - Copies []taskRefStorageCleanupCopy `json:"copies"` -} - -type taskRefStorageCleanupDeletedVersion struct { - BucketName string `json:"bucket_name"` - Key string `json:"key"` - VersionID string `json:"version_id"` - Size int64 `json:"size"` - DeletedAt string `json:"deleted_at"` -} - -type taskRefStorageCleanupCopy struct { - CopyIndex int `json:"copy_index"` - ProviderID *string `json:"provider_id,omitempty"` - DataSetID *string `json:"data_set_id,omitempty"` - ClientDataSetID *string `json:"client_data_set_id,omitempty"` - PieceID *string `json:"piece_id,omitempty"` - PieceCID string `json:"piece_cid"` - Status string `json:"status"` - DeleteTxHash *string `json:"delete_tx_hash,omitempty"` - LastError *string `json:"last_error,omitempty"` + Tasks []taskListItem `json:"tasks"` + NextCursor *int64 `json:"next_cursor,omitempty"` } func (s *Server) handleAPITasks(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - - taskType := r.URL.Query().Get("type") - stage := r.URL.Query().Get("stage") - status := r.URL.Query().Get("status") - if stage != "" && taskType == "" { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "stage filter requires type"}) - return - } - - limit := 20 - if v := r.URL.Query().Get("limit"); v != "" { - if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 1000 { - limit = n - } - } - - const maxOffset = 100000 - offset := 0 - if v := r.URL.Query().Get("offset"); v != "" { - if n, err := strconv.Atoi(v); err == nil && n >= 0 && n <= maxOffset { - offset = n - } - } - - tasks, total, err := s.repos.Tasks.List(ctx, taskType, stage, status, limit, offset) + filter, err := parseTaskListFilter(r) if err != nil { - s.logger.Error("api: failed to list tasks", "error", err) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) + writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) return } - progressByTaskID := s.taskProgresses(ctx, tasks) - bucketIDs := make([]int64, 0) - seenBucketIDs := make(map[int64]struct{}) - for i := range tasks { - if tasks[i].RefType == "bucket" && tasks[i].RefID > 0 { - if _, ok := seenBucketIDs[tasks[i].RefID]; !ok { - seenBucketIDs[tasks[i].RefID] = struct{}{} - bucketIDs = append(bucketIDs, tasks[i].RefID) - } - } - } - bucketNames, err := s.repos.Buckets.GetNamesByIDs(ctx, bucketIDs) + page, err := s.repos.Tasks.List(r.Context(), filter) if err != nil { - s.logger.Error("api: failed to load task bucket names", "error", err) + s.logger.Error("api: failed to list tasks", "error", err) writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) return } - - items := make([]taskListItem, 0, len(tasks)) - for i := range tasks { - item := taskListItemFromModel(&tasks[i], progressByTaskID[tasks[i].ID]) - if tasks[i].RefType == "bucket" { - item.BucketName = bucketNames[tasks[i].RefID] - } - items = append(items, item) - } - - writeJSON(w, http.StatusOK, taskListResponse{ - Tasks: items, - Total: total, - Limit: limit, - Offset: offset, - }) -} - -func taskListItemFromModel(t *model.Task, progress *taskProgressResponse) taskListItem { - item := taskListItem{ - ID: t.ID, - Type: string(t.Type), - Stage: taskStage(t), - UploadID: taskPayloadInt64(t.Payload, "upload_id"), - CopyIndex: taskPayloadInt(t.Payload, "copy_index"), - RefType: t.RefType, - RefID: t.RefID, - RefVersionID: t.RefVersionID, - Status: string(t.Status), - Progress: progress, - RetryCount: t.RetryCount, - MaxRetries: t.MaxRetries, - LastError: t.LastError, - StatusMessage: t.StatusMessage, - ScheduledAt: t.ScheduledAt.Format(time.RFC3339), - } - if t.WaitReason != nil { - v := string(*t.WaitReason) - item.WaitReason = &v - } - if t.ClaimedAt != nil { - v := t.ClaimedAt.Format(time.RFC3339) - item.ClaimedAt = &v - } - if t.CompletedAt != nil { - v := t.CompletedAt.Format(time.RFC3339) - item.CompletedAt = &v - } - return item -} - -func taskStage(task *model.Task) *string { - if task == nil { - return nil - } - if task.Stage != nil && *task.Stage != "" { - return task.Stage - } - if stage, ok := task.Payload["stage"].(string); ok && stage != "" { - return &stage + items := make([]taskListItem, 0, len(page.Tasks)) + for i := range page.Tasks { + items = append(items, s.taskListItem(&page.Tasks[i])) } - if task.Type == model.TaskTypeUpload { - stage := "prepare_upload" - return &stage + response := taskListResponse{Tasks: items} + if page.NextBeforeID > 0 { + response.NextCursor = &page.NextBeforeID } - return nil + writeJSON(w, http.StatusOK, response) } -func taskPayloadInt(payload map[string]interface{}, key string) *int { - if value := taskPayloadInt64(payload, key); value != nil { - n := int(*value) - return &n - } - return nil -} - -func taskPayloadInt64(payload map[string]interface{}, key string) *int64 { - if payload == nil { - return nil - } - raw, ok := payload[key] - if !ok { - return nil - } - switch v := raw.(type) { - case int: - n := int64(v) - return &n - case int64: - n := v - return &n - case float64: - n := int64(v) - return &n - case string: - n, err := strconv.ParseInt(v, 10, 64) - if err == nil { - return &n +func parseTaskListFilter(r *http.Request) (repository.TaskListFilter, error) { + for _, removed := range []string{"category", "stage", "offset"} { + if r.URL.Query().Has(removed) { + return repository.TaskListFilter{}, &taskQueryError{removed + " is no longer supported"} } } - return nil -} - -type taskProgressResponse struct { - Scope string `json:"scope"` - - Attempt int `json:"attempt,omitempty"` - UploadedBytes int64 `json:"uploaded_bytes,omitempty"` - TotalBytes int64 `json:"total_bytes,omitempty"` - Done bool `json:"done,omitempty"` - UpdatedAt string `json:"updated_at,omitempty"` - - Phase string `json:"phase,omitempty"` - SeedingComplete bool `json:"seeding_complete,omitempty"` - ItemsTotal int `json:"items_total,omitempty"` - ItemsProcessed int `json:"items_processed,omitempty"` - ItemsCopied int `json:"items_copied,omitempty"` - ItemsNoLongerNeeded int `json:"items_no_longer_needed,omitempty"` - ItemsPending int `json:"items_pending,omitempty"` - ItemsActive int `json:"items_active,omitempty"` - ItemsAttention int `json:"items_attention,omitempty"` - ItemsRetrying int `json:"items_retrying,omitempty"` - ItemsWaitingSource int `json:"items_waiting_source,omitempty"` - ItemsFailed int `json:"items_failed,omitempty"` - Percent *int `json:"percent,omitempty"` - NextRetryAt *string `json:"next_retry_at,omitempty"` -} - -func (p taskProgressResponse) MarshalJSON() ([]byte, error) { - if p.Scope == "provider_replacement" { - return json.Marshal(struct { - Scope string `json:"scope"` - Phase string `json:"phase"` - SeedingComplete bool `json:"seeding_complete"` - ItemsTotal int `json:"items_total"` - ItemsProcessed int `json:"items_processed"` - ItemsCopied int `json:"items_copied"` - ItemsNoLongerNeeded int `json:"items_no_longer_needed"` - ItemsPending int `json:"items_pending"` - ItemsActive int `json:"items_active"` - ItemsAttention int `json:"items_attention"` - ItemsRetrying int `json:"items_retrying"` - ItemsWaitingSource int `json:"items_waiting_source"` - ItemsFailed int `json:"items_failed"` - Percent *int `json:"percent,omitempty"` - NextRetryAt *string `json:"next_retry_at,omitempty"` - }{ - Scope: p.Scope, Phase: p.Phase, SeedingComplete: p.SeedingComplete, - ItemsTotal: p.ItemsTotal, ItemsProcessed: p.ItemsProcessed, - ItemsCopied: p.ItemsCopied, ItemsNoLongerNeeded: p.ItemsNoLongerNeeded, - ItemsPending: p.ItemsPending, ItemsActive: p.ItemsActive, - ItemsAttention: p.ItemsAttention, - ItemsRetrying: p.ItemsRetrying, ItemsWaitingSource: p.ItemsWaitingSource, - ItemsFailed: p.ItemsFailed, Percent: p.Percent, NextRetryAt: p.NextRetryAt, - }) + filter := repository.TaskListFilter{ + Type: model.TaskType(r.URL.Query().Get("type")), + Limit: 50, + HideHealthyRecurringSystem: true, } - return json.Marshal(struct { - Scope string `json:"scope"` - Attempt int `json:"attempt"` - UploadedBytes int64 `json:"uploaded_bytes"` - TotalBytes int64 `json:"total_bytes"` - Percent *int `json:"percent,omitempty"` - Done bool `json:"done"` - UpdatedAt string `json:"updated_at"` - }{ - Scope: p.Scope, Attempt: p.Attempt, UploadedBytes: p.UploadedBytes, - TotalBytes: p.TotalBytes, Percent: p.Percent, Done: p.Done, UpdatedAt: p.UpdatedAt, - }) -} - -func taskProgressFromUpload(progress *uploadProgressResponse) *taskProgressResponse { - if progress == nil { - return nil - } - return &taskProgressResponse{ - Scope: progress.Scope, - Attempt: progress.Attempt, - UploadedBytes: progress.UploadedBytes, - TotalBytes: progress.TotalBytes, - Percent: progress.Percent, - Done: progress.Done, - UpdatedAt: progress.UpdatedAt, - } -} - -func taskProgressFromReplacement(progress storagereplacement.ProgressSnapshot) *taskProgressResponse { - response := &taskProgressResponse{ - Scope: "provider_replacement", - Phase: string(progress.Phase), - SeedingComplete: progress.SeedingComplete, - ItemsTotal: progress.ItemsTotal, - ItemsProcessed: progress.ItemsProcessed, - ItemsCopied: progress.ItemsCopied, - ItemsNoLongerNeeded: progress.ItemsNoLongerNeeded, - ItemsPending: progress.ItemsPending, - ItemsActive: progress.ItemsActive, - ItemsAttention: progress.ItemsAttention, - ItemsRetrying: progress.ItemsRetrying, - ItemsWaitingSource: progress.ItemsWaitingSource, - ItemsFailed: progress.ItemsFailed, - Percent: progress.Percent, - } - if progress.NextRetryAt != nil { - value := progress.NextRetryAt.Format(time.RFC3339) - response.NextRetryAt = &value - } - return response -} - -func (s *Server) taskProgresses(ctx context.Context, tasks []model.Task) map[int64]*taskProgressResponse { - progressByTaskID := make(map[int64]*taskProgressResponse) - if s == nil || s.repos == nil || s.repos.Uploads == nil || len(tasks) == 0 { - return progressByTaskID - } - - taskUploadIDs := make(map[int64]int64) - taskVersionIDs := make(map[int64]string) - uploadIDSet := make(map[int64]struct{}) - versionIDSet := make(map[string]struct{}) - replacementTaskIDs := make(map[int64]int64) - replacementIDSet := make(map[int64]struct{}) - for i := range tasks { - task := &tasks[i] - if task.Stage != nil && (*task.Stage == storagereplacement.StageMigrate || *task.Stage == storagereplacement.StageRetire) { - if replacementID := taskPayloadInt64(task.Payload, "replacement_id"); replacementID != nil { - replacementTaskIDs[task.ID] = *replacementID - replacementIDSet[*replacementID] = struct{}{} - } - continue - } - if !taskWantsUploadProgress(task) { - continue - } - if uploadID := taskPayloadInt64(task.Payload, "upload_id"); uploadID != nil { - taskUploadIDs[task.ID] = *uploadID - uploadIDSet[*uploadID] = struct{}{} - continue - } - versionID := strings.TrimSpace(task.RefVersionID) - if versionID == "" { - continue + status := r.URL.Query().Get("status") + if filter.Type != "" && !validTaskType(filter.Type) { + return repository.TaskListFilter{}, &taskQueryError{"unknown task type"} + } + if status != "" && !validTaskStatus(status) { + return repository.TaskListFilter{}, &taskQueryError{"status must be pending, running, completed, failed, cancelled, or dismissed"} + } + switch status { + case "failed": + filter.Status = model.TaskStatusFailed + filter.Acknowledged = new(false) + case "dismissed": + filter.Status = model.TaskStatusFailed + filter.Acknowledged = new(true) + default: + filter.Status = model.TaskStatus(status) + } + if raw := r.URL.Query().Get("limit"); raw != "" { + limit, err := strconv.Atoi(raw) + if err != nil || limit < 1 || limit > 100 { + return repository.TaskListFilter{}, &taskQueryError{"limit must be between 1 and 100"} } - taskVersionIDs[task.ID] = versionID - versionIDSet[versionID] = struct{}{} + filter.Limit = limit } - - uploadsByID := make(map[int64]model.StorageUpload) - if len(uploadIDSet) > 0 { - uploadIDs := make([]int64, 0, len(uploadIDSet)) - for uploadID := range uploadIDSet { - uploadIDs = append(uploadIDs, uploadID) - } - var err error - uploadsByID, err = s.repos.Uploads.GetByIDs(ctx, uploadIDs) - if err != nil { - s.logger.Warn("api: failed to load task upload progress by upload id", "error", err) - uploadsByID = nil + if raw := r.URL.Query().Get("cursor"); raw != "" { + cursor, err := strconv.ParseInt(raw, 10, 64) + if err != nil || cursor < 1 { + return repository.TaskListFilter{}, &taskQueryError{"cursor must be a positive task ID"} } + filter.BeforeID = cursor } + return filter, nil +} - uploadsByVersionID := make(map[string]model.StorageUpload) - if len(versionIDSet) > 0 { - versionIDs := make([]string, 0, len(versionIDSet)) - for versionID := range versionIDSet { - versionIDs = append(versionIDs, versionID) - } - var err error - uploadsByVersionID, err = s.repos.Uploads.FindLatestUploadsBySourceVersions(ctx, versionIDs) - if err != nil { - s.logger.Warn("api: failed to load task upload progress by version id", "error", err) - uploadsByVersionID = nil - } - } +type taskQueryError struct{ message string } - for taskID, uploadID := range taskUploadIDs { - upload, ok := uploadsByID[uploadID] - if !ok { - continue - } - progressByTaskID[taskID] = taskProgressFromUpload(uploadProgressResponseFromUpload(&upload)) - } - for taskID, versionID := range taskVersionIDs { - upload, ok := uploadsByVersionID[versionID] - if !ok { - continue - } - progressByTaskID[taskID] = taskProgressFromUpload(uploadProgressResponseFromUpload(&upload)) - } - if len(replacementIDSet) > 0 { - replacementIDs := make([]int64, 0, len(replacementIDSet)) - for replacementID := range replacementIDSet { - replacementIDs = append(replacementIDs, replacementID) - } - progresses, err := s.repos.Replacements.ReplacementProgresses(ctx, replacementIDs) - if err != nil { - s.logger.Warn("api: failed to load task replacement progress", "error", err) - } else { - for taskID, replacementID := range replacementTaskIDs { - if progress, ok := progresses[replacementID]; ok { - progressByTaskID[taskID] = taskProgressFromReplacement(progress) - } - } - } - } - return progressByTaskID -} +func (e *taskQueryError) Error() string { return e.message } -func taskWantsUploadProgress(task *model.Task) bool { - if task == nil || task.Type != model.TaskTypeUpload { +func validTaskStatus(status string) bool { + switch status { + case string(model.TaskStatusPending), string(model.TaskStatusRunning), string(model.TaskStatusCompleted), + string(model.TaskStatusFailed), string(model.TaskStatusCancelled), "dismissed": + return true + default: return false } - stage := taskStage(task) - return stage != nil && (*stage == "ingress_store" || *stage == "") } -func (s *Server) handleAPITaskRefDetail(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) - if err != nil { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid id"}) - return - } - - task, err := s.repos.Tasks.GetByID(ctx, id) - if err != nil { - s.logger.Error("api: failed to get task ref detail", "error", err, "taskID", id) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) - return - } - if task == nil { - writeJSON(w, http.StatusNotFound, map[string]string{"error": "task not found"}) - return - } - - resp := taskRefDetailResponse{ - RefType: task.RefType, - RefID: task.RefID, - RefVersionID: task.RefVersionID, - } - if task.RefType == "bucket" { - bucket, bucketErr := s.repos.Buckets.GetByID(ctx, task.RefID) - if bucketErr != nil { - s.logger.Error("api: failed to load task bucket ref", "error", bucketErr, "taskID", id) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) - return - } - if bucket != nil { - resp.BucketName = bucket.Name - } - } - if task.RefType == "storage_upload" && task.Type == model.TaskTypeStorageCleanup { - copies, err := s.repos.StorageCleanup.ListCopiesForTask(ctx, task.ID) - if err != nil { - s.logger.Error("api: failed to list storage cleanup copies", "error", err, "taskID", id) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) - return - } - deletedVersions, err := s.taskStorageCleanupDeletedVersions(ctx, task) - if err != nil { - s.logger.Error("api: failed to list storage cleanup deleted versions", "error", err, "taskID", id) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) - return - } - resp.StorageCleanup = &taskRefStorageCleanupDetail{ - UploadID: task.RefID, - DeletedVersions: deletedVersions, - Copies: make([]taskRefStorageCleanupCopy, 0, len(copies)), - } - for _, copy := range copies { - resp.StorageCleanup.Copies = append(resp.StorageCleanup.Copies, taskRefStorageCleanupCopy{ - CopyIndex: copy.CopyIndex, - ProviderID: onChainIDStringPtr(copy.ProviderID), - DataSetID: onChainIDStringPtr(copy.DataSetID), - ClientDataSetID: onChainIDStringPtr(copy.ClientDataSetID), - PieceID: onChainIDStringPtr(copy.PieceID), - PieceCID: copy.PieceCID, - Status: string(copy.Status), - DeleteTxHash: copy.DeleteTxHash, - LastError: copy.LastError, - }) - } - writeJSON(w, http.StatusOK, resp) - return - } - if task.RefType != "object" { - writeJSON(w, http.StatusOK, resp) - return - } - - var version *model.ObjectVersion - if strings.TrimSpace(task.RefVersionID) != "" { - version, err = s.repos.Objects.GetVersionByID(ctx, task.RefVersionID) - } else { - version, err = s.repos.Objects.GetCurrentVersionByObjectID(ctx, task.RefID) - } - if err != nil { - s.logger.Error("api: failed to get task object ref", "error", err, "taskID", id) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) - return - } - if version == nil { - writeJSON(w, http.StatusNotFound, map[string]string{"error": "object details not found"}) - return - } - if version.ObjectID != task.RefID { - writeJSON(w, http.StatusNotFound, map[string]string{"error": "object details not found"}) - return - } - - bucket, err := s.repos.Buckets.GetByID(ctx, version.BucketID) - if err != nil { - s.logger.Error("api: failed to get task object bucket", "error", err, "taskID", id, "bucketID", version.BucketID) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) - return - } - if bucket == nil || !bucket.Status.IsAdminVisible() { - writeJSON(w, http.StatusNotFound, map[string]string{"error": "object details not found"}) - return - } - uploadInfo, err := s.objectAdminUploadInfo(ctx, *version) - if err != nil { - s.logger.Error("api: failed to get task object upload status", "error", err, "taskID", id, "versionID", version.VersionID) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) - return +func validTaskType(taskType model.TaskType) bool { + switch taskType { + case model.TaskTypeBucketProvision, + model.TaskTypeUploadPlan, + model.TaskTypeStorageDataSetEnsure, + model.TaskTypeStorageTransferPlan, + model.TaskTypeStorageStore, + model.TaskTypeStoragePull, + model.TaskTypeStorageCommitCoordinate, + model.TaskTypeStorageCommit, + model.TaskTypeProviderReplacementCoordinate, + model.TaskTypeCacheCapacityReconcile, + model.TaskTypeCacheEvict, + model.TaskTypeCacheReconcileDurability, + model.TaskTypeStorageCleanup, + model.TaskTypeStorageDataSetRetire, + model.TaskTypeWalletOperation, + model.TaskTypeObservabilityRefresh, + model.TaskTypeGC: + return true + default: + return false } +} - resp.Object = &taskRefObjectDetail{ - BucketName: bucket.Name, - Key: version.Key, - VersionID: version.VersionID, - Size: version.Size, - State: string(version.State), - Status: objectAdminStatusWithUpload(version.State, version.InCache, version.InFilecoin, uploadInfo.Status), - UploadStatus: uploadStatusString(uploadInfo.Status), - Progress: uploadInfo.Progress, - Location: objectLocation{Cache: version.InCache, Filecoin: version.InFilecoin}, - ContentType: version.ContentType, - UpdatedAt: version.UpdatedAt.Format(time.RFC3339), - } - writeJSON(w, http.StatusOK, resp) +func (s *Server) taskListItem(row *model.Task) taskListItem { + item := taskListItem{ + ID: row.ID, Type: string(row.Type), Operation: taskOperationLabel(row.Type), + Status: string(row.Status), Presentation: taskPresentationStatus(row, time.Now()), + SubjectType: row.SubjectType, SubjectKey: row.SubjectKey, + RetryCount: row.RetryCount, RetryLimit: row.RetryLimit, + WaitReason: row.WaitReason, FailureReason: row.FailureReason, + LastError: row.LastError, StatusMessage: row.StatusMessage, + AvailableAt: row.AvailableAt.Format(time.RFC3339), + CreatedAt: row.CreatedAt.Format(time.RFC3339), UpdatedAt: row.UpdatedAt.Format(time.RFC3339), + } + if s.taskService != nil { + item.Retryable = s.taskService.Retryable(row) + item.Acknowledgeable = s.taskService.Acknowledgeable(row) + } + item.StartedAt = formattedTime(row.StartedAt) + item.FinishedAt = formattedTime(row.FinishedAt) + item.AcknowledgedAt = formattedTime(row.AcknowledgedAt) + return item } -func (s *Server) taskStorageCleanupDeletedVersions(ctx context.Context, task *model.Task) ([]taskRefStorageCleanupDeletedVersion, error) { - if task == nil { - return nil, nil +func taskPresentationStatus(row *model.Task, now time.Time) string { + if row == nil { + return "" } - versionIDs := taskPayloadStringSlice(task.Payload, "deleted_source_versions") - if len(versionIDs) == 0 { - if versionID, ok := task.Payload["deleted_source_version"].(string); ok && strings.TrimSpace(versionID) != "" { - versionIDs = append(versionIDs, strings.TrimSpace(versionID)) - } - } - - type deletedVersionRow struct { - BucketName string `bun:"bucket_name"` - Key string `bun:"key"` - VersionID string `bun:"version_id"` - Size int64 `bun:"size"` - DeletedAt time.Time `bun:"deleted_at"` + if row.Status == model.TaskStatusFailed && row.AcknowledgedAt != nil { + return "dismissed" } - var rows []deletedVersionRow - q := s.db.NewSelect(). - TableExpr("object_deletions AS deletion"). - ColumnExpr("bucket.name AS bucket_name"). - ColumnExpr("deletion.key AS key"). - ColumnExpr("deletion.version_id AS version_id"). - ColumnExpr("deletion.size AS size"). - ColumnExpr("deletion.deleted_at AS deleted_at"). - Join("JOIN buckets AS bucket ON bucket.id = deletion.bucket_id") - if len(versionIDs) > 0 { - q = q.Where("deletion.version_id IN (?)", bun.List(versionIDs)) - } else { - q = q.Where("deletion.storage_upload_id = ?", task.RefID) + if row.Status != model.TaskStatusPending { + return string(row.Status) } - if err := q.OrderExpr("deletion.deleted_at DESC, deletion.id DESC").Scan(ctx, &rows); err != nil { - return nil, err + if row.WaitReason != nil && *row.WaitReason != "" { + return "waiting" } - - deletedVersions := make([]taskRefStorageCleanupDeletedVersion, 0, len(rows)) - for _, row := range rows { - deletedVersions = append(deletedVersions, taskRefStorageCleanupDeletedVersion{ - BucketName: row.BucketName, - Key: row.Key, - VersionID: row.VersionID, - Size: row.Size, - DeletedAt: row.DeletedAt.Format(time.RFC3339), - }) + if row.AvailableAt.After(now) { + return "scheduled" } - return deletedVersions, nil + return "queued" } -func taskPayloadStringSlice(payload map[string]interface{}, key string) []string { - if payload == nil { - return nil - } - raw, ok := payload[key] - if !ok { - return nil +func taskOperationLabel(taskType model.TaskType) string { + switch taskType { + case model.TaskTypeBucketProvision: + return "Prepare bucket storage" + case model.TaskTypeUploadPlan: + return "Prepare upload" + case model.TaskTypeStorageDataSetEnsure: + return "Prepare storage" + case model.TaskTypeStorageTransferPlan: + return "Plan storage transfer" + case model.TaskTypeStorageStore: + return "Store content" + case model.TaskTypeStoragePull: + return "Transfer stored content" + case model.TaskTypeStorageCommitCoordinate: + return "Prepare storage confirmation" + case model.TaskTypeStorageCommit: + return "Confirm storage" + case model.TaskTypeProviderReplacementCoordinate: + return "Replace storage provider" + case model.TaskTypeCacheCapacityReconcile: + return "Manage local cache capacity" + case model.TaskTypeCacheEvict: + return "Remove local cached copy" + case model.TaskTypeCacheReconcileDurability: + return "Review cache durability" + case model.TaskTypeStorageCleanup: + return "Remove remote storage copy" + case model.TaskTypeStorageDataSetRetire: + return "Retire storage service" + case model.TaskTypeWalletOperation: + return "Process wallet request" + case model.TaskTypeObservabilityRefresh: + return "Refresh storage health" + case model.TaskTypeGC: + return "Remove expired task records" + default: + return "Background operation" } +} - var values []string - switch v := raw.(type) { - case []string: - values = v - case []interface{}: - for _, item := range v { - value, ok := item.(string) - if !ok { - continue - } - values = append(values, value) - } - } - if len(values) == 0 { +func formattedTime(value *time.Time) *string { + if value == nil { return nil } + formatted := value.Format(time.RFC3339) + return &formatted +} - clean := make([]string, 0, len(values)) - seen := make(map[string]struct{}, len(values)) - for _, value := range values { - value = strings.TrimSpace(value) - if value == "" { - continue - } - if _, ok := seen[value]; ok { - continue - } - seen[value] = struct{}{} - clean = append(clean, value) - } - return clean +type taskStatsItem struct { + Type string `json:"type"` + Status string `json:"status"` + Count int64 `json:"count"` } func (s *Server) handleAPITaskStats(w http.ResponseWriter, r *http.Request) { - counts, err := s.repos.Tasks.CountByStatus(r.Context()) + counts, err := s.repos.Tasks.CountByPresentationStatus(r.Context()) if err != nil { - s.logger.Error("api: failed to get task stats", "error", err) + s.logger.Error("api: failed to count tasks", "error", err) writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) return } - if counts == nil { - counts = []repository.TaskStatusCount{} + items := make([]taskStatsItem, 0, len(counts)) + for _, count := range counts { + if model.TaskType(count.Type).IsRecurringSystem() && count.Status != string(model.TaskStatusFailed) && count.Status != "dismissed" { + continue + } + items = append(items, taskStatsItem{Type: count.Type, Status: count.Status, Count: count.Count}) } - writeJSON(w, http.StatusOK, counts) + writeJSON(w, http.StatusOK, items) } diff --git a/internal/admin/api_tasks_test.go b/internal/admin/api_tasks_test.go index 0fe8a27..1e3c961 100644 --- a/internal/admin/api_tasks_test.go +++ b/internal/admin/api_tasks_test.go @@ -3,7 +3,6 @@ package admin import ( "context" "encoding/json" - "errors" "net/http" "net/http/httptest" "strconv" @@ -14,768 +13,400 @@ import ( "github.com/strahe/synaps3/internal/config" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagereplacement" + taskengine "github.com/strahe/synaps3/internal/task" "github.com/strahe/synaps3/internal/testutil" - "github.com/uptrace/bun" ) -type failingTaskProgressUploadRepo struct { - repository.StorageUploadRepository +type adminTaskHandler struct { + definition taskengine.Definition } -func (r failingTaskProgressUploadRepo) GetByIDs(_ context.Context, _ []int64) (map[int64]model.StorageUpload, error) { - return nil, errors.New("progress lookup failed") +func (h adminTaskHandler) Definition() taskengine.Definition { return h.definition } +func (adminTaskHandler) Execute(context.Context, taskengine.Execution) taskengine.Result { + return taskengine.Complete("", nil) } -func TestAPIListExhaustedUsesTaskListDTO(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - task := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "object", - RefID: 1, - RefVersionID: "01J000000000000000TASKE1", - IdempotencyKey: "api-exhausted-list", - Status: model.TaskStatusExhausted, - ScheduledAt: time.Now(), - MaxRetries: 3, - RetryCount: 3, - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task: %v", err) - } - - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()) - mux := http.NewServeMux() - mux.HandleFunc("GET /admin/exhausted-tasks", srv.handleListExhausted) +func (adminTaskHandler) Recover(context.Context, taskengine.Execution) taskengine.Result { + return taskengine.Complete("", nil) +} - rr := httptest.NewRecorder() - mux.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/admin/exhausted-tasks", nil)) - if rr.Code != http.StatusOK { - t.Fatalf("status = %d, want 200; body=%s", rr.Code, rr.Body.String()) - } +func newAdminTestTaskService(t *testing.T, repos *repository.Repositories) *taskengine.Service { + t.Helper() - var tasks []taskListItem - if err := json.NewDecoder(rr.Body).Decode(&tasks); err != nil { - t.Fatalf("Decode: %v", err) - } - if len(tasks) != 1 { - t.Fatalf("task count = %d, want 1", len(tasks)) - } - if tasks[0].ID != task.ID { - t.Fatalf("task id = %d, want %d", tasks[0].ID, task.ID) - } - if tasks[0].Status != string(model.TaskStatusExhausted) { - t.Fatalf("task status = %q, want exhausted", tasks[0].Status) + registry := taskengine.NewRegistry() + limit := 5 + for _, taskType := range []model.TaskType{ + model.TaskTypeBucketProvision, + model.TaskTypeUploadPlan, + model.TaskTypeStorageDataSetEnsure, + model.TaskTypeStorageTransferPlan, + model.TaskTypeStorageStore, + model.TaskTypeStoragePull, + model.TaskTypeStorageCommitCoordinate, + model.TaskTypeStorageCommit, + model.TaskTypeProviderReplacementCoordinate, + model.TaskTypeCacheCapacityReconcile, + model.TaskTypeCacheEvict, + model.TaskTypeCacheReconcileDurability, + model.TaskTypeStorageCleanup, + model.TaskTypeStorageDataSetRetire, + model.TaskTypeWalletOperation, + model.TaskTypeObservabilityRefresh, + model.TaskTypeGC, + } { + definition := taskengine.Definition{ + Type: taskType, + InputVersion: 1, + Codec: taskengine.StrictJSONCodec[map[string]any](nil), + RetryLimit: &limit, + AllowRetry: taskType != model.TaskTypeProviderReplacementCoordinate, + } + if taskType == model.TaskTypeWalletOperation { + definition.CanManualRetry = func(task *model.Task) bool { + return task.FailureReason != nil && *task.FailureReason == "wallet_broadcast_not_started" + } + } + if taskType == model.TaskTypeStorageStore { + definition.CanManualRetry = func(task *model.Task) bool { + return task.FailureReason != nil && (*task.FailureReason == "store_not_started" || *task.FailureReason == "store_outcome_unknown") + } + } + if taskType == model.TaskTypeStorageDataSetRetire { + definition.CanManualRetry = func(task *model.Task) bool { + return task.FailureReason == nil || *task.FailureReason != "termination_outcome_unknown" + } + } + if err := registry.Register(adminTaskHandler{definition: definition}); err != nil { + t.Fatalf("Register(%s): %v", definition.Type, err) + } } - if tasks[0].Type != string(model.TaskTypeUpload) { - t.Fatalf("task type = %q, want upload", tasks[0].Type) + service, err := taskengine.NewService(registry, repos, 7*24*time.Hour) + if err != nil { + t.Fatalf("NewService: %v", err) } + return service } -func TestAPIListTasksIncludesBucketNameForBucketReferences(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := testutil.SeedBucket(t, db, "replacement-task-bucket") - stage := "retire_abandoned_target" - task := &model.Task{ - Type: model.TaskTypeStorageCleanup, Stage: &stage, RefType: "bucket", RefID: bucket.ID, - IdempotencyKey: "replacement-task-bucket-ref", Status: model.TaskStatusCompleted, - ScheduledAt: time.Now(), CompletedAt: func() *time.Time { now := time.Now(); return &now }(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task: %v", err) - } - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()) - rr := httptest.NewRecorder() - srv.handleAPITasks(rr, httptest.NewRequest(http.MethodGet, "/api/v1/tasks", nil)) +func TestAPITasksUsesCursorAndServerPresentation(t *testing.T) { + fixture := newAdminTaskFixture(t) + now := time.Now() + waiting := fixture.enqueue(t, model.TaskTypeCacheEvict, "waiting", now, "object_version", "version-3") + fixture.transition(t, waiting.ID, repository.TaskTransition{ + Status: model.TaskStatusPending, ResumeMode: model.TaskResumeModeRecover, + AvailableAt: now.Add(time.Minute), WaitReason: new("durability"), + }) + queued := fixture.enqueue(t, model.TaskTypeUploadPlan, "queued", now, "object_version", "version-1") + scheduled := fixture.enqueue(t, model.TaskTypeUploadPlan, "scheduled", now.Add(time.Hour), "object_version", "version-2") + + rr := fixture.request(http.MethodGet, "/api/v1/tasks?type=upload_plan&status=pending&limit=1", nil) if rr.Code != http.StatusOK { - t.Fatalf("status = %d body=%s, want 200", rr.Code, rr.Body.String()) + t.Fatalf("status = %d body=%s", rr.Code, rr.Body.String()) } - var body taskListResponse - if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { - t.Fatalf("Decode: %v", err) + var first taskListResponse + decodeJSON(t, rr, &first) + if len(first.Tasks) != 1 || first.Tasks[0].ID != scheduled.ID || first.Tasks[0].Presentation != "scheduled" { + t.Fatalf("first page = %#v", first) } - if len(body.Tasks) != 1 || body.Tasks[0].BucketName != bucket.Name { - t.Fatalf("tasks = %#v, want bucket name %q", body.Tasks, bucket.Name) + if first.NextCursor == nil || *first.NextCursor != scheduled.ID { + t.Fatalf("next cursor = %v, want %d", first.NextCursor, scheduled.ID) } -} - -func TestAPIRetryExhaustedHTTPStatuses(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - task := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "object", - RefID: 2, - RefVersionID: "01J000000000000000TASKE2", - IdempotencyKey: "api-exhausted-retry", - Status: model.TaskStatusExhausted, - ScheduledAt: time.Now(), - MaxRetries: 3, - RetryCount: 3, - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task: %v", err) + if strings.Contains(rr.Body.String(), `"total"`) || strings.Contains(rr.Body.String(), `"input"`) || strings.Contains(rr.Body.String(), `"resume_mode"`) || strings.Contains(rr.Body.String(), `"claim_generation"`) { + t.Fatalf("response exposes removed or internal fields: %s", rr.Body.String()) } - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()) - mux := http.NewServeMux() - mux.HandleFunc("POST /admin/exhausted-tasks/{id}/retry", srv.handleRetryExhausted) - - for _, tc := range []struct { - name string - path string - wantStatus int - }{ - {name: "invalid id", path: "/admin/exhausted-tasks/not-a-number/retry", wantStatus: http.StatusBadRequest}, - {name: "missing task", path: "/admin/exhausted-tasks/999999/retry", wantStatus: http.StatusNotFound}, - {name: "exhausted task", path: "/admin/exhausted-tasks/" + strconv.FormatInt(task.ID, 10) + "/retry", wantStatus: http.StatusOK}, - } { - t.Run(tc.name, func(t *testing.T) { - rr := httptest.NewRecorder() - mux.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, tc.path, nil)) - if rr.Code != tc.wantStatus { - t.Fatalf("status = %d, want %d; body=%s", rr.Code, tc.wantStatus, rr.Body.String()) - } - }) + rr = fixture.request(http.MethodGet, "/api/v1/tasks?type=upload_plan&status=pending&limit=1&cursor="+strconv.FormatInt(*first.NextCursor, 10), nil) + var second taskListResponse + decodeJSON(t, rr, &second) + if len(second.Tasks) != 1 || second.Tasks[0].ID != queued.ID || second.Tasks[0].Presentation != "queued" || second.NextCursor != nil { + t.Fatalf("second page = %#v", second) } - got, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got == nil { - t.Fatal("retried task was not found") - } - if got.Status != model.TaskStatusQueued { - t.Fatalf("task status = %s, want queued", got.Status) + rr = fixture.request(http.MethodGet, "/api/v1/tasks?type=cache_evict", nil) + var cachePage taskListResponse + decodeJSON(t, rr, &cachePage) + if len(cachePage.Tasks) != 1 || cachePage.Tasks[0].Presentation != "waiting" || cachePage.Tasks[0].WaitReason == nil || *cachePage.Tasks[0].WaitReason != "durability" { + t.Fatalf("waiting task = %#v", cachePage) } } -func TestAPITaskStats(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - tasks := []*model.Task{ - { - Type: model.TaskTypeUpload, - RefType: "object", - RefID: 1, - RefVersionID: "01J000000000000000STAT01", - IdempotencyKey: "api-task-stats-queued-1", - Status: model.TaskStatusQueued, - }, - { - Type: model.TaskTypeUpload, - RefType: "object", - RefID: 2, - RefVersionID: "01J000000000000000STAT02", - IdempotencyKey: "api-task-stats-queued-2", - Status: model.TaskStatusQueued, - }, - { - Type: model.TaskTypeUpload, - RefType: "object", - RefID: 3, - RefVersionID: "01J000000000000000STAT03", - IdempotencyKey: "api-task-stats-running", - Status: model.TaskStatusRunning, - }, - } - for _, task := range tasks { - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task %q: %v", task.IdempotencyKey, err) +func TestAPITasksRejectsRemovedAndInvalidFilters(t *testing.T) { + fixture := newAdminTaskFixture(t) + for _, path := range []string{ + "/api/v1/tasks?category=upload", + "/api/v1/tasks?stage=store", + "/api/v1/tasks?offset=10", + "/api/v1/tasks?type=future_task_type", + "/api/v1/tasks?status=waiting", + "/api/v1/tasks?limit=0", + "/api/v1/tasks?cursor=nope", + } { + rr := fixture.request(http.MethodGet, path, nil) + if rr.Code != http.StatusBadRequest { + t.Fatalf("%s status = %d body=%s", path, rr.Code, rr.Body.String()) } } +} - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()) - rr := httptest.NewRecorder() - srv.handleAPITaskStats(rr, httptest.NewRequest(http.MethodGet, "/api/v1/tasks/stats", nil)) - if rr.Code != http.StatusOK { - t.Fatalf("status = %d, want 200; body=%s", rr.Code, rr.Body.String()) +func TestAPITaskStatsUsesPresentationStatusContract(t *testing.T) { + fixture := newAdminTaskFixture(t) + first := fixture.enqueue(t, model.TaskTypeUploadPlan, "one", time.Now(), "", "") + second := fixture.enqueue(t, model.TaskTypeUploadPlan, "two", time.Now(), "", "") + fixture.enqueue(t, model.TaskTypeUploadPlan, "three", time.Now(), "", "") + fixture.transition(t, first.ID, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, + FailureReason: new("provider_error"), LastError: new("provider unavailable"), + }) + fixture.transition(t, second.ID, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, + FailureReason: new("old_error"), LastError: new("old failure"), + }) + if err := fixture.repos.Tasks.AcknowledgeFailed(t.Context(), second.ID, time.Hour); err != nil { + t.Fatalf("AcknowledgeFailed: %v", err) } - var body []repository.TaskStatusCount - if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { - t.Fatalf("Decode stats: %v", err) + rr := fixture.request(http.MethodGet, "/api/v1/tasks/stats", nil) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", rr.Code, rr.Body.String()) } + var body []taskStatsItem + decodeJSON(t, rr, &body) counts := make(map[[2]string]int64, len(body)) - for _, count := range body { - counts[[2]string{count.Type, count.Status}] = count.Count + for _, item := range body { + counts[[2]string{item.Type, item.Status}] = item.Count } - if got := counts[[2]string{string(model.TaskTypeUpload), string(model.TaskStatusQueued)}]; got != 2 { - t.Fatalf("queued upload count = %d, want 2", got) - } - if got := counts[[2]string{string(model.TaskTypeUpload), string(model.TaskStatusRunning)}]; got != 1 { - t.Fatalf("running upload count = %d, want 1", got) + if counts[[2]string{string(model.TaskTypeUploadPlan), string(model.TaskStatusPending)}] != 1 || + counts[[2]string{string(model.TaskTypeUploadPlan), string(model.TaskStatusFailed)}] != 1 || + counts[[2]string{string(model.TaskTypeUploadPlan), "dismissed"}] != 1 { + t.Fatalf("counts = %#v", counts) } } -func TestAPITasksStageFilter(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - ingressCommit := "ingress_commit" - peerCommit := "peer_commit" - - for _, task := range []*model.Task{ - { - Type: model.TaskTypeUpload, - Stage: &ingressCommit, - RefType: "object", - RefID: 1, - RefVersionID: "01J000000000000000TASKA01", - IdempotencyKey: "api-stage-primary", - Status: model.TaskStatusQueued, - ScheduledAt: time.Now(), - }, - { - Type: model.TaskTypeUpload, - Stage: &peerCommit, - RefType: "object", - RefID: 2, - RefVersionID: "01J000000000000000TASKA02", - IdempotencyKey: "api-stage-secondary", - Status: model.TaskStatusQueued, - ScheduledAt: time.Now(), - }, - { - Type: model.TaskTypeEvictCache, - RefType: "object", - RefID: 3, - RefVersionID: "01J000000000000000TASKA03", - IdempotencyKey: "api-stage-evict", - Status: model.TaskStatusQueued, - ScheduledAt: time.Now(), - }, - } { - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task %q: %v", task.IdempotencyKey, err) - } - } - - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()) - mux := http.NewServeMux() - mux.HandleFunc("GET /api/v1/tasks", srv.handleAPITasks) - ts := httptest.NewServer(mux) - defer ts.Close() - - resp, err := http.Get(ts.URL + "/api/v1/tasks?type=upload&stage=ingress_commit&status=queued") - if err != nil { - t.Fatalf("GET tasks: %v", err) - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d, want 200", resp.StatusCode) - } - var body taskListResponse - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("Decode: %v", err) - } - if body.Total != 1 || len(body.Tasks) != 1 { - t.Fatalf("tasks = total:%d items:%#v, want one", body.Total, body.Tasks) +func TestAPITasksSeparatesFailedAndDismissedFilters(t *testing.T) { + fixture := newAdminTaskFixture(t) + failed := fixture.enqueue(t, model.TaskTypeUploadPlan, "visible-failure", time.Now(), "", "") + dismissed := fixture.enqueue(t, model.TaskTypeUploadPlan, "dismissed-failure", time.Now(), "", "") + for _, taskRow := range []*model.Task{failed, dismissed} { + fixture.transition(t, taskRow.ID, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, + FailureReason: new("provider_error"), LastError: new("provider unavailable"), + }) } - if body.Tasks[0].Stage == nil || *body.Tasks[0].Stage != ingressCommit { - t.Fatalf("stage = %#v, want ingress_commit", body.Tasks[0].Stage) + if err := fixture.repos.Tasks.AcknowledgeFailed(t.Context(), dismissed.ID, time.Hour); err != nil { + t.Fatalf("AcknowledgeFailed: %v", err) } - resp, err = http.Get(ts.URL + "/api/v1/tasks?stage=ingress_commit") - if err != nil { - t.Fatalf("GET tasks without type: %v", err) - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusBadRequest { - t.Fatalf("status = %d, want 400", resp.StatusCode) + for _, tt := range []struct { + status string + wantID int64 + presentation string + }{ + {status: "failed", wantID: failed.ID, presentation: "failed"}, + {status: "dismissed", wantID: dismissed.ID, presentation: "dismissed"}, + } { + rr := fixture.request(http.MethodGet, "/api/v1/tasks?status="+tt.status, nil) + if rr.Code != http.StatusOK { + t.Fatalf("status=%s response = %d %s", tt.status, rr.Code, rr.Body.String()) + } + var page taskListResponse + decodeJSON(t, rr, &page) + if len(page.Tasks) != 1 || page.Tasks[0].ID != tt.wantID || page.Tasks[0].Status != string(model.TaskStatusFailed) || page.Tasks[0].Presentation != tt.presentation { + t.Fatalf("status=%s page = %#v", tt.status, page) + } } } -func TestAPITasksReturnsWaitingDetails(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - waitReason := model.TaskWaitReasonDependency - statusMessage := "waiting for all copies to commit" - - task := &model.Task{ - Type: model.TaskTypeEvictCache, - RefType: "object", - RefID: 1, - RefVersionID: "01J000000000000000TASKW01", - IdempotencyKey: "api-task-waiting", - Status: model.TaskStatusWaiting, - WaitReason: &waitReason, - StatusMessage: &statusMessage, - ScheduledAt: time.Now().Add(time.Minute), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task: %v", err) - } - - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()) - mux := http.NewServeMux() - mux.HandleFunc("GET /api/v1/tasks", srv.handleAPITasks) - ts := httptest.NewServer(mux) - defer ts.Close() +func TestAPITasksHideHealthyRecurringSystemWorkButKeepFailures(t *testing.T) { + fixture := newAdminTaskFixture(t) + healthy := fixture.enqueue(t, model.TaskTypeCacheCapacityReconcile, "healthy-system", time.Now().Add(time.Hour), "system", "cache-capacity") + failed := fixture.enqueue(t, model.TaskTypeObservabilityRefresh, "failed-system", time.Now(), "system", "observability") + fixture.transition(t, failed.ID, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, + FailureReason: new("refresh_failed"), LastError: new("health refresh failed"), + }) - resp, err := http.Get(ts.URL + "/api/v1/tasks?status=waiting") - if err != nil { - t.Fatalf("GET tasks: %v", err) - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d, want 200", resp.StatusCode) - } - var body taskListResponse - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("Decode: %v", err) - } - if body.Total != 1 || len(body.Tasks) != 1 { - t.Fatalf("tasks = total:%d items:%#v, want one", body.Total, body.Tasks) - } - got := body.Tasks[0] - if got.Status != string(model.TaskStatusWaiting) { - t.Fatalf("task status = %s, want waiting", got.Status) - } - if got.WaitReason == nil || *got.WaitReason != string(waitReason) { - t.Fatalf("wait_reason = %v, want %s", got.WaitReason, waitReason) - } - if got.StatusMessage == nil || *got.StatusMessage != statusMessage { - t.Fatalf("status_message = %v, want %s", got.StatusMessage, statusMessage) - } - if got.LastError != nil { - t.Fatalf("last_error = %v, want nil", got.LastError) + rr := fixture.request(http.MethodGet, "/api/v1/tasks", nil) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", rr.Code, rr.Body.String()) } -} - -func TestAPITasksShowsLegacyPayloadStage(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - _, err := db.ExecContext(context.Background(), ` - INSERT INTO tasks (type, ref_type, ref_id, ref_version_id, idempotency_key, payload, status, scheduled_at) - VALUES ('upload', 'object', 1, '01J000000000000000TASKB01', 'legacy-payload-stage', '{"stage":"peer_pull","upload_id":9,"copy_index":1}', 'queued', CURRENT_TIMESTAMP) - `) - if err != nil { - t.Fatalf("insert legacy task: %v", err) + var page taskListResponse + decodeJSON(t, rr, &page) + if len(page.Tasks) != 1 || page.Tasks[0].ID != failed.ID { + t.Fatalf("visible tasks = %#v; healthy task %d should be hidden", page.Tasks, healthy.ID) } - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()) - mux := http.NewServeMux() - mux.HandleFunc("GET /api/v1/tasks", srv.handleAPITasks) - ts := httptest.NewServer(mux) - defer ts.Close() - - resp, err := http.Get(ts.URL + "/api/v1/tasks?type=upload&status=queued") - if err != nil { - t.Fatalf("GET tasks: %v", err) + rr = fixture.request(http.MethodGet, "/api/v1/tasks/stats", nil) + var stats []taskStatsItem + decodeJSON(t, rr, &stats) + if len(stats) != 1 || stats[0].Type != string(model.TaskTypeObservabilityRefresh) || stats[0].Status != string(model.TaskStatusFailed) { + t.Fatalf("visible task stats = %#v", stats) } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d, want 200", resp.StatusCode) + if err := fixture.repos.Tasks.AcknowledgeFailed(t.Context(), failed.ID, time.Hour); err != nil { + t.Fatalf("acknowledge recurring failure: %v", err) } - var body taskListResponse - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("Decode: %v", err) + rr = fixture.request(http.MethodGet, "/api/v1/tasks?status=dismissed", nil) + decodeJSON(t, rr, &page) + if len(page.Tasks) != 1 || page.Tasks[0].ID != failed.ID || page.Tasks[0].Presentation != "dismissed" { + t.Fatalf("dismissed recurring tasks = %#v", page.Tasks) } - if body.Total != 1 || len(body.Tasks) != 1 { - t.Fatalf("tasks = total:%d items:%#v, want one", body.Total, body.Tasks) - } - if body.Tasks[0].Stage == nil || *body.Tasks[0].Stage != "peer_pull" { - t.Fatalf("stage = %#v, want peer_pull", body.Tasks[0].Stage) + rr = fixture.request(http.MethodGet, "/api/v1/tasks/stats", nil) + decodeJSON(t, rr, &stats) + if len(stats) != 1 || stats[0].Type != string(model.TaskTypeObservabilityRefresh) || stats[0].Status != "dismissed" { + t.Fatalf("dismissed recurring task stats = %#v", stats) } } -func TestAPITasksIncludesPrimaryTransferProgress(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := testutil.SeedBucket(t, db, "task-progress-bucket") - objectID, versionID := seedAdminObjectVersion(t, repos, bucket, "uploading.txt", 20, "etag-progress", "checksum-progress", "text/plain", "", model.ObjectStateUploading) - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: 20, - Checksum: "checksum-progress", - RequestedCopies: 3, +func TestAPITaskRetryAndAcknowledgeFollowDefinition(t *testing.T) { + fixture := newAdminTaskFixture(t) + retryable := fixture.enqueue(t, model.TaskTypeUploadPlan, "retryable", time.Now(), "", "") + fixture.transition(t, retryable.ID, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, + FailureReason: new("temporary"), LastError: new("temporary failure"), IncrementRetry: true, + }) + nonRetryable := fixture.enqueue(t, model.TaskTypeWalletOperation, "wallet", time.Now(), "", "") + fixture.transition(t, nonRetryable.ID, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, + FailureReason: new("unknown_outcome"), LastError: new("transaction outcome unknown"), }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - progressUpload, err := repos.Uploads.BeginIngressStoreProgress(ctx, upload.ID) - if err != nil { - t.Fatalf("BeginIngressStoreProgress: %v", err) - } - if _, err := repos.Uploads.RecordIngressStoreProgress(ctx, repository.RecordIngressStoreProgressInput{ - UploadID: upload.ID, - Attempt: progressUpload.IngressStoreAttempt, - BytesUploaded: 5, - }); err != nil { - t.Fatalf("RecordIngressStoreProgress: %v", err) - } - stage := "ingress_store" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objectID, - RefVersionID: versionID, - IdempotencyKey: "task-primary-progress", - Payload: map[string]interface{}{"upload_id": upload.ID}, - Status: model.TaskStatusRunning, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task: %v", err) - } - - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()) - mux := http.NewServeMux() - mux.HandleFunc("GET /api/v1/tasks", srv.handleAPITasks) - ts := httptest.NewServer(mux) - defer ts.Close() - resp, err := http.Get(ts.URL + "/api/v1/tasks?type=upload&status=running") - if err != nil { - t.Fatalf("GET tasks: %v", err) - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d, want 200", resp.StatusCode) - } - var body taskListResponse - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("Decode: %v", err) + rr := fixture.request(http.MethodPost, "/api/v1/tasks/"+strconv.FormatInt(retryable.ID, 10)+"/retry", nil) + if rr.Code != http.StatusOK { + t.Fatalf("retry status = %d body=%s", rr.Code, rr.Body.String()) } - if len(body.Tasks) != 1 || body.Tasks[0].Progress == nil { - t.Fatalf("tasks = %#v, want primary transfer progress", body.Tasks) + got, err := fixture.repos.Tasks.GetByID(t.Context(), retryable.ID) + if err != nil || got == nil || got.Status != model.TaskStatusPending || got.ResumeMode != model.TaskResumeModeRecover || got.RetryCount != 0 { + t.Fatalf("retried task = %#v err=%v", got, err) } - progress := body.Tasks[0].Progress - if progress.Scope != "ingress_store" || progress.Attempt != progressUpload.IngressStoreAttempt || progress.UploadedBytes != 5 || progress.TotalBytes != 20 || progress.Percent == nil || *progress.Percent != 25 || progress.Done { - t.Fatalf("progress = %#v, want 5/20 primary transfer progress", progress) + + rr = fixture.request(http.MethodPost, "/api/v1/tasks/"+strconv.FormatInt(nonRetryable.ID, 10)+"/retry", nil) + if rr.Code != http.StatusConflict || !strings.Contains(rr.Body.String(), `"code":"task_retry_unsupported"`) { + t.Fatalf("unsupported retry status = %d body=%s", rr.Code, rr.Body.String()) } -} -func TestAPITasksIncludesProviderReplacementProgress(t *testing.T) { - fixture := newReplacementAPIFixture(t, &stubProviderSelector{providers: []string{"202"}}) - start := fixture.start(t, `{"mode":"manual","provider_id":"202"}`) - if start.Code != http.StatusCreated { - t.Fatalf("start replacement status=%d body=%s", start.Code, start.Body.String()) - } - replacement := decodeReplacement(t, start) - if err := fixture.srv.repos.Tasks.Create(context.Background(), storagereplacement.NewRetireTask( - replacement.ID, fixture.bucket.ID, 5, time.Now(), - )); err != nil { - t.Fatalf("create retirement task: %v", err) + rr = fixture.request(http.MethodPost, "/api/v1/tasks/"+strconv.FormatInt(nonRetryable.ID, 10)+"/acknowledge", nil) + if rr.Code != http.StatusOK { + t.Fatalf("acknowledge status = %d body=%s", rr.Code, rr.Body.String()) } - mux := http.NewServeMux() - mux.HandleFunc("GET /api/v1/tasks", fixture.srv.handleAPITasks) - req := httptest.NewRequest(http.MethodGet, "/api/v1/tasks", nil) - rec := httptest.NewRecorder() - mux.ServeHTTP(rec, req) - if rec.Code != http.StatusOK { - t.Fatalf("tasks status=%d body=%s", rec.Code, rec.Body.String()) - } - for _, field := range []string{ - `"seeding_complete":false`, `"items_total":0`, `"items_processed":0`, - `"items_waiting_source":0`, `"items_failed":0`, - } { - if !strings.Contains(rec.Body.String(), field) { - t.Fatalf("tasks body missing %s: %s", field, rec.Body.String()) - } + got, err = fixture.repos.Tasks.GetByID(t.Context(), nonRetryable.ID) + if err != nil || got == nil || got.AcknowledgedAt == nil || got.RetentionUntil == nil { + t.Fatalf("acknowledged task = %#v err=%v", got, err) } + rr = fixture.request(http.MethodGet, "/api/v1/tasks?status=failed", nil) var body taskListResponse - if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { - t.Fatalf("decode tasks: %v", err) - } - if len(body.Tasks) != 2 { - t.Fatalf("tasks = %#v, want migration and retirement coordinators", body.Tasks) - } - for i := range body.Tasks { - if body.Tasks[i].Progress == nil || body.Tasks[i].Progress.Scope != "provider_replacement" { - t.Fatalf("task %d progress = %#v, want provider replacement progress", body.Tasks[i].ID, body.Tasks[i].Progress) + decodeJSON(t, rr, &body) + for _, item := range body.Tasks { + if item.ID == nonRetryable.ID && item.Presentation != "dismissed" { + t.Fatalf("acknowledged task presentation = %q, want dismissed", item.Presentation) } } } -func TestAPITasksSkipsProgressWhenLookupFails(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - stage := "ingress_store" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: 1, - RefVersionID: "01J000000000000000TASKP01", - IdempotencyKey: "task-progress-lookup-fails", - Payload: map[string]interface{}{"upload_id": int64(42)}, - Status: model.TaskStatusRunning, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task: %v", err) - } - repos.Uploads = failingTaskProgressUploadRepo{StorageUploadRepository: repos.Uploads} - - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()) - mux := http.NewServeMux() - mux.HandleFunc("GET /api/v1/tasks", srv.handleAPITasks) - ts := httptest.NewServer(mux) - defer ts.Close() +func TestAPITaskFlagsComeFromRegistry(t *testing.T) { + fixture := newAdminTaskFixture(t) + retryable := fixture.enqueue(t, model.TaskTypeUploadPlan, "flags-retry", time.Now(), "", "") + fixture.transition(t, retryable.ID, repository.TaskTransition{Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover}) + wallet := fixture.enqueue(t, model.TaskTypeWalletOperation, "flags-wallet", time.Now(), "", "") + fixture.transition(t, wallet.ID, repository.TaskTransition{Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover}) + replacement := fixture.enqueue(t, model.TaskTypeProviderReplacementCoordinate, "flags-replacement", time.Now(), "", "") + fixture.transition(t, replacement.ID, repository.TaskTransition{Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover}) + retirement := fixture.enqueue(t, model.TaskTypeStorageDataSetRetire, "flags-retirement", time.Now(), "", "") + fixture.transition(t, retirement.ID, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, + FailureReason: new("termination_outcome_unknown"), + }) - resp, err := http.Get(ts.URL + "/api/v1/tasks?type=upload&status=running") - if err != nil { - t.Fatalf("GET tasks: %v", err) - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - t.Fatalf("status = %d, want 200", resp.StatusCode) - } + rr := fixture.request(http.MethodGet, "/api/v1/tasks?status=failed", nil) var body taskListResponse - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("Decode: %v", err) + decodeJSON(t, rr, &body) + if len(body.Tasks) != 4 { + t.Fatalf("tasks = %#v", body.Tasks) } - if len(body.Tasks) != 1 { - t.Fatalf("tasks = %#v, want one task", body.Tasks) + byType := make(map[string]taskListItem, len(body.Tasks)) + for _, item := range body.Tasks { + byType[item.Type] = item } - if body.Tasks[0].Progress != nil { - t.Fatalf("progress = %#v, want omitted progress after lookup failure", body.Tasks[0].Progress) + if !byType[string(model.TaskTypeUploadPlan)].Retryable || !byType[string(model.TaskTypeUploadPlan)].Acknowledgeable { + t.Fatalf("upload flags = %#v", byType[string(model.TaskTypeUploadPlan)]) } -} - -func TestAPITaskRefDetailObject(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := testutil.SeedBucket(t, db, "task-ref-bucket") - objectID, versionID := seedAdminObjectVersion(t, repos, bucket, "folder/file.txt", 123, "etag", "checksum", "text/plain", "", model.ObjectStateStored) - task := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "object", - RefID: objectID, - RefVersionID: versionID, - IdempotencyKey: "task-ref-object", - Status: model.TaskStatusCompleted, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task: %v", err) - } - - body, status := getTaskRefDetail(t, db, repos, task.ID) - if status != http.StatusOK { - t.Fatalf("status = %d, want 200", status) - } - if body.RefType != "object" || body.RefID != objectID || body.RefVersionID != versionID { - t.Fatalf("ref detail = %#v, want object ref", body) - } - if body.Object == nil { - t.Fatal("object detail is nil") + if byType[string(model.TaskTypeWalletOperation)].Retryable || !byType[string(model.TaskTypeWalletOperation)].Acknowledgeable { + t.Fatalf("wallet flags = %#v", byType[string(model.TaskTypeWalletOperation)]) } - if body.Object.BucketName != bucket.Name || body.Object.Key != "folder/file.txt" || body.Object.VersionID != versionID { - t.Fatalf("object detail = %#v, want bucket/key/version", body.Object) + if byType[string(model.TaskTypeProviderReplacementCoordinate)].Retryable || + !byType[string(model.TaskTypeProviderReplacementCoordinate)].Acknowledgeable { + t.Fatalf("replacement flags = %#v", byType[string(model.TaskTypeProviderReplacementCoordinate)]) } - if body.Object.Size != 123 || body.Object.State != string(model.ObjectStateStored) || body.Object.Status != "success" { - t.Fatalf("object detail = %#v, want stored success size 123", body.Object) + if byType[string(model.TaskTypeStorageDataSetRetire)].Retryable || + !byType[string(model.TaskTypeStorageDataSetRetire)].Acknowledgeable { + t.Fatalf("retirement flags = %#v", byType[string(model.TaskTypeStorageDataSetRetire)]) } - if !body.Object.Location.Cache || !body.Object.Location.Filecoin { - t.Fatalf("location = %#v, want cache and filecoin", body.Object.Location) - } - rawBody, rawStatus := getTaskRefDetailUploadStatus(t, db, repos, task.ID) - if rawStatus != http.StatusOK { - t.Fatalf("raw status = %d, want 200", rawStatus) - } - if rawBody.Object == nil || rawBody.Object.UploadStatus != string(model.StorageUploadStatusComplete) { - t.Fatalf("task object upload_status = %#v, want complete", rawBody.Object) + rr = fixture.request(http.MethodPost, "/api/v1/tasks/"+strconv.FormatInt(retirement.ID, 10)+"/retry", nil) + if rr.Code != http.StatusConflict || !strings.Contains(rr.Body.String(), `"code":"task_retry_unsupported"`) { + t.Fatalf("retirement retry status = %d body=%s", rr.Code, rr.Body.String()) } } -func TestAPITaskRefDetailNotFound(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - if _, status := getTaskRefDetail(t, db, repos, 999); status != http.StatusNotFound { - t.Fatalf("missing task status = %d, want 404", status) - } - - task := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "object", - RefID: 123, - RefVersionID: "01J000000000000000MISSING", - IdempotencyKey: "task-ref-missing-object", - Status: model.TaskStatusQueued, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(context.Background(), task); err != nil { - t.Fatalf("Create task: %v", err) - } - if _, status := getTaskRefDetail(t, db, repos, task.ID); status != http.StatusNotFound { - t.Fatalf("missing object status = %d, want 404", status) - } +type adminTaskFixture struct { + t *testing.T + repos *repository.Repositories + service *taskengine.Service + server *Server } -func TestAPITaskRefDetailStorageCleanupIncludesDeletedVersionSnapshot(t *testing.T) { +func newAdminTaskFixture(t *testing.T) *adminTaskFixture { + t.Helper() db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := testutil.SeedBucket(t, db, "task-ref-storage-cleanup-bucket") - uploadID := int64(98) - versionID := "01J000000000000000DELETE1" - now := time.Now().UTC() - - task := &model.Task{ - Type: model.TaskTypeStorageCleanup, - RefType: "storage_upload", - RefID: uploadID, - RefVersionID: "", - IdempotencyKey: "task-ref-storage-cleanup", - Payload: map[string]interface{}{ - "storage_upload_id": uploadID, - "deleted_source_version": versionID, - "deleted_source_versions": []string{versionID}, - }, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: now, - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task: %v", err) - } - deletion := &model.ObjectDeletion{ - BucketID: bucket.ID, - ObjectID: 55, - Key: "folder/deleted.txt", - VersionID: versionID, - CacheKey: "cache/deleted", - StorageUploadID: &uploadID, - Size: 456, - Checksum: "checksum-deleted", - CacheCleanupStatus: model.CacheCleanupStatusSkipped, - CreatedAt: now, - UpdatedAt: now, - DeletedAt: now, - } - if _, err := db.NewInsert().Model(deletion).Exec(ctx); err != nil { - t.Fatalf("Insert object deletion: %v", err) - } - copy := &model.StorageCleanupCopy{ - TaskID: task.ID, - UploadID: uploadID, - CopyIndex: 0, - PieceCID: "bafkzcibedeleted", - Status: model.StorageCleanupCopyStatusPending, - CreatedAt: now, - UpdatedAt: now, - ScheduledAt: &now, - } - if _, err := db.NewInsert().Model(copy).Exec(ctx); err != nil { - t.Fatalf("Insert storage cleanup copy: %v", err) - } + service := newAdminTestTaskService(t, repos) + server := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()).WithTaskService(service) + return &adminTaskFixture{t: t, repos: repos, service: service, server: server} +} - body, status := getTaskRefDetail(t, db, repos, task.ID) - if status != http.StatusOK { - t.Fatalf("status = %d, want 200", status) - } - if body.StorageCleanup == nil { - t.Fatal("storage cleanup detail is nil") - } - if body.StorageCleanup.UploadID != uploadID || len(body.StorageCleanup.Copies) != 1 { - t.Fatalf("storage cleanup detail = %#v, want upload and copy snapshot", body.StorageCleanup) - } - if len(body.StorageCleanup.DeletedVersions) != 1 { - t.Fatalf("deleted versions = %#v, want one deleted version snapshot", body.StorageCleanup.DeletedVersions) - } - deleted := body.StorageCleanup.DeletedVersions[0] - if deleted.BucketName != bucket.Name || deleted.Key != "folder/deleted.txt" || deleted.VersionID != versionID { - t.Fatalf("deleted version = %#v, want bucket/key/version snapshot", deleted) - } - if deleted.Size != 456 || deleted.DeletedAt == "" { - t.Fatalf("deleted version = %#v, want size and deletion timestamp", deleted) +func (f *adminTaskFixture) enqueue(t *testing.T, taskType model.TaskType, key string, availableAt time.Time, subjectType, subjectKey string) *model.Task { + t.Helper() + row, _, err := f.service.Enqueue(t.Context(), taskengine.EnqueueRequest{ + Type: taskType, IdempotencyKey: key, Input: map[string]any{"key": key}, + AvailableAt: availableAt, SubjectType: subjectType, SubjectKey: subjectKey, + }) + if err != nil { + t.Fatalf("Enqueue(%s): %v", key, err) } + return row } -func TestAPITaskRefDetailNonObject(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - bucket := testutil.SeedBucket(t, db, "task-ref-non-object-bucket") - task := &model.Task{ - Type: model.TaskTypeEvictCache, - RefType: "bucket", - RefID: bucket.ID, - RefVersionID: "", - IdempotencyKey: "task-ref-bucket", - Status: model.TaskStatusQueued, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(context.Background(), task); err != nil { - t.Fatalf("Create task: %v", err) +func (f *adminTaskFixture) transition(t *testing.T, id int64, transition repository.TaskTransition) { + t.Helper() + claimed, err := f.repos.Tasks.ClaimNext(t.Context(), time.Minute) + if err != nil { + t.Fatalf("ClaimNext: %v", err) } - - body, status := getTaskRefDetail(t, db, repos, task.ID) - if status != http.StatusOK { - t.Fatalf("status = %d, want 200", status) + if claimed == nil || claimed.ID != id { + t.Fatalf("claimed = %#v, want task %d", claimed, id) } - if body.RefType != "bucket" || body.RefID != bucket.ID || body.BucketName != bucket.Name || body.Object != nil { - t.Fatalf("ref detail = %#v, want bucket ref without object", body) + if err := f.repos.Tasks.Settle(t.Context(), claimed.ID, claimed.ClaimGeneration, transition); err != nil { + t.Fatalf("Settle(%d): %v", id, err) } } -func getTaskRefDetail(t *testing.T, db *bun.DB, repos *repository.Repositories, taskID int64) (taskRefDetailResponse, int) { - t.Helper() - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()) +func (f *adminTaskFixture) request(method, path string, body *strings.Reader) *httptest.ResponseRecorder { + f.t.Helper() mux := http.NewServeMux() - mux.HandleFunc("GET /api/v1/tasks/{id}/ref-detail", srv.handleAPITaskRefDetail) - ts := httptest.NewServer(mux) - defer ts.Close() - - resp, err := http.Get(ts.URL + "/api/v1/tasks/" + strconv.FormatInt(taskID, 10) + "/ref-detail") - if err != nil { - t.Fatalf("GET task ref detail: %v", err) - } - defer func() { _ = resp.Body.Close() }() - var body taskRefDetailResponse - if resp.StatusCode == http.StatusOK { - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("Decode: %v", err) - } + mux.HandleFunc("GET /api/v1/tasks", f.server.handleAPITasks) + mux.HandleFunc("GET /api/v1/tasks/stats", f.server.handleAPITaskStats) + mux.HandleFunc("POST /api/v1/tasks/{id}/retry", f.server.handleAPITaskRetry) + mux.HandleFunc("POST /api/v1/tasks/{id}/acknowledge", f.server.handleAPITaskAcknowledge) + var requestBody *strings.Reader + if body != nil { + requestBody = body + } else { + requestBody = strings.NewReader("") } - return body, resp.StatusCode + rr := httptest.NewRecorder() + mux.ServeHTTP(rr, httptest.NewRequest(method, path, requestBody)) + return rr } -func getTaskRefDetailUploadStatus(t *testing.T, db *bun.DB, repos *repository.Repositories, taskID int64) (struct { - Object *struct { - UploadStatus string `json:"upload_status"` - } `json:"object"` -}, int, -) { +func decodeJSON(t *testing.T, rr *httptest.ResponseRecorder, target any) { t.Helper() - srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()) - mux := http.NewServeMux() - mux.HandleFunc("GET /api/v1/tasks/{id}/ref-detail", srv.handleAPITaskRefDetail) - ts := httptest.NewServer(mux) - defer ts.Close() - - resp, err := http.Get(ts.URL + "/api/v1/tasks/" + strconv.FormatInt(taskID, 10) + "/ref-detail") - if err != nil { - t.Fatalf("GET task ref detail upload status: %v", err) - } - defer func() { _ = resp.Body.Close() }() - var body struct { - Object *struct { - UploadStatus string `json:"upload_status"` - } `json:"object"` - } - if resp.StatusCode == http.StatusOK { - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { - t.Fatalf("Decode upload status: %v", err) - } + if err := json.NewDecoder(rr.Body).Decode(target); err != nil { + t.Fatalf("Decode: %v; body=%s", err, rr.Body.String()) } - return body, resp.StatusCode } diff --git a/internal/admin/api_wallet.go b/internal/admin/api_wallet.go index 99c343e..abdb327 100644 --- a/internal/admin/api_wallet.go +++ b/internal/admin/api_wallet.go @@ -14,6 +14,8 @@ import ( "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/synapse" + taskengine "github.com/strahe/synaps3/internal/task" + "github.com/strahe/synaps3/internal/walletoperation" ) // --- Response DTOs --- @@ -97,7 +99,6 @@ type walletOperationDTO struct { Status string `json:"status"` TxHash *string `json:"tx_hash,omitempty"` LastError *string `json:"last_error,omitempty"` - LeaseUntil *string `json:"lease_until,omitempty"` StartedAt *string `json:"started_at,omitempty"` SubmittedAt *string `json:"submitted_at,omitempty"` CompletedAt *string `json:"completed_at,omitempty"` @@ -169,11 +170,11 @@ func (s *Server) handleAPIWallet(w http.ResponseWriter, r *http.Request) { resp.PartialErrors["task_counts"] = "database query failed" } for _, tc := range taskCounts { - if model.TaskType(tc.Type) != model.TaskTypeUpload { + if !walletStorageTaskType(model.TaskType(tc.Type)) { continue } switch tc.Status { - case string(model.TaskStatusQueued), string(model.TaskStatusScheduled), string(model.TaskStatusWaiting), string(model.TaskStatusRunning): + case string(model.TaskStatusPending), string(model.TaskStatusRunning): biz.OnchainTasksPending += int(tc.Count) case string(model.TaskStatusCompleted): biz.OnchainTasksCompleted += int(tc.Count) @@ -184,6 +185,17 @@ func (s *Server) handleAPIWallet(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, resp) } +func walletStorageTaskType(taskType model.TaskType) bool { + switch taskType { + case model.TaskTypeStorageDataSetEnsure, model.TaskTypeStorageStore, model.TaskTypeStoragePull, + model.TaskTypeStorageCommit, model.TaskTypeStorageDataSetRetire, + model.TaskTypeProviderReplacementCoordinate: + return true + default: + return false + } +} + func (s *Server) handleAPIWalletFund(w http.ResponseWriter, r *http.Request) { s.handleAPIWalletOperation(w, r, model.WalletOperationTypeFund) } @@ -255,10 +267,34 @@ func (s *Server) handleAPIWalletOperation(w http.ResponseWriter, r *http.Request return } - op, _, err := s.repos.WalletOperations.CreateOrGet(r.Context(), repository.CreateWalletOperationInput{ - Type: opType, - ClientRequestID: clientRequestID, - Amount: amount, + if s.taskService == nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "wallet operations unavailable"}) + return + } + var op *model.WalletOperation + err = s.repos.WithTx(r.Context(), func(txRepos *repository.Repositories) error { + var createErr error + op, _, createErr = txRepos.WalletOperations.CreateOrGet(r.Context(), repository.CreateWalletOperationInput{ + Type: opType, ClientRequestID: clientRequestID, Amount: amount, + }) + if createErr != nil || op == nil || op.TaskID != nil || op.Status != model.WalletOperationStatusPending { + return createErr + } + taskRow, _, enqueueErr := s.taskService.EnqueueInTransaction(r.Context(), txRepos, taskengine.EnqueueRequest{ + Type: model.TaskTypeWalletOperation, + IdempotencyKey: walletoperation.TaskKey(op.ID), + Input: walletoperation.Input{OperationID: op.ID}, + SubjectType: "wallet_operation", + SubjectKey: strconv.FormatInt(op.ID, 10), + }) + if enqueueErr != nil { + return enqueueErr + } + if bindErr := txRepos.WalletOperations.BindTask(r.Context(), op.ID, taskRow.ID); bindErr != nil { + return bindErr + } + op.TaskID = &taskRow.ID + return nil }) if err != nil { if errors.Is(err, repository.ErrWalletOperationConflict) { @@ -352,7 +388,6 @@ func walletOperationToDTO(op *model.WalletOperation) walletOperationDTO { Status: string(op.Status), TxHash: op.TxHash, LastError: op.LastError, - LeaseUntil: timeToString(op.LeaseUntil), StartedAt: timeToString(op.StartedAt), SubmittedAt: timeToString(op.SubmittedAt), CompletedAt: timeToString(op.CompletedAt), diff --git a/internal/admin/api_wallet_test.go b/internal/admin/api_wallet_test.go index 5a5a3dc..cd5e7e5 100644 --- a/internal/admin/api_wallet_test.go +++ b/internal/admin/api_wallet_test.go @@ -9,7 +9,6 @@ import ( "net/http/httptest" "strings" "testing" - "time" "github.com/strahe/synaps3/internal/config" "github.com/strahe/synaps3/internal/db/repository" @@ -33,23 +32,16 @@ func TestHandleAPIWallet_ReturnsStructuredWalletResponse(t *testing.T) { ctx := context.Background() bucket := &model.Bucket{ - Name: "wallet-proofset", - Status: model.BucketStatusActive, + Name: "wallet-proofset", + Status: model.BucketStatusActive, + DefaultCopies: 1, MinimumDurableCopies: 1, } if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("creating bucket: %v", err) } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - ContentSize: 1, - Checksum: "wallet-checksum", - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("start upload attempt: %v", err) - } + upload := insertAdminStorageContentSnapshot(t, db, bucket.ID, "01J000000000000000WALLET0", 1, "wallet-checksum", 1) pieceCID := "piece-wallet" - seedAdminCommittedCopies(t, repos, bucket.ID, upload.ID, pieceCID, []adminStorageCopySeed{{ + seedAdminCommittedCopies(t, db, repos, bucket.ID, upload.ID, pieceCID, []adminStorageCopySeed{{ ProviderID: onChainID(t, "101"), DataSetID: onChainID(t, "1001"), PieceID: onChainIDPtr(t, "1"), @@ -57,43 +49,10 @@ func TestHandleAPIWallet_ReturnsStructuredWalletResponse(t *testing.T) { RetrievalURL: "https://provider.example/wallet", }}) - tasks := []*model.Task{ - { - Type: model.TaskTypeUpload, - RefType: "object", - RefID: 1, - RefVersionID: "01J000000000000000WALLET1", - IdempotencyKey: "wallet-upload-pending", - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - }, - { - Type: model.TaskTypeUpload, - RefType: "object", - RefID: 2, - RefVersionID: "01J000000000000000WALLET2", - IdempotencyKey: "wallet-upload-completed", - Status: model.TaskStatusCompleted, - MaxRetries: 5, - ScheduledAt: time.Now(), - }, - { - Type: model.TaskTypeEvictCache, - RefType: "object", - RefID: 3, - RefVersionID: "01J000000000000000WALLET3", - IdempotencyKey: "wallet-evict-pending", - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - }, - } - for _, task := range tasks { - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("creating task %q: %v", task.IdempotencyKey, err) - } - } + taskService := newAdminTestTaskService(t, repos) + overviewSeedTask(t, taskService, repos, model.TaskTypeStorageStore, "wallet-store-pending", model.TaskStatusPending) + overviewSeedTask(t, taskService, repos, model.TaskTypeStorageCommit, "wallet-commit-completed", model.TaskStatusCompleted) + overviewSeedTask(t, taskService, repos, model.TaskTypeCacheEvict, "wallet-evict-pending", model.TaskStatusPending) nonce := uint64(7) srv := newTestServer(":0", db, &stubCache{rootDir: t.TempDir()}, 1<<20, repos, nil, &stubWalletQuerier{ @@ -120,7 +79,7 @@ func TestHandleAPIWallet_ReturnsStructuredWalletResponse(t *testing.T) { LockupRatePerMonth: big.NewInt(172800), }, }, - }, config.DefaultFilecoinCopies, testLogger()) + }, config.DefaultFilecoinCopies, testLogger()).WithTaskService(taskService) req := httptest.NewRequest(http.MethodGet, "/api/v1/wallet", nil) rr := httptest.NewRecorder() @@ -393,6 +352,6 @@ func newWalletOperationTestServer(t *testing.T) (*Server, *repository.Repositori repos := repository.NewRepositories(db) srv := newTestServer("127.0.0.1:0", db, &stubCache{rootDir: t.TempDir()}, 1<<20, repos, nil, &stubWalletQuerier{ info: &synapse.WalletInfo{Address: "0xabc"}, - }, config.DefaultFilecoinCopies, testLogger()) + }, config.DefaultFilecoinCopies, testLogger()).WithTaskService(newAdminTestTaskService(t, repos)) return srv, repos } diff --git a/internal/admin/events_test.go b/internal/admin/events_test.go index 78e0d7a..52f23c5 100644 --- a/internal/admin/events_test.go +++ b/internal/admin/events_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "slices" "strings" "sync" "testing" @@ -50,7 +51,7 @@ func TestAdminEventHubDoesNotBlockOnSlowSubscribers(t *testing.T) { fast, unsubscribeFast := hub.subscribe() defer unsubscribeFast() - for i := 0; i < adminEventSubscriberBuffer; i++ { + for range adminEventSubscriberBuffer { hub.publish("provider_identity_updated", map[string]any{"provider_id": "101"}) select { case <-fast: @@ -193,7 +194,7 @@ func TestAdminEventsHandlerStreamsUploadProgressEvent(t *testing.T) { } events.Publish("upload_progress_updated", map[string]any{ - "upload_id": int64(11), + "content_id": int64(11), "version_id": "01J000000000000000PROG01", "bucket_name": "photos", "object_key": "image.jpg", @@ -218,7 +219,7 @@ func TestAdminEventsHandlerStreamsUploadProgressEvent(t *testing.T) { var payload struct { Seq uint64 `json:"seq"` Topic string `json:"topic"` - UploadID int64 `json:"upload_id"` + ContentID int64 `json:"content_id"` VersionID string `json:"version_id"` Progress struct { Scope string `json:"scope"` @@ -232,7 +233,7 @@ func TestAdminEventsHandlerStreamsUploadProgressEvent(t *testing.T) { if err := json.Unmarshal([]byte(strings.TrimPrefix(strings.TrimSpace(dataLine), "data: ")), &payload); err != nil { t.Fatalf("Unmarshal SSE data: %v", err) } - if payload.Seq != 1 || payload.Topic != "upload_progress_updated" || payload.UploadID != 11 || payload.VersionID != "01J000000000000000PROG01" || payload.Progress.Percent == nil || *payload.Progress.Percent != 40 { + if payload.Seq != 1 || payload.Topic != "upload_progress_updated" || payload.ContentID != 11 || payload.VersionID != "01J000000000000000PROG01" || payload.Progress.Percent == nil || *payload.Progress.Percent != 40 { t.Fatalf("payload = %#v, want upload progress event payload", payload) } } @@ -271,12 +272,7 @@ func readSSEEventLines(t *testing.T, reader *bufio.Reader) []string { } func containsLine(lines []string, want string) bool { - for _, line := range lines { - if line == want { - return true - } - } - return false + return slices.Contains(lines, want) } func findDataLine(lines []string) string { diff --git a/internal/admin/metrics.go b/internal/admin/metrics.go index cadeb1f..8fd3102 100644 --- a/internal/admin/metrics.go +++ b/internal/admin/metrics.go @@ -47,22 +47,6 @@ var ( Help: "Whether LRU cache eviction is paused because recent cache access could not be retained safely", }) - // WorkerTasksProcessed counts tasks processed by worker type and result. - WorkerTasksProcessed = promauto.NewCounterVec(prometheus.CounterOpts{ - Namespace: "synaps3", - Subsystem: "worker", - Name: "tasks_processed_total", - Help: "Total tasks processed by worker type and result", - }, []string{"worker", "result"}) - - // TasksExhaustedTotal counts tasks that entered exhausted status by worker type and task type. - TasksExhaustedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ - Namespace: "synaps3", - Subsystem: "worker", - Name: "tasks_exhausted_total", - Help: "Total tasks that entered exhausted status", - }, []string{"worker", "task_type"}) - // TaskQueueDepth tracks active task count by type and status. TaskQueueDepth = promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: "synaps3", @@ -71,15 +55,6 @@ var ( Help: "Number of tasks by type and status", }, []string{"type", "status"}) - // WorkerTaskDuration tracks per-task processing duration in seconds. - WorkerTaskDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "synaps3", - Subsystem: "worker", - Name: "task_duration_seconds", - Help: "Task processing duration in seconds by worker", - Buckets: prometheus.ExponentialBuckets(0.1, 2, 12), // 0.1s to ~200s - }, []string{"worker"}) - // ObjectStateDistribution tracks object count by state. ObjectStateDistribution = promauto.NewGaugeVec(prometheus.GaugeOpts{ Namespace: "synaps3", diff --git a/internal/admin/server.go b/internal/admin/server.go index e10c79e..b1823f4 100644 --- a/internal/admin/server.go +++ b/internal/admin/server.go @@ -22,8 +22,8 @@ import ( "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/objectreader" "github.com/strahe/synaps3/internal/observability" - "github.com/strahe/synaps3/internal/storagereplacement" "github.com/strahe/synaps3/internal/synapse" + taskengine "github.com/strahe/synaps3/internal/task" "github.com/strahe/synaps3/ui" "github.com/uptrace/bun" "github.com/versity/versitygw/auth" @@ -36,35 +36,31 @@ type WorkerHealthChecker interface { // Server provides /healthz and /metrics endpoints on a separate port. type Server struct { - addr string - db *bun.DB - cache cache.Cache - objectReader *objectreader.Reader - objectStorage synapse.StorageClient - cacheGate *cacheaccess.Gate - cacheAccessTracker *cacheaccess.Tracker - objectUploader objectUploader - objectVersionRestorer objectVersionRestorer - cacheMaxBytes int64 - repos *repository.Repositories - bucketLifecycle *bucketlifecycle.Service - workerHealth WorkerHealthChecker - wallet synapse.WalletQuerier - filecoinReadiness filecoinReadinessProbe - observability *observability.Service - taskDiagnosticChecker taskDiagnosticStatusChecker - providerIdentity providerIdentityLookup - events *EventHub - settings *SettingsService - auth *authService - trustedProxies []netip.Prefix - s3IAM auth.IAMService - s3RootAccess string - filecoinDefaultCopies int - evictMaxRetries int - storageCleanupMaxRetries int - uploadMaxRetries int - providerReplacementMaxRetries int + addr string + db *bun.DB + cache cache.Cache + objectReader *objectreader.Reader + objectStorage synapse.StorageClient + cacheGate *cacheaccess.Gate + cacheAccessTracker *cacheaccess.Tracker + objectUploader objectUploader + objectVersionRestorer objectVersionRestorer + cacheMaxBytes int64 + repos *repository.Repositories + taskService *taskengine.Service + bucketLifecycle *bucketlifecycle.Service + workerHealth WorkerHealthChecker + wallet synapse.WalletQuerier + filecoinReadiness filecoinReadinessProbe + observability *observability.Service + providerIdentity providerIdentityLookup + events *EventHub + settings *SettingsService + auth *authService + trustedProxies []netip.Prefix + s3IAM auth.IAMService + s3RootAccess string + filecoinDefaultCopies int // replacementSelector resolves an automatic replacement provider. Nil means // only an explicit Provider ID can be confirmed. replacementSelector providerReplacementSelector @@ -77,6 +73,12 @@ type Server struct { prevObjectLabels map[string]struct{} } +func (s *Server) WithTaskService(service *taskengine.Service) *Server { + s.taskService = service + s.bucketLifecycle.SetTaskService(service) + return s +} + // New creates a new admin HTTP server. func New( addr string, @@ -98,26 +100,21 @@ func New( panic("admin server requires a cache access tracker") } s := &Server{ - addr: addr, - db: db, - cache: c, - cacheGate: cacheGate, - cacheAccessTracker: cacheAccessTracker, - objectReader: objectreader.New(repos, c, nil, cacheGate, cacheAccessTracker, logger), - cacheMaxBytes: cacheMaxBytes, - repos: repos, - bucketLifecycle: bucketlifecycle.New(repos, c, logger), - workerHealth: wh, - wallet: newCachedWalletQuerier(wallet, walletCacheTTL, time.Now), - taskDiagnosticChecker: synapse.NewPDPStatusChecker(synapse.PDPStatusCheckerOptions{}), - events: newAdminEventHub(), - filecoinDefaultCopies: boundedBucketCopies(filecoinDefaultCopies), - evictMaxRetries: 5, - storageCleanupMaxRetries: 5, - uploadMaxRetries: 5, - providerReplacementMaxRetries: 5, - logger: logger, - startedAt: time.Now(), + addr: addr, + db: db, + cache: c, + cacheGate: cacheGate, + cacheAccessTracker: cacheAccessTracker, + objectReader: objectreader.New(repos, c, nil, cacheGate, cacheAccessTracker, logger), + cacheMaxBytes: cacheMaxBytes, + repos: repos, + bucketLifecycle: bucketlifecycle.New(repos, c, boundedBucketCopies(filecoinDefaultCopies), logger), + workerHealth: wh, + wallet: newCachedWalletQuerier(wallet, walletCacheTTL, time.Now), + events: newAdminEventHub(), + filecoinDefaultCopies: boundedBucketCopies(filecoinDefaultCopies), + logger: logger, + startedAt: time.Now(), } s.watchWalletOperationEvents() return s @@ -214,34 +211,6 @@ func (s *Server) WithS3IAM(iam auth.IAMService, rootAccess string) *Server { return s } -// WithUploadMaxRetries sets the retry budget replacement coordinators inherit. -func (s *Server) WithUploadMaxRetries(maxRetries int) *Server { - if maxRetries > 0 { - s.uploadMaxRetries = maxRetries - } - return s -} - -// WithProviderReplacementMaxRetries sets the provider replacement retry limit. -func (s *Server) WithProviderReplacementMaxRetries(maxRetries int) *Server { - if maxRetries >= 0 { - s.providerReplacementMaxRetries = maxRetries - } - return s -} - -// WithStorageCleanupMaxRetries configures max retries for storage cleanup tasks created by admin actions. -func (s *Server) WithStorageCleanupMaxRetries(maxRetries int) *Server { - s.storageCleanupMaxRetries = maxRetries - return s -} - -// WithEvictMaxRetries configures max retries for eviction tasks created by admin actions. -func (s *Server) WithEvictMaxRetries(maxRetries int) *Server { - s.evictMaxRetries = maxRetries - return s -} - // Run starts the admin HTTP server at its configured address. func (s *Server) Run(ctx context.Context) error { listener, err := net.Listen("tcp", s.addr) @@ -270,9 +239,6 @@ func (s *Server) Serve(ctx context.Context, listener net.Listener) error { } else { mux.HandleFunc("GET /healthz", s.handleHealthz) mux.Handle("GET /metrics", promhttp.Handler()) - mux.HandleFunc("GET /admin/exhausted-tasks", s.handleListExhausted) - mux.HandleFunc("POST /admin/exhausted-tasks/{id}/retry", s.handleRetryExhausted) - // Dashboard API mux.HandleFunc("GET /api/v1/overview", s.handleAPIOverview) mux.HandleFunc("GET /api/v1/events", s.handleAPIEvents) @@ -303,10 +269,8 @@ func (s *Server) Serve(ctx context.Context, listener net.Listener) error { mux.HandleFunc("POST /api/v1/buckets/{name}/objects/upload", s.handleAPIUploadObject) mux.HandleFunc("GET /api/v1/tasks", s.handleAPITasks) mux.HandleFunc("GET /api/v1/tasks/stats", s.handleAPITaskStats) - mux.HandleFunc("GET /api/v1/tasks/{id}/ref-detail", s.handleAPITaskRefDetail) - mux.HandleFunc("GET /api/v1/tasks/{id}/diagnostic", s.handleAPITaskDiagnostic) - mux.HandleFunc("POST /api/v1/tasks/{id}/diagnostic/refresh", s.handleAPITaskDiagnosticRefresh) - mux.HandleFunc("POST /api/v1/tasks/{id}/retry", s.handleRetryExhausted) // only retries exhausted tasks + mux.HandleFunc("POST /api/v1/tasks/{id}/retry", s.handleAPITaskRetry) + mux.HandleFunc("POST /api/v1/tasks/{id}/acknowledge", s.handleAPITaskAcknowledge) mux.HandleFunc("GET /api/v1/system/info", s.handleAPISystemInfo) mux.HandleFunc("GET /api/v1/workers", s.handleAPIWorkers) mux.HandleFunc("GET /api/v1/cache/stats", s.handleAPICacheStats) @@ -457,58 +421,54 @@ func (s *Server) cacheRootDir() string { return "" } -func (s *Server) handleListExhausted(w http.ResponseWriter, r *http.Request) { - const maxExhaustedLimit = 1000 - limit := 100 - if v := r.URL.Query().Get("limit"); v != "" { - if n, err := strconv.Atoi(v); err == nil && n > 0 { - limit = n - } - } - if limit > maxExhaustedLimit { - limit = maxExhaustedLimit - } - - tasks, err := s.repos.Tasks.ListExhausted(r.Context(), limit) - if err != nil { - s.logger.Error("failed to list exhausted tasks", "error", err) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) - return - } - - // Map to DTO to ensure consistent snake_case JSON and avoid exposing internal fields. - items := make([]taskListItem, 0, len(tasks)) - for i := range tasks { - items = append(items, taskListItemFromModel(&tasks[i], nil)) - } - - writeJSON(w, http.StatusOK, items) -} - -func (s *Server) handleRetryExhausted(w http.ResponseWriter, r *http.Request) { +func (s *Server) handleAPITaskRetry(w http.ResponseWriter, r *http.Request) { idStr := r.PathValue("id") id, err := strconv.ParseInt(idStr, 10, 64) - if err != nil { + if err != nil || id <= 0 { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid id"}) return } - - if err := s.repos.Tasks.RetryExhausted(r.Context(), id); err != nil { - s.logger.Error("failed to retry exhausted task", "taskID", id, "error", err) - if errors.Is(err, repository.ErrReplacementRetryUnsupported) { + if s.taskService == nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "task service unavailable"}) + return + } + if err := s.taskService.Retry(r.Context(), id); err != nil { + if errors.Is(err, taskengine.ErrRetryUnsupported) { writeJSON(w, http.StatusConflict, map[string]string{ - "error": "Retry this replacement from the bucket Details page, under Storage → Data Sets.", - "code": storagereplacement.CodeTaskRetryUnsupported, + "error": "This operation cannot be recovered from Tasks.", + "code": "task_retry_unsupported", }) } else if errors.Is(err, repository.ErrNotFound) { - writeJSON(w, http.StatusNotFound, map[string]string{"error": "not found or not in exhausted state"}) + writeJSON(w, http.StatusNotFound, map[string]string{"error": "failed task not found"}) } else { + s.logger.Error("api: failed to retry task", "taskID", id, "error", err) writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) } return } + writeJSON(w, http.StatusOK, map[string]string{"status": "pending"}) +} - writeJSON(w, http.StatusOK, map[string]string{"status": "requeued"}) +func (s *Server) handleAPITaskAcknowledge(w http.ResponseWriter, r *http.Request) { + id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) + if err != nil || id <= 0 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid id"}) + return + } + if s.taskService == nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "task service unavailable"}) + return + } + if err := s.taskService.Acknowledge(r.Context(), id); err != nil { + if errors.Is(err, repository.ErrNotFound) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "failed task not found"}) + } else { + s.logger.Error("api: failed to acknowledge task", "taskID", id, "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) + } + return + } + writeJSON(w, http.StatusOK, map[string]string{"status": "acknowledged"}) } func (s *Server) refreshMetricsLoop(ctx context.Context) { @@ -571,10 +531,7 @@ func (s *Server) refreshMetrics(ctx context.Context) { func isActiveTaskStatus(status string) bool { switch status { - case string(model.TaskStatusQueued), - string(model.TaskStatusScheduled), - string(model.TaskStatusWaiting), - string(model.TaskStatusRunning): + case string(model.TaskStatusPending), string(model.TaskStatusRunning): return true default: return false diff --git a/internal/admin/server_test.go b/internal/admin/server_test.go index 688221e..a58f2f8 100644 --- a/internal/admin/server_test.go +++ b/internal/admin/server_test.go @@ -9,7 +9,6 @@ import ( "net/http/httptest" "strings" "testing" - "time" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/strahe/synaps3/internal/cache" @@ -146,9 +145,7 @@ func TestMetrics_Endpoint(t *testing.T) { // Increment metrics so we can verify their presence. ObjectOperationsTotal.WithLabelValues("put", "success").Inc() - WorkerTasksProcessed.WithLabelValues("uploader", "success").Inc() - WorkerTaskDuration.WithLabelValues("uploader").Observe(0.5) - WorkerTaskDuration.WithLabelValues("uploader").Observe(0.5) + TaskQueueDepth.WithLabelValues("upload_plan", "pending").Set(1) mux := http.NewServeMux() mux.HandleFunc("GET /healthz", srv.handleHealthz) @@ -178,7 +175,7 @@ func TestMetrics_Endpoint(t *testing.T) { "synaps3_cache_used_bytes", "synaps3_cache_hits_total", "synaps3_cache_misses_total", - "synaps3_worker_tasks_processed_total", + "synaps3_task_queue_depth", } { if !strings.Contains(text, prefix) { t.Errorf("metrics output missing %q", prefix) @@ -326,36 +323,34 @@ func TestRefreshMetrics(t *testing.T) { ctx := context.Background() - // Seed a queued task. - task := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "object", - RefID: 1, - RefVersionID: "01J000000000000000TASK001", - IdempotencyKey: "test-refresh-task", - Status: model.TaskStatusQueued, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("seeding task: %v", err) - } + // Seed one pending storage operation. + taskService := newAdminTestTaskService(t, repos) + overviewSeedTask(t, taskService, repos, model.TaskTypeStorageStore, "test-refresh-task", model.TaskStatusPending) // Seed an object. - bucket := &model.Bucket{Name: "metrics-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "metrics-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if _, err := db.NewInsert().Model(bucket).Exec(ctx); err != nil { t.Fatalf("seeding bucket: %v", err) } + testutil.OpenBucketReplicaSlots(t, db, bucket.ID, bucket.DefaultCopies) versionID := model.NewVersionID() + content, err := repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, + ContentSize: 1, + Checksum: testutil.StorageChecksum("c"), + RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("seeding content: %v", err) + } version := &model.ObjectVersion{ VersionID: versionID, BucketID: bucket.ID, Key: "metrics.txt", + ContentID: &content.ID, Size: 1, ETag: "e", - Checksum: "c", ContentType: "text/plain", - CacheKey: ".versions/" + versionID, - State: model.ObjectStateCached, } if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { t.Fatalf("seeding object: %v", err) @@ -433,8 +428,6 @@ func TestWithSecurityHeadersSensitivePaths(t *testing.T) { "//api/v1/auth/session", "/admin/", "//admin/", - "/admin/exhausted-tasks", - "//admin/exhausted-tasks", "/metrics", "//metrics", "/healthz", diff --git a/internal/app/runtime.go b/internal/app/runtime.go index 52f4600..4ebc668 100644 --- a/internal/app/runtime.go +++ b/internal/app/runtime.go @@ -18,11 +18,13 @@ import ( "github.com/strahe/synaps3/internal/cacheaccess" "github.com/strahe/synaps3/internal/config" "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/observability" "github.com/strahe/synaps3/internal/s3access" "github.com/strahe/synaps3/internal/s3iam" - "github.com/strahe/synaps3/internal/state" "github.com/strahe/synaps3/internal/synapse" + "github.com/strahe/synaps3/internal/systemtask" + taskengine "github.com/strahe/synaps3/internal/task" "github.com/strahe/synaps3/internal/worker" "github.com/uptrace/bun" "github.com/versity/versitygw/auth" @@ -82,7 +84,6 @@ type Runtime struct { accessTracker *cacheaccess.Tracker iam auth.IAMService workers *worker.Manager - observer *observability.Runner s3Addresses []string logger *slog.Logger shutdown time.Duration @@ -113,13 +114,88 @@ func NewRuntime(ctx context.Context, opts RuntimeOptions) (_ *Runtime, err error } cacheGate := cacheaccess.NewGate() accessTracker := cacheaccess.NewTracker(cacheaccess.DefaultPersistenceInterval, repos.Objects) - stateMachine := state.NewObjectStateMachine() events := admin.NewEventHub() - appBackend := backend.New(repos, localCache, stateMachine, opts.Filecoin.Storage, cacheGate, accessTracker, logger, - backend.WithUploadMaxRetries(cfg.Worker.Upload.MaxRetries), - backend.WithEvictMaxRetries(cfg.Worker.Evictor.MaxRetries), - backend.WithStorageCleanupMaxRetries(cfg.Worker.StorageCleanup.MaxRetries), + observabilityService := newObservabilityService(cfg, repos, opts.Filecoin.Observability) + pdpStatusChecker := synapse.NewPDPStatusChecker(synapse.PDPStatusCheckerOptions{ + Timeout: 15 * time.Second, + AllowPrivateNetworks: cfg.Filecoin.AllowPrivateNetworks, + }) + registry := taskengine.NewRegistry() + handlers, err := worker.NewTaskHandlers(worker.TaskHandlerDependencies{ + Repositories: repos, + Events: events, + Cache: localCache, + CacheGate: cacheGate, + CacheTracker: accessTracker, + Storage: opts.Filecoin.Storage, + Wallet: opts.Filecoin.Wallet, + Receipts: opts.Filecoin.Receipts, + Terminator: opts.Filecoin.Terminator, + Epochs: opts.Filecoin.Epochs, + Observability: observabilityService, + CommitStatus: pdpStatusChecker, + ParkedPieces: pdpStatusChecker, + EvictionPolicy: evictionPolicy, + MaxCacheBytes: maxCacheBytes, + LRUHighPercent: cfg.Cache.LRUHighWatermarkPercent, + LRULowPercent: cfg.Cache.LRULowWatermarkPercent, + DefaultCopies: cfg.Filecoin.DefaultCopies, + MaxRetries: cfg.Worker.Tasks.MaxRetries, + Logger: logger, + }) + if err != nil { + return nil, fmt.Errorf("initializing task handlers: %w", err) + } + if err := handlers.RegisterCore(registry); err != nil { + return nil, fmt.Errorf("registering core task handlers: %w", err) + } + if err := handlers.RegisterStorage(registry); err != nil { + return nil, fmt.Errorf("registering storage task handlers: %w", err) + } + if err := handlers.RegisterReplacement(registry); err != nil { + return nil, fmt.Errorf("registering replacement task handlers: %w", err) + } + taskService, err := taskengine.NewService(registry, repos, cfg.Worker.Tasks.Retention) + if err != nil { + return nil, fmt.Errorf("initializing task service: %w", err) + } + handlers.SetTaskService(taskService) + engine, err := taskengine.NewEngine(taskengine.EngineConfig{ + Concurrency: cfg.Worker.Tasks.Concurrency, + PollInterval: cfg.Worker.Tasks.PollInterval, + LeaseDuration: cfg.Worker.Tasks.LeaseDuration, + Retention: cfg.Worker.Tasks.Retention, + ProviderMutationConcurrency: cfg.Worker.Tasks.ProviderMutationConcurrency, + DestructiveMutationConcurrency: cfg.Worker.Tasks.DestructiveMutationConcurrency, + OnTaskSettled: func(taskRow *model.Task, transition repository.TaskTransition) { + publishUploadTaskSettlement(events, taskRow, transition) + }, + }, repos, registry, logger) + if err != nil { + return nil, fmt.Errorf("initializing task engine: %w", err) + } + for _, recurring := range []taskengine.EnqueueRequest{ + { + Type: model.TaskTypeCacheCapacityReconcile, IdempotencyKey: systemtask.CacheCapacityKey, + Input: systemtask.Input{}, SubjectType: "system", SubjectKey: "cache-capacity", + }, + { + Type: model.TaskTypeObservabilityRefresh, IdempotencyKey: "system:observability-refresh", + Input: systemtask.Input{}, SubjectType: "system", SubjectKey: "observability", + }, + { + Type: model.TaskTypeGC, IdempotencyKey: "system:task-gc", + Input: systemtask.Input{}, SubjectType: "system", SubjectKey: "task-gc", + }, + } { + if _, _, err := taskService.Enqueue(ctx, recurring); err != nil { + return nil, fmt.Errorf("seeding recurring task %s: %w", recurring.Type, err) + } + } + appBackend := backend.New(repos, localCache, opts.Filecoin.Storage, cacheGate, accessTracker, logger, + backend.WithTaskService(taskService), backend.WithEvictionPolicy(evictionPolicy), + backend.WithDefaultCopies(cfg.Filecoin.DefaultCopies), ) iamService := s3iam.NewService(repos) @@ -156,46 +232,10 @@ func NewRuntime(ctx context.Context, opts RuntimeOptions) (_ *Runtime, err error return nil, fmt.Errorf("creating S3 server: %w", err) } - observabilityService := newObservabilityService(cfg, repos, opts.Filecoin.Observability) - uploader := worker.NewUploader(repos, localCache, opts.Filecoin.Storage, opts.Filecoin.WalletQuery, stateMachine, evictionPolicy, - cfg.Filecoin.DefaultCopies, cfg.Worker.Upload.Concurrency, cfg.Worker.Upload.PollInterval, logger, - worker.WithEvictMaxRetries(cfg.Worker.Evictor.MaxRetries), - worker.WithProviderReplacementMaxRetries(cfg.Worker.ProviderReplacement.MaxRetries), - worker.WithPDPStatusChecker(synapse.NewPDPStatusChecker(synapse.PDPStatusCheckerOptions{ - Timeout: 15 * time.Second, - AllowPrivateNetworks: cfg.Filecoin.AllowPrivateNetworks, - })), - worker.WithEventPublisher(events)) - manager := worker.NewManager(repos, logger, evictionPolicy, - uploader, - worker.NewProviderReplacementWorker(repos, uploader, - cfg.Worker.ProviderReplacement.Concurrency, - cfg.Worker.ProviderReplacement.PollInterval, - logger), - worker.NewEvictor(repos, localCache, cacheGate, accessTracker, stateMachine, - cfg.Worker.Evictor.Concurrency, cfg.Worker.Evictor.PollInterval, logger, - worker.WithCacheEvictionPolicy( - evictionPolicy, - maxCacheBytes, - cfg.Cache.LRUHighWatermarkPercent, - cfg.Cache.LRULowWatermarkPercent, - cfg.Worker.Evictor.MaxRetries, - )), - worker.NewStorageCleanupWorker(repos, opts.Filecoin.Storage, - cfg.Worker.StorageCleanup.Concurrency, cfg.Worker.StorageCleanup.PollInterval, logger, worker.WithServiceTermination(opts.Filecoin.Terminator, opts.Filecoin.Epochs)), - worker.NewWalletOperationRunner(repos, opts.Filecoin.Wallet, opts.Filecoin.Receipts, 5*time.Second, logger, - worker.WithWalletOperationEventPublisher(events)), - ).WithTaskMaxRetries(cfg.Worker.Upload.MaxRetries, cfg.Worker.Evictor.MaxRetries). - WithProviderReplacementRecovery( - cfg.Worker.ProviderReplacement.MaxRetries, - cfg.Worker.ProviderReplacement.PollInterval, - ) + manager := worker.NewManager(engine, logger) adminServer := admin.New(cfg.Admin.Addr, opts.Database, localCache, cacheGate, accessTracker, maxCacheBytes, repos, manager, opts.Filecoin.WalletQuery, cfg.Filecoin.DefaultCopies, logger). - WithTaskDiagnosticStatusChecker(synapse.NewPDPStatusChecker(synapse.PDPStatusCheckerOptions{ - AllowPrivateNetworks: cfg.Filecoin.AllowPrivateNetworks, - })). WithEventHub(events). WithObjectUploader(appBackend). WithObjectVersionRestorer(appBackend). @@ -203,10 +243,7 @@ func NewRuntime(ctx context.Context, opts RuntimeOptions) (_ *Runtime, err error WithSettings(opts.Settings). WithFilecoinReadiness(opts.Filecoin.Readiness). WithObservability(observabilityService). - WithEvictMaxRetries(cfg.Worker.Evictor.MaxRetries). - WithStorageCleanupMaxRetries(cfg.Worker.StorageCleanup.MaxRetries). - WithUploadMaxRetries(cfg.Worker.Upload.MaxRetries). - WithProviderReplacementMaxRetries(cfg.Worker.ProviderReplacement.MaxRetries). + WithTaskService(taskService). WithS3IAM(iamService, rootAccount.Access) if opts.ProviderIdentity != nil { adminServer.WithProviderIdentityResolver(opts.ProviderIdentity) @@ -245,13 +282,50 @@ func NewRuntime(ctx context.Context, opts RuntimeOptions) (_ *Runtime, err error accessTracker: accessTracker, iam: iamService, workers: manager, - observer: observability.NewRunner(observabilityService, logger), s3Addresses: s3Addresses, logger: logger, shutdown: shutdownTimeout, }, nil } +func publishUploadTaskSettlement(events admin.EventPublisher, taskRow *model.Task, transition repository.TaskTransition) { + if events == nil || taskRow == nil || transition.Status == model.TaskStatusPending || !uploadPipelineTask(taskRow.Type) { + return + } + payload := map[string]any{ + "task_id": taskRow.ID, + "task_type": taskRow.Type, + "status": transition.Status, + } + if taskRow.SubjectType != nil { + payload["subject_type"] = *taskRow.SubjectType + } + if taskRow.SubjectKey != nil { + payload["subject_key"] = *taskRow.SubjectKey + if taskRow.SubjectType != nil && *taskRow.SubjectType == "object_version" { + payload["version_id"] = *taskRow.SubjectKey + } + } + events.Publish("upload_state_changed", payload) +} + +func uploadPipelineTask(taskType model.TaskType) bool { + switch taskType { + case model.TaskTypeUploadPlan, + model.TaskTypeStorageDataSetEnsure, + model.TaskTypeStorageTransferPlan, + model.TaskTypeStorageStore, + model.TaskTypeStoragePull, + model.TaskTypeStorageCommitCoordinate, + model.TaskTypeStorageCommit, + model.TaskTypeProviderReplacementCoordinate, + model.TaskTypeStorageDataSetRetire: + return true + default: + return false + } +} + func validateOptions(opts RuntimeOptions) error { var missing []string dependencies := []struct { @@ -350,13 +424,6 @@ func (r *Runtime) Run(ctx context.Context) error { } return nil }) - group.Go(func() error { - r.observer.Run(groupCtx) - if groupCtx.Err() == nil { - return errors.New("observer stopped unexpectedly") - } - return nil - }) group.Go(func() error { r.accessTracker.Run(groupCtx, r.cacheGate, r.logger) return nil @@ -454,7 +521,7 @@ func newObservabilityService(cfg *config.Config, repos *repository.Repositories, return observability.NewService(observability.ServiceOptions{ Checker: checker, LocalDataSets: observability.LocalDataSetSourceFunc(func(ctx context.Context) ([]observability.LocalDataSet, error) { - summaries, err := repos.Uploads.ListDataSetSummaries(ctx, 0) + summaries, err := repos.Contents.ListDataSetSummaries(ctx, 0) if err != nil { return nil, err } diff --git a/internal/backend/backend.go b/internal/backend/backend.go index 494b7e5..2d6dd5e 100644 --- a/internal/backend/backend.go +++ b/internal/backend/backend.go @@ -8,8 +8,8 @@ import ( "github.com/strahe/synaps3/internal/cacheaccess" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/objectreader" - "github.com/strahe/synaps3/internal/state" "github.com/strahe/synaps3/internal/synapse" + taskengine "github.com/strahe/synaps3/internal/task" "github.com/versity/versitygw/backend" ) @@ -18,48 +18,34 @@ import ( type SynapseBackend struct { backend.BackendUnsupported // provides ErrNotImplemented for unimplemented methods - repos *repository.Repositories - cache cache.Cache - objectReader *objectreader.Reader - cacheGate *cacheaccess.Gate - cacheAccessTracker *cacheaccess.Tracker - bucketLifecycle *bucketlifecycle.Service - stateMachine *state.Machine - storage synapse.StorageClient - uploadMaxRetries int - evictMaxRetries int - storageCleanupMaxRetries int - evictionPolicy cache.EvictionPolicy - logger *slog.Logger + repos *repository.Repositories + cache cache.Cache + objectReader *objectreader.Reader + cacheGate *cacheaccess.Gate + cacheAccessTracker *cacheaccess.Tracker + bucketLifecycle *bucketlifecycle.Service + storage synapse.StorageClient + taskService *taskengine.Service + evictionPolicy cache.EvictionPolicy + defaultCopies int + logger *slog.Logger } -const ( - defaultUploadMaxRetries = 5 - defaultEvictMaxRetries = 3 - defaultStorageCleanupMaxRetries = 5 -) - // Option configures SynapseBackend runtime behavior. type Option func(*SynapseBackend) -// WithUploadMaxRetries configures max retries for newly-created upload tasks. -func WithUploadMaxRetries(maxRetries int) Option { - return func(b *SynapseBackend) { - b.uploadMaxRetries = maxRetries - } -} - -// WithEvictMaxRetries configures max retries for newly-created cache eviction tasks. -func WithEvictMaxRetries(maxRetries int) Option { +// WithTaskService configures the sole task creation boundary. +func WithTaskService(service *taskengine.Service) Option { return func(b *SynapseBackend) { - b.evictMaxRetries = maxRetries + b.taskService = service } } -// WithStorageCleanupMaxRetries configures max retries for newly-created storage cleanup tasks. -func WithStorageCleanupMaxRetries(maxRetries int) Option { +// WithDefaultCopies sets the replica target recorded on content the first time +// its bytes are written. A bucket override still wins. +func WithDefaultCopies(copies int) Option { return func(b *SynapseBackend) { - b.storageCleanupMaxRetries = maxRetries + b.defaultCopies = copies } } @@ -74,7 +60,6 @@ func WithEvictionPolicy(policy cache.EvictionPolicy) Option { func New( repos *repository.Repositories, c cache.Cache, - sm *state.Machine, sc synapse.StorageClient, cacheGate *cacheaccess.Gate, cacheAccessTracker *cacheaccess.Tracker, @@ -88,22 +73,19 @@ func New( panic("backend requires a cache access tracker") } b := &SynapseBackend{ - repos: repos, - cache: c, - cacheGate: cacheGate, - cacheAccessTracker: cacheAccessTracker, - bucketLifecycle: bucketlifecycle.New(repos, c, logger), - stateMachine: sm, - storage: sc, - uploadMaxRetries: defaultUploadMaxRetries, - evictMaxRetries: defaultEvictMaxRetries, - storageCleanupMaxRetries: defaultStorageCleanupMaxRetries, - evictionPolicy: cache.EvictionPolicyNone, - logger: logger, + repos: repos, + cache: c, + cacheGate: cacheGate, + cacheAccessTracker: cacheAccessTracker, + storage: sc, + evictionPolicy: cache.EvictionPolicyNone, + logger: logger, } for _, opt := range opts { opt(b) } + b.bucketLifecycle = bucketlifecycle.New(repos, c, b.defaultCopies, logger) + b.bucketLifecycle.SetTaskService(b.taskService) b.objectReader = objectreader.New( repos, c, diff --git a/internal/backend/bucket.go b/internal/backend/bucket.go index 74823fb..abaa50c 100644 --- a/internal/backend/bucket.go +++ b/internal/backend/bucket.go @@ -6,8 +6,8 @@ import ( "fmt" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/strahe/synaps3/internal/bucketlifecycle" "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" "github.com/versity/versitygw/auth" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3response" @@ -45,29 +45,18 @@ func (b *SynapseBackend) createBucketWithOwner(ctx context.Context, name, owner if owner != "" { ownerPtr = &owner } - var bucket *model.Bucket - err := b.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { - if owner != "" { - account, err := txRepos.S3Accounts.LockByAccessKey(ctx, owner) - if err != nil { - return err - } - if account == nil { - return auth.ErrNoSuchUser - } - } - bucket = &model.Bucket{ - Name: name, - ACL: acl, - OwnerAccessKey: ownerPtr, - Status: model.BucketStatusActive, - } - return txRepos.Buckets.Create(ctx, bucket) + bucket, err := b.bucketLifecycle.CreateWithOptions(ctx, bucketlifecycle.CreateOptions{ + Name: name, + ACL: acl, + OwnerAccessKey: ownerPtr, }) + if errors.Is(err, bucketlifecycle.ErrOwnerNotFound) { + return auth.ErrNoSuchUser + } if err != nil { return err } - b.bucketLifecycle.EnsureCacheBucketDir(ctx, bucket.Name) + b.logger.Info("bucket storage provisioning scheduled", "bucket", bucket.Name, "id", bucket.ID) return nil } diff --git a/internal/backend/bucket_policy_test.go b/internal/backend/bucket_policy_test.go index 61180ad..886397b 100644 --- a/internal/backend/bucket_policy_test.go +++ b/internal/backend/bucket_policy_test.go @@ -19,7 +19,7 @@ func TestGetBucketAcl_ExistingBucket(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() - bkt := &model.Bucket{Name: "acl-bucket", Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: "acl-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket: %v", err) } @@ -155,7 +155,7 @@ func TestPutBucketAcl_WritableBucket(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() - bkt := &model.Bucket{Name: "put-acl-bucket", Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: "put-acl-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket: %v", err) } @@ -170,7 +170,7 @@ func TestPutBucketAclPersistsACL(t *testing.T) { ctx := context.Background() seedS3Account(t, tb, "new-owner") - bkt := &model.Bucket{Name: "persist-acl-bucket", Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: "persist-acl-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket: %v", err) } @@ -213,7 +213,7 @@ func TestPutBucketAclPreservesCurrentOwnerWhenACLHasNoOwner(t *testing.T) { if err != nil { t.Fatalf("Marshal current ACL: %v", err) } - bkt := &model.Bucket{Name: "preserve-owner-acl-bucket", Status: model.BucketStatusActive, OwnerAccessKey: ¤tOwner, ACL: acl} + bkt := &model.Bucket{Name: "preserve-owner-acl-bucket", Status: model.BucketStatusActive, OwnerAccessKey: ¤tOwner, ACL: acl, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket: %v", err) } @@ -249,7 +249,7 @@ func TestPutBucketAclPreservesCurrentOwnerWhenACLHasNoOwner(t *testing.T) { func TestPutBucketAclRejectsUnknownOwner(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() - bkt := &model.Bucket{Name: "unknown-put-owner-bucket", Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: "unknown-put-owner-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket: %v", err) } @@ -276,7 +276,7 @@ func TestChangeBucketOwnerUpdatesACL(t *testing.T) { ctx := context.Background() seedS3Account(t, tb, "replacement-owner") - bkt := &model.Bucket{Name: "change-owner-bucket", Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: "change-owner-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket: %v", err) } @@ -313,7 +313,7 @@ func TestListBucketsAndOwnersReturnsStoredOwners(t *testing.T) { {name: "owner-list-unassigned"}, {name: "owner-list-malformed", acl: []byte("{")}, } { - bkt := &model.Bucket{Name: seed.name, Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: seed.name, Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} switch { case seed.acl != nil: bkt.ACL = seed.acl @@ -360,7 +360,7 @@ func TestPutBucketOwnershipControlsRejectsUnsupportedModes(t *testing.T) { if err != nil { t.Fatalf("Marshal ACL: %v", err) } - bkt := &model.Bucket{Name: "ownership-controls-bucket", Status: model.BucketStatusActive, OwnerAccessKey: &owner, ACL: acl} + bkt := &model.Bucket{Name: "ownership-controls-bucket", Status: model.BucketStatusActive, OwnerAccessKey: &owner, ACL: acl, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -429,7 +429,7 @@ func TestDeleteBucketOwnershipControlsIsCompatibilityNoop(t *testing.T) { if err != nil { t.Fatalf("Marshal ACL: %v", err) } - bkt := &model.Bucket{Name: "delete-ownership-controls-bucket", Status: model.BucketStatusActive, OwnerAccessKey: &owner, ACL: acl} + bkt := &model.Bucket{Name: "delete-ownership-controls-bucket", Status: model.BucketStatusActive, OwnerAccessKey: &owner, ACL: acl, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -465,7 +465,7 @@ func TestGetBucketPolicyMissingPolicyReturnsNoSuchBucketPolicy(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() - bkt := &model.Bucket{Name: "policy-bucket", Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: "policy-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket: %v", err) } @@ -508,7 +508,7 @@ func TestGetBucketTagging_ReturnsAPIError(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() - bkt := &model.Bucket{Name: "tag-bucket", Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: "tag-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket: %v", err) } @@ -545,7 +545,7 @@ func TestGetBucketVersioning_ExistingBucket(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() - bkt := &model.Bucket{Name: "ver-bucket", Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: "ver-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket: %v", err) } @@ -566,7 +566,7 @@ func TestPutBucketVersioningEnabledIsNoop(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() - bkt := &model.Bucket{Name: "put-ver-bucket", Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: "put-ver-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket: %v", err) } @@ -580,7 +580,7 @@ func TestPutBucketVersioningSuspendedRejected(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() - bkt := &model.Bucket{Name: "put-ver-suspended-bucket", Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: "put-ver-suspended-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket: %v", err) } @@ -621,7 +621,7 @@ func TestGetObjectLockConfiguration_ReturnsNotFound(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() - bkt := &model.Bucket{Name: "lock-bucket", Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: "lock-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket: %v", err) } @@ -647,7 +647,7 @@ func TestGetBucketOwnershipControls_ReturnsAclCompatibleOwnership(t *testing.T) tb := newTestBackend(t) ctx := context.Background() - bkt := &model.Bucket{Name: "own-bucket", Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: "own-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket: %v", err) } @@ -667,7 +667,7 @@ func TestGetObjectAcl_VisibleBucket(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() - bkt := &model.Bucket{Name: "obj-acl-bucket", Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: "obj-acl-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket: %v", err) } @@ -689,7 +689,7 @@ func TestPutObjectAcl_WritableBucket(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() - bkt := &model.Bucket{Name: "put-obj-acl", Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: "put-obj-acl", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket: %v", err) } diff --git a/internal/backend/bucket_test.go b/internal/backend/bucket_test.go index 0ec64f6..469cc12 100644 --- a/internal/backend/bucket_test.go +++ b/internal/backend/bucket_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "testing" + "github.com/strahe/synaps3/internal/bucketlifecycle" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/strahe/synaps3/internal/model" @@ -24,7 +26,7 @@ func TestCreateBucket_HappyPath(t *testing.T) { t.Fatalf("CreateBucket: %v", err) } - // Verify bucket in DB with status=creating. + // The namespace is visible while provider storage is prepared in the background. bucket, err := tb.repos.Buckets.GetByName(ctx, "my-bucket") if err != nil { t.Fatalf("GetByName: %v", err) @@ -32,9 +34,25 @@ func TestCreateBucket_HappyPath(t *testing.T) { if bucket == nil { t.Fatal("bucket not found in DB") } - if bucket.Status != model.BucketStatusActive { - t.Errorf("bucket status = %q, want %q", bucket.Status, model.BucketStatusActive) + if bucket.Status != model.BucketStatusProvisioning { + t.Errorf("bucket status = %q, want %q", bucket.Status, model.BucketStatusProvisioning) + } + provisionTask, err := tb.repos.Tasks.GetByIdentity(ctx, model.TaskTypeBucketProvision, bucketlifecycle.ProvisionKey(bucket.ID, bucket.DefaultCopies)) + if err != nil || provisionTask == nil || provisionTask.Status != model.TaskStatusPending { + t.Fatalf("bucket provision task = %#v, err=%v", provisionTask, err) + } +} + +func TestProvisioningBucketRejectsObjectWritesAsRetryable(t *testing.T) { + tb := newTestBackend(t) + ctx := t.Context() + if err := tb.backend.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String("warming-bucket")}, nil); err != nil { + t.Fatalf("CreateBucket: %v", err) } + _, err := tb.backend.CreateMultipartUpload(ctx, s3response.CreateMultipartUploadInput{ + Bucket: aws.String("warming-bucket"), Key: aws.String("object.bin"), + }) + requireAPIErrorCode(t, err, s3err.GetAPIError(s3err.ErrSlowDown)) } func TestCreateBucketRejectsMissingBucketInput(t *testing.T) { @@ -156,7 +174,7 @@ func TestHeadBucket_Exists(t *testing.T) { ctx := context.Background() // Seed an active bucket directly via repos. - bkt := &model.Bucket{Name: "head-bucket", Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: "head-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket: %v", err) } @@ -216,7 +234,7 @@ func TestDeleteBucket_NotSupported(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() - bkt := &model.Bucket{Name: "del-bucket", Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: "del-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket: %v", err) } @@ -246,7 +264,7 @@ func TestListBuckets_OnlyActive(t *testing.T) { // Seed active buckets. for _, name := range []string{"active-1", "active-2", "active-3"} { - b := &model.Bucket{Name: name, Status: model.BucketStatusActive} + b := &model.Bucket{Name: name, Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := tb.repos.Buckets.Create(ctx, b); err != nil { t.Fatalf("seeding bucket %q: %v", name, err) } @@ -285,7 +303,7 @@ func TestListBucketsFiltersNonAdminByOwnerAccessKey(t *testing.T) { {name: "legacy-root-bucket"}, {name: "malformed-acl-bucket", acl: []byte("{")}, } { - b := &model.Bucket{Name: seed.name, Status: model.BucketStatusActive} + b := &model.Bucket{Name: seed.name, Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} switch { case seed.acl != nil: b.ACL = seed.acl diff --git a/internal/backend/cache_write_gate_test.go b/internal/backend/cache_write_gate_test.go new file mode 100644 index 0000000..fb914f0 --- /dev/null +++ b/internal/backend/cache_write_gate_test.go @@ -0,0 +1,166 @@ +package backend_test + +import ( + "context" + "strings" + "sync" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/testutil" + "github.com/uptrace/bun" + "github.com/versity/versitygw/s3response" +) + +type blockingObjectVersionInsertHook struct { + once sync.Once + entered chan struct{} + release chan struct{} +} + +func (h *blockingObjectVersionInsertHook) BeforeQuery(ctx context.Context, event *bun.QueryEvent) context.Context { + query := strings.ToLower(event.Query) + if strings.Contains(query, `insert into "object_versions"`) || strings.Contains(query, "insert into object_versions") { + h.once.Do(func() { + close(h.entered) + select { + case <-h.release: + case <-ctx.Done(): + } + }) + } + return ctx +} + +func (*blockingObjectVersionInsertHook) AfterQuery(context.Context, *bun.QueryEvent) {} + +func seedExpectedWriteContent(t *testing.T, tb *testBackend, bucketName, body string) int64 { + t.Helper() + bucket, err := tb.repos.Buckets.GetByName(t.Context(), bucketName) + if err != nil || bucket == nil { + t.Fatalf("load bucket %q = %#v, err=%v", bucketName, bucket, err) + } + content, err := tb.repos.Contents.EnsureContent(t.Context(), repository.EnsureContentInput{ + BucketID: bucket.ID, ContentSize: int64(len(body)), Checksum: testutil.StorageChecksum(testSHA256Hex(body)), + RequestedCopies: bucket.DefaultCopies, + }) + if err != nil { + t.Fatalf("seed expected content: %v", err) + } + return content.ID +} + +func assertWriteHoldsContentGateThroughVersionInsert( + t *testing.T, + tb *testBackend, + contentID int64, + write func() error, +) { + t.Helper() + hook := &blockingObjectVersionInsertHook{entered: make(chan struct{}), release: make(chan struct{})} + tb.db.AddQueryHook(hook) + writeDone := make(chan error, 1) + go func() { writeDone <- write() }() + + select { + case <-hook.entered: + case <-time.After(2 * time.Second): + t.Fatal("write did not reach the object version transaction") + } + deletionStarted := make(chan struct{}) + deletionEntered := make(chan struct{}) + go func() { + close(deletionStarted) + tb.gate.GuardDeletion(model.ContentCacheKey(contentID), func() { close(deletionEntered) }) + }() + <-deletionStarted + select { + case <-deletionEntered: + close(hook.release) + t.Fatal("deletion entered while the cache write transaction was still blocked") + case <-time.After(50 * time.Millisecond): + } + close(hook.release) + select { + case err := <-writeDone: + if err != nil { + t.Fatalf("cache write: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("cache write did not finish after releasing the transaction hook") + } + select { + case <-deletionEntered: + case <-time.After(2 * time.Second): + t.Fatal("deletion did not enter after the cache write transaction finished") + } +} + +func TestPutObjectHoldsContentGateThroughVersionTransaction(t *testing.T) { + tb := newTestBackend(t) + bucketName := "put-cache-gate" + seedActiveBucket(t, tb, bucketName) + body := validTestObjectBody("put cache gate") + contentID := seedExpectedWriteContent(t, tb, bucketName, body) + assertWriteHoldsContentGateThroughVersionInsert(t, tb, contentID, func() error { + _, err := tb.backend.PutObject(t.Context(), s3response.PutObjectInput{ + Bucket: &bucketName, Key: aws.String("object.bin"), Body: strings.NewReader(body), + }) + return err + }) +} + +func TestCopyObjectHoldsContentGateThroughVersionTransaction(t *testing.T) { + tb := newTestBackend(t) + bucketName := "copy-cache-gate" + bucket := seedActiveBucket(t, tb, bucketName) + putValidTestObject(t, tb, bucketName, "source.bin", "copy cache gate") + source, err := tb.repos.Objects.GetCurrentVersionByBucketAndKey(t.Context(), bucket.ID, "source.bin") + if err != nil || source == nil || source.ContentID == nil { + t.Fatalf("load copy source = %#v, err=%v", source, err) + } + assertWriteHoldsContentGateThroughVersionInsert(t, tb, *source.ContentID, func() error { + copySource := bucketName + "/source.bin" + _, err := tb.backend.CopyObject(t.Context(), s3response.CopyObjectInput{ + Bucket: &bucketName, Key: aws.String("destination.bin"), CopySource: ©Source, + }) + return err + }) +} + +func TestCompleteMultipartHoldsContentGateThroughVersionTransaction(t *testing.T) { + tb := newTestBackend(t) + bucketName := "multipart-cache-gate" + seedActiveBucket(t, tb, bucketName) + key := "assembled.bin" + created, err := tb.backend.CreateMultipartUpload(t.Context(), s3response.CreateMultipartUploadInput{ + Bucket: &bucketName, Key: &key, + }) + if err != nil { + t.Fatalf("create multipart upload: %v", err) + } + body := validTestObjectBody("multipart cache gate") + partNumber := int32(1) + part, err := tb.backend.UploadPart(t.Context(), &s3.UploadPartInput{ + Bucket: &bucketName, Key: &key, UploadId: &created.UploadId, + PartNumber: &partNumber, Body: strings.NewReader(body), + }) + if err != nil { + t.Fatalf("upload multipart part: %v", err) + } + contentID := seedExpectedWriteContent(t, tb, bucketName, body) + assertWriteHoldsContentGateThroughVersionInsert(t, tb, contentID, func() error { + _, _, err := tb.backend.CompleteMultipartUpload(t.Context(), &s3.CompleteMultipartUploadInput{ + Bucket: &bucketName, Key: &key, UploadId: &created.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{{ + PartNumber: &partNumber, ETag: part.ETag, + }}}, + }) + return err + }) +} diff --git a/internal/backend/integration_test.go b/internal/backend/integration_test.go index 1ecda17..2c8d1b4 100644 --- a/internal/backend/integration_test.go +++ b/internal/backend/integration_test.go @@ -6,19 +6,19 @@ import ( "fmt" "io" "log/slog" + "strconv" "strings" "testing" "time" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - cid "github.com/ipfs/go-cid" - "github.com/multiformats/go-multihash" + "github.com/ipfs/go-cid" + multihash "github.com/multiformats/go-multihash" "github.com/strahe/synaps3/internal/backend" "github.com/strahe/synaps3/internal/cache" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/state" "github.com/strahe/synaps3/internal/testutil" "github.com/strahe/synapse-go/storage" "github.com/uptrace/bun" @@ -43,12 +43,11 @@ func newIntegrationBackend(t *testing.T) *integrationBackend { if err != nil { t.Fatalf("creating test cache: %v", err) } - sm := state.NewObjectStateMachine() sc := &testutil.MockStorageClient{} logger := slog.Default() cacheGate, accessTracker := newBackendCacheAccess(repos) - b := backend.New(repos, fsCache, sm, sc, cacheGate, accessTracker, logger) + b := backend.New(repos, fsCache, sc, cacheGate, accessTracker, logger, backend.WithTaskService(newBackendTaskService(t, repos))) return &integrationBackend{ backend: b, repos: repos, @@ -58,12 +57,22 @@ func newIntegrationBackend(t *testing.T) *integrationBackend { } } -// findTasks queries all tasks matching the given ref type and ref ID. +// findTasks queries the storage tasks reachable from one object. Ingest is +// scheduled per content now, so the object is reached through the contents its +// versions point at. func findTasks(t *testing.T, db *bun.DB, refType string, refID int64) []model.Task { t.Helper() + if refType != "object" { + t.Fatalf("unsupported test task subject %q", refType) + } var tasks []model.Task err := db.NewSelect().Model(&tasks). - Where("ref_type = ? AND ref_id = ?", refType, refID). + Where("task.subject_type = ?", "storage_content"). + Where(`EXISTS ( + SELECT 1 FROM object_versions AS task_version + WHERE CAST(task_version.content_id AS TEXT) = task.subject_key + AND task_version.object_id = ? + )`, refID). OrderExpr("id ASC"). Scan(context.Background()) if err != nil { @@ -72,6 +81,20 @@ func findTasks(t *testing.T, db *bun.DB, refType string, refID int64) []model.Ta return tasks } +// contentSubject is the task subject key for the content a version points at. +func contentSubject(t *testing.T, db *bun.DB, versionID string) string { + t.Helper() + var contentID int64 + if err := db.NewSelect(). + Table("object_versions"). + Column("content_id"). + Where("version_id = ?", versionID). + Scan(context.Background(), &contentID); err != nil { + t.Fatalf("reading content for version %s: %v", versionID, err) + } + return strconv.FormatInt(contentID, 10) +} + // putObject is a helper that calls PutObject with the given string body. func putObject(t *testing.T, b *backend.SynapseBackend, bucket, key, body string) s3response.PutObjectOutput { t.Helper() @@ -118,104 +141,62 @@ func TestIntegration_FullWritePath(t *testing.T) { if err != nil || obj == nil { t.Fatalf("expected object in DB, got err=%v obj=%v", err, obj) } + // Nothing has reached a provider yet, and pipeline position is read from + // the content's copies, so the object is cached and no further. if obj.State != model.ObjectStateCached { t.Fatalf("expected state=cached, got %s", obj.State) } - if obj.VersionID == "" { - t.Fatal("expected current version id") + if obj.VersionID == "" || obj.ContentID == nil { + t.Fatalf("expected a current version bound to content, got version=%q content=%v", obj.VersionID, obj.ContentID) + } + if !obj.InCache { + t.Fatal("expected the written bytes to be cached") } tasks := findTasks(t, ib.db, "object", obj.ObjectID) if len(tasks) != 1 { t.Fatalf("expected 1 upload task, got %d", len(tasks)) } - if tasks[0].Type != model.TaskTypeUpload { - t.Fatalf("expected task type upload, got %s", tasks[0].Type) - } - if tasks[0].RefVersionID != obj.VersionID { - t.Fatalf("expected task version=%s, got %s", obj.VersionID, tasks[0].RefVersionID) + if tasks[0].Type != model.TaskTypeUploadPlan { + t.Fatalf("expected task type upload_plan, got %s", tasks[0].Type) } - - // 2. Simulate uploader: cached → uploading - if err := ib.repos.Objects.UpdateVersionState(ctx, obj.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("cached→uploading: %v", err) + // Ingest is planned for the bytes, so the task names the content. + if want := contentSubject(t, ib.db, obj.VersionID); tasks[0].SubjectKey == nil || *tasks[0].SubjectKey != want { + t.Fatalf("expected task content=%s, got %v", want, tasks[0].SubjectKey) } - acceptBackendVersionUpload(t, ib.repos, obj.VersionID, "bafk2test123", "https://provider.example/pieces/test") + // 2. Simulate the uploader placing the content with a provider. + acceptBackendVersionUpload(t, ib.db, ib.repos, obj.VersionID, "bafk2test123", "https://provider.example/pieces/test") obj, _ = ib.repos.Objects.GetCurrentVersionByObjectID(ctx, obj.ObjectID) if obj.State != model.ObjectStateStored { t.Fatalf("expected state=stored, got %s", obj.State) } - if obj.StorageUploadID == nil || obj.PieceCID == nil || *obj.PieceCID != "bafk2test123" { - t.Fatalf("expected accepted upload with PieceCID=bafk2test123, got upload=%v piece=%v", obj.StorageUploadID, obj.PieceCID) + if obj.ContentID == nil || obj.PieceCID == nil || *obj.PieceCID != "bafk2test123" { + t.Fatalf("expected accepted content with PieceCID=bafk2test123, got content=%v piece=%v", obj.ContentID, obj.PieceCID) + } + if !obj.InFilecoin { + t.Fatal("expected a readable committed copy after acceptance") } - // 3. Simulate evictor: stored → cache_evicted, remove cache file - if err := ib.repos.Objects.UpdateVersionState(ctx, obj.VersionID, model.ObjectStateStored, model.ObjectStateCacheEvicted); err != nil { - t.Fatalf("stored→cache_evicted: %v", err) + // 3. Simulate the evictor: durability is unchanged, only the cached copy goes. + cacheKey := obj.CacheKey() + if err := ib.repos.Objects.ClearContentCachePresence(ctx, *obj.ContentID); err != nil { + t.Fatalf("mark cache absent: %v", err) } - if err := ib.cache.Delete(ctx, "test-bucket", obj.CacheKey); err != nil { + if err := ib.cache.Delete(ctx, "test-bucket", cacheKey); err != nil { t.Fatalf("cache delete: %v", err) } obj, _ = ib.repos.Objects.GetCurrentVersionByObjectID(ctx, obj.ObjectID) - if obj.State != model.ObjectStateCacheEvicted { - t.Fatalf("expected state=cache_evicted, got %s", obj.State) + if obj.State != model.ObjectStateStored || obj.InCache { + t.Fatalf("expected stored remote-only object, got state=%s in_cache=%v", obj.State, obj.InCache) } - if ib.cache.Exists(ctx, "test-bucket", obj.CacheKey) { + if ib.cache.Exists(ctx, "test-bucket", cacheKey) { t.Fatal("expected cache file to be gone") } } -func TestIntegration_OverwritePath(t *testing.T) { - ib := newIntegrationBackend(t) - ctx := context.Background() - - bucket := testutil.SeedBucket(t, ib.db, "bucket") - - // First write - putObject(t, ib.backend, "bucket", "key", "v1") - - obj, _ := ib.repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "key") - firstVersionID := obj.VersionID - if firstVersionID == "" { - t.Fatal("expected current version id after first put") - } - - tasks := findTasks(t, ib.db, "object", obj.ObjectID) - if len(tasks) != 1 || tasks[0].RefVersionID != firstVersionID { - t.Fatalf("expected 1 task with first version, got %d tasks", len(tasks)) - } - - // Overwrite - putObject(t, ib.backend, "bucket", "key", "v2") - - obj, _ = ib.repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "key") - secondVersionID := obj.VersionID - if secondVersionID == "" || secondVersionID == firstVersionID { - t.Fatalf("expected new current version after overwrite, first=%s second=%s", firstVersionID, secondVersionID) - } - - tasks = findTasks(t, ib.db, "object", obj.ObjectID) - if len(tasks) != 2 { - t.Fatalf("expected 2 tasks, got %d", len(tasks)) - } - if tasks[1].RefVersionID != secondVersionID { - t.Fatalf("expected second task version=%s, got %s", secondVersionID, tasks[1].RefVersionID) - } - - if tasks[0].RefVersionID != firstVersionID { - t.Fatalf("expected first task version=%s, got %s", firstVersionID, tasks[0].RefVersionID) - } - - // GetObject should return the current version. - body := getObjectBody(t, ib.backend, "bucket", "key") - if want := validTestObjectBody("v2"); body != want { - t.Fatalf("expected body=%q, got %q", want, body) - } -} - func TestIntegration_ColdReadAfterEviction(t *testing.T) { ib := newIntegrationBackend(t) ctx := context.Background() @@ -228,11 +209,6 @@ func TestIntegration_ColdReadAfterEviction(t *testing.T) { obj, _ := ib.repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "test-key") - // Simulate full pipeline to cache_evicted - if err := ib.repos.Objects.UpdateVersionState(ctx, obj.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatal(err) - } - // Create a valid CID for PieceCID mh, err := multihash.Sum([]byte("test"), multihash.SHA2_256, -1) if err != nil { @@ -240,16 +216,17 @@ func TestIntegration_ColdReadAfterEviction(t *testing.T) { } testPieceCID := cid.NewCidV1(cid.Raw, mh) - acceptBackendVersionUpload(t, ib.repos, obj.VersionID, testPieceCID.String(), "https://provider.example/pieces/test") - if err := ib.repos.Objects.UpdateVersionState(ctx, obj.VersionID, model.ObjectStateStored, model.ObjectStateCacheEvicted); err != nil { + // Simulate a stored object whose local cache has been removed. Residency is + // content-addressed, so both the record and the file are keyed on content. + acceptBackendVersionUpload(t, ib.db, ib.repos, obj.VersionID, testPieceCID.String(), "https://provider.example/pieces/test") + cacheKey := obj.CacheKey() + if err := ib.repos.Objects.ClearContentCachePresence(ctx, *obj.ContentID); err != nil { t.Fatal(err) } - - // Remove cache file - if err := ib.cache.Delete(ctx, "test-bucket", obj.CacheKey); err != nil { + if err := ib.cache.Delete(ctx, "test-bucket", cacheKey); err != nil { t.Fatal(err) } - if ib.cache.Exists(ctx, "test-bucket", obj.CacheKey) { + if ib.cache.Exists(ctx, "test-bucket", cacheKey) { t.Fatal("cache should be empty after eviction") } @@ -270,8 +247,8 @@ func TestIntegration_ColdReadAfterEviction(t *testing.T) { // Cache rehydration is async (TeeReader goroutine writes while body is consumed). // Poll with a timeout to avoid flakiness. rehydrated := false - for i := 0; i < 200; i++ { - if ib.cache.Exists(ctx, "test-bucket", obj.CacheKey) { + for range 200 { + if ib.cache.Exists(ctx, "test-bucket", cacheKey) { rehydrated = true break } @@ -282,6 +259,129 @@ func TestIntegration_ColdReadAfterEviction(t *testing.T) { } } +func TestIntegration_MultipartUpload_Abort(t *testing.T) { + ib := newIntegrationBackend(t) + ctx := context.Background() + + testutil.SeedBucket(t, ib.db, "bucket") + + // 1. CreateMultipartUpload + createOut, err := ib.backend.CreateMultipartUpload(ctx, s3response.CreateMultipartUploadInput{ + Bucket: new("bucket"), + Key: new("file"), + }) + if err != nil { + t.Fatalf("CreateMultipartUpload: %v", err) + } + uploadID := createOut.UploadId + + // 2. UploadPart + part1Num := int32(1) + _, err = ib.backend.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: new("bucket"), + Key: new("file"), + UploadId: &uploadID, + PartNumber: &part1Num, + Body: strings.NewReader("data"), + }) + if err != nil { + t.Fatalf("UploadPart: %v", err) + } + + // 3. AbortMultipartUpload + err = ib.backend.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: new("bucket"), + Key: new("file"), + UploadId: &uploadID, + }) + if err != nil { + t.Fatalf("AbortMultipartUpload: %v", err) + } + + // 4. Verify: upload should be in aborted status (getActiveUpload won't find it) + upload, err := ib.repos.Multiparts.GetByUploadID(ctx, uploadID) + if err != nil { + t.Fatalf("querying upload: %v", err) + } + if upload == nil { + t.Fatal("expected upload record to still exist (with aborted status)") + } + if upload.Status != model.MultipartStatusAborted { + t.Fatalf("expected status=aborted, got %s", upload.Status) + } + + // Parts should be cleaned up + parts, err := ib.repos.Multiparts.GetParts(ctx, uploadID, 0, 100) + if err != nil { + t.Fatalf("querying parts: %v", err) + } + if len(parts) != 0 { + t.Fatalf("expected 0 parts after abort, got %d", len(parts)) + } + + // No object should be created + var objects []model.Object + err = ib.db.NewSelect().Model(&objects). + Where("key = ?", "file"). + Scan(ctx) + if err != nil { + t.Fatalf("querying objects: %v", err) + } + if len(objects) != 0 { + t.Fatalf("expected 0 objects after abort, got %d", len(objects)) + } +} + +func TestIntegration_OverwritePath(t *testing.T) { + ib := newIntegrationBackend(t) + ctx := context.Background() + + bucket := testutil.SeedBucket(t, ib.db, "bucket") + + // First write + putObject(t, ib.backend, "bucket", "key", "v1") + + obj, _ := ib.repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "key") + firstVersionID := obj.VersionID + if firstVersionID == "" { + t.Fatal("expected current version id after first put") + } + + tasks := findTasks(t, ib.db, "object", obj.ObjectID) + firstSubject := contentSubject(t, ib.db, firstVersionID) + if len(tasks) != 1 || tasks[0].SubjectKey == nil || *tasks[0].SubjectKey != firstSubject { + t.Fatalf("expected 1 task with first content, got %d tasks", len(tasks)) + } + + // Overwrite + putObject(t, ib.backend, "bucket", "key", "v2") + + obj, _ = ib.repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "key") + secondVersionID := obj.VersionID + if secondVersionID == "" || secondVersionID == firstVersionID { + t.Fatalf("expected new current version after overwrite, first=%s second=%s", firstVersionID, secondVersionID) + } + + tasks = findTasks(t, ib.db, "object", obj.ObjectID) + if len(tasks) != 2 { + t.Fatalf("expected 2 tasks, got %d", len(tasks)) + } + secondSubject := contentSubject(t, ib.db, secondVersionID) + if tasks[1].SubjectKey == nil || *tasks[1].SubjectKey != secondSubject { + t.Fatalf("expected second task content=%s, got %v", secondSubject, tasks[1].SubjectKey) + } + + if tasks[0].SubjectKey == nil || *tasks[0].SubjectKey != firstSubject { + t.Fatalf("expected first task content=%s, got %v", firstSubject, tasks[0].SubjectKey) + } + + // GetObject should return the current version. + body := getObjectBody(t, ib.backend, "bucket", "key") + if want := validTestObjectBody("v2"); body != want { + t.Fatalf("expected body=%q, got %q", want, body) + } +} + func TestIntegration_CopyObjectPath(t *testing.T) { ib := newIntegrationBackend(t) ctx := context.Background() @@ -316,14 +416,27 @@ func TestIntegration_CopyObjectPath(t *testing.T) { t.Fatalf("expected dst etag=%s, got %s", srcETag, dstObj.ETag) } - if dstObj.State != model.ObjectStateUploading { - t.Fatalf("expected dst state=uploading, got %s", dstObj.State) + // Pipeline position is derived from the content's copies, and no copy exists + // until the ingest plan runs, so a fresh copy reads as cached. + if dstObj.State != model.ObjectStateCached { + t.Fatalf("expected dst state=cached, got %s", dstObj.State) } - // Same-bucket copies of content with an active upload follow the source task. + // Same-bucket copies of identical bytes resolve to one content, so they + // share its single ingest plan instead of scheduling a second. dstTasks := findTasks(t, ib.db, "object", dstObj.ObjectID) - if len(dstTasks) != 0 { - t.Fatalf("expected no destination upload task, got %d", len(dstTasks)) + if len(dstTasks) != 1 { + t.Fatalf("expected the destination to share one upload task, got %d", len(dstTasks)) + } + totalPlans, err := ib.db.NewSelect(). + Model((*model.Task)(nil)). + Where("type = ?", model.TaskTypeUploadPlan). + Count(ctx) + if err != nil { + t.Fatalf("counting upload tasks: %v", err) + } + if totalPlans != 1 { + t.Fatalf("upload task count = %d, want 1", totalPlans) } // GetObject on dest should return the same data @@ -341,8 +454,8 @@ func TestIntegration_DeletePath_CreatesDeleteMarker(t *testing.T) { putOut := putObject(t, ib.backend, "bucket", "key", "data") deleteOut, err := ib.backend.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: strPtr("bucket"), - Key: strPtr("key"), + Bucket: new("bucket"), + Key: new("key"), }) if err != nil { t.Fatalf("DeleteObject: %v", err) @@ -359,17 +472,17 @@ func TestIntegration_DeletePath_CreatesDeleteMarker(t *testing.T) { t.Fatalf("current version = %#v, want delete marker", current) } - if _, err := ib.backend.GetObject(ctx, &s3.GetObjectInput{Bucket: strPtr("bucket"), Key: strPtr("key")}); err == nil { + if _, err := ib.backend.GetObject(ctx, &s3.GetObjectInput{Bucket: new("bucket"), Key: new("key")}); err == nil { t.Fatal("GetObject after delete returned nil error") } - listOut, err := ib.backend.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: strPtr("bucket")}) + listOut, err := ib.backend.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: new("bucket")}) if err != nil { t.Fatalf("ListObjectsV2: %v", err) } if len(listOut.Contents) != 0 { t.Fatalf("ListObjectsV2 contents = %#v, want deleted object hidden", listOut.Contents) } - versionsOut, err := ib.backend.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: strPtr("bucket")}) + versionsOut, err := ib.backend.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: new("bucket")}) if err != nil { t.Fatalf("ListObjectVersions: %v", err) } @@ -381,8 +494,8 @@ func TestIntegration_DeletePath_CreatesDeleteMarker(t *testing.T) { } if _, err := ib.backend.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: strPtr("bucket"), - Key: strPtr("key"), + Bucket: new("bucket"), + Key: new("key"), VersionId: deleteOut.VersionId, }); err != nil { t.Fatalf("DeleteObject marker version: %v", err) @@ -396,26 +509,26 @@ func TestIntegration_BucketLifecycle(t *testing.T) { ib := newIntegrationBackend(t) ctx := context.Background() - // 1. CreateBucket — bucket should be immediately active + // 1. CreateBucket creates the namespace and schedules provider storage. err := ib.backend.CreateBucket(ctx, &s3.CreateBucketInput{ - Bucket: strPtr("my-bucket"), + Bucket: new("my-bucket"), }, nil) if err != nil { t.Fatalf("CreateBucket: %v", err) } - // Verify bucket in active status + // Provider storage is not ready until the provisioning task finishes. bkt, err := ib.repos.Buckets.GetByName(ctx, "my-bucket") if err != nil || bkt == nil { t.Fatalf("expected bucket, got err=%v", err) } - if bkt.Status != model.BucketStatusActive { - t.Fatalf("expected status=active, got %s", bkt.Status) + if bkt.Status != model.BucketStatusProvisioning { + t.Fatalf("expected status=provisioning, got %s", bkt.Status) } // 2. HeadBucket should succeed _, err = ib.backend.HeadBucket(ctx, &s3.HeadBucketInput{ - Bucket: strPtr("my-bucket"), + Bucket: new("my-bucket"), }) if err != nil { t.Fatalf("HeadBucket: %v", err) @@ -437,7 +550,11 @@ func TestIntegration_BucketLifecycle(t *testing.T) { t.Fatal("expected my-bucket in ListBuckets") } - // 4. PutObject should succeed on active bucket + if err := ib.repos.Buckets.UpdateStatus(ctx, bkt.ID, model.BucketStatusProvisioning, model.BucketStatusReady); err != nil { + t.Fatalf("mark test bucket ready: %v", err) + } + + // 4. PutObject should succeed after provisioning completes. putObject(t, ib.backend, "my-bucket", "temp-key", "temp") // 5. DeleteBucket should return error (not supported) @@ -455,21 +572,21 @@ func TestIntegration_MultipartUpload_HappyPath(t *testing.T) { // 1. CreateMultipartUpload createOut, err := ib.backend.CreateMultipartUpload(ctx, s3response.CreateMultipartUploadInput{ - Bucket: strPtr("bucket"), - Key: strPtr("big-file"), + Bucket: new("bucket"), + Key: new("big-file"), }) if err != nil { t.Fatalf("CreateMultipartUpload: %v", err) } - uploadID := createOut.UploadId + contentID := createOut.UploadId // 2. UploadPart 1 part1Num := int32(1) part1Body := validTestObjectBody("part1-data") part1Out, err := ib.backend.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: strPtr("bucket"), - Key: strPtr("big-file"), - UploadId: &uploadID, + Bucket: new("bucket"), + Key: new("big-file"), + UploadId: &contentID, PartNumber: &part1Num, Body: strings.NewReader(part1Body), }) @@ -481,9 +598,9 @@ func TestIntegration_MultipartUpload_HappyPath(t *testing.T) { part2Num := int32(2) part2Body := "part2-data" part2Out, err := ib.backend.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: strPtr("bucket"), - Key: strPtr("big-file"), - UploadId: &uploadID, + Bucket: new("bucket"), + Key: new("big-file"), + UploadId: &contentID, PartNumber: &part2Num, Body: strings.NewReader(part2Body), }) @@ -493,9 +610,9 @@ func TestIntegration_MultipartUpload_HappyPath(t *testing.T) { // 4. CompleteMultipartUpload _, _, err = ib.backend.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ - Bucket: strPtr("bucket"), - Key: strPtr("big-file"), - UploadId: &uploadID, + Bucket: new("bucket"), + Key: new("big-file"), + UploadId: &contentID, MultipartUpload: &types.CompletedMultipartUpload{ Parts: []types.CompletedPart{ {PartNumber: &part1Num, ETag: part1Out.ETag}, @@ -531,79 +648,6 @@ func TestIntegration_MultipartUpload_HappyPath(t *testing.T) { } } -func TestIntegration_MultipartUpload_Abort(t *testing.T) { - ib := newIntegrationBackend(t) - ctx := context.Background() - - testutil.SeedBucket(t, ib.db, "bucket") - - // 1. CreateMultipartUpload - createOut, err := ib.backend.CreateMultipartUpload(ctx, s3response.CreateMultipartUploadInput{ - Bucket: strPtr("bucket"), - Key: strPtr("file"), - }) - if err != nil { - t.Fatalf("CreateMultipartUpload: %v", err) - } - uploadID := createOut.UploadId - - // 2. UploadPart - part1Num := int32(1) - _, err = ib.backend.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: strPtr("bucket"), - Key: strPtr("file"), - UploadId: &uploadID, - PartNumber: &part1Num, - Body: strings.NewReader("data"), - }) - if err != nil { - t.Fatalf("UploadPart: %v", err) - } - - // 3. AbortMultipartUpload - err = ib.backend.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ - Bucket: strPtr("bucket"), - Key: strPtr("file"), - UploadId: &uploadID, - }) - if err != nil { - t.Fatalf("AbortMultipartUpload: %v", err) - } - - // 4. Verify: upload should be in aborted status (getActiveUpload won't find it) - upload, err := ib.repos.Multiparts.GetByUploadID(ctx, uploadID) - if err != nil { - t.Fatalf("querying upload: %v", err) - } - if upload == nil { - t.Fatal("expected upload record to still exist (with aborted status)") - } - if upload.Status != model.MultipartStatusAborted { - t.Fatalf("expected status=aborted, got %s", upload.Status) - } - - // Parts should be cleaned up - parts, err := ib.repos.Multiparts.GetParts(ctx, uploadID, 0, 100) - if err != nil { - t.Fatalf("querying parts: %v", err) - } - if len(parts) != 0 { - t.Fatalf("expected 0 parts after abort, got %d", len(parts)) - } - - // No object should be created - var objects []model.Object - err = ib.db.NewSelect().Model(&objects). - Where("key = ?", "file"). - Scan(ctx) - if err != nil { - t.Fatalf("querying objects: %v", err) - } - if len(objects) != 0 { - t.Fatalf("expected 0 objects after abort, got %d", len(objects)) - } -} - func TestIntegration_StringAndShutdown(t *testing.T) { ib := newIntegrationBackend(t) @@ -623,10 +667,10 @@ func TestIntegration_ListMultipartUploads(t *testing.T) { testutil.SeedBucket(t, ib.db, "bucket") // Create 3 multipart uploads - for i := 0; i < 3; i++ { + for i := range 3 { key := fmt.Sprintf("multi-key-%d", i) _, err := ib.backend.CreateMultipartUpload(ctx, s3response.CreateMultipartUploadInput{ - Bucket: strPtr("bucket"), + Bucket: new("bucket"), Key: &key, }) if err != nil { @@ -636,7 +680,7 @@ func TestIntegration_ListMultipartUploads(t *testing.T) { // ListMultipartUploads listOut, err := ib.backend.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ - Bucket: strPtr("bucket"), + Bucket: new("bucket"), }) if err != nil { t.Fatalf("ListMultipartUploads: %v", err) @@ -651,7 +695,7 @@ func TestIntegration_ListMultipartUploads(t *testing.T) { // Verify MaxUploads pagination maxUploads := int32(1) listOut2, err := ib.backend.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ - Bucket: strPtr("bucket"), + Bucket: new("bucket"), MaxUploads: &maxUploads, }) if err != nil { @@ -673,22 +717,22 @@ func TestIntegration_ListParts(t *testing.T) { // Create multipart upload createOut, err := ib.backend.CreateMultipartUpload(ctx, s3response.CreateMultipartUploadInput{ - Bucket: strPtr("bucket"), - Key: strPtr("parts-file"), + Bucket: new("bucket"), + Key: new("parts-file"), }) if err != nil { t.Fatalf("CreateMultipartUpload: %v", err) } - uploadID := createOut.UploadId + contentID := createOut.UploadId // Upload 3 parts for i := int32(1); i <= 3; i++ { partNum := i body := fmt.Sprintf("part-%d-data", i) _, err := ib.backend.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: strPtr("bucket"), - Key: strPtr("parts-file"), - UploadId: &uploadID, + Bucket: new("bucket"), + Key: new("parts-file"), + UploadId: &contentID, PartNumber: &partNum, Body: strings.NewReader(body), }) @@ -699,9 +743,9 @@ func TestIntegration_ListParts(t *testing.T) { // ListParts listOut, err := ib.backend.ListParts(ctx, &s3.ListPartsInput{ - Bucket: strPtr("bucket"), - Key: strPtr("parts-file"), - UploadId: &uploadID, + Bucket: new("bucket"), + Key: new("parts-file"), + UploadId: &contentID, }) if err != nil { t.Fatalf("ListParts: %v", err) @@ -724,9 +768,9 @@ func TestIntegration_ListParts(t *testing.T) { // ListParts with MaxParts pagination maxParts := int32(1) listOut2, err := ib.backend.ListParts(ctx, &s3.ListPartsInput{ - Bucket: strPtr("bucket"), - Key: strPtr("parts-file"), - UploadId: &uploadID, + Bucket: new("bucket"), + Key: new("parts-file"), + UploadId: &contentID, MaxParts: &maxParts, }) if err != nil { @@ -751,21 +795,21 @@ func TestIntegration_UploadPartCopy(t *testing.T) { // Create a multipart upload createOut, err := ib.backend.CreateMultipartUpload(ctx, s3response.CreateMultipartUploadInput{ - Bucket: strPtr("bucket"), - Key: strPtr("copy-dst"), + Bucket: new("bucket"), + Key: new("copy-dst"), }) if err != nil { t.Fatalf("CreateMultipartUpload: %v", err) } - uploadID := createOut.UploadId + contentID := createOut.UploadId // UploadPartCopy: copy source into part 1 partNum := int32(1) copySource := "bucket/copy-src" partCopyOut, err := ib.backend.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ - Bucket: strPtr("bucket"), - Key: strPtr("copy-dst"), - UploadId: &uploadID, + Bucket: new("bucket"), + Key: new("copy-dst"), + UploadId: &contentID, PartNumber: &partNum, CopySource: ©Source, }) @@ -778,9 +822,9 @@ func TestIntegration_UploadPartCopy(t *testing.T) { // Complete the multipart upload with the copied part _, _, err = ib.backend.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ - Bucket: strPtr("bucket"), - Key: strPtr("copy-dst"), - UploadId: &uploadID, + Bucket: new("bucket"), + Key: new("copy-dst"), + UploadId: &contentID, MultipartUpload: &types.CompletedMultipartUpload{ Parts: []types.CompletedPart{ {PartNumber: &partNum, ETag: partCopyOut.ETag}, @@ -811,7 +855,7 @@ func TestIntegration_CopyObject_MetadataMatch(t *testing.T) { Bucket: &srcBucket, Key: &srcKey, Body: strings.NewReader(validTestObjectBody("copy-me")), - ContentType: strPtr("text/plain"), + ContentType: new("text/plain"), }) if err != nil { t.Fatalf("PutObject: %v", err) @@ -858,8 +902,10 @@ func TestIntegration_CopyObject_MetadataMatch(t *testing.T) { if dstObj.ContentType != srcObj.ContentType { t.Fatalf("content-type mismatch: src=%s dst=%s", srcObj.ContentType, dstObj.ContentType) } - if dstObj.State != model.ObjectStateUploading { - t.Fatalf("expected dst state=uploading, got %s", dstObj.State) + // Pipeline position is derived from the content's copies, and no copy exists + // until the ingest plan runs, so a fresh copy reads as cached. + if dstObj.State != model.ObjectStateCached { + t.Fatalf("expected dst state=cached, got %s", dstObj.State) } if dstObj.VersionID == "" { t.Fatal("expected destination current version id") @@ -881,11 +927,11 @@ func TestIntegration_DeleteObjects_BatchCreatesDeleteMarkers(t *testing.T) { putObject(t, ib.backend, "bucket", "file-b", "bbb") out, err := ib.backend.DeleteObjects(ctx, &s3.DeleteObjectsInput{ - Bucket: strPtr("bucket"), + Bucket: new("bucket"), Delete: &types.Delete{ Objects: []types.ObjectIdentifier{ - {Key: strPtr("file-a")}, - {Key: strPtr("file-b")}, + {Key: new("file-a")}, + {Key: new("file-b")}, }, }, }) @@ -910,7 +956,7 @@ func TestIntegration_DeleteObjects_BatchCreatesDeleteMarkers(t *testing.T) { } } - listOut, err := ib.backend.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: strPtr("bucket")}) + listOut, err := ib.backend.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: new("bucket")}) if err != nil { t.Fatalf("ListObjectsV2: %v", err) } @@ -918,7 +964,7 @@ func TestIntegration_DeleteObjects_BatchCreatesDeleteMarkers(t *testing.T) { t.Fatalf("ListObjectsV2 contents = %#v, want hidden objects", listOut.Contents) } - versionsOut, err := ib.backend.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: strPtr("bucket")}) + versionsOut, err := ib.backend.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: new("bucket")}) if err != nil { t.Fatalf("ListObjectVersions: %v", err) } @@ -942,7 +988,7 @@ func TestIntegration_ListObjectsV2_Pagination(t *testing.T) { // Page 1: MaxKeys=2 maxKeys := int32(2) out1, err := ib.backend.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ - Bucket: strPtr("bucket"), + Bucket: new("bucket"), MaxKeys: &maxKeys, }) if err != nil { @@ -966,7 +1012,7 @@ func TestIntegration_ListObjectsV2_Pagination(t *testing.T) { // Page 2 out2, err := ib.backend.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ - Bucket: strPtr("bucket"), + Bucket: new("bucket"), MaxKeys: &maxKeys, ContinuationToken: out1.NextContinuationToken, }) @@ -985,7 +1031,7 @@ func TestIntegration_ListObjectsV2_Pagination(t *testing.T) { // Page 3 (last page, should have 1 object) out3, err := ib.backend.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ - Bucket: strPtr("bucket"), + Bucket: new("bucket"), MaxKeys: &maxKeys, ContinuationToken: out2.NextContinuationToken, }) @@ -1026,7 +1072,7 @@ func TestIntegration_HeadObject(t *testing.T) { Bucket: &bucketName, Key: &keyName, Body: strings.NewReader(content), - ContentType: strPtr("application/json"), + ContentType: new("application/json"), }) if err != nil { t.Fatalf("PutObject: %v", err) @@ -1034,8 +1080,8 @@ func TestIntegration_HeadObject(t *testing.T) { // HeadObject headOut, err := ib.backend.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: strPtr("bucket"), - Key: strPtr("head-key"), + Bucket: new("bucket"), + Key: new("head-key"), }) if err != nil { t.Fatalf("HeadObject: %v", err) @@ -1084,23 +1130,19 @@ func TestIntegration_GetObject_CacheMiss_NoPieceCID(t *testing.T) { } // Manually delete from cache to simulate a cache miss - if err := ib.cache.Delete(ctx, "bucket", obj.CacheKey); err != nil { + if err := ib.cache.Delete(ctx, "bucket", obj.CacheKey()); err != nil { t.Fatalf("cache delete: %v", err) } - if ib.cache.Exists(ctx, "bucket", obj.CacheKey) { + if ib.cache.Exists(ctx, "bucket", obj.CacheKey()) { t.Fatal("expected cache file to be gone") } // GetObject should fail — object is in DB but no cache and no PieceCID for SP download _, err = ib.backend.GetObject(ctx, &s3.GetObjectInput{ - Bucket: strPtr("bucket"), - Key: strPtr("evicted-key"), + Bucket: new("bucket"), + Key: new("evicted-key"), }) if err == nil { t.Fatal("expected GetObject to fail on cache miss with no PieceCID") } } - -func strPtr(s string) *string { - return &s -} diff --git a/internal/backend/multipart.go b/internal/backend/multipart.go index 5d3e156..ac2d5e5 100644 --- a/internal/backend/multipart.go +++ b/internal/backend/multipart.go @@ -26,7 +26,7 @@ func (b *SynapseBackend) CreateMultipartUpload(ctx context.Context, input s3resp return s3response.InitiateMultipartUploadResult{}, err } - bucket, err := b.requireActiveBucket(ctx, bucketName) + bucket, err := b.requireWritableBucket(ctx, bucketName) if err != nil { return s3response.InitiateMultipartUploadResult{}, err } @@ -276,13 +276,15 @@ func (b *SynapseBackend) CompleteMultipartUpload(ctx context.Context, input *s3. } versionID := model.NewVersionID() - cacheKey := versionCacheKey(versionID) - // Assemble parts into a version-specific cache key. - cacheInfo, _, err := b.cache.AssembleParts(ctx, bucketName, cacheKey, uploadID, partNumbers) + // Assemble into a staged file; the destination is content-addressed and + // only nameable once the assembled checksum resolves a content row. + staged, _, err := b.cache.AssemblePartsStaged(ctx, bucketName, stagingCacheKey(versionID), uploadID, partNumbers) if err != nil { return s3response.CompleteMultipartUploadResult{}, "", fmt.Errorf("assembling parts: %w", err) } + defer func() { _ = staged.Rollback() }() + cacheInfo := staged.Info // Compute S3 multipart ETag from DB-recorded ETags (source of truth, not re-derived from files) orderedETags := make([]string, len(partNumbers)) @@ -296,44 +298,56 @@ func (b *SynapseBackend) CompleteMultipartUpload(ctx context.Context, input *s3. // Atomic: create object version + enqueue any needed task + finalize upload status var objectID int64 - var createdState model.ObjectState - if err := b.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { - reuse, err := b.resolveVersionReuse(ctx, txRepos.Objects, upload.BucketID, cacheInfo.Size, cacheInfo.Checksum) - if err != nil { - return err + mpBucket, err := b.repos.Buckets.GetByID(ctx, upload.BucketID) + if err != nil { + return s3response.CompleteMultipartUploadResult{}, "", fmt.Errorf("loading bucket for multipart completion: %w", err) + } + content, err := b.ensureContentForBytes(ctx, b.repos, mpBucket, cacheInfo.Size, cacheInfo.Checksum) + if err != nil { + return s3response.CompleteMultipartUploadResult{}, "", err + } + cacheKey := model.ContentCacheKey(content.ID) + cacheCommitted := false + err = b.cacheGate.Commit(cacheKey, func() error { + if err := staged.CommitAs(bucketName, cacheKey); err != nil { + return fmt.Errorf("committing assembled cache file: %w", err) } + cacheCommitted = true + return b.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { + state, err := txRepos.Contents.ContentPipelineState(ctx, content.ID) + if err != nil { + return err + } - version := &model.ObjectVersion{ - VersionID: versionID, - BucketID: upload.BucketID, - Key: keyName, - Size: cacheInfo.Size, - ETag: s3ETag, - Checksum: cacheInfo.Checksum, - ContentType: upload.ContentType, - Metadata: upload.Metadata, - CacheKey: cacheKey, - MultipartUploadID: &uploadID, - StorageUploadID: reuse.StorageUploadID, - InCache: true, - State: reuse.State, - } - objectID, err = txRepos.Objects.CreateVersionAndSetCurrent(ctx, version) - if err != nil { - return fmt.Errorf("creating assembled object version: %w", err) - } - createdState = version.State - if err := b.enqueuePostWriteTask(ctx, txRepos, objectID, versionID, version.State); err != nil { - return err - } + version := &model.ObjectVersion{ + VersionID: versionID, + BucketID: upload.BucketID, + Key: keyName, + Size: cacheInfo.Size, + ETag: s3ETag, + ContentType: upload.ContentType, + Metadata: upload.Metadata, + MultipartUploadID: &uploadID, + ContentID: &content.ID, + } + objectID, err = txRepos.Objects.CreateVersionAndSetCurrent(ctx, version) + if err != nil { + return fmt.Errorf("creating assembled object version: %w", err) + } + if err := b.enqueuePostWriteTask(ctx, txRepos, objectID, versionID, version.ContentID, state); err != nil { + return err + } - return txRepos.Multiparts.SetStatus(ctx, uploadID, model.MultipartStatusCompleting, model.MultipartStatusCompleted) - }); err != nil { - b.deleteVersionCacheBestEffort(ctx, bucketName, cacheKey, "orphaned multipart version cache file after complete tx failure") + return txRepos.Multiparts.SetStatus(ctx, uploadID, model.MultipartStatusCompleting, model.MultipartStatusCompleted) + }) + }) + if err != nil { + if cacheCommitted { + b.releaseContentCacheIfUnreferenced(ctx, bucketName, content.ID, "orphaned content cache file after multipart complete tx failure") + } return s3response.CompleteMultipartUploadResult{}, "", err } completed = true - b.completeFollowerIfStoredReuseWonRace(ctx, upload.BucketID, bucketName, cacheInfo.Size, cacheInfo.Checksum, objectID, versionID, createdState) // Clean up multipart parts from cache (best-effort) _ = b.cache.DeleteUpload(ctx, uploadID) diff --git a/internal/backend/multipart_test.go b/internal/backend/multipart_test.go index 33dabd3..107e107 100644 --- a/internal/backend/multipart_test.go +++ b/internal/backend/multipart_test.go @@ -2,6 +2,7 @@ package backend_test import ( "context" + "errors" "fmt" "io" "strings" @@ -11,7 +12,6 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - synaps3backend "github.com/strahe/synaps3/internal/backend" "github.com/strahe/synaps3/internal/cache" "github.com/strahe/synaps3/internal/model" synaps3testutil "github.com/strahe/synaps3/internal/testutil" @@ -251,7 +251,7 @@ func TestUploadPartCopy_CopySourceVersionIDCopiesSpecifiedVersion(t *testing.T) // ---------- CompleteMultipartUpload ---------- func TestCompleteMultipartUpload_HappyPath(t *testing.T) { - tb := newTestBackendWithOptions(t, synaps3backend.WithUploadMaxRetries(12)) + tb := newTestBackend(t) ctx := context.Background() seedActiveBucket(t, tb, "cmp-bucket") @@ -324,15 +324,15 @@ func TestCompleteMultipartUpload_HappyPath(t *testing.T) { if obj.MultipartUploadID == nil || *obj.MultipartUploadID != uploadID { t.Fatalf("object multipart_upload_id = %v, want %s", obj.MultipartUploadID, uploadID) } - task, err := tb.repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) + task, err := tb.repos.Tasks.ClaimNext(ctx, time.Minute) if err != nil { - t.Fatalf("ClaimReady: %v", err) + t.Fatalf("ClaimNext: %v", err) } if task == nil { t.Fatal("expected upload task") } - if task.MaxRetries != 12 { - t.Fatalf("task MaxRetries = %d, want 12", task.MaxRetries) + if task.Type != model.TaskTypeUploadPlan || task.RetryLimit == nil || *task.RetryLimit != 5 { + t.Fatalf("task = %#v, want upload_plan with retry limit 5", task) } } @@ -374,9 +374,9 @@ func TestCompleteMultipartUploadRejectsFOCSizeBelowMinimum(t *testing.T) { func TestCompleteMultipartUploadRejectsFOCSizeAboveMaximum(t *testing.T) { assembleCalled := false mc := &synaps3testutil.MockCache{ - AssemblePartsFunc: func(_ context.Context, _, _, _ string, _ []int) (*cache.ObjectInfo, []string, error) { + AssemblePartsFunc: func(_ context.Context, _, _, _ string, _ []int) (*cache.StagedObject, []string, error) { assembleCalled = true - return &cache.ObjectInfo{Size: chain.MaxUploadSize + 1, ETag: strings.Repeat("a", 32), Checksum: "checksum"}, nil, nil + return nil, nil, errors.New("assemble must not run for an oversize object") }, } tb := newTestBackendWithMockCache(t, mc) @@ -492,7 +492,7 @@ func TestCompleteMultipartUploadIdenticalCurrentObjectCreatesNewVersion(t *testi taskCount, err := tb.db.NewSelect(). Model((*model.Task)(nil)). - Where("ref_type = ? AND ref_id = ?", "object", obj1.ObjectID). + Where("type = ?", model.TaskTypeUploadPlan). Count(ctx) if err != nil { t.Fatalf("counting upload tasks: %v", err) @@ -505,8 +505,10 @@ func TestCompleteMultipartUploadIdenticalCurrentObjectCreatesNewVersion(t *testi if err != nil || secondVersion == nil { t.Fatalf("second version: version=%v err=%v", secondVersion, err) } - if secondVersion.State != model.ObjectStateUploading { - t.Fatalf("second version state = %s, want uploading", secondVersion.State) + // Both versions share one content whose ingest plan has not produced a copy + // yet, so the derived position is still cached. + if secondVersion.State != model.ObjectStateCached { + t.Fatalf("second version state = %s, want cached", secondVersion.State) } } @@ -572,7 +574,7 @@ func TestListMultipartUploads_HappyPath(t *testing.T) { seedActiveBucket(t, tb, "lmu-bucket") ct := "application/octet-stream" - for i := 0; i < 3; i++ { + for i := range 3 { _, err := tb.backend.CreateMultipartUpload(ctx, s3response.CreateMultipartUploadInput{ Bucket: aws.String("lmu-bucket"), Key: aws.String(fmt.Sprintf("file-%d.bin", i)), diff --git a/internal/backend/object.go b/internal/backend/object.go index e323619..80ffa4d 100644 --- a/internal/backend/object.go +++ b/internal/backend/object.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" "path" + "slices" "strconv" "strings" "time" @@ -16,12 +17,16 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/strahe/synaps3/internal/admin" + "github.com/strahe/synaps3/internal/cacheeviction" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/objectdeletion" "github.com/strahe/synaps3/internal/objectkey" "github.com/strahe/synaps3/internal/objectlimits" "github.com/strahe/synaps3/internal/objectreader" + "github.com/strahe/synaps3/internal/storagecleanup" + "github.com/strahe/synaps3/internal/storagepipeline" + taskengine "github.com/strahe/synaps3/internal/task" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3response" ) @@ -34,13 +39,12 @@ func (b *SynapseBackend) PutObject(ctx context.Context, input s3response.PutObje return s3response.PutObjectOutput{}, err } - bucket, err := b.requireActiveBucket(ctx, bucketName) + bucket, err := b.requireWritableBucket(ctx, bucketName) if err != nil { return s3response.PutObjectOutput{}, err } versionID := model.NewVersionID() - cacheKey := versionCacheKey(versionID) if input.ContentLength != nil { if err := objectlimits.ValidateFOCUploadSize(*input.ContentLength); err != nil { @@ -49,8 +53,9 @@ func (b *SynapseBackend) PutObject(ctx context.Context, input s3response.PutObje } } - // Write to a version-specific cache key so overwrites cannot affect older tasks. - staged, err := b.cache.PutStaged(ctx, bucketName, cacheKey, objectlimits.LimitFOCUploadReader(input.Body)) + // Stage beside the content directory: the destination is content-addressed + // and only nameable once the staged checksum resolves a content row. + staged, err := b.cache.PutStaged(ctx, bucketName, stagingCacheKey(versionID), objectlimits.LimitFOCUploadReader(input.Body)) if err != nil { admin.ObjectOperationsTotal.WithLabelValues("put", "failure").Inc() if errors.Is(err, objectlimits.ErrTooLarge) { @@ -73,46 +78,52 @@ func (b *SynapseBackend) PutObject(ctx context.Context, input s3response.PutObje } contentType := stringOrDefault(input.ContentType, "application/octet-stream") - if err := staged.Commit(); err != nil { + content, err := b.ensureContentForBytes(ctx, b.repos, bucket, cacheInfo.Size, cacheInfo.Checksum) + if err != nil { admin.ObjectOperationsTotal.WithLabelValues("put", "failure").Inc() - return s3response.PutObjectOutput{}, fmt.Errorf("committing cache file: %w", err) + return s3response.PutObjectOutput{}, err } - - // Atomic transaction: create object version + enqueue any needed task. + cacheKey := model.ContentCacheKey(content.ID) var objectID int64 - var createdState model.ObjectState - if err := b.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { - reuse, err := b.resolveVersionReuse(ctx, txRepos.Objects, bucket.ID, cacheInfo.Size, cacheInfo.Checksum) - if err != nil { - return err + cacheCommitted := false + // The content gate stays held from the physical commit through the database + // transaction. A deletion therefore observes either the old state or the + // new file and its committed version/presence together. + err = b.cacheGate.Commit(cacheKey, func() error { + if err := staged.CommitAs(bucketName, cacheKey); err != nil { + return fmt.Errorf("committing cache file: %w", err) } + cacheCommitted = true + return b.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { + state, err := txRepos.Contents.ContentPipelineState(ctx, content.ID) + if err != nil { + return err + } - version := &model.ObjectVersion{ - VersionID: versionID, - BucketID: bucket.ID, - Key: keyName, - Size: cacheInfo.Size, - ETag: cacheInfo.ETag, - Checksum: cacheInfo.Checksum, - ContentType: contentType, - Metadata: meta, - CacheKey: cacheKey, - StorageUploadID: reuse.StorageUploadID, - InCache: true, - State: reuse.State, - } - objectID, err = txRepos.Objects.CreateVersionAndSetCurrent(ctx, version) - if err != nil { - return fmt.Errorf("creating object version: %w", err) - } - createdState = version.State - return b.enqueuePostWriteTask(ctx, txRepos, objectID, versionID, version.State) - }); err != nil { + version := &model.ObjectVersion{ + VersionID: versionID, + BucketID: bucket.ID, + Key: keyName, + Size: cacheInfo.Size, + ETag: cacheInfo.ETag, + ContentType: contentType, + Metadata: meta, + ContentID: &content.ID, + } + objectID, err = txRepos.Objects.CreateVersionAndSetCurrent(ctx, version) + if err != nil { + return fmt.Errorf("creating object version: %w", err) + } + return b.enqueuePostWriteTask(ctx, txRepos, objectID, versionID, version.ContentID, state) + }) + }) + if err != nil { admin.ObjectOperationsTotal.WithLabelValues("put", "failure").Inc() - b.deleteVersionCacheBestEffort(ctx, bucketName, cacheKey, "orphaned version cache file after put tx failure") + if cacheCommitted { + b.releaseContentCacheIfUnreferenced(ctx, bucketName, content.ID, "orphaned content cache file after put tx failure") + } return s3response.PutObjectOutput{}, err } - b.completeFollowerIfStoredReuseWonRace(ctx, bucket.ID, bucketName, cacheInfo.Size, cacheInfo.Checksum, objectID, versionID, createdState) b.logger.Info("object stored", "bucket", bucketName, "key", keyName, "size", cacheInfo.Size, "versionID", versionID) admin.ObjectOperationsTotal.WithLabelValues("put", "success").Inc() @@ -430,7 +441,7 @@ func (b *SynapseBackend) DeleteObject(ctx context.Context, input *s3.DeleteObjec ) } - bucket, err := b.requireActiveBucket(ctx, *input.Bucket) + bucket, err := b.requireWritableBucket(ctx, *input.Bucket) if err != nil { return nil, err } @@ -448,7 +459,7 @@ func (b *SynapseBackend) DeleteObjects(ctx context.Context, input *s3.DeleteObje ) } - bucket, err := b.requireActiveBucket(ctx, *input.Bucket) + bucket, err := b.requireWritableBucket(ctx, *input.Bucket) if err != nil { return s3response.DeleteResult{}, err } @@ -499,11 +510,16 @@ func (b *SynapseBackend) deleteObjectInBucket(ctx context.Context, bucket *model return nil, s3err.GetAPIError(s3err.ErrNoSuchVersion) } if !version.IsDeleteMarker { - result, err := b.repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: bucket.ID, - Key: key, - VersionID: versionID, - StorageCleanupMaxRetries: &b.storageCleanupMaxRetries, + var result repository.DeleteObjectVersionResult + err := b.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { + var deleteErr error + result, deleteErr = txRepos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ + BucketID: bucket.ID, Key: key, VersionID: versionID, + }) + if deleteErr != nil { + return deleteErr + } + return b.bindStorageCleanupTask(ctx, txRepos, result.StorageCleanup) }) if err != nil { switch { @@ -525,7 +541,7 @@ func (b *SynapseBackend) deleteObjectInBucket(ctx context.Context, bucket *model return nil, fmt.Errorf("permanently deleting object version: %w", err) } } - b.recordPermanentDeleteCacheCleanup(ctx, bucket.Name, versionID, result.CacheKey) + b.releaseContentCache(ctx, bucket.Name, result.ContentID) return &s3.DeleteObjectOutput{ VersionId: &versionID, }, nil @@ -560,8 +576,7 @@ func deleteObjectsDeletedObject(obj types.ObjectIdentifier, out *s3.DeleteObject } func deleteObjectsEntryError(key *string, versionID *string, err error) types.Error { - var s3Err s3err.S3Error - if errors.As(err, &s3Err) { + if s3Err, ok := errors.AsType[s3err.S3Error](err); ok { apiErr := s3Err.BaseError() code := apiErr.Code message := apiErr.Description @@ -582,18 +597,24 @@ func deleteObjectsEntryError(key *string, versionID *string, err error) types.Er } } -func (b *SynapseBackend) recordPermanentDeleteCacheCleanup(ctx context.Context, bucketName string, versionID string, cacheKey string) { - objectdeletion.RecordCacheCleanup( +// releaseContentCache frees cached bytes only once the deletion removed the +// last live reference to that content. Residency is content-addressed, so bytes +// another version still names must survive this deletion. +func (b *SynapseBackend) releaseContentCache(ctx context.Context, bucketName string, contentID *int64) { + if contentID == nil { + return + } + if _, err := objectdeletion.ReleaseContentCache( ctx, b.cache, b.cacheGate, b.cacheAccessTracker, b.repos.Objects, - b.logger, bucketName, - versionID, - cacheKey, - ) + *contentID, + ); err != nil { + b.logger.Warn("releasing content cache failed", "bucket", bucketName, "contentID", *contentID, "error", err) + } } func (b *SynapseBackend) CopyObject(ctx context.Context, input s3response.CopyObjectInput) (s3response.CopyObjectOutput, error) { @@ -623,7 +644,7 @@ func (b *SynapseBackend) CopyObject(ctx context.Context, input s3response.CopyOb return s3response.CopyObjectOutput{}, err } - dstBucket, err := b.requireActiveBucket(ctx, dstBucketName) + dstBucket, err := b.requireWritableBucket(ctx, dstBucketName) if err != nil { return s3response.CopyObjectOutput{}, err } @@ -696,11 +717,10 @@ func (b *SynapseBackend) copyObjectVersion(ctx context.Context, input copyObject if err != nil { return copyObjectVersionResult{}, err } - defer func() { _ = srcResult.Body.Close() }() versionID := model.NewVersionID() - cacheKey := versionCacheKey(versionID) - staged, err := b.cache.PutStaged(ctx, input.DestinationBucket.Name, cacheKey, objectlimits.LimitFOCUploadReader(srcResult.Body)) + staged, err := b.cache.PutStaged(ctx, input.DestinationBucket.Name, stagingCacheKey(versionID), objectlimits.LimitFOCUploadReader(srcResult.Body)) + _ = srcResult.Body.Close() if err != nil { return copyObjectVersionResult{}, fmt.Errorf("staging copy destination: %w", err) } @@ -718,70 +738,61 @@ func (b *SynapseBackend) copyObjectVersion(ctx context.Context, input copyObject } contentType = stringOrDefault(input.ContentType, "application/octet-stream") } else { - for key, value := range input.SourceVersion.Metadata { - metadata[key] = value - } + maps.Copy(metadata, input.SourceVersion.Metadata) } - if err := staged.Commit(); err != nil { - return copyObjectVersionResult{}, fmt.Errorf("committing copy cache: %w", err) + // Content is bucket-scoped, so a copy always resolves content in the + // destination bucket, whether or not the source shares it. + content, err := b.ensureContentForBytes(ctx, b.repos, input.DestinationBucket, cacheInfo.Size, cacheInfo.Checksum) + if err != nil { + return copyObjectVersionResult{}, err } - + cacheKey := model.ContentCacheKey(content.ID) var objectID int64 - var createdState model.ObjectState - if err := b.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { - reuse := versionStorageReuse{State: model.ObjectStateCached} - if input.SourceBucket.ID == input.DestinationBucket.ID { - var err error - reuse, err = b.resolveVersionReuse(ctx, txRepos.Objects, input.DestinationBucket.ID, cacheInfo.Size, cacheInfo.Checksum) + cacheCommitted := false + err = b.cacheGate.Commit(cacheKey, func() error { + if err := staged.CommitAs(input.DestinationBucket.Name, cacheKey); err != nil { + return fmt.Errorf("committing copy cache: %w", err) + } + cacheCommitted = true + return b.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { + state, err := txRepos.Contents.ContentPipelineState(ctx, content.ID) if err != nil { return err } - } - version := &model.ObjectVersion{ - VersionID: versionID, - BucketID: input.DestinationBucket.ID, - Key: input.DestinationKey, - Size: cacheInfo.Size, - ETag: cacheInfo.ETag, - Checksum: cacheInfo.Checksum, - ContentType: contentType, - Metadata: metadata, - CacheKey: cacheKey, - StorageUploadID: reuse.StorageUploadID, - InCache: true, - State: reuse.State, - } - if input.Restore == nil { - objectID, err = txRepos.Objects.CreateVersionAndSetCurrent(ctx, version) - } else { - objectID, err = txRepos.Objects.CreateRestoredVersionAndSetCurrent( - ctx, - version, - input.Restore.SourceVersionID, - input.Restore.ExpectedCurrentVersionID, - ) - } - if err != nil { - return fmt.Errorf("creating copy destination version: %w", err) + version := &model.ObjectVersion{ + VersionID: versionID, + BucketID: input.DestinationBucket.ID, + Key: input.DestinationKey, + Size: cacheInfo.Size, + ETag: cacheInfo.ETag, + ContentType: contentType, + Metadata: metadata, + ContentID: &content.ID, + } + if input.Restore == nil { + objectID, err = txRepos.Objects.CreateVersionAndSetCurrent(ctx, version) + } else { + objectID, err = txRepos.Objects.CreateRestoredVersionAndSetCurrent( + ctx, + version, + input.Restore.SourceVersionID, + input.Restore.ExpectedCurrentVersionID, + ) + } + if err != nil { + return fmt.Errorf("creating copy destination version: %w", err) + } + return b.enqueuePostWriteTask(ctx, txRepos, objectID, versionID, version.ContentID, state) + }) + }) + if err != nil { + if cacheCommitted { + b.releaseContentCacheIfUnreferenced(ctx, input.DestinationBucket.Name, content.ID, "orphaned content cache file after copy tx failure") } - createdState = version.State - return b.enqueuePostWriteTask(ctx, txRepos, objectID, versionID, version.State) - }); err != nil { - b.deleteVersionCacheBestEffort(ctx, input.DestinationBucket.Name, cacheKey, "orphaned version cache file after copy tx failure") return copyObjectVersionResult{}, err } - b.completeFollowerIfStoredReuseWonRace( - ctx, - input.DestinationBucket.ID, - input.DestinationBucket.Name, - cacheInfo.Size, - cacheInfo.Checksum, - objectID, - versionID, - createdState, - ) return copyObjectVersionResult{ SourceVersionID: input.SourceVersion.VersionID, @@ -991,7 +1002,6 @@ func (b *SynapseBackend) ListObjectsV2(ctx context.Context, input *s3.ListObject } // getBucket retrieves a bucket visible to S3 clients. -// Rejects deleted, create_failed, and delete_failed statuses. func (b *SynapseBackend) getBucket(ctx context.Context, name string) (*model.Bucket, error) { bucket, err := b.repos.Buckets.GetByName(ctx, name) if err != nil { @@ -1003,16 +1013,20 @@ func (b *SynapseBackend) getBucket(ctx context.Context, name string) (*model.Buc return bucket, nil } -// requireActiveBucket retrieves a bucket that accepts write operations. -// Active and creating buckets are writable; deleting/failed buckets are rejected. -func (b *SynapseBackend) requireActiveBucket(ctx context.Context, name string) (*model.Bucket, error) { +// requireWritableBucket retrieves a bucket that accepts write operations. +func (b *SynapseBackend) requireWritableBucket(ctx context.Context, name string) (*model.Bucket, error) { bucket, err := b.repos.Buckets.GetByName(ctx, name) if err != nil { return nil, fmt.Errorf("querying bucket: %w", err) } - if bucket == nil || !bucket.Status.IsWritable() { + if bucket == nil || !bucket.Status.IsVisible() { return nil, s3err.GetAPIError(s3err.ErrNoSuchBucket) } + if !bucket.Status.IsWritable() { + apiErr := s3err.GetAPIError(s3err.ErrSlowDown) + apiErr.Description = "The bucket is still being prepared. Please retry shortly." + return nil, apiErr + } return bucket, nil } @@ -1071,15 +1085,10 @@ func objectPartsRequested(attrs []types.ObjectAttributes) bool { if len(attrs) == 0 { return true } - for _, attr := range attrs { - if attr == types.ObjectAttributesObjectParts { - return true - } - } - return false + return slices.Contains(attrs, types.ObjectAttributesObjectParts) } -func (b *SynapseBackend) getObjectAttributeParts(ctx context.Context, uploadID *string, input *s3.GetObjectAttributesInput) (*s3response.ObjectParts, error) { +func (b *SynapseBackend) getObjectAttributeParts(ctx context.Context, contentID *string, input *s3.GetObjectAttributesInput) (*s3response.ObjectParts, error) { maxParts := 1000 if input.MaxParts != nil { maxParts = int(*input.MaxParts) @@ -1104,11 +1113,11 @@ func (b *SynapseBackend) getObjectAttributeParts(ctx context.Context, uploadID * MaxParts: maxParts, PartNumberMarker: partMarker, } - if uploadID == nil || *uploadID == "" { + if contentID == nil || *contentID == "" { return result, nil } - parts, err := b.repos.Multiparts.GetParts(ctx, *uploadID, partMarker, maxParts+1) + parts, err := b.repos.Multiparts.GetParts(ctx, *contentID, partMarker, maxParts+1) if err != nil { return nil, fmt.Errorf("listing object attribute parts: %w", err) } @@ -1168,162 +1177,103 @@ func (b *SynapseBackend) versionForRead(ctx context.Context, bucketID int64, key return version, nil } -type versionStorageReuse struct { - State model.ObjectState - StorageUploadID *int64 -} - -func (b *SynapseBackend) resolveVersionReuse(ctx context.Context, objects repository.ObjectRepository, bucketID int64, size int64, checksum string) (versionStorageReuse, error) { - reuse := versionStorageReuse{State: model.ObjectStateCached} +// ensureContentForBytes binds this write to the content identity for its bytes, +// creating that identity on first sight. The bytes have their own row with a +// unique key, so a second write of the same content is a single upsert and the +// pipeline position is read from that content's copies rather than inferred +// from whatever version happened to be found first. +func (b *SynapseBackend) ensureContentForBytes( + ctx context.Context, + repos *repository.Repositories, + bucket *model.Bucket, + size int64, + checksum string, +) (*model.StorageContent, error) { if checksum == "" { - return reuse, nil + return nil, errors.New("cannot identify content without a checksum") } - - stored, err := objects.FindReusableStoredVersion(ctx, bucketID, size, checksum) - if err != nil { - return reuse, err + requestedCopies := b.defaultCopies + if bucket.DefaultCopies > 0 { + requestedCopies = bucket.DefaultCopies } - if stored != nil { - reuse.State = model.ObjectStateStored - reuse.StorageUploadID = stored.StorageUploadID - return reuse, nil - } - - replicating, err := objects.FindReusableReplicatingVersion(ctx, bucketID, size, checksum) - if err != nil { - return reuse, err - } - if replicating != nil { - reuse.State = model.ObjectStateReplicating - reuse.StorageUploadID = replicating.StorageUploadID - return reuse, nil - } - - active, err := objects.FindReusableActiveUploadVersion(ctx, bucketID, size, checksum) - if err != nil { - return reuse, err - } - if active != nil { - reuse.State = model.ObjectStateUploading - } - return reuse, nil -} - -func (b *SynapseBackend) reusableStoredVersion(ctx context.Context, bucketID int64, size int64, checksum string) (*model.ObjectVersion, error) { - if checksum == "" { - return nil, nil - } - return b.repos.Objects.FindReusableStoredVersion(ctx, bucketID, size, checksum) + return repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, + ContentSize: size, + Checksum: checksum, + RequestedCopies: model.ClampStorageCopies(requestedCopies), + }) } -func (b *SynapseBackend) completeFollowerIfStoredReuseWonRace(ctx context.Context, bucketID int64, bucketName string, size int64, checksum string, objectID int64, versionID string, createdState model.ObjectState) { - if createdState == model.ObjectStateReplicating { - b.completeReplicatingFollowerIfUploadFinalized(ctx, bucketName, versionID) - return +func (b *SynapseBackend) enqueuePostWriteTask(ctx context.Context, repos *repository.Repositories, _ int64, versionID string, contentID *int64, state model.ObjectState) error { + if b.taskService == nil { + return errors.New("task service is unavailable") } - if createdState != model.ObjectStateUploading || checksum == "" { - return - } - - reusable, err := b.reusableStoredVersion(ctx, bucketID, size, checksum) - if err != nil { - b.logger.Warn("checking stored reuse after active upload follower write", "bucket", bucketName, "versionID", versionID, "error", err) - return - } - if reusable == nil || reusable.StorageUploadID == nil { - replicating, repErr := b.repos.Objects.FindReusableReplicatingVersion(ctx, bucketID, size, checksum) - if repErr != nil { - b.logger.Warn("checking replicating reuse after active upload follower write", "bucket", bucketName, "versionID", versionID, "error", repErr) - return - } - if replicating == nil || replicating.StorageUploadID == nil { - return - } - if refs, bindErr := b.repos.Uploads.BindReadableUploadForVersion(ctx, repository.BindReadableUploadForVersionInput{ - UploadID: *replicating.StorageUploadID, - BucketID: bucketID, - ContentSize: size, - Checksum: checksum, - VersionID: versionID, - }); bindErr != nil { - b.logger.Debug("active upload follower was not ready for readable copy reuse", "bucket", bucketName, "versionID", versionID, "error", bindErr) - } else if len(refs) > 0 { - b.completeReplicatingFollowerIfUploadFinalized(ctx, bucketName, versionID) + switch state { + case model.ObjectStateCached: + if contentID == nil { + return nil } - return - } - - if err := b.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { - if err := txRepos.Objects.SetVersionStorageUploadAndTransition( - ctx, - versionID, - *reusable.StorageUploadID, - model.ObjectStateUploading, - model.ObjectStateStored, - ); err != nil { - return err + _, _, err := b.taskService.EnqueueOrReactivateTerminalInTransaction(ctx, repos, taskengine.EnqueueRequest{ + Type: model.TaskTypeUploadPlan, + IdempotencyKey: storagepipeline.UploadPlanKey(*contentID), + Input: storagepipeline.UploadPlanInput{ContentID: *contentID}, + SubjectType: "storage_content", + SubjectKey: strconv.FormatInt(*contentID, 10), + }) + return err + case model.ObjectStateStored: + if !b.evictionPolicy.EnqueuesAfterUploadEviction() || contentID == nil { + return nil } - return b.enqueuePostWriteTask( - ctx, - txRepos, - objectID, - versionID, - model.ObjectStateStored, - ) - }); err != nil { - b.logger.Debug("active upload follower already handled or still pending", "bucket", bucketName, "versionID", versionID, "error", err) - return + return b.enqueueEvictionTask(ctx, repos, *contentID) + default: + return nil } } -func (b *SynapseBackend) completeReplicatingFollowerIfUploadFinalized(ctx context.Context, bucketName string, versionID string) { - version, err := b.repos.Objects.GetVersionByID(ctx, versionID) +// enqueueEvictionTask schedules cache removal for one content payload. Several +// versions can name the same bytes, so the unit of eviction is the content. +func (b *SynapseBackend) enqueueEvictionTask(ctx context.Context, repos *repository.Repositories, contentID int64) error { + reservation, err := repos.CacheEvictions.PrepareEviction(ctx, contentID) if err != nil { - b.logger.Warn("checking replicating reuse after follower write", "bucket", bucketName, "versionID", versionID, "error", err) - return + return err } - if version == nil || version.State != model.ObjectStateReplicating || version.StorageUploadID == nil { - return + if reservation.ActiveTaskID != nil { + return nil } - _, _, err = b.repos.Uploads.FinalizeUploadIfTargetCopiesMet( - ctx, - repository.NewFinalizeUploadInput( - *version.StorageUploadID, - b.evictionPolicy.EnqueuesAfterUploadEviction(), - b.evictMaxRetries, - ), - ) + generation := reservation.Generation + taskRow, _, err := b.taskService.EnqueueInTransaction(ctx, repos, taskengine.EnqueueRequest{ + Type: model.TaskTypeCacheEvict, + IdempotencyKey: cacheeviction.EvictTaskKey(contentID, generation), + Input: cacheeviction.EvictInput{ContentID: contentID, Generation: generation}, + SubjectType: "storage_content", + SubjectKey: strconv.FormatInt(contentID, 10), + }) if err != nil { - b.logger.Warn("finalizing replicating reuse after follower write", "bucket", bucketName, "versionID", versionID, "uploadID", *version.StorageUploadID, "error", err) - return + return err } + return repos.CacheEvictions.BindEvictionTask(ctx, contentID, generation, taskRow.ID) } -func (b *SynapseBackend) enqueuePostWriteTask(ctx context.Context, repos *repository.Repositories, objectID int64, versionID string, state model.ObjectState) error { - switch state { - case model.ObjectStateCached: - stage := "prepare_upload" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objectID, - RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("upload:%s", versionID), - Status: model.TaskStatusQueued, - MaxRetries: b.uploadMaxRetries, - ScheduledAt: time.Now(), - } - return repos.Tasks.Create(ctx, task) - case model.ObjectStateStored: - if !b.evictionPolicy.EnqueuesAfterUploadEviction() { - return nil - } - _, err := repos.CacheEvictions.EnsureAfterUploadTask(ctx, objectID, versionID, b.evictMaxRetries) - return err - default: +func (b *SynapseBackend) bindStorageCleanupTask(ctx context.Context, repos *repository.Repositories, cleanup *repository.StorageCleanupReservation) error { + if cleanup == nil || cleanup.TaskID != nil { return nil } + taskRow, _, err := b.taskService.EnqueueInTransaction(ctx, repos, taskengine.EnqueueRequest{ + Type: model.TaskTypeStorageCleanup, + IdempotencyKey: storagecleanup.TaskKey(cleanup.ContentID, cleanup.Generation), + Input: storagecleanup.Input{ContentID: cleanup.ContentID, Generation: cleanup.Generation}, + SubjectType: "storage_content", + SubjectKey: strconv.FormatInt(cleanup.ContentID, 10), + }) + if err != nil { + return err + } + if err := repos.StorageCleanup.BindTask(ctx, cleanup.ContentID, cleanup.Generation, taskRow.ID); err != nil { + return err + } + cleanup.TaskID = &taskRow.ID + return nil } func (b *SynapseBackend) objectReaderError(err error) error { @@ -1348,15 +1298,30 @@ func (b *SynapseBackend) objectReaderError(err error) error { } } -func versionCacheKey(versionID string) string { - return path.Join(".versions", versionID) +// stagingCacheKey names a not-yet-committed write inside the content directory, +// so the staged file and its final content-addressed name share a directory and +// the commit stays a single rename. +func stagingCacheKey(versionID string) string { + return path.Join(".contents", ".staging-"+versionID) } -func (b *SynapseBackend) deleteVersionCacheBestEffort(ctx context.Context, bucketName, cacheKey, message string) { +// releaseContentCacheIfUnreferenced drops cached bytes left behind by a failed +// write. Residency is content-addressed, so the file may already back a version +// a concurrent writer created on the same content and can only go when nothing +// names it. +func (b *SynapseBackend) releaseContentCacheIfUnreferenced(ctx context.Context, bucketName string, contentID int64, message string) { cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) defer cancel() - if cleanupErr := b.cache.Delete(cleanupCtx, bucketName, cacheKey); cleanupErr != nil { - b.logger.Warn(message, "bucket", bucketName, "cacheKey", cacheKey, "error", cleanupErr) + if _, err := objectdeletion.ReleaseContentCache( + cleanupCtx, + b.cache, + b.cacheGate, + b.cacheAccessTracker, + b.repos.Objects, + bucketName, + contentID, + ); err != nil { + b.logger.Warn(message, "bucket", bucketName, "contentID", contentID, "error", err) } } diff --git a/internal/backend/object_internal_test.go b/internal/backend/object_internal_test.go index 1025090..2480943 100644 --- a/internal/backend/object_internal_test.go +++ b/internal/backend/object_internal_test.go @@ -1,221 +1 @@ package backend - -import ( - "context" - "log/slog" - "testing" - - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/testutil" -) - -func TestCompleteFollowerIfStoredReuseWonRaceFinalizesReplicatingFollower(t *testing.T) { - ctx := context.Background() - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - bucket := &model.Bucket{Name: "replicating-finalize-race-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("create bucket: %v", err) - } - - sourceID := model.NewVersionID() - source := &model.ObjectVersion{ - VersionID: sourceID, - BucketID: bucket.ID, - Key: "file.txt", - Size: 9, - ETag: "source-etag", - Checksum: "shared-checksum", - ContentType: "text/plain", - CacheKey: ".versions/" + sourceID, - State: model.ObjectStateCached, - } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, source); err != nil { - t.Fatalf("create source version: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, sourceID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("source uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, sourceID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("source committing: %v", err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: sourceID, - ContentSize: source.Size, - Checksum: source.Checksum, - RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("start upload: %v", err) - } - primary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("primary binding: %v", err) - } - secondary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "202"), CopyIndex: 1, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("secondary binding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: primary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001")}); err != nil { - t.Fatalf("primary ready: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: secondary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "2002")}); err != nil { - t.Fatalf("secondary ready: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - {StorageDataSetID: secondary.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202")}, - }); err != nil { - t.Fatalf("create copy rows: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{UploadID: upload.ID, CopyIndex: 0, PieceCID: "bafk2bzacedummy", PieceID: onChainIDPtr(t, "1"), RetrievalURL: "https://provider.example/primary"}); err != nil { - t.Fatalf("primary committed: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: source.Size, - Checksum: source.Checksum, - }); err != nil { - t.Fatalf("bind primary committed: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{UploadID: upload.ID, CopyIndex: 1, PieceCID: "bafk2bzacedummy", PieceID: onChainIDPtr(t, "2"), RetrievalURL: "https://provider.example/secondary"}); err != nil { - t.Fatalf("secondary committed: %v", err) - } - - uploadID := upload.ID - followerID := model.NewVersionID() - follower := &model.ObjectVersion{ - VersionID: followerID, - BucketID: bucket.ID, - Key: "file.txt", - Size: source.Size, - ETag: "follower-etag", - Checksum: source.Checksum, - ContentType: "text/plain", - CacheKey: ".versions/" + followerID, - StorageUploadID: &uploadID, - State: model.ObjectStateReplicating, - } - followerObjectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, follower) - if err != nil { - t.Fatalf("create follower version: %v", err) - } - - b := &SynapseBackend{repos: repos, logger: slog.Default()} - b.completeFollowerIfStoredReuseWonRace(ctx, bucket.ID, bucket.Name, follower.Size, follower.Checksum, followerObjectID, followerID, model.ObjectStateReplicating) - - got, err := repos.Objects.GetVersionByID(ctx, followerID) - if err != nil || got == nil { - t.Fatalf("get follower version: version=%v err=%v", got, err) - } - if got.State != model.ObjectStateStored { - t.Fatalf("follower state = %s, want stored", got.State) - } -} - -func TestCompleteFollowerIfStoredReuseWonRaceFinalizesAfterBindingFollower(t *testing.T) { - ctx := context.Background() - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - bucket := &model.Bucket{Name: "active-follower-finalize-race-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("create bucket: %v", err) - } - - sourceID := model.NewVersionID() - source := &model.ObjectVersion{ - VersionID: sourceID, - BucketID: bucket.ID, - Key: "file.txt", - Size: 9, - ETag: "source-etag", - Checksum: "shared-checksum", - ContentType: "text/plain", - CacheKey: ".versions/" + sourceID, - State: model.ObjectStateCached, - } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, source); err != nil { - t.Fatalf("create source version: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, sourceID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("source uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, sourceID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("source committing: %v", err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: sourceID, - ContentSize: source.Size, - Checksum: source.Checksum, - RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("start upload: %v", err) - } - primary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("primary binding: %v", err) - } - secondary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "202"), CopyIndex: 1, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("secondary binding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: primary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001")}); err != nil { - t.Fatalf("primary ready: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: secondary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "2002")}); err != nil { - t.Fatalf("secondary ready: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - {StorageDataSetID: secondary.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202")}, - }); err != nil { - t.Fatalf("create copy rows: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{UploadID: upload.ID, CopyIndex: 0, PieceCID: "bafk2bzacedummy", PieceID: onChainIDPtr(t, "1"), RetrievalURL: "https://provider.example/primary"}); err != nil { - t.Fatalf("primary committed: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: source.Size, - Checksum: source.Checksum, - }); err != nil { - t.Fatalf("bind primary committed: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{UploadID: upload.ID, CopyIndex: 1, PieceCID: "bafk2bzacedummy", PieceID: onChainIDPtr(t, "2"), RetrievalURL: "https://provider.example/secondary"}); err != nil { - t.Fatalf("secondary committed: %v", err) - } - - followerID := model.NewVersionID() - follower := &model.ObjectVersion{ - VersionID: followerID, - BucketID: bucket.ID, - Key: "file.txt", - Size: source.Size, - ETag: "follower-etag", - Checksum: source.Checksum, - ContentType: "text/plain", - CacheKey: ".versions/" + followerID, - State: model.ObjectStateUploading, - } - followerObjectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, follower) - if err != nil { - t.Fatalf("create follower version: %v", err) - } - - b := &SynapseBackend{repos: repos, logger: slog.Default()} - b.completeFollowerIfStoredReuseWonRace(ctx, bucket.ID, bucket.Name, follower.Size, follower.Checksum, followerObjectID, followerID, model.ObjectStateUploading) - - got, err := repos.Objects.GetVersionByID(ctx, followerID) - if err != nil || got == nil { - t.Fatalf("get follower version: version=%v err=%v", got, err) - } - if got.State != model.ObjectStateStored { - t.Fatalf("follower state = %s, want stored", got.State) - } -} diff --git a/internal/backend/object_test.go b/internal/backend/object_test.go index 63c8ad1..5352bf1 100644 --- a/internal/backend/object_test.go +++ b/internal/backend/object_test.go @@ -3,12 +3,14 @@ package backend_test import ( "bytes" "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "io" "maps" "os" - "path" + "strconv" "strings" "sync" "testing" @@ -20,15 +22,17 @@ import ( "github.com/ipfs/go-cid" mh "github.com/multiformats/go-multihash" "github.com/prometheus/client_golang/prometheus" - synaps3backend "github.com/strahe/synaps3/internal/backend" + "github.com/strahe/synaps3/internal/backend" "github.com/strahe/synaps3/internal/cache" "github.com/strahe/synaps3/internal/cacheeviction" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/objectreader" + "github.com/strahe/synaps3/internal/storagepipeline" synaps3testutil "github.com/strahe/synaps3/internal/testutil" "github.com/strahe/synapse-go/chain" "github.com/strahe/synapse-go/storage" + "github.com/uptrace/bun" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3response" ) @@ -39,7 +43,20 @@ import ( func seedActiveBucket(t *testing.T, tb *testBackend, name string) *model.Bucket { t.Helper() ctx := context.Background() - bkt := &model.Bucket{Name: name, Status: model.BucketStatusActive} + bkt := &model.Bucket{Name: name, Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { + t.Fatalf("seeding bucket %q: %v", name, err) + } + return bkt +} + +// seedActiveBucketWithCopies seeds a bucket whose durability policy is the given +// copy count. Content freezes requested_copies from the policy at first ingest, +// so a test that wants partially replicated content sets it on the bucket. +func seedActiveBucketWithCopies(t *testing.T, tb *testBackend, name string, copies int) *model.Bucket { + t.Helper() + ctx := context.Background() + bkt := &model.Bucket{Name: name, Status: model.BucketStatusActive, DefaultCopies: copies, MinimumDurableCopies: copies} if err := tb.repos.Buckets.Create(ctx, bkt); err != nil { t.Fatalf("seeding bucket %q: %v", name, err) } @@ -60,8 +77,17 @@ func putTestObjectOutput(t *testing.T, tb *testBackend, bucket, key, body string t.Fatalf("getting seeded bucket %q: bucket=%v err=%v", bucket, bkt, err) } versionID := model.NewVersionID() - cacheKey := path.Join(".versions", versionID) - info, err := tb.cache.Put(ctx, bucket, cacheKey, strings.NewReader(body)) + sum := sha256.Sum256([]byte(body)) + content, err := tb.repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bkt.ID, + ContentSize: int64(len(body)), + Checksum: hex.EncodeToString(sum[:]), + RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("seeding content %s/%s: %v", bucket, key, err) + } + info, err := tb.cache.Put(ctx, bucket, model.ContentCacheKey(content.ID), strings.NewReader(body)) if err != nil { t.Fatalf("seeding cache object %s/%s: %v", bucket, key, err) } @@ -69,12 +95,10 @@ func putTestObjectOutput(t *testing.T, tb *testBackend, bucket, key, body string VersionID: versionID, BucketID: bkt.ID, Key: key, + ContentID: &content.ID, Size: info.Size, ETag: info.ETag, - Checksum: info.Checksum, ContentType: "text/plain", - CacheKey: cacheKey, - State: model.ObjectStateCached, }); err != nil { t.Fatalf("seeding object version %s/%s: %v", bucket, key, err) } @@ -217,7 +241,7 @@ func putValidTestObjectOutput(t *testing.T, tb *testBackend, bucket, key, body s } func ptrInt64(v int64) *int64 { - return &v + return new(v) } func assertS3ErrorCode(t *testing.T, err error, wantCode s3err.ErrorCode) { @@ -234,11 +258,18 @@ func assertS3ErrorCode(t *testing.T, err error, wantCode s3err.ErrorCode) { } } +// touchVersionLifecycle forces updated_at to diverge from created_at so a reader +// that reported the wrong one would be caught. A version is immutable once +// written now that its mutable state lives on the content and the cache entry, +// so the divergence has to be staged by the test itself. func touchVersionLifecycle(t *testing.T, tb *testBackend, ctx context.Context, versionID string) *model.ObjectVersion { t.Helper() - time.Sleep(5 * time.Millisecond) - if err := tb.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("touch version lifecycle: %v", err) + if _, err := tb.db.NewUpdate(). + Model((*model.ObjectVersion)(nil)). + Set("updated_at = ?", time.Now().Add(time.Second)). + Where("version_id = ?", versionID). + Exec(ctx); err != nil { + t.Fatalf("touching version %s: %v", versionID, err) } version, err := tb.repos.Objects.GetVersionByID(ctx, versionID) if err != nil || version == nil { @@ -269,72 +300,85 @@ func (r *getCurrentVersionByBucketAndKeyAfterReadRepo) GetCurrentVersionByBucket func seedBackendObjectVersion(t *testing.T, tb *testBackend, bucket *model.Bucket, key string, size int64, etag, checksum, contentType string, state model.ObjectState, pieceCID, retrievalURL *string) (int64, string) { t.Helper() versionID := model.NewVersionID() - createState := state - if state == model.ObjectStateStored || state == model.ObjectStateCacheEvicted { - createState = model.ObjectStateUploading + if checksum == "" { + checksum = "checksum-" + versionID + } + content, err := tb.repos.Contents.EnsureContent(context.Background(), repository.EnsureContentInput{ + BucketID: bucket.ID, + ContentSize: size, + Checksum: synaps3testutil.StorageChecksum(checksum), + RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("seeding content: %v", err) } version := &model.ObjectVersion{ VersionID: versionID, BucketID: bucket.ID, Key: key, + ContentID: &content.ID, Size: size, ETag: etag, - Checksum: checksum, ContentType: contentType, - CacheKey: ".versions/" + versionID, - State: createState, } objID, err := tb.repos.Objects.CreateVersionAndSetCurrent(context.Background(), version) if err != nil { t.Fatalf("seeding object version: %v", err) } - if (state == model.ObjectStateStored || state == model.ObjectStateCacheEvicted) && pieceCID != nil && retrievalURL != nil { - acceptBackendVersionUpload(t, tb.repos, versionID, *pieceCID, *retrievalURL) - if state == model.ObjectStateCacheEvicted { - if err := tb.repos.Objects.UpdateVersionState(context.Background(), versionID, model.ObjectStateStored, model.ObjectStateCacheEvicted); err != nil { - t.Fatalf("restore seeded cache evicted version: %v", err) - } - } + if state == model.ObjectStateStored && pieceCID != nil && retrievalURL != nil { + acceptBackendVersionUpload(t, tb.db, tb.repos, versionID, *pieceCID, *retrievalURL) } return objID, versionID } -func acceptBackendVersionUpload(t *testing.T, repos *repository.Repositories, versionID string, pieceCID string, retrievalURL string) *model.StorageUpload { +// contentSubjectForVersion is the task subject key for the content backing a +// version. Ingest tasks are keyed on the bytes, not on one version of them. +func contentSubjectForVersion(t *testing.T, tb *testBackend, versionID string) string { + t.Helper() + version, err := tb.repos.Objects.GetVersionByID(t.Context(), versionID) + if err != nil || version == nil || version.ContentID == nil { + t.Fatalf("content for version %s: version=%v err=%v", versionID, version, err) + } + return strconv.FormatInt(*version.ContentID, 10) +} + +// contentForVersion returns the content a seeded version already points at. The +// bytes are their own row now, so upload seeding attaches copies to that row +// rather than minting a second identity for the same version. +func contentForVersion(t *testing.T, repos *repository.Repositories, version *model.ObjectVersion) *model.StorageContent { + t.Helper() + if version.ContentID == nil { + t.Fatalf("version %s has no content", version.VersionID) + } + content, err := repos.Contents.GetByID(t.Context(), *version.ContentID) + if err != nil || content == nil { + t.Fatalf("get content %d: content=%v err=%v", *version.ContentID, content, err) + } + return content +} + +func acceptBackendVersionUpload(t *testing.T, db *bun.DB, repos *repository.Repositories, versionID string, pieceCID string, retrievalURL string) *model.StorageContent { t.Helper() ctx := context.Background() version, err := repos.Objects.GetVersionByID(ctx, versionID) if err != nil || version == nil { t.Fatalf("get version for upload accept: version=%v err=%v", version, err) } - if version.State == model.ObjectStateUploading { - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("mark committing: %v", err) - } - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: version.BucketID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("start upload attempt: %v", err) - } + upload := contentForVersion(t, repos, version) providerID := onChainID(t, "101") - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: version.BucketID, - ProviderID: providerID, - CopyIndex: 0, - CreatedByUploadID: upload.ID, + binding, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: version.BucketID, + ProviderID: providerID, + CopyIndex: 0, + CreatedByContentID: upload.ID, }) if err != nil { t.Fatalf("ensure dataset binding: %v", err) } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001")}); err != nil { + if err := repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, ContentID: upload.ID, DataSetID: onChainID(t, "1001")}); err != nil { t.Fatalf("mark dataset ready: %v", err) } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ + if err := repos.Contents.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, @@ -342,24 +386,20 @@ func acceptBackendVersionUpload(t *testing.T, repos *repository.Repositories, ve }}); err != nil { t.Fatalf("create upload copy: %v", err) } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, + synaps3testutil.CommitStorageCopy(t, db, repos, repository.MarkUploadCopyCommittedInput{ + ContentID: upload.ID, CopyIndex: 0, PieceCID: pieceCID, PieceID: onChainIDPtr(t, "1"), RetrievalURL: retrievalURL, - }); err != nil { - t.Fatalf("mark copy committed: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: version.BucketID, - ContentSize: version.Size, - Checksum: version.Checksum, + }) + if _, err := repos.Contents.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ + ContentID: upload.ID, + BucketID: version.BucketID, }); err != nil { t.Fatalf("bind readable upload: %v", err) } - if finalized, _, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}); err != nil { + if finalized, _, err := repos.Contents.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{ContentID: upload.ID}); err != nil { t.Fatalf("finalize upload: %v", err) } else if !finalized { t.Fatal("finalize upload = false, want true") @@ -367,82 +407,63 @@ func acceptBackendVersionUpload(t *testing.T, repos *repository.Repositories, ve return upload } -func bindBackendPrimaryCommittedUpload(t *testing.T, repos *repository.Repositories, versionID string, pieceCID string, retrievalURL string) *model.StorageUpload { +func bindBackendPrimaryCommittedUpload(t *testing.T, db *bun.DB, repos *repository.Repositories, versionID string, pieceCID string, retrievalURL string) *model.StorageContent { t.Helper() ctx := context.Background() version, err := repos.Objects.GetVersionByID(ctx, versionID) if err != nil || version == nil { t.Fatalf("get version for primary bind: version=%v err=%v", version, err) } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("mark committing: %v", err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: version.BucketID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("start upload attempt: %v", err) - } - primary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: version.BucketID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, + upload := contentForVersion(t, repos, version) + primary, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: version.BucketID, + ProviderID: onChainID(t, "101"), + CopyIndex: 0, + CreatedByContentID: upload.ID, }) if err != nil { t.Fatalf("ensure primary dataset binding: %v", err) } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + if err := repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ ID: primary.ID, - UploadID: upload.ID, + ContentID: upload.ID, DataSetID: onChainID(t, "1001"), }); err != nil { t.Fatalf("mark primary dataset ready: %v", err) } - secondary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: version.BucketID, - ProviderID: onChainID(t, "202"), - CopyIndex: 1, - CreatedByUploadID: upload.ID, + secondary, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: version.BucketID, + ProviderID: onChainID(t, "202"), + CopyIndex: 1, + CreatedByContentID: upload.ID, }) if err != nil { t.Fatalf("ensure secondary dataset binding: %v", err) } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ + if err := repos.Contents.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, {StorageDataSetID: secondary.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202")}, }); err != nil { t.Fatalf("create upload copy rows: %v", err) } - if err := repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: upload.ID, + if err := repos.Contents.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ + ContentID: upload.ID, CopyIndex: 0, PieceCID: pieceCID, RetrievalURL: retrievalURL, }); err != nil { t.Fatalf("mark primary piece ready: %v", err) } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, + synaps3testutil.CommitStorageCopy(t, db, repos, repository.MarkUploadCopyCommittedInput{ + ContentID: upload.ID, CopyIndex: 0, PieceCID: pieceCID, PieceID: onChainIDPtr(t, "2001"), RetrievalURL: retrievalURL, - }); err != nil { - t.Fatalf("mark primary committed: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: version.BucketID, - ContentSize: version.Size, - Checksum: version.Checksum, + }) + if _, err := repos.Contents.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ + ContentID: upload.ID, + BucketID: version.BucketID, }); err != nil { t.Fatalf("bind primary committed upload: %v", err) } @@ -514,7 +535,7 @@ func TestPutObject_HappyPath(t *testing.T) { } // Verify cache file exists. - if !tb.cache.Exists(ctx, "put-bucket", obj.CacheKey) { + if !tb.cache.Exists(ctx, "put-bucket", obj.CacheKey()) { t.Error("cache file does not exist") } } @@ -665,8 +686,8 @@ func requireObjectKeyInvalidArgument(t *testing.T, err error) { } } -func TestPutObjectUsesConfiguredUploadMaxRetries(t *testing.T) { - tb := newTestBackendWithOptions(t, synaps3backend.WithUploadMaxRetries(11)) +func TestPutObjectEnqueuesRegisteredUploadPlan(t *testing.T) { + tb := newTestBackend(t) ctx := context.Background() seedActiveBucket(t, tb, "put-retries-bucket") @@ -679,15 +700,15 @@ func TestPutObjectUsesConfiguredUploadMaxRetries(t *testing.T) { t.Fatalf("PutObject: %v", err) } - task, err := tb.repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) + task, err := tb.repos.Tasks.ClaimNext(ctx, time.Minute) if err != nil { - t.Fatalf("ClaimReady: %v", err) + t.Fatalf("ClaimNext: %v", err) } if task == nil { t.Fatal("expected upload task") } - if task.MaxRetries != 11 { - t.Fatalf("task MaxRetries = %d, want 11", task.MaxRetries) + if task.Type != model.TaskTypeUploadPlan || task.RetryLimit == nil || *task.RetryLimit != 5 { + t.Fatalf("task = %#v, want upload_plan with retry limit 5", task) } } @@ -790,7 +811,7 @@ func TestPutObjectIdenticalCurrentObjectCreatesNewVersion(t *testing.T) { taskCount, err := tb.db.NewSelect(). Model((*model.Task)(nil)). - Where("ref_type = ? AND ref_id = ?", "object", obj1.ObjectID). + Where("type = ?", model.TaskTypeUploadPlan). Count(ctx) if err != nil { t.Fatalf("counting upload tasks: %v", err) @@ -803,61 +824,223 @@ func TestPutObjectIdenticalCurrentObjectCreatesNewVersion(t *testing.T) { if err != nil || secondVersion == nil { t.Fatalf("second version: version=%v err=%v", secondVersion, err) } - if secondVersion.State != model.ObjectStateUploading { - t.Fatalf("second version state = %s, want uploading", secondVersion.State) + // Both versions share one content whose ingest plan has not produced a copy + // yet, so the derived position is still cached. + if secondVersion.State != model.ObjectStateCached { + t.Fatalf("second version state = %s, want cached", secondVersion.State) } } -func TestPutObjectIdenticalUploadingContentFollowsActiveUploadTask(t *testing.T) { +func TestPutObjectFreezesRequestedCopiesPerContent(t *testing.T) { tb := newTestBackend(t) - ctx := context.Background() - seedActiveBucket(t, tb, "uploading-reuse-bucket") + ctx := t.Context() + bucket := seedActiveBucket(t, tb, "requested-copies-freeze") - firstOut := putValidTestObjectOutput(t, tb, "uploading-reuse-bucket", "file.txt", "same data") - bkt, _ := tb.repos.Buckets.GetByName(ctx, "uploading-reuse-bucket") - firstObj, err := tb.repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bkt.ID, "file.txt") - if err != nil || firstObj == nil { - t.Fatalf("current object after first put: obj=%v err=%v", firstObj, err) + first := putValidTestObjectOutput(t, tb, bucket.Name, "first.txt", "first content") + firstVersion, err := tb.repos.Objects.GetVersionByID(ctx, first.VersionID) + if err != nil || firstVersion == nil { + t.Fatalf("GetVersionByID(first): version=%v err=%v", firstVersion, err) + } + firstContent := contentForVersion(t, tb.repos, firstVersion) + if firstContent.RequestedCopies != 1 { + t.Fatalf("first requested_copies = %d, want 1", firstContent.RequestedCopies) } - claimed, err := tb.repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil { - t.Fatalf("claim upload task: %v", err) + twoCopies := 2 + if _, err := tb.repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, + SetDefaultCopies: true, + DefaultCopies: &twoCopies, + }); err != nil { + t.Fatalf("UpdateCopyPolicy: %v", err) } - if claimed == nil || claimed.RefVersionID != firstOut.VersionID { - t.Fatalf("claimed task = %#v, want version %s", claimed, firstOut.VersionID) + + second := putValidTestObjectOutput(t, tb, bucket.Name, "second.txt", "second content") + secondVersion, err := tb.repos.Objects.GetVersionByID(ctx, second.VersionID) + if err != nil || secondVersion == nil { + t.Fatalf("GetVersionByID(second): version=%v err=%v", secondVersion, err) } - if err := tb.repos.Objects.UpdateVersionState(ctx, firstOut.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark first version uploading: %v", err) + secondContent := contentForVersion(t, tb.repos, secondVersion) + if secondContent.RequestedCopies != 2 { + t.Fatalf("second requested_copies = %d, want 2", secondContent.RequestedCopies) } - tasksBefore, _, err := tb.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "", "", 10, 0) - if err != nil { - t.Fatalf("list tasks before second put: %v", err) + deduplicated := putValidTestObjectOutput(t, tb, bucket.Name, "deduplicated.txt", "first content") + deduplicatedVersion, err := tb.repos.Objects.GetVersionByID(ctx, deduplicated.VersionID) + if err != nil || deduplicatedVersion == nil { + t.Fatalf("GetVersionByID(deduplicated): version=%v err=%v", deduplicatedVersion, err) + } + deduplicatedContent := contentForVersion(t, tb.repos, deduplicatedVersion) + if deduplicatedContent.ID != firstContent.ID { + t.Fatalf("deduplicated content id = %d, want original %d", deduplicatedContent.ID, firstContent.ID) } + if deduplicatedContent.RequestedCopies != 1 { + t.Fatalf("deduplicated requested_copies = %d, want frozen value 1", deduplicatedContent.RequestedCopies) + } +} - secondOut := putValidTestObjectOutput(t, tb, "uploading-reuse-bucket", "file.txt", "same data") - if secondOut.VersionID == "" || secondOut.VersionID == firstOut.VersionID { - t.Fatalf("second VersionID = %q, first = %q", secondOut.VersionID, firstOut.VersionID) +func TestPutObjectReactivatesTerminalUploadPlan(t *testing.T) { + for _, terminalStatus := range []model.TaskStatus{model.TaskStatusFailed, model.TaskStatusCancelled} { + t.Run(string(terminalStatus), func(t *testing.T) { + tb := newTestBackend(t) + ctx := t.Context() + bucket := seedActiveBucket(t, tb, "reactivate-plan-"+string(terminalStatus)) + first := putValidTestObjectOutput(t, tb, bucket.Name, "first.bin", "reactivated content") + version, err := tb.repos.Objects.GetVersionByID(ctx, first.VersionID) + if err != nil || version == nil || version.ContentID == nil { + t.Fatalf("first version = %#v, err=%v", version, err) + } + taskRow, err := tb.repos.Tasks.GetByIdentity(ctx, model.TaskTypeUploadPlan, storagepipeline.UploadPlanKey(*version.ContentID)) + if err != nil || taskRow == nil { + t.Fatalf("upload plan = %#v, err=%v", taskRow, err) + } + claimed, err := tb.repos.Tasks.ClaimNext(ctx, time.Minute) + if err != nil || claimed == nil || claimed.ID != taskRow.ID { + t.Fatalf("claimed upload plan = %#v, err=%v", claimed, err) + } + if err := tb.repos.Tasks.WriteCheckpoint(ctx, claimed.ID, claimed.ClaimGeneration, []byte(`{"old":true}`)); err != nil { + t.Fatalf("write old checkpoint: %v", err) + } + if err := tb.repos.Tasks.RequestCancellation(ctx, claimed.ID, "old owner stopped"); err != nil { + t.Fatalf("request old cancellation: %v", err) + } + transition := repository.TaskTransition{ + Status: terminalStatus, ResumeMode: model.TaskResumeModeRecover, + FailureReason: new("old_plan_failed"), LastError: new("old upload plan failed"), IncrementRetry: true, + } + if terminalStatus == model.TaskStatusCancelled { + transition.FailureReason = nil + transition.LastError = nil + transition.RetentionUntil = new(time.Now().Add(time.Hour)) + } + if err := tb.repos.Tasks.Settle(ctx, claimed.ID, claimed.ClaimGeneration, transition); err != nil { + t.Fatalf("settle old upload plan: %v", err) + } + if terminalStatus == model.TaskStatusFailed { + if err := tb.repos.Tasks.AcknowledgeFailed(ctx, claimed.ID, time.Hour); err != nil { + t.Fatalf("acknowledge old upload plan: %v", err) + } + } + + second := putValidTestObjectOutput(t, tb, bucket.Name, "second.bin", "reactivated content") + secondVersion, err := tb.repos.Objects.GetVersionByID(ctx, second.VersionID) + if err != nil || secondVersion == nil || secondVersion.ContentID == nil || *secondVersion.ContentID != *version.ContentID { + t.Fatalf("second version = %#v, err=%v", secondVersion, err) + } + reactivated, err := tb.repos.Tasks.GetByID(ctx, taskRow.ID) + if err != nil || reactivated == nil { + t.Fatalf("reactivated task = %#v, err=%v", reactivated, err) + } + if reactivated.Status != model.TaskStatusPending || reactivated.ResumeMode != model.TaskResumeModeExecute || + reactivated.RetryCount != 0 || len(reactivated.Checkpoint) != 0 || reactivated.FailureReason != nil || + reactivated.CancellationRequestedAt != nil || reactivated.CancellationReason != nil || reactivated.AcknowledgedAt != nil || reactivated.RetentionUntil != nil { + t.Fatalf("reactivated task retained terminal state: %#v", reactivated) + } + }) } +} - secondVersion, err := tb.repos.Objects.GetVersionByID(ctx, secondOut.VersionID) - if err != nil || secondVersion == nil { - t.Fatalf("second version: version=%v err=%v", secondVersion, err) +func TestPutObjectRejectsCompletedUploadPlanForCachedContent(t *testing.T) { + tb := newTestBackend(t) + ctx := t.Context() + bucket := seedActiveBucket(t, tb, "completed-cached-plan") + first := putValidTestObjectOutput(t, tb, bucket.Name, "first.bin", "completed cached content") + version, err := tb.repos.Objects.GetVersionByID(ctx, first.VersionID) + if err != nil || version == nil || version.ContentID == nil { + t.Fatalf("first version = %#v, err=%v", version, err) + } + taskRow, err := tb.repos.Tasks.GetByIdentity(ctx, model.TaskTypeUploadPlan, storagepipeline.UploadPlanKey(*version.ContentID)) + if err != nil || taskRow == nil { + t.Fatalf("upload plan = %#v, err=%v", taskRow, err) + } + claimed, err := tb.repos.Tasks.ClaimNext(ctx, time.Minute) + if err != nil || claimed == nil || claimed.ID != taskRow.ID { + t.Fatalf("claimed upload plan = %#v, err=%v", claimed, err) + } + if err := tb.repos.Tasks.Settle(ctx, claimed.ID, claimed.ClaimGeneration, repository.TaskTransition{ + Status: model.TaskStatusCompleted, ResumeMode: model.TaskResumeModeRecover, RetentionUntil: new(time.Now().Add(time.Hour)), + }); err != nil { + t.Fatalf("complete inconsistent upload plan: %v", err) } - if secondVersion.State != model.ObjectStateUploading { - t.Fatalf("second version state = %s, want uploading", secondVersion.State) + contentType := "text/plain" + validBody := validTestObjectBody("completed cached content") + _, err = tb.backend.PutObject(ctx, s3response.PutObjectInput{ + Bucket: &bucket.Name, Key: new("second.bin"), Body: strings.NewReader(validBody), ContentType: &contentType, + }) + if !errors.Is(err, repository.ErrConflict) { + t.Fatalf("PutObject completed/cached error = %v, want conflict", err) } - if secondVersion.PieceCID != nil || secondVersion.RetrievalURL != nil { - t.Fatalf("second version storage = piece:%v url:%v, want unset while upload runs", secondVersion.PieceCID, secondVersion.RetrievalURL) +} + +func TestPutObjectIdenticalStoredContentQueuesAfterUploadEviction(t *testing.T) { + tb := newTestBackendWithOptions(t, backend.WithEvictionPolicy(cache.EvictionPolicyAfterUpload)) + ctx := context.Background() + seedActiveBucket(t, tb, "stored-reuse-evict-bucket") + + putValidTestObject(t, tb, "stored-reuse-evict-bucket", "file.txt", "same data") + bkt, _ := tb.repos.Buckets.GetByName(ctx, "stored-reuse-evict-bucket") + firstObj, err := tb.repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bkt.ID, "file.txt") + if err != nil || firstObj == nil { + t.Fatalf("current object after first put: obj=%v err=%v", firstObj, err) } + acceptBackendVersionUpload(t, tb.db, tb.repos, firstObj.VersionID, "piece-shared", "https://provider.example/shared") - tasksAfter, _, err := tb.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "", "", 10, 0) + putValidTestObject(t, tb, "stored-reuse-evict-bucket", "file.txt", "same data") + page, err := tb.repos.Tasks.List(ctx, repository.TaskListFilter{Type: model.TaskTypeCacheEvict, Limit: 10}) if err != nil { - t.Fatalf("list tasks after second put: %v", err) + t.Fatalf("claim evict task: %v", err) + } + if len(page.Tasks) != 1 { + t.Fatal("expected evict task for reused stored content") + } + // Eviction frees one cache file, and that file belongs to the content, so + // the task names the content rather than either version of it. + task := &page.Tasks[0] + contentID := *firstObj.ContentID + if task.SubjectKey == nil || *task.SubjectKey != strconv.FormatInt(contentID, 10) { + t.Fatalf("evict task content = %v, want %d", task.SubjectKey, contentID) } - if len(tasksAfter) != len(tasksBefore) { - t.Fatalf("upload task count changed from %d to %d", len(tasksBefore), len(tasksAfter)) + input, err := cacheeviction.ParseEvictInput(task) + if err != nil || input.ContentID != contentID || input.AccessedAt != nil { + t.Fatalf("evict input = %#v, err=%v", input, err) + } + if task.IdempotencyKey != cacheeviction.EvictTaskKey(contentID, input.Generation) { + t.Fatalf("evict task key = %q", task.IdempotencyKey) + } + if task.RetryLimit == nil || *task.RetryLimit != 5 { + t.Fatalf("evict task retry limit = %v, want 5", task.RetryLimit) + } +} + +func TestPutObjectIdenticalStoredContentDoesNotQueueImmediateEvictionOutsideAfterUpload(t *testing.T) { + for _, policy := range []cache.EvictionPolicy{cache.EvictionPolicyLRU, cache.EvictionPolicyNone} { + t.Run(string(policy), func(t *testing.T) { + tb := newTestBackendWithOptions(t, backend.WithEvictionPolicy(policy)) + ctx := context.Background() + bucketName := "stored-reuse-" + string(policy) + "-bucket" + seedActiveBucket(t, tb, bucketName) + + putValidTestObject(t, tb, bucketName, "file.txt", "same data") + bucket, _ := tb.repos.Buckets.GetByName(ctx, bucketName) + first, err := tb.repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "file.txt") + if err != nil || first == nil { + t.Fatalf("current object after first put: object=%v err=%v", first, err) + } + acceptBackendVersionUpload(t, tb.db, tb.repos, first.VersionID, "piece-"+string(policy), "https://provider.example/"+string(policy)) + + second := putValidTestObjectOutput(t, tb, bucketName, "file.txt", "same data") + stored, err := tb.repos.Objects.GetVersionByID(ctx, second.VersionID) + if err != nil || stored == nil || stored.State != model.ObjectStateStored { + t.Fatalf("reused version = %#v err=%v, want stored", stored, err) + } + page, err := tb.repos.Tasks.List(ctx, repository.TaskListFilter{Type: model.TaskTypeCacheEvict, Limit: 10}) + if err != nil { + t.Fatalf("List eviction tasks: %v", err) + } + if len(page.Tasks) != 0 { + t.Fatalf("policy %s immediate eviction tasks=%#v, want none", policy, page.Tasks) + } + }) } } @@ -872,12 +1055,9 @@ func TestPutObjectIdenticalStoredContentReusesChainStorage(t *testing.T) { if err != nil || firstObj == nil { t.Fatalf("current object after first put: obj=%v err=%v", firstObj, err) } - if err := tb.repos.Objects.UpdateVersionState(ctx, firstObj.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark first version uploading: %v", err) - } - acceptBackendVersionUpload(t, tb.repos, firstObj.VersionID, "piece-shared", "https://provider.example/shared") + acceptBackendVersionUpload(t, tb.db, tb.repos, firstObj.VersionID, "piece-shared", "https://provider.example/shared") - tasksBefore, _, err := tb.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "", "", 10, 0) + before, err := tb.repos.Tasks.List(ctx, repository.TaskListFilter{Type: model.TaskTypeUploadPlan, Limit: 10}) if err != nil { t.Fatalf("list tasks before second put: %v", err) } @@ -886,6 +1066,8 @@ func TestPutObjectIdenticalStoredContentReusesChainStorage(t *testing.T) { t.Fatalf("second VersionID = %q, first = %q", secondOut.VersionID, firstOut.VersionID) } + // The rewritten bytes resolve to the content that is already stored, so the + // new version inherits its durability instead of re-uploading. secondVersion, err := tb.repos.Objects.GetVersionByID(ctx, secondOut.VersionID) if err != nil || secondVersion == nil { t.Fatalf("second version: version=%v err=%v", secondVersion, err) @@ -893,23 +1075,23 @@ func TestPutObjectIdenticalStoredContentReusesChainStorage(t *testing.T) { if secondVersion.State != model.ObjectStateStored { t.Fatalf("second version state = %s, want stored", secondVersion.State) } - if secondVersion.StorageUploadID == nil { - t.Fatal("second version storage_upload_id is nil, want reused upload") + if secondVersion.ContentID == nil || firstObj.ContentID == nil || *secondVersion.ContentID != *firstObj.ContentID { + t.Fatalf("second version content = %v, want the first version's %v", secondVersion.ContentID, firstObj.ContentID) } - tasksAfter, _, err := tb.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "", "", 10, 0) + after, err := tb.repos.Tasks.List(ctx, repository.TaskListFilter{Type: model.TaskTypeUploadPlan, Limit: 10}) if err != nil { t.Fatalf("list tasks after second put: %v", err) } - if len(tasksAfter) != len(tasksBefore) { - t.Fatalf("upload task count changed from %d to %d", len(tasksBefore), len(tasksAfter)) + if len(after.Tasks) != len(before.Tasks) { + t.Fatalf("upload task count changed from %d to %d", len(before.Tasks), len(after.Tasks)) } } func TestPutObjectIdenticalReplicatingContentReusesPrimaryCommittedUpload(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() - seedActiveBucket(t, tb, "replicating-reuse-bucket") + seedActiveBucketWithCopies(t, tb, "replicating-reuse-bucket", 3) firstOut := putValidTestObjectOutput(t, tb, "replicating-reuse-bucket", "file.txt", "same data") bkt, _ := tb.repos.Buckets.GetByName(ctx, "replicating-reuse-bucket") @@ -917,9 +1099,9 @@ func TestPutObjectIdenticalReplicatingContentReusesPrimaryCommittedUpload(t *tes if err != nil || firstObj == nil { t.Fatalf("current object after first put: obj=%v err=%v", firstObj, err) } - upload := bindBackendPrimaryCommittedUpload(t, tb.repos, firstObj.VersionID, buildDummyCID(t), "https://provider.example/primary") + upload := bindBackendPrimaryCommittedUpload(t, tb.db, tb.repos, firstObj.VersionID, buildDummyCID(t), "https://provider.example/primary") - tasksBefore, _, err := tb.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "", "", 10, 0) + before, err := tb.repos.Tasks.List(ctx, repository.TaskListFilter{Type: model.TaskTypeUploadPlan, Limit: 10}) if err != nil { t.Fatalf("list tasks before second put: %v", err) } @@ -935,94 +1117,19 @@ func TestPutObjectIdenticalReplicatingContentReusesPrimaryCommittedUpload(t *tes if secondVersion.State != model.ObjectStateReplicating { t.Fatalf("second version state = %s, want replicating", secondVersion.State) } - if secondVersion.StorageUploadID == nil || *secondVersion.StorageUploadID != upload.ID || !secondVersion.InFilecoin { - t.Fatalf("second version storage = upload:%v in_filecoin:%v, want upload %d readable", secondVersion.StorageUploadID, secondVersion.InFilecoin, upload.ID) + if secondVersion.ContentID == nil || *secondVersion.ContentID != upload.ID || !secondVersion.InFilecoin { + t.Fatalf("second version storage = upload:%v in_filecoin:%v, want upload %d readable", secondVersion.ContentID, secondVersion.InFilecoin, upload.ID) } - tasksAfter, _, err := tb.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "", "", 10, 0) + after, err := tb.repos.Tasks.List(ctx, repository.TaskListFilter{Type: model.TaskTypeUploadPlan, Limit: 10}) if err != nil { t.Fatalf("list tasks after second put: %v", err) } - if len(tasksAfter) != len(tasksBefore) { - t.Fatalf("upload task count changed from %d to %d", len(tasksBefore), len(tasksAfter)) + if len(after.Tasks) != len(before.Tasks) { + t.Fatalf("upload task count changed from %d to %d", len(before.Tasks), len(after.Tasks)) } } -func TestPutObjectIdenticalStoredContentQueuesAfterUploadEviction(t *testing.T) { - tb := newTestBackendWithOptions(t, synaps3backend.WithEvictionPolicy(cache.EvictionPolicyAfterUpload), synaps3backend.WithEvictMaxRetries(9)) - ctx := context.Background() - seedActiveBucket(t, tb, "stored-reuse-evict-bucket") - - putValidTestObject(t, tb, "stored-reuse-evict-bucket", "file.txt", "same data") - bkt, _ := tb.repos.Buckets.GetByName(ctx, "stored-reuse-evict-bucket") - firstObj, err := tb.repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bkt.ID, "file.txt") - if err != nil || firstObj == nil { - t.Fatalf("current object after first put: obj=%v err=%v", firstObj, err) - } - if err := tb.repos.Objects.UpdateVersionState(ctx, firstObj.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark first version uploading: %v", err) - } - acceptBackendVersionUpload(t, tb.repos, firstObj.VersionID, "piece-shared", "https://provider.example/shared") - - secondOut := putValidTestObjectOutput(t, tb, "stored-reuse-evict-bucket", "file.txt", "same data") - task, err := tb.repos.Tasks.ClaimReady(ctx, model.TaskTypeEvictCache, time.Minute) - if err != nil { - t.Fatalf("claim evict task: %v", err) - } - if task == nil { - t.Fatal("expected evict task for reused stored version") - } - if task.RefVersionID != secondOut.VersionID { - t.Fatalf("evict task version = %s, want %s", task.RefVersionID, secondOut.VersionID) - } - if task.Stage == nil || *task.Stage != cacheeviction.StageAfterUpload { - t.Fatalf("evict task stage = %v, want %s", task.Stage, cacheeviction.StageAfterUpload) - } - if task.IdempotencyKey != "evict_cache:"+secondOut.VersionID { - t.Fatalf("evict task key = %q, want stable after_upload version key", task.IdempotencyKey) - } - if task.MaxRetries != 9 { - t.Fatalf("evict task MaxRetries = %d, want 9", task.MaxRetries) - } -} - -func TestPutObjectIdenticalStoredContentDoesNotQueueImmediateEvictionOutsideAfterUpload(t *testing.T) { - for _, policy := range []cache.EvictionPolicy{cache.EvictionPolicyLRU, cache.EvictionPolicyNone} { - t.Run(string(policy), func(t *testing.T) { - tb := newTestBackendWithOptions(t, synaps3backend.WithEvictionPolicy(policy)) - ctx := context.Background() - bucketName := "stored-reuse-" + string(policy) + "-bucket" - seedActiveBucket(t, tb, bucketName) - - putValidTestObject(t, tb, bucketName, "file.txt", "same data") - bucket, _ := tb.repos.Buckets.GetByName(ctx, bucketName) - first, err := tb.repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "file.txt") - if err != nil || first == nil { - t.Fatalf("current object after first put: object=%v err=%v", first, err) - } - if err := tb.repos.Objects.UpdateVersionState(ctx, first.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark first version uploading: %v", err) - } - acceptBackendVersionUpload(t, tb.repos, first.VersionID, "piece-"+string(policy), "https://provider.example/"+string(policy)) - - second := putValidTestObjectOutput(t, tb, bucketName, "file.txt", "same data") - stored, err := tb.repos.Objects.GetVersionByID(ctx, second.VersionID) - if err != nil || stored == nil || stored.State != model.ObjectStateStored { - t.Fatalf("reused version = %#v err=%v, want stored", stored, err) - } - tasks, total, err := tb.repos.Tasks.List(ctx, string(model.TaskTypeEvictCache), "", "", 10, 0) - if err != nil { - t.Fatalf("List eviction tasks: %v", err) - } - if total != 0 || len(tasks) != 0 { - t.Fatalf("policy %s immediate eviction tasks total=%d tasks=%#v, want none", policy, total, tasks) - } - }) - } -} - -// ---------- GetObject ---------- - func TestGetObject_FromCache(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() @@ -1723,7 +1830,7 @@ func TestListObjects_Pagination(t *testing.T) { ctx := context.Background() seedActiveBucket(t, tb, "list-bucket") - for i := 0; i < 5; i++ { + for i := range 5 { key := fmt.Sprintf("obj-%02d", i) putTestObject(t, tb, "list-bucket", key, "data") } @@ -1836,7 +1943,7 @@ func TestListObjectsV2_ContinuationToken(t *testing.T) { ctx := context.Background() seedActiveBucket(t, tb, "v2-bucket") - for i := 0; i < 5; i++ { + for i := range 5 { putTestObject(t, tb, "v2-bucket", fmt.Sprintf("key-%02d", i), "data") } @@ -2206,15 +2313,143 @@ func TestDeleteObject_MissingKeyCreatesDeleteMarker(t *testing.T) { } } +func TestDeleteObject_DataVersionPermanentDeleteReportsActiveStorageWork(t *testing.T) { + tb := newTestBackend(t) + ctx := context.Background() + seedActiveBucket(t, tb, "delete-active-storage-work-bucket") + // The put leaves an ingest plan scheduled against the content, which is the + // storage work that must block the delete. + putOut := putValidTestObjectOutput(t, tb, "delete-active-storage-work-bucket", "file.txt", "data") + _, err := tb.backend.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String("delete-active-storage-work-bucket"), + Key: aws.String("file.txt"), + VersionId: aws.String(putOut.VersionID), + }) + var apiErr s3err.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("DeleteObject error = %T %v, want s3 API error", err, err) + } + if apiErr.Code != "InvalidRequest" || apiErr.Description != "The object version cannot be deleted while storage is still in progress or a Filecoin transaction is awaiting confirmation. Try again later." { + t.Fatalf("DeleteObject API error = %#v, want actionable storage-work conflict", apiErr) + } +} + +func TestDeleteObjects_DataVersionReportsActiveStorageWorkPerEntry(t *testing.T) { + tb := newTestBackend(t) + ctx := context.Background() + seedActiveBucket(t, tb, "delete-objects-active-storage-work-bucket") + putOut := putValidTestObjectOutput(t, tb, "delete-objects-active-storage-work-bucket", "file.txt", "data") + out, err := tb.backend.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: aws.String("delete-objects-active-storage-work-bucket"), + Delete: &types.Delete{Objects: []types.ObjectIdentifier{ + {Key: aws.String("file.txt"), VersionId: aws.String(putOut.VersionID)}, + }}, + }) + if err != nil { + t.Fatalf("DeleteObjects(active storage work): %v", err) + } + if len(out.Deleted) != 0 || len(out.Error) != 1 { + t.Fatalf("DeleteObjects(active storage work) = %#v, want one entry error", out) + } + entryErr := out.Error[0] + if entryErr.Key == nil || *entryErr.Key != "file.txt" || entryErr.VersionId == nil || *entryErr.VersionId != putOut.VersionID { + t.Fatalf("entry identity = key:%v version:%v, want file.txt/%s", entryErr.Key, entryErr.VersionId, putOut.VersionID) + } + if entryErr.Code == nil || *entryErr.Code != "InvalidRequest" { + t.Fatalf("entry code = %v, want InvalidRequest", entryErr.Code) + } + wantMessage := "The object version cannot be deleted while storage is still in progress or a Filecoin transaction is awaiting confirmation. Try again later." + if entryErr.Message == nil || *entryErr.Message != wantMessage { + t.Fatalf("entry message = %v, want %q", entryErr.Message, wantMessage) + } + got, err := tb.repos.Objects.GetVersionByID(ctx, putOut.VersionID) + if err != nil || got == nil { + t.Fatalf("version after rejected DeleteObjects = %#v err=%v, want retained", got, err) + } +} + +func TestDeleteObjects_DataVersionPermanentDeleteRemovesHiddenVersion(t *testing.T) { + tb := newTestBackend(t) + ctx := context.Background() + seedActiveBucket(t, tb, "delete-objects-data-version-bucket") + + putOut := putTestObjectOutput(t, tb, "delete-objects-data-version-bucket", "file.txt", "data") + marker, err := tb.backend.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String("delete-objects-data-version-bucket"), + Key: aws.String("file.txt"), + }) + if err != nil { + t.Fatalf("DeleteObject(marker): %v", err) + } + versionBeforeDelete, err := tb.repos.Objects.GetVersionByID(ctx, putOut.VersionID) + if err != nil || versionBeforeDelete == nil { + t.Fatalf("GetVersionByID(before delete): version=%v err=%v", versionBeforeDelete, err) + } + cacheKey := versionBeforeDelete.CacheKey() + if !tb.cache.Exists(ctx, "delete-objects-data-version-bucket", cacheKey) { + t.Fatal("expected cache file before permanent delete") + } + + out, err := tb.backend.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: aws.String("delete-objects-data-version-bucket"), + Delete: &types.Delete{ + Objects: []types.ObjectIdentifier{ + {Key: aws.String("file.txt"), VersionId: aws.String(putOut.VersionID)}, + }, + }, + }) + if err != nil { + t.Fatalf("DeleteObjects(data version): %v", err) + } + if len(out.Error) != 0 { + t.Fatalf("Error = %#v, want none", out.Error) + } + if len(out.Deleted) != 1 { + t.Fatalf("Deleted = %#v, want one entry", out.Deleted) + } + deleted := out.Deleted[0] + if deleted.VersionId == nil || *deleted.VersionId != putOut.VersionID { + t.Fatalf("deleted VersionId = %v, want %s", deleted.VersionId, putOut.VersionID) + } + if deleted.DeleteMarker == nil || *deleted.DeleteMarker { + t.Fatalf("deleted DeleteMarker = %v, want false", deleted.DeleteMarker) + } + + removed, err := tb.repos.Objects.GetVersionByID(ctx, putOut.VersionID) + if err != nil { + t.Fatalf("GetVersionByID(after delete): %v", err) + } + if removed != nil { + t.Fatalf("deleted data version still exists: %#v", removed) + } + if tb.cache.Exists(ctx, "delete-objects-data-version-bucket", cacheKey) { + t.Fatal("cache file still exists after permanent delete") + } + var tombstones int + if err := tb.db.NewRaw(`SELECT COUNT(*) FROM object_deletions WHERE version_id = ?`, putOut.VersionID).Scan(ctx, &tombstones); err != nil { + t.Fatalf("count object deletions: %v", err) + } + if tombstones != 1 { + t.Fatalf("object deletion tombstones = %d, want 1", tombstones) + } + + versionsOut, err := tb.backend.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: aws.String("delete-objects-data-version-bucket"), + }) + if err != nil { + t.Fatalf("ListObjectVersions: %v", err) + } + if len(versionsOut.Versions) != 0 || len(versionsOut.DeleteMarkers) != 1 || *versionsOut.DeleteMarkers[0].VersionId != *marker.VersionId { + t.Fatalf("versions=%#v markers=%#v, want only marker %s", versionsOut.Versions, versionsOut.DeleteMarkers, *marker.VersionId) + } +} + func TestDeleteObject_DataVersionPermanentDeleteRemovesHiddenVersion(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() seedActiveBucket(t, tb, "delete-data-version-bucket") putOut := putTestObjectOutput(t, tb, "delete-data-version-bucket", "file.txt", "data") - if _, err := tb.db.NewRaw(`UPDATE tasks SET status = ? WHERE ref_type = ? AND ref_version_id = ?`, model.TaskStatusCompleted, "object", putOut.VersionID).Exec(ctx); err != nil { - t.Fatalf("complete upload task: %v", err) - } marker, err := tb.backend.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: aws.String("delete-data-version-bucket"), Key: aws.String("file.txt"), @@ -2230,7 +2465,8 @@ func TestDeleteObject_DataVersionPermanentDeleteRemovesHiddenVersion(t *testing. if err != nil || versionBeforeDelete == nil { t.Fatalf("GetVersionByID(before delete): version=%v err=%v", versionBeforeDelete, err) } - if !tb.cache.Exists(ctx, "delete-data-version-bucket", versionBeforeDelete.CacheKey) { + cacheKey := versionBeforeDelete.CacheKey() + if !tb.cache.Exists(ctx, "delete-data-version-bucket", cacheKey) { t.Fatal("expected cache file before permanent delete") } @@ -2256,16 +2492,20 @@ func TestDeleteObject_DataVersionPermanentDeleteRemovesHiddenVersion(t *testing. if deleted != nil { t.Fatalf("deleted data version still exists: %#v", deleted) } - if tb.cache.Exists(ctx, "delete-data-version-bucket", versionBeforeDelete.CacheKey) { + // That version held the last reference to those bytes, so the shared cache + // file goes with it. + if tb.cache.Exists(ctx, "delete-data-version-bucket", cacheKey) { t.Fatal("cache file still exists after permanent delete") } - var cacheStatus string - if err := tb.db.NewRaw(`SELECT cache_cleanup_status FROM object_deletions WHERE version_id = ?`, putOut.VersionID).Scan(ctx, &cacheStatus); err != nil { - t.Fatalf("object deletion cache status: %v", err) + // object_deletions is an append-only tombstone now; the record of the + // deletion is the row itself, not a cleanup status on it. + var tombstones int + if err := tb.db.NewRaw(`SELECT COUNT(*) FROM object_deletions WHERE version_id = ?`, putOut.VersionID).Scan(ctx, &tombstones); err != nil { + t.Fatalf("count object deletions: %v", err) } - if cacheStatus != string(model.CacheCleanupStatusDeleted) { - t.Fatalf("cache cleanup status = %q, want %q", cacheStatus, model.CacheCleanupStatusDeleted) + if tombstones != 1 { + t.Fatalf("object deletion tombstones = %d, want 1", tombstones) } versionsOut, err := tb.backend.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ @@ -2279,49 +2519,14 @@ func TestDeleteObject_DataVersionPermanentDeleteRemovesHiddenVersion(t *testing. } } -func TestDeleteObject_DataVersionPermanentDeleteReportsActiveStorageWork(t *testing.T) { - tb := newTestBackend(t) - ctx := context.Background() - seedActiveBucket(t, tb, "delete-active-storage-work-bucket") - putOut := putTestObjectOutput(t, tb, "delete-active-storage-work-bucket", "file.txt", "data") - version, err := tb.repos.Objects.GetVersionByID(ctx, putOut.VersionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%#v err=%v", version, err) - } - if err := tb.repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("UpdateVersionState(uploading): %v", err) - } - stage := "prepare_upload" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "object", RefID: version.ObjectID, RefVersionID: version.VersionID, - IdempotencyKey: "upload:" + version.VersionID, Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := tb.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Tasks.Create: %v", err) - } - - _, err = tb.backend.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: aws.String("delete-active-storage-work-bucket"), - Key: aws.String("file.txt"), - VersionId: aws.String(putOut.VersionID), - }) - apiErr, ok := err.(s3err.APIError) - if !ok { - t.Fatalf("DeleteObject error = %T %v, want s3 API error", err, err) - } - if apiErr.Code != "InvalidRequest" || apiErr.Description != "The object version cannot be deleted while storage is still in progress or a Filecoin transaction is awaiting confirmation. Try again later." { - t.Fatalf("DeleteObject API error = %#v, want actionable storage-work conflict", apiErr) - } -} - func TestDeleteObject_DataVersionPermanentDeleteRemovesCurrentVisibleVersion(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() seedActiveBucket(t, tb, "delete-current-data-version-bucket") putOut := putTestObjectOutput(t, tb, "delete-current-data-version-bucket", "file.txt", "data") - if _, err := tb.db.NewRaw(`UPDATE tasks SET status = ? WHERE ref_type = ? AND ref_version_id = ?`, model.TaskStatusCompleted, "object", putOut.VersionID).Exec(ctx); err != nil { - t.Fatalf("complete upload task: %v", err) + if _, err := tb.db.NewRaw(`DELETE FROM tasks WHERE subject_type = ? AND subject_key = ?`, "object_version", putOut.VersionID).Exec(ctx); err != nil { + t.Fatalf("remove upload task: %v", err) } out, err := tb.backend.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: aws.String("delete-current-data-version-bucket"), @@ -2526,136 +2731,6 @@ func TestDeleteObjects_DeleteMarkerVersionRestoresObject(t *testing.T) { } } -func TestDeleteObjects_DataVersionPermanentDeleteRemovesHiddenVersion(t *testing.T) { - tb := newTestBackend(t) - ctx := context.Background() - seedActiveBucket(t, tb, "delete-objects-data-version-bucket") - - putOut := putTestObjectOutput(t, tb, "delete-objects-data-version-bucket", "file.txt", "data") - if _, err := tb.db.NewRaw(`UPDATE tasks SET status = ? WHERE ref_type = ? AND ref_version_id = ?`, model.TaskStatusCompleted, "object", putOut.VersionID).Exec(ctx); err != nil { - t.Fatalf("complete upload task: %v", err) - } - marker, err := tb.backend.DeleteObject(ctx, &s3.DeleteObjectInput{ - Bucket: aws.String("delete-objects-data-version-bucket"), - Key: aws.String("file.txt"), - }) - if err != nil { - t.Fatalf("DeleteObject(marker): %v", err) - } - versionBeforeDelete, err := tb.repos.Objects.GetVersionByID(ctx, putOut.VersionID) - if err != nil || versionBeforeDelete == nil { - t.Fatalf("GetVersionByID(before delete): version=%v err=%v", versionBeforeDelete, err) - } - if !tb.cache.Exists(ctx, "delete-objects-data-version-bucket", versionBeforeDelete.CacheKey) { - t.Fatal("expected cache file before permanent delete") - } - - out, err := tb.backend.DeleteObjects(ctx, &s3.DeleteObjectsInput{ - Bucket: aws.String("delete-objects-data-version-bucket"), - Delete: &types.Delete{ - Objects: []types.ObjectIdentifier{ - {Key: aws.String("file.txt"), VersionId: aws.String(putOut.VersionID)}, - }, - }, - }) - if err != nil { - t.Fatalf("DeleteObjects(data version): %v", err) - } - if len(out.Error) != 0 { - t.Fatalf("Error = %#v, want none", out.Error) - } - if len(out.Deleted) != 1 { - t.Fatalf("Deleted = %#v, want one entry", out.Deleted) - } - deleted := out.Deleted[0] - if deleted.VersionId == nil || *deleted.VersionId != putOut.VersionID { - t.Fatalf("deleted VersionId = %v, want %s", deleted.VersionId, putOut.VersionID) - } - if deleted.DeleteMarker == nil || *deleted.DeleteMarker { - t.Fatalf("deleted DeleteMarker = %v, want false", deleted.DeleteMarker) - } - - removed, err := tb.repos.Objects.GetVersionByID(ctx, putOut.VersionID) - if err != nil { - t.Fatalf("GetVersionByID(after delete): %v", err) - } - if removed != nil { - t.Fatalf("deleted data version still exists: %#v", removed) - } - if tb.cache.Exists(ctx, "delete-objects-data-version-bucket", versionBeforeDelete.CacheKey) { - t.Fatal("cache file still exists after permanent delete") - } - var cacheStatus string - if err := tb.db.NewRaw(`SELECT cache_cleanup_status FROM object_deletions WHERE version_id = ?`, putOut.VersionID).Scan(ctx, &cacheStatus); err != nil { - t.Fatalf("object deletion cache status: %v", err) - } - if cacheStatus != string(model.CacheCleanupStatusDeleted) { - t.Fatalf("cache cleanup status = %q, want %q", cacheStatus, model.CacheCleanupStatusDeleted) - } - - versionsOut, err := tb.backend.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ - Bucket: aws.String("delete-objects-data-version-bucket"), - }) - if err != nil { - t.Fatalf("ListObjectVersions: %v", err) - } - if len(versionsOut.Versions) != 0 || len(versionsOut.DeleteMarkers) != 1 || *versionsOut.DeleteMarkers[0].VersionId != *marker.VersionId { - t.Fatalf("versions=%#v markers=%#v, want only marker %s", versionsOut.Versions, versionsOut.DeleteMarkers, *marker.VersionId) - } -} - -func TestDeleteObjects_DataVersionReportsActiveStorageWorkPerEntry(t *testing.T) { - tb := newTestBackend(t) - ctx := context.Background() - seedActiveBucket(t, tb, "delete-objects-active-storage-work-bucket") - putOut := putTestObjectOutput(t, tb, "delete-objects-active-storage-work-bucket", "file.txt", "data") - version, err := tb.repos.Objects.GetVersionByID(ctx, putOut.VersionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%#v err=%v", version, err) - } - if err := tb.repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("UpdateVersionState(uploading): %v", err) - } - stage := "prepare_upload" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "object", RefID: version.ObjectID, RefVersionID: version.VersionID, - IdempotencyKey: "upload:" + version.VersionID, Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := tb.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Tasks.Create: %v", err) - } - - out, err := tb.backend.DeleteObjects(ctx, &s3.DeleteObjectsInput{ - Bucket: aws.String("delete-objects-active-storage-work-bucket"), - Delete: &types.Delete{Objects: []types.ObjectIdentifier{ - {Key: aws.String("file.txt"), VersionId: aws.String(putOut.VersionID)}, - }}, - }) - if err != nil { - t.Fatalf("DeleteObjects(active storage work): %v", err) - } - if len(out.Deleted) != 0 || len(out.Error) != 1 { - t.Fatalf("DeleteObjects(active storage work) = %#v, want one entry error", out) - } - entryErr := out.Error[0] - if entryErr.Key == nil || *entryErr.Key != "file.txt" || entryErr.VersionId == nil || *entryErr.VersionId != putOut.VersionID { - t.Fatalf("entry identity = key:%v version:%v, want file.txt/%s", entryErr.Key, entryErr.VersionId, putOut.VersionID) - } - if entryErr.Code == nil || *entryErr.Code != "InvalidRequest" { - t.Fatalf("entry code = %v, want InvalidRequest", entryErr.Code) - } - wantMessage := "The object version cannot be deleted while storage is still in progress or a Filecoin transaction is awaiting confirmation. Try again later." - if entryErr.Message == nil || *entryErr.Message != wantMessage { - t.Fatalf("entry message = %v, want %q", entryErr.Message, wantMessage) - } - got, err := tb.repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil || got == nil { - t.Fatalf("version after rejected DeleteObjects = %#v err=%v, want retained", got, err) - } -} - -// ---------- CopyObject ---------- - func TestCopyObject_HappyPath(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() @@ -2745,20 +2820,27 @@ func TestCopyObjectIdenticalCurrentObjectCreatesNewVersion(t *testing.T) { t.Fatalf("object version count = %d, want 2", versionCount) } + // Ingest is scheduled per content and both copies carry the same bytes into + // the same bucket, so the two versions share one plan. + if first, second := contentSubjectForVersion(t, tb, obj1.VersionID), contentSubjectForVersion(t, tb, obj2.VersionID); first != second { + t.Fatalf("identical copies resolved to contents %s and %s, want one", first, second) + } taskCount, err := tb.db.NewSelect(). Model((*model.Task)(nil)). - Where("ref_type = ? AND ref_id = ?", "object", obj1.ObjectID). + Where("type = ?", model.TaskTypeUploadPlan). + Where("subject_type = ?", "storage_content"). + Where("subject_key = ?", contentSubjectForVersion(t, tb, obj2.VersionID)). Count(ctx) if err != nil { t.Fatalf("counting upload tasks: %v", err) } - if taskCount != 2 { - t.Fatalf("task count = %d, want 2", taskCount) + if taskCount != 1 { + t.Fatalf("task count = %d, want 1", taskCount) } } -func TestCopyObjectUsesConfiguredUploadMaxRetries(t *testing.T) { - tb := newTestBackendWithOptions(t, synaps3backend.WithUploadMaxRetries(13)) +func TestCopyObjectEnqueuesRegisteredUploadPlan(t *testing.T) { + tb := newTestBackend(t) ctx := context.Background() seedActiveBucket(t, tb, "copy-retry-src") seedActiveBucket(t, tb, "copy-retry-dst") @@ -2782,22 +2864,19 @@ func TestCopyObjectUsesConfiguredUploadMaxRetries(t *testing.T) { t.Fatalf("GetByBucketAndKey: %v", err) } - tasks, _, err := tb.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "", string(model.TaskStatusQueued), 10, 0) + page, err := tb.repos.Tasks.List(ctx, repository.TaskListFilter{Type: model.TaskTypeUploadPlan, Status: model.TaskStatusPending, Limit: 10}) if err != nil { t.Fatalf("List tasks: %v", err) } - for _, task := range tasks { - if task.RefType == "object" && task.RefID == dstObj.ObjectID { - if task.MaxRetries != 13 { - t.Fatalf("copy upload task MaxRetries = %d, want 13", task.MaxRetries) - } - if task.Stage == nil || *task.Stage != "prepare_upload" { - t.Fatalf("copy upload task Stage = %#v, want prepare_upload", task.Stage) + for _, task := range page.Tasks { + if task.SubjectType != nil && task.SubjectKey != nil && *task.SubjectType == "storage_content" && *task.SubjectKey == contentSubjectForVersion(t, tb, dstObj.VersionID) { + if task.RetryLimit == nil || *task.RetryLimit != 5 { + t.Fatalf("copy upload task retry limit = %v, want 5", task.RetryLimit) } return } } - t.Fatalf("copy upload task for object %d not found in %#v", dstObj.ObjectID, tasks) + t.Fatalf("copy upload task for object %d not found in %#v", dstObj.ObjectID, page.Tasks) } func TestCopyObject_MetadataReplace(t *testing.T) { @@ -3011,8 +3090,11 @@ func TestRestoreObjectVersionCreatesNewCurrentAndPreservesHistory(t *testing.T) if current.VersionID != versionID { t.Fatalf("current version = %s, want %s", current.VersionID, versionID) } - if current.CacheKey == sourceBefore.CacheKey || !tb.cache.Exists(ctx, bucketName, current.CacheKey) { - t.Fatalf("restored cache key = %q, source = %q, exists=%v", current.CacheKey, sourceBefore.CacheKey, tb.cache.Exists(ctx, bucketName, current.CacheKey)) + // A restore rewrites the same bytes, which resolve to the source's + // content, so the two versions share one cache file rather than + // each keeping a private copy. + if current.CacheKey() != sourceBefore.CacheKey() || !tb.cache.Exists(ctx, bucketName, current.CacheKey()) { + t.Fatalf("restored cache key = %q, source = %q, exists=%v", current.CacheKey(), sourceBefore.CacheKey(), tb.cache.Exists(ctx, bucketName, current.CacheKey())) } if current.ContentType != sourceBefore.ContentType { t.Fatalf("content type = %q, want %q", current.ContentType, sourceBefore.ContentType) @@ -3170,22 +3252,30 @@ func TestRestoreObjectVersionRejectsUnavailableSources(t *testing.T) { ctx := context.Background() bucket := seedActiveBucket(t, tb, "restore-unreadable-source") versionID := "01J0000000000000000000BR00" + unreadableSize := int64(len(validTestObjectBody("unreadable"))) + unreadableContent, err := tb.repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, + ContentSize: unreadableSize, + Checksum: synaps3testutil.StorageChecksum("unreadable-checksum"), + RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("create unreadable content: %v", err) + } if _, err := tb.repos.Objects.CreateVersionAndSetCurrent(ctx, &model.ObjectVersion{ VersionID: versionID, BucketID: bucket.ID, Key: "file.txt", - Size: int64(len(validTestObjectBody("unreadable"))), + ContentID: &unreadableContent.ID, + Size: unreadableSize, ETag: "unreadable-etag", - Checksum: "unreadable-checksum", ContentType: "text/plain", - CacheKey: ".versions/" + versionID, - State: model.ObjectStateCached, }); err != nil { t.Fatalf("create unreadable version: %v", err) } current := putValidTestObjectOutput(t, tb, bucket.Name, "file.txt", "current") - _, err := tb.backend.RestoreObjectVersion(ctx, bucket.Name, "file.txt", versionID, current.VersionID) + _, err = tb.backend.RestoreObjectVersion(ctx, bucket.Name, "file.txt", versionID, current.VersionID) if err == nil { t.Fatal("RestoreObjectVersion succeeded for unreadable source") } @@ -3202,7 +3292,18 @@ func TestRestoreObjectVersionRejectsUnavailableSources(t *testing.T) { ctx := context.Background() bucket := seedActiveBucket(t, tb, "restore-permanently-deleted") seed := func(versionID, body string) { - info, err := tb.cache.Put(ctx, bucket.Name, ".versions/"+versionID, strings.NewReader(validTestObjectBody(body))) + payload := validTestObjectBody(body) + sum := sha256.Sum256([]byte(payload)) + content, err := tb.repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, + ContentSize: int64(len(payload)), + Checksum: hex.EncodeToString(sum[:]), + RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("ensure content: %v", err) + } + info, err := tb.cache.Put(ctx, bucket.Name, model.ContentCacheKey(content.ID), strings.NewReader(payload)) if err != nil { t.Fatalf("cache Put: %v", err) } @@ -3210,12 +3311,10 @@ func TestRestoreObjectVersionRejectsUnavailableSources(t *testing.T) { VersionID: versionID, BucketID: bucket.ID, Key: "file.txt", + ContentID: &content.ID, Size: info.Size, ETag: info.ETag, - Checksum: info.Checksum, ContentType: "text/plain", - CacheKey: ".versions/" + versionID, - State: model.ObjectStateCached, }); err != nil { t.Fatalf("create version: %v", err) } @@ -3347,8 +3446,18 @@ func TestRestoreObjectVersionCASConflictCleansCommittedCache(t *testing.T) { var usedAfterConcurrentWrite int64 changingCache.onGet = func() { versionID := model.NewVersionID() - cacheKey := ".versions/" + versionID - info, err := baseCache.Put(ctx, bucket.Name, cacheKey, strings.NewReader(validTestObjectBody("concurrent"))) + payload := validTestObjectBody("concurrent") + sum := sha256.Sum256([]byte(payload)) + content, err := tb.repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, + ContentSize: int64(len(payload)), + Checksum: hex.EncodeToString(sum[:]), + RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("ensure concurrent content: %v", err) + } + info, err := baseCache.Put(ctx, bucket.Name, model.ContentCacheKey(content.ID), strings.NewReader(payload)) if err != nil { t.Fatalf("cache concurrent version: %v", err) } @@ -3356,12 +3465,10 @@ func TestRestoreObjectVersionCASConflictCleansCommittedCache(t *testing.T) { VersionID: versionID, BucketID: bucket.ID, Key: "file.txt", + ContentID: &content.ID, Size: info.Size, ETag: info.ETag, - Checksum: info.Checksum, ContentType: "text/plain", - CacheKey: cacheKey, - State: model.ObjectStateCached, }); err != nil { t.Fatalf("create concurrent version: %v", err) } @@ -3400,8 +3507,18 @@ func TestCopyObjectBindsImplicitCurrentReadToResolvedVersion(t *testing.T) { ObjectRepository: baseObjects, afterFirstRead: func() { newVersionID := model.NewVersionID() - cacheKey := path.Join(".versions", newVersionID) - info, err := tb.cache.Put(ctx, "copy-implicit-race-src", cacheKey, strings.NewReader("new")) + sum := sha256.Sum256([]byte("new")) + content, err := tb.repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: srcBkt.ID, + ContentSize: int64(len("new")), + Checksum: hex.EncodeToString(sum[:]), + RequestedCopies: 1, + }) + if err != nil { + hookErr = err + return + } + info, err := tb.cache.Put(ctx, "copy-implicit-race-src", model.ContentCacheKey(content.ID), strings.NewReader("new")) if err != nil { hookErr = err return @@ -3410,12 +3527,10 @@ func TestCopyObjectBindsImplicitCurrentReadToResolvedVersion(t *testing.T) { VersionID: newVersionID, BucketID: srcBkt.ID, Key: "original.txt", + ContentID: &content.ID, Size: info.Size, ETag: info.ETag, - Checksum: info.Checksum, ContentType: "text/plain", - CacheKey: cacheKey, - State: model.ObjectStateCached, }) }, } diff --git a/internal/backend/onchain_id_internal_test.go b/internal/backend/onchain_id_internal_test.go deleted file mode 100644 index cfe1c86..0000000 --- a/internal/backend/onchain_id_internal_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package backend - -import ( - "testing" - - "github.com/strahe/synaps3/internal/types" -) - -func onChainID(t *testing.T, value string) types.OnChainID { - t.Helper() - id, err := types.ParseOnChainID("test id", value) - if err != nil { - t.Fatalf("parse on-chain id %q: %v", value, err) - } - return id -} - -func onChainIDPtr(t *testing.T, value string) *types.OnChainID { - t.Helper() - id := onChainID(t, value) - return &id -} diff --git a/internal/backend/testutil_test.go b/internal/backend/testutil_test.go index 9db8798..475550e 100644 --- a/internal/backend/testutil_test.go +++ b/internal/backend/testutil_test.go @@ -1,6 +1,7 @@ package backend_test import ( + "context" "crypto/sha256" "encoding/hex" "errors" @@ -8,14 +9,15 @@ import ( "net/http" "strings" "testing" + "time" "github.com/strahe/synaps3/internal/backend" "github.com/strahe/synaps3/internal/cache" "github.com/strahe/synaps3/internal/cacheaccess" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/state" "github.com/strahe/synaps3/internal/synapse" + taskengine "github.com/strahe/synaps3/internal/task" "github.com/strahe/synaps3/internal/testutil" "github.com/strahe/synapse-go/chain" "github.com/uptrace/bun" @@ -28,8 +30,10 @@ type testBackend struct { backend *backend.SynapseBackend repos *repository.Repositories cache cache.Cache + gate *cacheaccess.Gate storage *testutil.MockStorageClient db *bun.DB + tasks *taskengine.Service } // newTestBackend constructs a SynapseBackend backed by in-memory SQLite @@ -43,15 +47,15 @@ func newTestBackendWithOptions(t *testing.T, opts ...backend.Option) *testBacken db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) fsCache := newTestCache(t, 1<<30) // 1 GB - sm := state.NewObjectStateMachine() sc := &testutil.MockStorageClient{} logger := slog.Default() cacheGate, accessTracker := newBackendCacheAccess(repos) + taskService := newBackendTaskService(t, repos) + opts = append(opts, backend.WithTaskService(taskService)) b := backend.New( repos, fsCache, - sm, sc, cacheGate, accessTracker, @@ -62,8 +66,10 @@ func newTestBackendWithOptions(t *testing.T, opts ...backend.Option) *testBacken backend: b, repos: repos, cache: fsCache, + gate: cacheGate, storage: sc, db: db, + tasks: taskService, } } @@ -73,18 +79,20 @@ func newTestBackendWithMockCache(t *testing.T, mc *testutil.MockCache) *testBack t.Helper() db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) - sm := state.NewObjectStateMachine() sc := &testutil.MockStorageClient{} logger := slog.Default() cacheGate, accessTracker := newBackendCacheAccess(repos) - b := backend.New(repos, mc, sm, sc, cacheGate, accessTracker, logger) + taskService := newBackendTaskService(t, repos) + b := backend.New(repos, mc, sc, cacheGate, accessTracker, logger, backend.WithTaskService(taskService)) return &testBackend{ backend: b, repos: repos, cache: mc, + gate: cacheGate, storage: sc, db: db, + tasks: taskService, } } @@ -92,18 +100,20 @@ func newTestBackendWithCache(t *testing.T, c cache.Cache) *testBackend { t.Helper() db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) - sm := state.NewObjectStateMachine() sc := &testutil.MockStorageClient{} logger := slog.Default() cacheGate, accessTracker := newBackendCacheAccess(repos) - b := backend.New(repos, c, sm, sc, cacheGate, accessTracker, logger) + taskService := newBackendTaskService(t, repos) + b := backend.New(repos, c, sc, cacheGate, accessTracker, logger, backend.WithTaskService(taskService)) return &testBackend{ backend: b, repos: repos, cache: c, + gate: cacheGate, storage: sc, db: db, + tasks: taskService, } } @@ -113,19 +123,60 @@ func newTestBackendWithSDK(t *testing.T, sc synapse.StorageClient) *testBackend db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) fsCache := newTestCache(t, 1<<30) - sm := state.NewObjectStateMachine() logger := slog.Default() cacheGate, accessTracker := newBackendCacheAccess(repos) - b := backend.New(repos, fsCache, sm, sc, cacheGate, accessTracker, logger) + taskService := newBackendTaskService(t, repos) + b := backend.New(repos, fsCache, sc, cacheGate, accessTracker, logger, backend.WithTaskService(taskService)) return &testBackend{ backend: b, repos: repos, cache: fsCache, + gate: cacheGate, db: db, + tasks: taskService, } } +type backendTestTaskHandler struct { + definition taskengine.Definition +} + +func (h backendTestTaskHandler) Definition() taskengine.Definition { return h.definition } +func (backendTestTaskHandler) Execute(context.Context, taskengine.Execution) taskengine.Result { + return taskengine.Complete("", nil) +} + +func (backendTestTaskHandler) Recover(context.Context, taskengine.Execution) taskengine.Result { + return taskengine.Complete("", nil) +} + +func newBackendTaskService(t *testing.T, repos *repository.Repositories) *taskengine.Service { + t.Helper() + registry := taskengine.NewRegistry() + retryLimit := 5 + for _, taskType := range []model.TaskType{ + model.TaskTypeBucketProvision, + model.TaskTypeUploadPlan, + model.TaskTypeCacheEvict, + model.TaskTypeStorageCleanup, + } { + err := registry.Register(backendTestTaskHandler{definition: taskengine.Definition{ + Type: taskType, InputVersion: 1, + Codec: taskengine.StrictJSONCodec[map[string]any](nil), + RetryLimit: &retryLimit, AllowRetry: true, + }}) + if err != nil { + t.Fatalf("registering test task type %s: %v", taskType, err) + } + } + service, err := taskengine.NewService(registry, repos, 7*24*time.Hour) + if err != nil { + t.Fatalf("creating task service: %v", err) + } + return service +} + func newBackendCacheAccess( repos *repository.Repositories, ) (*cacheaccess.Gate, *cacheaccess.Tracker) { diff --git a/internal/bucketlifecycle/provision.go b/internal/bucketlifecycle/provision.go new file mode 100644 index 0000000..cab37f0 --- /dev/null +++ b/internal/bucketlifecycle/provision.go @@ -0,0 +1,31 @@ +package bucketlifecycle + +import ( + "fmt" + + "github.com/strahe/synaps3/internal/db/repository" +) + +// ProvisionInput identifies the bucket whose provider storage must be ready. +type ProvisionInput struct { + BucketID int64 `json:"bucket_id"` +} + +func ValidateProvisionInput(input ProvisionInput) error { + if input.BucketID < 1 { + return fmt.Errorf("bucketID must be positive: %w", repository.ErrInvalidInput) + } + return nil +} + +// ProvisionKey scopes provisioning to a replica target, so raising a bucket's +// target schedules a fresh run instead of colliding with the completed one that +// provisioned the smaller target. +// +// This is only sound while the target can never go down: a target that returned +// to an earlier value would rebuild a key whose task already completed, and the +// slots opened for it would never be provisioned. Whoever implements lowering +// has to replace this with a monotonic generation first. +func ProvisionKey(bucketID int64, copies int) string { + return fmt.Sprintf("bucket:%d:provision:%d", bucketID, copies) +} diff --git a/internal/bucketlifecycle/service.go b/internal/bucketlifecycle/service.go index 647c959..d08d44c 100644 --- a/internal/bucketlifecycle/service.go +++ b/internal/bucketlifecycle/service.go @@ -5,34 +5,55 @@ import ( "errors" "fmt" "log/slog" + "strconv" "github.com/strahe/synaps3/internal/cache" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" + taskengine "github.com/strahe/synaps3/internal/task" ) // Service coordinates bucket lifecycle operations shared by multiple entrypoints. type Service struct { - repos *repository.Repositories - cache cache.Cache - logger *slog.Logger + repos *repository.Repositories + cache cache.Cache + tasks *taskengine.Service + // defaultCopies is the configured replica target a bucket materialises when + // the caller does not name one. A bucket's policy is stored, not inherited, + // so later config changes leave existing buckets alone. + defaultCopies int + logger *slog.Logger } var ( ErrBucketNotFound = errors.New("bucket not found") ErrBucketNotEmpty = errors.New("bucket not empty") ErrDeleteNotSupported = errors.New("bucket deletion is not supported") + ErrOwnerNotFound = errors.New("bucket owner not found") ) +type CreateOptions struct { + Name string + ACL []byte + OwnerAccessKey *string + DefaultCopies *int + MinimumDurableCopies *int +} + type DeleteOptions struct { Recursive bool } -func New(repos *repository.Repositories, c cache.Cache, logger *slog.Logger) *Service { +func (s *Service) SetTaskService(service *taskengine.Service) { + s.tasks = service +} + +func New(repos *repository.Repositories, c cache.Cache, defaultCopies int, logger *slog.Logger) *Service { return &Service{ - repos: repos, - cache: c, - logger: logger, + repos: repos, + cache: c, + defaultCopies: model.ClampStorageCopies(defaultCopies), + logger: logger, } } @@ -41,22 +62,69 @@ func (s *Service) Create(ctx context.Context, name string) (*model.Bucket, error } func (s *Service) CreateWithACL(ctx context.Context, name string, acl []byte) (*model.Bucket, error) { + return s.CreateWithOptions(ctx, CreateOptions{Name: name, ACL: acl}) +} + +func (s *Service) CreateWithOptions(ctx context.Context, options CreateOptions) (*model.Bucket, error) { + if s.tasks == nil { + return nil, errors.New("bucket lifecycle requires a task service") + } + // The durability policy is materialised here: a bucket that exists knows how + // many replica slots it has without anyone re-reading configuration. + defaultCopies := s.defaultCopies + if options.DefaultCopies != nil { + defaultCopies = *options.DefaultCopies + } + if !model.ValidStorageCopies(defaultCopies) { + return nil, fmt.Errorf("creating bucket %q: replica target %d out of range: %w", options.Name, defaultCopies, repository.ErrInvalidInput) + } + minimumDurableCopies := defaultCopies + if options.MinimumDurableCopies != nil { + minimumDurableCopies = *options.MinimumDurableCopies + } + if !model.ValidStorageCopies(minimumDurableCopies) || minimumDurableCopies > defaultCopies { + return nil, fmt.Errorf("creating bucket %q: minimum durable copies %d out of range: %w", options.Name, minimumDurableCopies, repository.ErrInvalidInput) + } bucket := &model.Bucket{ - Name: name, - ACL: acl, - Status: model.BucketStatusActive, + Name: options.Name, + ACL: options.ACL, + OwnerAccessKey: options.OwnerAccessKey, + DefaultCopies: defaultCopies, + MinimumDurableCopies: minimumDurableCopies, + Status: model.BucketStatusProvisioning, } - if err := s.repos.Buckets.Create(ctx, bucket); err != nil { - return nil, fmt.Errorf("creating bucket %q: %w", name, err) + if err := s.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { + if options.OwnerAccessKey != nil { + owner, err := txRepos.S3Accounts.LockByAccessKey(ctx, *options.OwnerAccessKey) + if err != nil { + return err + } + if owner == nil { + return ErrOwnerNotFound + } + } + if err := txRepos.Buckets.Create(ctx, bucket); err != nil { + return err + } + _, _, err := s.tasks.EnqueueInTransaction(ctx, txRepos, taskengine.EnqueueRequest{ + Type: model.TaskTypeBucketProvision, + IdempotencyKey: ProvisionKey(bucket.ID, bucket.DefaultCopies), + Input: ProvisionInput{BucketID: bucket.ID}, + SubjectType: "bucket", + SubjectKey: strconv.FormatInt(bucket.ID, 10), + }) + return err + }); err != nil { + return nil, fmt.Errorf("creating bucket %q: %w", options.Name, err) } - if err := s.cache.CreateBucketDir(ctx, name); err != nil && s.logger != nil { - s.logger.Warn("pre-creating cache dir failed (non-fatal)", "bucket", name, "error", err) + if err := s.cache.CreateBucketDir(ctx, options.Name); err != nil && s.logger != nil { + s.logger.Warn("pre-creating cache dir failed (non-fatal)", "bucket", options.Name, "error", err) } if s.logger != nil { - s.logger.Info("bucket created", "bucket", name, "id", bucket.ID) + s.logger.Info("bucket created", "bucket", options.Name, "id", bucket.ID) } return bucket, nil } @@ -70,3 +138,20 @@ func (s *Service) EnsureCacheBucketDir(ctx context.Context, name string) { func (s *Service) Delete(_ context.Context, _ string, _ DeleteOptions) (*model.Bucket, error) { return nil, ErrDeleteNotSupported } + +// ScheduleProvision enqueues provisioning for the bucket's current replica +// target. It is idempotent per target: raising the target schedules the slots +// that were just opened, lowering it schedules a run that finds nothing to do. +func (s *Service) ScheduleProvision(ctx context.Context, repos *repository.Repositories, bucket *model.Bucket) error { + if s.tasks == nil { + return errors.New("bucket lifecycle requires a task service") + } + _, _, err := s.tasks.EnqueueInTransaction(ctx, repos, taskengine.EnqueueRequest{ + Type: model.TaskTypeBucketProvision, + IdempotencyKey: ProvisionKey(bucket.ID, bucket.DefaultCopies), + Input: ProvisionInput{BucketID: bucket.ID}, + SubjectType: "bucket", + SubjectKey: strconv.FormatInt(bucket.ID, 10), + }) + return err +} diff --git a/internal/bucketlifecycle/service_test.go b/internal/bucketlifecycle/service_test.go index a55c46d..6de069c 100644 --- a/internal/bucketlifecycle/service_test.go +++ b/internal/bucketlifecycle/service_test.go @@ -6,12 +6,48 @@ import ( "errors" "log/slog" "testing" + "time" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" + taskengine "github.com/strahe/synaps3/internal/task" "github.com/strahe/synaps3/internal/testutil" ) +type provisionTestHandler struct{ definition taskengine.Definition } + +func (h provisionTestHandler) Definition() taskengine.Definition { return h.definition } +func (provisionTestHandler) Execute(context.Context, taskengine.Execution) taskengine.Result { + return taskengine.Complete("", nil) +} + +func (provisionTestHandler) Recover(context.Context, taskengine.Execution) taskengine.Result { + return taskengine.Complete("", nil) +} + +func newLifecycleTestService(t *testing.T, repos *repository.Repositories, c *testutil.MockCache, logger *slog.Logger) *Service { + t.Helper() + registry := taskengine.NewRegistry() + retryLimit := 5 + err := registry.Register(provisionTestHandler{definition: taskengine.Definition{ + Type: model.TaskTypeBucketProvision, InputVersion: 1, + Codec: taskengine.StrictJSONCodec(func(input *ProvisionInput) error { + return ValidateProvisionInput(*input) + }), + RetryLimit: &retryLimit, AllowRetry: true, + }}) + if err != nil { + t.Fatalf("register bucket provision handler: %v", err) + } + tasks, err := taskengine.NewService(registry, repos, 7*24*time.Hour) + if err != nil { + t.Fatalf("create task service: %v", err) + } + service := New(repos, c, 2, logger) + service.SetTaskService(tasks) + return service +} + func TestServiceCreatePrecreatesCacheBucketDir(t *testing.T) { repos := testutil.NewTestRepos(t) ctx := context.Background() @@ -25,7 +61,7 @@ func TestServiceCreatePrecreatesCacheBucketDir(t *testing.T) { return nil }, } - s := New(repos, mockCache, slog.Default()) + s := newLifecycleTestService(t, repos, mockCache, slog.Default()) bucket, err := s.Create(ctx, "test-bucket") if err != nil { @@ -35,8 +71,8 @@ func TestServiceCreatePrecreatesCacheBucketDir(t *testing.T) { if bucket.Name != "test-bucket" { t.Fatalf("bucket name = %q, want %q", bucket.Name, "test-bucket") } - if bucket.Status != model.BucketStatusActive { - t.Fatalf("bucket status = %s, want %s", bucket.Status, model.BucketStatusActive) + if bucket.Status != model.BucketStatusProvisioning { + t.Fatalf("bucket status = %s, want %s", bucket.Status, model.BucketStatusProvisioning) } if !cacheCalled { t.Fatal("cache directory was not pre-created") @@ -48,8 +84,12 @@ func TestServiceCreatePrecreatesCacheBucketDir(t *testing.T) { if persisted == nil { t.Fatal("persisted bucket is nil") } - if persisted.Status != model.BucketStatusActive { - t.Fatalf("persisted bucket status = %s, want %s", persisted.Status, model.BucketStatusActive) + if persisted.Status != model.BucketStatusProvisioning { + t.Fatalf("persisted bucket status = %s, want %s", persisted.Status, model.BucketStatusProvisioning) + } + taskRow, err := repos.Tasks.GetByIdentity(ctx, model.TaskTypeBucketProvision, ProvisionKey(bucket.ID, bucket.DefaultCopies)) + if err != nil || taskRow == nil || taskRow.Status != model.TaskStatusPending { + t.Fatalf("bucket provision task = %#v, err=%v", taskRow, err) } } @@ -62,7 +102,7 @@ func TestServiceCreateKeepsBucketWhenCacheDirPrecreateFails(t *testing.T) { }, } var logBuf bytes.Buffer - s := New(repos, mockCache, slog.New(slog.NewTextHandler(&logBuf, nil))) + s := newLifecycleTestService(t, repos, mockCache, slog.New(slog.NewTextHandler(&logBuf, nil))) bucket, err := s.Create(ctx, "test-bucket-cache-error") if err != nil { @@ -82,15 +122,15 @@ func TestServiceCreateKeepsBucketWhenCacheDirPrecreateFails(t *testing.T) { if persisted == nil { t.Fatal("persisted bucket is nil") } - if persisted.Status != model.BucketStatusActive { - t.Fatalf("persisted bucket status = %s, want %s", persisted.Status, model.BucketStatusActive) + if persisted.Status != model.BucketStatusProvisioning { + t.Fatalf("persisted bucket status = %s, want %s", persisted.Status, model.BucketStatusProvisioning) } } func TestServiceCreateWithACLPersistsACL(t *testing.T) { repos := testutil.NewTestRepos(t) ctx := context.Background() - s := New(repos, &testutil.MockCache{}, slog.Default()) + s := newLifecycleTestService(t, repos, &testutil.MockCache{}, slog.Default()) acl := []byte(`{"Owner":"owner-access"}`) bucket, err := s.CreateWithACL(ctx, "test-bucket-acl", acl) @@ -116,7 +156,7 @@ func TestServiceCreateWithACLPersistsACL(t *testing.T) { func TestServiceCreateReturnsErrorWhenBucketCreateFails(t *testing.T) { repos := testutil.NewTestRepos(t) ctx := context.Background() - if err := repos.Buckets.Create(ctx, &model.Bucket{Name: "test-bucket-duplicate", Status: model.BucketStatusActive}); err != nil { + if err := repos.Buckets.Create(ctx, &model.Bucket{Name: "test-bucket-duplicate", Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8}); err != nil { t.Fatalf("seed bucket: %v", err) } var cacheCalled bool @@ -126,7 +166,7 @@ func TestServiceCreateReturnsErrorWhenBucketCreateFails(t *testing.T) { return nil }, } - s := New(repos, mockCache, slog.Default()) + s := newLifecycleTestService(t, repos, mockCache, slog.Default()) _, err := s.Create(ctx, "test-bucket-duplicate") if !errors.Is(err, repository.ErrAlreadyExists) { @@ -137,6 +177,34 @@ func TestServiceCreateReturnsErrorWhenBucketCreateFails(t *testing.T) { } } +func TestServiceCreateRollsBackBucketWhenProvisionTaskCannotBeEnqueued(t *testing.T) { + repos := testutil.NewTestRepos(t) + registry := taskengine.NewRegistry() + tasks, err := taskengine.NewService(registry, repos, time.Hour) + if err != nil { + t.Fatalf("create task service: %v", err) + } + var cacheCalled bool + mockCache := &testutil.MockCache{CreateBucketDirFunc: func(context.Context, string) error { + cacheCalled = true + return nil + }} + service := New(repos, mockCache, 2, slog.Default()) + service.SetTaskService(tasks) + + _, err = service.Create(t.Context(), "atomic-create") + if !errors.Is(err, taskengine.ErrUnknownType) { + t.Fatalf("Create error = %v, want unknown task type", err) + } + stored, getErr := repos.Buckets.GetByName(t.Context(), "atomic-create") + if getErr != nil || stored != nil { + t.Fatalf("rolled-back bucket = %#v, err=%v", stored, getErr) + } + if cacheCalled { + t.Fatal("cache directory was created after transaction rollback") + } +} + func TestServiceEnsureCacheBucketDirCallsCache(t *testing.T) { repos := testutil.NewTestRepos(t) var cacheCalled bool @@ -149,7 +217,7 @@ func TestServiceEnsureCacheBucketDirCallsCache(t *testing.T) { return nil }, } - s := New(repos, mockCache, slog.Default()) + s := newLifecycleTestService(t, repos, mockCache, slog.Default()) s.EnsureCacheBucketDir(context.Background(), "test-bucket") @@ -160,7 +228,7 @@ func TestServiceEnsureCacheBucketDirCallsCache(t *testing.T) { func TestServiceDeleteReturnsUnsupported(t *testing.T) { repos := testutil.NewTestRepos(t) - s := New(repos, &testutil.MockCache{}, slog.Default()) + s := newLifecycleTestService(t, repos, &testutil.MockCache{}, slog.Default()) _, err := s.Delete(context.Background(), "test-bucket", DeleteOptions{}) if !errors.Is(err, ErrDeleteNotSupported) { diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 393d1cd..468d42b 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -26,6 +26,7 @@ type ObjectInfo struct { type StagedObject struct { Info *ObjectInfo commit func() error + commitAs func(bucket, key string) error rollback func() error } @@ -33,6 +34,11 @@ type StagedObject struct { // Must be called exactly once. After Commit, Rollback is a no-op. func (s *StagedObject) Commit() error { return s.commit() } +// CommitAs commits to a different final path than the one the object was staged +// under. Cache residency is content-addressed, so writers stage the bytes first +// and only learn the destination once the checksum has produced a content row. +func (s *StagedObject) CommitAs(bucket, key string) error { return s.commitAs(bucket, key) } + // Rollback removes the staged temp file without affecting the final cache path. // Safe to call multiple times and after Commit (no-op if already committed). func (s *StagedObject) Rollback() error { return s.rollback() } @@ -82,11 +88,12 @@ type Cache interface { // Returns ObjectInfo with the part's Size, ETag (MD5), and Checksum (SHA-256). PutPart(ctx context.Context, uploadID string, partNumber int, r io.Reader) (*ObjectInfo, error) - // AssembleParts concatenates the specified parts in order into a final - // object at bucket/key. Returns ObjectInfo for the assembled object and - // the ordered list of individual part MD5 hex digests (for S3 ETag computation). - // The part files are NOT deleted; call DeleteUpload to clean up. - AssembleParts(ctx context.Context, bucket, key, uploadID string, partNumbers []int) (*ObjectInfo, []string, error) + // AssemblePartsStaged concatenates the specified parts in order into a + // staged file next to bucket/key, plus the ordered list of individual part + // MD5 hex digests (for S3 ETag computation). The caller names the final + // destination through Commit or CommitAs once the assembled checksum has + // resolved a content row. The part files are NOT deleted; call DeleteUpload. + AssemblePartsStaged(ctx context.Context, bucket, key, uploadID string, partNumbers []int) (*StagedObject, []string, error) // DeleteUpload removes all part files for the given upload ID. DeleteUpload(ctx context.Context, uploadID string) error diff --git a/internal/cache/filesystem.go b/internal/cache/filesystem.go index 0be0755..7641b25 100644 --- a/internal/cache/filesystem.go +++ b/internal/cache/filesystem.go @@ -193,6 +193,21 @@ func (f *Filesystem) Put(ctx context.Context, bucket, key string, r io.Reader) ( return staged.Info, nil } +// stagedDestination resolves the final path for a staged commit and makes sure +// its directory exists, so a staged file can be named after its content once the +// checksum is known. +func (f *Filesystem) stagedDestination(bucket, key string) (string, string, error) { + dst, err := f.safePath(bucket, key) + if err != nil { + return "", "", err + } + dir := filepath.Dir(dst) + if err := ensurePrivateDir(dir); err != nil { + return "", "", fmt.Errorf("mkdir %s: %w", dir, err) + } + return dst, dir, nil +} + func (f *Filesystem) PutStaged(ctx context.Context, bucket, key string, r io.Reader) (*StagedObject, error) { dst, err := f.safePath(bucket, key) if err != nil { @@ -271,60 +286,68 @@ func (f *Filesystem) PutStaged(ctx context.Context, bucket, key string, r io.Rea var stateMu sync.Mutex committed := false - return &StagedObject{ - Info: info, - commit: func() error { - stateMu.Lock() - defer stateMu.Unlock() + commitAs := func(destBucket, destKey string) error { + stateMu.Lock() + defer stateMu.Unlock() - if committed { - return nil - } + if committed { + return nil + } - // Acquire shard lock for atomic rename and accounting. - mu := f.shardFor(bucket, key) - mu.Lock() - defer mu.Unlock() + destPath, destDir, err := f.stagedDestination(destBucket, destKey) + if err != nil { + return err + } - // Determine old file size for accounting. - var oldSize int64 - if stat, statErr := os.Stat(dst); statErr == nil { - oldSize = stat.Size() - } + // Acquire shard lock for atomic rename and accounting. + mu := f.shardFor(destBucket, destKey) + mu.Lock() + defer mu.Unlock() - // Reserve capacity. - delta := n - oldSize - var reserved bool - if f.maxBytes > 0 && delta > 0 { - newUsed := f.usedBytes.Add(delta) - if newUsed > f.maxBytes { - f.usedBytes.Add(-delta) - return ErrCacheFull - } - reserved = true - } + // Determine old file size for accounting. + var oldSize int64 + if stat, statErr := os.Stat(destPath); statErr == nil { + oldSize = stat.Size() + } - if err := os.Rename(tmpPath, dst); err != nil { - if reserved { - f.usedBytes.Add(-delta) - } - return fmt.Errorf("renaming temp to final: %w", err) + // Reserve capacity. + delta := n - oldSize + var reserved bool + if f.maxBytes > 0 && delta > 0 { + newUsed := f.usedBytes.Add(delta) + if newUsed > f.maxBytes { + f.usedBytes.Add(-delta) + return ErrCacheFull } + reserved = true + } - // Fsync parent directory to ensure the rename is durable. - if err := fsyncDir(dir); err != nil { - slog.Warn("fsync parent dir failed", "dir", dir, "error", err) + if err := os.Rename(tmpPath, destPath); err != nil { + if reserved { + f.usedBytes.Add(-delta) } + return fmt.Errorf("renaming temp to final: %w", err) + } - // Apply remaining accounting (shrink or unlimited mode). - if !reserved { - f.usedBytes.Add(delta) - } + // Fsync parent directory to ensure the rename is durable. + if err := fsyncDir(destDir); err != nil { + slog.Warn("fsync parent dir failed", "dir", destDir, "error", err) + } - committed = true - slog.Debug("cached object", "bucket", bucket, "key", key, "size", n) - return nil - }, + // Apply remaining accounting (shrink or unlimited mode). + if !reserved { + f.usedBytes.Add(delta) + } + + committed = true + info.Path = destPath + slog.Debug("cached object", "bucket", destBucket, "key", destKey, "size", n) + return nil + } + return &StagedObject{ + Info: info, + commit: func() error { return commitAs(bucket, key) }, + commitAs: commitAs, rollback: func() error { stateMu.Lock() defer stateMu.Unlock() @@ -578,7 +601,7 @@ func (f *Filesystem) PutPart(_ context.Context, uploadID string, partNumber int, }, nil } -func (f *Filesystem) AssembleParts(_ context.Context, bucket, key, uploadID string, partNumbers []int) (*ObjectInfo, []string, error) { +func (f *Filesystem) AssemblePartsStaged(_ context.Context, bucket, key, uploadID string, partNumbers []int) (*StagedObject, []string, error) { dst, err := f.safePath(bucket, key) if err != nil { return nil, nil, err @@ -588,15 +611,6 @@ func (f *Filesystem) AssembleParts(_ context.Context, bucket, key, uploadID stri return nil, nil, fmt.Errorf("mkdir %s: %w", dir, err) } - mu := f.shardFor(bucket, key) - mu.Lock() - defer mu.Unlock() - - var oldSize int64 - if stat, statErr := os.Stat(dst); statErr == nil { - oldSize = stat.Size() - } - file, err := os.CreateTemp(dir, ".synaps3-*.tmp") if err != nil { return nil, nil, fmt.Errorf("creating temp file: %w", err) @@ -637,53 +651,91 @@ func (f *Filesystem) AssembleParts(_ context.Context, bucket, key, uploadID stri partETags = append(partETags, hex.EncodeToString(partMD5.Sum(nil))) } - delta := totalSize - oldSize - var reserved bool - if f.maxBytes > 0 && delta > 0 { - newUsed := f.usedBytes.Add(delta) - if newUsed > f.maxBytes { - f.usedBytes.Add(-delta) - return nil, nil, ErrCacheFull - } - reserved = true - } - if err := file.Sync(); err != nil { - if reserved { - f.usedBytes.Add(-delta) - } return nil, nil, fmt.Errorf("fsync assembled file: %w", err) } if err := file.Close(); err != nil { - if reserved { - f.usedBytes.Add(-delta) - } return nil, nil, fmt.Errorf("closing assembled file: %w", err) } file = nil - if err := os.Rename(tmpPath, dst); err != nil { - if reserved { - f.usedBytes.Add(-delta) - } - _ = os.Remove(tmpPath) - return nil, nil, fmt.Errorf("renaming assembled to final: %w", err) - } - - if err := fsyncDir(dir); err != nil { - slog.Warn("fsync dir after assemble failed", "dir", dir, "error", err) - } - - if !reserved { - f.usedBytes.Add(delta) - } - info := &ObjectInfo{ Path: dst, Size: totalSize, Checksum: hex.EncodeToString(sha256Hash.Sum(nil)), } - return info, partETags, nil + + var stateMu sync.Mutex + committed := false + commitAs := func(destBucket, destKey string) error { + stateMu.Lock() + defer stateMu.Unlock() + + if committed { + return nil + } + + destPath, destDir, err := f.stagedDestination(destBucket, destKey) + if err != nil { + return err + } + + mu := f.shardFor(destBucket, destKey) + mu.Lock() + defer mu.Unlock() + + var oldSize int64 + if stat, statErr := os.Stat(destPath); statErr == nil { + oldSize = stat.Size() + } + + delta := totalSize - oldSize + var reserved bool + if f.maxBytes > 0 && delta > 0 { + newUsed := f.usedBytes.Add(delta) + if newUsed > f.maxBytes { + f.usedBytes.Add(-delta) + return ErrCacheFull + } + reserved = true + } + + if err := os.Rename(tmpPath, destPath); err != nil { + if reserved { + f.usedBytes.Add(-delta) + } + return fmt.Errorf("renaming assembled to final: %w", err) + } + + if err := fsyncDir(destDir); err != nil { + slog.Warn("fsync dir after assemble failed", "dir", destDir, "error", err) + } + + if !reserved { + f.usedBytes.Add(delta) + } + + committed = true + info.Path = destPath + return nil + } + return &StagedObject{ + Info: info, + commit: func() error { return commitAs(bucket, key) }, + commitAs: commitAs, + rollback: func() error { + stateMu.Lock() + defer stateMu.Unlock() + + if committed { + return nil + } + if err := os.Remove(tmpPath); err != nil && !os.IsNotExist(err) { + return err + } + return nil + }, + }, partETags, nil } func (f *Filesystem) DeleteUpload(_ context.Context, uploadID string) error { diff --git a/internal/cache/filesystem_test.go b/internal/cache/filesystem_test.go index 5cd1538..643012c 100644 --- a/internal/cache/filesystem_test.go +++ b/internal/cache/filesystem_test.go @@ -476,8 +476,17 @@ func TestAssemblePartsCapacityEnforcement(t *testing.T) { t.Fatalf("UsedBytes after parts = %d, want 10", fs.UsedBytes()) } - if _, _, err := fs.AssembleParts(ctx, "bkt", "key", "up-assemble-cap", []int{1, 2}); err != ErrCacheFull { - t.Fatalf("AssembleParts over capacity err = %v, want ErrCacheFull", err) + // Capacity is reserved when the staged file claims its destination, so the + // refusal surfaces at commit rather than during assembly. + staged, _, err := fs.AssemblePartsStaged(ctx, "bkt", "key", "up-assemble-cap", []int{1, 2}) + if err != nil { + t.Fatalf("AssemblePartsStaged: %v", err) + } + if err := staged.Commit(); err != ErrCacheFull { + t.Fatalf("assembled commit over capacity err = %v, want ErrCacheFull", err) + } + if err := staged.Rollback(); err != nil { + t.Fatalf("Rollback: %v", err) } if fs.UsedBytes() != 10 { t.Fatalf("UsedBytes after failed assemble = %d, want 10", fs.UsedBytes()) @@ -531,7 +540,7 @@ func TestConcurrentPuts(t *testing.T) { var wg sync.WaitGroup errs := make([]error, goroutines) - for i := 0; i < goroutines; i++ { + for i := range goroutines { wg.Add(1) go func(idx int) { defer wg.Done() @@ -621,10 +630,14 @@ func TestPutPartAndAssemble(t *testing.T) { } // Assemble - assembled, partETags, err := fs.AssembleParts(ctx, "bkt", "assembled-key", "upload-1", []int{1, 2}) + staged, partETags, err := fs.AssemblePartsStaged(ctx, "bkt", "assembled-key", "upload-1", []int{1, 2}) if err != nil { - t.Fatalf("AssembleParts: %v", err) + t.Fatalf("AssemblePartsStaged: %v", err) } + if err := staged.Commit(); err != nil { + t.Fatalf("commit assembled: %v", err) + } + assembled := staged.Info expectedData := append([]byte{}, part1Data...) expectedData = append(expectedData, part2Data...) @@ -727,9 +740,12 @@ func TestAssemblePartsUsedBytes(t *testing.T) { _, _ = fs.PutPart(ctx, "up-ub", 2, bytes.NewReader(p2)) partsSize := int64(len(p1) + len(p2)) - _, _, err := fs.AssembleParts(ctx, "bkt", "key", "up-ub", []int{1, 2}) + staged, _, err := fs.AssemblePartsStaged(ctx, "bkt", "key", "up-ub", []int{1, 2}) if err != nil { - t.Fatalf("AssembleParts: %v", err) + t.Fatalf("AssemblePartsStaged: %v", err) + } + if err := staged.Commit(); err != nil { + t.Fatalf("commit assembled: %v", err) } // UsedBytes = parts + assembled object diff --git a/internal/cacheaccess/gate.go b/internal/cacheaccess/gate.go index 1aab78f..c0420a2 100644 --- a/internal/cacheaccess/gate.go +++ b/internal/cacheaccess/gate.go @@ -28,47 +28,56 @@ type gateShard struct { } // Gate serializes cache opens and commits with physical deletion for the same -// object version. Idle entries are removed as soon as their last user exits. +// content cache key. Idle entries are removed as soon as their last user exits. type Gate struct { shards [gateShardCount]gateShard } -// NewGate creates a cache read/delete gate. +// NewGate creates a cache read/commit/delete gate. func NewGate() *Gate { return &Gate{} } // Open protects a successful cache open until the returned body is closed. func (g *Gate) Open( - versionID string, + cacheKey string, open func() (io.ReadCloser, *cache.ObjectInfo, error), ) (*OpenedCacheEntry, error) { - shard, entry := g.acquire(versionID) - entry.mu.RLock() + release := g.HoldRead(cacheKey) body, info, err := open() if err != nil { - entry.mu.RUnlock() - g.release(shard, versionID, entry) + release() return nil, err } return &OpenedCacheEntry{ Body: &guardedReadCloser{ - body: body, - release: func() { - entry.mu.RUnlock() - g.release(shard, versionID, entry) - }, + body: body, + release: release, }, Info: info, }, nil } -// Commit serializes a local cache commit with deletion for the same version. -func (g *Gate) Commit(versionID string, commit func() error) error { - shard, entry := g.acquire(versionID) - defer g.release(shard, versionID, entry) +// HoldRead protects a cache entry from deletion across multiple opens. The +// returned release function is idempotent and must be called by the holder. +func (g *Gate) HoldRead(cacheKey string) func() { + shard, entry := g.acquire(cacheKey) + entry.mu.RLock() + var once sync.Once + return func() { + once.Do(func() { + entry.mu.RUnlock() + g.release(shard, cacheKey, entry) + }) + } +} + +// Commit serializes a local cache commit with deletion for the same cache key. +func (g *Gate) Commit(cacheKey string, commit func() error) error { + shard, entry := g.acquire(cacheKey) + defer g.release(shard, cacheKey, entry) entry.mu.RLock() defer entry.mu.RUnlock() @@ -76,58 +85,55 @@ func (g *Gate) Commit(versionID string, commit func() error) error { } // GuardDeletion waits for open bodies and serializes final checks and physical -// deletion for one version. -func (g *Gate) GuardDeletion(versionID string, remove func()) { - shard, entry := g.acquire(versionID) - defer g.release(shard, versionID, entry) +// deletion for one cache key. +func (g *Gate) GuardDeletion(cacheKey string, remove func()) { + shard, entry := g.acquire(cacheKey) + defer g.release(shard, cacheKey, entry) entry.mu.Lock() defer entry.mu.Unlock() remove() } -func (g *Gate) guardAccess(versionID string, access func()) { - shard, entry := g.acquire(versionID) - defer g.release(shard, versionID, entry) - - entry.mu.RLock() - defer entry.mu.RUnlock() +func (g *Gate) guardAccess(cacheKey string, access func()) { + release := g.HoldRead(cacheKey) + defer release() access() } -func (g *Gate) acquire(versionID string) (*gateShard, *gateEntry) { - shard := g.shard(versionID) +func (g *Gate) acquire(cacheKey string) (*gateShard, *gateEntry) { + shard := g.shard(cacheKey) shard.mu.Lock() defer shard.mu.Unlock() if shard.entries == nil { shard.entries = make(map[string]*gateEntry) } - entry := shard.entries[versionID] + entry := shard.entries[cacheKey] if entry == nil { entry = &gateEntry{} - shard.entries[versionID] = entry + shard.entries[cacheKey] = entry } entry.refs++ return shard, entry } -func (*Gate) release(shard *gateShard, versionID string, entry *gateEntry) { +func (*Gate) release(shard *gateShard, cacheKey string, entry *gateEntry) { shard.mu.Lock() defer shard.mu.Unlock() entry.refs-- - if entry.refs == 0 && shard.entries[versionID] == entry { - delete(shard.entries, versionID) + if entry.refs == 0 && shard.entries[cacheKey] == entry { + delete(shard.entries, cacheKey) } } -func (g *Gate) shard(versionID string) *gateShard { +func (g *Gate) shard(cacheKey string) *gateShard { if g == nil { panic("nil cache access gate") } h := fnv.New32a() - _, _ = h.Write([]byte(versionID)) + _, _ = h.Write([]byte(cacheKey)) return &g.shards[h.Sum32()%gateShardCount] } diff --git a/internal/cacheaccess/gate_test.go b/internal/cacheaccess/gate_test.go index b1e5bf3..8dc755e 100644 --- a/internal/cacheaccess/gate_test.go +++ b/internal/cacheaccess/gate_test.go @@ -44,6 +44,21 @@ func TestGateWaitsForOpenedBodyBeforeDeletion(t *testing.T) { waitForGateSignal(t, deleteDone, time.Second, "deletion completion") } +func TestGateHoldReadProtectsMultipleOpensAsOneOperation(t *testing.T) { + gate := NewGate() + release := gate.HoldRead("content-1") + deleteStarted := make(chan struct{}) + go gate.GuardDeletion("content-1", func() { close(deleteStarted) }) + select { + case <-deleteStarted: + t.Fatal("deletion entered while the shared read guard was held") + case <-time.After(50 * time.Millisecond): + } + release() + release() + waitForGateSignal(t, deleteStarted, time.Second, "deletion after shared read guard") +} + func TestGateBlocksOpenWhileDeletionRuns(t *testing.T) { gate := NewGate() deleteStarted := make(chan struct{}) diff --git a/internal/cacheaccess/tracker.go b/internal/cacheaccess/tracker.go index 2cc150c..7e08d58 100644 --- a/internal/cacheaccess/tracker.go +++ b/internal/cacheaccess/tracker.go @@ -2,6 +2,7 @@ package cacheaccess import ( "context" + "encoding/binary" "errors" "fmt" "hash/fnv" @@ -11,6 +12,7 @@ import ( "time" "github.com/strahe/synaps3/internal/cacheeviction" + "github.com/strahe/synaps3/internal/model" ) const ( @@ -26,9 +28,11 @@ const ( var ErrLRUAccessUncertain = errors.New("cache access is not reliable enough for LRU eviction") // Store persists foreground cache access and cache commit metadata. +// Store persists cache recency. The unit is the content payload, because one +// cache file backs every version that shares those bytes. type Store interface { - RecordVersionCacheAccess(context.Context, string, time.Time) error - RecordVersionCacheCommit(context.Context, string, time.Time) error + RecordContentCacheAccess(context.Context, int64, time.Time) error + RecordContentCacheCommit(context.Context, int64, time.Time) error } type trackerEntry struct { @@ -41,7 +45,7 @@ type trackerEntry struct { type trackerShard struct { mu sync.Mutex - entries map[string]*trackerEntry + entries map[int64]*trackerEntry } // Tracker coalesces durable access-time writes and retains the latest exact @@ -114,28 +118,28 @@ func (t *Tracker) Run(ctx context.Context, gate *Gate, logger *slog.Logger) { // RecordAccess records one successful foreground cache open. func (t *Tracker) RecordAccess( ctx context.Context, - versionID string, + contentID int64, durableAccess *time.Time, ) error { - return t.record(ctx, versionID, durableAccess, false) + return t.record(ctx, contentID, durableAccess, false) } // RecordCommit records a successful local cache commit and forces the presence // and access metadata write. func (t *Tracker) RecordCommit( ctx context.Context, - versionID string, + contentID int64, durableAccess *time.Time, ) error { - return t.record(ctx, versionID, durableAccess, true) + return t.record(ctx, contentID, durableAccess, true) } // Latest returns the latest exact in-process access for one version. -func (t *Tracker) Latest(versionID string) time.Time { - shard := t.shard(versionID) +func (t *Tracker) Latest(contentID int64) time.Time { + shard := t.shard(contentID) shard.mu.Lock() defer shard.mu.Unlock() - if entry := shard.entries[versionID]; entry != nil { + if entry := shard.entries[contentID]; entry != nil { return entry.lastAccess } return time.Time{} @@ -143,16 +147,16 @@ func (t *Tracker) Latest(versionID string) time.Time { // FlushWhileGuarded persists one version's latest access. The caller must hold // the version's cache gate. -func (t *Tracker) FlushWhileGuarded(ctx context.Context, versionID string) error { - return t.flush(ctx, versionID, cacheeviction.NormalizeAccessTime(t.now()), true) +func (t *Tracker) FlushWhileGuarded(ctx context.Context, contentID int64) error { + return t.flush(ctx, contentID, cacheeviction.NormalizeAccessTime(t.now()), true) } // Forget removes tracking state after the cache entry is deleted or its // version is permanently removed. -func (t *Tracker) Forget(versionID string) { - shard := t.shard(versionID) +func (t *Tracker) Forget(contentID int64) { + shard := t.shard(contentID) shard.mu.Lock() - delete(shard.entries, versionID) + delete(shard.entries, contentID) shard.mu.Unlock() } @@ -164,25 +168,25 @@ func (t *Tracker) SafeForLRU() bool { func (t *Tracker) record( ctx context.Context, - versionID string, + contentID int64, durableAccess *time.Time, requireCommit bool, ) error { now := cacheeviction.NormalizeAccessTime(t.now()) - shard := t.shard(versionID) + shard := t.shard(contentID) shard.mu.Lock() if shard.entries == nil { - shard.entries = make(map[string]*trackerEntry) + shard.entries = make(map[int64]*trackerEntry) } - entry := shard.entries[versionID] + entry := shard.entries[contentID] if entry == nil { t.evictOldestCleanLocked(shard) if t.maxEntriesPerShard > 0 && len(shard.entries) >= t.maxEntriesPerShard { shard.mu.Unlock() - return t.persistOverflow(ctx, versionID, durableAccess, now, requireCommit) + return t.persistOverflow(ctx, contentID, durableAccess, now, requireCommit) } entry = &trackerEntry{} - shard.entries[versionID] = entry + shard.entries[contentID] = entry } if durableAccess != nil { @@ -210,20 +214,20 @@ func (t *Tracker) record( if !shouldPersist { return nil } - return t.persistAndAcknowledge(ctx, shard, versionID, entry, accessedAt, commit) + return t.persistAndAcknowledge(ctx, shard, contentID, entry, accessedAt, commit) } func (t *Tracker) persistOverflow( ctx context.Context, - versionID string, + contentID int64, durableAccess *time.Time, now time.Time, commit bool, ) error { accessedAt := nextAccessTime(now, time.Time{}, durableAccess) - if err := t.persist(ctx, versionID, accessedAt, commit); err != nil { + if err := t.persist(ctx, contentID, accessedAt, commit); err != nil { t.unsafeForLRU.Store(true) - return fmt.Errorf("%w: persisting access for %s: %w", ErrLRUAccessUncertain, versionID, err) + return fmt.Errorf("%w: persisting access for content %d: %w", ErrLRUAccessUncertain, contentID, err) } return nil } @@ -231,18 +235,18 @@ func (t *Tracker) persistOverflow( func (t *Tracker) persistAndAcknowledge( ctx context.Context, shard *trackerShard, - versionID string, + contentID int64, entry *trackerEntry, accessedAt time.Time, commit bool, ) error { - if err := t.persist(ctx, versionID, accessedAt, commit); err != nil { + if err := t.persist(ctx, contentID, accessedAt, commit); err != nil { return err } shard.mu.Lock() defer shard.mu.Unlock() - if shard.entries[versionID] != entry { + if shard.entries[contentID] != entry { return nil } if accessedAt.After(entry.lastPersisted) { @@ -256,30 +260,30 @@ func (t *Tracker) persistAndAcknowledge( func (t *Tracker) persist( ctx context.Context, - versionID string, + contentID int64, accessedAt time.Time, commit bool, ) error { if commit { - return t.store.RecordVersionCacheCommit(ctx, versionID, accessedAt) + return t.store.RecordContentCacheCommit(ctx, contentID, accessedAt) } - return t.store.RecordVersionCacheAccess(ctx, versionID, accessedAt) + return t.store.RecordContentCacheAccess(ctx, contentID, accessedAt) } func (t *Tracker) sweep(ctx context.Context, gate *Gate) (int, error) { now := cacheeviction.NormalizeAccessTime(t.now()) - versionIDs := t.snapshotVersionIDs() + contentIDs := t.snapshotContentIDs() var ( failed int firstErr error ) - for _, versionID := range versionIDs { + for _, contentID := range contentIDs { if ctx.Err() != nil { break } var err error - gate.guardAccess(versionID, func() { - err = t.flush(ctx, versionID, now, false) + gate.guardAccess(model.ContentCacheKey(contentID), func() { + err = t.flush(ctx, contentID, now, false) }) if err != nil { failed++ @@ -291,28 +295,28 @@ func (t *Tracker) sweep(ctx context.Context, gate *Gate) (int, error) { return failed, firstErr } -func (t *Tracker) snapshotVersionIDs() []string { - var versionIDs []string +func (t *Tracker) snapshotContentIDs() []int64 { + var contentIDs []int64 for index := range t.shards { shard := &t.shards[index] shard.mu.Lock() - for versionID := range shard.entries { - versionIDs = append(versionIDs, versionID) + for contentID := range shard.entries { + contentIDs = append(contentIDs, contentID) } shard.mu.Unlock() } - return versionIDs + return contentIDs } func (t *Tracker) flush( ctx context.Context, - versionID string, + contentID int64, now time.Time, force bool, ) error { - shard := t.shard(versionID) + shard := t.shard(contentID) shard.mu.Lock() - entry := shard.entries[versionID] + entry := shard.entries[contentID] if entry == nil { shard.mu.Unlock() return nil @@ -334,7 +338,7 @@ func (t *Tracker) flush( if err := t.persistAndAcknowledge( ctx, shard, - versionID, + contentID, entry, accessedAt, commit, @@ -345,13 +349,13 @@ func (t *Tracker) flush( shard.mu.Lock() defer shard.mu.Unlock() - if shard.entries[versionID] != entry { + if shard.entries[contentID] != entry { return nil } clean := !entry.lastAccess.After(entry.lastPersisted) && entry.commitRequired.IsZero() idle := !entry.lastTouched.IsZero() && now.Sub(entry.lastTouched) >= t.idleRetention if clean && idle { - delete(shard.entries, versionID) + delete(shard.entries, contentID) } return nil } @@ -361,14 +365,14 @@ func (t *Tracker) evictOldestCleanLocked(shard *trackerShard) { return } var ( - oldestID string + oldestID int64 oldestEntry *trackerEntry oldestTouched time.Time ) - for versionID, entry := range shard.entries { + for contentID, entry := range shard.entries { clean := !entry.lastAccess.After(entry.lastPersisted) && entry.commitRequired.IsZero() if clean && (oldestEntry == nil || entry.lastTouched.Before(oldestTouched)) { - oldestID = versionID + oldestID = contentID oldestEntry = entry oldestTouched = entry.lastTouched } @@ -393,11 +397,13 @@ func nextAccessTime(now, inMemory time.Time, durable *time.Time) time.Time { return now } -func (t *Tracker) shard(versionID string) *trackerShard { +func (t *Tracker) shard(contentID int64) *trackerShard { if t == nil { panic("nil cache access tracker") } h := fnv.New32a() - _, _ = h.Write([]byte(versionID)) + var buf [8]byte + binary.LittleEndian.PutUint64(buf[:], uint64(contentID)) + _, _ = h.Write(buf[:]) return &t.shards[h.Sum32()%trackerShardCount] } diff --git a/internal/cacheaccess/tracker_test.go b/internal/cacheaccess/tracker_test.go index 3dd0115..d529e0e 100644 --- a/internal/cacheaccess/tracker_test.go +++ b/internal/cacheaccess/tracker_test.go @@ -3,48 +3,49 @@ package cacheaccess import ( "context" "errors" - "fmt" "sync" "testing" "time" + + "github.com/strahe/synaps3/internal/model" ) type recordingAccessStore struct { mu sync.Mutex fail bool - accessCalls map[string][]time.Time - commitCalls map[string][]time.Time + accessCalls map[int64][]time.Time + commitCalls map[int64][]time.Time } func newRecordingAccessStore() *recordingAccessStore { return &recordingAccessStore{ - accessCalls: make(map[string][]time.Time), - commitCalls: make(map[string][]time.Time), + accessCalls: make(map[int64][]time.Time), + commitCalls: make(map[int64][]time.Time), } } -func (s *recordingAccessStore) RecordVersionCacheAccess( +func (s *recordingAccessStore) RecordContentCacheAccess( _ context.Context, - versionID string, + contentID int64, accessedAt time.Time, ) error { s.mu.Lock() defer s.mu.Unlock() - s.accessCalls[versionID] = append(s.accessCalls[versionID], accessedAt) + s.accessCalls[contentID] = append(s.accessCalls[contentID], accessedAt) if s.fail { return errors.New("database unavailable") } return nil } -func (s *recordingAccessStore) RecordVersionCacheCommit( +func (s *recordingAccessStore) RecordContentCacheCommit( _ context.Context, - versionID string, + contentID int64, accessedAt time.Time, ) error { s.mu.Lock() defer s.mu.Unlock() - s.commitCalls[versionID] = append(s.commitCalls[versionID], accessedAt) + s.commitCalls[contentID] = append(s.commitCalls[contentID], accessedAt) if s.fail { return errors.New("database unavailable") } @@ -57,10 +58,10 @@ func (s *recordingAccessStore) setFail(fail bool) { s.mu.Unlock() } -func (s *recordingAccessStore) callCounts(versionID string) (int, int) { +func (s *recordingAccessStore) callCounts(contentID int64) (int, int) { s.mu.Lock() defer s.mu.Unlock() - return len(s.accessCalls[versionID]), len(s.commitCalls[versionID]) + return len(s.accessCalls[contentID]), len(s.commitCalls[contentID]) } func TestTrackerCoalescesAccessWritesAndKeepsLatestTimestamp(t *testing.T) { @@ -69,26 +70,26 @@ func TestTrackerCoalescesAccessWritesAndKeepsLatestTimestamp(t *testing.T) { now := time.Date(2026, time.July, 28, 1, 0, 0, 0, time.UTC) tracker.now = func() time.Time { return now } - if err := tracker.RecordAccess(context.Background(), "version-1", nil); err != nil { + if err := tracker.RecordAccess(context.Background(), 1, nil); err != nil { t.Fatalf("first RecordAccess: %v", err) } now = now.Add(10 * time.Second) - if err := tracker.RecordAccess(context.Background(), "version-1", nil); err != nil { + if err := tracker.RecordAccess(context.Background(), 1, nil); err != nil { t.Fatalf("second RecordAccess: %v", err) } - accessCalls, commitCalls := store.callCounts("version-1") + accessCalls, commitCalls := store.callCounts(1) if accessCalls != 1 || commitCalls != 0 { t.Fatalf("store calls = access %d commit %d, want 1 and 0", accessCalls, commitCalls) } - if got := tracker.Latest("version-1"); !got.Equal(now) { + if got := tracker.Latest(1); !got.Equal(now) { t.Fatalf("Latest = %s, want %s", got, now) } now = now.Add(time.Minute) - if err := tracker.RecordAccess(context.Background(), "version-1", nil); err != nil { + if err := tracker.RecordAccess(context.Background(), 1, nil); err != nil { t.Fatalf("third RecordAccess: %v", err) } - accessCalls, _ = store.callCounts("version-1") + accessCalls, _ = store.callCounts(1) if accessCalls != 2 { t.Fatalf("access writes = %d, want 2", accessCalls) } @@ -102,7 +103,7 @@ func TestTrackerCommitFailureRetriesAsCommitDuringSweep(t *testing.T) { now := time.Date(2026, time.July, 28, 2, 0, 0, 0, time.UTC) tracker.now = func() time.Time { return now } - if err := tracker.RecordCommit(context.Background(), "version-commit", nil); err == nil { + if err := tracker.RecordCommit(context.Background(), 2, nil); err == nil { t.Fatal("RecordCommit error = nil, want persistence failure") } store.setFail(false) @@ -111,7 +112,7 @@ func TestTrackerCommitFailureRetriesAsCommitDuringSweep(t *testing.T) { if err != nil || failed != 0 { t.Fatalf("sweep: failed=%d err=%v", failed, err) } - accessCalls, commitCalls := store.callCounts("version-commit") + accessCalls, commitCalls := store.callCounts(2) if accessCalls != 0 || commitCalls != 2 { t.Fatalf("store calls = access %d commit %d, want 0 and 2", accessCalls, commitCalls) } @@ -122,7 +123,7 @@ func TestTrackerSweepDoesNotPersistForgottenCommitAfterDeletion(t *testing.T) { store.setFail(true) tracker := NewTracker(0, store) gate := NewGate() - if err := tracker.RecordCommit(context.Background(), "version-deleted", nil); err == nil { + if err := tracker.RecordCommit(context.Background(), 3, nil); err == nil { t.Fatal("RecordCommit error = nil, want persistence failure") } store.setFail(false) @@ -131,10 +132,10 @@ func TestTrackerSweepDoesNotPersistForgottenCommitAfterDeletion(t *testing.T) { releaseDelete := make(chan struct{}) deleteDone := make(chan struct{}) go func() { - gate.GuardDeletion("version-deleted", func() { + gate.GuardDeletion(model.ContentCacheKey(3), func() { close(deleteStarted) <-releaseDelete - tracker.Forget("version-deleted") + tracker.Forget(3) }) close(deleteDone) }() @@ -161,7 +162,7 @@ func TestTrackerSweepDoesNotPersistForgottenCommitAfterDeletion(t *testing.T) { case <-time.After(time.Second): t.Fatal("timed out waiting for sweep") } - accessCalls, commitCalls := store.callCounts("version-deleted") + accessCalls, commitCalls := store.callCounts(3) if accessCalls != 0 || commitCalls != 1 { t.Fatalf("store calls = access %d commit %d, want only the initial failed commit", accessCalls, commitCalls) } @@ -175,11 +176,11 @@ func TestTrackerStaysBoundedAndPausesLRUWhenDirtyAccessCannotPersist(t *testing. now := time.Date(2026, time.July, 28, 3, 0, 0, 0, time.UTC) tracker.now = func() time.Time { return now } - versionIDs := sameTrackerShardVersionIDs(tracker, 3) - if err := tracker.RecordAccess(context.Background(), versionIDs[0], nil); err == nil { + contentIDs := sameTrackerShardContentIDs(tracker, 3) + if err := tracker.RecordAccess(context.Background(), contentIDs[0], nil); err == nil { t.Fatal("first RecordAccess error = nil, want persistence failure") } - err := tracker.RecordAccess(context.Background(), versionIDs[1], nil) + err := tracker.RecordAccess(context.Background(), contentIDs[1], nil) if !errors.Is(err, ErrLRUAccessUncertain) { t.Fatalf("overflow error = %v, want ErrLRUAccessUncertain", err) } @@ -187,7 +188,7 @@ func TestTrackerStaysBoundedAndPausesLRUWhenDirtyAccessCannotPersist(t *testing. t.Fatal("SafeForLRU = true after an access could not be retained") } - shard := tracker.shard(versionIDs[0]) + shard := tracker.shard(contentIDs[0]) shard.mu.Lock() entryCount := len(shard.entries) shard.mu.Unlock() @@ -195,7 +196,7 @@ func TestTrackerStaysBoundedAndPausesLRUWhenDirtyAccessCannotPersist(t *testing. t.Fatalf("tracked entries = %d, want hard cap 1", entryCount) } - _ = tracker.RecordAccess(context.Background(), versionIDs[2], nil) + _ = tracker.RecordAccess(context.Background(), contentIDs[2], nil) shard.mu.Lock() entryCount = len(shard.entries) shard.mu.Unlock() @@ -210,20 +211,20 @@ func TestTrackerEvictsCleanEntryAtShardCapacity(t *testing.T) { tracker.maxEntriesPerShard = 1 now := time.Date(2026, time.July, 28, 4, 0, 0, 0, time.UTC) tracker.now = func() time.Time { return now } - versionIDs := sameTrackerShardVersionIDs(tracker, 2) + contentIDs := sameTrackerShardContentIDs(tracker, 2) - if err := tracker.RecordAccess(context.Background(), versionIDs[0], nil); err != nil { + if err := tracker.RecordAccess(context.Background(), contentIDs[0], nil); err != nil { t.Fatalf("first RecordAccess: %v", err) } now = now.Add(time.Second) - if err := tracker.RecordAccess(context.Background(), versionIDs[1], nil); err != nil { + if err := tracker.RecordAccess(context.Background(), contentIDs[1], nil); err != nil { t.Fatalf("second RecordAccess: %v", err) } - if got := tracker.Latest(versionIDs[0]); !got.IsZero() { + if got := tracker.Latest(contentIDs[0]); !got.IsZero() { t.Fatalf("evicted clean entry access = %s, want zero", got) } - if got := tracker.Latest(versionIDs[1]); !got.Equal(now) { + if got := tracker.Latest(contentIDs[1]); !got.Equal(now) { t.Fatalf("retained entry access = %s, want %s", got, now) } if !tracker.SafeForLRU() { @@ -234,11 +235,11 @@ func TestTrackerEvictsCleanEntryAtShardCapacity(t *testing.T) { func TestTrackerForgetRemovesVersionState(t *testing.T) { store := newRecordingAccessStore() tracker := NewTracker(time.Minute, store) - if err := tracker.RecordAccess(context.Background(), "version-forget", nil); err != nil { + if err := tracker.RecordAccess(context.Background(), 4, nil); err != nil { t.Fatalf("RecordAccess: %v", err) } - tracker.Forget("version-forget") - if got := tracker.Latest("version-forget"); !got.IsZero() { + tracker.Forget(4) + if got := tracker.Latest(4); !got.IsZero() { t.Fatalf("Latest after Forget = %s, want zero", got) } } @@ -248,24 +249,23 @@ func TestTrackerAdvancesPastEqualDurableTimestamp(t *testing.T) { tracker := NewTracker(time.Minute, store) durable := time.Date(2026, time.July, 28, 5, 0, 0, 123456000, time.UTC) tracker.now = func() time.Time { return durable } - if err := tracker.RecordAccess(context.Background(), "version-equal", &durable); err != nil { + if err := tracker.RecordAccess(context.Background(), 5, &durable); err != nil { t.Fatalf("RecordAccess: %v", err) } want := durable.Add(time.Microsecond) - if got := tracker.Latest("version-equal"); !got.Equal(want) { + if got := tracker.Latest(5); !got.Equal(want) { t.Fatalf("Latest = %s, want %s", got, want) } } -func sameTrackerShardVersionIDs(tracker *Tracker, count int) []string { - first := "version-shard" +func sameTrackerShardContentIDs(tracker *Tracker, count int) []int64 { + const first int64 = 1000 target := tracker.shard(first) - versionIDs := []string{first} - for index := 0; len(versionIDs) < count; index++ { - candidate := fmt.Sprintf("version-shard-%d", index) + contentIDs := []int64{first} + for candidate := first + 1; len(contentIDs) < count; candidate++ { if tracker.shard(candidate) == target { - versionIDs = append(versionIDs, candidate) + contentIDs = append(contentIDs, candidate) } } - return versionIDs + return contentIDs } diff --git a/internal/cacheeviction/task.go b/internal/cacheeviction/task.go index 2077313..1b3da58 100644 --- a/internal/cacheeviction/task.go +++ b/internal/cacheeviction/task.go @@ -1,160 +1,112 @@ package cacheeviction import ( + "encoding/json" "errors" "fmt" + "strconv" "time" "github.com/strahe/synaps3/internal/model" ) -const ( - StageLRU = "lru" - StageAfterUpload = "after_upload" - StageReconcileBucketDurability = "reconcile_bucket_durability" - - lruAccessedAtPayloadKey = "cache_accessed_at" - deleteAuthorizedPayloadKey = "delete_authorized" - lruTaskKeyPrefix = "evict_cache:lru:" - afterUploadTaskKeyPrefix = "evict_cache:" - bucketDurabilityTaskKeyPrefix = "evict_cache:bucket_durability:" -) +func EvictTaskKey(contentID, generation int64) string { + return EvictTaskKeyPrefix + strconv.FormatInt(contentID, 10) + ":" + strconv.FormatInt(generation, 10) +} -// ErrDurabilityThreshold means the current Bucket policy does not authorize deletion. -var ErrDurabilityThreshold = errors.New("minimum durable copies not met") +func DurabilityTaskKey(bucketID, generation int64) string { + return DurabilityTaskKeyPrefix + strconv.FormatInt(bucketID, 10) + ":" + strconv.FormatInt(generation, 10) +} -// ErrNoLongerEligible means a planned cache entry no longer matches the deletion contract. -var ErrNoLongerEligible = errors.New("cache entry is no longer eligible") +const ( + EvictTaskKeyPrefix = "cache-evict:" + DurabilityTaskKeyPrefix = "cache-durability:" +) -// ErrAccessChanged means an LRU candidate was accessed after it was planned. -var ErrAccessChanged = errors.New("cache access snapshot changed") +var ( + ErrDurabilityThreshold = errors.New("minimum durable copies not met") + ErrNoLongerEligible = errors.New("cache entry is no longer eligible") + ErrAccessChanged = errors.New("cache access snapshot changed") +) -// Candidate is the persisted snapshot needed to plan one LRU eviction. +// Candidate is one cached content payload, not one object version: residency +// is content-addressed, so several versions of identical bytes share a single +// eviction decision. type Candidate struct { - ObjectID int64 `bun:"object_id"` - VersionID string `bun:"version_id"` - Size int64 `bun:"size"` + ContentID int64 `bun:"content_id"` + BucketID int64 `bun:"bucket_id"` + Size int64 `bun:"content_size"` AccessedAt time.Time `bun:"cache_accessed_at"` } -// LRUTaskPayload is the typed boundary for an LRU task's persisted payload. -type LRUTaskPayload struct { - AccessedAt time.Time +// EvictInput identifies one cache generation. AccessedAt is present for an LRU +// authorization and omitted when remote durability directly authorizes removal. +type EvictInput struct { + ContentID int64 `json:"content_id"` + Generation int64 `json:"generation"` + AccessedAt *time.Time `json:"accessed_at,omitempty"` +} + +type DurabilityInput struct { + BucketID int64 `json:"bucket_id"` + Generation int64 `json:"generation"` } -// AuthorizedDeletion is the persisted decision needed to remove one cache -// entry outside the database transaction that approved it. type AuthorizedDeletion struct { - Version model.ObjectVersion + Content model.StorageContent BucketName string } -// NormalizeAccessTime matches the timestamp precision supported by both -// PostgreSQL and SQLite persistence paths. func NormalizeAccessTime(value time.Time) time.Time { return value.UTC().Truncate(time.Microsecond) } -// NewLRUTask builds the stable task for one candidate access snapshot. -func NewLRUTask(candidate Candidate, maxRetries int, scheduledAt time.Time) *model.Task { - stage := StageLRU - payload := LRUTaskPayload{ - AccessedAt: NormalizeAccessTime(candidate.AccessedAt), +func ParseEvictInput(task *model.Task) (EvictInput, error) { + if task == nil { + return EvictInput{}, errors.New("nil cache eviction task") } - return &model.Task{ - Type: model.TaskTypeEvictCache, - Stage: &stage, - RefType: "object", - RefID: candidate.ObjectID, - RefVersionID: candidate.VersionID, - IdempotencyKey: lruTaskKeyPrefix + candidate.VersionID, - Payload: payload.taskPayload(), - Status: model.TaskStatusQueued, - MaxRetries: maxRetries, - ScheduledAt: scheduledAt, + var input EvictInput + if err := json.Unmarshal(task.Input, &input); err != nil { + return EvictInput{}, fmt.Errorf("decoding cache eviction input: %w", err) } -} - -// NewAfterUploadTask builds the stable task for post-upload eviction. -func NewAfterUploadTask(objectID int64, versionID string, maxRetries int, scheduledAt time.Time) *model.Task { - stage := StageAfterUpload - return &model.Task{ - Type: model.TaskTypeEvictCache, - Stage: &stage, - RefType: "object", - RefID: objectID, - RefVersionID: versionID, - IdempotencyKey: afterUploadTaskKeyPrefix + versionID, - Status: model.TaskStatusQueued, - MaxRetries: maxRetries, - ScheduledAt: scheduledAt, + if input.ContentID < 1 || input.Generation < 1 { + return EvictInput{}, errors.New("cache eviction input is incomplete") } -} - -// NewBucketDurabilityTask builds the singleton reconciliation task for one bucket. -func NewBucketDurabilityTask(bucketID int64, maxRetries int, scheduledAt time.Time) *model.Task { - stage := StageReconcileBucketDurability - return &model.Task{ - Type: model.TaskTypeEvictCache, - Stage: &stage, - RefType: "bucket", - RefID: bucketID, - IdempotencyKey: fmt.Sprintf("%s%d", bucketDurabilityTaskKeyPrefix, bucketID), - Status: model.TaskStatusQueued, - MaxRetries: maxRetries, - ScheduledAt: scheduledAt, + if input.AccessedAt != nil { + normalized := NormalizeAccessTime(*input.AccessedAt) + input.AccessedAt = &normalized } + return input, nil } -// ParseLRUTaskPayload validates and decodes the persisted LRU access snapshot. -func ParseLRUTaskPayload(task *model.Task) (LRUTaskPayload, error) { +func ParseDurabilityInput(task *model.Task) (DurabilityInput, error) { if task == nil { - return LRUTaskPayload{}, errors.New("nil LRU eviction task") + return DurabilityInput{}, errors.New("nil durability reconciliation task") } - raw, ok := task.Payload[lruAccessedAtPayloadKey] - if !ok { - return LRUTaskPayload{}, errors.New("LRU eviction task is missing cache_accessed_at") + var input DurabilityInput + if err := json.Unmarshal(task.Input, &input); err != nil { + return DurabilityInput{}, fmt.Errorf("decoding durability input: %w", err) } - value, ok := raw.(string) - if !ok { - return LRUTaskPayload{}, fmt.Errorf("LRU eviction task cache_accessed_at has type %T, want string", raw) + if input.BucketID < 1 || input.Generation < 1 { + return DurabilityInput{}, errors.New("durability input is incomplete") } - accessedAt, err := time.Parse(time.RFC3339Nano, value) - if err != nil { - return LRUTaskPayload{}, fmt.Errorf("parsing LRU eviction task cache_accessed_at: %w", err) - } - return LRUTaskPayload{AccessedAt: NormalizeAccessTime(accessedAt)}, nil + return input, nil } -// DeleteAuthorized reports whether the task has crossed the durable deletion -// authorization boundary. -func DeleteAuthorized(task *model.Task) (bool, error) { - if task == nil || task.Payload == nil { - return false, nil - } - raw, ok := task.Payload[deleteAuthorizedPayloadKey] - if !ok { - return false, nil - } - authorized, ok := raw.(bool) - if !ok { - return false, fmt.Errorf("cache eviction task delete_authorized has type %T, want bool", raw) +func ValidateEvictInput(input *EvictInput) error { + if input == nil || input.ContentID < 1 || input.Generation < 1 { + return errors.New("content_id and generation are required") } - return authorized, nil -} - -// WithDeleteAuthorization copies payload before recording an authorization. -func WithDeleteAuthorization(payload map[string]any) map[string]any { - out := make(map[string]any, len(payload)+1) - for key, value := range payload { - out[key] = value + if input.AccessedAt != nil { + normalized := NormalizeAccessTime(*input.AccessedAt) + input.AccessedAt = &normalized } - out[deleteAuthorizedPayloadKey] = true - return out + return nil } -func (p LRUTaskPayload) taskPayload() map[string]any { - return map[string]any{ - lruAccessedAtPayloadKey: NormalizeAccessTime(p.AccessedAt).Format(time.RFC3339Nano), +func ValidateDurabilityInput(input *DurabilityInput) error { + if input == nil || input.BucketID < 1 || input.Generation < 1 { + return errors.New("bucket_id and generation are required") } + return nil } diff --git a/internal/cacheeviction/task_test.go b/internal/cacheeviction/task_test.go index 49f9406..e5bb484 100644 --- a/internal/cacheeviction/task_test.go +++ b/internal/cacheeviction/task_test.go @@ -1,6 +1,7 @@ package cacheeviction_test import ( + "encoding/json" "testing" "time" @@ -8,56 +9,47 @@ import ( "github.com/strahe/synaps3/internal/model" ) -func TestLRUTaskPayloadRoundTripUsesDatabasePrecision(t *testing.T) { +func TestEvictInputRoundTripNormalizesAccessSnapshot(t *testing.T) { accessedAt := time.Date(2026, time.July, 28, 8, 9, 10, 123456789, time.FixedZone("test", 8*60*60)) - task := cacheeviction.NewLRUTask(cacheeviction.Candidate{ - ObjectID: 12, - VersionID: "01J0000000000000000000LRU1", - Size: 42, - AccessedAt: accessedAt, - }, 3, time.Now()) - - payload, err := cacheeviction.ParseLRUTaskPayload(task) + raw, err := json.Marshal(cacheeviction.EvictInput{ + ContentID: 41, Generation: 3, AccessedAt: &accessedAt, + }) if err != nil { - t.Fatalf("ParseLRUTaskPayload: %v", err) + t.Fatalf("Marshal: %v", err) } - want := accessedAt.UTC().Truncate(time.Microsecond) - if !payload.AccessedAt.Equal(want) { - t.Fatalf("accessed at = %v, want %v", payload.AccessedAt, want) + + got, err := cacheeviction.ParseEvictInput(&model.Task{Input: raw}) + if err != nil { + t.Fatalf("ParseEvictInput: %v", err) } - if task.Stage == nil || *task.Stage != cacheeviction.StageLRU { - t.Fatalf("stage = %v, want %s", task.Stage, cacheeviction.StageLRU) + if want := accessedAt.UTC().Truncate(time.Microsecond); got.AccessedAt == nil || !got.AccessedAt.Equal(want) { + t.Fatalf("accessed at = %v, want %v", got.AccessedAt, want) } - if task.IdempotencyKey != "evict_cache:lru:"+task.RefVersionID { - t.Fatalf("idempotency key = %q", task.IdempotencyKey) + if got.ContentID != 41 || got.Generation != 3 { + t.Fatalf("input = %#v", got) } } -func TestParseLRUTaskPayloadRejectsMalformedSnapshot(t *testing.T) { - tests := []struct { - name string - payload map[string]any - }{ - {name: "missing", payload: nil}, - {name: "wrong type", payload: map[string]any{"cache_accessed_at": float64(1)}}, - {name: "invalid timestamp", payload: map[string]any{"cache_accessed_at": "not-a-time"}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, err := cacheeviction.ParseLRUTaskPayload(&model.Task{Payload: tt.payload}) - if err == nil { - t.Fatal("ParseLRUTaskPayload returned nil error") +func TestParseEvictInputRejectsIncompleteIdentity(t *testing.T) { + for name, task := range map[string]*model.Task{ + "nil task": nil, + "missing input": {}, + "missing content": {Input: json.RawMessage(`{"generation":1}`)}, + "zero generation": {Input: json.RawMessage(`{"content_id":41}`)}, + } { + t.Run(name, func(t *testing.T) { + if _, err := cacheeviction.ParseEvictInput(task); err == nil { + t.Fatal("ParseEvictInput succeeded") } }) } } -func TestAfterUploadTaskUsesIndependentStableKey(t *testing.T) { - task := cacheeviction.NewAfterUploadTask(12, "01J0000000000000000000POST", 5, time.Now()) - if task.Stage == nil || *task.Stage != cacheeviction.StageAfterUpload { - t.Fatalf("stage = %v, want %s", task.Stage, cacheeviction.StageAfterUpload) +func TestCacheTaskKeysIncludeGeneration(t *testing.T) { + if cacheeviction.EvictTaskKey(41, 1) == cacheeviction.EvictTaskKey(41, 2) { + t.Fatal("eviction generations share an idempotency key") } - if task.IdempotencyKey != "evict_cache:"+task.RefVersionID { - t.Fatalf("idempotency key = %q", task.IdempotencyKey) + if cacheeviction.DurabilityTaskKey(7, 1) == cacheeviction.DurabilityTaskKey(7, 2) { + t.Fatal("durability generations share an idempotency key") } } diff --git a/internal/config/config.go b/internal/config/config.go index bf6beb7..f0a9e1b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -86,16 +86,17 @@ type CacheConfig struct { } type WorkerConfig struct { - Upload WorkerPoolConfig `koanf:"upload"` - ProviderReplacement WorkerPoolConfig `koanf:"provider_replacement"` - Evictor WorkerPoolConfig `koanf:"evictor"` - StorageCleanup WorkerPoolConfig `koanf:"storage_cleanup"` + Tasks TaskWorkerConfig `koanf:"tasks"` } -type WorkerPoolConfig struct { - Concurrency int `koanf:"concurrency"` - PollInterval time.Duration `koanf:"poll_interval"` - MaxRetries int `koanf:"max_retries"` +type TaskWorkerConfig struct { + Concurrency int `koanf:"concurrency"` + PollInterval time.Duration `koanf:"poll_interval"` + LeaseDuration time.Duration `koanf:"lease_duration"` + MaxRetries int `koanf:"max_retries"` + Retention time.Duration `koanf:"retention"` + ProviderMutationConcurrency int `koanf:"provider_mutation_concurrency"` + DestructiveMutationConcurrency int `koanf:"destructive_mutation_concurrency"` } type LoggingConfig struct { @@ -187,25 +188,14 @@ func defaultConfig() *Config { LRULowWatermarkPercent: 80, }, Worker: WorkerConfig{ - Upload: WorkerPoolConfig{ - Concurrency: 4, - PollInterval: 5 * time.Second, - MaxRetries: 5, - }, - ProviderReplacement: WorkerPoolConfig{ - Concurrency: 4, - PollInterval: 5 * time.Second, - MaxRetries: 5, - }, - Evictor: WorkerPoolConfig{ - Concurrency: 2, - PollInterval: time.Minute, - MaxRetries: 3, - }, - StorageCleanup: WorkerPoolConfig{ - Concurrency: 2, - PollInterval: time.Minute, - MaxRetries: 5, + Tasks: TaskWorkerConfig{ + Concurrency: 12, + PollInterval: 5 * time.Second, + LeaseDuration: 5 * time.Minute, + MaxRetries: 5, + Retention: 7 * 24 * time.Hour, + ProviderMutationConcurrency: 4, + DestructiveMutationConcurrency: 2, }, }, Logging: LoggingConfig{ @@ -325,6 +315,16 @@ func loadWithOptions(path string, includeEnv, applyRuntimeDefaults bool) (*Confi if err := k.Unmarshal("", cfg); err != nil { return nil, PersistedFieldPresence{}, fmt.Errorf("unmarshalling config: %w", err) } + for _, legacy := range []string{ + "worker.upload", + "worker.provider_replacement", + "worker.evictor", + "worker.storage_cleanup", + } { + if k.Exists(legacy) { + return nil, PersistedFieldPresence{}, fmt.Errorf("configuration key %q is no longer supported; configure worker.tasks instead", legacy) + } + } cfg.Normalize() if applyRuntimeDefaults { if err := applyDefaultRuntimePaths(cfg, k.Exists("database.dsn"), k.Exists("cache.dir")); err != nil { @@ -491,22 +491,32 @@ func (c *Config) FieldValidationErrors() []FieldError { add("s3.region", "must be non-empty") } - // Worker pools. - validatePool := func(name string, p WorkerPoolConfig) { - if p.Concurrency < 1 { - add(fmt.Sprintf("worker.%s.concurrency", name), fmt.Sprintf("must be >= 1, got %d", p.Concurrency)) - } - if p.PollInterval <= 0 { - add(fmt.Sprintf("worker.%s.poll_interval", name), fmt.Sprintf("must be > 0, got %s", p.PollInterval)) - } - if p.MaxRetries < 0 { - add(fmt.Sprintf("worker.%s.max_retries", name), fmt.Sprintf("must be >= 0, got %d", p.MaxRetries)) - } + // Universal task engine. + tasks := c.Worker.Tasks + if tasks.Concurrency < 1 { + add("worker.tasks.concurrency", fmt.Sprintf("must be >= 1, got %d", tasks.Concurrency)) + } + if tasks.PollInterval <= 0 { + add("worker.tasks.poll_interval", fmt.Sprintf("must be > 0, got %s", tasks.PollInterval)) + } + if tasks.LeaseDuration <= 0 { + add("worker.tasks.lease_duration", fmt.Sprintf("must be > 0, got %s", tasks.LeaseDuration)) + } + if tasks.LeaseDuration <= tasks.PollInterval { + add("worker.tasks.lease_duration", "must be greater than worker.tasks.poll_interval") + } + if tasks.MaxRetries < 0 { + add("worker.tasks.max_retries", fmt.Sprintf("must be >= 0, got %d", tasks.MaxRetries)) + } + if tasks.Retention <= 0 { + add("worker.tasks.retention", fmt.Sprintf("must be > 0, got %s", tasks.Retention)) + } + if tasks.ProviderMutationConcurrency < 1 { + add("worker.tasks.provider_mutation_concurrency", fmt.Sprintf("must be >= 1, got %d", tasks.ProviderMutationConcurrency)) + } + if tasks.DestructiveMutationConcurrency < 1 { + add("worker.tasks.destructive_mutation_concurrency", fmt.Sprintf("must be >= 1, got %d", tasks.DestructiveMutationConcurrency)) } - validatePool("upload", c.Worker.Upload) - validatePool("provider_replacement", c.Worker.ProviderReplacement) - validatePool("evictor", c.Worker.Evictor) - validatePool("storage_cleanup", c.Worker.StorageCleanup) // Logging. switch c.Logging.Level { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ea83537..f1690d6 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -196,7 +196,7 @@ func TestValidate_InvalidNetwork(t *testing.T) { func TestValidate_WorkerConcurrency_Zero(t *testing.T) { cfg := validConfig() - cfg.Worker.Upload.Concurrency = 0 + cfg.Worker.Tasks.Concurrency = 0 err := cfg.Validate() if err == nil { @@ -209,7 +209,7 @@ func TestValidate_WorkerConcurrency_Zero(t *testing.T) { func TestValidate_WorkerPollInterval_Zero(t *testing.T) { cfg := validConfig() - cfg.Worker.Evictor.PollInterval = 0 + cfg.Worker.Tasks.PollInterval = 0 err := cfg.Validate() if err == nil { @@ -275,9 +275,9 @@ func TestValidate_EditableSettingsFields(t *testing.T) { }, { name: "worker max retries", - field: "worker.upload.max_retries", + field: "worker.tasks.max_retries", mutate: func(cfg *Config) { - cfg.Worker.Upload.MaxRetries = -1 + cfg.Worker.Tasks.MaxRetries = -1 }, }, { @@ -420,11 +420,11 @@ func TestLoad_DefaultConfig(t *testing.T) { if cfg.Cache.MaxSizeGB != def.Cache.MaxSizeGB { t.Errorf("Cache.MaxSizeGB = %d, want %d", cfg.Cache.MaxSizeGB, def.Cache.MaxSizeGB) } - if cfg.Worker.Upload.Concurrency != def.Worker.Upload.Concurrency { - t.Errorf("Worker.Upload.Concurrency = %d, want %d", cfg.Worker.Upload.Concurrency, def.Worker.Upload.Concurrency) + if cfg.Worker.Tasks.Concurrency != def.Worker.Tasks.Concurrency { + t.Errorf("Worker.Tasks.Concurrency = %d, want %d", cfg.Worker.Tasks.Concurrency, def.Worker.Tasks.Concurrency) } - if cfg.Worker.Upload.PollInterval != def.Worker.Upload.PollInterval { - t.Errorf("Worker.Upload.PollInterval = %s, want %s", cfg.Worker.Upload.PollInterval, def.Worker.Upload.PollInterval) + if cfg.Worker.Tasks.PollInterval != def.Worker.Tasks.PollInterval { + t.Errorf("Worker.Tasks.PollInterval = %s, want %s", cfg.Worker.Tasks.PollInterval, def.Worker.Tasks.PollInterval) } if cfg.Filecoin.DefaultCopies != 3 { t.Errorf("Filecoin.DefaultCopies = %d, want 3", cfg.Filecoin.DefaultCopies) @@ -509,13 +509,13 @@ func TestLoad_EnvOverrideUnderscoreFields(t *testing.T) { t.Setenv("SYNAPS3_CACHE_EVICTION_POLICY", "After_Upload") t.Setenv("SYNAPS3_CACHE_LRU_HIGH_WATERMARK_PERCENT", "88") t.Setenv("SYNAPS3_CACHE_LRU_LOW_WATERMARK_PERCENT", "73") - t.Setenv("SYNAPS3_WORKER_UPLOAD_POLL_INTERVAL", "9s") - t.Setenv("SYNAPS3_WORKER_UPLOAD_MAX_RETRIES", "8") - t.Setenv("SYNAPS3_WORKER_PROVIDER_REPLACEMENT_CONCURRENCY", "7") - t.Setenv("SYNAPS3_WORKER_PROVIDER_REPLACEMENT_POLL_INTERVAL", "11s") - t.Setenv("SYNAPS3_WORKER_PROVIDER_REPLACEMENT_MAX_RETRIES", "9") - t.Setenv("SYNAPS3_WORKER_EVICTOR_POLL_INTERVAL", "2m") - t.Setenv("SYNAPS3_WORKER_EVICTOR_MAX_RETRIES", "6") + t.Setenv("SYNAPS3_WORKER_TASKS_CONCURRENCY", "7") + t.Setenv("SYNAPS3_WORKER_TASKS_POLL_INTERVAL", "9s") + t.Setenv("SYNAPS3_WORKER_TASKS_LEASE_DURATION", "2m") + t.Setenv("SYNAPS3_WORKER_TASKS_MAX_RETRIES", "8") + t.Setenv("SYNAPS3_WORKER_TASKS_RETENTION", "96h") + t.Setenv("SYNAPS3_WORKER_TASKS_PROVIDER_MUTATION_CONCURRENCY", "3") + t.Setenv("SYNAPS3_WORKER_TASKS_DESTRUCTIVE_MUTATION_CONCURRENCY", "2") t.Setenv("SYNAPS3_LOGGING_S3_ACCESS_ENABLED", "false") t.Setenv("SYNAPS3_LOGGING_S3_ACCESS_LEVEL", "debug") t.Setenv("SYNAPS3_ADMIN_AUTH_PASSWORD_HASH", "$2a$10$7EqJtq98hPqEX7fNZaFWoOhi6r4aIvJrDWHtqK4V0GaQYe7TzTx6W") @@ -547,16 +547,14 @@ func TestLoad_EnvOverrideUnderscoreFields(t *testing.T) { cfg.Cache.LRULowWatermarkPercent != 73 { t.Fatalf("cache config = %#v, want env values", cfg.Cache) } - if cfg.Worker.Upload.PollInterval != 9*time.Second || cfg.Worker.Upload.MaxRetries != 8 { - t.Fatalf("upload worker = %#v, want env values", cfg.Worker.Upload) - } - if cfg.Worker.ProviderReplacement.Concurrency != 7 || - cfg.Worker.ProviderReplacement.PollInterval != 11*time.Second || - cfg.Worker.ProviderReplacement.MaxRetries != 9 { - t.Fatalf("provider replacement worker = %#v, want independent env values", cfg.Worker.ProviderReplacement) - } - if cfg.Worker.Evictor.PollInterval != 2*time.Minute || cfg.Worker.Evictor.MaxRetries != 6 { - t.Fatalf("evictor worker = %#v, want env values", cfg.Worker.Evictor) + if cfg.Worker.Tasks.Concurrency != 7 || + cfg.Worker.Tasks.PollInterval != 9*time.Second || + cfg.Worker.Tasks.LeaseDuration != 2*time.Minute || + cfg.Worker.Tasks.MaxRetries != 8 || + cfg.Worker.Tasks.Retention != 96*time.Hour || + cfg.Worker.Tasks.ProviderMutationConcurrency != 3 || + cfg.Worker.Tasks.DestructiveMutationConcurrency != 2 { + t.Fatalf("task worker = %#v, want env values", cfg.Worker.Tasks) } if cfg.Logging.S3Access.Enabled || cfg.Logging.S3Access.Level != "debug" { t.Fatalf("s3 access logging = %#v, want disabled debug", cfg.Logging.S3Access) @@ -853,7 +851,7 @@ func TestValidate_TLS_Disabled_NoCerts(t *testing.T) { func TestValidate_WorkerPollInterval_Negative(t *testing.T) { cfg := validConfig() - cfg.Worker.Evictor.PollInterval = -1 * time.Second + cfg.Worker.Tasks.PollInterval = -1 * time.Second err := cfg.Validate() if err == nil { diff --git a/internal/config/metadata.go b/internal/config/metadata.go index 16fa677..fdc9ddf 100644 --- a/internal/config/metadata.go +++ b/internal/config/metadata.go @@ -163,77 +163,33 @@ var fieldMetadataByPath = map[string]FieldMetadata{ Env: "SYNAPS3_CACHE_LRU_LOW_WATERMARK_PERCENT", Editable: true, }, - "worker.upload.concurrency": { - Label: "Upload Concurrency", - Description: "Number of upload worker jobs that may run concurrently.", - Env: "SYNAPS3_WORKER_UPLOAD_CONCURRENCY", - Editable: true, + "worker.tasks.concurrency": { + Label: "Task Concurrency", Description: "Maximum background operations that may run at once. Restart required.", + Env: "SYNAPS3_WORKER_TASKS_CONCURRENCY", Editable: true, }, - "worker.upload.poll_interval": { - Label: "Upload Poll Interval", - Description: "Interval between upload worker polling cycles.", - Env: "SYNAPS3_WORKER_UPLOAD_POLL_INTERVAL", - Editable: true, + "worker.tasks.poll_interval": { + Label: "Task Poll Interval", Description: "Interval between checks for ready background operations. Restart required.", + Env: "SYNAPS3_WORKER_TASKS_POLL_INTERVAL", Editable: true, }, - "worker.upload.max_retries": { - Label: "Upload Max Retries", - Description: "Maximum retry attempts for failed upload work.", - Env: "SYNAPS3_WORKER_UPLOAD_MAX_RETRIES", - Editable: true, + "worker.tasks.lease_duration": { + Label: "Task Lease Duration", Description: "Time another process waits before recovering interrupted background work. Restart required.", + Env: "SYNAPS3_WORKER_TASKS_LEASE_DURATION", Editable: true, }, - "worker.provider_replacement.concurrency": { - Label: "Provider Replacement Concurrency", - Description: "Number of provider replacement transfers that may run concurrently. Restart required.", - Env: "SYNAPS3_WORKER_PROVIDER_REPLACEMENT_CONCURRENCY", - Editable: true, + "worker.tasks.max_retries": { + Label: "Task Max Retries", Description: "Default retry limit for background operations. Restart required.", + Env: "SYNAPS3_WORKER_TASKS_MAX_RETRIES", Editable: true, }, - "worker.provider_replacement.poll_interval": { - Label: "Provider Replacement Poll Interval", - Description: "Interval between checks for provider replacement work. Restart required.", - Env: "SYNAPS3_WORKER_PROVIDER_REPLACEMENT_POLL_INTERVAL", - Editable: true, + "worker.tasks.retention": { + Label: "Task Retention", Description: "How long finished background operations remain visible. Restart required.", + Env: "SYNAPS3_WORKER_TASKS_RETENTION", Editable: true, }, - "worker.provider_replacement.max_retries": { - Label: "Provider Replacement Max Retries", - Description: "Maximum retry attempts for each provider replacement copy. Restart required.", - Env: "SYNAPS3_WORKER_PROVIDER_REPLACEMENT_MAX_RETRIES", - Editable: true, - }, - "worker.evictor.concurrency": { - Label: "Evictor Concurrency", - Description: "Number of cache eviction jobs that may run concurrently.", - Env: "SYNAPS3_WORKER_EVICTOR_CONCURRENCY", - Editable: true, + "worker.tasks.provider_mutation_concurrency": { + Label: "Storage Mutation Concurrency", Description: "Maximum concurrent requests that change remote storage. Restart required.", + Env: "SYNAPS3_WORKER_TASKS_PROVIDER_MUTATION_CONCURRENCY", Editable: true, }, - "worker.evictor.poll_interval": { - Label: "Evictor Poll Interval", - Description: "Interval between cache evictor polling cycles.", - Env: "SYNAPS3_WORKER_EVICTOR_POLL_INTERVAL", - Editable: true, - }, - "worker.evictor.max_retries": { - Label: "Evictor Max Retries", - Description: "Maximum retry attempts for failed eviction work.", - Env: "SYNAPS3_WORKER_EVICTOR_MAX_RETRIES", - Editable: true, - }, - "worker.storage_cleanup.concurrency": { - Label: "Replica Cleanup Concurrency", - Description: "Number of remote replica cleanup jobs that may run concurrently.", - Env: "SYNAPS3_WORKER_STORAGE_CLEANUP_CONCURRENCY", - Editable: true, - }, - "worker.storage_cleanup.poll_interval": { - Label: "Replica Cleanup Poll Interval", - Description: "Interval between remote replica cleanup polling cycles.", - Env: "SYNAPS3_WORKER_STORAGE_CLEANUP_POLL_INTERVAL", - Editable: true, - }, - "worker.storage_cleanup.max_retries": { - Label: "Replica Cleanup Max Retries", - Description: "Maximum retry attempts for failed remote replica cleanup work.", - Env: "SYNAPS3_WORKER_STORAGE_CLEANUP_MAX_RETRIES", - Editable: true, + "worker.tasks.destructive_mutation_concurrency": { + Label: "Removal Concurrency", Description: "Maximum concurrent remote cleanup and retirement requests. Restart required.", + Env: "SYNAPS3_WORKER_TASKS_DESTRUCTIVE_MUTATION_CONCURRENCY", Editable: true, }, "logging.level": { Label: "Level", diff --git a/internal/config/persistence.go b/internal/config/persistence.go index 404702c..3fb0445 100644 --- a/internal/config/persistence.go +++ b/internal/config/persistence.go @@ -285,35 +285,15 @@ func renderTOMLConfig(cfg *Config, presence PersistedFieldPresence, saveMode boo }, }, { - Name: "worker.upload", + Name: "worker.tasks", Fields: []initFieldDescriptor{ - {Field: "worker.upload.concurrency", Key: "concurrency", Value: strconv.Itoa(cfg.Worker.Upload.Concurrency), Enabled: saveMode}, - {Field: "worker.upload.poll_interval", Key: "poll_interval", Value: quoteTOMLString(cfg.Worker.Upload.PollInterval.String()), Enabled: saveMode}, - {Field: "worker.upload.max_retries", Key: "max_retries", Value: strconv.Itoa(cfg.Worker.Upload.MaxRetries), Enabled: saveMode}, - }, - }, - { - Name: "worker.provider_replacement", - Fields: []initFieldDescriptor{ - {Field: "worker.provider_replacement.concurrency", Key: "concurrency", Value: strconv.Itoa(cfg.Worker.ProviderReplacement.Concurrency), Enabled: saveMode}, - {Field: "worker.provider_replacement.poll_interval", Key: "poll_interval", Value: quoteTOMLString(cfg.Worker.ProviderReplacement.PollInterval.String()), Enabled: saveMode}, - {Field: "worker.provider_replacement.max_retries", Key: "max_retries", Value: strconv.Itoa(cfg.Worker.ProviderReplacement.MaxRetries), Enabled: saveMode}, - }, - }, - { - Name: "worker.evictor", - Fields: []initFieldDescriptor{ - {Field: "worker.evictor.concurrency", Key: "concurrency", Value: strconv.Itoa(cfg.Worker.Evictor.Concurrency), Enabled: saveMode}, - {Field: "worker.evictor.poll_interval", Key: "poll_interval", Value: quoteTOMLString(cfg.Worker.Evictor.PollInterval.String()), Enabled: saveMode}, - {Field: "worker.evictor.max_retries", Key: "max_retries", Value: strconv.Itoa(cfg.Worker.Evictor.MaxRetries), Enabled: saveMode}, - }, - }, - { - Name: "worker.storage_cleanup", - Fields: []initFieldDescriptor{ - {Field: "worker.storage_cleanup.concurrency", Key: "concurrency", Value: strconv.Itoa(cfg.Worker.StorageCleanup.Concurrency), Enabled: saveMode}, - {Field: "worker.storage_cleanup.poll_interval", Key: "poll_interval", Value: quoteTOMLString(cfg.Worker.StorageCleanup.PollInterval.String()), Enabled: saveMode}, - {Field: "worker.storage_cleanup.max_retries", Key: "max_retries", Value: strconv.Itoa(cfg.Worker.StorageCleanup.MaxRetries), Enabled: saveMode}, + {Field: "worker.tasks.concurrency", Key: "concurrency", Value: strconv.Itoa(cfg.Worker.Tasks.Concurrency), Enabled: saveMode}, + {Field: "worker.tasks.poll_interval", Key: "poll_interval", Value: quoteTOMLString(cfg.Worker.Tasks.PollInterval.String()), Enabled: saveMode}, + {Field: "worker.tasks.lease_duration", Key: "lease_duration", Value: quoteTOMLString(cfg.Worker.Tasks.LeaseDuration.String()), Enabled: saveMode}, + {Field: "worker.tasks.max_retries", Key: "max_retries", Value: strconv.Itoa(cfg.Worker.Tasks.MaxRetries), Enabled: saveMode}, + {Field: "worker.tasks.retention", Key: "retention", Value: quoteTOMLString(cfg.Worker.Tasks.Retention.String()), Enabled: saveMode}, + {Field: "worker.tasks.provider_mutation_concurrency", Key: "provider_mutation_concurrency", Value: strconv.Itoa(cfg.Worker.Tasks.ProviderMutationConcurrency), Enabled: saveMode}, + {Field: "worker.tasks.destructive_mutation_concurrency", Key: "destructive_mutation_concurrency", Value: strconv.Itoa(cfg.Worker.Tasks.DestructiveMutationConcurrency), Enabled: saveMode}, }, }, { diff --git a/internal/config/persistence_test.go b/internal/config/persistence_test.go index 444ba24..cf663a9 100644 --- a/internal/config/persistence_test.go +++ b/internal/config/persistence_test.go @@ -166,8 +166,8 @@ func TestInitAppDataDir_DefaultCreatesReferenceConfigAndRuntimeDirs(t *testing.T if loaded.Server.Port != defaults.Server.Port { t.Fatalf("Server.Port = %q, want default %q", loaded.Server.Port, defaults.Server.Port) } - if loaded.Worker.Upload.Concurrency != defaults.Worker.Upload.Concurrency { - t.Fatalf("Worker.Upload.Concurrency = %d, want default %d", loaded.Worker.Upload.Concurrency, defaults.Worker.Upload.Concurrency) + if loaded.Worker.Tasks.Concurrency != defaults.Worker.Tasks.Concurrency { + t.Fatalf("Worker.Tasks.Concurrency = %d, want default %d", loaded.Worker.Tasks.Concurrency, defaults.Worker.Tasks.Concurrency) } if loaded.Logging.Level != defaults.Logging.Level { t.Fatalf("Logging.Level = %q, want default %q", loaded.Logging.Level, defaults.Logging.Level) @@ -238,8 +238,7 @@ func TestInitAppDataDir_WritesCommentedReferenceConfig(t *testing.T) { "[server]", "# port = \":8080\"", "[server.tls]", - "[worker.upload]", - "[worker.provider_replacement]", + "[worker.tasks]", "[logging]", "[logging.s3_access]", "[admin.auth]", @@ -297,7 +296,7 @@ func TestInitAppDataDir_CommentedFieldsCanBeUncommentedInPlace(t *testing.T) { } text := strings.ReplaceAll(string(data), "# port = \":8080\"", "port = \":9191\"") text = strings.ReplaceAll(text, "# level = \"info\"", "level = \"debug\"") - text = strings.ReplaceAll(text, "# concurrency = 4", "concurrency = 6") + text = strings.ReplaceAll(text, "# concurrency = 12", "concurrency = 6") if err := os.WriteFile(result.ConfigPath, []byte(text), 0o600); err != nil { t.Fatalf("WriteFile: %v", err) } @@ -312,8 +311,8 @@ func TestInitAppDataDir_CommentedFieldsCanBeUncommentedInPlace(t *testing.T) { if loaded.Logging.Level != "debug" { t.Fatalf("Logging.Level = %q, want debug", loaded.Logging.Level) } - if loaded.Worker.Upload.Concurrency != 6 { - t.Fatalf("Worker.Upload.Concurrency = %d, want 6", loaded.Worker.Upload.Concurrency) + if loaded.Worker.Tasks.Concurrency != 6 { + t.Fatalf("Worker.Tasks.Concurrency = %d, want 6", loaded.Worker.Tasks.Concurrency) } } @@ -328,8 +327,8 @@ func TestSaveGeneratedTOML_RoundTripsWithCommentsAndUsesPrivatePermissions(t *te cfg.Cache.EvictionPolicy = "NONE" cfg.Cache.LRUHighWatermarkPercent = 87 cfg.Cache.LRULowWatermarkPercent = 72 - cfg.Worker.Upload.PollInterval = 7 * time.Second - cfg.Worker.Evictor.PollInterval = 2 * time.Minute + cfg.Worker.Tasks.PollInterval = 7 * time.Second + cfg.Worker.Tasks.LeaseDuration = 2 * time.Minute cfg.Logging.S3Access.Enabled = false cfg.Logging.S3Access.Level = "debug" @@ -363,15 +362,13 @@ func TestSaveGeneratedTOML_RoundTripsWithCommentsAndUsesPrivatePermissions(t *te "interval = \"5m0s\"", "timeout = \"5s\"", "concurrency = 8", - "[worker.upload]", + "[worker.tasks]", "poll_interval = \"7s\"", - "[worker.provider_replacement]", "[cache]", "eviction_policy = \"none\"", "lru_high_watermark_percent = 87", "lru_low_watermark_percent = 72", - "[worker.evictor]", - "poll_interval = \"2m0s\"", + "lease_duration = \"2m0s\"", "[logging.s3_access]", "enabled = false", "level = \"debug\"", @@ -391,11 +388,11 @@ func TestSaveGeneratedTOML_RoundTripsWithCommentsAndUsesPrivatePermissions(t *te if loaded.Server.Port != cfg.Server.Port { t.Fatalf("Server.Port = %q, want %q", loaded.Server.Port, cfg.Server.Port) } - if loaded.Worker.Upload.PollInterval != cfg.Worker.Upload.PollInterval { - t.Fatalf("Upload poll interval = %s, want %s", loaded.Worker.Upload.PollInterval, cfg.Worker.Upload.PollInterval) + if loaded.Worker.Tasks.PollInterval != cfg.Worker.Tasks.PollInterval { + t.Fatalf("Task poll interval = %s, want %s", loaded.Worker.Tasks.PollInterval, cfg.Worker.Tasks.PollInterval) } - if loaded.Worker.Evictor.PollInterval != cfg.Worker.Evictor.PollInterval { - t.Fatalf("Evictor poll interval = %s, want %s", loaded.Worker.Evictor.PollInterval, cfg.Worker.Evictor.PollInterval) + if loaded.Worker.Tasks.LeaseDuration != cfg.Worker.Tasks.LeaseDuration { + t.Fatalf("Task lease duration = %s, want %s", loaded.Worker.Tasks.LeaseDuration, cfg.Worker.Tasks.LeaseDuration) } if loaded.Cache.EvictionPolicy != "none" || loaded.Cache.LRUHighWatermarkPercent != 87 || diff --git a/internal/db/db.go b/internal/db/db.go index aaa302c..7be5bcd 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -25,6 +25,8 @@ import ( const migrationUnlockTimeout = 5 * time.Second +const sqlitePreflightTimeout = 5 * time.Second + // New creates a Bun database connection based on the provided configuration. func New(cfg config.DatabaseConfig) (*bun.DB, error) { var ( @@ -42,6 +44,9 @@ func New(cfg config.DatabaseConfig) (*bun.DB, error) { db = bun.NewDB(sqldb, pgdialect.New()) case "sqlite": + if err := validateExistingSQLiteTarget(cfg.DSN); err != nil { + return nil, err + } if err := ensureSQLiteDir(cfg.DSN); err != nil { return nil, err } @@ -61,8 +66,60 @@ func New(cfg config.DatabaseConfig) (*bun.DB, error) { return db, nil } +// validateExistingSQLiteTarget runs before the normal connection can apply a +// persistent journal-mode pragma. An incompatible database is therefore +// rejected without changing the preserved file. +func validateExistingSQLiteTarget(dsn string) (retErr error) { + path, ok, err := sqliteFilePath(dsn) + if err != nil { + return fmt.Errorf("resolving sqlite database path: %w", err) + } + if !ok { + return nil + } + info, err := os.Stat(path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("inspecting sqlite database: %w", err) + } + if info.IsDir() { + return fmt.Errorf("sqlite database path is a directory: %s", path) + } + absolutePath, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("resolving sqlite database absolute path: %w", err) + } + readOnlyURL := url.URL{Scheme: "file", Path: filepath.ToSlash(absolutePath)} + query := readOnlyURL.Query() + query.Set("mode", "ro") + query.Add("_pragma", "busy_timeout(5000)") + readOnlyURL.RawQuery = query.Encode() + + sqldb, err := sql.Open("sqlite", readOnlyURL.String()) + if err != nil { + return fmt.Errorf("opening sqlite database for compatibility check: %w", err) + } + readOnlyDB := bun.NewDB(sqldb, sqlitedialect.New()) + defer func() { + if err := readOnlyDB.Close(); err != nil { + retErr = errors.Join(retErr, fmt.Errorf("closing sqlite compatibility check: %w", err)) + } + }() + ctx, cancel := context.WithTimeout(context.Background(), sqlitePreflightTimeout) + defer cancel() + if err := migrations.ValidateTarget(ctx, readOnlyDB); err != nil { + return fmt.Errorf("validating existing sqlite database: %w", err) + } + return nil +} + // RunMigrations initialises the Bun migrator and applies all pending migrations. func RunMigrations(ctx context.Context, db *bun.DB) (retErr error) { + if err := migrations.ValidateTarget(ctx, db); err != nil { + return err + } migrator := migrations.NewMigrator(db) if err := migrator.Init(ctx); err != nil { @@ -91,11 +148,30 @@ func RunMigrations(ctx context.Context, db *bun.DB) (retErr error) { slog.Info("no new migrations to apply") } + // Statistics are an operational concern, not part of the frozen DDL, and the + // initial migration returns early on an already-migrated database. Refreshing + // them here means a partial index such as idx_tasks_pending is costed against + // what the table actually holds rather than against a default guess. + if err := analyzeSchema(ctx, db); err != nil { + slog.Warn("refreshing planner statistics failed (non-fatal)", "error", err) + } + + return nil +} + +// analyzeSchema refreshes query planner statistics for the whole database. +func analyzeSchema(ctx context.Context, db *bun.DB) error { + if _, err := db.ExecContext(ctx, "ANALYZE"); err != nil { + return fmt.Errorf("analyzing schema: %w", err) + } return nil } // ForceUnlockMigrations releases a migration lock left by a killed run. func ForceUnlockMigrations(ctx context.Context, db *bun.DB) error { + if err := migrations.ValidateTarget(ctx, db); err != nil { + return err + } migrator := migrations.NewMigrator(db) if err := migrator.Init(ctx); err != nil { return fmt.Errorf("initializing migrator: %w", err) diff --git a/internal/db/db_test.go b/internal/db/db_test.go index 6ee04ad..cee1d7e 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -1,8 +1,10 @@ package db import ( + "bytes" "context" "database/sql" + "errors" "fmt" "os" "path/filepath" @@ -77,6 +79,92 @@ func TestForceUnlockMigrationsClearsALockLeftByAKilledRun(t *testing.T) { } } +func TestForceUnlockMigrationsRejectsLegacyDatabaseWithoutModification(t *testing.T) { + cfg := config.DatabaseConfig{ + Driver: "sqlite", DSN: "file:" + filepath.Join(t.TempDir(), "legacy-force-unlock.db"), + MaxOpenConns: 1, MaxIdleConns: 1, + } + database, err := New(cfg) + if err != nil { + t.Fatalf("New() error = %v", err) + } + t.Cleanup(func() { _ = database.Close() }) + if _, err := database.ExecContext(t.Context(), `CREATE TABLE legacy_tasks (id INTEGER PRIMARY KEY, status TEXT NOT NULL)`); err != nil { + t.Fatalf("create legacy table: %v", err) + } + if _, err := database.ExecContext(t.Context(), `INSERT INTO legacy_tasks (id, status) VALUES (1, 'running')`); err != nil { + t.Fatalf("seed legacy table: %v", err) + } + + err = ForceUnlockMigrations(t.Context(), database) + if !errors.Is(err, migrations.ErrIncompatibleDatabase) { + t.Fatalf("ForceUnlockMigrations() error = %v, want incompatible database", err) + } + var status string + if err := database.NewRaw(`SELECT status FROM legacy_tasks WHERE id = 1`).Scan(t.Context(), &status); err != nil { + t.Fatalf("read legacy row: %v", err) + } + if status != "running" { + t.Fatalf("legacy row status = %q, want unchanged", status) + } + var migrationTables int + if err := database.NewRaw(`SELECT COUNT(*) FROM sqlite_schema WHERE type = 'table' AND name LIKE 'bun_migration%'`).Scan(t.Context(), &migrationTables); err != nil { + t.Fatalf("count migration tables: %v", err) + } + if migrationTables != 0 { + t.Fatalf("force unlock created %d migration metadata tables in legacy database", migrationTables) + } +} + +func TestNewRejectsLegacySQLiteBeforeApplyingPersistentPragmas(t *testing.T) { + databasePath := filepath.Join(t.TempDir(), "preserved-legacy.db") + legacy, err := sql.Open("sqlite", databasePath) + if err != nil { + t.Fatalf("open legacy sqlite database: %v", err) + } + if _, err := legacy.Exec(`CREATE TABLE legacy_tasks (id INTEGER PRIMARY KEY, status TEXT NOT NULL)`); err != nil { + _ = legacy.Close() + t.Fatalf("create legacy task table: %v", err) + } + if _, err := legacy.Exec(`INSERT INTO legacy_tasks (id, status) VALUES (1, 'running')`); err != nil { + _ = legacy.Close() + t.Fatalf("seed legacy task: %v", err) + } + if err := legacy.Close(); err != nil { + t.Fatalf("close legacy sqlite database: %v", err) + } + before, err := os.ReadFile(databasePath) + if err != nil { + t.Fatalf("read legacy sqlite database before preflight: %v", err) + } + + database, err := New(config.DatabaseConfig{ + Driver: "sqlite", + DSN: "file:" + filepath.ToSlash(databasePath) + "?_pragma=journal_mode(WAL)", + MaxOpenConns: 2, + MaxIdleConns: 2, + }) + if database != nil { + _ = database.Close() + t.Fatal("New returned a connection for an incompatible database") + } + if !errors.Is(err, migrations.ErrIncompatibleDatabase) { + t.Fatalf("New error = %v, want incompatible database", err) + } + after, err := os.ReadFile(databasePath) + if err != nil { + t.Fatalf("read legacy sqlite database after preflight: %v", err) + } + if !bytes.Equal(after, before) { + t.Fatal("legacy sqlite database changed during compatibility preflight") + } + for _, suffix := range []string{"-wal", "-shm"} { + if _, statErr := os.Stat(databasePath + suffix); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("legacy sqlite sidecar %s exists after rejection: %v", suffix, statErr) + } + } +} + func TestRunMigrationsSerializesConcurrentRunnersAndUnlocksAfterCancellation(t *testing.T) { cfg := config.DatabaseConfig{ Driver: "sqlite", @@ -143,20 +231,15 @@ func TestNew_SQLiteConcurrentClaimsDoNotBusy(t *testing.T) { } repos := repository.NewRepositories(db) - for i := 0; i < 100; i++ { + for i := range 100 { versionID := fmt.Sprintf("01J00000000000000000%06d", i+1) task := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "object", - RefID: int64(i + 1), - RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("upload:%s", versionID), - Status: model.TaskStatusQueued, - MaxRetries: 3, - ScheduledAt: time.Now(), + Type: model.TaskTypeUploadPlan, IdempotencyKey: fmt.Sprintf("upload-plan:%s", versionID), + InputVersion: 1, Input: []byte(fmt.Sprintf(`{"version_id":%q}`, versionID)), InputHash: versionID, + Status: model.TaskStatusPending, ResumeMode: model.TaskResumeModeExecute, AvailableAt: time.Now(), } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create() error = %v", err) + if _, created, err := repos.Tasks.Enqueue(ctx, task); err != nil || !created { + t.Fatalf("Enqueue() created=%t error=%v", created, err) } } @@ -164,17 +247,15 @@ func TestNew_SQLiteConcurrentClaimsDoNotBusy(t *testing.T) { var claimedCount atomic.Int64 var wg sync.WaitGroup for range 8 { - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { for { - task, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) + task, err := repos.Tasks.ClaimNext(ctx, time.Minute) if err != nil { if strings.Contains(err.Error(), "SQLITE_BUSY") || strings.Contains(err.Error(), "database is locked") { busyCount.Add(1) return } - t.Errorf("ClaimReady() unexpected error = %v", err) + t.Errorf("ClaimNext() unexpected error = %v", err) return } if task == nil { @@ -182,7 +263,7 @@ func TestNew_SQLiteConcurrentClaimsDoNotBusy(t *testing.T) { } claimedCount.Add(1) } - }() + }) } wg.Wait() @@ -198,42 +279,55 @@ func TestRunMigrations_ObjectVersionSchema(t *testing.T) { db := newMigratedSQLiteDB(t, "schema.db") objectColumns := sqliteColumns(t, db, "objects") - for _, column := range []string{"current_version_id", "size", "e_tag", "checksum", "cache_key", "in_cache", "in_filecoin", "state"} { + for _, column := range []string{"size", "e_tag", "checksum", "cache_key", "in_cache", "in_filecoin", "state"} { if objectColumns[column] { t.Fatalf("objects.%s should not exist", column) } } - for _, column := range []string{"bucket_id", "key"} { + // "Current" is one pointer on the object, not a flag repeated per version. + for _, column := range []string{"bucket_id", "key", "current_version_id"} { if !objectColumns[column] { t.Fatalf("objects.%s should exist", column) } } versionColumns := sqliteColumns(t, db, "object_versions") - for _, column := range []string{"is_current", "in_cache", "storage_upload_id", "multipart_upload_id", "cache_accessed_at"} { + for _, column := range []string{"content_id", "multipart_upload_id"} { if !versionColumns[column] { t.Fatalf("object_versions.%s should exist", column) } } - for _, column := range []string{"piece_cid", "retrieval_url", "in_filecoin"} { + // Bytes, durability and cache residency belong to the content and its cache + // entry; a version that carried them would be a second authority for facts + // those rows already own. + for _, column := range []string{ + "piece_cid", "retrieval_url", "in_filecoin", "checksum", "state", + "in_cache", "cache_accessed_at", "cache_key", "is_current", + } { if versionColumns[column] { t.Fatalf("object_versions.%s should not exist", column) } } + cacheColumns := sqliteColumns(t, db, "object_cache") + for _, column := range []string{"content_id", "in_cache", "cache_accessed_at", "cache_active_task_id"} { + if !cacheColumns[column] { + t.Fatalf("object_cache.%s should exist", column) + } + } bucketColumns := sqliteColumns(t, db, "buckets") if bucketColumns["proof_set_id"] { t.Fatal("buckets.proof_set_id should not exist") } - for _, table := range []string{"storage_uploads", "storage_data_sets", "storage_upload_copies", "storage_upload_failures"} { + for _, table := range []string{"storage_contents", "storage_data_sets", "storage_copies"} { if columns := sqliteColumns(t, db, table); len(columns) == 0 { t.Fatalf("%s table should exist", table) } } - copyColumns := sqliteColumns(t, db, "storage_upload_copies") - if !copyColumns["is_new_data_set"] { - t.Fatal("storage_upload_copies.is_new_data_set should exist") + copyColumns := sqliteColumns(t, db, "storage_copies") + if copyColumns["is_new_data_set"] { + t.Fatal("storage_copies.is_new_data_set should not exist") } partColumns := sqliteColumns(t, db, "multipart_parts") if !partColumns["checksum"] { @@ -245,14 +339,12 @@ func TestRunMigrations_ObjectVersionSchema(t *testing.T) { t.Fatal("idx_objects_bucket_key should exist") } versionIndexes := sqliteIndexes(t, db, "object_versions") - if !versionIndexes["idx_object_versions_current_unique"] { - t.Fatal("idx_object_versions_current_unique should exist") + objectIndexes := sqliteIndexes(t, db, "objects") + if !objectIndexes["idx_objects_current_version"] { + t.Fatal("idx_objects_current_version should exist") } - if !versionIndexes["idx_object_versions_current_bucket_key"] { - t.Fatal("idx_object_versions_current_bucket_key should exist") - } - if !versionIndexes["idx_object_versions_storage_upload"] { - t.Fatal("idx_object_versions_storage_upload should exist") + if !versionIndexes["idx_object_versions_content"] { + t.Fatal("idx_object_versions_content should exist") } if !versionIndexes["idx_object_versions_multipart_upload"] { t.Fatal("idx_object_versions_multipart_upload should exist") @@ -260,125 +352,132 @@ func TestRunMigrations_ObjectVersionSchema(t *testing.T) { if !versionIndexes["idx_object_versions_object_created"] { t.Fatal("idx_object_versions_object_created should exist") } - if !versionIndexes["idx_object_versions_cache_lru"] { - t.Fatal("idx_object_versions_cache_lru should exist") + cacheIndexes := sqliteIndexes(t, db, "object_cache") + if !cacheIndexes["idx_object_cache_lru"] { + t.Fatal("idx_object_cache_lru should exist") } taskIndexes := sqliteIndexes(t, db, "tasks") - if !taskIndexes["idx_tasks_type_ready_scheduled"] { - t.Fatal("idx_tasks_type_ready_scheduled should exist") + for _, name := range []string{ + "idx_tasks_pending", + "idx_tasks_recovery", + "idx_tasks_gc", + "idx_tasks_type_status_id", + "idx_tasks_subject", + } { + if !taskIndexes[name] { + t.Fatalf("%s should exist", name) + } } - uploadCopyIndexes := sqliteIndexes(t, db, "storage_upload_copies") - if !uploadCopyIndexes["idx_storage_upload_copies_status_data_set_upload"] { - t.Fatal("idx_storage_upload_copies_status_data_set_upload should exist") + uploadCopyIndexes := sqliteIndexes(t, db, "storage_copies") + if !uploadCopyIndexes["idx_storage_copies_status_data_set_content"] { + t.Fatal("idx_storage_copies_status_data_set_content should exist") } } func TestRunMigrations_ObjectVersionCurrentAndForeignKeyConstraints(t *testing.T) { db := newMigratedSQLiteDB(t, "object-version-constraints.db") - ctx := context.Background() - - mustExec(t, db, `INSERT INTO buckets (id, name) VALUES (1, 'bucket-a')`) - mustExec(t, db, `INSERT INTO buckets (id, name) VALUES (2, 'bucket-b')`) - mustExec(t, db, `INSERT INTO objects (id, bucket_id, key) VALUES (1, 1, 'file.txt')`) - mustExec(t, db, `INSERT INTO object_versions (version_id, object_id, bucket_id, key, size, e_tag, checksum, cache_key, is_current) VALUES ('v1', 1, 1, 'file.txt', 1, 'etag-1', 'sum-1', '.versions/v1', TRUE)`) - if _, err := db.ExecContext(ctx, `INSERT INTO object_versions (version_id, object_id, bucket_id, key, size, e_tag, checksum, cache_key, is_current) VALUES ('v2', 1, 1, 'file.txt', 2, 'etag-2', 'sum-2', '.versions/v2', TRUE)`); err == nil { - t.Fatal("expected second current version for one object to fail") - } - mustExec(t, db, `INSERT INTO object_versions (version_id, object_id, bucket_id, key, size, e_tag, checksum, cache_key, is_current) VALUES ('v3', 1, 1, 'file.txt', 3, 'etag-3', 'sum-3', '.versions/v3', FALSE)`) - - if _, err := db.ExecContext(ctx, `INSERT INTO object_versions (version_id, object_id, bucket_id, key, size, e_tag, checksum, cache_key) VALUES ('wrong-bucket', 1, 2, 'file.txt', 1, 'etag-x', 'sum-x', '.versions/wrong-bucket')`); err == nil { - t.Fatal("expected object_versions object/bucket/key mismatch to fail") - } - if _, err := db.ExecContext(ctx, `INSERT INTO object_versions (version_id, object_id, bucket_id, key, size, e_tag, checksum, cache_key, state) VALUES ('stored-without-upload', 1, 1, 'file.txt', 1, 'etag-x', 'sum-x', '.versions/stored-without-upload', 'stored')`); err == nil { - t.Fatal("expected stored object version without storage_upload_id to fail") - } - if _, err := db.ExecContext(ctx, `INSERT INTO object_versions (version_id, object_id, bucket_id, key, size, e_tag, checksum, content_type, cache_key, is_delete_marker, in_cache, state) VALUES ('bad-marker', 1, 1, 'file.txt', 0, 'etag-marker', '', '', '', TRUE, FALSE, 'cached')`); err == nil { - t.Fatal("expected malformed delete marker insert to fail") - } - if _, err := db.ExecContext(ctx, `UPDATE object_versions SET is_delete_marker = TRUE WHERE version_id = 'v3'`); err == nil { - t.Fatal("expected malformed delete marker update to fail") - } - if _, err := db.ExecContext(ctx, `INSERT INTO object_versions (version_id, object_id, bucket_id, key, size, e_tag, checksum, cache_key) VALUES ('empty-cache-key', 1, 1, 'file.txt', 1, 'etag-x', 'sum-x', '')`); err == nil { - t.Fatal("expected data version without cache_key to fail") - } - mustExec(t, db, `INSERT INTO multipart_uploads (bucket_id, key, upload_id, status) VALUES (1, 'file.txt', 'upload-valid', 'completed')`) - mustExec(t, db, `INSERT INTO object_versions (version_id, object_id, bucket_id, key, size, e_tag, checksum, cache_key, multipart_upload_id) VALUES ('multipart-valid', 1, 1, 'file.txt', 1, 'etag-mp', 'sum-mp', '.versions/multipart-valid', 'upload-valid')`) - if _, err := db.ExecContext(ctx, `INSERT INTO object_versions (version_id, object_id, bucket_id, key, size, e_tag, checksum, cache_key, multipart_upload_id) VALUES ('multipart-missing', 1, 1, 'file.txt', 1, 'etag-missing', 'sum-missing', '.versions/multipart-missing', 'upload-missing')`); err == nil { - t.Fatal("expected object_versions multipart_upload_id without upload to fail") - } - mustExec(t, db, `INSERT INTO object_versions (version_id, object_id, bucket_id, key, size, e_tag, checksum, content_type, cache_key, is_delete_marker, in_cache, state) VALUES ('marker-ok', 1, 1, 'file.txt', 0, '', '', '', '', TRUE, FALSE, 'cached')`) - - mustExec(t, db, `INSERT INTO storage_uploads (id, bucket_id, source_version_id, content_size, checksum, status, piece_cid, requested_copies) VALUES (1, 1, 'v-upload', 1, 'sum-1', 'complete', 'bafk2bzacefake', 1)`) - if _, err := db.ExecContext(ctx, `INSERT INTO object_versions (version_id, object_id, bucket_id, key, size, e_tag, checksum, cache_key, state, storage_upload_id) VALUES ('cached-with-upload', 1, 1, 'file.txt', 1, 'etag-x', 'sum-x', '.versions/cached-with-upload', 'cached', 1)`); err == nil { - t.Fatal("expected cached object version with storage_upload_id to fail") - } + mustExec(t, db, `INSERT INTO buckets (id, name, default_copies, minimum_durable_copies, created_at, updated_at) VALUES (1, 'bucket-a', 8, 8, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO bucket_replica_slots (bucket_id, copy_index, created_at, updated_at) SELECT 1, value, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP FROM (SELECT 0 AS value UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7)`) + mustExec(t, db, `INSERT INTO buckets (id, name, default_copies, minimum_durable_copies, created_at, updated_at) VALUES (2, 'bucket-b', 8, 8, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO bucket_replica_slots (bucket_id, copy_index, created_at, updated_at) SELECT 2, value, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP FROM (SELECT 0 AS value UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7)`) + mustExec(t, db, `INSERT INTO objects (id, bucket_id, key, created_at, updated_at) VALUES (1, 1, 'file.txt', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + // A data version points at the content holding its bytes, and the composite + // foreign key welds the size it repeats to that content's size. + mustExec(t, db, `INSERT INTO storage_contents (id, bucket_id, content_size, checksum, requested_copies, created_at, updated_at) VALUES (1, 1, 1, printf('%064x', 1), 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO storage_contents (id, bucket_id, content_size, checksum, requested_copies, created_at, updated_at) VALUES (2, 1, 2, printf('%064x', 2), 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO storage_contents (id, bucket_id, content_size, checksum, requested_copies, created_at, updated_at) VALUES (3, 1, 3, printf('%064x', 3), 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO storage_contents (id, bucket_id, content_size, checksum, requested_copies, created_at, updated_at) VALUES (4, 2, 1, printf('%064x', 4), 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO object_versions (version_id, object_id, bucket_id, key, content_id, size, e_tag, created_at, updated_at) VALUES ('v1', 1, 1, 'file.txt', 1, 1, 'etag-1', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO object_versions (version_id, object_id, bucket_id, key, content_id, size, e_tag, created_at, updated_at) VALUES ('v3', 1, 1, 'file.txt', 3, 3, 'etag-3', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + // One column holds one value, so "at most one current version" needs no + // partial unique index; pointing at a version of another object is refused + // by the pointer's composite foreign key. + mustExec(t, db, `UPDATE objects SET current_version_id = 'v1' WHERE id = 1`) + mustReject(t, db, "expected a pointer to a missing version to fail", `UPDATE objects SET current_version_id = 'v-missing' WHERE id = 1`) + mustReject(t, db, "expected deleting the pointed-at version to fail", `DELETE FROM object_versions WHERE version_id = 'v1'`) + mustExec(t, db, `INSERT INTO objects (id, bucket_id, key, created_at, updated_at) VALUES (2, 1, 'other.txt', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustReject(t, db, "expected pointing at another object's version to fail", `UPDATE objects SET current_version_id = 'v1' WHERE id = 2`) + mustExec(t, db, `UPDATE objects SET current_version_id = 'v3' WHERE id = 1`) + mustExec(t, db, `DELETE FROM object_versions WHERE version_id = 'v1'`) + mustExec(t, db, `INSERT INTO object_versions (version_id, object_id, bucket_id, key, content_id, size, e_tag, created_at, updated_at) VALUES ('v1', 1, 1, 'file.txt', 1, 1, 'etag-1', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `UPDATE objects SET current_version_id = 'v1' WHERE id = 1`) + + mustReject(t, db, "expected object_versions object/bucket/key mismatch to fail", `INSERT INTO object_versions (version_id, object_id, bucket_id, key, content_id, size, e_tag, created_at, updated_at) VALUES ('wrong-bucket', 1, 2, 'file.txt', 4, 1, 'etag-x', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustReject(t, db, "expected data object version without content_id to fail", `INSERT INTO object_versions (version_id, object_id, bucket_id, key, size, e_tag, created_at, updated_at) VALUES ('data-without-content', 1, 1, 'file.txt', 1, 'etag-x', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustReject(t, db, "expected version whose size disagrees with its content to fail", `INSERT INTO object_versions (version_id, object_id, bucket_id, key, content_id, size, e_tag, created_at, updated_at) VALUES ('size-drift', 1, 1, 'file.txt', 1, 99, 'etag-x', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustReject(t, db, "expected version pointing at another bucket's content to fail", `INSERT INTO object_versions (version_id, object_id, bucket_id, key, content_id, size, e_tag, created_at, updated_at) VALUES ('cross-bucket-content', 1, 1, 'file.txt', 4, 1, 'etag-x', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustReject(t, db, "expected malformed delete marker insert to fail", `INSERT INTO object_versions (version_id, object_id, bucket_id, key, content_id, size, e_tag, content_type, is_delete_marker, created_at, updated_at) VALUES ('bad-marker', 1, 1, 'file.txt', 1, 1, 'etag-marker', '', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustReject(t, db, "expected malformed delete marker update to fail", `UPDATE object_versions SET is_delete_marker = TRUE WHERE version_id = 'v3'`) + mustExec(t, db, `INSERT INTO multipart_uploads (bucket_id, key, upload_id, status, created_at, updated_at) VALUES (1, 'file.txt', 'upload-valid', 'completed', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO storage_contents (id, bucket_id, content_size, checksum, requested_copies, created_at, updated_at) VALUES (5, 1, 1, printf('%064x', 5), 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO object_versions (version_id, object_id, bucket_id, key, content_id, size, e_tag, multipart_upload_id, created_at, updated_at) VALUES ('multipart-valid', 1, 1, 'file.txt', 5, 1, 'etag-mp', 'upload-valid', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustReject(t, db, "expected object_versions multipart_upload_id without upload to fail", `INSERT INTO object_versions (version_id, object_id, bucket_id, key, content_id, size, e_tag, multipart_upload_id, created_at, updated_at) VALUES ('multipart-missing', 1, 1, 'file.txt', 5, 1, 'etag-missing', 'upload-missing', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO object_versions (version_id, object_id, bucket_id, key, size, e_tag, content_type, is_delete_marker, created_at, updated_at) VALUES ('marker-ok', 1, 1, 'file.txt', 0, '', '', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + + // Residency belongs to the content, so a cache entry cannot name bytes that + // do not exist and cannot be recorded twice for one content. + mustExec(t, db, `INSERT INTO object_cache (content_id, in_cache, created_at, updated_at) VALUES (1, TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustReject(t, db, "expected a second cache entry for one content to fail", `INSERT INTO object_cache (content_id, in_cache, created_at, updated_at) VALUES (1, FALSE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustReject(t, db, "expected a cache entry for missing content to fail", `INSERT INTO object_cache (content_id, in_cache, created_at, updated_at) VALUES (9999, TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) } func TestRunMigrations_StorageProvenanceConstraints(t *testing.T) { db := newMigratedSQLiteDB(t, "storage-provenance-constraints.db") - ctx := context.Background() - - mustExec(t, db, `INSERT INTO buckets (id, name) VALUES (1, 'bucket-a')`) - mustExec(t, db, `INSERT INTO buckets (id, name) VALUES (2, 'bucket-b')`) - mustExec(t, db, `INSERT INTO storage_uploads (id, bucket_id, source_version_id, content_size, checksum, status, piece_cid, requested_copies) VALUES (1, 1, 'v1', 10, 'sum-1', 'complete', 'bafk2bzacefake', 2)`) - mustExec(t, db, `INSERT INTO storage_uploads (id, bucket_id, source_version_id, content_size, checksum, status, requested_copies) VALUES (2, 2, 'v2', 10, 'sum-2', 'running', 3)`) - mustExec(t, db, `INSERT INTO storage_data_sets (id, bucket_id, provider_id, copy_index, generation, is_current, data_set_id, status, created_by_upload_id, last_used_upload_id) VALUES (1, 1, '101', 0, 1, TRUE, '1001', 'ready', 1, 1)`) - if _, err := db.ExecContext(ctx, `INSERT INTO storage_data_sets (bucket_id, provider_id, copy_index, generation, is_current, data_set_id, status, created_by_upload_id, last_used_upload_id) VALUES (2, '101', 0, 1, TRUE, '1001', 'ready', 2, 2)`); err == nil { - t.Fatal("expected provider/data_set reuse across buckets to fail") - } - if _, err := db.ExecContext(ctx, `INSERT INTO storage_data_sets (bucket_id, provider_id, copy_index, generation, is_current, status, created_by_upload_id, last_used_upload_id) VALUES (1, '202', 0, 1, TRUE, 'pending', 1, 1)`); err == nil { - t.Fatal("expected duplicate bucket/copy_index binding to fail") - } - mustExec(t, db, `INSERT INTO storage_data_sets (id, bucket_id, provider_id, copy_index, generation, is_current, status, created_by_upload_id, last_used_upload_id) VALUES (2, 1, '202', 1, 1, TRUE, 'pending', 1, 1)`) - mustExec(t, db, `INSERT INTO storage_upload_copies (upload_id, copy_index, provider_id, piece_id, transfer_method, status, retrieval_url, storage_data_set_id) VALUES (1, 0, '101', '2001', 'ingress', 'committed', 'https://provider.example/piece', 1)`) - if _, err := db.ExecContext(ctx, `INSERT INTO storage_upload_copies (upload_id, copy_index, provider_id, transfer_method, storage_data_set_id) VALUES (1, 0, '101', 'peer_pull', 1)`); err == nil { - t.Fatal("expected duplicate copy for one data set to fail") - } + mustExec(t, db, `INSERT INTO buckets (id, name, default_copies, minimum_durable_copies, created_at, updated_at) VALUES (1, 'bucket-a', 8, 8, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO bucket_replica_slots (bucket_id, copy_index, created_at, updated_at) SELECT 1, value, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP FROM (SELECT 0 AS value UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7)`) + mustExec(t, db, `INSERT INTO buckets (id, name, default_copies, minimum_durable_copies, created_at, updated_at) VALUES (2, 'bucket-b', 8, 8, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO bucket_replica_slots (bucket_id, copy_index, created_at, updated_at) SELECT 2, value, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP FROM (SELECT 0 AS value UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7)`) + mustExec(t, db, `INSERT INTO storage_contents (id, bucket_id, content_size, checksum, piece_cid, requested_copies, created_at, updated_at) VALUES (1, 1, 10, printf('%064x', 1), 'bafk2bzacefake', 2, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO storage_contents (id, bucket_id, content_size, checksum, requested_copies, created_at, updated_at) VALUES (2, 2, 10, printf('%064x', 2), 3, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO storage_data_sets (id, bucket_id, provider_id, copy_index, generation, is_current, data_set_id, status, created_by_content_id, last_used_content_id, created_at, updated_at) VALUES (1, 1, '101', 0, 1, TRUE, '1001', 'ready', 1, 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustReject(t, db, "expected provider/data_set reuse across buckets to fail", `INSERT INTO storage_data_sets (bucket_id, provider_id, copy_index, generation, is_current, data_set_id, status, created_by_content_id, last_used_content_id, created_at, updated_at) VALUES (2, '101', 0, 1, TRUE, '1001', 'ready', 2, 2, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustReject(t, db, "expected duplicate bucket/copy_index binding to fail", `INSERT INTO storage_data_sets (bucket_id, provider_id, copy_index, generation, is_current, status, created_by_content_id, last_used_content_id, created_at, updated_at) VALUES (1, '202', 0, 1, TRUE, 'pending', 1, 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + + mustExec(t, db, `INSERT INTO storage_data_sets (id, bucket_id, provider_id, copy_index, generation, is_current, status, created_by_content_id, last_used_content_id, created_at, updated_at) VALUES (2, 1, '202', 1, 1, TRUE, 'pending', 1, 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + // Committed is a projection of a confirmed ledger row, so the evidence has + // to exist before the copy can claim it. + mustReject(t, db, "expected a committed copy without confirmed evidence to fail", `INSERT INTO storage_copies (content_id, bucket_id, content_size, copy_index, provider_id, piece_id, transfer_method, status, retrieval_url, storage_data_set_id, created_at, updated_at) VALUES (1, 1, 10, 0, '101', '2001', 'ingress', 'committed', 'https://provider.example/piece', 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO storage_copies (content_id, bucket_id, content_size, copy_index, provider_id, piece_id, transfer_method, status, retrieval_url, storage_data_set_id, created_at, updated_at) VALUES (1, 1, 10, 0, '101', '2001', 'ingress', 'piece_ready', 'https://provider.example/piece', 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO storage_commit_attempts (attempt_id, content_id, storage_data_set_id, status, extra_data_hex, transaction_id, confirmed_transaction_id, attempted_at, resolved_at, created_at, updated_at) VALUES ('attempt-1', 1, 1, 'confirmed', 'abcd', 'tx-1', 'tx-1', current_timestamp, current_timestamp, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustReject(t, db, "expected a copy to refuse projecting an unconfirmed status", `UPDATE storage_copies SET status = 'committed', confirmed_attempt_id = 'attempt-1', confirmed_attempt_status = 'attempted' WHERE content_id = 1 AND copy_index = 0`) + mustReject(t, db, "expected a copy to refuse projecting a missing attempt", `UPDATE storage_copies SET status = 'committed', confirmed_attempt_id = 'attempt-missing', confirmed_attempt_status = 'confirmed' WHERE content_id = 1 AND copy_index = 0`) + mustExec(t, db, `UPDATE storage_copies SET status = 'committed', confirmed_attempt_id = 'attempt-1', confirmed_attempt_status = 'confirmed' WHERE content_id = 1 AND copy_index = 0`) + mustReject(t, db, "expected a referenced attempt to refuse leaving confirmed", `UPDATE storage_commit_attempts SET status = 'released' WHERE attempt_id = 'attempt-1'`) + mustReject(t, db, "expected a referenced attempt to refuse deletion", `DELETE FROM storage_commit_attempts WHERE attempt_id = 'attempt-1'`) + mustReject(t, db, "expected duplicate copy for one data set to fail", `INSERT INTO storage_copies (content_id, bucket_id, content_size, copy_index, provider_id, transfer_method, storage_data_set_id, created_at, updated_at) VALUES (1, 1, 10, 0, '101', 'peer_pull', 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) // A replica slot holds both generations while a provider replacement migrates. - mustExec(t, db, `INSERT INTO storage_data_sets (id, bucket_id, provider_id, copy_index, generation, is_current, status, created_by_upload_id, last_used_upload_id) VALUES (3, 1, '303', 0, 2, FALSE, 'pending', 1, 1)`) - mustExec(t, db, `INSERT INTO storage_upload_copies (upload_id, copy_index, provider_id, transfer_method, storage_data_set_id) VALUES (1, 0, '303', 'peer_pull', 3)`) - // Unbound copies still cannot duplicate a slot, which distinct NULL data set - // ids would otherwise allow. - mustExec(t, db, `INSERT INTO storage_upload_copies (upload_id, copy_index, transfer_method) VALUES (1, 3, 'peer_pull')`) - if _, err := db.ExecContext(ctx, `INSERT INTO storage_upload_copies (upload_id, copy_index, transfer_method) VALUES (1, 3, 'peer_pull')`); err == nil { - t.Fatal("expected duplicate unbound copy for one slot to fail") - } - mustExec(t, db, `INSERT INTO storage_upload_copies (upload_id, copy_index, provider_id, transfer_method, storage_data_set_id) VALUES (1, 1, '202', 'peer_pull', 2)`) - if _, err := db.ExecContext(ctx, `UPDATE storage_upload_copies SET status = 'committed' WHERE upload_id = 1 AND copy_index = 1`); err == nil { - t.Fatal("expected committed copy without piece identity to fail") - } - mustExec(t, db, `INSERT INTO storage_upload_copies (upload_id, copy_index, transfer_method) VALUES (1, 2, 'peer_pull')`) - if _, err := db.ExecContext(ctx, `UPDATE storage_upload_copies SET status = 'committed', piece_id = '2002', retrieval_url = 'https://provider.example/missing-binding' WHERE upload_id = 1 AND copy_index = 2`); err == nil { - t.Fatal("expected committed copy without provider and data set binding to fail") - } - mustExec(t, db, `UPDATE storage_upload_copies SET status = 'committed', piece_id = '0', retrieval_url = 'https://provider.example/zero-piece' WHERE upload_id = 1 AND copy_index = 1`) - mustExec(t, db, `INSERT INTO storage_upload_failures (upload_id, attempt_index, provider_id, transfer_method, stage, error_message, explicit) VALUES (1, 0, '202', 'peer_pull', 'pull', 'pull failed', FALSE)`) - if _, err := db.ExecContext(ctx, `INSERT INTO storage_upload_failures (upload_id, attempt_index, transfer_method) VALUES (1, 0, 'peer_pull')`); err == nil { - t.Fatal("expected duplicate failure attempt_index for upload to fail") - } - if _, err := db.ExecContext(ctx, `INSERT INTO storage_upload_failures (upload_id, attempt_index, transfer_method) VALUES (1, 1, 'legacy')`); err == nil { - t.Fatal("expected invalid failure transfer_method to fail") - } + mustExec(t, db, `INSERT INTO storage_data_sets (id, bucket_id, provider_id, copy_index, generation, is_current, status, created_by_content_id, last_used_content_id, created_at, updated_at) VALUES (3, 1, '303', 0, 2, FALSE, 'pending', 1, 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO storage_copies (content_id, bucket_id, content_size, copy_index, provider_id, transfer_method, storage_data_set_id, created_at, updated_at) VALUES (1, 1, 10, 0, '303', 'peer_pull', 3, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + // A copy is always born bound to exactly one data set; the NOT NULL on + // storage_data_set_id is what enforces that, so it is the assertion here. + mustRejectRequiredColumn(t, db, "expected copy without a data set and provider binding to fail", `INSERT INTO storage_copies (content_id, bucket_id, content_size, copy_index, transfer_method, created_at, updated_at) VALUES (1, 1, 10, 3, 'peer_pull', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO storage_copies (content_id, bucket_id, content_size, copy_index, provider_id, transfer_method, storage_data_set_id, created_at, updated_at) VALUES (1, 1, 10, 1, '202', 'peer_pull', 2, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustReject(t, db, "expected committed copy without piece identity to fail", `UPDATE storage_copies SET status = 'committed' WHERE content_id = 1 AND copy_index = 1`) + mustExec(t, db, `INSERT INTO storage_commit_attempts (attempt_id, content_id, storage_data_set_id, status, extra_data_hex, transaction_id, confirmed_transaction_id, attempted_at, resolved_at, created_at, updated_at) VALUES ('attempt-2', 1, 2, 'confirmed', 'abcd', 'tx-2', 'tx-2', current_timestamp, current_timestamp, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `UPDATE storage_copies SET status = 'committed', piece_id = '0', retrieval_url = 'https://provider.example/zero-piece', confirmed_attempt_id = 'attempt-2', confirmed_attempt_status = 'confirmed' WHERE content_id = 1 AND copy_index = 1`) + mustExec(t, db, `INSERT INTO storage_cleanup_copies (content_id, bucket_id, copy_index, provider_id, storage_data_set_id, piece_id, piece_cid, created_at, updated_at) VALUES (1, 1, 0, '101', 1, '2001', 'bafk2bzacefake', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustReject(t, db, "expected duplicate physical cleanup identity to fail", `INSERT INTO storage_cleanup_copies (content_id, bucket_id, copy_index, provider_id, storage_data_set_id, piece_id, piece_cid, created_at, updated_at) VALUES (1, 1, 0, '101', 1, '2001', 'bafk2bzaceduplicate', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + // Cleanup evidence requires both the data set and the piece it removed; + // each NOT NULL is the invariant under test, so supply every other column. + mustRejectRequiredColumn(t, db, "expected cleanup evidence without storage_data_set_id to fail", `INSERT INTO storage_cleanup_copies (content_id, bucket_id, copy_index, provider_id, piece_id, piece_cid, created_at, updated_at) VALUES (1, 1, 0, '101', '2002', 'bafk2bzacemissingdataset', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustRejectRequiredColumn(t, db, "expected cleanup evidence without piece_id to fail", `INSERT INTO storage_cleanup_copies (content_id, bucket_id, copy_index, provider_id, storage_data_set_id, piece_cid, created_at, updated_at) VALUES (1, 1, 0, '101', 1, 'bafk2bzacemissingpiece', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) } func TestRunMigrations_TaskAndMultipartConstraints(t *testing.T) { db := newMigratedSQLiteDB(t, "task-multipart-constraints.db") ctx := context.Background() - mustExec(t, db, `INSERT INTO buckets (id, name) VALUES (1, 'bucket-a')`) + mustExec(t, db, `INSERT INTO buckets (id, name, default_copies, minimum_durable_copies, created_at, updated_at) VALUES (1, 'bucket-a', 8, 8, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO bucket_replica_slots (bucket_id, copy_index, created_at, updated_at) SELECT 1, value, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP FROM (SELECT 0 AS value UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7)`) - if _, err := db.ExecContext(ctx, `INSERT INTO tasks (type, ref_type, ref_id, ref_version_id, idempotency_key) VALUES ('upload', 'object', 1, '', 'upload:missing-version')`); err == nil { - t.Fatal("expected object task without ref_version_id to fail") - } - mustExec(t, db, `INSERT INTO tasks (type, ref_type, ref_id, ref_version_id, idempotency_key) VALUES ('upload', 'bucket', 1, '', 'bucket:allowed')`) + mustReject(t, db, "expected task with incomplete subject identity to fail", `INSERT INTO tasks (type, idempotency_key, input_version, input_hash, subject_type, available_at, created_at, updated_at) VALUES ('custom', 'invalid-subject', 1, 'hash', 'object_version', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO tasks (type, idempotency_key, input_version, input_hash, subject_type, subject_key, available_at, created_at, updated_at) VALUES ('future_extension', 'open-type', 1, 'hash', 'bucket', '1', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - mustExec(t, db, `INSERT INTO multipart_uploads (bucket_id, key, upload_id) VALUES (1, 'large.bin', 'upload-1')`) - mustExec(t, db, `INSERT INTO multipart_parts (upload_id, part_number, size, e_tag) VALUES ('upload-1', 1, 10, 'part-etag')`) - if _, err := db.ExecContext(ctx, `INSERT INTO multipart_parts (upload_id, part_number, size, e_tag) VALUES ('upload-1', 10001, 10, 'bad-part')`); err == nil { - t.Fatal("expected multipart part_number > 10000 to fail") - } + mustExec(t, db, `INSERT INTO multipart_uploads (bucket_id, key, upload_id, created_at, updated_at) VALUES (1, 'large.bin', 'upload-1', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO multipart_parts (upload_id, part_number, size, e_tag, created_at) VALUES ('upload-1', 1, 10, 'part-etag', CURRENT_TIMESTAMP)`) + mustReject(t, db, "expected multipart part_number > 10000 to fail", `INSERT INTO multipart_parts (upload_id, part_number, size, e_tag, created_at) VALUES ('upload-1', 10001, 10, 'bad-part', CURRENT_TIMESTAMP)`) mustExec(t, db, `DELETE FROM multipart_uploads WHERE upload_id = 'upload-1'`) var partCount int @@ -644,15 +743,79 @@ func newMigratedSQLiteDB(t *testing.T, filename string) *bun.DB { return db } -func mustExec(t *testing.T, db *bun.DB, query string, args ...interface{}) { +func mustExec(t *testing.T, db *bun.DB, query string, args ...any) { t.Helper() if _, err := db.ExecContext(context.Background(), query, args...); err != nil { t.Fatalf("exec %q: %v", query, err) } } +// mustReject asserts a statement is refused by the constraint it targets. A +// statement that omits a required column is refused by the null constraint +// before that constraint is ever evaluated, so the case would keep passing +// while proving nothing. +func mustReject(t *testing.T, db *bun.DB, message, query string, args ...any) { + t.Helper() + _, err := db.ExecContext(context.Background(), query, args...) + if err == nil { + t.Fatal(message) + } + if rejectedByNullConstraint(err) { + t.Fatalf("%s: rejected by a null constraint instead of the constraint under test\nquery: %s\nerror: %v", message, query, err) + } + if rejectedByMissingSchema(err) { + t.Fatalf("%s: never reached the constraint under test because the schema has no such table or column\nquery: %s\nerror: %v", message, query, err) + } +} + +// rejectedByMissingSchema reports a statement that never reached the constraint +// under test because it names a table or column the schema does not have. Such a +// statement fails, so a negative assertion keeps passing while proving nothing — +// exactly how three stale cases survived a column being removed. +func rejectedByMissingSchema(err error) bool { + for _, missing := range []string{ + "no such table", // SQLite + "no such column", // SQLite + "has no column named", // SQLite, INSERT column list + "does not exist", // PostgreSQL, relation/column + "undefined_table", // PostgreSQL, SQLSTATE name + "undefined_column", // PostgreSQL, SQLSTATE name + } { + if strings.Contains(err.Error(), missing) { + return true + } + } + return false +} + +// mustRejectRequiredColumn asserts the opposite of mustReject: the omitted +// column is itself the invariant under test, so a null-constraint rejection is +// the expected outcome rather than a false pass. +func mustRejectRequiredColumn(t *testing.T, db *bun.DB, message, query string, args ...any) { + t.Helper() + _, err := db.ExecContext(context.Background(), query, args...) + if err == nil { + t.Fatal(message) + } + if !rejectedByNullConstraint(err) { + t.Fatalf("%s: not rejected by a null constraint\nquery: %s\nerror: %v", message, query, err) + } +} + +func rejectedByNullConstraint(err error) bool { + for _, nullConstraint := range []string{ + "NOT NULL constraint failed", // SQLite + "null value in column", // PostgreSQL + } { + if strings.Contains(err.Error(), nullConstraint) { + return true + } + } + return false +} + type sqliteQueryer interface { - QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryContext(context.Context, string, ...any) (*sql.Rows, error) } func sqliteColumns(t *testing.T, db sqliteQueryer, table string) map[string]bool { diff --git a/internal/db/migrations/2026040501_init.go b/internal/db/migrations/2026040501_init.go deleted file mode 100644 index 4c9c68e..0000000 --- a/internal/db/migrations/2026040501_init.go +++ /dev/null @@ -1,830 +0,0 @@ -package migrations - -import ( - "context" - "fmt" - "time" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" -) - -func init() { - Migrations.MustRegister( - transactionalMigration(up2026040501Init), - transactionalMigration(down2026040501Init), - ) -} - -type s3Account2026040501 struct { - bun.BaseModel `bun:"table:s3_accounts"` - - AccessKey string `bun:",pk"` - SecretKey string `bun:",notnull"` - Role string `bun:",notnull"` - IsRoot bool `bun:",notnull,default:false"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` -} - -type bucket2026040501 struct { - bun.BaseModel `bun:"table:buckets"` - - ID int64 `bun:",pk,autoincrement"` - Name string `bun:",unique,notnull"` - ACL []byte `bun:",nullzero"` - OwnerAccessKey *string `bun:",nullzero"` - Status string `bun:",notnull,default:'active'"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` -} - -type object2026040501 struct { - bun.BaseModel `bun:"table:objects"` - - ID int64 `bun:",pk,autoincrement"` - BucketID int64 `bun:",notnull"` - Key string `bun:",notnull"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` -} - -type objectVersion2026040501 struct { - bun.BaseModel `bun:"table:object_versions"` - - VersionID string `bun:",pk"` - ObjectID int64 `bun:",notnull"` - BucketID int64 `bun:",notnull"` - Key string `bun:",notnull"` - Size int64 `bun:",notnull"` - ETag string `bun:",notnull"` - Checksum string `bun:",notnull"` - ContentType string `bun:",notnull,default:'application/octet-stream'"` - Metadata map[string]string `bun:"type:jsonb"` - CacheKey string `bun:",notnull"` - StorageUploadID *int64 `bun:",nullzero"` - PieceCID *string `bun:",scanonly"` - RetrievalURL *string `bun:",scanonly"` - InCache bool `bun:",notnull,default:true"` - InFilecoin bool `bun:",scanonly"` - IsCurrent bool `bun:",notnull,default:false"` - IsDeleteMarker bool `bun:",notnull,default:false"` - State string `bun:",notnull,default:'cached'"` - FailedAtState *string `bun:",nullzero"` - LastError *string `bun:",nullzero"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` -} - -type objectDeletion2026040501 struct { - bun.BaseModel `bun:"table:object_deletions"` - - ID int64 `bun:",pk,autoincrement"` - BucketID int64 `bun:",notnull"` - ObjectID int64 `bun:",notnull"` - Key string `bun:",notnull"` - VersionID string `bun:",unique,notnull"` - CacheKey string `bun:",notnull"` - StorageUploadID *int64 `bun:",nullzero"` - Size int64 `bun:",notnull"` - Checksum string `bun:",notnull"` - CacheCleanupStatus string `bun:",notnull,default:'pending'"` - CacheError *string `bun:",nullzero"` - CacheCleanedAt *time.Time `bun:",nullzero"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - DeletedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` -} - -type task2026040501 struct { - bun.BaseModel `bun:"table:tasks"` - - ID int64 `bun:",pk,autoincrement"` - Type string `bun:",notnull"` - Stage *string `bun:",nullzero"` - RefType string `bun:",notnull"` - RefID int64 `bun:",notnull"` - RefVersionID string `bun:",notnull"` - IdempotencyKey string `bun:",unique,notnull"` - Payload map[string]any `bun:"type:jsonb"` - Status string `bun:",notnull,default:'queued'"` - RetryCount int `bun:",notnull,default:0"` - MaxRetries int `bun:",notnull,default:5"` - LastError *string `bun:",nullzero"` - StatusMessage *string `bun:",nullzero"` - WaitReason *string `bun:",nullzero"` - ScheduledAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - ClaimedAt *time.Time `bun:",nullzero"` - LeaseUntil *time.Time `bun:",nullzero"` - StartedAt *time.Time `bun:",nullzero"` - CompletedAt *time.Time `bun:",nullzero"` -} - -type storageCleanupCopy2026040501 struct { - bun.BaseModel `bun:"table:storage_cleanup_copies"` - - ID int64 `bun:",pk,autoincrement"` - TaskID int64 `bun:",notnull"` - UploadID int64 `bun:",notnull"` - CopyIndex int `bun:",notnull"` - ProviderID *string `bun:"type:text"` - StorageDataSetID *int64 `bun:",nullzero"` - DataSetID *string `bun:"type:text"` - ClientDataSetID *string `bun:"type:text"` - PieceID *string `bun:"type:text"` - PieceCID string `bun:",notnull"` - RetrievalURL *string `bun:",nullzero"` - Status string `bun:",notnull,default:'pending'"` - DeleteTxHash *string `bun:",nullzero"` - LastError *string `bun:",nullzero"` - ScheduledAt *time.Time `bun:",nullzero"` - RemovedAt *time.Time `bun:",nullzero"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` -} - -type multipartUpload2026040501 struct { - bun.BaseModel `bun:"table:multipart_uploads"` - - ID int64 `bun:",pk,autoincrement"` - BucketID int64 `bun:",notnull"` - Key string `bun:",notnull"` - UploadID string `bun:",notnull,unique"` - ContentType string `bun:",notnull,default:'application/octet-stream'"` - Metadata map[string]string `bun:"type:jsonb"` - Status string `bun:",notnull,default:'initiated'"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` -} - -type multipartPart2026040501 struct { - bun.BaseModel `bun:"table:multipart_parts"` - - ID int64 `bun:",pk,autoincrement"` - UploadID string `bun:",notnull"` - PartNumber int `bun:",notnull"` - Size int64 `bun:",notnull"` - ETag string `bun:",notnull"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` -} - -type walletOperation2026040501 struct { - bun.BaseModel `bun:"table:wallet_operations"` - - ID int64 `bun:",pk,autoincrement"` - Type string `bun:",notnull"` - ClientRequestID string `bun:",notnull"` - Amount string `bun:",notnull"` - Status string `bun:",notnull,default:'pending'"` - TxHash *string `bun:",nullzero"` - LastError *string `bun:",nullzero"` - LeaseUntil *time.Time `bun:",nullzero"` - StartedAt *time.Time `bun:",nullzero"` - SubmittedAt *time.Time `bun:",nullzero"` - CompletedAt *time.Time `bun:",nullzero"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` -} - -type storageUpload2026040501 struct { - bun.BaseModel `bun:"table:storage_uploads"` - - ID int64 `bun:",pk,autoincrement"` - BucketID int64 `bun:",notnull"` - SourceTaskID *int64 `bun:",nullzero"` - SourceVersionID string `bun:",nullzero"` - ContentSize int64 `bun:",notnull"` - Checksum string `bun:",notnull"` - Status string `bun:",notnull,default:'running'"` - PieceCID *string `bun:",nullzero"` - RequestedCopies int `bun:",notnull"` - IngressBytesTransferred int64 `bun:",notnull,default:0"` - IngressStoreAttempt int `bun:",notnull,default:0"` - ProgressUpdatedAt *time.Time `bun:",nullzero"` - RawResultJSON []byte `bun:"type:jsonb,nullzero"` - ErrorMessage *string `bun:",nullzero"` - AcceptError *string `bun:",nullzero"` - AcceptedAt *time.Time `bun:",nullzero"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` -} - -type storageDataSet2026040501 struct { - bun.BaseModel `bun:"table:storage_data_sets"` - - ID int64 `bun:",pk,autoincrement"` - BucketID int64 `bun:",notnull"` - ProviderID string `bun:"type:text,notnull"` - CopyIndex int `bun:",notnull"` - DataSetID *string `bun:"type:text"` - ClientDataSetID *string `bun:"type:text"` - Status string `bun:",notnull,default:'pending'"` - CreateTransactionID *string `bun:",nullzero"` - CreateStatusURL *string `bun:",nullzero"` - CreatedByUploadID *int64 `bun:",nullzero"` - LastUsedUploadID *int64 `bun:",nullzero"` - LastError *string `bun:",nullzero"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` -} - -type storageUploadCopy2026040501 struct { - bun.BaseModel `bun:"table:storage_upload_copies"` - - ID int64 `bun:",pk,autoincrement"` - UploadID int64 `bun:",notnull"` - CopyIndex int `bun:",notnull"` - ProviderID *string `bun:"type:text"` - DataSetID *string `bun:"type:text,scanonly"` - PieceID *string `bun:"type:text"` - TransferMethod string `bun:",notnull"` - Status string `bun:",notnull,default:'pending'"` - RetrievalURL *string `bun:",nullzero"` - IsNewDataSet bool `bun:",notnull,default:false"` - StorageDataSetID *int64 `bun:",nullzero"` - CommitExtraDataHex *string `bun:",nullzero"` - CommitTransactionID *string `bun:",nullzero"` - LastError *string `bun:",nullzero"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` -} - -type storageUploadFailure2026040501 struct { - bun.BaseModel `bun:"table:storage_upload_failures"` - - ID int64 `bun:",pk,autoincrement"` - UploadID int64 `bun:",notnull"` - AttemptIndex int `bun:",notnull"` - ProviderID *string `bun:"type:text"` - TransferMethod string `bun:",notnull"` - Stage *string `bun:",nullzero"` - ErrorMessage *string `bun:",nullzero"` - Explicit bool `bun:",notnull,default:false"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` -} - -// up2026040501Init is the frozen development-preview baseline schema. -// Follow-up schema changes must be added as separate migrations. -func up2026040501Init(ctx context.Context, db bun.IDB) error { - // IAM and bucket ownership tables come first because buckets reference S3 accounts. - if _, err := db.NewCreateTable(). - Model((*s3Account2026040501)(nil)). - IfNotExists(). - ColumnExpr("CONSTRAINT chk_s3_accounts_role CHECK (role IN ('admin', 'user', 'userplus'))"). - Exec(ctx); err != nil { - return fmt.Errorf("creating s3_accounts table: %w", err) - } - - if _, err := db.NewCreateTable(). - Model((*bucket2026040501)(nil)). - IfNotExists(). - ForeignKey("(owner_access_key) REFERENCES s3_accounts(access_key) ON UPDATE CASCADE ON DELETE RESTRICT"). - ColumnExpr("CONSTRAINT chk_buckets_status CHECK (status IN ('active'))"). - Exec(ctx); err != nil { - return fmt.Errorf("creating buckets table: %w", err) - } - - // Objects are stable key identities; all mutable version data lives in object_versions. - if _, err := db.NewCreateTable(). - Model((*object2026040501)(nil)). - IfNotExists(). - ForeignKey("(bucket_id) REFERENCES buckets(id) ON UPDATE CASCADE ON DELETE RESTRICT"). - Exec(ctx); err != nil { - return fmt.Errorf("creating objects table: %w", err) - } - - if _, err := db.NewCreateIndex(). - Model((*object2026040501)(nil)). - Index("idx_objects_bucket_key"). - Column("bucket_id", "key"). - Unique(). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating unique index on objects bucket/key: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*object2026040501)(nil)). - Index("idx_objects_id_bucket_key"). - Column("id", "bucket_id", "key"). - Unique(). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating unique index on objects identity tuple: %w", err) - } - - if err := createStorageProvenanceTables(ctx, db); err != nil { - return err - } - - // Object versions are the source of truth for current data and lifecycle. - if _, err := db.NewCreateTable(). - Model((*objectVersion2026040501)(nil)). - IfNotExists(). - ForeignKey("(object_id, bucket_id, key) REFERENCES objects(id, bucket_id, key) ON UPDATE CASCADE ON DELETE CASCADE"). - ForeignKey("(storage_upload_id) REFERENCES storage_uploads(id) ON UPDATE CASCADE ON DELETE RESTRICT"). - ColumnExpr("CONSTRAINT chk_object_versions_state CHECK (state IN ('cached', 'uploading', 'committing', 'replicating', 'stored', 'failed', 'cache_evicted'))"). - ColumnExpr("CONSTRAINT chk_object_versions_size CHECK (size >= 0)"). - // Committing tracks the active upload through storage_uploads.source_version_id. - // storage_upload_id is set only after a committed copy makes the version readable. - ColumnExpr("CONSTRAINT chk_object_versions_storage_upload_state CHECK ((state IN ('replicating', 'stored', 'cache_evicted') AND storage_upload_id IS NOT NULL) OR (state IN ('cached', 'uploading', 'committing', 'failed') AND storage_upload_id IS NULL))"). - ColumnExpr("CONSTRAINT chk_object_versions_delete_marker_shape CHECK ((is_delete_marker = TRUE AND size = 0 AND e_tag = '' AND checksum = '' AND content_type = '' AND cache_key = '' AND storage_upload_id IS NULL AND in_cache = FALSE AND state = 'cached' AND failed_at_state IS NULL AND last_error IS NULL) OR (is_delete_marker = FALSE AND e_tag <> '' AND checksum <> '' AND cache_key <> ''))"). - Exec(ctx); err != nil { - return fmt.Errorf("creating object_versions table: %w", err) - } - - // Current-object reads use partial indexes over object_versions.is_current. - if _, err := db.NewCreateIndex(). - Model((*objectVersion2026040501)(nil)). - Index("idx_object_versions_current_unique"). - Column("object_id"). - Where(boolTrueWhere(db, "is_current")). - Unique(). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating current version unique index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*objectVersion2026040501)(nil)). - Index("idx_object_versions_current_bucket_key"). - Column("bucket_id", "key"). - Where(boolTrueWhere(db, "is_current")). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating current version listing index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*objectVersion2026040501)(nil)). - Index("idx_object_versions_bucket_key_created"). - ColumnExpr("bucket_id, key, created_at DESC, version_id DESC"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating version history index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*objectVersion2026040501)(nil)). - Index("idx_object_versions_object_created"). - ColumnExpr("object_id, created_at DESC, version_id DESC"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating object version history index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*objectVersion2026040501)(nil)). - Index("idx_object_versions_state_updated"). - Column("state", "updated_at"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating version state index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*objectVersion2026040501)(nil)). - Index("idx_object_versions_content_reuse"). - ColumnExpr("bucket_id, size, checksum, state, created_at DESC, version_id DESC"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating version content reuse index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*objectVersion2026040501)(nil)). - Index("idx_object_versions_storage_upload"). - Column("storage_upload_id"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating version storage upload index: %w", err) - } - - if _, err := db.NewCreateTable(). - Model((*objectDeletion2026040501)(nil)). - IfNotExists(). - ColumnExpr("CONSTRAINT chk_object_deletions_size CHECK (size >= 0)"). - ColumnExpr("CONSTRAINT chk_object_deletions_cache_status CHECK (cache_cleanup_status IN ('pending', 'deleted', 'skipped', 'failed'))"). - Exec(ctx); err != nil { - return fmt.Errorf("creating object_deletions table: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*objectDeletion2026040501)(nil)). - Index("idx_object_deletions_bucket_key_created"). - Column("bucket_id", "key", "created_at"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating object deletion bucket key index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*objectDeletion2026040501)(nil)). - Index("idx_object_deletions_storage_upload"). - Column("storage_upload_id"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating object deletion storage upload index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*objectDeletion2026040501)(nil)). - Index("idx_object_deletions_bucket_created"). - ColumnExpr("bucket_id, created_at DESC, id DESC"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating object deletion bucket created index: %w", err) - } - - // Tasks are queue/audit records with polymorphic references, so no FK is declared here. - if _, err := db.NewCreateTable(). - Model((*task2026040501)(nil)). - IfNotExists(). - ColumnExpr(`CONSTRAINT chk_tasks_type CHECK ("type" IN ('upload', 'evict_cache', 'storage_cleanup'))`). - ColumnExpr("CONSTRAINT chk_tasks_status CHECK (status IN ('queued', 'scheduled', 'running', 'waiting', 'completed', 'failed', 'exhausted', 'cancelled'))"). - ColumnExpr("CONSTRAINT chk_tasks_wait_reason CHECK (wait_reason IS NULL OR (status = 'waiting' AND wait_reason IN ('dependency', 'external_confirmation')))"). - ColumnExpr("CONSTRAINT chk_tasks_ref_type CHECK (ref_type IN ('object', 'bucket', 'storage_upload'))"). - ColumnExpr("CONSTRAINT chk_tasks_object_ref_version CHECK (ref_type <> 'object' OR ref_version_id <> '')"). - ColumnExpr("CONSTRAINT chk_tasks_retry_count CHECK (retry_count >= 0)"). - ColumnExpr("CONSTRAINT chk_tasks_max_retries CHECK (max_retries >= 0)"). - Exec(ctx); err != nil { - return fmt.Errorf("creating tasks table: %w", err) - } - - if _, err := db.NewCreateIndex(). - Model((*task2026040501)(nil)). - Index("idx_tasks_type_status_scheduled"). - Column("type", "status", "scheduled_at"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating task polling index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*task2026040501)(nil)). - Index("idx_tasks_type_ready_scheduled"). - Column("type", "scheduled_at", "id"). - Where("status IN ('queued', 'scheduled', 'waiting')"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating ready task polling index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*task2026040501)(nil)). - Index("idx_tasks_type_stage_status_scheduled"). - Column("type", "stage", "status", "scheduled_at"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating task stage index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*task2026040501)(nil)). - Index("idx_tasks_lease_until"). - Column("lease_until"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating task lease index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*task2026040501)(nil)). - Index("idx_tasks_ref_status"). - Column("ref_type", "ref_id", "status"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating task ref status index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*task2026040501)(nil)). - Index("idx_tasks_ref_version_type_status"). - Column("ref_type", "ref_version_id", "type", "status"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating task ref version index: %w", err) - } - if _, err := db.NewCreateTable(). - Model((*storageCleanupCopy2026040501)(nil)). - IfNotExists(). - ForeignKey("(task_id) REFERENCES tasks(id) ON UPDATE CASCADE ON DELETE CASCADE"). - ColumnExpr("CONSTRAINT chk_storage_cleanup_copies_copy_index CHECK (copy_index >= 0)"). - ColumnExpr("CONSTRAINT chk_storage_cleanup_copies_status CHECK (status IN ('pending', 'delete_scheduled', 'removed', 'failed', 'unsupported'))"). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage_cleanup_copies table: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*storageCleanupCopy2026040501)(nil)). - Index("idx_storage_cleanup_copies_task_copy"). - Column("task_id", "copy_index"). - Unique(). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage cleanup copy task index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*storageCleanupCopy2026040501)(nil)). - Index("idx_storage_cleanup_copies_upload_status"). - Column("upload_id", "status"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage cleanup copy upload index: %w", err) - } - if err := createWalletOperationTables(ctx, db); err != nil { - return err - } - - // Secondary IAM and bucket indexes are grouped here to keep table creation order clear. - if err := createS3AccountIndexes(ctx, db); err != nil { - return err - } - if err := createBucketIndexes(ctx, db); err != nil { - return err - } - - // Multipart uploads keep parts separate and cascade-delete parts with their upload. - if _, err := db.NewCreateTable(). - Model((*multipartUpload2026040501)(nil)). - IfNotExists(). - ForeignKey("(bucket_id) REFERENCES buckets(id) ON UPDATE CASCADE ON DELETE RESTRICT"). - ColumnExpr("CONSTRAINT chk_multipart_uploads_status CHECK (status IN ('initiated', 'completing', 'completed', 'aborted'))"). - Exec(ctx); err != nil { - return fmt.Errorf("creating multipart_uploads table: %w", err) - } - - if _, err := db.NewCreateTable(). - Model((*multipartPart2026040501)(nil)). - IfNotExists(). - ForeignKey("(upload_id) REFERENCES multipart_uploads(upload_id) ON UPDATE CASCADE ON DELETE CASCADE"). - ColumnExpr("CONSTRAINT chk_multipart_parts_part_number CHECK (part_number >= 1 AND part_number <= 10000)"). - ColumnExpr("CONSTRAINT chk_multipart_parts_size CHECK (size >= 0)"). - Exec(ctx); err != nil { - return fmt.Errorf("creating multipart_parts table: %w", err) - } - - if _, err := db.NewCreateIndex(). - Model((*multipartUpload2026040501)(nil)). - Index("idx_multipart_uploads_bucket_status_key_upload"). - Column("bucket_id", "status", "key", "upload_id"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating multipart upload listing index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*multipartPart2026040501)(nil)). - Index("idx_multipart_parts_upload_part"). - Column("upload_id", "part_number"). - Unique(). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating multipart part unique index: %w", err) - } - - return nil -} - -// down2026040501Init drops the baseline schema in reverse dependency order. -func down2026040501Init(ctx context.Context, db bun.IDB) error { - for _, m := range []interface{}{ - (*multipartPart2026040501)(nil), - (*multipartUpload2026040501)(nil), - (*walletOperation2026040501)(nil), - (*storageCleanupCopy2026040501)(nil), - (*task2026040501)(nil), - (*objectDeletion2026040501)(nil), - (*storageUploadFailure2026040501)(nil), - (*storageUploadCopy2026040501)(nil), - (*objectVersion2026040501)(nil), - (*storageDataSet2026040501)(nil), - (*storageUpload2026040501)(nil), - (*object2026040501)(nil), - (*bucket2026040501)(nil), - (*s3Account2026040501)(nil), - } { - if _, err := db.NewDropTable().Model(m).IfExists().Exec(ctx); err != nil { - return fmt.Errorf("dropping table %T: %w", m, err) - } - } - return nil -} - -func createWalletOperationTables(ctx context.Context, db bun.IDB) error { - if _, err := db.NewCreateTable(). - Model((*walletOperation2026040501)(nil)). - IfNotExists(). - ColumnExpr(`CONSTRAINT chk_wallet_operations_type CHECK ("type" IN ('fund', 'withdraw'))`). - ColumnExpr("CONSTRAINT chk_wallet_operations_status CHECK (status IN ('pending', 'running', 'submitted', 'confirmed', 'failed', 'unknown'))"). - ColumnExpr("CONSTRAINT chk_wallet_operations_amount CHECK (" + walletOperationAmountCheck(db) + ")"). - Exec(ctx); err != nil { - return fmt.Errorf("creating wallet_operations table: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*walletOperation2026040501)(nil)). - Index("idx_wallet_operations_request"). - Column("type", "client_request_id"). - Unique(). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating wallet operation request index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*walletOperation2026040501)(nil)). - Index("idx_wallet_operations_status_created"). - Column("status", "created_at", "id"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating wallet operation status index: %w", err) - } - return nil -} - -func walletOperationAmountCheck(db bun.IDB) string { - if db.Dialect().Name() == dialect.PG { - return `amount ~ '^[1-9][0-9]*$'` - } - return `amount GLOB '[1-9]*' AND amount NOT GLOB '*[^0-9]*'` -} - -func createS3AccountIndexes(ctx context.Context, db bun.IDB) error { - if _, err := db.NewCreateIndex(). - Model((*s3Account2026040501)(nil)). - Index("idx_s3_accounts_is_root"). - Column("is_root"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating s3 account root index: %w", err) - } - - if _, err := db.NewCreateIndex(). - Model((*s3Account2026040501)(nil)). - Index("idx_s3_accounts_single_root"). - Column("is_root"). - Where(boolTrueWhere(db, "is_root")). - Unique(). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating single root account index: %w", err) - } - return nil -} - -func createBucketIndexes(ctx context.Context, db bun.IDB) error { - if _, err := db.NewCreateIndex(). - Model((*bucket2026040501)(nil)). - Index("idx_buckets_owner_access_key"). - Column("owner_access_key"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating bucket owner index: %w", err) - } - return nil -} - -func createStorageProvenanceTables(ctx context.Context, db bun.IDB) error { - if _, err := db.NewCreateTable(). - Model((*storageUpload2026040501)(nil)). - IfNotExists(). - ForeignKey("(bucket_id) REFERENCES buckets(id) ON UPDATE CASCADE ON DELETE RESTRICT"). - ColumnExpr("CONSTRAINT chk_storage_uploads_status CHECK (status IN ('running', 'ingress_ready', 'readable', 'complete', 'failed', 'rejected', 'superseded'))"). - ColumnExpr("CONSTRAINT chk_storage_uploads_content_size CHECK (content_size >= 0)"). - ColumnExpr("CONSTRAINT chk_storage_uploads_requested_copies CHECK (requested_copies >= 0)"). - ColumnExpr("CONSTRAINT chk_storage_uploads_ingress_bytes CHECK (ingress_bytes_transferred >= 0 AND ingress_bytes_transferred <= content_size)"). - ColumnExpr("CONSTRAINT chk_storage_uploads_ingress_attempt CHECK (ingress_store_attempt >= 0)"). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage_uploads table: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*storageUpload2026040501)(nil)). - Index("idx_storage_uploads_task_version_status"). - Column("source_task_id", "source_version_id", "status", "accepted_at"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage upload task/version index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*storageUpload2026040501)(nil)). - Index("idx_storage_uploads_source_version_id"). - Column("source_version_id", "id"). - Where("source_version_id <> ''"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage upload source version index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*storageUpload2026040501)(nil)). - Index("idx_storage_uploads_content_status"). - Column("bucket_id", "content_size", "checksum", "status", "accepted_at"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage upload content index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*storageUpload2026040501)(nil)). - Index("idx_storage_uploads_active_source_version"). - Column("source_version_id"). - Where("source_version_id <> '' AND status IN ('running', 'ingress_ready', 'readable')"). - Unique(). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating active storage upload source version index: %w", err) - } - - if _, err := db.NewCreateTable(). - Model((*storageDataSet2026040501)(nil)). - IfNotExists(). - ForeignKey("(bucket_id) REFERENCES buckets(id) ON UPDATE CASCADE ON DELETE RESTRICT"). - ForeignKey("(created_by_upload_id) REFERENCES storage_uploads(id) ON UPDATE CASCADE ON DELETE SET NULL"). - ForeignKey("(last_used_upload_id) REFERENCES storage_uploads(id) ON UPDATE CASCADE ON DELETE SET NULL"). - ColumnExpr("CONSTRAINT chk_storage_data_sets_copy_index CHECK (copy_index >= 0)"). - ColumnExpr("CONSTRAINT chk_storage_data_sets_status CHECK (status IN ('pending', 'creating', 'ready', 'failed', 'unavailable', 'draining', 'retired'))"). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage_data_sets table: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*storageDataSet2026040501)(nil)). - Index("idx_storage_data_sets_provider_data_set"). - Column("provider_id", "data_set_id"). - Where("data_set_id IS NOT NULL AND data_set_id <> ''"). - Unique(). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage data set provider unique index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*storageDataSet2026040501)(nil)). - Index("idx_storage_data_sets_bucket_provider"). - Column("bucket_id", "provider_id"). - Unique(). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage data set bucket provider index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*storageDataSet2026040501)(nil)). - Index("idx_storage_data_sets_bucket_copy_index"). - Column("bucket_id", "copy_index"). - Unique(). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage data set bucket copy index: %w", err) - } - - if _, err := db.NewCreateTable(). - Model((*storageUploadCopy2026040501)(nil)). - IfNotExists(). - ForeignKey("(upload_id) REFERENCES storage_uploads(id) ON UPDATE CASCADE ON DELETE CASCADE"). - ForeignKey("(storage_data_set_id) REFERENCES storage_data_sets(id) ON UPDATE CASCADE ON DELETE RESTRICT"). - ColumnExpr("CONSTRAINT chk_storage_upload_copies_copy_index CHECK (copy_index >= 0)"). - ColumnExpr("CONSTRAINT chk_storage_upload_copies_status CHECK (status IN ('pending', 'piece_ready', 'committing', 'committed', 'failed'))"). - ColumnExpr("CONSTRAINT chk_storage_upload_copies_transfer_method CHECK (transfer_method IN ('ingress', 'peer_pull'))"). - ColumnExpr("CONSTRAINT chk_storage_upload_copies_committed_shape CHECK (status <> 'committed' OR (storage_data_set_id IS NOT NULL AND provider_id IS NOT NULL AND provider_id <> '' AND piece_id IS NOT NULL AND piece_id <> '' AND retrieval_url IS NOT NULL AND retrieval_url <> ''))"). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage_upload_copies table: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*storageUploadCopy2026040501)(nil)). - Index("idx_storage_upload_copies_upload_index"). - Column("upload_id", "copy_index"). - Unique(). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage upload copy unique index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*storageUploadCopy2026040501)(nil)). - Index("idx_storage_upload_copies_upload_transfer_method_index"). - Column("upload_id", "transfer_method", "copy_index"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage upload copy transfer method index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*storageUploadCopy2026040501)(nil)). - Index("idx_storage_upload_copies_status_data_set_upload"). - Column("status", "storage_data_set_id", "upload_id"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage upload copy dataset summary index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*storageUploadCopy2026040501)(nil)). - Index("idx_storage_upload_copies_status_piece_identity_upload"). - Column("status", "provider_id", "piece_id", "upload_id"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage upload copy piece identity index: %w", err) - } - - if _, err := db.NewCreateTable(). - Model((*storageUploadFailure2026040501)(nil)). - IfNotExists(). - ForeignKey("(upload_id) REFERENCES storage_uploads(id) ON UPDATE CASCADE ON DELETE CASCADE"). - ColumnExpr("CONSTRAINT chk_storage_upload_failures_transfer_method CHECK (transfer_method IN ('ingress', 'peer_pull'))"). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage_upload_failures table: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*storageUploadFailure2026040501)(nil)). - Index("idx_storage_upload_failures_upload_attempt"). - Column("upload_id", "attempt_index"). - Unique(). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage upload failure unique index: %w", err) - } - return nil -} - -// boolTrueWhere emits portable partial-index predicates for PostgreSQL and SQLite. -func boolTrueWhere(db bun.IDB, column string) string { - if db.Dialect().Name() == dialect.PG { - return column + " IS TRUE" - } - return column + " = TRUE" -} diff --git a/internal/db/migrations/2026051301_bucket_default_copies.go b/internal/db/migrations/2026051301_bucket_default_copies.go deleted file mode 100644 index 93600df..0000000 --- a/internal/db/migrations/2026051301_bucket_default_copies.go +++ /dev/null @@ -1,42 +0,0 @@ -package migrations - -import ( - "context" - "fmt" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" -) - -func init() { - Migrations.MustRegister( - transactionalMigration(up2026051301BucketDefaultCopies), - transactionalMigration(down2026051301BucketDefaultCopies), - ) -} - -func up2026051301BucketDefaultCopies(ctx context.Context, db bun.IDB) error { - exists, err := columnExists(ctx, db, "buckets", "default_copies") - if err != nil || exists { - return err - } - query := "ALTER TABLE buckets ADD COLUMN default_copies INTEGER CHECK (default_copies IS NULL OR (default_copies >= 1 AND default_copies <= 8))" - if db.Dialect().Name() == dialect.PG { - query = "ALTER TABLE buckets ADD COLUMN default_copies INTEGER CONSTRAINT chk_buckets_default_copies CHECK (default_copies IS NULL OR (default_copies >= 1 AND default_copies <= 8))" - } - if _, err := db.ExecContext(ctx, query); err != nil { - return fmt.Errorf("adding buckets.default_copies: %w", err) - } - return nil -} - -func down2026051301BucketDefaultCopies(ctx context.Context, db bun.IDB) error { - exists, err := columnExists(ctx, db, "buckets", "default_copies") - if err != nil || !exists { - return err - } - if _, err := db.ExecContext(ctx, "ALTER TABLE buckets DROP COLUMN default_copies"); err != nil { - return fmt.Errorf("dropping buckets.default_copies: %w", err) - } - return nil -} diff --git a/internal/db/migrations/2026051701_observability_states.go b/internal/db/migrations/2026051701_observability_states.go deleted file mode 100644 index c8861ac..0000000 --- a/internal/db/migrations/2026051701_observability_states.go +++ /dev/null @@ -1,140 +0,0 @@ -package migrations - -import ( - "context" - "fmt" - "time" - - "github.com/uptrace/bun" -) - -func init() { - Migrations.MustRegister( - transactionalMigration(up2026051701ObservabilityStates), - transactionalMigration(down2026051701ObservabilityStates), - ) -} - -type observabilityProviderState2026051701 struct { - bun.BaseModel `bun:"table:observability_provider_states"` - - ProviderID string `bun:"provider_id,pk,type:text"` - Status string `bun:"status,notnull"` - ReasonCodes []string `bun:"reason_codes,type:jsonb,notnull"` - Active *bool `bun:"active"` - HasPDP *bool `bun:"has_pdp"` - ServiceURL *string `bun:"service_url"` - HealthStatus *string `bun:"health_status"` - LastCheckedAt time.Time `bun:"last_checked_at,nullzero,notnull"` - LastError *string `bun:"last_error,nullzero"` - Evidence map[string]any `bun:"evidence_json,type:jsonb,notnull"` - CreatedAt time.Time `bun:"created_at,nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:"updated_at,nullzero,notnull,default:current_timestamp"` -} - -type observabilityDataSetState2026051701 struct { - bun.BaseModel `bun:"table:observability_data_set_states"` - - LocalDataSetID int64 `bun:"local_data_set_id,pk"` - BucketID int64 `bun:"bucket_id,notnull"` - BucketName string `bun:"bucket_name,notnull"` - CopyIndex int `bun:"copy_index,notnull"` - ProviderID string `bun:"provider_id,type:text,notnull"` - ChainDataSetID *string `bun:"chain_data_set_id,type:text"` - ClientDataSetID *string `bun:"client_data_set_id,type:text"` - LocalStatus string `bun:"local_status,notnull"` - Status string `bun:"status,notnull"` - ReasonCodes []string `bun:"reason_codes,type:jsonb,notnull"` - ActivePieceCount *int64 `bun:"active_piece_count,nullzero"` - LastCheckedAt time.Time `bun:"last_checked_at,nullzero,notnull"` - LastError *string `bun:"last_error,nullzero"` - Evidence map[string]any `bun:"evidence_json,type:jsonb,notnull"` - CreatedAt time.Time `bun:"created_at,nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:"updated_at,nullzero,notnull,default:current_timestamp"` -} - -type observabilityCollectionState2026051701 struct { - bun.BaseModel `bun:"table:observability_collection_states"` - - CollectionType string `bun:"collection_type,pk,type:text"` - LastCheckedAt time.Time `bun:"last_checked_at,nullzero,notnull"` - CreatedAt time.Time `bun:"created_at,nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:"updated_at,nullzero,notnull,default:current_timestamp"` -} - -func up2026051701ObservabilityStates(ctx context.Context, db bun.IDB) error { - if _, err := db.NewCreateTable(). - Model((*observabilityCollectionState2026051701)(nil)). - IfNotExists(). - ColumnExpr("CONSTRAINT chk_observability_collection_type CHECK (collection_type IN ('providers', 'data_sets'))"). - Exec(ctx); err != nil { - return fmt.Errorf("creating observability_collection_states table: %w", err) - } - - if _, err := db.NewCreateTable(). - Model((*observabilityProviderState2026051701)(nil)). - IfNotExists(). - ColumnExpr("CONSTRAINT chk_observability_provider_status CHECK (status IN ('available', 'degraded', 'unavailable', 'unknown'))"). - Exec(ctx); err != nil { - return fmt.Errorf("creating observability_provider_states table: %w", err) - } - - if _, err := db.NewCreateIndex(). - Model((*observabilityProviderState2026051701)(nil)). - Index("idx_observability_provider_states_status"). - Column("status", "last_checked_at"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating observability provider status index: %w", err) - } - - if _, err := db.NewCreateTable(). - Model((*observabilityDataSetState2026051701)(nil)). - IfNotExists(). - ForeignKey("(local_data_set_id) REFERENCES storage_data_sets(id) ON UPDATE CASCADE ON DELETE CASCADE"). - ColumnExpr("CONSTRAINT chk_observability_data_set_status CHECK (status IN ('available', 'degraded', 'unavailable', 'unknown'))"). - Exec(ctx); err != nil { - return fmt.Errorf("creating observability_data_set_states table: %w", err) - } - - if _, err := db.NewCreateIndex(). - Model((*observabilityDataSetState2026051701)(nil)). - Index("idx_observability_data_set_states_bucket_status"). - Column("bucket_id", "status", "last_checked_at"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating observability data set bucket status index: %w", err) - } - if _, err := db.NewCreateIndex(). - Model((*observabilityDataSetState2026051701)(nil)). - Index("idx_observability_data_set_states_provider_status"). - Column("provider_id", "status", "last_checked_at"). - IfNotExists(). - Exec(ctx); err != nil { - return fmt.Errorf("creating observability data set provider status index: %w", err) - } - - return nil -} - -func down2026051701ObservabilityStates(ctx context.Context, db bun.IDB) error { - if _, err := db.NewDropTable(). - Model((*observabilityDataSetState2026051701)(nil)). - IfExists(). - Exec(ctx); err != nil { - return fmt.Errorf("dropping observability_data_set_states table: %w", err) - } - if _, err := db.NewDropTable(). - Model((*observabilityProviderState2026051701)(nil)). - IfExists(). - Exec(ctx); err != nil { - return fmt.Errorf("dropping observability_provider_states table: %w", err) - } - if _, err := db.NewDropTable(). - Model((*observabilityCollectionState2026051701)(nil)). - IfExists(). - Exec(ctx); err != nil { - return fmt.Errorf("dropping observability_collection_states table: %w", err) - } - return nil -} diff --git a/internal/db/migrations/2026060401_multipart_object_parts.go b/internal/db/migrations/2026060401_multipart_object_parts.go deleted file mode 100644 index c0e69b2..0000000 --- a/internal/db/migrations/2026060401_multipart_object_parts.go +++ /dev/null @@ -1,114 +0,0 @@ -package migrations - -import ( - "context" - "fmt" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" -) - -func init() { - Migrations.MustRegister( - transactionalMigration(up2026060401MultipartObjectParts), - transactionalMigration(down2026060401MultipartObjectParts), - ) -} - -func up2026060401MultipartObjectParts(ctx context.Context, db bun.IDB) error { - done, err := multipartObjectPartsReached2026060401(ctx, db, true) - if err != nil || done { - return err - } - addMultipartUploadID := "ALTER TABLE object_versions ADD COLUMN multipart_upload_id TEXT REFERENCES multipart_uploads(upload_id) ON UPDATE CASCADE ON DELETE RESTRICT" - if db.Dialect().Name() == dialect.PG { - addMultipartUploadID = "ALTER TABLE object_versions ADD COLUMN multipart_upload_id TEXT" - } - if _, err := db.ExecContext(ctx, addMultipartUploadID); err != nil { - return fmt.Errorf("adding object_versions.multipart_upload_id: %w", err) - } - if db.Dialect().Name() == dialect.PG { - if _, err := db.ExecContext(ctx, "ALTER TABLE object_versions ADD CONSTRAINT fk_object_versions_multipart_upload_id FOREIGN KEY (multipart_upload_id) REFERENCES multipart_uploads(upload_id) ON UPDATE CASCADE ON DELETE RESTRICT"); err != nil { - return fmt.Errorf("adding object_versions multipart upload foreign key: %w", err) - } - } - if _, err := db.ExecContext(ctx, "CREATE INDEX IF NOT EXISTS idx_object_versions_multipart_upload ON object_versions (multipart_upload_id)"); err != nil { - return fmt.Errorf("creating object_versions multipart upload index: %w", err) - } - if _, err := db.ExecContext(ctx, "ALTER TABLE multipart_parts ADD COLUMN checksum TEXT"); err != nil { - return fmt.Errorf("adding multipart_parts.checksum: %w", err) - } - return nil -} - -func down2026060401MultipartObjectParts(ctx context.Context, db bun.IDB) error { - done, err := multipartObjectPartsReached2026060401(ctx, db, false) - if err != nil || done { - return err - } - if _, err := db.ExecContext(ctx, "DROP INDEX IF EXISTS idx_object_versions_multipart_upload"); err != nil { - return fmt.Errorf("dropping object_versions multipart upload index: %w", err) - } - if db.Dialect().Name() == dialect.PG { - if _, err := db.ExecContext(ctx, "ALTER TABLE object_versions DROP CONSTRAINT IF EXISTS fk_object_versions_multipart_upload_id"); err != nil { - return fmt.Errorf("dropping object_versions multipart upload foreign key: %w", err) - } - } - if _, err := db.ExecContext(ctx, "ALTER TABLE object_versions DROP COLUMN multipart_upload_id"); err != nil { - return fmt.Errorf("dropping object_versions.multipart_upload_id: %w", err) - } - if _, err := db.ExecContext(ctx, "ALTER TABLE multipart_parts DROP COLUMN checksum"); err != nil { - return fmt.Errorf("dropping multipart_parts.checksum: %w", err) - } - return nil -} - -func multipartObjectPartsReached2026060401(ctx context.Context, db bun.IDB, wantPresent bool) (bool, error) { - multipartUploadID, err := columnExists(ctx, db, "object_versions", "multipart_upload_id") - if err != nil { - return false, fmt.Errorf("checking object_versions.multipart_upload_id: %w", err) - } - checksum, err := columnExists(ctx, db, "multipart_parts", "checksum") - if err != nil { - return false, fmt.Errorf("checking multipart_parts.checksum: %w", err) - } - index, err := indexExists(ctx, db, "idx_object_versions_multipart_upload") - if err != nil { - return false, fmt.Errorf("checking idx_object_versions_multipart_upload: %w", err) - } - foreignKey, err := multipartUploadForeignKeyExists2026060401(ctx, db) - if err != nil { - return false, fmt.Errorf("checking object_versions multipart upload foreign key: %w", err) - } - if multipartUploadID == wantPresent && checksum == wantPresent && index == wantPresent && foreignKey == wantPresent { - return true, nil - } - if multipartUploadID != wantPresent && checksum != wantPresent && index != wantPresent && foreignKey != wantPresent { - return false, nil - } - return false, fmt.Errorf( - "2026060401_multipart_object_parts has partial schema state: multipart_upload_id=%t, checksum=%t, index=%t, foreign_key=%t", - multipartUploadID, - checksum, - index, - foreignKey, - ) -} - -func multipartUploadForeignKeyExists2026060401(ctx context.Context, db bun.IDB) (bool, error) { - if db.Dialect().Name() == dialect.PG { - return queryExists(ctx, db, `SELECT COUNT(*) - FROM pg_constraint c - JOIN pg_class t ON t.oid = c.conrelid - JOIN pg_namespace n ON n.oid = t.relnamespace - WHERE n.nspname = current_schema() - AND t.relname = 'object_versions' - AND c.conname = 'fk_object_versions_multipart_upload_id'`) - } - return queryExists(ctx, db, `SELECT COUNT(*) FROM pragma_foreign_key_list('object_versions') - WHERE "table" = 'multipart_uploads' - AND "from" = 'multipart_upload_id' - AND "to" = 'upload_id' - AND on_update = 'CASCADE' - AND on_delete = 'RESTRICT'`) -} diff --git a/internal/db/migrations/2026061101_postgres_prefix_indexes.go b/internal/db/migrations/2026061101_postgres_prefix_indexes.go deleted file mode 100644 index d057d69..0000000 --- a/internal/db/migrations/2026061101_postgres_prefix_indexes.go +++ /dev/null @@ -1,61 +0,0 @@ -package migrations - -import ( - "context" - "fmt" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" -) - -func init() { - Migrations.MustRegister( - transactionalMigration(up2026061101PostgresPrefixIndexes), - transactionalMigration(down2026061101PostgresPrefixIndexes), - ) -} - -func up2026061101PostgresPrefixIndexes(ctx context.Context, db bun.IDB) error { - if db.Dialect().Name() != dialect.PG { - return nil - } - statements := []struct { - name string - sql string - }{ - { - name: "current object version prefix index", - sql: `CREATE INDEX IF NOT EXISTS idx_object_versions_current_bucket_delete_key_c ON object_versions (bucket_id, is_delete_marker, (key COLLATE "C")) WHERE is_current = TRUE`, - }, - { - name: "version history prefix index", - sql: `CREATE INDEX IF NOT EXISTS idx_object_versions_bucket_key_created_c ON object_versions (bucket_id, (key COLLATE "C"), created_at DESC, version_id DESC)`, - }, - { - name: "multipart upload prefix index", - sql: `CREATE INDEX IF NOT EXISTS idx_multipart_uploads_bucket_status_key_upload_c ON multipart_uploads (bucket_id, status, (key COLLATE "C"), upload_id)`, - }, - } - for _, stmt := range statements { - if _, err := db.ExecContext(ctx, stmt.sql); err != nil { - return fmt.Errorf("creating %s: %w", stmt.name, err) - } - } - return nil -} - -func down2026061101PostgresPrefixIndexes(ctx context.Context, db bun.IDB) error { - if db.Dialect().Name() != dialect.PG { - return nil - } - for _, name := range []string{ - "idx_multipart_uploads_bucket_status_key_upload_c", - "idx_object_versions_bucket_key_created_c", - "idx_object_versions_current_bucket_delete_key_c", - } { - if _, err := db.ExecContext(ctx, "DROP INDEX IF EXISTS "+name); err != nil { - return fmt.Errorf("dropping %s: %w", name, err) - } - } - return nil -} diff --git a/internal/db/migrations/2026062201_wallet_approve.go b/internal/db/migrations/2026062201_wallet_approve.go deleted file mode 100644 index b639f50..0000000 --- a/internal/db/migrations/2026062201_wallet_approve.go +++ /dev/null @@ -1,202 +0,0 @@ -package migrations - -import ( - "context" - "fmt" - "strings" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" -) - -func init() { - Migrations.MustRegister( - transactionalMigration(up2026062201WalletApprove), - transactionalMigration(down2026062201WalletApprove), - ) -} - -func up2026062201WalletApprove(ctx context.Context, db bun.IDB) error { - done, err := walletApproveReached2026062201(ctx, db, true) - if err != nil || done { - return err - } - if db.Dialect().Name() == dialect.PG { - if _, err := db.ExecContext(ctx, "ALTER TABLE wallet_operations DROP CONSTRAINT chk_wallet_operations_type"); err != nil { - return fmt.Errorf("dropping wallet operation type constraint: %w", err) - } - if _, err := db.ExecContext(ctx, "ALTER TABLE wallet_operations DROP CONSTRAINT chk_wallet_operations_amount"); err != nil { - return fmt.Errorf("dropping wallet operation amount constraint: %w", err) - } - if _, err := db.ExecContext(ctx, `ALTER TABLE wallet_operations ADD CONSTRAINT chk_wallet_operations_type CHECK ("type" IN ('fund', 'withdraw', 'approve'))`); err != nil { - return fmt.Errorf("adding wallet operation type constraint: %w", err) - } - if _, err := db.ExecContext(ctx, `ALTER TABLE wallet_operations ADD CONSTRAINT chk_wallet_operations_amount CHECK ((type = 'approve' AND amount = '0') OR (type IN ('fund', 'withdraw') AND amount ~ '^[1-9][0-9]*$'))`); err != nil { - return fmt.Errorf("adding wallet operation amount constraint: %w", err) - } - return nil - } - return rebuildSQLiteWalletOperations(ctx, db, true) -} - -func down2026062201WalletApprove(ctx context.Context, db bun.IDB) error { - done, err := walletApproveReached2026062201(ctx, db, false) - if err != nil || done { - return err - } - var approveCount int - if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM wallet_operations WHERE type = 'approve'").Scan(&approveCount); err != nil { - return fmt.Errorf("counting approve wallet operations: %w", err) - } - if approveCount > 0 { - return fmt.Errorf("cannot remove wallet approve support while approve operations exist") - } - - if db.Dialect().Name() == dialect.PG { - if _, err := db.ExecContext(ctx, "ALTER TABLE wallet_operations DROP CONSTRAINT chk_wallet_operations_type"); err != nil { - return fmt.Errorf("dropping wallet operation type constraint: %w", err) - } - if _, err := db.ExecContext(ctx, "ALTER TABLE wallet_operations DROP CONSTRAINT chk_wallet_operations_amount"); err != nil { - return fmt.Errorf("dropping wallet operation amount constraint: %w", err) - } - if _, err := db.ExecContext(ctx, `ALTER TABLE wallet_operations ADD CONSTRAINT chk_wallet_operations_type CHECK ("type" IN ('fund', 'withdraw'))`); err != nil { - return fmt.Errorf("restoring wallet operation type constraint: %w", err) - } - if _, err := db.ExecContext(ctx, `ALTER TABLE wallet_operations ADD CONSTRAINT chk_wallet_operations_amount CHECK (amount ~ '^[1-9][0-9]*$')`); err != nil { - return fmt.Errorf("restoring wallet operation amount constraint: %w", err) - } - return nil - } - return rebuildSQLiteWalletOperations(ctx, db, false) -} - -func rebuildSQLiteWalletOperations(ctx context.Context, db bun.IDB, allowApprove bool) error { - typeCheck := `"type" IN ('fund', 'withdraw')` - amountCheck := `amount GLOB '[1-9]*' AND amount NOT GLOB '*[^0-9]*'` - if allowApprove { - typeCheck = `"type" IN ('fund', 'withdraw', 'approve')` - amountCheck = `(("type" = 'approve' AND amount = '0') OR ("type" IN ('fund', 'withdraw') AND amount GLOB '[1-9]*' AND amount NOT GLOB '*[^0-9]*'))` - } - - statements := []string{ - "DROP INDEX IF EXISTS idx_wallet_operations_request", - "DROP INDEX IF EXISTS idx_wallet_operations_status_created", - "ALTER TABLE wallet_operations RENAME TO wallet_operations_2026062201_old", - fmt.Sprintf(`CREATE TABLE wallet_operations ( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - type TEXT NOT NULL, - client_request_id TEXT NOT NULL, - amount TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - tx_hash TEXT, - last_error TEXT, - lease_until TIMESTAMP, - started_at TIMESTAMP, - submitted_at TIMESTAMP, - completed_at TIMESTAMP, - created_at TIMESTAMP NOT NULL DEFAULT current_timestamp, - updated_at TIMESTAMP NOT NULL DEFAULT current_timestamp, - CONSTRAINT chk_wallet_operations_type CHECK (%s), - CONSTRAINT chk_wallet_operations_status CHECK (status IN ('pending', 'running', 'submitted', 'confirmed', 'failed', 'unknown')), - CONSTRAINT chk_wallet_operations_amount CHECK (%s) - )`, typeCheck, amountCheck), - `INSERT INTO wallet_operations ( - id, type, client_request_id, amount, status, tx_hash, last_error, lease_until, - started_at, submitted_at, completed_at, created_at, updated_at - ) - SELECT - id, type, client_request_id, amount, status, tx_hash, last_error, lease_until, - started_at, submitted_at, completed_at, created_at, updated_at - FROM wallet_operations_2026062201_old`, - "DROP TABLE wallet_operations_2026062201_old", - "CREATE UNIQUE INDEX idx_wallet_operations_request ON wallet_operations (type, client_request_id)", - "CREATE INDEX idx_wallet_operations_status_created ON wallet_operations (status, created_at, id)", - } - for _, stmt := range statements { - if _, err := db.ExecContext(ctx, stmt); err != nil { - return fmt.Errorf("rebuilding wallet_operations table: %w", err) - } - } - return nil -} - -func walletApproveReached2026062201(ctx context.Context, db bun.IDB, wantApprove bool) (bool, error) { - typeConstraint, err := walletConstraintExists2026062201(ctx, db, "chk_wallet_operations_type") - if err != nil { - return false, fmt.Errorf("checking wallet operation type constraint: %w", err) - } - amountConstraint, err := walletConstraintExists2026062201(ctx, db, "chk_wallet_operations_amount") - if err != nil { - return false, fmt.Errorf("checking wallet operation amount constraint: %w", err) - } - typeAllowsApprove, err := walletConstraintAllowsApprove2026062201( - ctx, - db, - "chk_wallet_operations_type", - `"type" IN ('fund', 'withdraw', 'approve')`, - ) - if err != nil { - return false, fmt.Errorf("checking wallet operation type constraint definition: %w", err) - } - amountAllowsApprove, err := walletConstraintAllowsApprove2026062201( - ctx, - db, - "chk_wallet_operations_amount", - `"type" = 'approve'`, - ) - if err != nil { - return false, fmt.Errorf("checking wallet operation amount constraint definition: %w", err) - } - - if typeConstraint && amountConstraint && typeAllowsApprove == amountAllowsApprove { - return typeAllowsApprove == wantApprove, nil - } - return false, fmt.Errorf( - "2026062201_wallet_approve has partial schema state: type_constraint=%t, amount_constraint=%t, type_allows_approve=%t, amount_allows_approve=%t", - typeConstraint, - amountConstraint, - typeAllowsApprove, - amountAllowsApprove, - ) -} - -func walletConstraintExists2026062201(ctx context.Context, db bun.IDB, name string) (bool, error) { - if db.Dialect().Name() == dialect.PG { - return queryExists(ctx, db, `SELECT COUNT(*) - FROM pg_constraint c - JOIN pg_class t ON t.oid = c.conrelid - JOIN pg_namespace n ON n.oid = t.relnamespace - WHERE n.nspname = current_schema() - AND t.relname = 'wallet_operations' - AND c.conname = ?`, name) - } - var schemaSQL string - if err := db.NewRaw("SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = 'wallet_operations'").Scan(ctx, &schemaSQL); err != nil { - return false, err - } - return strings.Contains(schemaSQL, name), nil -} - -func walletConstraintAllowsApprove2026062201( - ctx context.Context, - db bun.IDB, - name, sqliteNeedle string, -) (bool, error) { - var definition string - if db.Dialect().Name() == dialect.PG { - return queryExists(ctx, db, `SELECT COUNT(*) - FROM pg_constraint c - JOIN pg_class t ON t.oid = c.conrelid - JOIN pg_namespace n ON n.oid = t.relnamespace - WHERE n.nspname = current_schema() - AND t.relname = 'wallet_operations' - AND c.conname = ? - AND position('approve' IN pg_get_constraintdef(c.oid)) > 0`, name) - } - if err := db.NewRaw(`SELECT COALESCE(( - SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = 'wallet_operations' - ), '')`).Scan(ctx, &definition); err != nil { - return false, err - } - return strings.Contains(definition, sqliteNeedle), nil -} diff --git a/internal/db/migrations/2026072801_cache_lru.go b/internal/db/migrations/2026072801_cache_lru.go deleted file mode 100644 index b133d3e..0000000 --- a/internal/db/migrations/2026072801_cache_lru.go +++ /dev/null @@ -1,79 +0,0 @@ -package migrations - -import ( - "context" - "fmt" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" -) - -func init() { - Migrations.MustRegister( - transactionalMigration(up2026072801CacheLRU), - transactionalMigration(down2026072801CacheLRU), - ) -} - -func up2026072801CacheLRU(ctx context.Context, db bun.IDB) error { - done, err := cacheLRUReached2026072801(ctx, db, true) - if err != nil || done { - return err - } - columnType := "TIMESTAMP" - if db.Dialect().Name() == dialect.PG { - columnType = "TIMESTAMPTZ" - } - if _, err := db.ExecContext(ctx, "ALTER TABLE object_versions ADD COLUMN cache_accessed_at "+columnType); err != nil { - return fmt.Errorf("adding object_versions.cache_accessed_at: %w", err) - } - if _, err := db.ExecContext( - ctx, - "UPDATE object_versions SET cache_accessed_at = created_at WHERE cache_accessed_at IS NULL", - ); err != nil { - return fmt.Errorf("initializing object version cache access timestamps: %w", err) - } - if _, err := db.ExecContext( - ctx, - "CREATE INDEX idx_object_versions_cache_lru ON object_versions (in_cache, cache_accessed_at, created_at, version_id)", - ); err != nil { - return fmt.Errorf("creating object version cache LRU index: %w", err) - } - return nil -} - -func down2026072801CacheLRU(ctx context.Context, db bun.IDB) error { - done, err := cacheLRUReached2026072801(ctx, db, false) - if err != nil || done { - return err - } - if _, err := db.ExecContext(ctx, "DROP INDEX IF EXISTS idx_object_versions_cache_lru"); err != nil { - return fmt.Errorf("dropping object version cache LRU index: %w", err) - } - if _, err := db.ExecContext(ctx, "ALTER TABLE object_versions DROP COLUMN cache_accessed_at"); err != nil { - return fmt.Errorf("dropping object_versions.cache_accessed_at: %w", err) - } - return nil -} - -func cacheLRUReached2026072801(ctx context.Context, db bun.IDB, wantPresent bool) (bool, error) { - columnPresent, err := columnExists(ctx, db, "object_versions", "cache_accessed_at") - if err != nil { - return false, fmt.Errorf("checking object_versions.cache_accessed_at: %w", err) - } - indexPresent, err := indexExists(ctx, db, "idx_object_versions_cache_lru") - if err != nil { - return false, fmt.Errorf("checking idx_object_versions_cache_lru: %w", err) - } - if columnPresent == wantPresent && indexPresent == wantPresent { - return true, nil - } - if columnPresent != wantPresent && indexPresent != wantPresent { - return false, nil - } - return false, fmt.Errorf( - "2026072801_cache_lru has partial schema state: cache_accessed_at=%t, index=%t", - columnPresent, - indexPresent, - ) -} diff --git a/internal/db/migrations/2026081901_bucket_minimum_durable_copies.go b/internal/db/migrations/2026081901_bucket_minimum_durable_copies.go deleted file mode 100644 index d834d54..0000000 --- a/internal/db/migrations/2026081901_bucket_minimum_durable_copies.go +++ /dev/null @@ -1,42 +0,0 @@ -package migrations - -import ( - "context" - "fmt" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" -) - -func init() { - Migrations.MustRegister( - transactionalMigration(up2026081901BucketMinimumDurableCopies), - transactionalMigration(down2026081901BucketMinimumDurableCopies), - ) -} - -func up2026081901BucketMinimumDurableCopies(ctx context.Context, db bun.IDB) error { - exists, err := columnExists(ctx, db, "buckets", "minimum_durable_copies") - if err != nil || exists { - return err - } - query := "ALTER TABLE buckets ADD COLUMN minimum_durable_copies INTEGER CHECK (minimum_durable_copies IS NULL OR (minimum_durable_copies >= 1 AND minimum_durable_copies <= 8))" - if db.Dialect().Name() == dialect.PG { - query = "ALTER TABLE buckets ADD COLUMN minimum_durable_copies INTEGER CONSTRAINT chk_buckets_minimum_durable_copies CHECK (minimum_durable_copies IS NULL OR (minimum_durable_copies >= 1 AND minimum_durable_copies <= 8))" - } - if _, err := db.ExecContext(ctx, query); err != nil { - return fmt.Errorf("adding buckets.minimum_durable_copies: %w", err) - } - return nil -} - -func down2026081901BucketMinimumDurableCopies(ctx context.Context, db bun.IDB) error { - exists, err := columnExists(ctx, db, "buckets", "minimum_durable_copies") - if err != nil || !exists { - return err - } - if _, err := db.ExecContext(ctx, "ALTER TABLE buckets DROP COLUMN minimum_durable_copies"); err != nil { - return fmt.Errorf("dropping buckets.minimum_durable_copies: %w", err) - } - return nil -} diff --git a/internal/db/migrations/2026082101_storage_data_set_generations.go b/internal/db/migrations/2026082101_storage_data_set_generations.go deleted file mode 100644 index b7ee569..0000000 --- a/internal/db/migrations/2026082101_storage_data_set_generations.go +++ /dev/null @@ -1,427 +0,0 @@ -package migrations - -import ( - "context" - "fmt" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" -) - -func init() { - Migrations.MustRegister( - transactionalMigration(up2026082101StorageDataSetGenerations), - transactionalMigration(down2026082101StorageDataSetGenerations), - ) -} - -// A replica slot may now own several data set generations so an operator can -// replace a provider while the previous generation stays readable. -func up2026082101StorageDataSetGenerations(ctx context.Context, db bun.IDB) error { - done, err := storageDataSetGenerationsReached2026082101(ctx, db, true) - if done { - return nil - } - if err != nil { - if dataErr := assertStorageGenerationPreconditions2026082101(ctx, db); dataErr != nil { - return dataErr - } - return err - } - if err := assertStorageGenerationPreconditions2026082101(ctx, db); err != nil { - return err - } - pg := db.Dialect().Name() == dialect.PG - - isCurrentColumn := "INTEGER NOT NULL DEFAULT 1" - if pg { - isCurrentColumn = "BOOLEAN NOT NULL DEFAULT TRUE" - } - // Neither column carries a CHECK: SQLite cannot add one later and a - // column-level CHECK blocks DROP COLUMN on rollback. The partial unique - // indexes below enforce the invariants on both dialects instead. - statements := []string{ - "ALTER TABLE storage_data_sets ADD COLUMN generation INTEGER NOT NULL DEFAULT 1", - "ALTER TABLE storage_data_sets ADD COLUMN is_current " + isCurrentColumn, - "DROP INDEX IF EXISTS idx_storage_data_sets_bucket_copy_index", - "DROP INDEX IF EXISTS idx_storage_data_sets_bucket_provider", - "DROP INDEX IF EXISTS idx_storage_upload_copies_upload_index", - - `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_data_sets_bucket_copy_current - ON storage_data_sets (bucket_id, copy_index) WHERE is_current`, - `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_data_sets_bucket_copy_generation - ON storage_data_sets (bucket_id, copy_index, generation)`, - // A historical generation never blocks reusing its provider; only a - // live slot does. - `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_data_sets_bucket_provider_current - ON storage_data_sets (bucket_id, provider_id) WHERE is_current`, - - `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_upload_copies_upload_data_set - ON storage_upload_copies (upload_id, storage_data_set_id) WHERE storage_data_set_id IS NOT NULL`, - // NULLs compare distinct in unique indexes on both dialects, so the - // index above would let unbound duplicates accumulate per slot. - `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_upload_copies_upload_slot_unbound - ON storage_upload_copies (upload_id, copy_index) WHERE storage_data_set_id IS NULL`, - `CREATE INDEX IF NOT EXISTS idx_storage_upload_copies_upload_slot - ON storage_upload_copies (upload_id, copy_index)`, - `CREATE INDEX IF NOT EXISTS idx_storage_uploads_bucket_id - ON storage_uploads (bucket_id, id)`, - - storageReplacementsTableSQL2026082101(pg), - storageReplacementItemsTableSQL2026082101(pg), - - `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_replacements_active_source - ON storage_replacements (source_data_set_id) WHERE status NOT IN ('completed', 'superseded')`, - `CREATE INDEX IF NOT EXISTS idx_storage_replacements_bucket_slot - ON storage_replacements (bucket_id, copy_index, id)`, - `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_replacements_bucket_request - ON storage_replacements (bucket_id, client_request_id)`, - `CREATE INDEX IF NOT EXISTS idx_storage_replacement_items_due - ON storage_replacement_items (replacement_id, scheduled_at, id) - WHERE status IN ('pending', 'retrying', 'waiting_source') AND claimed_at IS NULL AND max_retries IS NOT NULL`, - `CREATE INDEX IF NOT EXISTS idx_storage_replacement_items_lease - ON storage_replacement_items (replacement_id, lease_until, id) - WHERE status = 'running' AND max_retries IS NOT NULL`, - `CREATE INDEX IF NOT EXISTS idx_storage_replacement_items_state - ON storage_replacement_items (replacement_id, status, id)`, - `CREATE INDEX IF NOT EXISTS idx_storage_replacement_items_upload_id - ON storage_replacement_items (upload_id)`, - `CREATE INDEX IF NOT EXISTS idx_storage_replacements_dispatch - ON storage_replacements (last_dispatched_at, id) WHERE status = 'migrating'`, - } - for _, query := range statements { - if _, err := db.ExecContext(ctx, query); err != nil { - return fmt.Errorf("adding storage data set generations: %w", err) - } - } - return nil -} - -// Rollback only succeeds while every slot still owns a single generation. Once -// a replacement has produced a second generation the original unique indexes -// cannot be restored, and refusing is the correct outcome. -func down2026082101StorageDataSetGenerations(ctx context.Context, db bun.IDB) error { - done, err := storageDataSetGenerationsReached2026082101(ctx, db, false) - if err != nil || done { - return err - } - if err := assertStorageGenerationRollbackPreconditions2026082101(ctx, db); err != nil { - return err - } - statements := []string{ - "DROP INDEX IF EXISTS idx_storage_replacements_dispatch", - "DROP INDEX IF EXISTS idx_storage_replacement_items_lease", - "DROP INDEX IF EXISTS idx_storage_replacement_items_due", - "DROP INDEX IF EXISTS idx_storage_replacement_items_state", - "DROP INDEX IF EXISTS idx_storage_replacement_items_upload_id", - "DROP INDEX IF EXISTS idx_storage_uploads_bucket_id", - "DROP TABLE IF EXISTS storage_replacement_items", - "DROP TABLE IF EXISTS storage_replacements", - - // SQLite refuses to drop a column named by an index or by a partial - // index predicate, so the indexes go first. - "DROP INDEX IF EXISTS idx_storage_upload_copies_upload_slot", - "DROP INDEX IF EXISTS idx_storage_upload_copies_upload_slot_unbound", - "DROP INDEX IF EXISTS idx_storage_upload_copies_upload_data_set", - "DROP INDEX IF EXISTS idx_storage_data_sets_bucket_provider_current", - "DROP INDEX IF EXISTS idx_storage_data_sets_bucket_copy_generation", - "DROP INDEX IF EXISTS idx_storage_data_sets_bucket_copy_current", - - "ALTER TABLE storage_data_sets DROP COLUMN generation", - "ALTER TABLE storage_data_sets DROP COLUMN is_current", - - `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_data_sets_bucket_copy_index - ON storage_data_sets (bucket_id, copy_index)`, - `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_data_sets_bucket_provider - ON storage_data_sets (bucket_id, provider_id)`, - `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_upload_copies_upload_index - ON storage_upload_copies (upload_id, copy_index)`, - } - for _, query := range statements { - if _, err := db.ExecContext(ctx, query); err != nil { - return fmt.Errorf("removing storage data set generations: %w", err) - } - } - return nil -} - -// The new unique indexes would fail mid-migration on a database that violated -// an invariant the old schema never enforced. Reporting the offending rows up -// front keeps the failure actionable. -func assertStorageGenerationPreconditions2026082101(ctx context.Context, db bun.IDB) error { - checks := []struct { - description string - query string - }{ - { - description: "storage upload copies sharing one data set", - query: `SELECT COUNT(*) FROM ( - SELECT upload_id, storage_data_set_id FROM storage_upload_copies - WHERE storage_data_set_id IS NOT NULL - GROUP BY upload_id, storage_data_set_id HAVING COUNT(*) > 1 - ) AS duplicates`, - }, - { - description: "unbound storage upload copies sharing one replica slot", - query: `SELECT COUNT(*) FROM ( - SELECT upload_id, copy_index FROM storage_upload_copies - WHERE storage_data_set_id IS NULL - GROUP BY upload_id, copy_index HAVING COUNT(*) > 1 - ) AS duplicates`, - }, - { - description: "storage data sets sharing one replica slot", - query: `SELECT COUNT(*) FROM ( - SELECT bucket_id, copy_index FROM storage_data_sets - GROUP BY bucket_id, copy_index HAVING COUNT(*) > 1 - ) AS duplicates`, - }, - } - for _, check := range checks { - var count int - if err := db.NewRaw(check.query).Scan(ctx, &count); err != nil { - return fmt.Errorf("checking %s: %w", check.description, err) - } - if count > 0 { - return fmt.Errorf("cannot add storage data set generations: found %d %s", count, check.description) - } - } - return nil -} - -func assertStorageGenerationRollbackPreconditions2026082101(ctx context.Context, db bun.IDB) error { - checks := []struct { - description string - query string - }{ - { - description: "duplicate (bucket_id, copy_index) groups in storage_data_sets", - query: `SELECT COUNT(*) FROM ( - SELECT bucket_id, copy_index FROM storage_data_sets - GROUP BY bucket_id, copy_index HAVING COUNT(*) > 1 - ) AS duplicates`, - }, - { - description: "duplicate (bucket_id, provider_id) groups in storage_data_sets", - query: `SELECT COUNT(*) FROM ( - SELECT bucket_id, provider_id FROM storage_data_sets - GROUP BY bucket_id, provider_id HAVING COUNT(*) > 1 - ) AS duplicates`, - }, - { - description: "duplicate (upload_id, copy_index) groups in storage_upload_copies", - query: `SELECT COUNT(*) FROM ( - SELECT upload_id, copy_index FROM storage_upload_copies - GROUP BY upload_id, copy_index HAVING COUNT(*) > 1 - ) AS duplicates`, - }, - } - for _, check := range checks { - var count int - if err := db.NewRaw(check.query).Scan(ctx, &count); err != nil { - return fmt.Errorf("checking %s: %w", check.description, err) - } - if count > 0 { - return fmt.Errorf("cannot remove storage data set generations: found %d %s", count, check.description) - } - } - return nil -} - -func storageDataSetGenerationsReached2026082101( - ctx context.Context, - db bun.IDB, - wantMigrated bool, -) (bool, error) { - generation, err := columnExists(ctx, db, "storage_data_sets", "generation") - if err != nil { - return false, fmt.Errorf("checking storage_data_sets.generation: %w", err) - } - isCurrent, err := columnExists(ctx, db, "storage_data_sets", "is_current") - if err != nil { - return false, fmt.Errorf("checking storage_data_sets.is_current: %w", err) - } - legacyIndexes := []string{ - "idx_storage_data_sets_bucket_copy_index", - "idx_storage_data_sets_bucket_provider", - "idx_storage_upload_copies_upload_index", - } - migratedIndexes := []string{ - "idx_storage_data_sets_bucket_copy_current", - "idx_storage_data_sets_bucket_copy_generation", - "idx_storage_data_sets_bucket_provider_current", - "idx_storage_upload_copies_upload_data_set", - "idx_storage_upload_copies_upload_slot_unbound", - "idx_storage_upload_copies_upload_slot", - "idx_storage_uploads_bucket_id", - "idx_storage_replacements_active_source", - "idx_storage_replacements_bucket_slot", - "idx_storage_replacements_bucket_request", - "idx_storage_replacement_items_due", - "idx_storage_replacement_items_lease", - "idx_storage_replacement_items_state", - "idx_storage_replacement_items_upload_id", - "idx_storage_replacements_dispatch", - } - legacyIndexCount, err := existingIndexes2026082101(ctx, db, legacyIndexes) - if err != nil { - return false, err - } - migratedIndexCount, err := existingIndexes2026082101(ctx, db, migratedIndexes) - if err != nil { - return false, err - } - replacementTables := 0 - for _, table := range []string{"storage_replacements", "storage_replacement_items"} { - exists, err := tableExists(ctx, db, table) - if err != nil { - return false, fmt.Errorf("checking %s: %w", table, err) - } - if exists { - replacementTables++ - } - } - replacementColumns := []struct { - table string - column string - }{ - {"storage_replacements", "state_version"}, - {"storage_replacements", "last_dispatched_at"}, - {"storage_replacement_items", "scheduled_at"}, - {"storage_replacement_items", "retry_count"}, - {"storage_replacement_items", "max_retries"}, - {"storage_replacement_items", "claimed_at"}, - {"storage_replacement_items", "lease_until"}, - } - replacementColumnCount := 0 - for _, item := range replacementColumns { - exists, err := columnExists(ctx, db, item.table, item.column) - if err != nil { - return false, fmt.Errorf("checking %s.%s: %w", item.table, item.column, err) - } - if exists { - replacementColumnCount++ - } - } - - legacy := !generation && !isCurrent && legacyIndexCount == len(legacyIndexes) && migratedIndexCount == 0 && replacementTables == 0 && replacementColumnCount == 0 - migrated := generation && isCurrent && legacyIndexCount == 0 && migratedIndexCount == len(migratedIndexes) && replacementTables == 2 && replacementColumnCount == len(replacementColumns) - if (wantMigrated && migrated) || (!wantMigrated && legacy) { - return true, nil - } - if (wantMigrated && legacy) || (!wantMigrated && migrated) { - return false, nil - } - return false, fmt.Errorf( - "2026082101_storage_data_set_generations has partial schema state: generation=%t, is_current=%t, legacy_indexes=%d/%d, migrated_indexes=%d/%d, replacement_tables=%d/2, replacement_columns=%d/%d", - generation, - isCurrent, - legacyIndexCount, - len(legacyIndexes), - migratedIndexCount, - len(migratedIndexes), - replacementTables, - replacementColumnCount, - len(replacementColumns), - ) -} - -func existingIndexes2026082101(ctx context.Context, db bun.IDB, names []string) (int, error) { - count := 0 - for _, name := range names { - exists, err := indexExists(ctx, db, name) - if err != nil { - return 0, fmt.Errorf("checking %s: %w", name, err) - } - if exists { - count++ - } - } - return count, nil -} - -func storageReplacementsTableSQL2026082101(pg bool) string { - identity := "id INTEGER PRIMARY KEY AUTOINCREMENT" - reference := "INTEGER" - timestamp := "TIMESTAMP" - boolean := "INTEGER NOT NULL DEFAULT 0" - if pg { - identity = "id BIGSERIAL PRIMARY KEY" - reference = "BIGINT" - timestamp = "TIMESTAMPTZ" - boolean = "BOOLEAN NOT NULL DEFAULT FALSE" - } - return fmt.Sprintf(`CREATE TABLE IF NOT EXISTS storage_replacements ( - %[1]s, - bucket_id %[2]s NOT NULL REFERENCES buckets (id) ON UPDATE CASCADE ON DELETE RESTRICT, - copy_index INTEGER NOT NULL, - source_data_set_id %[2]s NOT NULL REFERENCES storage_data_sets (id) ON UPDATE CASCADE ON DELETE RESTRICT, - target_data_set_id %[2]s NOT NULL REFERENCES storage_data_sets (id) ON UPDATE CASCADE ON DELETE RESTRICT, - selection_mode TEXT NOT NULL, - requested_provider_id TEXT, - client_request_id TEXT NOT NULL, - status TEXT NOT NULL, - wait_reason TEXT, - failure_reason TEXT, - last_error TEXT, - items_total INTEGER NOT NULL DEFAULT 0, - items_copied INTEGER NOT NULL DEFAULT 0, - seed_cursor_upload_id %[2]s NOT NULL DEFAULT 0, - seeding_complete %[4]s, - state_version %[2]s NOT NULL DEFAULT 1, - last_dispatched_at %[3]s, - termination_tx_hash TEXT, - termination_epoch %[2]s, - termination_observed_at %[3]s, - abandoned_termination_tx_hash TEXT, - abandoned_termination_epoch %[2]s, - abandoned_termination_observed_at %[3]s, - superseded_by_id %[2]s REFERENCES storage_replacements (id) ON UPDATE CASCADE ON DELETE SET NULL, - confirmed_at %[3]s NOT NULL DEFAULT CURRENT_TIMESTAMP, - created_at %[3]s NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at %[3]s NOT NULL DEFAULT CURRENT_TIMESTAMP, - CONSTRAINT chk_storage_replacements_copy_index CHECK (copy_index >= 0), - CONSTRAINT chk_storage_replacements_selection_mode CHECK (selection_mode IN ('automatic', 'manual')), - CONSTRAINT chk_storage_replacements_status CHECK (status IN ('preparing_target', 'migrating', 'waiting', 'retiring', 'cleanup_attention', 'failed', 'completed', 'superseded')), - CONSTRAINT chk_storage_replacements_wait_reason CHECK (wait_reason IS NULL OR wait_reason IN ('readable_source', 'target', 'target_creating', 'target_writable', 'funding', 'provider', 'termination_epoch', 'source_writes', 'coverage')), - CONSTRAINT chk_storage_replacements_failure_reason CHECK (failure_reason IS NULL OR failure_reason IN ('target_in_use')), - CONSTRAINT chk_storage_replacements_client_request_id CHECK (length(client_request_id) BETWEEN 1 AND 128), - CONSTRAINT chk_storage_replacements_distinct_data_sets CHECK (source_data_set_id <> target_data_set_id), - CONSTRAINT chk_storage_replacements_items CHECK (items_total >= 0 AND items_copied >= 0 AND items_copied <= items_total) - )`, identity, reference, timestamp, boolean) -} - -func storageReplacementItemsTableSQL2026082101(pg bool) string { - identity := "id INTEGER PRIMARY KEY AUTOINCREMENT" - reference := "INTEGER" - timestamp := "TIMESTAMP" - if pg { - identity = "id BIGSERIAL PRIMARY KEY" - reference = "BIGINT" - timestamp = "TIMESTAMPTZ" - } - return fmt.Sprintf(`CREATE TABLE IF NOT EXISTS storage_replacement_items ( - %[1]s, - replacement_id %[2]s NOT NULL REFERENCES storage_replacements (id) ON UPDATE CASCADE ON DELETE CASCADE, - upload_id %[2]s NOT NULL REFERENCES storage_uploads (id) ON UPDATE CASCADE ON DELETE RESTRICT, - target_copy_id %[2]s REFERENCES storage_upload_copies (id) ON UPDATE CASCADE ON DELETE SET NULL, - status TEXT NOT NULL, - attempts INTEGER NOT NULL DEFAULT 0, - scheduled_at %[3]s NOT NULL DEFAULT CURRENT_TIMESTAMP, - retry_count INTEGER NOT NULL DEFAULT 0, - max_retries INTEGER, - claimed_at %[3]s, - lease_until %[3]s, - last_error TEXT, - created_at %[3]s NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at %[3]s NOT NULL DEFAULT CURRENT_TIMESTAMP, - CONSTRAINT chk_storage_replacement_items_status CHECK (status IN ('pending', 'running', 'retrying', 'waiting_source', 'copied', 'cancelled', 'failed')), - CONSTRAINT chk_storage_replacement_items_attempts CHECK (attempts >= 0), - CONSTRAINT chk_storage_replacement_items_retry CHECK (retry_count >= 0 AND (max_retries IS NULL OR max_retries >= 0)), - CONSTRAINT chk_storage_replacement_items_claim CHECK ( - (status = 'running' AND claimed_at IS NOT NULL AND lease_until IS NOT NULL) - OR (status <> 'running' AND claimed_at IS NULL AND lease_until IS NULL) - ), - CONSTRAINT uq_storage_replacement_items_upload UNIQUE (replacement_id, upload_id) - )`, identity, reference, timestamp) -} diff --git a/internal/db/migrations/2026083001_durable_storage_commits.go b/internal/db/migrations/2026083001_durable_storage_commits.go deleted file mode 100644 index f97db68..0000000 --- a/internal/db/migrations/2026083001_durable_storage_commits.go +++ /dev/null @@ -1,206 +0,0 @@ -package migrations - -import ( - "context" - "fmt" - "slices" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" -) - -const durableStorageCommitMigration2026083001 = "2026083001_durable_storage_commits" - -var durableStorageCommitColumns2026083001 = []string{ - "commit_ready_at", - "commit_attempt_id", - "commit_attempted_at", - "commit_submission_json", - "commit_confirmed_transaction_id", - "commit_attention_code", - "commit_attention_at", -} - -var durableStorageCommitIndexes2026083001 = []string{ - "idx_storage_upload_copies_commit_attempt_data_set", - "idx_storage_upload_copies_commit_ready", -} - -func init() { - Migrations.MustRegister( - transactionalMigration(up2026083001DurableStorageCommits), - transactionalMigration(down2026083001DurableStorageCommits), - ) -} - -func up2026083001DurableStorageCommits(ctx context.Context, db bun.IDB) error { - columns, indexes, err := durableStorageCommitSchemaCounts2026083001(ctx, db) - if err != nil { - return err - } - if err := assertDurableStorageCommitUpgradePreconditions2026083001(ctx, db); err != nil { - return err - } - if columns == len(durableStorageCommitColumns2026083001) && indexes == len(durableStorageCommitIndexes2026083001) { - return nil - } - if columns != 0 || indexes != 0 { - return fmt.Errorf( - "%s has partial schema state: columns=%d/%d indexes=%d/%d", - durableStorageCommitMigration2026083001, - columns, - len(durableStorageCommitColumns2026083001), - indexes, - len(durableStorageCommitIndexes2026083001), - ) - } - timestampType := "TIMESTAMP" - if db.Dialect().Name() == dialect.PG { - timestampType = "TIMESTAMPTZ" - } - statements := []string{ - "ALTER TABLE storage_upload_copies ADD COLUMN commit_ready_at " + timestampType, - "ALTER TABLE storage_upload_copies ADD COLUMN commit_attempt_id TEXT", - "ALTER TABLE storage_upload_copies ADD COLUMN commit_attempted_at " + timestampType, - "ALTER TABLE storage_upload_copies ADD COLUMN commit_submission_json TEXT", - "ALTER TABLE storage_upload_copies ADD COLUMN commit_confirmed_transaction_id TEXT", - "ALTER TABLE storage_upload_copies ADD COLUMN commit_attention_code TEXT", - "ALTER TABLE storage_upload_copies ADD COLUMN commit_attention_at " + timestampType, - `CREATE INDEX idx_storage_upload_copies_commit_attempt_data_set - ON storage_upload_copies (storage_data_set_id, commit_attempt_id) - WHERE commit_attempt_id IS NOT NULL`, - `CREATE INDEX idx_storage_upload_copies_commit_ready - ON storage_upload_copies (storage_data_set_id, commit_ready_at, id) - WHERE status = 'piece_ready' AND commit_attempt_id IS NULL AND commit_ready_at IS NOT NULL`, - } - for _, statement := range statements { - if _, err := db.ExecContext(ctx, statement); err != nil { - return fmt.Errorf("applying %s: %w", durableStorageCommitMigration2026083001, err) - } - } - return nil -} - -func down2026083001DurableStorageCommits(ctx context.Context, db bun.IDB) error { - columns, indexes, err := durableStorageCommitSchemaCounts2026083001(ctx, db) - if err != nil { - return err - } - if columns == 0 && indexes == 0 { - return nil - } - if columns != len(durableStorageCommitColumns2026083001) || indexes != len(durableStorageCommitIndexes2026083001) { - return fmt.Errorf( - "%s has partial schema state: columns=%d/%d indexes=%d/%d", - durableStorageCommitMigration2026083001, - columns, - len(durableStorageCommitColumns2026083001), - indexes, - len(durableStorageCommitIndexes2026083001), - ) - } - var active int - if err := db.NewRaw(`SELECT COUNT(*) FROM storage_upload_copies - WHERE commit_attempt_id IS NOT NULL OR commit_submission_json IS NOT NULL`).Scan(ctx, &active); err != nil { - return fmt.Errorf("checking %s rollback preconditions: %w", durableStorageCommitMigration2026083001, err) - } - if active > 0 { - return fmt.Errorf( - "cannot rollback %s while %d durable commit attempts or submissions remain", - durableStorageCommitMigration2026083001, - active, - ) - } - for _, index := range slices.Backward(durableStorageCommitIndexes2026083001) { - if _, err := db.ExecContext(ctx, "DROP INDEX IF EXISTS "+index); err != nil { - return fmt.Errorf("dropping %s index %s: %w", durableStorageCommitMigration2026083001, index, err) - } - } - for _, column := range slices.Backward(durableStorageCommitColumns2026083001) { - if _, err := db.ExecContext(ctx, "ALTER TABLE storage_upload_copies DROP COLUMN "+column); err != nil { - return fmt.Errorf("dropping %s column %s: %w", durableStorageCommitMigration2026083001, column, err) - } - } - return nil -} - -// strandedCommitTransactionFilter2026083001 matches copies that kept a commit -// transaction after leaving 'committing'. Older code could send a submitted copy -// back to 'piece_ready' without clearing its transaction, and from there on to -// 'failed', leaving evidence of a submission whose outcome was never resolved. -const strandedCommitTransactionFilter2026083001 = " FROM storage_upload_copies" + - " WHERE status NOT IN ('committed', 'committing')" + - " AND commit_transaction_id IS NOT NULL AND commit_transaction_id <> ''" - -func assertDurableStorageCommitUpgradePreconditions2026083001(ctx context.Context, db bun.IDB) error { - attemptColumnExists, err := columnExists(ctx, db, "storage_upload_copies", "commit_attempt_id") - if err != nil { - return fmt.Errorf("checking durable attempt column before %s: %w", durableStorageCommitMigration2026083001, err) - } - // On a re-entrant run the durable columns already exist, so exclude copies the - // new model owns and judge only the rows the old code could have left behind. - legacyOnly := "" - if attemptColumnExists { - legacyOnly = " AND (commit_attempt_id IS NULL OR commit_attempt_id = '')" - } - - var committing int - if err := db.NewRaw( - "SELECT COUNT(*) FROM storage_upload_copies WHERE status = 'committing'"+legacyOnly, - ).Scan(ctx, &committing); err != nil { - return fmt.Errorf("checking existing committing copies before %s: %w", durableStorageCommitMigration2026083001, err) - } - if committing > 0 { - return fmt.Errorf( - "cannot apply %s while %d legacy committing copies remain; drain storage commit work before upgrading", - durableStorageCommitMigration2026083001, - committing, - ) - } - - // The durable model cannot tell this evidence apart from a copy that never - // submitted anything, so a later permanent delete would discard a piece the - // provider may still be paid to keep. Refuse instead of guessing. - var stranded int - if err := db.NewRaw( - "SELECT COUNT(*)"+strandedCommitTransactionFilter2026083001+legacyOnly, - ).Scan(ctx, &stranded); err != nil { - return fmt.Errorf("checking stranded commit transactions before %s: %w", durableStorageCommitMigration2026083001, err) - } - if stranded > 0 { - return fmt.Errorf( - "cannot apply %s while %d copies keep a commit transaction outside 'committed' and 'committing'; "+ - "review them with [SELECT id, upload_id, copy_index, status, commit_transaction_id%s%s], "+ - "resolve each transaction and clear commit_transaction_id, or reset the development database", - durableStorageCommitMigration2026083001, - stranded, - strandedCommitTransactionFilter2026083001, - legacyOnly, - ) - } - return nil -} - -func durableStorageCommitSchemaCounts2026083001(ctx context.Context, db bun.IDB) (int, int, error) { - columns := 0 - for _, column := range durableStorageCommitColumns2026083001 { - exists, err := columnExists(ctx, db, "storage_upload_copies", column) - if err != nil { - return 0, 0, fmt.Errorf("checking %s column %s: %w", durableStorageCommitMigration2026083001, column, err) - } - if exists { - columns++ - } - } - indexes := 0 - for _, index := range durableStorageCommitIndexes2026083001 { - exists, err := indexExists(ctx, db, index) - if err != nil { - return 0, 0, fmt.Errorf("checking %s index %s: %w", durableStorageCommitMigration2026083001, index, err) - } - if exists { - indexes++ - } - } - return columns, indexes, nil -} diff --git a/internal/db/migrations/2026090101_initial_schema.go b/internal/db/migrations/2026090101_initial_schema.go new file mode 100644 index 0000000..316c0f7 --- /dev/null +++ b/internal/db/migrations/2026090101_initial_schema.go @@ -0,0 +1,151 @@ +package migrations + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/uptrace/bun" +) + +func init() { + Migrations.MustRegister( + transactionalMigration(up2026090101InitialSchema), + func(context.Context, *bun.DB) error { + return errors.New("the initial schema cannot be rolled back; create a new empty database") + }, + ) +} + +func up2026090101InitialSchema(ctx context.Context, db bun.IDB) error { + complete, err := initialSchemaPostStateComplete(ctx, db) + if err != nil { + return fmt.Errorf("checking initial schema post-state: %w", err) + } + if complete { + return nil + } + count, err := applicationTableCount(ctx, db) + if err != nil { + return fmt.Errorf("checking initial schema pre-state: %w", err) + } + if count != 0 { + return incompatibleDatabaseError() + } + steps := []struct { + name string + up migrationBody + }{ + {"tasks", createTaskSchema}, + {"identity and object roots", createCoreRootSchema}, + {"storage ledgers", createStorageSchema}, + {"object lifecycle", createObjectLifecycleSchema}, + {"wallet operations", createWalletSchema}, + {"observability", createObservabilitySchema}, + {"PostgreSQL prefix indexes", createPostgresPrefixIndexes}, + } + for _, step := range steps { + if err := step.up(ctx, db); err != nil { + return fmt.Errorf("creating %s: %w", step.name, err) + } + } + return nil +} + +// taskPayload2026090101 holds the JSON a task carries. It lives beside the task +// rather than in it because lease renewal updates an indexed column on every +// heartbeat, and PostgreSQL copies the whole row — JSON included — each time. +type taskPayload2026090101 struct { + bun.BaseModel `bun:"table:task_payloads"` + + TaskID int64 `bun:",pk"` + Input json.RawMessage `bun:"input_json,type:jsonb,notnull"` + Checkpoint json.RawMessage `bun:"checkpoint_json,type:jsonb"` +} + +type task2026090101 struct { + bun.BaseModel `bun:"table:tasks"` + + ID int64 `bun:",pk,autoincrement,identity"` + Type string `bun:"type:text,notnull"` + IdempotencyKey string `bun:"type:text,notnull"` + InputVersion int `bun:"type:integer,notnull"` + InputHash string `bun:"type:text,notnull"` + SubjectType *string `bun:"type:text"` + SubjectKey *string `bun:"type:text"` + + Status string `bun:"type:text,notnull,default:'pending'"` + ResumeMode string `bun:"type:text,notnull,default:'execute'"` + AvailableAt time.Time `bun:",notnull"` + WaitReason *string `bun:"type:text"` + RetryCount int `bun:"type:integer,notnull,default:0"` + RetryLimit *int `bun:"type:integer"` + FailureReason *string `bun:"type:text"` + LastError *string `bun:"type:text"` + StatusMessage *string `bun:"type:text"` + + CancellationRequestedAt *time.Time + CancellationReason *string `bun:"type:text"` + ClaimGeneration int64 `bun:",notnull,default:0"` + ClaimedAt *time.Time + LeaseUntil *time.Time + StartedAt *time.Time + FinishedAt *time.Time + AcknowledgedAt *time.Time + RetentionUntil *time.Time + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +func createTaskSchema(ctx context.Context, db bun.IDB) error { + if err := createInitialTable(ctx, db, initialTableSpec{ + name: "tasks", + model: (*task2026090101)(nil), + constraints: []string{ + "CONSTRAINT uq_tasks_type_key UNIQUE (type, idempotency_key)", + "CONSTRAINT chk_tasks_identity CHECK (type <> '' AND idempotency_key <> '' AND input_hash <> '' AND input_version >= 1)", + "CONSTRAINT chk_tasks_status CHECK (status IN ('pending', 'running', 'completed', 'failed', 'cancelled'))", + "CONSTRAINT chk_tasks_resume_mode CHECK (resume_mode IN ('execute', 'recover'))", + "CONSTRAINT chk_tasks_subject CHECK ((subject_type IS NULL AND subject_key IS NULL) OR (subject_type IS NOT NULL AND subject_type <> '' AND subject_key IS NOT NULL AND subject_key <> ''))", + "CONSTRAINT chk_tasks_reason_codes CHECK ((wait_reason IS NULL OR wait_reason <> '') AND (failure_reason IS NULL OR failure_reason <> ''))", + "CONSTRAINT chk_tasks_retry CHECK (retry_count >= 0 AND (retry_limit IS NULL OR (retry_limit >= 0 AND retry_count <= retry_limit)))", + "CONSTRAINT chk_tasks_generation CHECK (claim_generation >= 0)", + `CONSTRAINT chk_tasks_claim CHECK ( + (status = 'running' AND claimed_at IS NOT NULL AND lease_until IS NOT NULL AND claim_generation > 0) + OR (status <> 'running' AND claimed_at IS NULL AND lease_until IS NULL) + )`, + `CONSTRAINT chk_tasks_finished CHECK ( + (status IN ('completed', 'failed', 'cancelled') AND finished_at IS NOT NULL) + OR (status IN ('pending', 'running') AND finished_at IS NULL) + )`, + `CONSTRAINT chk_tasks_retention CHECK ( + (status IN ('completed', 'cancelled') AND retention_until IS NOT NULL) + OR (status = 'failed' AND ((acknowledged_at IS NULL AND retention_until IS NULL) OR (acknowledged_at IS NOT NULL AND retention_until IS NOT NULL))) + OR (status IN ('pending', 'running') AND acknowledged_at IS NULL AND retention_until IS NULL) + )`, + }, + }); err != nil { + return err + } + if err := createInitialTable(ctx, db, initialTableSpec{ + name: "task_payloads", + model: (*taskPayload2026090101)(nil), + jsonColumns: initialJSONColumns("task_payloads"), + foreignKeys: []string{ + "(task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE CASCADE", + }, + }); err != nil { + return err + } + return createInitialIndexes(ctx, db, + initialIndexSpec{name: "idx_tasks_pending", table: "tasks", columns: []string{"available_at", "id"}, where: "status = 'pending'"}, + initialIndexSpec{name: "idx_tasks_recovery", table: "tasks", columns: []string{"lease_until", "id"}, where: "status = 'running'"}, + initialIndexSpec{name: "idx_tasks_gc", table: "tasks", columns: []string{"retention_until", "id"}, where: "retention_until IS NOT NULL"}, + initialIndexSpec{name: "idx_tasks_type_status_id", table: "tasks", columns: []string{"type", "status", "id"}}, + initialIndexSpec{name: "idx_tasks_type_id", table: "tasks", columns: []string{"type", "id"}}, + initialIndexSpec{name: "idx_tasks_status_id", table: "tasks", columns: []string{"status", "id"}}, + initialIndexSpec{name: "idx_tasks_subject", table: "tasks", columns: []string{"subject_type", "subject_key", "id"}}, + ) +} diff --git a/internal/db/migrations/2026999999_transaction_semantics_test.go b/internal/db/migrations/2026999999_transaction_semantics_test.go index cafa98a..fbfc288 100644 --- a/internal/db/migrations/2026999999_transaction_semantics_test.go +++ b/internal/db/migrations/2026999999_transaction_semantics_test.go @@ -82,6 +82,43 @@ func TestMigrationRepairsMarkerForCompletePostState(t *testing.T) { testMigrationDialects(t, testMigrationRepairsMarkerForCompletePostState) } +func TestInitialBaselineRepairsMissingMarkerOnlyForCompletePostState(t *testing.T) { + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + ctx := t.Context() + if err := runMigrationBody(ctx, db, up2026090101InitialSchema); err != nil { + t.Fatalf("simulate committed baseline without marker: %v", err) + } + migrator := NewMigrator(db) + if err := migrator.Init(ctx); err != nil { + t.Fatalf("initialize migrator: %v", err) + } + if err := ValidateTarget(ctx, db); err != nil { + t.Fatalf("validate complete baseline without marker: %v", err) + } + if _, err := migrator.Migrate(ctx); err != nil { + t.Fatalf("repair baseline marker: %v", err) + } + assertAppliedMigrationCount(t, ctx, migrator, 1) + }) +} + +func TestInitialBaselineRejectsPartialPostStateWithEmptyMarker(t *testing.T) { + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + ctx := t.Context() + migrator := NewMigrator(db) + if err := migrator.Init(ctx); err != nil { + t.Fatalf("initialize migrator: %v", err) + } + if _, err := db.ExecContext(ctx, "CREATE TABLE tasks (id INTEGER PRIMARY KEY)"); err != nil { + t.Fatalf("create partial baseline: %v", err) + } + if err := ValidateTarget(ctx, db); !errors.Is(err, ErrIncompatibleDatabase) { + t.Fatalf("validate partial baseline = %v, want ErrIncompatibleDatabase", err) + } + assertAppliedMigrationCount(t, ctx, migrator, 0) + }) +} + func testMigrationRepairsMarkerForCompletePostState(t *testing.T, db *bun.DB) { ctx := context.Background() executions := 0 diff --git a/internal/db/migrations/bucket_minimum_durable_copies_test.go b/internal/db/migrations/bucket_minimum_durable_copies_test.go deleted file mode 100644 index 7bda13b..0000000 --- a/internal/db/migrations/bucket_minimum_durable_copies_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package migrations - -import ( - "context" - "database/sql" - "testing" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect/sqlitedialect" - - _ "modernc.org/sqlite" -) - -func TestBucketMinimumDurableCopiesMigrationAddsNullableBoundedColumn(t *testing.T) { - sqldb, err := sql.Open("sqlite", "file:bucket_minimum_durable_copies?mode=memory&cache=shared") - if err != nil { - t.Fatalf("open sqlite: %v", err) - } - sqldb.SetMaxOpenConns(1) - db := bun.NewDB(sqldb, sqlitedialect.New()) - t.Cleanup(func() { _ = db.Close() }) - ctx := context.Background() - - if _, err := db.ExecContext(ctx, `CREATE TABLE buckets (id INTEGER PRIMARY KEY, name TEXT NOT NULL)`); err != nil { - t.Fatalf("create buckets: %v", err) - } - if _, err := db.ExecContext(ctx, `INSERT INTO buckets (id, name) VALUES (1, 'existing')`); err != nil { - t.Fatalf("insert existing bucket: %v", err) - } - if err := up2026081901BucketMinimumDurableCopies(ctx, db); err != nil { - t.Fatalf("up migration: %v", err) - } - if !sqliteColumnExists(t, db, "buckets", "minimum_durable_copies") { - t.Fatal("buckets.minimum_durable_copies column missing") - } - var minimum sql.NullInt64 - if err := db.NewRaw(`SELECT minimum_durable_copies FROM buckets WHERE id = 1`).Scan(ctx, &minimum); err != nil { - t.Fatalf("select existing bucket minimum: %v", err) - } - if minimum.Valid { - t.Fatalf("existing bucket minimum = %d, want NULL", minimum.Int64) - } - for _, invalid := range []int{0, 9} { - if _, err := db.ExecContext(ctx, `UPDATE buckets SET minimum_durable_copies = ? WHERE id = 1`, invalid); err == nil { - t.Fatalf("minimum_durable_copies=%d accepted, want check failure", invalid) - } - } - if _, err := db.ExecContext(ctx, `UPDATE buckets SET minimum_durable_copies = 2 WHERE id = 1`); err != nil { - t.Fatalf("set valid minimum: %v", err) - } - if err := down2026081901BucketMinimumDurableCopies(ctx, db); err != nil { - t.Fatalf("down migration: %v", err) - } - if sqliteColumnExists(t, db, "buckets", "minimum_durable_copies") { - t.Fatal("buckets.minimum_durable_copies still exists after down migration") - } -} diff --git a/internal/db/migrations/cache_lru_test.go b/internal/db/migrations/cache_lru_test.go deleted file mode 100644 index b84e6c6..0000000 --- a/internal/db/migrations/cache_lru_test.go +++ /dev/null @@ -1,122 +0,0 @@ -package migrations - -import ( - "context" - "database/sql" - "strings" - "testing" - "time" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect/sqlitedialect" - - _ "modernc.org/sqlite" -) - -func TestCacheLRUMigrationAddsAccessTracking(t *testing.T) { - sqldb, err := sql.Open("sqlite", "file:cache_lru_migration?mode=memory&cache=shared") - if err != nil { - t.Fatalf("open sqlite: %v", err) - } - sqldb.SetMaxOpenConns(1) - db := bun.NewDB(sqldb, sqlitedialect.New()) - t.Cleanup(func() { _ = db.Close() }) - ctx := context.Background() - - if _, err := db.ExecContext(ctx, `CREATE TABLE object_versions ( - version_id TEXT PRIMARY KEY, - in_cache BOOLEAN NOT NULL DEFAULT TRUE, - state TEXT NOT NULL, - created_at TIMESTAMP NOT NULL - )`); err != nil { - t.Fatalf("create object_versions: %v", err) - } - createdAt := time.Date(2026, time.July, 1, 12, 0, 0, 0, time.UTC) - if _, err := db.ExecContext( - ctx, - `INSERT INTO object_versions (version_id, in_cache, state, created_at) VALUES (?, TRUE, 'stored', ?)`, - "version-before-lru-migration", - createdAt, - ); err != nil { - t.Fatalf("insert pre-migration object version: %v", err) - } - - if err := up2026072801CacheLRU(ctx, db); err != nil { - t.Fatalf("up migration: %v", err) - } - if !sqliteColumnExists(t, db, "object_versions", "cache_accessed_at") { - t.Fatal("object_versions.cache_accessed_at column missing") - } - if !sqliteIndexExists(t, db, "idx_object_versions_cache_lru") { - t.Fatal("idx_object_versions_cache_lru index missing") - } - var accessedAt time.Time - if err := db.NewRaw( - `SELECT cache_accessed_at FROM object_versions WHERE version_id = ?`, - "version-before-lru-migration", - ).Scan(ctx, &accessedAt); err != nil { - t.Fatalf("select initialized cache access time: %v", err) - } - if !accessedAt.Equal(createdAt) { - t.Fatalf("initialized cache access time = %v, want created_at %v", accessedAt, createdAt) - } - - rows, err := db.QueryContext(ctx, `PRAGMA index_info('idx_object_versions_cache_lru')`) - if err != nil { - t.Fatalf("inspect LRU index: %v", err) - } - var indexColumns []string - for rows.Next() { - var sequence, columnID int - var name string - if err := rows.Scan(&sequence, &columnID, &name); err != nil { - _ = rows.Close() - t.Fatalf("scan LRU index column: %v", err) - } - indexColumns = append(indexColumns, name) - } - if err := rows.Close(); err != nil { - t.Fatalf("close LRU index rows: %v", err) - } - wantColumns := []string{"in_cache", "cache_accessed_at", "created_at", "version_id"} - if strings.Join(indexColumns, ",") != strings.Join(wantColumns, ",") { - t.Fatalf("LRU index columns = %v, want %v", indexColumns, wantColumns) - } - - planRows, err := db.QueryContext(ctx, `EXPLAIN QUERY PLAN - SELECT version_id - FROM object_versions - WHERE in_cache = TRUE - AND state IN ('stored', 'cache_evicted') - AND cache_accessed_at IS NOT NULL - ORDER BY cache_accessed_at, created_at, version_id - LIMIT 100`) - if err != nil { - t.Fatalf("explain LRU candidate order: %v", err) - } - for planRows.Next() { - var id, parent, unused int - var detail string - if err := planRows.Scan(&id, &parent, &unused, &detail); err != nil { - _ = planRows.Close() - t.Fatalf("scan LRU query plan: %v", err) - } - if strings.Contains(detail, "USE TEMP B-TREE") { - _ = planRows.Close() - t.Fatalf("LRU candidate order uses a temporary sort: %s", detail) - } - } - if err := planRows.Close(); err != nil { - t.Fatalf("close LRU query plan rows: %v", err) - } - - if err := down2026072801CacheLRU(ctx, db); err != nil { - t.Fatalf("down migration: %v", err) - } - if sqliteColumnExists(t, db, "object_versions", "cache_accessed_at") { - t.Fatal("object_versions.cache_accessed_at still exists after down migration") - } - if sqliteIndexExists(t, db, "idx_object_versions_cache_lru") { - t.Fatal("idx_object_versions_cache_lru still exists after down migration") - } -} diff --git a/internal/db/migrations/durable_storage_commits_test.go b/internal/db/migrations/durable_storage_commits_test.go deleted file mode 100644 index 2da982e..0000000 --- a/internal/db/migrations/durable_storage_commits_test.go +++ /dev/null @@ -1,225 +0,0 @@ -package migrations - -import ( - "context" - "strings" - "testing" - "time" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" -) - -func TestDurableStorageCommitsMigration(t *testing.T) { - testMigrationDialects(t, func(t *testing.T, db *bun.DB) { - createDurableStorageCommitLegacyTable(t, db) - if err := runMigrationBody(t.Context(), db, up2026083001DurableStorageCommits); err != nil { - t.Fatalf("up migration: %v", err) - } - for _, column := range durableStorageCommitColumns2026083001 { - exists, err := columnExists(t.Context(), db, "storage_upload_copies", column) - if err != nil || !exists { - t.Fatalf("column %s exists=%v err=%v", column, exists, err) - } - } - for _, index := range durableStorageCommitIndexes2026083001 { - exists, err := indexExists(t.Context(), db, index) - if err != nil || !exists { - t.Fatalf("index %s exists=%v err=%v", index, exists, err) - } - } - if db.Dialect().Name() == dialect.PG { - want := time.Date(2026, time.August, 30, 17, 45, 12, 123456000, time.FixedZone("UTC+08", 8*60*60)) - if _, err := db.ExecContext(t.Context(), `INSERT INTO storage_upload_copies - (id, status, commit_ready_at, commit_attempted_at, commit_attention_at) - VALUES (?, 'piece_ready', ?, ?, ?)`, 1, want, want, want); err != nil { - t.Fatalf("insert PostgreSQL commit timestamps: %v", err) - } - var got struct { - ReadyAt time.Time `bun:"commit_ready_at"` - AttemptedAt time.Time `bun:"commit_attempted_at"` - AttentionAt time.Time `bun:"commit_attention_at"` - } - if err := db.NewRaw(`SELECT commit_ready_at, commit_attempted_at, commit_attention_at - FROM storage_upload_copies WHERE id = ?`, 1).Scan(t.Context(), &got); err != nil { - t.Fatalf("scan PostgreSQL commit timestamps: %v", err) - } - if !got.ReadyAt.Equal(want) || !got.AttemptedAt.Equal(want) || !got.AttentionAt.Equal(want) { - t.Fatalf("PostgreSQL timestamp round trip = %s/%s/%s, want instant %s", - got.ReadyAt, got.AttemptedAt, got.AttentionAt, want) - } - } - if err := runMigrationBody(t.Context(), db, up2026083001DurableStorageCommits); err != nil { - t.Fatalf("idempotent up migration: %v", err) - } - }) -} - -func TestDurableStorageCommitsMigrationRejectsLegacyCommittingRowsAtomically(t *testing.T) { - testMigrationDialects(t, func(t *testing.T, db *bun.DB) { - createDurableStorageCommitLegacyTable(t, db) - mustExecMigrationTest(t, db, "INSERT INTO storage_upload_copies (id, status) VALUES (1, 'committing')") - - err := runMigrationBody(t.Context(), db, up2026083001DurableStorageCommits) - if err == nil || !strings.Contains(err.Error(), "legacy committing copies") { - t.Fatalf("up migration error = %v, want legacy committing refusal", err) - } - assertNoDurableStorageCommitDDL(t, db) - }) -} - -func TestDurableStorageCommitsMigrationRechecksLegacyRowsForCompleteSchema(t *testing.T) { - testMigrationDialects(t, func(t *testing.T, db *bun.DB) { - createDurableStorageCommitLegacyTable(t, db) - if err := runMigrationBody(t.Context(), db, up2026083001DurableStorageCommits); err != nil { - t.Fatalf("initial up migration: %v", err) - } - mustExecMigrationTest(t, db, "INSERT INTO storage_upload_copies (id, status) VALUES (1, 'committing')") - - err := runMigrationBody(t.Context(), db, up2026083001DurableStorageCommits) - if err == nil || !strings.Contains(err.Error(), "legacy committing copies") { - t.Fatalf("complete-schema recovery error = %v, want legacy committing refusal", err) - } - mustExecMigrationTest(t, db, "DELETE FROM storage_upload_copies") - mustExecMigrationTest(t, db, `INSERT INTO storage_upload_copies - (id, status, commit_attempt_id, commit_attempted_at) - VALUES (2, 'committing', 'durable-attempt', CURRENT_TIMESTAMP)`) - if err := runMigrationBody(t.Context(), db, up2026083001DurableStorageCommits); err != nil { - t.Fatalf("complete-schema recovery rejected durable committing row: %v", err) - } - }) -} - -func TestDurableStorageCommitsMigrationRejectsPartialSchema(t *testing.T) { - testMigrationDialects(t, func(t *testing.T, db *bun.DB) { - createDurableStorageCommitLegacyTable(t, db) - mustExecMigrationTest(t, db, "ALTER TABLE storage_upload_copies ADD COLUMN commit_ready_at TIMESTAMP") - - err := runMigrationBody(t.Context(), db, up2026083001DurableStorageCommits) - if err == nil || !strings.Contains(err.Error(), "partial schema state") { - t.Fatalf("up migration error = %v, want partial schema refusal", err) - } - exists, checkErr := columnExists(t.Context(), db, "storage_upload_copies", "commit_attempt_id") - if checkErr != nil || exists { - t.Fatalf("failed migration added commit_attempt_id=%v err=%v", exists, checkErr) - } - }) -} - -func TestDurableStorageCommitsMigrationRollbackFence(t *testing.T) { - testMigrationDialects(t, func(t *testing.T, db *bun.DB) { - createDurableStorageCommitLegacyTable(t, db) - if err := runMigrationBody(t.Context(), db, up2026083001DurableStorageCommits); err != nil { - t.Fatalf("up migration: %v", err) - } - mustExecMigrationTest(t, db, `INSERT INTO storage_upload_copies - (id, status, storage_data_set_id, commit_attempt_id, commit_attempted_at) - VALUES (1, 'committing', 9, 'attempt-1', CURRENT_TIMESTAMP)`) - if err := runMigrationBody(t.Context(), db, down2026083001DurableStorageCommits); err == nil || !strings.Contains(err.Error(), "durable commit attempts") { - t.Fatalf("down migration error = %v, want active attempt refusal", err) - } - mustExecMigrationTest(t, db, "DELETE FROM storage_upload_copies") - mustExecMigrationTest(t, db, `INSERT INTO storage_upload_copies - (id, status, commit_submission_json) - VALUES (2, 'committing', '{"version":1,"submission":{}}')`) - if err := runMigrationBody(t.Context(), db, down2026083001DurableStorageCommits); err == nil || !strings.Contains(err.Error(), "durable commit attempts") { - t.Fatalf("down migration error = %v, want durable submission refusal", err) - } - mustExecMigrationTest(t, db, "DELETE FROM storage_upload_copies") - if err := runMigrationBody(t.Context(), db, down2026083001DurableStorageCommits); err != nil { - t.Fatalf("down migration after drain: %v", err) - } - assertNoDurableStorageCommitDDL(t, db) - }) -} - -func createDurableStorageCommitLegacyTable(t *testing.T, db *bun.DB) { - t.Helper() - idColumn := "INTEGER PRIMARY KEY" - if db.Dialect().Name() == dialect.PG { - idColumn = "BIGINT PRIMARY KEY" - } - mustExecMigrationTest(t, db, `CREATE TABLE storage_upload_copies ( - id `+idColumn+`, - status TEXT NOT NULL, - storage_data_set_id BIGINT, - commit_extra_data_hex TEXT, - commit_transaction_id TEXT - )`) -} - -func assertNoDurableStorageCommitDDL(t *testing.T, db *bun.DB) { - t.Helper() - for _, column := range durableStorageCommitColumns2026083001 { - exists, err := columnExists(context.Background(), db, "storage_upload_copies", column) - if err != nil { - t.Fatalf("check column %s: %v", column, err) - } - if exists { - t.Fatalf("column %s remains", column) - } - } - for _, index := range durableStorageCommitIndexes2026083001 { - exists, err := indexExists(context.Background(), db, index) - if err != nil { - t.Fatalf("check index %s: %v", index, err) - } - if exists { - t.Fatalf("index %s remains", index) - } - } -} - -func TestDurableStorageCommitsMigrationRejectsStrandedCommitTransactions(t *testing.T) { - for _, tc := range []struct { - name string - row string - }{ - {name: "piece ready", row: "(1, 'piece_ready', '0xstranded')"}, - {name: "failed", row: "(1, 'failed', '0xstranded')"}, - } { - t.Run(tc.name, func(t *testing.T) { - testMigrationDialects(t, func(t *testing.T, db *bun.DB) { - createDurableStorageCommitLegacyTable(t, db) - mustExecMigrationTest(t, db, - "INSERT INTO storage_upload_copies (id, status, commit_transaction_id) VALUES "+tc.row) - - err := runMigrationBody(t.Context(), db, up2026083001DurableStorageCommits) - if err == nil || !strings.Contains(err.Error(), "keep a commit transaction outside") { - t.Fatalf("up migration error = %v, want stranded transaction refusal", err) - } - assertNoDurableStorageCommitDDL(t, db) - }) - }) - } -} - -func TestDurableStorageCommitsMigrationAcceptsResolvedCommitTransactions(t *testing.T) { - testMigrationDialects(t, func(t *testing.T, db *bun.DB) { - createDurableStorageCommitLegacyTable(t, db) - mustExecMigrationTest(t, db, - "INSERT INTO storage_upload_copies (id, status, commit_transaction_id) VALUES (1, 'committed', '0xdone')") - - if err := runMigrationBody(t.Context(), db, up2026083001DurableStorageCommits); err != nil { - t.Fatalf("up migration rejected a committed transaction: %v", err) - } - }) -} - -func TestDurableStorageCommitsMigrationIgnoresDurableTransactionsOnRecovery(t *testing.T) { - testMigrationDialects(t, func(t *testing.T, db *bun.DB) { - createDurableStorageCommitLegacyTable(t, db) - if err := runMigrationBody(t.Context(), db, up2026083001DurableStorageCommits); err != nil { - t.Fatalf("initial up migration: %v", err) - } - // The durable model owns this row, so a recovery run must judge it by the - // attempt token rather than treating its transaction as legacy evidence. - mustExecMigrationTest(t, db, `INSERT INTO storage_upload_copies - (id, status, commit_transaction_id, commit_attempt_id, commit_attempted_at) - VALUES (1, 'failed', '0xdurable', 'durable-attempt', CURRENT_TIMESTAMP)`) - - if err := runMigrationBody(t.Context(), db, up2026083001DurableStorageCommits); err != nil { - t.Fatalf("recovery run rejected a durable commit transaction: %v", err) - } - }) -} diff --git a/internal/db/migrations/migrations.go b/internal/db/migrations/migrations.go index 0cba0e8..32c63a9 100644 --- a/internal/db/migrations/migrations.go +++ b/internal/db/migrations/migrations.go @@ -2,15 +2,51 @@ package migrations import ( "context" + "errors" + "fmt" + "slices" "github.com/uptrace/bun" "github.com/uptrace/bun/dialect" "github.com/uptrace/bun/migrate" ) +// Bun stores the numeric migration name and derives "initial_schema" as its +// human-readable comment from the filename. +const InitialSchemaName = "2026090101" + +var ErrIncompatibleDatabase = errors.New("database contains an incompatible SynapS3 schema") + // Migrations is the global registry of database migrations. var Migrations = migrate.NewMigrations() +var initialSchemaTableNames = []string{ + "bucket_replica_slots", + "buckets", + "multipart_parts", + "multipart_uploads", + "object_cache", + "object_deletions", + "object_versions", + "objects", + "observability_collection_states", + "observability_data_set_states", + "observability_provider_states", + "s3_accounts", + "storage_cleanup_copies", + "storage_commit_attempts", + "storage_contents", + "storage_copies", + "storage_data_set_terminations", + "storage_data_sets", + "storage_pull_attempts", + "storage_replacement_items", + "storage_replacements", + "task_payloads", + "tasks", + "wallet_operations", +} + type migrationBody func(context.Context, bun.IDB) error // NewMigrator is the only supported migrator constructor. Migration markers @@ -19,12 +55,167 @@ func NewMigrator(db *bun.DB) *migrate.Migrator { return newMigrator(db, Migrations) } +// ValidateTarget accepts an empty application database or a database whose +// applied migrations are an exact ordered prefix of the current registry. +func ValidateTarget(ctx context.Context, db bun.IDB) error { + return validateTarget(ctx, db, Migrations) +} + +func validateTarget(ctx context.Context, db bun.IDB, registry *migrate.Migrations) error { + if !validMigrationRegistry(registry) { + return incompatibleDatabaseError() + } + markerExists, err := tableExists(ctx, db, "bun_migrations") + if err != nil { + return fmt.Errorf("checking migration metadata: %w", err) + } + if markerExists { + var names []string + if err := db.NewRaw("SELECT name FROM bun_migrations ORDER BY id").Scan(ctx, &names); err != nil { + return fmt.Errorf("reading migration metadata: %w", err) + } + if len(names) == 0 { + complete, err := initialSchemaPostStateComplete(ctx, db) + if err != nil { + return fmt.Errorf("checking initial schema post-state: %w", err) + } + if complete { + return nil + } + } else if appliedMigrationPrefix(names, registry) { + return nil + } else { + return incompatibleDatabaseError() + } + } + + count, err := applicationTableCount(ctx, db) + if err != nil { + return fmt.Errorf("checking database contents: %w", err) + } + if count != 0 { + return incompatibleDatabaseError() + } + return nil +} + +func appliedMigrationPrefix(names []string, registry *migrate.Migrations) bool { + registered := registry.Sorted() + if len(names) > len(registered) { + return false + } + for i, migration := range registered { + if i < len(names) && names[i] != migration.Name { + return false + } + } + return true +} + +func validMigrationRegistry(registry *migrate.Migrations) bool { + registered := registry.Sorted() + if len(registered) == 0 || registered[0].Name != InitialSchemaName { + return false + } + for i := 1; i < len(registered); i++ { + if registered[i-1].Name >= registered[i].Name { + return false + } + } + return true +} + +func applicationTableCount(ctx context.Context, db bun.IDB) (int, error) { + names, err := applicationTableNames(ctx, db) + return len(names), err +} + +func applicationTableNames(ctx context.Context, db bun.IDB) ([]string, error) { + query := `SELECT name FROM sqlite_schema + WHERE type = 'table' + AND name NOT LIKE 'sqlite_%' + AND name NOT IN ('bun_migrations', 'bun_migration_locks') + ORDER BY name` + if db.Dialect().Name() == dialect.PG { + query = `SELECT table_name FROM information_schema.tables + WHERE table_schema = current_schema() + AND table_type = 'BASE TABLE' + AND table_name NOT IN ('bun_migrations', 'bun_migration_locks') + ORDER BY table_name` + } + var names []string + if err := db.NewRaw(query).Scan(ctx, &names); err != nil { + return nil, err + } + return names, nil +} + +func initialSchemaPostStateComplete(ctx context.Context, db bun.IDB) (bool, error) { + tables, err := applicationTableNames(ctx, db) + if err != nil || !slices.Equal(tables, initialSchemaTableNames) { + return false, err + } + for _, column := range []struct { + table string + name string + }{ + {"multipart_uploads", "upload_id"}, + {"storage_contents", "checksum"}, + {"storage_copies", "content_id"}, + {"storage_copies", "storage_data_set_id"}, + {"storage_commit_attempts", "attempt_id"}, + {"storage_replacement_items", "target_data_set_id"}, + {"storage_cleanup_copies", "bucket_id"}, + {"object_versions", "content_id"}, + {"object_cache", "content_id"}, + } { + exists, err := columnExists(ctx, db, column.table, column.name) + if err != nil || !exists { + return false, err + } + } + for _, column := range []struct { + table string + name string + }{ + {"multipart_uploads", "id"}, + {"storage_copies", "commit_attempt_id"}, + {"storage_copies", "commit_transaction_id"}, + {"storage_copies", "upload_id"}, + {"storage_replacement_items", "target_copy_id"}, + {"object_versions", "state"}, + {"object_versions", "storage_upload_id"}, + {"storage_data_sets", "repair_task_id"}, + } { + exists, err := columnExists(ctx, db, column.table, column.name) + if err != nil || exists { + return false, err + } + } + for _, index := range []string{ + "idx_storage_commit_attempts_unresolved_copy", + "idx_storage_data_sets_bucket_provider_active", + "idx_observability_data_set_states_provider_status", + } { + exists, err := indexExists(ctx, db, index) + if err != nil || !exists { + return false, err + } + } + return true, nil +} + +func incompatibleDatabaseError() error { + return fmt.Errorf("%w; keep the existing database as a read-only backup and configure a new empty database", ErrIncompatibleDatabase) +} + func newMigrator(db *bun.DB, registry *migrate.Migrations) *migrate.Migrator { return migrate.NewMigrator(db, registry, migrate.WithMarkAppliedOnSuccess(true)) } -// transactionalMigration keeps every migration body atomic while still -// allowing the migrator to repair a missing marker after a committed DDL body. +// transactionalMigration keeps every migration body atomic. The frozen +// baseline separately recognizes its complete post-state so Bun can repair a +// marker lost after the DDL transaction committed. func transactionalMigration(body migrationBody) migrate.MigrationFunc { return func(ctx context.Context, db *bun.DB) error { return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { diff --git a/internal/db/migrations/s3_accounts_test.go b/internal/db/migrations/s3_accounts_test.go deleted file mode 100644 index 932c942..0000000 --- a/internal/db/migrations/s3_accounts_test.go +++ /dev/null @@ -1,258 +0,0 @@ -package migrations - -import ( - "context" - "database/sql" - "strings" - "testing" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect/sqlitedialect" - - _ "modernc.org/sqlite" -) - -func TestS3AccountsMigrationCreatesAccountAndOwnerSchema(t *testing.T) { - sqldb, err := sql.Open("sqlite", "file::memory:?cache=shared&_pragma=foreign_keys(1)") - if err != nil { - t.Fatalf("open sqlite: %v", err) - } - db := bun.NewDB(sqldb, sqlitedialect.New()) - t.Cleanup(func() { _ = db.Close() }) - - ctx := context.Background() - migrator := NewMigrator(db) - if err := migrator.Init(ctx); err != nil { - t.Fatalf("init migrator: %v", err) - } - if _, err := migrator.Migrate(ctx); err != nil { - t.Fatalf("first migrate: %v", err) - } - if _, err := migrator.Migrate(ctx); err != nil { - t.Fatalf("second migrate: %v", err) - } - - if !sqliteTableExists(t, db, "s3_accounts") { - t.Fatal("s3_accounts table missing") - } - for _, column := range []string{"access_key", "secret_key", "role", "is_root", "created_at", "updated_at"} { - if !sqliteColumnExists(t, db, "s3_accounts", column) { - t.Fatalf("s3_accounts.%s column missing", column) - } - } - if !sqliteColumnExists(t, db, "buckets", "owner_access_key") { - t.Fatal("buckets.owner_access_key column missing") - } - if !sqliteIndexExists(t, db, "idx_buckets_owner_access_key") { - t.Fatal("idx_buckets_owner_access_key index missing") - } - if !sqliteIndexExists(t, db, "idx_s3_accounts_is_root") { - t.Fatal("idx_s3_accounts_is_root index missing") - } - if !sqliteIndexExists(t, db, "idx_s3_accounts_single_root") { - t.Fatal("idx_s3_accounts_single_root index missing") - } - if !sqliteForeignKeyExists(t, db, "buckets", "owner_access_key", "s3_accounts") { - t.Fatal("buckets.owner_access_key foreign key missing") - } - - if _, err := db.ExecContext(ctx, `INSERT INTO s3_accounts (access_key, secret_key, role, is_root) VALUES ('root-a', 'secret-a', 'admin', TRUE)`); err != nil { - t.Fatalf("insert first root: %v", err) - } - if _, err := db.ExecContext(ctx, `INSERT INTO s3_accounts (access_key, secret_key, role, is_root) VALUES ('root-b', 'secret-b', 'admin', TRUE)`); err == nil { - t.Fatal("expected duplicate root insert to fail") - } -} - -func TestMigrationCreatesStorageUploadSourceVersionIndex(t *testing.T) { - sqldb, err := sql.Open("sqlite", "file::memory:?cache=shared&_pragma=foreign_keys(1)") - if err != nil { - t.Fatalf("open sqlite: %v", err) - } - db := bun.NewDB(sqldb, sqlitedialect.New()) - t.Cleanup(func() { _ = db.Close() }) - - ctx := context.Background() - migrator := NewMigrator(db) - if err := migrator.Init(ctx); err != nil { - t.Fatalf("init migrator: %v", err) - } - if _, err := migrator.Migrate(ctx); err != nil { - t.Fatalf("migrate: %v", err) - } - - if !sqliteIndexExists(t, db, "idx_storage_uploads_source_version_id") { - t.Fatal("idx_storage_uploads_source_version_id index missing") - } -} - -func TestBucketDefaultCopiesMigrationAddsNullableBoundedColumn(t *testing.T) { - sqldb, err := sql.Open("sqlite", "file::memory:?cache=shared&_pragma=foreign_keys(1)") - if err != nil { - t.Fatalf("open sqlite: %v", err) - } - db := bun.NewDB(sqldb, sqlitedialect.New()) - t.Cleanup(func() { _ = db.Close() }) - - ctx := context.Background() - migrator := NewMigrator(db) - if err := migrator.Init(ctx); err != nil { - t.Fatalf("init migrator: %v", err) - } - if _, err := migrator.Migrate(ctx); err != nil { - t.Fatalf("migrate: %v", err) - } - - if !sqliteColumnExists(t, db, "buckets", "default_copies") { - t.Fatal("buckets.default_copies column missing") - } - if _, err := db.ExecContext(ctx, `INSERT INTO buckets (name, status, default_copies) VALUES ('inherit-bucket', 'active', NULL)`); err != nil { - t.Fatalf("insert inherited bucket: %v", err) - } - if _, err := db.ExecContext(ctx, `INSERT INTO buckets (name, status, default_copies) VALUES ('one-copy-bucket', 'active', 1)`); err != nil { - t.Fatalf("insert one-copy bucket: %v", err) - } - if _, err := db.ExecContext(ctx, `INSERT INTO buckets (name, status, default_copies) VALUES ('eight-copy-bucket', 'active', 8)`); err != nil { - t.Fatalf("insert eight-copy bucket: %v", err) - } - for _, tc := range []struct { - name string - copies int - }{ - {name: "zero-copy-bucket", copies: 0}, - {name: "nine-copy-bucket", copies: 9}, - } { - if _, err := db.ExecContext(ctx, `INSERT INTO buckets (name, status, default_copies) VALUES (?, 'active', ?)`, tc.name, tc.copies); err == nil { - t.Fatalf("insert %s with default_copies=%d succeeded, want constraint failure", tc.name, tc.copies) - } - } - - if _, err := migrator.Migrate(ctx); err != nil { - t.Fatalf("second migrate: %v", err) - } -} - -func TestWalletApproveMigrationUpdatesSQLiteConstraints(t *testing.T) { - sqldb, err := sql.Open("sqlite", "file::memory:?cache=shared&_pragma=foreign_keys(1)") - if err != nil { - t.Fatalf("open sqlite: %v", err) - } - db := bun.NewDB(sqldb, sqlitedialect.New()) - t.Cleanup(func() { _ = db.Close() }) - - ctx := context.Background() - migrator := NewMigrator(db) - if err := migrator.Init(ctx); err != nil { - t.Fatalf("init migrator: %v", err) - } - if _, err := migrator.Migrate(ctx); err != nil { - t.Fatalf("migrate: %v", err) - } - - if _, err := db.ExecContext(ctx, `INSERT INTO wallet_operations (type, client_request_id, amount) VALUES ('approve', 'approve-1', '0')`); err != nil { - t.Fatalf("insert approve amount 0: %v", err) - } - for _, tc := range []struct { - name string - opType string - amount string - }{ - {name: "approve positive", opType: "approve", amount: "1"}, - {name: "fund zero", opType: "fund", amount: "0"}, - {name: "withdraw zero", opType: "withdraw", amount: "0"}, - } { - t.Run(tc.name, func(t *testing.T) { - _, err := db.ExecContext(ctx, `INSERT INTO wallet_operations (type, client_request_id, amount) VALUES (?, ?, ?)`, tc.opType, tc.name, tc.amount) - if err == nil { - t.Fatalf("insert type=%s amount=%s succeeded, want constraint failure", tc.opType, tc.amount) - } - }) - } - - if _, err := migrator.Rollback(ctx); err == nil { - t.Fatal("rollback with approve operation succeeded, want refusal") - } -} - -func TestWalletApproveMigrationReportsMissingPostgresConstraint(t *testing.T) { - db := newPostgresMigrationDB(t) - ctx := context.Background() - if err := runMigrationBody(ctx, db, up2026040501Init); err != nil { - t.Fatalf("create initial schema: %v", err) - } - if _, err := db.ExecContext(ctx, "ALTER TABLE wallet_operations DROP CONSTRAINT chk_wallet_operations_type"); err != nil { - t.Fatalf("drop wallet operation type constraint: %v", err) - } - - err := up2026062201WalletApprove(ctx, db) - if err == nil { - t.Fatal("migration accepted a missing wallet operation constraint") - } - if !strings.Contains(err.Error(), "partial schema state") { - t.Fatalf("migration error = %v, want partial schema diagnosis", err) - } - if strings.Contains(err.Error(), "no rows") { - t.Fatalf("migration error leaked missing-row detail: %v", err) - } -} - -func sqliteTableExists(t *testing.T, db *bun.DB, table string) bool { - t.Helper() - var count int - if err := db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?", table).Scan(&count); err != nil { - t.Fatalf("query table %s: %v", table, err) - } - return count > 0 -} - -func sqliteColumnExists(t *testing.T, db *bun.DB, table, column string) bool { - t.Helper() - rows, err := db.Query("PRAGMA table_info(" + table + ")") - if err != nil { - t.Fatalf("query columns for %s: %v", table, err) - } - defer func() { _ = rows.Close() }() - for rows.Next() { - var cid int - var name, columnType string - var notNull int - var defaultValue sql.NullString - var pk int - if err := rows.Scan(&cid, &name, &columnType, ¬Null, &defaultValue, &pk); err != nil { - t.Fatalf("scan column for %s: %v", table, err) - } - if name == column { - return true - } - } - return false -} - -func sqliteIndexExists(t *testing.T, db *bun.DB, index string) bool { - t.Helper() - var count int - if err := db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = ?", index).Scan(&count); err != nil { - t.Fatalf("query index %s: %v", index, err) - } - return count > 0 -} - -func sqliteForeignKeyExists(t *testing.T, db *bun.DB, table, column, refTable string) bool { - t.Helper() - rows, err := db.Query("PRAGMA foreign_key_list(" + table + ")") - if err != nil { - t.Fatalf("query foreign keys for %s: %v", table, err) - } - defer func() { _ = rows.Close() }() - for rows.Next() { - var id, seq int - var fkTable, from, to, onUpdate, onDelete, match string - if err := rows.Scan(&id, &seq, &fkTable, &from, &to, &onUpdate, &onDelete, &match); err != nil { - t.Fatalf("scan foreign key for %s: %v", table, err) - } - if fkTable == refTable && from == column { - return true - } - } - return false -} diff --git a/internal/db/migrations/schema_behavior_test.go b/internal/db/migrations/schema_behavior_test.go new file mode 100644 index 0000000..0bcf222 --- /dev/null +++ b/internal/db/migrations/schema_behavior_test.go @@ -0,0 +1,649 @@ +package migrations + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + "testing" + + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" +) + +func TestBaselineConstraintsRejectInvalidWrites(t *testing.T) { + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatalf("create initial schema: %v", err) + } + bucketID := insertBaselineTestBucket(t, db, "constraint-bucket") + + mustRejectStatement(t, db, `INSERT INTO tasks + (type, idempotency_key, input_version, input_hash, status, available_at, created_at, updated_at) + VALUES ('test', 'invalid-status', 1, 'hash', 'unknown', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + for _, checksum := range []string{ + "", + "checksum", + strings.Repeat("A", 64), + "sha256:" + strings.Repeat("a", 64), + strings.Repeat("a", 63) + "g", + } { + mustRejectStatement(t, db, `INSERT INTO storage_contents + (bucket_id, content_size, checksum, requested_copies, created_at, updated_at) + VALUES (?, 1, ?, 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, bucketID, checksum) + } + mustRejectStatement(t, db, `INSERT INTO storage_data_sets + (bucket_id, provider_id, copy_index, generation, is_current, create_transaction_id, created_at, updated_at) + VALUES (?, 'provider', 0, 1, FALSE, '', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, bucketID) + mustRejectStatement(t, db, `INSERT INTO wallet_operations + (type, client_request_id, amount, created_at, updated_at) + VALUES ('fund', 'invalid-amount', '0', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustRejectStatement(t, db, `INSERT INTO storage_data_sets + (bucket_id, provider_id, copy_index, generation, is_current, status, created_at, updated_at) + VALUES (?, 'ready-without-id', 1, 1, FALSE, 'ready', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, bucketID) + mustRejectStatement(t, db, `INSERT INTO wallet_operations + (type, client_request_id, amount, status, tx_hash, created_at, updated_at) + VALUES ('fund', 'submitted-without-time', '1', 'submitted', 'tx-1', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustRejectStatement(t, db, `INSERT INTO wallet_operations + (type, client_request_id, amount, status, submitted_at, created_at, updated_at) + VALUES ('fund', 'submitted-without-hash', '1', 'submitted', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + if _, err := db.Exec(`INSERT INTO wallet_operations + (type, client_request_id, amount, status, tx_hash, submitted_at, created_at, updated_at) + VALUES ('fund', 'submitted-complete', '1', 'submitted', 'tx-complete', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`); err != nil { + t.Fatalf("insert valid submitted wallet operation: %v", err) + } + + // A data version is bytes plus a name, so it cannot exist without the + // content that holds those bytes, and a delete marker cannot carry one. + if _, err := db.ExecContext(t.Context(), + `INSERT INTO objects (id, bucket_id, key, created_at, updated_at) VALUES (1, ?, 'failure-shape.txt', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, bucketID); err != nil { + t.Fatalf("insert object: %v", err) + } + contentID := insertBaselineTestContent(t, db, bucketID, "v-shape-origin") + mustRejectStatement(t, db, `INSERT INTO object_versions + (version_id, object_id, bucket_id, key, content_id, size, e_tag, content_type, + metadata, is_delete_marker, created_at, updated_at) + VALUES ('v-data-without-content', 1, ?, 'failure-shape.txt', NULL, 1, 'etag', + 'application/octet-stream', '{}', FALSE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, bucketID) + mustRejectStatement(t, db, `INSERT INTO object_versions + (version_id, object_id, bucket_id, key, content_id, size, e_tag, content_type, + metadata, is_delete_marker, created_at, updated_at) + VALUES ('v-marker-with-content', 1, ?, 'failure-shape.txt', ?, 0, '', + '', '{}', TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, bucketID, contentID) + + dataSetID := insertBaselineTestDataSet(t, db, bucketID, "cleanup-provider", 2, 1, false) + var cleanupID int64 + if err := db.QueryRow(`INSERT INTO storage_cleanup_copies + (content_id, bucket_id, copy_index, provider_id, storage_data_set_id, piece_id, piece_cid, created_at, updated_at) + VALUES (?, ?, 2, 'cleanup-provider', ?, 'piece-1', 'piece-cid-1', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + RETURNING id`, contentID, bucketID, dataSetID).Scan(&cleanupID); err != nil { + t.Fatalf("insert cleanup copy: %v", err) + } + mustRejectStatement(t, db, `UPDATE storage_cleanup_copies + SET status = 'delete_scheduled', delete_tx_hash = 'delete-tx' WHERE id = ?`, cleanupID) + mustRejectStatement(t, db, `UPDATE storage_cleanup_copies + SET status = 'delete_scheduled', scheduled_at = CURRENT_TIMESTAMP WHERE id = ?`, cleanupID) + if _, err := db.Exec(`UPDATE storage_cleanup_copies + SET status = 'delete_scheduled', delete_tx_hash = 'delete-tx', scheduled_at = CURRENT_TIMESTAMP + WHERE id = ?`, cleanupID); err != nil { + t.Fatalf("schedule valid cleanup copy: %v", err) + } + + taskID := insertBaselineTestTask(t, db, "valid-before-update") + mustRejectStatement(t, db, `UPDATE tasks SET claim_generation = -1 WHERE id = ?`, taskID) + mustRejectStatement(t, db, `UPDATE tasks SET retry_limit = 0, retry_count = 1 WHERE id = ?`, taskID) + mustRejectStatement(t, db, `UPDATE buckets SET default_copies = 2, minimum_durable_copies = 3 WHERE id = ?`, bucketID) + var generation int64 + if err := db.NewRaw(`SELECT claim_generation FROM tasks WHERE id = ?`, taskID).Scan(t.Context(), &generation); err != nil { + t.Fatalf("read task after rejected update: %v", err) + } + if generation != 0 { + t.Fatalf("claim_generation = %d after rejected update, want 0", generation) + } + }) +} + +func TestBaselineStorageIdentityAndLedgerConstraints(t *testing.T) { + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatalf("create initial schema: %v", err) + } + bucketA := insertBaselineTestBucket(t, db, "identity-a") + bucketB := insertBaselineTestBucket(t, db, "identity-b") + contentA := insertBaselineTestContent(t, db, bucketA, "upload-a") + contentA2 := insertBaselineTestContent(t, db, bucketA, "upload-a2") + contentB := insertBaselineTestContent(t, db, bucketB, "upload-b") + source := insertBaselineTestDataSet(t, db, bucketA, "101", 0, 1, true) + copyID := insertBaselineTestCopy(t, db, contentA, bucketA, source, 0, "101", "ingress") + mustRejectStatement(t, db, `UPDATE storage_copies + SET confirmed_attempt_status = 'confirmed' WHERE id = ?`, copyID) + + mustRejectStatement(t, db, `INSERT INTO storage_copies + (content_id, bucket_id, content_size, storage_data_set_id, copy_index, provider_id, transfer_method, created_at, updated_at) + VALUES (?, ?, 1, ?, 0, '101', 'ingress', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, contentB, bucketB, source) + mustRejectStatement(t, db, `INSERT INTO storage_copies + (content_id, bucket_id, content_size, storage_data_set_id, copy_index, provider_id, transfer_method, created_at, updated_at) + VALUES (?, ?, 1, ?, 1, '101', 'ingress', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, contentA2, bucketA, source) + mustRejectStatement(t, db, `INSERT INTO storage_copies + (content_id, bucket_id, content_size, storage_data_set_id, copy_index, provider_id, transfer_method, created_at, updated_at) + VALUES (?, ?, 1, ?, 0, '202', 'ingress', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, contentA2, bucketA, source) + // A copy is always born bound to exactly one data set; the NOT NULL on + // storage_data_set_id is what enforces that, so it is the assertion here. + mustRejectRequiredColumn(t, db, `INSERT INTO storage_copies + (content_id, bucket_id, content_size, copy_index, transfer_method, created_at, updated_at) + VALUES (?, ?, 1, 0, 'ingress', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, contentA2, bucketA) + + mustRejectStatement(t, db, `INSERT INTO storage_data_sets + (bucket_id, provider_id, copy_index, generation, is_current, status, created_at, updated_at) + VALUES (?, '101', 1, 1, FALSE, 'draining', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, bucketA) + if _, err := db.Exec(`INSERT INTO storage_data_sets + (bucket_id, provider_id, copy_index, generation, is_current, status, created_at, updated_at) + VALUES (?, '101', 1, 1, FALSE, 'retired', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, bucketA); err != nil { + t.Fatalf("reuse retired provider: %v", err) + } + mustRejectStatement(t, db, `INSERT INTO storage_data_sets + (bucket_id, provider_id, copy_index, generation, is_current, data_set_id, status, created_at, updated_at) + VALUES (?, '202', 0, 2, TRUE, '2002', 'ready', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, bucketA) + // A generation that failed, drained or retired has given up its slot. + // Letting one stay current is what stalled bucket provisioning forever. + for _, endedStatus := range []string{"failed", "draining", "retired"} { + mustRejectStatement(t, db, `INSERT INTO storage_data_sets + (bucket_id, provider_id, copy_index, generation, is_current, status, created_at, updated_at) + VALUES (?, '909', 5, 1, TRUE, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, bucketA, endedStatus) + } + bucketBSource := insertBaselineTestDataSet(t, db, bucketB, "101", 0, 1, true) + + createdByContent := insertBaselineTestContent(t, db, bucketB, "provenance-created-by") + if _, err := db.Exec(`INSERT INTO storage_data_sets + (bucket_id, provider_id, copy_index, generation, is_current, status, created_by_content_id, created_at, updated_at) + VALUES (?, 'provenance-created-by', 1, 1, FALSE, 'retired', ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, bucketB, createdByContent); err != nil { + t.Fatalf("insert created-by provenance: %v", err) + } + mustRejectStatement(t, db, `DELETE FROM storage_contents WHERE id = ?`, createdByContent) + + lastUsedContent := insertBaselineTestContent(t, db, bucketB, "provenance-last-used") + if _, err := db.Exec(`INSERT INTO storage_data_sets + (bucket_id, provider_id, copy_index, generation, is_current, status, last_used_content_id, created_at, updated_at) + VALUES (?, 'provenance-last-used', 2, 1, FALSE, 'retired', ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, bucketB, lastUsedContent); err != nil { + t.Fatalf("insert last-used provenance: %v", err) + } + mustRejectStatement(t, db, `DELETE FROM storage_contents WHERE id = ?`, lastUsedContent) + + if _, err := db.Exec(`INSERT INTO storage_commit_attempts + (attempt_id, content_id, storage_data_set_id, created_at, updated_at) + VALUES ('attempt-1', ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, contentA, source); err != nil { + t.Fatalf("insert first unresolved attempt: %v", err) + } + mustRejectStatement(t, db, `INSERT INTO storage_commit_attempts + (attempt_id, content_id, storage_data_set_id, created_at, updated_at) + VALUES ('attempt-2', ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, contentA, source) + if _, err := db.Exec(`UPDATE storage_commit_attempts + SET status = 'released', release_reason = 'before_submit_canceled', resolved_at = current_timestamp + WHERE attempt_id = 'attempt-1'`); err != nil { + t.Fatalf("release reserved attempt: %v", err) + } + if _, err := db.Exec(`INSERT INTO storage_commit_attempts + (attempt_id, content_id, storage_data_set_id, created_at, updated_at) + VALUES ('attempt-2', ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, contentA, source); err != nil { + t.Fatalf("insert unresolved attempt after release: %v", err) + } + mustRejectStatement(t, db, `UPDATE storage_commit_attempts + SET status = 'attempted', attempted_at = current_timestamp + WHERE attempt_id = 'attempt-2'`) + mustRejectStatement(t, db, `UPDATE storage_commit_attempts + SET extra_data_hex = 'abcd' + WHERE attempt_id = 'attempt-2'`) + mustRejectStatement(t, db, `UPDATE storage_commit_attempts + SET status = 'attempted', attempted_at = current_timestamp, + extra_data_hex = 'abcd', submission_json = '{}' + WHERE attempt_id = 'attempt-2'`) + mustRejectStatement(t, db, `INSERT INTO storage_commit_attempts + (attempt_id, content_id, storage_data_set_id, created_at, updated_at) + VALUES ('cross-copy', ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, contentA2, source) + + target := insertBaselineTestDataSet(t, db, bucketA, "202", 0, 2, false) + var replacementID int64 + if err := db.QueryRow(`INSERT INTO storage_replacements + (bucket_id, copy_index, source_data_set_id, target_data_set_id, + selection_mode, client_request_id, status, created_at, updated_at) + VALUES (?, 0, ?, ?, 'manual', 'replacement-1', 'preparing_target', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + RETURNING id`, bucketA, source, target).Scan(&replacementID); err != nil { + t.Fatalf("insert replacement: %v", err) + } + bucketBTarget := insertBaselineTestDataSet(t, db, bucketB, "303", 0, 2, false) + if _, err := db.Exec(`INSERT INTO storage_replacements + (bucket_id, copy_index, source_data_set_id, target_data_set_id, + selection_mode, client_request_id, status, superseded_by_id, created_at, updated_at) + VALUES (?, 0, ?, ?, 'manual', 'replacement-superseded', 'superseded', ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, + bucketB, bucketBSource, bucketBTarget, replacementID); err != nil { + t.Fatalf("insert superseded replacement provenance: %v", err) + } + mustRejectStatement(t, db, `DELETE FROM storage_replacements WHERE id = ?`, replacementID) + mustRejectStatement(t, db, `INSERT INTO storage_replacements + (bucket_id, copy_index, source_data_set_id, target_data_set_id, + selection_mode, client_request_id, status, created_at, updated_at) + VALUES (?, 0, ?, ?, 'manual', 'replacement-2', 'waiting', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, bucketA, source, target) + if _, err := db.Exec(`UPDATE storage_replacements SET status = 'failed' WHERE id = ?`, replacementID); err != nil { + t.Fatalf("mark replacement retryable: %v", err) + } + mustRejectStatement(t, db, `INSERT INTO storage_replacements + (bucket_id, copy_index, source_data_set_id, target_data_set_id, + selection_mode, client_request_id, status, created_at, updated_at) + VALUES (?, 0, ?, ?, 'manual', 'replacement-3', 'preparing_target', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, bucketA, source, target) + mustRejectStatement(t, db, `INSERT INTO storage_replacement_items + (replacement_id, content_id, target_data_set_id, created_at, updated_at) + VALUES (?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, replacementID, contentA, target) + mustRejectStatement(t, db, `INSERT INTO storage_copies + (content_id, bucket_id, content_size, storage_data_set_id, copy_index, provider_id, transfer_method, created_at, updated_at) + VALUES (?, ?, 1, ?, 0, '202', 'ingress', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, contentA, bucketA, target) + insertBaselineTestCopy(t, db, contentA, bucketA, target, 0, "202", "peer_pull") + if _, err := db.Exec(`INSERT INTO storage_replacement_items + (replacement_id, content_id, target_data_set_id, created_at, updated_at) + VALUES (?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, replacementID, contentA, target); err != nil { + t.Fatalf("insert replacement item after target copy: %v", err) + } + + // A copy can have only one unresolved pull attempt: a second source can + // be tried only after the first attempt is abandoned. + if _, err := db.Exec(`INSERT INTO storage_pull_attempts + (attempt_id, content_id, storage_data_set_id, status, + source_provider_id, source_data_set_id, source_piece_id, source_piece_cid, source_retrieval_url, attempted_at, created_at, updated_at) + VALUES ('pull-1', ?, ?, 'attempted', '301', '3001', '4001', 'bafk2bzacepull', 'https://source.example/piece', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, + contentA, source); err != nil { + t.Fatalf("insert first pull attempt: %v", err) + } + mustRejectStatement(t, db, `INSERT INTO storage_pull_attempts + (attempt_id, content_id, storage_data_set_id, status, + source_provider_id, source_data_set_id, source_piece_id, source_piece_cid, source_retrieval_url, attempted_at, created_at, updated_at) + VALUES ('pull-2', ?, ?, 'attempted', '302', '3002', '4002', 'bafk2bzacepull2', 'https://source.example/other', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, + contentA, source) + mustRejectStatement(t, db, `UPDATE storage_pull_attempts SET status = 'abandoned' WHERE attempt_id = 'pull-1'`) + if _, err := db.Exec(`UPDATE storage_pull_attempts + SET status = 'abandoned', resolved_at = current_timestamp WHERE attempt_id = 'pull-1'`); err != nil { + t.Fatalf("abandon first pull attempt: %v", err) + } + if _, err := db.Exec(`INSERT INTO storage_pull_attempts + (attempt_id, content_id, storage_data_set_id, status, + source_provider_id, source_data_set_id, source_piece_id, source_piece_cid, source_retrieval_url, attempted_at, created_at, updated_at) + VALUES ('pull-2', ?, ?, 'attempted', '302', '3002', '4002', 'bafk2bzacepull2', 'https://source.example/other', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, + contentA, source); err != nil { + t.Fatalf("insert second pull attempt after abandon: %v", err) + } + // A successful pull resolves without changing status, and that also + // frees the slot. + if _, err := db.Exec(`UPDATE storage_pull_attempts + SET resolved_at = current_timestamp WHERE attempt_id = 'pull-2'`); err != nil { + t.Fatalf("resolve second pull attempt: %v", err) + } + mustRejectStatement(t, db, `INSERT INTO storage_pull_attempts + (attempt_id, content_id, storage_data_set_id, status, + source_provider_id, source_data_set_id, source_piece_id, source_piece_cid, source_retrieval_url, attempted_at, created_at, updated_at) + VALUES ('', ?, ?, 'attempted', '303', '3003', '4003', 'bafk2bzacepull3', 'https://source.example/third', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, + contentA, source) + + taskID := insertBaselineTestTask(t, db, "owner-unique") + if _, err := db.Exec(`UPDATE buckets SET durability_task_id = ? WHERE id = ?`, taskID, bucketA); err != nil { + t.Fatalf("bind first task owner: %v", err) + } + mustRejectStatement(t, db, `UPDATE buckets SET durability_task_id = ? WHERE id = ?`, taskID, bucketB) + }) +} + +func TestBaselineIngressRemainsUniqueAfterGenerationRetirement(t *testing.T) { + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatalf("create initial schema: %v", err) + } + bucketID := insertBaselineTestBucket(t, db, "retired-ingress") + contentID := insertBaselineTestContent(t, db, bucketID, "retired-ingress-content") + retiredDataSetID := insertBaselineTestDataSet(t, db, bucketID, "101", 0, 1, true) + copyID := insertBaselineTestCopy(t, db, contentID, bucketID, retiredDataSetID, 0, "101", "ingress") + if _, err := db.Exec(`UPDATE storage_copies SET status = 'failed', last_error = 'retired generation' WHERE id = ?`, copyID); err != nil { + t.Fatalf("fail original ingress copy: %v", err) + } + if _, err := db.Exec(`UPDATE storage_data_sets SET status = 'retired', is_current = FALSE WHERE id = ?`, retiredDataSetID); err != nil { + t.Fatalf("retire original ingress data set: %v", err) + } + + currentDataSetID := insertBaselineTestDataSet(t, db, bucketID, "202", 0, 2, true) + mustRejectStatement(t, db, `INSERT INTO storage_copies + (content_id, bucket_id, content_size, storage_data_set_id, copy_index, provider_id, transfer_method, created_at, updated_at) + VALUES (?, ?, 1, ?, 0, '202', 'ingress', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, contentID, bucketID, currentDataSetID) + insertBaselineTestCopy(t, db, contentID, bucketID, currentDataSetID, 0, "202", "peer_pull") + }) +} + +func TestBaselineIdentitySupportsGenerationAndBackfill(t *testing.T) { + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatalf("create initial schema: %v", err) + } + first := insertBaselineTestTask(t, db, "identity-first") + if _, err := db.Exec(`DELETE FROM tasks WHERE id = ?`, first); err != nil { + t.Fatalf("delete first identity row: %v", err) + } + second := insertBaselineTestTask(t, db, "identity-second") + if second <= first { + t.Fatalf("generated ID %d reused deleted ID %d", second, first) + } + + const backfilledID int64 = 5_000_000_000 + if _, err := db.Exec(`INSERT INTO tasks + (id, type, idempotency_key, input_version, input_hash, available_at, created_at, updated_at) + VALUES (?, 'test', 'identity-backfill', 1, 'hash', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, backfilledID); err != nil { + t.Fatalf("backfill explicit identity: %v", err) + } + var storedID int64 + if err := db.NewRaw(`SELECT id FROM tasks WHERE id = ?`, backfilledID).Scan(t.Context(), &storedID); err != nil { + t.Fatalf("read backfilled identity: %v", err) + } + if storedID != backfilledID { + t.Fatalf("backfilled ID = %d, want %d", storedID, backfilledID) + } + + afterBackfill := insertBaselineTestTask(t, db, "identity-after-backfill") + if afterBackfill <= 0 || afterBackfill == backfilledID { + t.Fatalf("generated ID after backfill = %d", afterBackfill) + } + if db.Dialect().Name() == dialect.SQLite && afterBackfill <= backfilledID { + t.Fatalf("SQLite generated ID after backfill = %d, want > %d", afterBackfill, backfilledID) + } + + assertBaselineIdentityTypes(t, db) + }) +} + +func TestBaselineStoresLargeGeneration(t *testing.T) { + const large = int64(5_000_000_000) + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatalf("create initial schema: %v", err) + } + bucketID := insertBaselineTestBucket(t, db, "large-value-bucket") + sourceID := insertBaselineTestDataSet(t, db, bucketID, "provider-source", 0, large, true) + var generation int64 + if err := db.NewRaw(`SELECT generation FROM storage_data_sets WHERE id = ?`, sourceID).Scan(t.Context(), &generation); err != nil { + t.Fatalf("read large generation: %v", err) + } + if generation != large { + t.Fatalf("generation = %d, want %d", generation, large) + } + }) +} + +func insertBaselineTestTask(t *testing.T, db *bun.DB, key string) int64 { + t.Helper() + var id int64 + if err := db.QueryRow(`INSERT INTO tasks + (type, idempotency_key, input_version, input_hash, available_at, created_at, updated_at) + VALUES ('test', ?, 1, 'hash', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + RETURNING id`, key).Scan(&id); err != nil { + t.Fatalf("insert baseline test task %q: %v", key, err) + } + if _, err := db.Exec(`INSERT INTO task_payloads (task_id, input_json) VALUES (?, '{}')`, id); err != nil { + t.Fatalf("insert baseline test task payload %q: %v", key, err) + } + return id +} + +func insertBaselineTestBucket(t *testing.T, db *bun.DB, name string) int64 { + t.Helper() + return insertBaselineTestBucketWithSlots(t, db, name, 8) +} + +// insertBaselineTestBucketWithSlots opens exactly the given number of replica +// slots, so a test can reach for an index the bucket never opened. +func insertBaselineTestBucketWithSlots(t *testing.T, db *bun.DB, name string, slots int) int64 { + t.Helper() + var id int64 + if err := db.QueryRow(`INSERT INTO buckets (name, default_copies, minimum_durable_copies, created_at, updated_at) + VALUES (?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) RETURNING id`, name, slots, slots).Scan(&id); err != nil { + t.Fatalf("insert baseline test bucket %q: %v", name, err) + } + for copyIndex := range slots { + if _, err := db.Exec(`INSERT INTO bucket_replica_slots (bucket_id, copy_index, created_at, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, id, copyIndex); err != nil { + t.Fatalf("insert baseline test replica slot %d: %v", copyIndex, err) + } + } + return id +} + +func insertBaselineTestDataSet(t *testing.T, db *bun.DB, bucketID int64, provider string, copyIndex int, generation int64, current bool) int64 { + t.Helper() + var id int64 + if err := db.QueryRow(`INSERT INTO storage_data_sets + (bucket_id, provider_id, copy_index, generation, is_current, data_set_id, status, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, 'ready', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + RETURNING id`, bucketID, provider, copyIndex, generation, current, + fmt.Sprintf("%d-%s-%d-%d", bucketID, provider, copyIndex, generation)).Scan(&id); err != nil { + t.Fatalf("insert baseline test data set: %v", err) + } + return id +} + +func insertBaselineTestContent(t *testing.T, db *bun.DB, bucketID int64, identity string) int64 { + t.Helper() + digest := sha256.Sum256([]byte(identity)) + var id int64 + if err := db.QueryRow(`INSERT INTO storage_contents + (bucket_id, content_size, checksum, requested_copies, created_at, updated_at) + VALUES (?, 1, ?, 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + RETURNING id`, bucketID, hex.EncodeToString(digest[:])).Scan(&id); err != nil { + t.Fatalf("insert baseline test content: %v", err) + } + return id +} + +func insertBaselineTestCopy( + t *testing.T, + db *bun.DB, + contentID int64, + bucketID int64, + storageDataSetID int64, + copyIndex int, + provider string, + transferMethod string, +) int64 { + t.Helper() + var id int64 + if err := db.QueryRow(`INSERT INTO storage_copies + (content_id, bucket_id, content_size, storage_data_set_id, copy_index, provider_id, transfer_method, created_at, updated_at) + VALUES (?, ?, 1, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + RETURNING id`, contentID, bucketID, storageDataSetID, copyIndex, provider, transferMethod).Scan(&id); err != nil { + t.Fatalf("insert baseline test copy: %v", err) + } + return id +} + +func mustRejectStatement(t *testing.T, db *bun.DB, query string, args ...any) { + t.Helper() + _, err := db.Exec(query, args...) + if err == nil { + t.Fatalf("statement unexpectedly succeeded: %s", query) + } + requireConstraintRejection(t, err, query) +} + +// mustRejectRequiredColumn asserts the opposite of mustRejectStatement: the +// omitted column is itself the invariant under test, so a null-constraint +// rejection is the expected outcome rather than a false pass. +func mustRejectRequiredColumn(t *testing.T, db *bun.DB, query string, args ...any) { + t.Helper() + _, err := db.Exec(query, args...) + if err == nil { + t.Fatalf("statement unexpectedly succeeded: %s", query) + } + if !rejectedByNullConstraint(err) { + t.Fatalf("statement was not rejected by a null constraint: %s\nerror: %v", query, err) + } +} + +// requireConstraintRejection keeps a negative assertion honest. A statement +// that omits a required column is rejected by the null constraint before the +// constraint under test is ever evaluated, so the case would keep passing +// while proving nothing. +func requireConstraintRejection(t *testing.T, err error, query string) { + t.Helper() + if rejectedByNullConstraint(err) { + t.Fatalf("statement was rejected by a null constraint instead of the constraint under test: %s\nerror: %v", query, err) + } + if rejectedByMissingSchema(err) { + t.Fatalf("statement never reached the constraint under test because the schema has no such table or column: %s\nerror: %v", query, err) + } +} + +// rejectedByMissingSchema reports a statement that never reached the constraint +// under test because it names a table or column the schema does not have. Such a +// statement fails, so a negative assertion keeps passing while proving nothing — +// exactly how three stale cases survived a column being removed. +func rejectedByMissingSchema(err error) bool { + for _, missing := range []string{ + "no such table", // SQLite + "no such column", // SQLite + "has no column named", // SQLite, INSERT column list + "does not exist", // PostgreSQL, relation/column + "undefined_table", // PostgreSQL, SQLSTATE name + "undefined_column", // PostgreSQL, SQLSTATE name + } { + if strings.Contains(err.Error(), missing) { + return true + } + } + return false +} + +func rejectedByNullConstraint(err error) bool { + for _, nullConstraint := range []string{ + "NOT NULL constraint failed", // SQLite + "null value in column", // PostgreSQL + } { + if strings.Contains(err.Error(), nullConstraint) { + return true + } + } + return false +} + +func assertBaselineIdentityTypes(t *testing.T, db *bun.DB) { + t.Helper() + if db.Dialect().Name() == dialect.PG { + var idType, inputVersionType, generationType, identity string + if err := db.QueryRow(`SELECT + (SELECT data_type FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'tasks' AND column_name = 'id'), + (SELECT data_type FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'tasks' AND column_name = 'input_version'), + (SELECT data_type FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'storage_data_sets' AND column_name = 'generation'), + (SELECT is_identity FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = 'tasks' AND column_name = 'id')`). + Scan(&idType, &inputVersionType, &generationType, &identity); err != nil { + t.Fatalf("read PostgreSQL identity types: %v", err) + } + if idType != "bigint" || inputVersionType != "integer" || generationType != "bigint" || identity != "YES" { + t.Fatalf("PostgreSQL types = id:%s input:%s generation:%s identity:%s", idType, inputVersionType, generationType, identity) + } + return + } + for table, column := range map[string]string{ + "tasks": "id", + "tasks/input": "input_version", + "storage_data_sets": "generation", + } { + tableName := table + if table == "tasks/input" { + tableName = "tasks" + } + var sqlType string + if err := db.NewRaw(`SELECT type FROM pragma_table_info(?) WHERE name = ?`, tableName, column).Scan(t.Context(), &sqlType); err != nil { + t.Fatalf("read SQLite type for %s.%s: %v", tableName, column, err) + } + if sqlType != "INTEGER" { + t.Fatalf("SQLite type for %s.%s = %s, want INTEGER", tableName, column, sqlType) + } + } +} + +// A replica index is a foreign key into the bucket's own slots, not a bare +// integer bounded by a range check. Every other seed in this package opens all +// eight slots, so this is the only case that reaches the constraint. +func TestBaselineReplicaIndexRequiresAnOpenSlot(t *testing.T) { + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatalf("create initial schema: %v", err) + } + bucketID := insertBaselineTestBucketWithSlots(t, db, "two-slot-bucket", 2) + + // The bucket opened 0 and 1, so a data set on 5 has no slot to belong to. + mustRejectStatement(t, db, `INSERT INTO storage_data_sets + (bucket_id, provider_id, copy_index, generation, is_current, status, created_at, updated_at) + VALUES (?, 'provider-a', 5, 1, TRUE, 'pending', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, bucketID) + + // Same index on a bucket that did open it is accepted, which is what + // makes the rejection above about the slot rather than the range. + wideBucketID := insertBaselineTestBucketWithSlots(t, db, "eight-slot-bucket", 8) + if _, err := db.Exec(`INSERT INTO storage_data_sets + (bucket_id, provider_id, copy_index, generation, is_current, status, created_at, updated_at) + VALUES (?, 'provider-a', 5, 1, TRUE, 'pending', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, wideBucketID); err != nil { + t.Fatalf("insert data set on an open slot: %v", err) + } + + // A slot cannot be dropped while a generation still stands on it. + mustRejectStatement(t, db, `DELETE FROM bucket_replica_slots WHERE bucket_id = ? AND copy_index = 5`, wideBucketID) + }) +} + +// A termination names one data set through the role it ended, and the composite +// foreign keys keep that data set inside the replacement it belongs to. +func TestBaselineTerminationBelongsToItsReplacementRole(t *testing.T) { + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatalf("create initial schema: %v", err) + } + bucketID := insertBaselineTestBucket(t, db, "termination-bucket") + source := insertBaselineTestDataSet(t, db, bucketID, "101", 0, 1, true) + target := insertBaselineTestDataSet(t, db, bucketID, "202", 0, 2, false) + stranger := insertBaselineTestDataSet(t, db, bucketID, "303", 1, 1, true) + var replacementID int64 + if err := db.QueryRow(`INSERT INTO storage_replacements + (bucket_id, copy_index, source_data_set_id, target_data_set_id, + selection_mode, client_request_id, status, created_at, updated_at) + VALUES (?, 0, ?, ?, 'manual', 'termination-request', 'retiring', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + RETURNING id`, bucketID, source, target).Scan(&replacementID); err != nil { + t.Fatalf("insert replacement: %v", err) + } + + // The role decides which data set column carries the subject. + mustRejectStatement(t, db, `INSERT INTO storage_data_set_terminations + (replacement_id, role, abandoned_target_data_set_id, epoch, created_at, updated_at) + VALUES (?, 'source', ?, 84, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, replacementID, target) + mustRejectStatement(t, db, `INSERT INTO storage_data_set_terminations + (replacement_id, role, source_data_set_id, abandoned_target_data_set_id, epoch, created_at, updated_at) + VALUES (?, 'source', ?, ?, 84, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, replacementID, source, target) + mustRejectStatement(t, db, `INSERT INTO storage_data_set_terminations + (replacement_id, role, source_data_set_id, epoch, created_at, updated_at) + VALUES (?, 'both', ?, 84, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, replacementID, source) + + // A source termination cannot name a data set this replacement never + // held as its source, even one that exists in the same bucket. + mustRejectStatement(t, db, `INSERT INTO storage_data_set_terminations + (replacement_id, role, source_data_set_id, epoch, created_at, updated_at) + VALUES (?, 'source', ?, 84, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, replacementID, stranger) + // The replacement's own target is not its source either. + mustRejectStatement(t, db, `INSERT INTO storage_data_set_terminations + (replacement_id, role, source_data_set_id, epoch, created_at, updated_at) + VALUES (?, 'source', ?, 84, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, replacementID, target) + + if _, err := db.Exec(`INSERT INTO storage_data_set_terminations + (replacement_id, role, source_data_set_id, epoch, created_at, updated_at) + VALUES (?, 'source', ?, 84, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, replacementID, source); err != nil { + t.Fatalf("insert source termination: %v", err) + } + // One end of term per role: a second would mean paying twice. + mustRejectStatement(t, db, `INSERT INTO storage_data_set_terminations + (replacement_id, role, source_data_set_id, epoch, created_at, updated_at) + VALUES (?, 'source', ?, 90, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, replacementID, source) + // The abandoned target is a separate role and still fits. + if _, err := db.Exec(`INSERT INTO storage_data_set_terminations + (replacement_id, role, abandoned_target_data_set_id, epoch, created_at, updated_at) + VALUES (?, 'abandoned_target', ?, 90, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, replacementID, target); err != nil { + t.Fatalf("insert abandoned target termination: %v", err) + } + }) +} diff --git a/internal/db/migrations/schema_builder.go b/internal/db/migrations/schema_builder.go new file mode 100644 index 0000000..ccd4b0d --- /dev/null +++ b/internal/db/migrations/schema_builder.go @@ -0,0 +1,193 @@ +package migrations + +import ( + "context" + "fmt" + "strings" + + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" +) + +type initialTableSpec struct { + name string + model any + constraints []string + foreignKeys []string + // forwardForeignKeys reference a table this one is created before. SQLite + // resolves a foreign key parent when rows are written, so the constraint can + // be declared inline; PostgreSQL requires the parent to exist already, so + // there the same constraint is added by addForwardForeignKey once both + // tables are in place. + forwardForeignKeys []initialForwardForeignKey + jsonColumns []initialJSONColumnSpec +} + +type initialForwardForeignKey struct { + name string + definition string +} + +type initialJSONShape string + +const ( + initialJSONAny initialJSONShape = "any" + initialJSONObject initialJSONShape = "object" + initialJSONArray initialJSONShape = "array" +) + +type initialJSONColumnSpec struct { + name string + shape initialJSONShape + nullable bool + // text marks a column already declared as text rather than jsonb. Its shape + // still has to hold, so PostgreSQL casts before asking. + text bool +} + +func initialJSONColumns(table string) []initialJSONColumnSpec { + switch table { + case "storage_commit_attempts": + return []initialJSONColumnSpec{{name: "submission_json", shape: initialJSONObject, nullable: true, text: true}} + case "task_payloads": + return []initialJSONColumnSpec{ + {name: "input_json", shape: initialJSONObject}, + {name: "checkpoint_json", shape: initialJSONObject, nullable: true}, + } + case "multipart_uploads", "object_versions": + return []initialJSONColumnSpec{{name: "metadata", shape: initialJSONObject}} + case "observability_provider_states", "observability_data_set_states": + return []initialJSONColumnSpec{ + {name: "reason_codes", shape: initialJSONArray}, + {name: "evidence_json", shape: initialJSONObject}, + } + default: + return nil + } +} + +func isInitialJSONColumn(table, column string) bool { + for _, candidate := range initialJSONColumns(table) { + if candidate.name == column { + return true + } + } + return false +} + +func createInitialTable(ctx context.Context, db bun.IDB, spec initialTableSpec) error { + query := db.NewCreateTable().Model(spec.model) + for _, constraint := range spec.constraints { + query.ColumnExpr(constraint) + } + for _, foreignKey := range spec.foreignKeys { + query.ForeignKey(foreignKey) + } + if db.Dialect().Name() == dialect.SQLite { + for _, foreignKey := range spec.forwardForeignKeys { + query.ColumnExpr("CONSTRAINT " + foreignKey.name + " FOREIGN KEY " + foreignKey.definition) + } + } + for _, column := range spec.jsonColumns { + query.ColumnExpr(initialJSONConstraint(db.Dialect().Name(), spec.name, column)) + } + + var err error + if db.Dialect().Name() == dialect.SQLite && len(spec.jsonColumns) > 0 { + ddl := query.String() + for _, column := range spec.jsonColumns { + if column.text { + continue + } + jsonType := fmt.Sprintf(`"%s" jsonb`, column.name) + if strings.Count(ddl, jsonType) != 1 { + return fmt.Errorf("rendering SQLite JSON column %s.%s", spec.name, column.name) + } + ddl = strings.Replace(ddl, jsonType, fmt.Sprintf(`"%s" text`, column.name), 1) + } + _, err = db.ExecContext(ctx, ddl) + } else { + _, err = query.Exec(ctx) + } + if err != nil { + return fmt.Errorf("creating table %s: %w", spec.name, err) + } + return nil +} + +func initialJSONConstraint(dialectName dialect.Name, table string, column initialJSONColumnSpec) string { + name := fmt.Sprintf("chk_%s_%s_json", table, column.name) + if dialectName == dialect.PG { + // text::jsonb is immutable, so a CHECK may cast a text column here. + value := column.name + if column.text { + value = column.name + "::jsonb" + } + expression := fmt.Sprintf("jsonb_typeof(%s) IS NOT NULL", value) + if column.shape != initialJSONAny { + expression = fmt.Sprintf("jsonb_typeof(%s) = '%s'", value, column.shape) + } + if column.nullable { + expression = fmt.Sprintf("%s IS NULL OR %s", column.name, expression) + } + return fmt.Sprintf("CONSTRAINT %s CHECK (%s)", name, expression) + } + + expression := fmt.Sprintf("json_valid(%s)", column.name) + if column.shape != initialJSONAny { + expression = fmt.Sprintf( + "CASE WHEN json_valid(%s) THEN json_type(%s) = '%s' ELSE FALSE END", + column.name, + column.name, + column.shape, + ) + } + if column.nullable { + expression = fmt.Sprintf("%s IS NULL OR (%s)", column.name, expression) + } + return fmt.Sprintf("CONSTRAINT %s CHECK (%s)", name, expression) +} + +type initialIndexSpec struct { + name string + table string + columns []string + where string + unique bool +} + +func createInitialIndexes(ctx context.Context, db bun.IDB, specs ...initialIndexSpec) error { + for _, spec := range specs { + query := db.NewCreateIndex().Index(spec.name).Table(spec.table) + if spec.unique { + query.Unique() + } + for _, column := range spec.columns { + query.ColumnExpr(column) + } + if spec.where != "" { + query.Where(spec.where) + } + if _, err := query.Exec(ctx); err != nil { + return fmt.Errorf("creating index %s: %w", spec.name, err) + } + } + return nil +} + +// addForwardForeignKey completes a constraint that could not be declared while +// the child table was created because its parent did not exist yet. Only +// PostgreSQL needs it; SQLite already carries the constraint inline. +func addForwardForeignKey(ctx context.Context, db bun.IDB, table string, foreignKey initialForwardForeignKey) error { + if db.Dialect().Name() != dialect.PG { + return nil + } + statement := fmt.Sprintf( + "ALTER TABLE %s ADD CONSTRAINT %s FOREIGN KEY %s", + table, foreignKey.name, foreignKey.definition, + ) + if _, err := db.ExecContext(ctx, statement); err != nil { + return fmt.Errorf("adding foreign key %s on %s: %w", foreignKey.name, table, err) + } + return nil +} diff --git a/internal/db/migrations/schema_core.go b/internal/db/migrations/schema_core.go new file mode 100644 index 0000000..024465e --- /dev/null +++ b/internal/db/migrations/schema_core.go @@ -0,0 +1,341 @@ +package migrations + +import ( + "context" + "encoding/json" + "time" + + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" +) + +type s3Account2026090101 struct { + bun.BaseModel `bun:"table:s3_accounts"` + + AccessKey string `bun:"type:text,pk"` + SecretKey string `bun:"type:text,notnull"` + Role string `bun:"type:text,notnull"` + IsRoot bool `bun:",notnull,default:false"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type bucket2026090101 struct { + bun.BaseModel `bun:"table:buckets"` + + ID int64 `bun:",pk,autoincrement,identity"` + Name string `bun:"type:text,notnull,unique"` + ACL []byte + OwnerAccessKey *string `bun:"type:text"` + DefaultCopies int `bun:"type:integer,notnull"` + MinimumDurableCopies int `bun:"type:integer,notnull"` + DurabilityGeneration int64 `bun:",notnull,default:0"` + DurabilityTaskID *int64 + Status string `bun:"type:text,notnull,default:'provisioning'"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +// bucketReplicaSlot2026090101 gives a replica slot a row of its own. copy_index +// used to be a bare integer repeated across five tables and bounded only by a +// range check; as a table it becomes a foreign key target, and a slot keeps its +// history because retired generations still reference it. +type bucketReplicaSlot2026090101 struct { + bun.BaseModel `bun:"table:bucket_replica_slots"` + + ID int64 `bun:",pk,autoincrement,identity"` + BucketID int64 `bun:",notnull"` + CopyIndex int `bun:"type:integer,notnull"` + Status string `bun:"type:text,notnull,default:'active'"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type object2026090101 struct { + bun.BaseModel `bun:"table:objects"` + + ID int64 `bun:",pk,autoincrement,identity"` + BucketID int64 `bun:",notnull"` + Key string `bun:"type:text,notnull"` + // CurrentVersionID points at the version S3 reads serve. One column can hold + // one value, so "at most one current version" is structural here rather than + // a partial unique index over every version of the object. + CurrentVersionID *string `bun:"type:text"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type multipartUpload2026090101 struct { + bun.BaseModel `bun:"table:multipart_uploads"` + + UploadID string `bun:"type:text,pk"` + BucketID int64 `bun:",notnull"` + Key string `bun:"type:text,notnull"` + ContentType string `bun:"type:text,notnull,default:'application/octet-stream'"` + Metadata json.RawMessage `bun:"type:jsonb,notnull,default:'{}'"` + Status string `bun:"type:text,notnull,default:'initiated'"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type multipartPart2026090101 struct { + bun.BaseModel `bun:"table:multipart_parts"` + + ID int64 `bun:",pk,autoincrement,identity"` + UploadID string `bun:"type:text,notnull"` + PartNumber int `bun:"type:integer,notnull"` + Size int64 `bun:",notnull"` + ETag string `bun:"e_tag,type:text,notnull"` + Checksum *string `bun:"type:text"` + CreatedAt time.Time `bun:",notnull"` +} + +func createCoreRootSchema(ctx context.Context, db bun.IDB) error { + tables := []initialTableSpec{ + { + name: "s3_accounts", + model: (*s3Account2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_s3_accounts_identity CHECK (access_key <> '' AND secret_key <> '')", + "CONSTRAINT chk_s3_accounts_role CHECK (role IN ('admin', 'user', 'userplus'))", + }, + }, + { + name: "buckets", + model: (*bucket2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_buckets_identity CHECK (name <> '' AND (owner_access_key IS NULL OR owner_access_key <> ''))", + "CONSTRAINT chk_buckets_status CHECK (status IN ('provisioning', 'ready'))", + // The durability policy is materialised from configuration when + // the bucket is created, so "how many replicas does this bucket + // want" never depends on reading config at query time. + "CONSTRAINT chk_buckets_default_copies CHECK (default_copies BETWEEN 1 AND 8)", + "CONSTRAINT chk_buckets_minimum_durable_copies CHECK (minimum_durable_copies BETWEEN 1 AND 8)", + "CONSTRAINT chk_buckets_explicit_copy_policy CHECK (minimum_durable_copies <= default_copies)", + "CONSTRAINT chk_buckets_durability_generation CHECK (durability_generation >= 0)", + }, + foreignKeys: []string{ + "(owner_access_key) REFERENCES s3_accounts (access_key) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(durability_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }, + { + name: "bucket_replica_slots", + model: (*bucketReplicaSlot2026090101)(nil), + constraints: []string{ + "CONSTRAINT uq_bucket_replica_slots_identity UNIQUE (bucket_id, copy_index)", + "CONSTRAINT chk_bucket_replica_slots_copy_index CHECK (copy_index BETWEEN 0 AND 7)", + // A shrunk slot is decommissioned, never deleted: retired data + // set generations still point at it. + "CONSTRAINT chk_bucket_replica_slots_status CHECK (status IN ('active', 'decommissioned'))", + }, + foreignKeys: []string{ + "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }, + { + name: "objects", + model: (*object2026090101)(nil), + constraints: []string{ + "CONSTRAINT uq_objects_identity UNIQUE (id, bucket_id, key)", + "CONSTRAINT chk_objects_identity CHECK (key <> '')", + }, + foreignKeys: []string{ + "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + forwardForeignKeys: []initialForwardForeignKey{objectCurrentVersionForeignKey2026090101()}, + }, + { + name: "multipart_uploads", + model: (*multipartUpload2026090101)(nil), + jsonColumns: initialJSONColumns("multipart_uploads"), + constraints: []string{ + "CONSTRAINT uq_multipart_uploads_identity UNIQUE (upload_id, bucket_id, key)", + "CONSTRAINT chk_multipart_uploads_identity CHECK (key <> '' AND upload_id <> '')", + "CONSTRAINT chk_multipart_uploads_status CHECK (status IN ('initiated', 'completing', 'completed', 'aborted'))", + }, + foreignKeys: []string{ + "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }, + { + name: "multipart_parts", + model: (*multipartPart2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_multipart_parts_identity CHECK (upload_id <> '' AND e_tag <> '' AND (checksum IS NULL OR checksum <> ''))", + "CONSTRAINT chk_multipart_parts_part_number CHECK (part_number BETWEEN 1 AND 10000)", + "CONSTRAINT chk_multipart_parts_size CHECK (size >= 0)", + }, + foreignKeys: []string{ + "(upload_id) REFERENCES multipart_uploads (upload_id) ON UPDATE RESTRICT ON DELETE CASCADE", + }, + }, + } + for _, table := range tables { + if err := createInitialTable(ctx, db, table); err != nil { + return err + } + } + indexes := []initialIndexSpec{ + {name: "idx_objects_bucket_key", table: "objects", columns: []string{"bucket_id", "key"}, unique: true}, + {name: "idx_objects_current_version", table: "objects", columns: []string{"current_version_id"}, where: "current_version_id IS NOT NULL"}, + {name: "idx_s3_accounts_single_root", table: "s3_accounts", columns: []string{"is_root"}, where: "is_root = TRUE", unique: true}, + {name: "idx_buckets_owner_access_key", table: "buckets", columns: []string{"owner_access_key"}}, + {name: "idx_buckets_durability_task", table: "buckets", columns: []string{"durability_task_id"}, where: "durability_task_id IS NOT NULL", unique: true}, + {name: "idx_multipart_parts_upload_part", table: "multipart_parts", columns: []string{"upload_id", "part_number"}, unique: true}, + } + if db.Dialect().Name() != dialect.PG { + indexes = append(indexes, initialIndexSpec{name: "idx_multipart_uploads_bucket_status_key_upload", table: "multipart_uploads", columns: []string{"bucket_id", "status", "key", "upload_id"}}) + } + return createInitialIndexes(ctx, db, indexes...) +} + +// objectVersion2026090101 carries S3 naming semantics only. Pipeline state is +// a function of the copy rows and stays a query; cache residency belongs to +// object_cache; the bytes themselves are storage_contents. A NULL content_id +// is exactly a delete marker. +type objectVersion2026090101 struct { + bun.BaseModel `bun:"table:object_versions"` + + VersionID string `bun:"type:text,pk"` + ObjectID int64 `bun:",notnull"` + BucketID int64 `bun:",notnull"` + Key string `bun:"type:text,notnull"` + ContentID *int64 + Size int64 `bun:",notnull"` + ETag string `bun:"e_tag,type:text,notnull"` + ContentType string `bun:"type:text,notnull,default:'application/octet-stream'"` + Metadata json.RawMessage `bun:"type:jsonb,notnull,default:'{}'"` + MultipartUploadID *string `bun:"type:text"` + IsDeleteMarker bool `bun:",notnull,default:false"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +// objectCache2026090101 records local cache residency per content, not per +// version, so identical bytes written under several keys share one file. The +// cache key is derived from content_id and therefore not stored. +type objectCache2026090101 struct { + bun.BaseModel `bun:"table:object_cache"` + + ContentID int64 `bun:",pk"` + InCache bool `bun:",notnull"` + CacheAccessedAt *time.Time + CachePresenceGeneration int64 `bun:",notnull,default:0"` + CacheOperationGeneration int64 `bun:",notnull,default:0"` + CacheActiveTaskID *int64 + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +// objectDeletion2026090101 is an append-only tombstone. Cache cleanup is not +// tracked here: with content-keyed residency the trigger is the content's +// reference count reaching zero, not the removal of one version. +type objectDeletion2026090101 struct { + bun.BaseModel `bun:"table:object_deletions"` + + ID int64 `bun:",pk,autoincrement,identity"` + BucketID int64 `bun:",notnull"` + ObjectID int64 `bun:",notnull"` + Key string `bun:"type:text,notnull"` + VersionID string `bun:"type:text,notnull,unique"` + ContentID *int64 + Size int64 `bun:",notnull"` + DeletedAt time.Time `bun:",notnull"` +} + +func createObjectLifecycleSchema(ctx context.Context, db bun.IDB) error { + if err := createInitialTable(ctx, db, initialTableSpec{ + name: "object_versions", + model: (*objectVersion2026090101)(nil), + jsonColumns: initialJSONColumns("object_versions"), + constraints: []string{ + "CONSTRAINT chk_object_versions_identity CHECK (version_id <> '' AND key <> '' AND (multipart_upload_id IS NULL OR multipart_upload_id <> ''))", + "CONSTRAINT chk_object_versions_size CHECK (size >= 0)", + // Candidate key for the objects.current_version_id pointer, which + // names both the version and the object it must belong to. + "CONSTRAINT uq_object_versions_object_identity UNIQUE (version_id, object_id)", + // A delete marker is exactly a version with no content. + "CONSTRAINT chk_object_versions_delete_marker_shape CHECK ((is_delete_marker = TRUE AND content_id IS NULL AND size = 0 AND e_tag = '' AND content_type = '') OR (is_delete_marker = FALSE AND content_id IS NOT NULL AND e_tag <> ''))", + "CONSTRAINT fk_object_versions_object FOREIGN KEY (object_id, bucket_id, key) REFERENCES objects (id, bucket_id, key) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + foreignKeys: []string{ + "(multipart_upload_id, bucket_id, key) REFERENCES multipart_uploads (upload_id, bucket_id, key) ON UPDATE RESTRICT ON DELETE RESTRICT", + // size repeats the content size so listings need no join; the + // composite key makes that repetition unable to drift. + "(content_id, bucket_id, size) REFERENCES storage_contents (id, bucket_id, content_size) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }); err != nil { + return err + } + if err := createInitialTable(ctx, db, initialTableSpec{ + name: "object_cache", + model: (*objectCache2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_object_cache_generation CHECK (cache_presence_generation >= 0 AND cache_operation_generation >= 0)", + }, + foreignKeys: []string{ + "(content_id) REFERENCES storage_contents (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(cache_active_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }); err != nil { + return err + } + if err := createInitialTable(ctx, db, initialTableSpec{ + name: "object_deletions", + model: (*objectDeletion2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_object_deletions_identity CHECK (key <> '' AND version_id <> '')", + "CONSTRAINT chk_object_deletions_size CHECK (size >= 0)", + }, + foreignKeys: []string{ + "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(content_id, bucket_id) REFERENCES storage_contents (id, bucket_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }); err != nil { + return err + } + indexes := []initialIndexSpec{ + {name: "idx_object_versions_object_created", table: "object_versions", columns: []string{"object_id", "created_at DESC", "version_id DESC"}}, + // Content reference count and foreign key coverage in one index. + {name: "idx_object_versions_content", table: "object_versions", columns: []string{"content_id"}}, + {name: "idx_object_versions_multipart_upload", table: "object_versions", columns: []string{"multipart_upload_id"}}, + {name: "idx_object_cache_lru", table: "object_cache", columns: []string{"cache_accessed_at", "content_id"}, where: "in_cache = TRUE"}, + {name: "idx_object_cache_active_task", table: "object_cache", columns: []string{"cache_active_task_id"}, where: "cache_active_task_id IS NOT NULL", unique: true}, + {name: "idx_object_deletions_bucket_key_deleted", table: "object_deletions", columns: []string{"bucket_id", "key", "deleted_at"}}, + {name: "idx_object_deletions_content", table: "object_deletions", columns: []string{"content_id"}}, + {name: "idx_object_deletions_bucket_deleted", table: "object_deletions", columns: []string{"bucket_id", "deleted_at DESC", "id DESC"}}, + } + if db.Dialect().Name() != dialect.PG { + indexes = append(indexes, initialIndexSpec{name: "idx_object_versions_bucket_key_created", table: "object_versions", columns: []string{"bucket_id", "key", "created_at DESC", "version_id DESC"}}) + } + if err := createInitialIndexes(ctx, db, indexes...); err != nil { + return err + } + // objects was created before object_versions existed, so PostgreSQL takes + // the pointer constraint here. + return addForwardForeignKey(ctx, db, "objects", objectCurrentVersionForeignKey2026090101()) +} + +// objectCurrentVersionForeignKey2026090101 keeps an object from pointing at a +// version of some other object, and keeps the pointed-at version from being +// deleted before the pointer moves. +func objectCurrentVersionForeignKey2026090101() initialForwardForeignKey { + return initialForwardForeignKey{ + name: "fk_objects_current_version", + definition: "(current_version_id, id) REFERENCES object_versions (version_id, object_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + } +} + +func createPostgresPrefixIndexes(ctx context.Context, db bun.IDB) error { + if db.Dialect().Name() != dialect.PG { + return nil + } + return createInitialIndexes(ctx, db, + // Listing current objects orders by a C-collated key, so the objects + // unique index needs a C-collated companion to drive it. + initialIndexSpec{name: "idx_objects_bucket_key_c", table: "objects", columns: []string{"bucket_id", `(key COLLATE "C")`}}, + initialIndexSpec{name: "idx_object_versions_bucket_key_created", table: "object_versions", columns: []string{"bucket_id", `(key COLLATE "C")`, "created_at DESC", "version_id DESC"}}, + initialIndexSpec{name: "idx_multipart_uploads_bucket_status_key_upload", table: "multipart_uploads", columns: []string{"bucket_id", "status", `(key COLLATE "C")`, "upload_id"}}, + ) +} diff --git a/internal/db/migrations/schema_ddl_test.go b/internal/db/migrations/schema_ddl_test.go new file mode 100644 index 0000000..af719f4 --- /dev/null +++ b/internal/db/migrations/schema_ddl_test.go @@ -0,0 +1,200 @@ +package migrations + +import ( + "database/sql" + "strings" + "testing" + + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect/pgdialect" + "github.com/uptrace/bun/dialect/sqlitedialect" +) + +func TestFrozenBaselineDDLUsesNativeIdentityAndExplicitTypes(t *testing.T) { + models := []any{ + (*task2026090101)(nil), + (*bucket2026090101)(nil), + (*object2026090101)(nil), + (*multipartPart2026090101)(nil), + (*storageContent2026090101)(nil), + (*storageDataSet2026090101)(nil), + (*storageCopy2026090101)(nil), + (*storageReplacement2026090101)(nil), + (*storageReplacementItem2026090101)(nil), + (*storageCleanupCopy2026090101)(nil), + (*objectDeletion2026090101)(nil), + (*walletOperation2026090101)(nil), + } + + postgres := newOfflinePostgresDDLDB(t) + for _, model := range models { + ddl := postgres.NewCreateTable().Model(model).String() + if !strings.Contains(strings.ToLower(ddl), `"id" bigint not null generated by default as identity`) { + t.Errorf("PostgreSQL identity DDL missing from %s", ddl) + } + } + postgresTaskDDL := postgres.NewCreateTable().Model((*task2026090101)(nil)).String() + postgresMultipartDDL := postgres.NewCreateTable().Model((*multipartUpload2026090101)(nil)).String() + postgresDataSetDDL := postgres.NewCreateTable().Model((*storageDataSet2026090101)(nil)).String() + postgresObjectVersionDDL := postgres.NewCreateTable().Model((*objectVersion2026090101)(nil)).String() + postgresReplacementDDL := postgres.NewCreateTable().Model((*storageReplacement2026090101)(nil)).String() + postgresTaskPayloadDDL := postgres.NewCreateTable().Model((*taskPayload2026090101)(nil)).String() + assertDDLContains(t, postgresTaskDDL, `"type" text not null`, `"input_version" integer not null`, `"retry_limit" integer`) + assertDDLNotContains(t, postgresTaskDDL, `"input_json"`, `"checkpoint_json"`, `"retry_limit" integer not null`) + assertDDLContains(t, postgresTaskPayloadDDL, `"input_json" jsonb not null`, `"checkpoint_json" jsonb`) + assertDDLNotContains(t, postgresTaskPayloadDDL, `"checkpoint_json" jsonb not null`) + assertDDLContains(t, postgresDataSetDDL, `"generation" bigint not null default 1`, `"copy_index" integer not null`) + assertDDLContains(t, postgresReplacementDDL, `"items_total" integer not null default 0`, `"items_copied" integer not null default 0`) + assertDDLContains(t, postgresObjectVersionDDL, `"metadata" jsonb not null default '{}'`) + assertDDLContains(t, postgresMultipartDDL, `primary key ("upload_id")`) + assertDDLNotContains(t, postgresMultipartDDL, `"id" bigint`) + + sqlite := newOfflineSQLiteDDLDB(t) + for _, model := range models { + ddl := sqlite.NewCreateTable().Model(model).String() + if !strings.Contains(strings.ToLower(ddl), `"id" integer not null primary key autoincrement`) { + t.Errorf("SQLite identity DDL missing from %s", ddl) + } + } + sqliteTaskDDL := sqlite.NewCreateTable().Model((*task2026090101)(nil)).String() + sqliteMultipartDDL := sqlite.NewCreateTable().Model((*multipartUpload2026090101)(nil)).String() + sqliteDataSetDDL := sqlite.NewCreateTable().Model((*storageDataSet2026090101)(nil)).String() + sqliteReplacementDDL := sqlite.NewCreateTable().Model((*storageReplacement2026090101)(nil)).String() + assertDDLContains(t, sqliteTaskDDL, `"type" text not null`, `"input_version" integer not null`, `"retry_limit" integer`) + assertDDLNotContains(t, sqliteTaskDDL, `"retry_limit" integer not null`) + assertDDLContains(t, sqliteDataSetDDL, `"generation" integer not null default 1`, `"copy_index" integer not null`) + assertDDLContains(t, sqliteReplacementDDL, `"items_total" integer not null default 0`, `"items_copied" integer not null default 0`) + assertDDLContains(t, sqliteMultipartDDL, `primary key ("upload_id")`) + assertDDLNotContains(t, sqliteMultipartDDL, `"id" integer`) +} + +func TestSQLiteBaselineJSONUsesTextAndValidatesShape(t *testing.T) { + db := newSQLiteMigrationDB(t, "sqlite_json_contract") + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatalf("create initial schema: %v", err) + } + + for table, columns := range map[string][]string{ + "task_payloads": {"input_json", "checkpoint_json"}, + "multipart_uploads": {"metadata"}, + "object_versions": {"metadata"}, + "observability_provider_states": {"reason_codes", "evidence_json"}, + "observability_data_set_states": {"reason_codes", "evidence_json"}, + } { + applied := appliedSQLiteColumns(t, db, table) + for _, name := range columns { + found := false + for _, column := range applied { + if column.Name == name { + found = true + if column.Type != "text" { + t.Errorf("%s.%s type = %s, want text", table, name, column.Type) + } + } + } + if !found { + t.Errorf("missing JSON column %s.%s", table, name) + } + } + } + + if _, err := db.Exec(`INSERT INTO tasks + (id, type, idempotency_key, input_version, input_hash, available_at, created_at, updated_at) + VALUES (1, 'json-test', 'valid', 1, 'hash', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`); err != nil { + t.Fatalf("insert task: %v", err) + } + if _, err := db.Exec(`INSERT INTO task_payloads (task_id, input_json) VALUES (1, '{}')`); err != nil { + t.Fatalf("insert valid object JSON: %v", err) + } + if _, err := db.Exec(`INSERT INTO task_payloads (task_id, input_json) VALUES (1, '[]')`); err == nil { + t.Fatal("object-shaped task input accepted an array") + } + if _, err := db.Exec(`INSERT INTO task_payloads (task_id, input_json) VALUES (1, '{')`); err == nil { + t.Fatal("task input accepted invalid JSON") + } +} + +func TestAppliedBaselineCarriesNoTimestampDefaults(t *testing.T) { + // A DEFAULT current_timestamp column is written by the database whenever bun + // omits it, and SQLite renders that as second-granularity local text while + // bun renders a fractional offset timestamp. One column would then hold two + // encodings whose lexical order disagrees with time inside the same second. + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatalf("create initial schema: %v", err) + } + for _, table := range applicationSchemaTables(t, db) { + for _, column := range appliedTableColumns(t, db, table) { + if strings.Contains(strings.ToLower(column.Default), "current_timestamp") { + t.Errorf("%s.%s still defaults to the database clock: %s", table, column.Name, column.Default) + } + } + } + }) +} + +func TestFrozenBaselineIndexDDLRetainsPredicates(t *testing.T) { + for name, db := range map[string]*bun.DB{ + "Postgres": newOfflinePostgresDDLDB(t), + "SQLite": newOfflineSQLiteDDLDB(t), + } { + t.Run(name, func(t *testing.T) { + query := db.NewCreateIndex(). + Index("idx_tasks_pending"). + Table("tasks"). + ColumnExpr("available_at"). + ColumnExpr("id"). + Where("status = 'pending'") + buf, err := query.AppendQuery(db.QueryGen(), nil) + if err != nil { + t.Fatalf("render index DDL: %v", err) + } + ddl := string(buf) + assertDDLContains(t, ddl, `"idx_tasks_pending"`, `available_at, id`, `where (status = 'pending')`) + }) + } +} + +func newOfflinePostgresDDLDB(t *testing.T) *bun.DB { + t.Helper() + sqldb, err := sql.Open("pgx", "postgres://invalid") + if err != nil { + t.Fatalf("open offline PostgreSQL dialect: %v", err) + } + db := bun.NewDB(sqldb, pgdialect.New()) + t.Cleanup(func() { _ = db.Close() }) + return db +} + +func newOfflineSQLiteDDLDB(t *testing.T) *bun.DB { + t.Helper() + sqldb, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("open offline SQLite dialect: %v", err) + } + db := bun.NewDB(sqldb, sqlitedialect.New()) + t.Cleanup(func() { _ = db.Close() }) + return db +} + +func assertDDLContains(t *testing.T, ddl string, fragments ...string) { + t.Helper() + ddl = strings.ToLower(ddl) + for _, fragment := range fragments { + fragment = strings.ToLower(fragment) + if !strings.Contains(ddl, fragment) { + t.Errorf("DDL %q does not contain %q", ddl, fragment) + } + } +} + +func assertDDLNotContains(t *testing.T, ddl string, fragments ...string) { + t.Helper() + ddl = strings.ToLower(ddl) + for _, fragment := range fragments { + fragment = strings.ToLower(fragment) + if strings.Contains(ddl, fragment) { + t.Errorf("DDL %q unexpectedly contains %q", ddl, fragment) + } + } +} diff --git a/internal/db/migrations/schema_enum_alignment_test.go b/internal/db/migrations/schema_enum_alignment_test.go new file mode 100644 index 0000000..26241a4 --- /dev/null +++ b/internal/db/migrations/schema_enum_alignment_test.go @@ -0,0 +1,228 @@ +package migrations + +import ( + "go/ast" + "go/build" + "go/constant" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "testing" + + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" +) + +type closedEnumCheck struct { + directory string + typeName string + table string + constraint string +} + +func TestClosedLifecycleEnumsMatchAppliedChecks(t *testing.T) { + checks := []closedEnumCheck{ + {directory: "model", typeName: "TaskStatus", table: "tasks", constraint: "chk_tasks_status"}, + {directory: "model", typeName: "TaskResumeMode", table: "tasks", constraint: "chk_tasks_resume_mode"}, + {directory: "model", typeName: "BucketStatus", table: "buckets", constraint: "chk_buckets_status"}, + {directory: "model", typeName: "BucketReplicaSlotStatus", table: "bucket_replica_slots", constraint: "chk_bucket_replica_slots_status"}, + {directory: "model", typeName: "MultipartStatus", table: "multipart_uploads", constraint: "chk_multipart_uploads_status"}, + {directory: "model", typeName: "StorageDataSetStatus", table: "storage_data_sets", constraint: "chk_storage_data_sets_status"}, + {directory: "model", typeName: "StorageCopyTransferMethod", table: "storage_copies", constraint: "chk_storage_copies_transfer_method"}, + {directory: "model", typeName: "StorageCopyStatus", table: "storage_copies", constraint: "chk_storage_copies_status"}, + {directory: "model", typeName: "StorageCleanupCopyStatus", table: "storage_cleanup_copies", constraint: "chk_storage_cleanup_copies_status"}, + {directory: "model", typeName: "WalletOperationType", table: "wallet_operations", constraint: "chk_wallet_operations_type"}, + {directory: "model", typeName: "WalletOperationStatus", table: "wallet_operations", constraint: "chk_wallet_operations_status"}, + {directory: "observability", typeName: "CollectionType", table: "observability_collection_states", constraint: "chk_observability_collection_type"}, + {directory: "observability", typeName: "Status", table: "observability_provider_states", constraint: "chk_observability_provider_status"}, + {directory: "observability", typeName: "Status", table: "observability_data_set_states", constraint: "chk_observability_data_set_status"}, + {directory: "storagecommit", typeName: "AttemptStatus", table: "storage_commit_attempts", constraint: "chk_storage_commit_attempts_status"}, + {directory: "storagepull", typeName: "AttemptStatus", table: "storage_pull_attempts", constraint: "chk_storage_pull_attempts_status"}, + {directory: "storagereplacement", typeName: "SelectionMode", table: "storage_replacements", constraint: "chk_storage_replacements_selection_mode"}, + {directory: "storagereplacement", typeName: "Status", table: "storage_replacements", constraint: "chk_storage_replacements_status"}, + {directory: "storagereplacement", typeName: "TerminationRole", table: "storage_data_set_terminations", constraint: "chk_storage_data_set_terminations_role"}, + {directory: "storagereplacement", typeName: "ItemStatus", table: "storage_replacement_items", constraint: "chk_storage_replacement_items_status"}, + } + + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatalf("create initial schema: %v", err) + } + for _, check := range checks { + t.Run(check.table+"/"+check.typeName, func(t *testing.T) { + goValues := typedStringConstants(t, filepath.Join("..", "..", check.directory), check.typeName) + ddlValues := appliedCheckStringValues(t, db, check.table, check.constraint) + if !slices.Equal(goValues, ddlValues) { + t.Fatalf("%s constants = %v, %s.%s values = %v", check.typeName, goValues, check.table, check.constraint, ddlValues) + } + }) + } + }) +} + +func typedStringConstants(t *testing.T, directory, typeName string) []string { + t.Helper() + fset := token.NewFileSet() + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatalf("read %s: %v", directory, err) + } + var packageName string + var declarations []ast.Decl + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".go" || strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + matches, err := build.Default.MatchFile(directory, entry.Name()) + if err != nil { + t.Fatalf("match build constraints for %s: %v", entry.Name(), err) + } + if !matches { + continue + } + file, err := parser.ParseFile(fset, filepath.Join(directory, entry.Name()), nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", entry.Name(), err) + } + if packageName == "" { + packageName = file.Name.Name + } else if packageName != file.Name.Name { + t.Fatalf("parse %s: found package %s alongside %s", directory, file.Name.Name, packageName) + } + for _, declaration := range file.Decls { + general, ok := declaration.(*ast.GenDecl) + if !ok { + continue + } + switch general.Tok { + case token.TYPE: + for _, spec := range general.Specs { + typeSpec, ok := spec.(*ast.TypeSpec) + if ok && typeSpec.Name.Name == typeName { + declarations = append(declarations, &ast.GenDecl{Tok: token.TYPE, Specs: []ast.Spec{typeSpec}}) + } + } + case token.CONST: + if constDeclarationDefinesType(general, typeName) { + // Keep the complete group so go/types can apply ConstSpec + // inheritance and iota semantics before we filter by type. + declarations = append(declarations, general) + } + } + } + } + if len(declarations) == 0 { + t.Fatalf("no declarations found for %s in %s", typeName, directory) + } + synthetic := &ast.File{Name: ast.NewIdent(packageName), Decls: declarations} + typedPackage, err := new(types.Config).Check(packageName, fset, []*ast.File{synthetic}, nil) + if err != nil { + t.Fatalf("type-check %s.%s: %v", packageName, typeName, err) + } + var values []string + for _, name := range typedPackage.Scope().Names() { + object, ok := typedPackage.Scope().Lookup(name).(*types.Const) + if !ok { + continue + } + named, ok := object.Type().(*types.Named) + if !ok || named.Obj().Name() != typeName || object.Val().Kind() != constant.String { + continue + } + values = append(values, constant.StringVal(object.Val())) + } + slices.Sort(values) + return slices.Compact(values) +} + +func constDeclarationDefinesType(declaration *ast.GenDecl, typeName string) bool { + for _, spec := range declaration.Specs { + valueSpec, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + identifier, ok := valueSpec.Type.(*ast.Ident) + if ok && identifier.Name == typeName { + return true + } + } + return false +} + +var singleQuotedSQLValue = regexp.MustCompile(`'(?:''|[^'])*'`) + +func appliedCheckStringValues(t *testing.T, db *bun.DB, table, constraint string) []string { + t.Helper() + var definition string + if db.Dialect().Name() == dialect.PG { + if err := db.NewRaw(`SELECT pg_get_constraintdef(constraint_info.oid) + FROM pg_constraint AS constraint_info + JOIN pg_class AS table_info ON table_info.oid = constraint_info.conrelid + JOIN pg_namespace AS namespace_info ON namespace_info.oid = table_info.relnamespace + WHERE namespace_info.nspname = current_schema() + AND table_info.relname = ? + AND constraint_info.conname = ?`, table, constraint).Scan(t.Context(), &definition); err != nil { + t.Fatalf("read PostgreSQL check %s.%s: %v", table, constraint, err) + } + } else { + var ddl string + if err := db.NewRaw(`SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = ?`, table).Scan(t.Context(), &ddl); err != nil { + t.Fatalf("read SQLite DDL for %s: %v", table, err) + } + definition = namedCheckExpression(t, ddl, constraint) + } + values := make([]string, 0) + for _, quoted := range singleQuotedSQLValue.FindAllString(definition, -1) { + values = append(values, strings.ReplaceAll(quoted[1:len(quoted)-1], "''", "'")) + } + slices.Sort(values) + return slices.Compact(values) +} + +func namedCheckExpression(t *testing.T, ddl, constraint string) string { + t.Helper() + lower := strings.ToLower(ddl) + start := strings.Index(lower, "constraint "+strings.ToLower(constraint)) + if start < 0 { + t.Fatalf("constraint %s not found in %s", constraint, ddl) + } + check := strings.Index(lower[start:], "check") + if check < 0 { + t.Fatalf("CHECK keyword for %s not found in %s", constraint, ddl) + } + open := strings.Index(ddl[start+check:], "(") + if open < 0 { + t.Fatalf("CHECK expression for %s not found in %s", constraint, ddl) + } + open += start + check + depth := 0 + inString := false + for index := open; index < len(ddl); index++ { + switch ddl[index] { + case '\'': + if inString && index+1 < len(ddl) && ddl[index+1] == '\'' { + index++ + continue + } + inString = !inString + case '(': + if !inString { + depth++ + } + case ')': + if !inString { + depth-- + if depth == 0 { + return ddl[open+1 : index] + } + } + } + } + t.Fatalf("unterminated CHECK expression for %s in %s", constraint, ddl) + return "" +} diff --git a/internal/db/migrations/schema_fk_index_test.go b/internal/db/migrations/schema_fk_index_test.go new file mode 100644 index 0000000..e07e5e4 --- /dev/null +++ b/internal/db/migrations/schema_fk_index_test.go @@ -0,0 +1,162 @@ +package migrations + +import ( + "database/sql" + "strconv" + "testing" + + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" +) + +func TestEveryForeignKeyHasLeadingChildIndexCoverage(t *testing.T) { + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatalf("create initial schema: %v", err) + } + for _, table := range applicationSchemaTables(t, db) { + indexed := appliedLeadingIndexColumns(t, db, table) + for _, foreignKey := range appliedForeignKeyLeadingColumns(t, db, table) { + if !indexed[foreignKey.column] { + t.Errorf("%s foreign key %s begins with unindexed child column %s", table, foreignKey.name, foreignKey.column) + } + } + } + }) +} + +type appliedForeignKeyLeadingColumn struct { + name string + column string +} + +func appliedForeignKeyLeadingColumns(t *testing.T, db *bun.DB, table string) []appliedForeignKeyLeadingColumn { + t.Helper() + if db.Dialect().Name() == dialect.PG { + rows, err := db.Query(`SELECT constraint_info.conname, column_info.attname + FROM pg_constraint AS constraint_info + JOIN pg_class AS table_info ON table_info.oid = constraint_info.conrelid + JOIN pg_namespace AS namespace_info ON namespace_info.oid = table_info.relnamespace + CROSS JOIN LATERAL unnest(constraint_info.conkey) + WITH ORDINALITY AS key_column(attnum, position) + JOIN pg_attribute AS column_info + ON column_info.attrelid = table_info.oid AND column_info.attnum = key_column.attnum + WHERE constraint_info.contype = 'f' + AND namespace_info.nspname = current_schema() + AND table_info.relname = ? + AND key_column.position = 1 + ORDER BY constraint_info.conname`, table) + if err != nil { + t.Fatalf("read PostgreSQL foreign keys for %s: %v", table, err) + } + defer func() { _ = rows.Close() }() + var result []appliedForeignKeyLeadingColumn + for rows.Next() { + var item appliedForeignKeyLeadingColumn + if err := rows.Scan(&item.name, &item.column); err != nil { + t.Fatalf("scan PostgreSQL foreign key for %s: %v", table, err) + } + result = append(result, item) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate PostgreSQL foreign keys for %s: %v", table, err) + } + return result + } + + rows, err := db.Query(`SELECT id, "from" FROM pragma_foreign_key_list(?) WHERE seq = 0 ORDER BY id`, table) + if err != nil { + t.Fatalf("read SQLite foreign keys for %s: %v", table, err) + } + defer func() { _ = rows.Close() }() + var result []appliedForeignKeyLeadingColumn + for rows.Next() { + var id int + var column string + if err := rows.Scan(&id, &column); err != nil { + t.Fatalf("scan SQLite foreign key for %s: %v", table, err) + } + result = append(result, appliedForeignKeyLeadingColumn{name: table + "#" + strconv.Itoa(id), column: column}) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate SQLite foreign keys for %s: %v", table, err) + } + return result +} + +func appliedLeadingIndexColumns(t *testing.T, db *bun.DB, table string) map[string]bool { + t.Helper() + indexed := make(map[string]bool) + if db.Dialect().Name() == dialect.PG { + rows, err := db.Query(`SELECT column_info.attname + FROM pg_index AS index_info + JOIN pg_class AS table_info ON table_info.oid = index_info.indrelid + JOIN pg_namespace AS namespace_info ON namespace_info.oid = table_info.relnamespace + JOIN pg_attribute AS column_info + ON column_info.attrelid = table_info.oid + AND column_info.attnum = index_info.indkey[0] + WHERE namespace_info.nspname = current_schema() + AND table_info.relname = ?`, table) + if err != nil { + t.Fatalf("read PostgreSQL indexes for %s: %v", table, err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var column string + if err := rows.Scan(&column); err != nil { + t.Fatalf("scan PostgreSQL index for %s: %v", table, err) + } + indexed[column] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate PostgreSQL indexes for %s: %v", table, err) + } + return indexed + } + + rows, err := db.Query(`SELECT name, pk FROM pragma_table_info(?)`, table) + if err != nil { + t.Fatalf("read SQLite primary key for %s: %v", table, err) + } + for rows.Next() { + var column string + var primaryKey int + if err := rows.Scan(&column, &primaryKey); err != nil { + _ = rows.Close() + t.Fatalf("scan SQLite primary key for %s: %v", table, err) + } + if primaryKey == 1 { + indexed[column] = true + } + } + if err := rows.Close(); err != nil { + t.Fatalf("close SQLite primary key rows for %s: %v", table, err) + } + + indexRows, err := db.Query(`SELECT name FROM pragma_index_list(?)`, table) + if err != nil { + t.Fatalf("read SQLite index list for %s: %v", table, err) + } + var indexNames []string + for indexRows.Next() { + var name string + if err := indexRows.Scan(&name); err != nil { + _ = indexRows.Close() + t.Fatalf("scan SQLite index list for %s: %v", table, err) + } + indexNames = append(indexNames, name) + } + if err := indexRows.Close(); err != nil { + t.Fatalf("close SQLite index list for %s: %v", table, err) + } + for _, index := range indexNames { + var column sql.NullString + if err := db.NewRaw(`SELECT name FROM pragma_index_xinfo(?) WHERE seqno = 0 AND "key" = 1`, index).Scan(t.Context(), &column); err != nil { + t.Fatalf("read SQLite leading index column for %s: %v", index, err) + } + if column.Valid { + indexed[column.String] = true + } + } + return indexed +} diff --git a/internal/db/migrations/schema_integrity_test.go b/internal/db/migrations/schema_integrity_test.go index b89e75e..7f04786 100644 --- a/internal/db/migrations/schema_integrity_test.go +++ b/internal/db/migrations/schema_integrity_test.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "database/sql" + "errors" "fmt" "go/parser" "go/token" @@ -15,22 +16,34 @@ import ( "testing" "time" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/stdlib" "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" "github.com/uptrace/bun/dialect/pgdialect" "github.com/uptrace/bun/dialect/sqlitedialect" "github.com/uptrace/bun/migrate" - - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/stdlib" _ "modernc.org/sqlite" ) const ( - initialSQLiteSchemaFingerprint = "a76c972a38ca0fd958625a4e1a6ca69f69a7dc67ffe7abe0bb0bff06481044bb" - initialPostgresSchemaFingerprint = "91bc1d871d84e05ae3f995a10c14a21276fbe39eb12df4aed3aab9138a5ea964" + initialPortableSchemaFingerprint = "82c448b3d6eb4bc627cc6ae913091c21c8b440903dd09a85032a653515a80323" ) +func TestMigrationRegistryStartsWithUniqueOrderedBaseline(t *testing.T) { + migrations := Migrations.Sorted() + if len(migrations) == 0 { + t.Fatal("migration registry is empty") + } + if migrations[0].Name != InitialSchemaName { + t.Fatalf("migration name = %q, want %q", migrations[0].Name, InitialSchemaName) + } + for i := 1; i < len(migrations); i++ { + if migrations[i-1].Name >= migrations[i].Name { + t.Fatalf("migration names are not unique and ordered: %q then %q", migrations[i-1].Name, migrations[i].Name) + } + } +} + func TestMigrationFilesDoNotImportRuntimePackages(t *testing.T) { files, err := filepath.Glob("*.go") if err != nil { @@ -56,196 +69,390 @@ func TestMigrationFilesDoNotImportRuntimePackages(t *testing.T) { } } -func TestInitialSchemaFingerprintSQLite(t *testing.T) { - db := newSQLiteMigrationDB(t, "initial_schema_fingerprint") - ctx := context.Background() - if err := runMigrationBody(ctx, db, up2026040501Init); err != nil { - t.Fatalf("create initial schema: %v", err) - } - got, _ := sqliteSchemaFingerprint(t, db, false) - if got != initialSQLiteSchemaFingerprint { - t.Fatalf("initial SQLite schema fingerprint = %s, want %s", got, initialSQLiteSchemaFingerprint) - } -} - func TestInitialSchemaFingerprintPostgres(t *testing.T) { db := newPostgresMigrationDB(t) - ctx := context.Background() - if err := runMigrationBody(ctx, db, up2026040501Init); err != nil { + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { t.Fatalf("create initial schema: %v", err) } - got, _ := postgresSchemaFingerprint(t, db, false) - if got != initialPostgresSchemaFingerprint { - t.Fatalf("initial PostgreSQL schema fingerprint = %s, want %s", got, initialPostgresSchemaFingerprint) + got, schema := portableSchemaFingerprint(t, db) + sqliteDB := newSQLiteMigrationDB(t, "postgres_portable_schema_comparison") + if err := runMigrationBody(t.Context(), sqliteDB, up2026090101InitialSchema); err != nil { + t.Fatalf("create comparison SQLite schema: %v", err) + } + sqliteFingerprint, sqliteSchema := portableSchemaFingerprint(t, sqliteDB) + if got != sqliteFingerprint { + t.Fatalf( + "portable schema differs by dialect: PostgreSQL=%s SQLite=%s\n%s", + got, + sqliteFingerprint, + semanticSchemaDifference(sqliteSchema, schema), + ) + } + if got != initialPortableSchemaFingerprint { + t.Fatalf("initial PostgreSQL portable schema fingerprint = %s, want %s\n%s", got, initialPortableSchemaFingerprint, schema) } } -func TestLegacyMigrationUpgradePreservesDataAndIsIdempotent(t *testing.T) { - testMigrationDialects(t, testLegacyMigrationUpgradePreservesDataAndIsIdempotent) +func semanticSchemaDifference(want, got string) string { + wantLines := strings.Split(want, "\n") + gotLines := strings.Split(got, "\n") + wantSet := make(map[string]struct{}, len(wantLines)) + gotSet := make(map[string]struct{}, len(gotLines)) + for _, line := range wantLines { + wantSet[line] = struct{}{} + } + for _, line := range gotLines { + gotSet[line] = struct{}{} + } + difference := make([]string, 0) + for _, line := range wantLines { + if _, ok := gotSet[line]; !ok { + difference = append(difference, "- "+line) + } + } + for _, line := range gotLines { + if _, ok := wantSet[line]; !ok { + difference = append(difference, "+ "+line) + } + } + return strings.Join(difference, "\n") } -func testLegacyMigrationUpgradePreservesDataAndIsIdempotent(t *testing.T, db *bun.DB) { - ctx := context.Background() - migrator := NewMigrator(db) - if err := migrator.Init(ctx); err != nil { - t.Fatalf("initialize legacy migrator: %v", err) - } - if err := runMigrationBody(ctx, db, up2026040501Init); err != nil { - t.Fatalf("create legacy initial schema: %v", err) +func TestInitialSchemaPortableFingerprintSQLite(t *testing.T) { + db := newSQLiteMigrationDB(t, "portable_schema_fingerprint") + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatalf("create initial schema: %v", err) } - markAppliedMigration(t, ctx, migrator, "2026040501", 1) - seedLegacyMigrationData(t, db) - walletBefore := readLegacyWalletRow(t, db) - if _, err := migrator.Migrate(ctx); err != nil { - t.Fatalf("upgrade legacy schema: %v", err) + got, schema := portableSchemaFingerprint(t, db) + if got != initialPortableSchemaFingerprint { + t.Fatalf("initial SQLite portable schema fingerprint = %s, want %s\n%s", got, initialPortableSchemaFingerprint, schema) } +} - // 2026062201 rebuilds wallet_operations on SQLite by copying rows. - if walletAfter := readLegacyWalletRow(t, db); walletAfter != walletBefore { - t.Fatalf("wallet row changed across the upgrade:\n before=%s\n after =%s", walletBefore, walletAfter) +func TestInitialSchemaContractSQLite(t *testing.T) { + db := newSQLiteMigrationDB(t, "initial_schema_contract") + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatalf("create initial schema: %v", err) } - - var generation int - var current bool - if err := db.NewRaw("SELECT generation, is_current FROM storage_data_sets WHERE id = 1").Scan(ctx, &generation, ¤t); err != nil { - t.Fatalf("read upgraded legacy data: %v", err) + for _, table := range []string{ + "s3_accounts", "buckets", "bucket_replica_slots", "objects", "object_versions", "object_cache", "object_deletions", + "multipart_uploads", "multipart_parts", "storage_contents", "storage_data_sets", + "storage_copies", "storage_commit_attempts", "storage_replacements", + "storage_pull_attempts", "storage_replacement_items", "storage_cleanup_copies", "wallet_operations", "tasks", + "observability_collection_states", "observability_provider_states", "observability_data_set_states", + "task_payloads", "storage_data_set_terminations", + } { + if exists, err := tableExists(t.Context(), db, table); err != nil || !exists { + t.Errorf("table %s exists=%t err=%v", table, exists, err) + } } - if generation != 1 || !current { - t.Fatalf("upgraded data generation/current = %d/%v, want 1/true", generation, current) + for _, column := range []struct{ table, name string }{ + {"tasks", "claim_generation"}, + {"task_payloads", "checkpoint_json"}, + {"storage_data_set_terminations", "epoch"}, + {"storage_copies", "active_task_id"}, + {"storage_copies", "bucket_id"}, + {"storage_copies", "content_size"}, + {"storage_copies", "storage_data_set_id"}, + {"storage_copies", "ingress_bytes_transferred"}, + {"storage_commit_attempts", "attempt_id"}, + {"storage_pull_attempts", "attempt_id"}, + {"storage_pull_attempts", "source_piece_cid"}, + {"storage_contents", "content_size"}, + {"storage_data_sets", "ensure_task_id"}, + {"storage_replacement_items", "target_data_set_id"}, + {"storage_cleanup_copies", "bucket_id"}, + {"object_versions", "content_id"}, + {"object_cache", "cache_active_task_id"}, + {"wallet_operations", "broadcast_attempted_at"}, + } { + if exists, err := columnExists(t.Context(), db, column.table, column.name); err != nil || !exists { + t.Errorf("column %s.%s exists=%t err=%v", column.table, column.name, exists, err) + } } - group, err := migrator.Migrate(ctx) - if err != nil { - t.Fatalf("repeat migrations: %v", err) + for _, index := range []string{ + "idx_tasks_pending", + "idx_tasks_recovery", + "idx_tasks_gc", + "idx_storage_copies_commit_ready", + "idx_storage_commit_attempts_unresolved_copy", + "idx_storage_copies_ingress_content", + "idx_storage_data_sets_bucket_provider_active", + "idx_storage_replacements_active_bucket_slot", + "idx_wallet_operations_recent", + } { + if exists, err := indexExists(t.Context(), db, index); err != nil || !exists { + t.Errorf("index %s exists=%t err=%v", index, exists, err) + } } - if len(group.Migrations) != 0 { - t.Fatalf("repeat migrations applied %d migrations, want none", len(group.Migrations)) + for _, index := range []string{ + "idx_storage_data_sets_replica_slot", + "idx_storage_replacements_replica_slot", + } { + if exists, err := indexExists(t.Context(), db, index); err != nil || exists { + t.Errorf("removed index %s exists=%t err=%v", index, exists, err) + } + } + for _, column := range []struct{ table, name string }{ + {"tasks", "stage"}, + {"tasks", "category"}, + {"tasks", "parent_task_id"}, + {"tasks", "workflow_id"}, + {"tasks", "priority"}, + {"tasks", "lane"}, + {"storage_replacement_items", "claimed_at"}, + {"storage_replacement_items", "lease_until"}, + {"storage_replacement_items", "scheduled_at"}, + {"storage_replacement_items", "retry_count"}, + {"storage_replacement_items", "task_id"}, + {"storage_replacement_items", "target_copy_id"}, + {"multipart_uploads", "id"}, + {"storage_copies", "commit_attempt_id"}, + {"storage_copies", "commit_attempted_at"}, + {"storage_copies", "commit_transaction_id"}, + {"storage_copies", "commit_submission_json"}, + {"storage_copies", "commit_confirmed_transaction_id"}, + {"storage_copies", "commit_attention_code"}, + {"storage_copies", "commit_attention_at"}, + {"storage_copies", "is_new_data_set"}, + // Pull identity is a ledger row now, not five nullable copy columns. + {"storage_copies", "pull_request_id"}, + {"storage_copies", "pull_source_provider_id"}, + {"storage_copies", "pull_source_data_set_id"}, + {"storage_copies", "pull_source_piece_id"}, + {"storage_copies", "pull_source_retrieval_url"}, + {"storage_pull_attempts", "request_id"}, + // Pipeline position, cache residency and the content status column are + // derived or moved; reintroducing any of them re-creates a second + // authority for a fact the copies or object_cache already own. + {"object_versions", "is_current"}, + {"object_versions", "state"}, + {"object_versions", "failed_at_state"}, + {"object_versions", "last_error"}, + {"object_versions", "checksum"}, + {"object_versions", "storage_upload_id"}, + {"object_versions", "cache_key"}, + {"object_versions", "in_cache"}, + {"object_versions", "cache_accessed_at"}, + {"object_versions", "cache_presence_generation"}, + {"object_versions", "cache_operation_generation"}, + {"object_versions", "cache_active_task_id"}, + {"storage_contents", "status"}, + {"storage_contents", "state"}, + {"storage_contents", "disposition"}, + {"storage_contents", "superseded_by_id"}, + {"storage_contents", "committed_slots"}, + {"storage_contents", "ingress_bytes_transferred"}, + {"storage_contents", "created_from_version_id"}, + {"observability_provider_states", "created_at"}, + {"observability_provider_states", "updated_at"}, + {"observability_data_set_states", "created_at"}, + {"observability_data_set_states", "updated_at"}, + {"object_deletions", "cache_cleanup_status"}, + {"object_deletions", "cache_error"}, + // The JSON a task carries and the terminations a replacement records + // are rows of their own; putting either back re-creates the write + // amplification and the repeated column group they were split out of. + {"tasks", "input_json"}, + {"tasks", "checkpoint_json"}, + {"storage_replacements", "termination_tx_hash"}, + {"storage_replacements", "termination_epoch"}, + {"storage_replacements", "termination_observed_at"}, + {"storage_replacements", "abandoned_termination_tx_hash"}, + {"storage_replacements", "abandoned_termination_epoch"}, + {"storage_replacements", "abandoned_termination_observed_at"}, + {"storage_replacements", "confirmed_at"}, + } { + if exists, err := columnExists(t.Context(), db, column.table, column.name); err != nil || exists { + t.Errorf("removed column %s.%s exists=%t err=%v", column.table, column.name, exists, err) + } } } -func TestFreshMigrationRollbackRemovesSchema(t *testing.T) { - testMigrationDialects(t, testFreshMigrationRollbackRemovesSchema) +func TestInitialSchemaTaskOwnerForeignKeysAreRestrictive(t *testing.T) { + db := newSQLiteMigrationDB(t, "task_owner_foreign_keys") + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatalf("create initial schema: %v", err) + } + want := map[string]map[string]bool{ + "buckets": {"durability_task_id": false}, + "object_cache": {"cache_active_task_id": false}, + "storage_contents": {"cleanup_task_id": false}, + "storage_data_sets": {"ensure_task_id": false, "retirement_task_id": false}, + "storage_copies": {"active_task_id": false}, + "storage_replacements": {"task_id": false}, + "wallet_operations": {"task_id": false}, + } + for table, columns := range want { + rows, err := db.Query(`SELECT "from", "table", on_delete FROM pragma_foreign_key_list(?)`, table) + if err != nil { + t.Fatalf("query foreign keys for %s: %v", table, err) + } + for rows.Next() { + var from, target, onDelete string + if err := rows.Scan(&from, &target, &onDelete); err != nil { + _ = rows.Close() + t.Fatalf("scan foreign key for %s: %v", table, err) + } + if _, tracked := columns[from]; !tracked || target != "tasks" { + continue + } + if onDelete != "RESTRICT" { + _ = rows.Close() + t.Fatalf("%s.%s ON DELETE = %s, want RESTRICT", table, from, onDelete) + } + columns[from] = true + } + if err := rows.Close(); err != nil { + t.Fatalf("close foreign keys for %s: %v", table, err) + } + for column, found := range columns { + if !found { + t.Errorf("missing task ownership foreign key %s.%s", table, column) + } + } + } } -func testFreshMigrationRollbackRemovesSchema(t *testing.T, db *bun.DB) { - ctx := context.Background() - migrator := NewMigrator(db) - if err := migrator.Init(ctx); err != nil { - t.Fatalf("initialize fresh migrator: %v", err) +func TestValidateTargetRejectsLegacyDatabaseWithoutModification(t *testing.T) { + db := newSQLiteMigrationDB(t, "legacy_rejection") + if _, err := db.Exec(`CREATE TABLE tasks (id INTEGER PRIMARY KEY, status TEXT)`); err != nil { + t.Fatalf("seed legacy table: %v", err) } - if _, err := migrator.Migrate(ctx); err != nil { - t.Fatalf("migrate fresh schema: %v", err) + if _, err := db.Exec(`INSERT INTO tasks (id, status) VALUES (7, 'running')`); err != nil { + t.Fatalf("seed legacy row: %v", err) } - if _, err := migrator.Rollback(ctx); err != nil { - t.Fatalf("rollback empty fresh schema: %v", err) + err := ValidateTarget(t.Context(), db) + if !errors.Is(err, ErrIncompatibleDatabase) { + t.Fatalf("ValidateTarget error = %v, want ErrIncompatibleDatabase", err) } - domainTables, err := countDomainTables(ctx, db) - if err != nil { - t.Fatalf("count tables after rollback: %v", err) - } - if domainTables != 0 { - t.Fatalf("rollback left %d domain tables, want none", domainTables) + var status string + if err := db.NewRaw(`SELECT status FROM tasks WHERE id = 7`).Scan(t.Context(), &status); err != nil { + t.Fatalf("read legacy row after rejection: %v", err) } - applied, err := migrator.AppliedMigrations(ctx) - if err != nil { - t.Fatalf("read migrations after rollback: %v", err) + if status != "running" { + t.Fatalf("legacy row status = %q, want unchanged", status) } - if len(applied) != 0 { - t.Fatalf("rollback left %d applied migrations, want none", len(applied)) + if exists, err := tableExists(t.Context(), db, "bun_migrations"); err != nil || exists { + t.Fatalf("migration marker exists=%t err=%v after rejection", exists, err) } } -func markAppliedMigration(t *testing.T, ctx context.Context, migrator interface { - MarkApplied(context.Context, *migrate.Migration) error -}, name string, groupID int64, -) { - t.Helper() - for _, migration := range Migrations.Sorted() { - if migration.Name != name { - continue - } - migration.GroupID = groupID - if err := migrator.MarkApplied(ctx, &migration); err != nil { - t.Fatalf("mark migration %s applied: %v", name, err) - } - return - } - t.Fatalf("migration %s not found", name) -} +func TestValidateTargetAcceptsOnlyAppliedMigrationPrefixes(t *testing.T) { + registry := migrate.NewMigrations() + registry.Add(migrate.Migration{Name: InitialSchemaName}) + registry.Add(migrate.Migration{Name: "2026090201"}) + registry.Add(migrate.Migration{Name: "2026090301"}) -func seedLegacyMigrationData(t *testing.T, db *bun.DB) { - t.Helper() - ctx := context.Background() - statements := []string{ - `INSERT INTO buckets (id, name) VALUES (1, 'legacy-migration-bucket')`, - `INSERT INTO storage_uploads - (id, bucket_id, content_size, checksum, requested_copies) - VALUES (1, 1, 1, 'checksum', 1)`, - `INSERT INTO storage_data_sets - (id, bucket_id, provider_id, copy_index, status) - VALUES (1, 1, '101', 0, 'ready')`, - `INSERT INTO wallet_operations - (id, type, client_request_id, amount, status, tx_hash, last_error, - lease_until, started_at, submitted_at, completed_at, created_at, updated_at) - VALUES (1, 'fund', 'legacy-fund', '1234', 'confirmed', '0xfeed', 'boom', - '2026-01-01 01:00:00', '2026-01-02 02:00:00', '2026-01-03 03:00:00', - '2026-01-04 04:00:00', '2026-01-05 05:00:00', '2026-01-06 06:00:00')`, - } - for _, statement := range statements { - if _, err := db.ExecContext(ctx, statement); err != nil { - t.Fatalf("seed legacy migration data: %v", err) - } - } -} + tests := []struct { + name string + applied []string + wantErr bool + }{ + {name: "metadata only"}, + {name: "baseline", applied: []string{InitialSchemaName}}, + {name: "longer prefix", applied: []string{InitialSchemaName, "2026090201"}}, + {name: "full registry", applied: []string{InitialSchemaName, "2026090201", "2026090301"}}, + {name: "legacy marker", applied: []string{"2026040501"}, wantErr: true}, + {name: "unknown marker", applied: []string{InitialSchemaName, "2026090250"}, wantErr: true}, + {name: "duplicate marker", applied: []string{InitialSchemaName, InitialSchemaName}, wantErr: true}, + {name: "out of order", applied: []string{"2026090201", InitialSchemaName}, wantErr: true}, + {name: "gap", applied: []string{InitialSchemaName, "2026090301"}, wantErr: true}, + {name: "longer than registry", applied: []string{InitialSchemaName, "2026090201", "2026090301", "2026090401"}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + db := newSQLiteMigrationDB(t, "migration_prefix_"+strings.ReplaceAll(tt.name, " ", "_")) + if err := newMigrator(db, registry).Init(t.Context()); err != nil { + t.Fatalf("initialize migration metadata: %v", err) + } + for _, name := range tt.applied { + if _, err := db.Exec(`INSERT INTO bun_migrations (name, group_id) VALUES (?, 1)`, name); err != nil { + t.Fatalf("insert migration marker %q: %v", name, err) + } + } -func readLegacyWalletRow(t *testing.T, db *bun.DB) string { - t.Helper() - var ( - opType, requestID, amount, status string - txHash, lastError *string - lease, started, submitted, done *time.Time - created, updated time.Time - ) - if err := db.NewRaw(`SELECT type, client_request_id, amount, status, tx_hash, last_error, - lease_until, started_at, submitted_at, completed_at, created_at, updated_at - FROM wallet_operations WHERE id = 1`). - Scan(context.Background(), &opType, &requestID, &amount, &status, &txHash, &lastError, - &lease, &started, &submitted, &done, &created, &updated); err != nil { - t.Fatalf("read wallet row: %v", err) - } - return fmt.Sprintf("%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s", - opType, requestID, amount, status, derefString(txHash), derefString(lastError), - formatTime(lease), formatTime(started), formatTime(submitted), formatTime(done), - formatTime(&created), formatTime(&updated)) + err := validateTarget(t.Context(), db, registry) + if tt.wantErr { + if !errors.Is(err, ErrIncompatibleDatabase) { + t.Fatalf("validateTarget error = %v, want ErrIncompatibleDatabase", err) + } + } else if err != nil { + t.Fatalf("validateTarget error = %v", err) + } + + var names []string + if err := db.NewRaw(`SELECT name FROM bun_migrations ORDER BY id`).Scan(t.Context(), &names); err != nil { + t.Fatalf("read migration markers: %v", err) + } + if !slices.Equal(names, tt.applied) { + t.Fatalf("migration markers after validation = %v, want unchanged %v", names, tt.applied) + } + }) + } } -func derefString(v *string) string { - if v == nil { - return "" +func TestValidateTargetRejectsInvalidMigrationRegistry(t *testing.T) { + tests := []struct { + name string + registry *migrate.Migrations + }{ + {name: "empty", registry: migrate.NewMigrations()}, + {name: "missing baseline", registry: migrationRegistryForTest("2026090201")}, + {name: "duplicate", registry: migrationRegistryForTest(InitialSchemaName, InitialSchemaName)}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + db := newSQLiteMigrationDB(t, "invalid_registry_"+strings.ReplaceAll(tt.name, " ", "_")) + if err := validateTarget(t.Context(), db, tt.registry); !errors.Is(err, ErrIncompatibleDatabase) { + t.Fatalf("validateTarget error = %v, want ErrIncompatibleDatabase", err) + } + if count, err := applicationTableCount(t.Context(), db); err != nil || count != 0 { + t.Fatalf("application table count after rejection = %d, err=%v", count, err) + } + }) } - return *v } -func formatTime(v *time.Time) string { - if v == nil { - return "" +func migrationRegistryForTest(names ...string) *migrate.Migrations { + registry := migrate.NewMigrations() + for _, name := range names { + registry.Add(migrate.Migration{Name: name}) } - return v.UTC().Format(time.RFC3339Nano) + return registry } -func countDomainTables(ctx context.Context, db *bun.DB) (int, error) { - query := `SELECT COUNT(*) FROM sqlite_schema - WHERE type = 'table' AND name NOT LIKE 'sqlite_%' - AND name NOT IN ('bun_migrations', 'bun_migration_locks')` - if db.Dialect().Name() == dialect.PG { - query = `SELECT COUNT(*) FROM information_schema.tables - WHERE table_schema = current_schema() - AND table_name NOT IN ('bun_migrations', 'bun_migration_locks')` - } - var count int - if err := db.NewRaw(query).Scan(ctx, &count); err != nil { - return 0, err - } - return count, nil +func TestFreshBaselineIsIdempotentAndCannotRollback(t *testing.T) { + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + ctx := t.Context() + if err := ValidateTarget(ctx, db); err != nil { + t.Fatalf("validate empty target: %v", err) + } + migrator := NewMigrator(db) + if err := migrator.Init(ctx); err != nil { + t.Fatalf("initialize migrator: %v", err) + } + first, err := migrator.Migrate(ctx) + if err != nil { + t.Fatalf("migrate fresh schema: %v", err) + } + if len(first.Migrations) != 1 || first.Migrations[0].Name != InitialSchemaName { + t.Fatalf("first migration group = %#v", first.Migrations) + } + second, err := migrator.Migrate(ctx) + if err != nil { + t.Fatalf("repeat migration: %v", err) + } + if len(second.Migrations) != 0 { + t.Fatalf("repeat migration applied %d migrations", len(second.Migrations)) + } + if _, err := migrator.Rollback(ctx); err == nil { + t.Fatal("initial schema rollback succeeded") + } + if exists, err := tableExists(ctx, db, "tasks"); err != nil || !exists { + t.Fatalf("tasks table exists=%t err=%v after rejected rollback", exists, err) + } + }) } func runMigrationBody(ctx context.Context, db *bun.DB, body migrationBody) error { @@ -311,70 +518,3 @@ func normalizedFingerprint(lines []string) (string, string) { payload := strings.Join(lines, "\n") return fmt.Sprintf("%x", sha256.Sum256([]byte(payload))), payload } - -func sqliteSchemaFingerprint(t *testing.T, db *bun.DB, includeMigrations bool) (string, string) { - t.Helper() - query := `SELECT type, name, tbl_name, COALESCE(sql, '') - FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%'` - if !includeMigrations { - query += ` AND name NOT IN ('bun_migrations', 'bun_migration_locks')` - } - query += ` ORDER BY type, name` - rows, err := db.Query(query) - if err != nil { - t.Fatalf("query SQLite schema: %v", err) - } - defer func() { _ = rows.Close() }() - var lines []string - for rows.Next() { - var kind, name, table, ddl string - if err := rows.Scan(&kind, &name, &table, &ddl); err != nil { - t.Fatalf("scan SQLite schema: %v", err) - } - lines = append(lines, strings.Join([]string{kind, name, table, strings.Join(strings.Fields(ddl), " ")}, "|")) - } - if err := rows.Err(); err != nil { - t.Fatalf("iterate SQLite schema: %v", err) - } - return normalizedFingerprint(lines) -} - -func postgresSchemaFingerprint(t *testing.T, db *bun.DB, includeMigrations bool) (string, string) { - t.Helper() - var lines []string - queries := []string{ - `SELECT 'column|' || table_name || '|' || lpad(ordinal_position::text, 4, '0') || '|' || - column_name || '|' || data_type || '|' || udt_name || '|' || is_nullable || '|' || - coalesce(replace(column_default, current_schema() || '.', '.'), '') - FROM information_schema.columns WHERE table_schema = current_schema()`, - `SELECT 'constraint|' || t.relname || '|' || c.conname || '|' || c.contype::text || '|' || - pg_get_constraintdef(c.oid, true) - FROM pg_constraint c JOIN pg_class t ON t.oid = c.conrelid - JOIN pg_namespace n ON n.oid = t.relnamespace WHERE n.nspname = current_schema()`, - `SELECT 'index|' || tablename || '|' || indexname || '|' || - replace(indexdef, current_schema() || '.', '.') - FROM pg_indexes WHERE schemaname = current_schema()`, - } - for _, query := range queries { - rows, err := db.Query(query) - if err != nil { - t.Fatalf("query PostgreSQL schema: %v", err) - } - for rows.Next() { - var line string - if err := rows.Scan(&line); err != nil { - _ = rows.Close() - t.Fatalf("scan PostgreSQL schema: %v", err) - } - if !includeMigrations && (strings.Contains(line, "|bun_migrations|") || strings.Contains(line, "|bun_migration_locks|")) { - continue - } - lines = append(lines, strings.Join(strings.Fields(line), " ")) - } - if err := rows.Close(); err != nil { - t.Fatalf("close PostgreSQL schema rows: %v", err) - } - } - slices.Sort(lines) - return normalizedFingerprint(lines) -} diff --git a/internal/db/migrations/schema_model_alignment_test.go b/internal/db/migrations/schema_model_alignment_test.go new file mode 100644 index 0000000..cc02c9f --- /dev/null +++ b/internal/db/migrations/schema_model_alignment_test.go @@ -0,0 +1,305 @@ +package migrations + +import ( + "database/sql" + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "reflect" + "regexp" + "slices" + "strconv" + "strings" + "testing" + + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/observability" + "github.com/strahe/synaps3/internal/storagecommit" + "github.com/strahe/synaps3/internal/storagepull" + "github.com/strahe/synaps3/internal/storagereplacement" + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" +) + +func TestRuntimeModelsMatchAppliedBaseline(t *testing.T) { + models := runtimePersistentModels() + runtimeTablesFromAST := runtimePersistentModelTablesFromAST(t) + + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + registeredTables := make([]string, 0, len(models)) + for _, runtimeModel := range models { + typ := reflect.TypeOf(runtimeModel) + if typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + registeredTables = append(registeredTables, db.Table(typ).Name) + } + slices.Sort(registeredTables) + if !slices.Equal(registeredTables, runtimeTablesFromAST) { + t.Fatalf("runtime model registry tables = %v, AST-discovered tables = %v", registeredTables, runtimeTablesFromAST) + } + + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatalf("create initial schema: %v", err) + } + appliedTables := applicationSchemaTables(t, db) + if !slices.Equal(appliedTables, registeredTables) { + t.Fatalf("applied tables = %v, runtime model tables = %v", appliedTables, registeredTables) + } + for _, runtimeModel := range models { + assertRuntimeModelMatchesTable(t, db, runtimeModel) + } + }) +} + +func runtimePersistentModels() []any { + return []any{ + (*model.Task)(nil), + (*model.TaskPayload)(nil), + (*model.S3Account)(nil), + (*model.Bucket)(nil), + (*model.BucketReplicaSlot)(nil), + (*model.Object)(nil), + (*model.MultipartUpload)(nil), + (*model.MultipartPart)(nil), + (*model.StorageContent)(nil), + (*model.StorageDataSet)(nil), + (*model.StorageCopy)(nil), + (*storagecommit.Attempt)(nil), + (*storagepull.Attempt)(nil), + (*model.ObjectVersion)(nil), + (*model.ObjectCache)(nil), + (*model.ObjectDeletion)(nil), + (*model.StorageCleanupCopy)(nil), + (*storagereplacement.Replacement)(nil), + (*storagereplacement.Termination)(nil), + (*storagereplacement.Item)(nil), + (*model.WalletOperation)(nil), + (*observability.CollectionState)(nil), + (*observability.ProviderState)(nil), + (*observability.DataSetState)(nil), + } +} + +func runtimePersistentModelTablesFromAST(t *testing.T) []string { + t.Helper() + internalRoot := filepath.Clean(filepath.Join("..", "..")) + migrationRoot := filepath.Clean(filepath.Join(internalRoot, "db", "migrations")) + tables := make(map[string]string) + err := filepath.WalkDir(internalRoot, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + if filepath.Clean(path) == migrationRoot { + return filepath.SkipDir + } + return nil + } + if filepath.Ext(path) != ".go" || strings.HasSuffix(path, "_test.go") { + return nil + } + parsed, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.SkipObjectResolution) + if err != nil { + return err + } + ast.Inspect(parsed, func(node ast.Node) bool { + structType, ok := node.(*ast.StructType) + if !ok { + return true + } + for _, field := range structType.Fields.List { + if field.Tag == nil { + continue + } + rawTag, err := strconv.Unquote(field.Tag.Value) + if err != nil { + continue + } + for option := range strings.SplitSeq(reflect.StructTag(rawTag).Get("bun"), ",") { + table, found := strings.CutPrefix(option, "table:") + if !found || table == "" { + continue + } + if previous, exists := tables[table]; exists { + t.Errorf("persistent table %s is declared in both %s and %s", table, previous, path) + } else { + tables[table] = path + } + } + } + return true + }) + return nil + }) + if err != nil { + t.Fatalf("discover runtime persistent models: %v", err) + } + result := make([]string, 0, len(tables)) + for table := range tables { + result = append(result, table) + } + slices.Sort(result) + return result +} + +type appliedColumn struct { + Name string + Type string + NotNull bool + Default string + PrimaryKey bool + Generated bool +} + +func assertRuntimeModelMatchesTable(t *testing.T, db *bun.DB, runtimeModel any) { + t.Helper() + typ := reflect.TypeOf(runtimeModel) + if typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + table := db.Table(typ) + actual := appliedTableColumns(t, db, table.Name) + if len(actual) != len(table.Fields) { + t.Errorf("%s column count = %d, runtime fields = %d", table.Name, len(actual), len(table.Fields)) + } + limit := min(len(actual), len(table.Fields)) + for i := range limit { + field := table.Fields[i] + got := actual[i] + want := appliedColumn{ + Name: field.Name, + Type: normalizedSQLType(field.CreateTableSQLType), + NotNull: field.NotNull, + Default: got.Default, + PrimaryKey: field.IsPK, + Generated: field.AutoIncrement && field.Identity, + } + if db.Dialect().Name() == dialect.SQLite && isInitialJSONColumn(table.Name, field.Name) { + want.Type = "text" + } + // A runtime default changes Bun insert behavior by omitting zero values. + // It is optional, but when present it must match the frozen DDL default. + if field.SQLDefault != "" { + want.Default = normalizedSQLDefault(field.SQLDefault) + } + if got != want { + t.Errorf("%s column %d mismatch\n got: %#v\nwant: %#v", table.Name, i+1, got, want) + } + } +} + +func appliedTableColumns(t *testing.T, db *bun.DB, table string) []appliedColumn { + t.Helper() + if db.Dialect().Name() == dialect.PG { + return appliedPostgresColumns(t, db, table) + } + return appliedSQLiteColumns(t, db, table) +} + +func appliedSQLiteColumns(t *testing.T, db *bun.DB, table string) []appliedColumn { + t.Helper() + var tableDDL string + if err := db.NewRaw(`SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = ?`, table).Scan(t.Context(), &tableDDL); err != nil { + t.Fatalf("read SQLite table DDL for %s: %v", table, err) + } + rows, err := db.Query(`SELECT name, type, "notnull", dflt_value, pk FROM pragma_table_info(?) ORDER BY cid`, table) + if err != nil { + t.Fatalf("read SQLite columns for %s: %v", table, err) + } + defer func() { _ = rows.Close() }() + var columns []appliedColumn + for rows.Next() { + var name, sqlType string + var notNull, primaryKey int + var defaultValue sql.NullString + if err := rows.Scan(&name, &sqlType, ¬Null, &defaultValue, &primaryKey); err != nil { + t.Fatalf("scan SQLite column for %s: %v", table, err) + } + columns = append(columns, appliedColumn{ + Name: name, + Type: normalizedSQLType(sqlType), + NotNull: notNull != 0, + Default: normalizedSQLDefault(defaultValue.String), + PrimaryKey: primaryKey != 0, + Generated: primaryKey != 0 && name == "id" && strings.Contains(strings.ToUpper(tableDDL), "AUTOINCREMENT"), + }) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate SQLite columns for %s: %v", table, err) + } + return columns +} + +func appliedPostgresColumns(t *testing.T, db *bun.DB, table string) []appliedColumn { + t.Helper() + rows, err := db.Query(`SELECT column_info.column_name, + column_info.data_type, + column_info.is_nullable = 'NO', + column_info.column_default, + column_info.is_identity = 'YES', + EXISTS ( + SELECT 1 + FROM information_schema.table_constraints AS table_constraint + JOIN information_schema.key_column_usage AS key_column + ON key_column.constraint_schema = table_constraint.constraint_schema + AND key_column.constraint_name = table_constraint.constraint_name + WHERE table_constraint.table_schema = current_schema() + AND table_constraint.table_name = column_info.table_name + AND table_constraint.constraint_type = 'PRIMARY KEY' + AND key_column.column_name = column_info.column_name + ) + FROM information_schema.columns AS column_info + WHERE column_info.table_schema = current_schema() AND column_info.table_name = ? + ORDER BY column_info.ordinal_position`, table) + if err != nil { + t.Fatalf("read PostgreSQL columns for %s: %v", table, err) + } + defer func() { _ = rows.Close() }() + var columns []appliedColumn + for rows.Next() { + var name, sqlType string + var notNull, generated, primaryKey bool + var defaultValue sql.NullString + if err := rows.Scan(&name, &sqlType, ¬Null, &defaultValue, &generated, &primaryKey); err != nil { + t.Fatalf("scan PostgreSQL column for %s: %v", table, err) + } + columns = append(columns, appliedColumn{ + Name: name, + Type: normalizedSQLType(sqlType), + NotNull: notNull, + Default: normalizedSQLDefault(defaultValue.String), + PrimaryKey: primaryKey, + Generated: generated, + }) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate PostgreSQL columns for %s: %v", table, err) + } + return columns +} + +func normalizedSQLType(value string) string { + value = strings.ToLower(strings.TrimSpace(value)) + switch value { + case "timestamptz", "timestamp with time zone", "timestamp": + return "timestamp" + case "bytea", "blob": + return "binary" + default: + return value + } +} + +var postgresDefaultCast = regexp.MustCompile(`::[a-zA-Z0-9_\"]+`) + +func normalizedSQLDefault(value string) string { + value = strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(value)), " ")) + value = postgresDefaultCast.ReplaceAllString(value, "") + for len(value) >= 2 && value[0] == '(' && value[len(value)-1] == ')' { + value = strings.TrimSpace(value[1 : len(value)-1]) + } + return value +} diff --git a/internal/db/migrations/schema_query_plan_test.go b/internal/db/migrations/schema_query_plan_test.go new file mode 100644 index 0000000..0da3335 --- /dev/null +++ b/internal/db/migrations/schema_query_plan_test.go @@ -0,0 +1,328 @@ +package migrations + +import ( + "context" + "strings" + "testing" + + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" +) + +func TestBaselineRepresentativeQueriesUseSupportingIndexes(t *testing.T) { + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatalf("create initial schema: %v", err) + } + seedObjectPlanBacklog(t, db) + seedTaskPlanBacklog(t, db) + seedWalletPlanBacklog(t, db) + + // Listing current objects walks the objects unique key and follows each + // pointer, so the driving index is on objects rather than a partial + // index over every version. + currentListQuery := `SELECT object_version.version_id + FROM objects AS current_object + JOIN object_versions AS object_version + ON object_version.version_id = current_object.current_version_id + WHERE current_object.bucket_id = 1 AND object_version.is_delete_marker = FALSE + AND current_object.key >= 'a' + ORDER BY current_object.key ASC LIMIT 100` + versionListQuery := `SELECT version_id FROM object_versions + WHERE bucket_id = 1 AND key >= 'a' + ORDER BY key ASC, created_at DESC, version_id DESC LIMIT 100` + currentListIndex := "idx_objects_bucket_key" + if db.Dialect().Name() == dialect.PG { + currentListQuery = `SELECT object_version.version_id + FROM objects AS current_object + JOIN object_versions AS object_version + ON object_version.version_id = current_object.current_version_id + WHERE current_object.bucket_id = 1 AND object_version.is_delete_marker = FALSE + AND current_object.key COLLATE "C" >= 'a' COLLATE "C" + ORDER BY current_object.key COLLATE "C" ASC LIMIT 100` + versionListQuery = `SELECT version_id FROM object_versions + WHERE bucket_id = 1 AND key COLLATE "C" >= 'a' COLLATE "C" + ORDER BY key COLLATE "C" ASC, created_at DESC, version_id DESC LIMIT 100` + currentListIndex = "idx_objects_bucket_key_c" + } + + plans := []struct { + name string + indexNames []string + query string + }{ + { + name: "ListObjectsV2", + indexNames: []string{currentListIndex}, + query: currentListQuery, + }, + { + name: "current object lookup", + indexNames: []string{"idx_objects_bucket_key"}, + query: `SELECT current_version_id FROM objects + WHERE bucket_id = 1 AND key = 'key' LIMIT 1`, + }, + { + name: "version listing", + indexNames: []string{"idx_object_versions_bucket_key_created"}, + query: versionListQuery, + }, + { + // Residency is content-addressed, so the eviction scan reads + // object_cache and never touches object_versions. + name: "cache LRU", + indexNames: []string{"idx_object_cache_lru"}, + query: `SELECT content_id FROM object_cache + WHERE in_cache = TRUE + ORDER BY cache_accessed_at, content_id LIMIT 100`, + }, + { + name: "expired task recovery", + indexNames: []string{"idx_tasks_recovery"}, + query: `SELECT id FROM tasks + WHERE status = 'running' AND lease_until <= '9999-12-31 00:00:00' + ORDER BY lease_until, id LIMIT 1`, + }, + { + name: "pending task claim", + indexNames: []string{"idx_tasks_pending"}, + query: `SELECT id FROM tasks + WHERE status = 'pending' AND available_at <= '9999-12-31 00:00:00' + ORDER BY available_at, id LIMIT 1`, + }, + { + name: "commit FIFO", + indexNames: []string{ + "idx_storage_copies_commit_ready", + "idx_storage_commit_attempts_unresolved_copy", + }, + query: `SELECT storage_copy.id FROM storage_copies AS storage_copy + WHERE storage_copy.storage_data_set_id = 1 + AND storage_copy.status = 'piece_ready' + AND storage_copy.commit_ready_at IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM storage_commit_attempts AS active_attempt + WHERE active_attempt.content_id = storage_copy.content_id + AND active_attempt.storage_data_set_id = storage_copy.storage_data_set_id + AND active_attempt.resolved_at IS NULL + ) + ORDER BY storage_copy.commit_ready_at, storage_copy.id LIMIT 1`, + }, + { + name: "replacement progress", + indexNames: []string{"idx_storage_replacement_items_state"}, + query: `SELECT id FROM storage_replacement_items + WHERE replacement_id = 1 AND status = 'pending' ORDER BY id LIMIT 100`, + }, + { + name: "recent wallet operations", + indexNames: []string{"idx_wallet_operations_recent"}, + query: `SELECT id FROM wallet_operations + ORDER BY created_at DESC, id DESC LIMIT 100`, + }, + } + + for _, plan := range plans { + t.Run(plan.name, func(t *testing.T) { + got := explainQueryPlan(t, db, plan.query) + for _, indexName := range plan.indexNames { + if !strings.Contains(got, indexName) { + t.Fatalf("plan does not use %s:\n%s", indexName, got) + } + } + }) + } + }) +} + +func seedObjectPlanBacklog(t *testing.T, db *bun.DB) { + t.Helper() + if _, err := db.ExecContext(t.Context(), `INSERT INTO buckets (id, name, default_copies, minimum_durable_copies, created_at, updated_at) + VALUES (1, 'query-plan-bucket', 1, 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`); err != nil { + t.Fatalf("seed query-plan bucket: %v", err) + } + if _, err := db.ExecContext(t.Context(), `INSERT INTO bucket_replica_slots (bucket_id, copy_index, created_at, updated_at) VALUES (1, 0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`); err != nil { + t.Fatalf("seed query-plan replica slot: %v", err) + } + var insertObjects string + if db.Dialect().Name() == dialect.PG { + insertObjects = `INSERT INTO objects (id, bucket_id, key, created_at, updated_at) + SELECT value, 1, + CASE WHEN value = 1 THEN 'key' ELSE 'key-' || lpad(value::text, 6, '0') END, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM generate_series(1, 512) AS series(value)` + } else { + insertObjects = `WITH RECURSIVE sequence(value) AS ( + SELECT 1 UNION ALL SELECT value + 1 FROM sequence WHERE value < 512 + ) + INSERT INTO objects (id, bucket_id, key, created_at, updated_at) + SELECT value, 1, + CASE WHEN value = 1 THEN 'key' ELSE 'key-' || printf('%06d', value) END, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM sequence` + } + if _, err := db.ExecContext(t.Context(), insertObjects); err != nil { + t.Fatalf("seed query-plan objects: %v", err) + } + // Bytes own their identity, so each seeded version needs a content row and + // residency belongs to that content rather than to the version. + checksumExpression := "printf('%064x', id)" + if db.Dialect().Name() == dialect.PG { + checksumExpression = "lpad(to_hex(id), 64, '0')" + } + if _, err := db.ExecContext(t.Context(), `INSERT INTO storage_contents ( + id, bucket_id, checksum, content_size, requested_copies + , created_at, updated_at) + SELECT id, bucket_id, `+checksumExpression+`, 1, 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP FROM objects`); err != nil { + t.Fatalf("seed query-plan contents: %v", err) + } + if _, err := db.ExecContext(t.Context(), `INSERT INTO object_cache ( + content_id, in_cache, cache_accessed_at + , created_at, updated_at) + SELECT id, TRUE, '2026-01-01 00:00:00', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP FROM storage_contents`); err != nil { + t.Fatalf("seed query-plan cache entries: %v", err) + } + if _, err := db.ExecContext(t.Context(), `INSERT INTO object_versions ( + version_id, object_id, bucket_id, key, content_id, size, e_tag, + is_delete_marker + , created_at, updated_at) + SELECT 'version-' || id, id, bucket_id, key, id, 1, 'etag-' || id, FALSE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM objects`); err != nil { + t.Fatalf("seed query-plan object versions: %v", err) + } + // "Current" is the object's pointer now. + if _, err := db.ExecContext(t.Context(), ` + UPDATE objects SET current_version_id = 'version-' || id`); err != nil { + t.Fatalf("point query-plan objects at their versions: %v", err) + } + var insertHistory string + if db.Dialect().Name() == dialect.PG { + insertHistory = `INSERT INTO object_versions ( + version_id, object_id, bucket_id, key, content_id, size, e_tag, + is_delete_marker, created_at + , updated_at) + SELECT 'history-' || object_info.id || '-' || generation, object_info.id, + object_info.bucket_id, object_info.key, object_info.id, 1, + 'history-etag-' || generation, FALSE, '2025-01-01 00:00:00', CURRENT_TIMESTAMP + FROM objects AS object_info CROSS JOIN generate_series(1, 4) AS series(generation)` + } else { + insertHistory = `WITH RECURSIVE generations(generation) AS ( + SELECT 1 UNION ALL SELECT generation + 1 FROM generations WHERE generation < 4 + ) + INSERT INTO object_versions ( + version_id, object_id, bucket_id, key, content_id, size, e_tag, + is_delete_marker, created_at + , updated_at) + SELECT 'history-' || object_info.id || '-' || generation, object_info.id, + object_info.bucket_id, object_info.key, object_info.id, 1, + 'history-etag-' || generation, FALSE, '2025-01-01 00:00:00', CURRENT_TIMESTAMP + FROM objects AS object_info CROSS JOIN generations` + } + if _, err := db.ExecContext(t.Context(), insertHistory); err != nil { + t.Fatalf("seed query-plan version history: %v", err) + } + for _, table := range []string{"objects", "object_versions", "object_cache"} { + if _, err := db.ExecContext(t.Context(), "ANALYZE "+table); err != nil { + t.Fatalf("analyze query-plan table %s: %v", table, err) + } + } +} + +func seedTaskPlanBacklog(t *testing.T, db *bun.DB) { + t.Helper() + var statements []string + if db.Dialect().Name() == dialect.PG { + statements = []string{ + `INSERT INTO tasks (type, idempotency_key, input_version, input_hash, status, available_at, created_at, updated_at) + SELECT 'plan', 'pending-' || value, 1, 'hash', 'pending', '2026-01-01 00:00:00', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM generate_series(1, 512) AS series(value)`, + `INSERT INTO tasks (type, idempotency_key, input_version, input_hash, status, + available_at, resume_mode, claim_generation, claimed_at, lease_until, created_at, updated_at) + SELECT 'plan', 'running-' || value, 1, 'hash', 'running', '2026-01-01 00:00:00', + 'recover', 1, '2026-01-01 00:00:00', '2026-01-01 00:01:00', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM generate_series(1, 512) AS series(value)`, + } + } else { + statements = []string{ + `WITH RECURSIVE sequence(value) AS ( + SELECT 1 UNION ALL SELECT value + 1 FROM sequence WHERE value < 512 + ) + INSERT INTO tasks (type, idempotency_key, input_version, input_hash, status, available_at, created_at, updated_at) + SELECT 'plan', 'pending-' || value, 1, 'hash', 'pending', '2026-01-01 00:00:00', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM sequence`, + `WITH RECURSIVE sequence(value) AS ( + SELECT 1 UNION ALL SELECT value + 1 FROM sequence WHERE value < 512 + ) + INSERT INTO tasks (type, idempotency_key, input_version, input_hash, status, + available_at, resume_mode, claim_generation, claimed_at, lease_until, created_at, updated_at) + SELECT 'plan', 'running-' || value, 1, 'hash', 'running', '2026-01-01 00:00:00', + 'recover', 1, '2026-01-01 00:00:00', '2026-01-01 00:01:00', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM sequence`, + } + } + statements = append(statements, + `INSERT INTO task_payloads (task_id, input_json) SELECT id, '{}' FROM tasks`) + for _, statement := range statements { + if _, err := db.ExecContext(t.Context(), statement); err != nil { + t.Fatalf("seed task query-plan backlog: %v", err) + } + } + if _, err := db.ExecContext(t.Context(), "ANALYZE tasks"); err != nil { + t.Fatalf("analyze task query-plan backlog: %v", err) + } +} + +func seedWalletPlanBacklog(t *testing.T, db *bun.DB) { + t.Helper() + var statement string + if db.Dialect().Name() == dialect.PG { + statement = `INSERT INTO wallet_operations + (type, client_request_id, amount, created_at, updated_at) + SELECT 'fund', 'wallet-' || value, '1', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM generate_series(1, 512) AS series(value)` + } else { + statement = `WITH RECURSIVE sequence(value) AS ( + SELECT 1 UNION ALL SELECT value + 1 FROM sequence WHERE value < 512 + ) + INSERT INTO wallet_operations + (type, client_request_id, amount, created_at, updated_at) + SELECT 'fund', 'wallet-' || value, '1', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM sequence` + } + if _, err := db.ExecContext(t.Context(), statement); err != nil { + t.Fatalf("seed wallet query-plan backlog: %v", err) + } + if _, err := db.ExecContext(t.Context(), "ANALYZE wallet_operations"); err != nil { + t.Fatalf("analyze wallet query-plan backlog: %v", err) + } +} + +func explainQueryPlan(t *testing.T, db *bun.DB, query string) string { + t.Helper() + if db.Dialect().Name() == dialect.PG { + var plan []string + err := db.RunInTx(t.Context(), nil, func(ctx context.Context, tx bun.Tx) error { + if _, err := tx.ExecContext(ctx, "SET LOCAL enable_seqscan = off"); err != nil { + return err + } + return tx.NewRaw("EXPLAIN (COSTS OFF) "+query).Scan(ctx, &plan) + }) + if err != nil { + t.Fatalf("explain PostgreSQL query: %v", err) + } + return strings.Join(plan, "\n") + } + + var rows []struct { + ID int `bun:"id"` + Parent int `bun:"parent"` + NotUsed int `bun:"notused"` + Detail string `bun:"detail"` + } + if err := db.NewRaw("EXPLAIN QUERY PLAN "+query).Scan(t.Context(), &rows); err != nil { + t.Fatalf("explain SQLite query: %v", err) + } + details := make([]string, 0, len(rows)) + for _, row := range rows { + details = append(details, row.Detail) + } + return strings.Join(details, "\n") +} diff --git a/internal/db/migrations/schema_semantic_fingerprint_test.go b/internal/db/migrations/schema_semantic_fingerprint_test.go new file mode 100644 index 0000000..cdae914 --- /dev/null +++ b/internal/db/migrations/schema_semantic_fingerprint_test.go @@ -0,0 +1,382 @@ +package migrations + +import ( + "database/sql" + "fmt" + "regexp" + "slices" + "strings" + "testing" + + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" +) + +func portableSchemaFingerprint(t *testing.T, db *bun.DB) (string, string) { + t.Helper() + tables := applicationSchemaTables(t, db) + lines := make([]string, 0, len(tables)*16) + for _, table := range tables { + for position, column := range appliedTableColumns(t, db, table) { + lines = append(lines, fmt.Sprintf( + "column|%s|%04d|%s|%s|%t|%s|%t|%t", + table, + position+1, + column.Name, + portableColumnTypeFamily(table, column.Name, column.Type), + column.NotNull, + column.Default, + column.PrimaryKey, + column.Generated, + )) + } + } + lines = append(lines, semanticConstraintLines(t, db, tables)...) + lines = append(lines, semanticUniqueConstraintLines(t, db, tables)...) + lines = append(lines, semanticForeignKeyLines(t, db, tables)...) + lines = append(lines, semanticIndexLines(t, db)...) + slices.Sort(lines) + return normalizedFingerprint(lines) +} + +func semanticUniqueConstraintLines(t *testing.T, db *bun.DB, tables []string) []string { + t.Helper() + if db.Dialect().Name() == dialect.PG { + rows, err := db.Query(`SELECT table_info.relname, + string_agg(column_info.attname, ',' ORDER BY key_column.position) + FROM pg_constraint AS constraint_info + JOIN pg_class AS table_info ON table_info.oid = constraint_info.conrelid + JOIN pg_namespace AS namespace_info ON namespace_info.oid = table_info.relnamespace + CROSS JOIN LATERAL unnest(constraint_info.conkey) + WITH ORDINALITY AS key_column(attnum, position) + JOIN pg_attribute AS column_info + ON column_info.attrelid = table_info.oid AND column_info.attnum = key_column.attnum + WHERE constraint_info.contype = 'u' AND namespace_info.nspname = current_schema() + GROUP BY table_info.relname, constraint_info.conname + ORDER BY table_info.relname, constraint_info.conname`) + if err != nil { + t.Fatalf("read PostgreSQL unique constraints: %v", err) + } + defer func() { _ = rows.Close() }() + var lines []string + for rows.Next() { + var table, columns string + if err := rows.Scan(&table, &columns); err != nil { + t.Fatalf("scan PostgreSQL unique constraint: %v", err) + } + lines = append(lines, "unique|"+table+"|"+columns) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate PostgreSQL unique constraints: %v", err) + } + return lines + } + + var lines []string + for _, table := range tables { + rows, err := db.Query(`SELECT name FROM pragma_index_list(?) WHERE origin = 'u' ORDER BY seq`, table) + if err != nil { + t.Fatalf("read SQLite unique constraints for %s: %v", table, err) + } + var indexes []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + _ = rows.Close() + t.Fatalf("scan SQLite unique constraint for %s: %v", table, err) + } + indexes = append(indexes, name) + } + if err := rows.Close(); err != nil { + t.Fatalf("close SQLite unique constraints for %s: %v", table, err) + } + for _, index := range indexes { + var columns []string + if err := db.NewRaw(`SELECT name FROM pragma_index_info(?) ORDER BY seqno`, index).Scan(t.Context(), &columns); err != nil { + t.Fatalf("read SQLite unique columns for %s: %v", index, err) + } + lines = append(lines, "unique|"+table+"|"+strings.Join(columns, ",")) + } + } + return lines +} + +func applicationSchemaTables(t *testing.T, db *bun.DB) []string { + t.Helper() + query := `SELECT name FROM sqlite_schema + WHERE type = 'table' AND name NOT LIKE 'sqlite_%' + AND name NOT IN ('bun_migrations', 'bun_migration_locks') + ORDER BY name` + if db.Dialect().Name() == dialect.PG { + query = `SELECT table_name FROM information_schema.tables + WHERE table_schema = current_schema() AND table_type = 'BASE TABLE' + AND table_name NOT IN ('bun_migrations', 'bun_migration_locks') + ORDER BY table_name` + } + var tables []string + if err := db.NewRaw(query).Scan(t.Context(), &tables); err != nil { + t.Fatalf("read application schema tables: %v", err) + } + return tables +} + +func semanticConstraintLines(t *testing.T, db *bun.DB, tables []string) []string { + t.Helper() + var lines []string + if db.Dialect().Name() == dialect.PG { + rows, err := db.Query(`SELECT table_info.relname, constraint_info.conname + FROM pg_constraint AS constraint_info + JOIN pg_class AS table_info ON table_info.oid = constraint_info.conrelid + JOIN pg_namespace AS namespace_info ON namespace_info.oid = table_info.relnamespace + WHERE namespace_info.nspname = current_schema() + AND (constraint_info.conname LIKE 'chk_%' OR constraint_info.conname LIKE 'uq_%' OR constraint_info.conname LIKE 'fk_%') + ORDER BY table_info.relname, constraint_info.conname`) + if err != nil { + t.Fatalf("read PostgreSQL constraints: %v", err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var table, name string + if err := rows.Scan(&table, &name); err != nil { + t.Fatalf("scan PostgreSQL constraint: %v", err) + } + lines = append(lines, "constraint|"+table+"|"+name) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate PostgreSQL constraints: %v", err) + } + return lines + } + + constraintPattern := regexp.MustCompile(`(?i)CONSTRAINT\s+([A-Za-z0-9_]+)`) + for _, table := range tables { + var ddl string + if err := db.NewRaw(`SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = ?`, table).Scan(t.Context(), &ddl); err != nil { + t.Fatalf("read SQLite DDL for %s: %v", table, err) + } + for _, match := range constraintPattern.FindAllStringSubmatch(ddl, -1) { + lines = append(lines, "constraint|"+table+"|"+match[1]) + } + } + return lines +} + +func semanticForeignKeyLines(t *testing.T, db *bun.DB, tables []string) []string { + t.Helper() + var lines []string + if db.Dialect().Name() == dialect.PG { + rows, err := db.Query(`SELECT source_table.relname, + source_column.attname, + target_table.relname, + target_column.attname, + CASE constraint_info.confupdtype + WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' + WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' + END, + CASE constraint_info.confdeltype + WHEN 'a' THEN 'NO ACTION' WHEN 'r' THEN 'RESTRICT' WHEN 'c' THEN 'CASCADE' + WHEN 'n' THEN 'SET NULL' WHEN 'd' THEN 'SET DEFAULT' + END + FROM pg_constraint AS constraint_info + JOIN pg_class AS source_table ON source_table.oid = constraint_info.conrelid + JOIN pg_namespace AS source_namespace ON source_namespace.oid = source_table.relnamespace + JOIN pg_class AS target_table ON target_table.oid = constraint_info.confrelid + CROSS JOIN LATERAL unnest(constraint_info.conkey, constraint_info.confkey) + WITH ORDINALITY AS key_pair(source_attnum, target_attnum, position) + JOIN pg_attribute AS source_column + ON source_column.attrelid = source_table.oid AND source_column.attnum = key_pair.source_attnum + JOIN pg_attribute AS target_column + ON target_column.attrelid = target_table.oid AND target_column.attnum = key_pair.target_attnum + WHERE constraint_info.contype = 'f' AND source_namespace.nspname = current_schema() + ORDER BY source_table.relname, constraint_info.conname, key_pair.position`) + if err != nil { + t.Fatalf("read PostgreSQL foreign keys: %v", err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var sourceTable, sourceColumn, targetTable, targetColumn, onUpdate, onDelete string + if err := rows.Scan(&sourceTable, &sourceColumn, &targetTable, &targetColumn, &onUpdate, &onDelete); err != nil { + t.Fatalf("scan PostgreSQL foreign key: %v", err) + } + lines = append(lines, semanticForeignKeyLine(sourceTable, sourceColumn, targetTable, targetColumn, onUpdate, onDelete)) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate PostgreSQL foreign keys: %v", err) + } + return lines + } + + for _, table := range tables { + rows, err := db.Query(`SELECT "from", "table", "to", on_update, on_delete FROM pragma_foreign_key_list(?) ORDER BY id, seq`, table) + if err != nil { + t.Fatalf("read SQLite foreign keys for %s: %v", table, err) + } + for rows.Next() { + var sourceColumn, targetTable, targetColumn, onUpdate, onDelete string + if err := rows.Scan(&sourceColumn, &targetTable, &targetColumn, &onUpdate, &onDelete); err != nil { + _ = rows.Close() + t.Fatalf("scan SQLite foreign key for %s: %v", table, err) + } + lines = append(lines, semanticForeignKeyLine(table, sourceColumn, targetTable, targetColumn, onUpdate, onDelete)) + } + if err := rows.Close(); err != nil { + t.Fatalf("close SQLite foreign keys for %s: %v", table, err) + } + } + return lines +} + +func semanticForeignKeyLine(sourceTable, sourceColumn, targetTable, targetColumn, onUpdate, onDelete string) string { + return fmt.Sprintf("foreign-key|%s|%s|%s|%s|%s|%s", sourceTable, sourceColumn, targetTable, targetColumn, strings.ToUpper(onUpdate), strings.ToUpper(onDelete)) +} + +func semanticIndexLines(t *testing.T, db *bun.DB) []string { + t.Helper() + if db.Dialect().Name() == dialect.PG { + return semanticPostgresIndexLines(t, db) + } + return semanticSQLiteIndexLines(t, db) +} + +func semanticPostgresIndexLines(t *testing.T, db *bun.DB) []string { + t.Helper() + rows, err := db.Query(`SELECT table_info.relname, + index_info.relname, + index_meta.indisunique, + COALESCE(( + SELECT string_agg( + pg_get_indexdef(index_meta.indexrelid, position, TRUE) || + CASE WHEN (index_meta.indoption[position - 1] & 1) = 1 THEN ' DESC' ELSE '' END, + '|' ORDER BY position + ) + FROM generate_series(1, index_meta.indnkeyatts) AS position + ), ''), + COALESCE(pg_get_expr(index_meta.indpred, index_meta.indrelid), '') + FROM pg_index AS index_meta + JOIN pg_class AS table_info ON table_info.oid = index_meta.indrelid + JOIN pg_namespace AS namespace_info ON namespace_info.oid = table_info.relnamespace + JOIN pg_class AS index_info ON index_info.oid = index_meta.indexrelid + WHERE namespace_info.nspname = current_schema() + AND index_info.relname LIKE 'idx_%' + AND index_info.relname NOT LIKE '%\_c' ESCAPE '\' + ORDER BY table_info.relname, index_info.relname`) + if err != nil { + t.Fatalf("read PostgreSQL indexes: %v", err) + } + defer func() { _ = rows.Close() }() + var lines []string + for rows.Next() { + var table, name, columns, predicate string + var unique bool + if err := rows.Scan(&table, &name, &unique, &columns, &predicate); err != nil { + t.Fatalf("scan PostgreSQL index: %v", err) + } + lines = append(lines, semanticIndexLine(table, name, unique, strings.Split(columns, "|"), predicate)) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate PostgreSQL indexes: %v", err) + } + return lines +} + +func semanticSQLiteIndexLines(t *testing.T, db *bun.DB) []string { + t.Helper() + rows, err := db.Query(`SELECT tbl_name, name, sql FROM sqlite_schema + WHERE type = 'index' AND name LIKE 'idx_%' + ORDER BY tbl_name, name`) + if err != nil { + t.Fatalf("read SQLite indexes: %v", err) + } + var indexes []struct { + table string + name string + ddl string + } + for rows.Next() { + var table, name, ddl string + if err := rows.Scan(&table, &name, &ddl); err != nil { + t.Fatalf("scan SQLite index: %v", err) + } + indexes = append(indexes, struct { + table string + name string + ddl string + }{table: table, name: name, ddl: ddl}) + } + if err := rows.Close(); err != nil { + t.Fatalf("close SQLite indexes: %v", err) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate SQLite indexes: %v", err) + } + + var lines []string + for _, index := range indexes { + table, name, ddl := index.table, index.name, index.ddl + columnRows, err := db.Query(`SELECT name, "desc" FROM pragma_index_xinfo(?) WHERE "key" = 1 ORDER BY seqno`, name) + if err != nil { + t.Fatalf("read SQLite index columns for %s: %v", name, err) + } + var columns []string + for columnRows.Next() { + var column sql.NullString + var descending bool + if err := columnRows.Scan(&column, &descending); err != nil { + _ = columnRows.Close() + t.Fatalf("scan SQLite index column for %s: %v", name, err) + } + value := column.String + if descending { + value += " DESC" + } + columns = append(columns, value) + } + if err := columnRows.Close(); err != nil { + t.Fatalf("close SQLite index columns for %s: %v", name, err) + } + predicate := "" + if where := strings.Index(strings.ToUpper(ddl), " WHERE "); where >= 0 { + predicate = ddl[where+len(" WHERE "):] + } + unique := strings.HasPrefix(strings.ToUpper(ddl), "CREATE UNIQUE INDEX") + lines = append(lines, semanticIndexLine(table, name, unique, columns, predicate)) + } + return lines +} + +func semanticIndexLine(table, name string, unique bool, columns []string, predicate string) string { + for i := range columns { + columns[i] = normalizedSQLExpression(columns[i]) + } + return fmt.Sprintf("index|%s|%s|%t|%s|%s", table, name, unique, strings.Join(columns, ","), normalizedSQLExpression(predicate)) +} + +func normalizedSQLExpression(value string) string { + value = strings.ToLower(value) + value = postgresDefaultCast.ReplaceAllString(value, "") + value = strings.ReplaceAll(value, ` collate "c"`, "") + value = strings.ReplaceAll(value, "= any (array[", " in ") + value = strings.ReplaceAll(value, "<> all (array[", " not in ") + value = strings.NewReplacer( + "(", " ", ")", " ", "[", " ", "]", " ", ",", " ", `"`, "", + ).Replace(value) + return strings.Join(strings.Fields(value), " ") +} + +func portableSQLTypeFamily(value string) string { + switch value { + case "bigint", "integer": + return "integer" + case "jsonb", "json": + return "json" + default: + return value + } +} + +func portableColumnTypeFamily(table, column, value string) string { + if isInitialJSONColumn(table, column) { + return "json" + } + return portableSQLTypeFamily(value) +} diff --git a/internal/db/migrations/schema_storage.go b/internal/db/migrations/schema_storage.go new file mode 100644 index 0000000..2444308 --- /dev/null +++ b/internal/db/migrations/schema_storage.go @@ -0,0 +1,508 @@ +package migrations + +import ( + "context" + "time" + + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" +) + +// storageContent2026090101 is the identity of one bucket-scoped byte payload. +// Durability, pipeline state and ingress progress are deliberately absent: the +// first two are functions of the copy rows and stay queries, while progress +// belongs to the concrete ingress transfer that produced it. +type storageContent2026090101 struct { + bun.BaseModel `bun:"table:storage_contents"` + + ID int64 `bun:",pk,autoincrement,identity"` + BucketID int64 `bun:",notnull"` + Checksum string `bun:"type:text,notnull"` + ContentSize int64 `bun:",notnull"` + PieceCID *string `bun:"type:text"` + RequestedCopies int `bun:"type:integer,notnull"` + ErrorMessage *string `bun:"type:text"` + AcceptedAt *time.Time + CleanupGeneration int64 `bun:",notnull,default:0"` + CleanupTaskID *int64 + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type storageDataSet2026090101 struct { + bun.BaseModel `bun:"table:storage_data_sets"` + + ID int64 `bun:",pk,autoincrement,identity"` + BucketID int64 `bun:",notnull"` + ProviderID string `bun:"type:text,notnull"` + CopyIndex int `bun:"type:integer,notnull"` + Generation int64 `bun:",notnull,default:1"` + IsCurrent bool `bun:",notnull"` + DataSetID *string `bun:"type:text"` + ClientDataSetID *string `bun:"type:text"` + Status string `bun:"type:text,notnull,default:'pending'"` + CreateTransactionID *string `bun:"type:text"` + CreateStatusURL *string `bun:"type:text"` + CreatedByContentID *int64 + LastUsedContentID *int64 + LastError *string `bun:"type:text"` + EnsureTaskID *int64 + RetirementGeneration int64 `bun:",notnull,default:0"` + RetirementTaskID *int64 + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type storageCopy2026090101 struct { + bun.BaseModel `bun:"table:storage_copies"` + + ID int64 `bun:",pk,autoincrement,identity"` + ContentID int64 `bun:",notnull"` + BucketID int64 `bun:",notnull"` + // ContentSize repeats storage_contents.content_size so the ingress bound + // stays a local CHECK. A composite foreign key keeps the copy from + // drifting away from the content it transfers. + ContentSize int64 `bun:",notnull"` + StorageDataSetID int64 `bun:",notnull"` + CopyIndex int `bun:"type:integer,notnull"` + ProviderID string `bun:"type:text,notnull"` + PieceID *string `bun:"type:text"` + TransferMethod string `bun:"type:text,notnull"` + Status string `bun:"type:text,notnull,default:'pending'"` + RetrievalURL *string `bun:"type:text"` + CommitExtraDataHex *string `bun:"type:text"` + CommitReadyAt *time.Time + // The confirmed commit attempt this copy projects. Its status is repeated so + // a composite foreign key can require the referenced attempt to be + // confirmed, which is what keeps the projection from drifting. + ConfirmedAttemptID *string `bun:"type:text"` + ConfirmedAttemptStatus *string `bun:"type:text"` + IngressBytesTransferred int64 `bun:",notnull,default:0"` + IngressStoreAttempt int `bun:"type:integer,notnull,default:0"` + ProgressUpdatedAt *time.Time + WorkGeneration int64 `bun:",notnull,default:0"` + ActiveTaskID *int64 + LastError *string `bun:"type:text"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type storageCommitAttempt2026090101 struct { + bun.BaseModel `bun:"table:storage_commit_attempts"` + + AttemptID string `bun:"type:text,pk"` + ContentID int64 `bun:",notnull"` + StorageDataSetID int64 `bun:",notnull"` + Status string `bun:"type:text,notnull,default:'reserved'"` + ExtraDataHex *string `bun:"type:text"` + TransactionID *string `bun:"type:text"` + SubmissionJSON *string `bun:"type:text"` + ConfirmedTransactionID *string `bun:"type:text"` + AttentionCode *string `bun:"type:text"` + AttentionAt *time.Time + ReleaseReason *string `bun:"type:text"` + LastError *string `bun:"type:text"` + AttemptedAt *time.Time + ResolvedAt *time.Time + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +// storagePullAttempt2026090101 is the ledger of provider-side copy requests. +// Every source column is NOT NULL, which replaces the all-or-nothing check that +// used to guard five nullable columns on the copy. +type storagePullAttempt2026090101 struct { + bun.BaseModel `bun:"table:storage_pull_attempts"` + + AttemptID string `bun:"type:text,pk"` + ContentID int64 `bun:",notnull"` + StorageDataSetID int64 `bun:",notnull"` + Status string `bun:"type:text,notnull"` + SourceProviderID string `bun:"type:text,notnull"` + SourceDataSetID string `bun:"type:text,notnull"` + SourcePieceID string `bun:"type:text,notnull"` + SourcePieceCID string `bun:"type:text,notnull"` + SourceRetrievalURL string `bun:"type:text,notnull"` + LastError *string `bun:"type:text"` + AttemptedAt time.Time `bun:",notnull"` + ResolvedAt *time.Time + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type storageReplacement2026090101 struct { + bun.BaseModel `bun:"table:storage_replacements"` + + ID int64 `bun:",pk,autoincrement,identity"` + BucketID int64 `bun:",notnull"` + CopyIndex int `bun:"type:integer,notnull"` + SourceDataSetID int64 `bun:",notnull"` + TargetDataSetID int64 `bun:",notnull"` + SelectionMode string `bun:"type:text,notnull"` + RequestedProviderID *string `bun:"type:text"` + ClientRequestID string `bun:"type:text,notnull"` + Status string `bun:"type:text,notnull"` + WaitReason *string `bun:"type:text"` + FailureReason *string `bun:"type:text"` + LastError *string `bun:"type:text"` + ItemsTotal int `bun:"type:integer,notnull,default:0"` + ItemsCopied int `bun:"type:integer,notnull,default:0"` + SeedCursorContentID int64 `bun:",notnull,default:0"` + SeedingComplete bool `bun:",notnull,default:false"` + TaskGeneration int64 `bun:",notnull,default:1"` + TaskID *int64 + SupersededByID *int64 + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +// storageDataSetTermination2026090101 records one data set's end of term. A +// replacement terminates the source it replaces and, once superseded, the +// target it abandoned; a third kind of termination adds a row here instead of +// another repeated column group on storage_replacements. +type storageDataSetTermination2026090101 struct { + bun.BaseModel `bun:"table:storage_data_set_terminations"` + + ID int64 `bun:",pk,autoincrement,identity"` + ReplacementID int64 `bun:",notnull"` + Role string `bun:"type:text,notnull"` + // Exactly one of the data set columns is set, chosen by role. Each carries a + // composite foreign key back to the matching column on the replacement, so a + // row cannot name a data set the replacement never held in that role. + SourceDataSetID *int64 + AbandonedTargetDataSetID *int64 + TxHash *string `bun:"type:text"` + Epoch int64 `bun:",notnull"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +func storageDataSetTerminationTable2026090101() initialTableSpec { + return initialTableSpec{ + name: "storage_data_set_terminations", + model: (*storageDataSetTermination2026090101)(nil), + constraints: []string{ + "CONSTRAINT uq_storage_data_set_terminations_role UNIQUE (replacement_id, role)", + "CONSTRAINT chk_storage_data_set_terminations_role CHECK (role IN ('source', 'abandoned_target'))", + "CONSTRAINT chk_storage_data_set_terminations_subject CHECK ((role = 'source') = (source_data_set_id IS NOT NULL) AND (role = 'abandoned_target') = (abandoned_target_data_set_id IS NOT NULL))", + "CONSTRAINT chk_storage_data_set_terminations_epoch CHECK (epoch >= 0)", + "CONSTRAINT chk_storage_data_set_terminations_tx_hash CHECK (tx_hash IS NULL OR tx_hash <> '')", + }, + foreignKeys: []string{ + "(replacement_id) REFERENCES storage_replacements (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(replacement_id, source_data_set_id) REFERENCES storage_replacements (id, source_data_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(replacement_id, abandoned_target_data_set_id) REFERENCES storage_replacements (id, target_data_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + } +} + +type storageReplacementItem2026090101 struct { + bun.BaseModel `bun:"table:storage_replacement_items"` + + ID int64 `bun:",pk,autoincrement,identity"` + ReplacementID int64 `bun:",notnull"` + ContentID int64 `bun:",notnull"` + // TargetDataSetID is written when the item is seeded, together with the + // pending copy it names. A nullable column would make the composite foreign + // key below skip validation entirely whenever it was unset. + TargetDataSetID int64 `bun:",notnull"` + Status string `bun:"type:text,notnull,default:'pending'"` + LastError *string `bun:"type:text"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type storageCleanupCopy2026090101 struct { + bun.BaseModel `bun:"table:storage_cleanup_copies"` + + ID int64 `bun:",pk,autoincrement,identity"` + ContentID int64 `bun:",notnull"` + BucketID int64 `bun:",notnull"` + CopyIndex int `bun:"type:integer,notnull"` + ProviderID string `bun:"type:text,notnull"` + StorageDataSetID int64 `bun:",notnull"` + DataSetID *string `bun:"type:text"` + ClientDataSetID *string `bun:"type:text"` + PieceID string `bun:"type:text,notnull"` + PieceCID string `bun:"type:text,notnull"` + RetrievalURL *string `bun:"type:text"` + Status string `bun:"type:text,notnull,default:'pending'"` + DeleteTxHash *string `bun:"type:text"` + LastError *string `bun:"type:text"` + ScheduledAt *time.Time + RemovedAt *time.Time + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +func createStorageSchema(ctx context.Context, db bun.IDB) error { + checksumCheck := "length(checksum) = 64 AND checksum NOT GLOB '*[^0-9a-f]*'" + if db.Dialect().Name() == dialect.PG { + checksumCheck = "checksum ~ '^[0-9a-f]{64}$'" + } + tables := []initialTableSpec{ + { + name: "storage_contents", + model: (*storageContent2026090101)(nil), + constraints: []string{ + // Content dedup is a unique-key lookup, not an index scan. + "CONSTRAINT uq_storage_contents_bytes UNIQUE (bucket_id, checksum, content_size)", + // Candidate keys for the composite foreign keys that pin + // denormalized identity on object_versions and storage_copies. + "CONSTRAINT uq_storage_contents_id_bucket UNIQUE (id, bucket_id)", + "CONSTRAINT uq_storage_contents_addr UNIQUE (id, bucket_id, content_size)", + "CONSTRAINT chk_storage_contents_identity CHECK ((" + checksumCheck + ") AND (piece_cid IS NULL OR piece_cid <> ''))", + "CONSTRAINT chk_storage_contents_content_size CHECK (content_size >= 0)", + "CONSTRAINT chk_storage_contents_requested_copies CHECK (requested_copies BETWEEN 1 AND 8)", + "CONSTRAINT chk_storage_contents_cleanup_generation CHECK (cleanup_generation >= 0)", + }, + foreignKeys: []string{ + "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(cleanup_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }, + { + name: "storage_data_sets", + model: (*storageDataSet2026090101)(nil), + constraints: []string{ + "CONSTRAINT uq_storage_data_sets_id_bucket_slot UNIQUE (id, bucket_id, copy_index)", + "CONSTRAINT uq_storage_data_sets_identity UNIQUE (id, bucket_id, copy_index, provider_id)", + "CONSTRAINT chk_storage_data_sets_identity CHECK (provider_id <> '' AND (data_set_id IS NULL OR data_set_id <> '') AND (client_data_set_id IS NULL OR client_data_set_id <> '') AND (create_transaction_id IS NULL OR create_transaction_id <> '') AND (create_status_url IS NULL OR create_status_url <> ''))", + // The replica slot is a row, so the index is a foreign key rather than a range check. + "CONSTRAINT fk_storage_data_sets_replica_slot FOREIGN KEY (bucket_id, copy_index) REFERENCES bucket_replica_slots (bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", + "CONSTRAINT chk_storage_data_sets_generation CHECK (generation >= 1 AND retirement_generation >= 0)", + "CONSTRAINT chk_storage_data_sets_status CHECK (status IN ('pending', 'creating', 'ready', 'failed', 'draining', 'retired'))", + "CONSTRAINT chk_storage_data_sets_ready_identity CHECK (status <> 'ready' OR data_set_id IS NOT NULL)", + "CONSTRAINT chk_storage_data_sets_current_shape CHECK (status NOT IN ('failed', 'draining', 'retired') OR is_current = FALSE)", + }, + foreignKeys: []string{ + "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(created_by_content_id, bucket_id) REFERENCES storage_contents (id, bucket_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(last_used_content_id, bucket_id) REFERENCES storage_contents (id, bucket_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(ensure_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(retirement_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }, + { + name: "storage_copies", + model: (*storageCopy2026090101)(nil), + constraints: []string{ + "CONSTRAINT uq_storage_copies_content_data_set UNIQUE (content_id, storage_data_set_id)", + // The replica slot is a row, so the index is a foreign key rather than a range check. + "CONSTRAINT fk_storage_copies_replica_slot FOREIGN KEY (bucket_id, copy_index) REFERENCES bucket_replica_slots (bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", + "CONSTRAINT chk_storage_copies_work_generation CHECK (work_generation >= 0)", + "CONSTRAINT chk_storage_copies_status CHECK (status IN ('pending', 'piece_ready', 'committing', 'committed', 'failed'))", + // The projected status is pinned to a literal so the composite + // foreign key below can only ever reach a confirmed attempt. + "CONSTRAINT chk_storage_copies_confirmed_attempt_status CHECK (confirmed_attempt_status IS NULL OR confirmed_attempt_status = 'confirmed')", + // Committed and "has confirmed evidence" are the same fact. The + // foreign key alone would still allow a committed copy with no + // evidence at all, so both directions are stated here. + "CONSTRAINT chk_storage_copies_committed_evidence CHECK ((status = 'committed') = (confirmed_attempt_id IS NOT NULL))", + "CONSTRAINT chk_storage_copies_confirmed_attempt_shape CHECK ((confirmed_attempt_id IS NULL AND confirmed_attempt_status IS NULL) OR (confirmed_attempt_id IS NOT NULL AND confirmed_attempt_id <> '' AND confirmed_attempt_status IS NOT NULL))", + "CONSTRAINT chk_storage_copies_transfer_method CHECK (transfer_method IN ('ingress', 'peer_pull'))", + "CONSTRAINT chk_storage_copies_optional_identity CHECK (provider_id <> '' AND (piece_id IS NULL OR piece_id <> '') AND (retrieval_url IS NULL OR retrieval_url <> '') AND (commit_extra_data_hex IS NULL OR commit_extra_data_hex <> ''))", + "CONSTRAINT chk_storage_copies_committed_shape CHECK (status <> 'committed' OR (piece_id IS NOT NULL AND piece_id <> '' AND retrieval_url IS NOT NULL AND retrieval_url <> ''))", + "CONSTRAINT chk_storage_copies_commit_ready CHECK (commit_ready_at IS NULL OR status IN ('piece_ready', 'committing', 'committed'))", + "CONSTRAINT chk_storage_copies_content_size CHECK (content_size >= 0)", + // Ingress progress belongs to the transfer that produces it, so + // only an ingress copy may carry it. + "CONSTRAINT chk_storage_copies_ingress_progress CHECK (transfer_method = 'ingress' OR (ingress_bytes_transferred = 0 AND ingress_store_attempt = 0 AND progress_updated_at IS NULL))", + "CONSTRAINT chk_storage_copies_ingress_bytes CHECK (ingress_bytes_transferred >= 0 AND ingress_bytes_transferred <= content_size)", + "CONSTRAINT chk_storage_copies_ingress_attempt CHECK (ingress_store_attempt >= 0)", + }, + foreignKeys: []string{ + "(content_id, bucket_id, content_size) REFERENCES storage_contents (id, bucket_id, content_size) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(storage_data_set_id, bucket_id, copy_index, provider_id) REFERENCES storage_data_sets (id, bucket_id, copy_index, provider_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(active_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + forwardForeignKeys: []initialForwardForeignKey{storageCopyConfirmedAttemptForeignKey2026090101()}, + }, + storageCommitAttemptTable2026090101(), + storagePullAttemptTable2026090101(), + storageReplacementTable2026090101(), + storageDataSetTerminationTable2026090101(), + storageReplacementItemTable2026090101(), + storageCleanupCopyTable2026090101(), + } + for _, table := range tables { + if err := createInitialTable(ctx, db, table); err != nil { + return err + } + } + if err := createInitialIndexes(ctx, db, storageIndexes2026090101()...); err != nil { + return err + } + // storage_copies was created before storage_commit_attempts existed, so + // PostgreSQL takes the projection constraint here. + return addForwardForeignKey(ctx, db, "storage_copies", storageCopyConfirmedAttemptForeignKey2026090101()) +} + +// storageCopyConfirmedAttemptForeignKey2026090101 welds the copy's committed +// state to the ledger row that proves it. storage_copies is created before +// storage_commit_attempts, so the constraint is forward-declared. +func storageCopyConfirmedAttemptForeignKey2026090101() initialForwardForeignKey { + return initialForwardForeignKey{ + name: "fk_storage_copies_confirmed_attempt", + definition: "(confirmed_attempt_id, confirmed_attempt_status) REFERENCES storage_commit_attempts (attempt_id, status) ON UPDATE RESTRICT ON DELETE RESTRICT", + } +} + +func storageCommitAttemptTable2026090101() initialTableSpec { + return initialTableSpec{ + name: "storage_commit_attempts", + jsonColumns: initialJSONColumns("storage_commit_attempts"), + model: (*storageCommitAttempt2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_storage_commit_attempts_identity CHECK (attempt_id <> '' AND (extra_data_hex IS NULL OR extra_data_hex <> '') AND (transaction_id IS NULL OR transaction_id <> '') AND (submission_json IS NULL OR submission_json <> '') AND (confirmed_transaction_id IS NULL OR confirmed_transaction_id <> '') AND (attention_code IS NULL OR attention_code <> '') AND (release_reason IS NULL OR release_reason <> ''))", + "CONSTRAINT chk_storage_commit_attempts_status CHECK (status IN ('reserved', 'attempted', 'confirmed', 'released', 'rejected'))", + // Candidate key for the copy's confirmed-attempt projection. + "CONSTRAINT uq_storage_commit_attempts_status UNIQUE (attempt_id, status)", + "CONSTRAINT chk_storage_commit_attempts_resolution CHECK ((status IN ('reserved', 'attempted') AND resolved_at IS NULL) OR (status IN ('confirmed', 'released', 'rejected') AND resolved_at IS NOT NULL))", + `CONSTRAINT chk_storage_commit_attempts_evidence_shape CHECK ( + (status = 'reserved' AND attempted_at IS NULL AND extra_data_hex IS NULL AND transaction_id IS NULL AND submission_json IS NULL AND confirmed_transaction_id IS NULL AND attention_code IS NULL AND attention_at IS NULL AND last_error IS NULL) + OR (status = 'attempted' AND attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL AND confirmed_transaction_id IS NULL AND last_error IS NULL) + OR (status = 'confirmed' AND attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL AND transaction_id IS NOT NULL AND confirmed_transaction_id IS NOT NULL AND last_error IS NULL) + OR (status = 'released' AND confirmed_transaction_id IS NULL AND last_error IS NULL AND ((attempted_at IS NULL AND extra_data_hex IS NULL AND transaction_id IS NULL AND submission_json IS NULL AND attention_code IS NULL AND attention_at IS NULL) OR (attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL))) + OR (status = 'rejected' AND attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL AND confirmed_transaction_id IS NULL AND last_error IS NOT NULL AND last_error <> '') + )`, + "CONSTRAINT chk_storage_commit_attempts_submission CHECK (submission_json IS NULL OR transaction_id IS NOT NULL)", + "CONSTRAINT chk_storage_commit_attempts_attention CHECK ((attention_code IS NULL AND attention_at IS NULL) OR (attention_code IS NOT NULL AND attention_at IS NOT NULL AND attempted_at IS NOT NULL))", + "CONSTRAINT chk_storage_commit_attempts_release CHECK ((status = 'released' AND release_reason IS NOT NULL) OR (status <> 'released' AND release_reason IS NULL))", + }, + foreignKeys: []string{ + "(content_id, storage_data_set_id) REFERENCES storage_copies (content_id, storage_data_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + } +} + +func storagePullAttemptTable2026090101() initialTableSpec { + return initialTableSpec{ + name: "storage_pull_attempts", + model: (*storagePullAttempt2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_storage_pull_attempts_identity CHECK (attempt_id <> '' AND source_provider_id <> '' AND source_data_set_id <> '' AND source_piece_id <> '' AND source_piece_cid <> '' AND source_retrieval_url <> '')", + "CONSTRAINT chk_storage_pull_attempts_status CHECK (status IN ('attempted', 'abandoned'))", + // An error only makes sense on a request nobody will observe again; + // reopening a copy carries no error string. + "CONSTRAINT chk_storage_pull_attempts_error CHECK (last_error IS NULL OR (status = 'abandoned' AND last_error <> ''))", + "CONSTRAINT chk_storage_pull_attempts_abandoned CHECK (status <> 'abandoned' OR resolved_at IS NOT NULL)", + }, + foreignKeys: []string{ + "(content_id, storage_data_set_id) REFERENCES storage_copies (content_id, storage_data_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + } +} + +func storageReplacementTable2026090101() initialTableSpec { + return initialTableSpec{ + name: "storage_replacements", + model: (*storageReplacement2026090101)(nil), + constraints: []string{ + "CONSTRAINT uq_storage_replacements_id_source UNIQUE (id, source_data_set_id)", + "CONSTRAINT uq_storage_replacements_id_target UNIQUE (id, target_data_set_id)", + "CONSTRAINT chk_storage_replacements_identity CHECK (client_request_id <> '' AND (requested_provider_id IS NULL OR requested_provider_id <> ''))", + // The replica slot is a row, so the index is a foreign key rather than a range check. + "CONSTRAINT fk_storage_replacements_replica_slot FOREIGN KEY (bucket_id, copy_index) REFERENCES bucket_replica_slots (bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", + "CONSTRAINT chk_storage_replacements_selection_mode CHECK (selection_mode IN ('automatic', 'manual'))", + "CONSTRAINT chk_storage_replacements_status CHECK (status IN ('preparing_target', 'migrating', 'waiting', 'retiring', 'cleanup_attention', 'failed', 'completed', 'superseded'))", + "CONSTRAINT chk_storage_replacements_wait_reason CHECK (wait_reason IS NULL OR wait_reason <> '')", + "CONSTRAINT chk_storage_replacements_failure_reason CHECK (failure_reason IS NULL OR failure_reason <> '')", + "CONSTRAINT chk_storage_replacements_client_request_id CHECK (length(client_request_id) BETWEEN 1 AND 128)", + "CONSTRAINT chk_storage_replacements_distinct_data_sets CHECK (source_data_set_id <> target_data_set_id)", + "CONSTRAINT chk_storage_replacements_items CHECK (items_total >= 0 AND items_copied >= 0 AND items_copied <= items_total)", + "CONSTRAINT chk_storage_replacements_generation CHECK (task_generation >= 1)", + }, + foreignKeys: []string{ + "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(source_data_set_id, bucket_id, copy_index) REFERENCES storage_data_sets (id, bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(target_data_set_id, bucket_id, copy_index) REFERENCES storage_data_sets (id, bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(superseded_by_id) REFERENCES storage_replacements (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + } +} + +func storageReplacementItemTable2026090101() initialTableSpec { + return initialTableSpec{ + name: "storage_replacement_items", + model: (*storageReplacementItem2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_storage_replacement_items_status CHECK (status IN ('pending', 'copied', 'cancelled', 'attention'))", + "CONSTRAINT uq_storage_replacement_items_content UNIQUE (replacement_id, content_id)", + }, + foreignKeys: []string{ + "(replacement_id) REFERENCES storage_replacements (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(content_id) REFERENCES storage_contents (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(replacement_id, target_data_set_id) REFERENCES storage_replacements (id, target_data_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(content_id, target_data_set_id) REFERENCES storage_copies (content_id, storage_data_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + } +} + +func storageCleanupCopyTable2026090101() initialTableSpec { + return initialTableSpec{ + name: "storage_cleanup_copies", + model: (*storageCleanupCopy2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_storage_cleanup_copies_identity CHECK (provider_id <> '' AND piece_id <> '' AND piece_cid <> '' AND (data_set_id IS NULL OR data_set_id <> '') AND (client_data_set_id IS NULL OR client_data_set_id <> '') AND (delete_tx_hash IS NULL OR delete_tx_hash <> ''))", + // The replica slot is a row, so the index is a foreign key rather than a range check. + "CONSTRAINT fk_storage_cleanup_copies_replica_slot FOREIGN KEY (bucket_id, copy_index) REFERENCES bucket_replica_slots (bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", + "CONSTRAINT chk_storage_cleanup_copies_status CHECK (status IN ('pending', 'delete_scheduled', 'removed', 'failed', 'unsupported'))", + "CONSTRAINT chk_storage_cleanup_copies_delete_scheduled CHECK (status <> 'delete_scheduled' OR (delete_tx_hash IS NOT NULL AND scheduled_at IS NOT NULL))", + "CONSTRAINT uq_storage_cleanup_copies_physical UNIQUE (content_id, storage_data_set_id, piece_id)", + }, + foreignKeys: []string{ + "(content_id, bucket_id) REFERENCES storage_contents (id, bucket_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(storage_data_set_id, bucket_id, copy_index, provider_id) REFERENCES storage_data_sets (id, bucket_id, copy_index, provider_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + } +} + +func storageIndexes2026090101() []initialIndexSpec { + return []initialIndexSpec{ + {name: "idx_storage_contents_bucket_id", table: "storage_contents", columns: []string{"bucket_id", "id"}}, + {name: "idx_storage_contents_cleanup_task", table: "storage_contents", columns: []string{"cleanup_task_id"}, where: "cleanup_task_id IS NOT NULL", unique: true}, + {name: "idx_storage_data_sets_provider_data_set", table: "storage_data_sets", columns: []string{"provider_id", "data_set_id"}, where: "data_set_id IS NOT NULL", unique: true}, + {name: "idx_storage_data_sets_bucket_copy_current", table: "storage_data_sets", columns: []string{"bucket_id", "copy_index"}, where: "is_current = TRUE", unique: true}, + {name: "idx_storage_data_sets_bucket_copy_generation", table: "storage_data_sets", columns: []string{"bucket_id", "copy_index", "generation"}, unique: true}, + {name: "idx_storage_data_sets_bucket_provider_active", table: "storage_data_sets", columns: []string{"bucket_id", "provider_id"}, where: "status <> 'retired'", unique: true}, + {name: "idx_storage_data_sets_created_by_content", table: "storage_data_sets", columns: []string{"created_by_content_id"}}, + {name: "idx_storage_data_sets_last_used_content", table: "storage_data_sets", columns: []string{"last_used_content_id"}}, + {name: "idx_storage_data_sets_ensure_task", table: "storage_data_sets", columns: []string{"ensure_task_id"}, where: "ensure_task_id IS NOT NULL", unique: true}, + {name: "idx_storage_data_sets_retirement_task", table: "storage_data_sets", columns: []string{"retirement_task_id"}, where: "retirement_task_id IS NOT NULL", unique: true}, + {name: "idx_storage_copies_content_slot", table: "storage_copies", columns: []string{"content_id", "copy_index"}}, + {name: "idx_storage_copies_data_set_identity", table: "storage_copies", columns: []string{"storage_data_set_id", "bucket_id", "copy_index", "provider_id"}}, + {name: "idx_storage_copies_content_transfer_method_index", table: "storage_copies", columns: []string{"content_id", "transfer_method", "copy_index"}}, + {name: "idx_storage_copies_ingress_content", table: "storage_copies", columns: []string{"content_id"}, where: "transfer_method = 'ingress'", unique: true}, + {name: "idx_storage_copies_status_data_set_content", table: "storage_copies", columns: []string{"status", "storage_data_set_id", "content_id"}}, + {name: "idx_storage_copies_status_piece_identity_content", table: "storage_copies", columns: []string{"status", "provider_id", "piece_id", "content_id"}}, + {name: "idx_storage_copies_commit_ready", table: "storage_copies", columns: []string{"storage_data_set_id", "commit_ready_at", "id"}, where: "status = 'piece_ready' AND commit_ready_at IS NOT NULL"}, + {name: "idx_storage_copies_active_task", table: "storage_copies", columns: []string{"active_task_id"}, where: "active_task_id IS NOT NULL", unique: true}, + {name: "idx_storage_commit_attempts_unresolved_copy", table: "storage_commit_attempts", columns: []string{"content_id", "storage_data_set_id"}, where: "resolved_at IS NULL", unique: true}, + {name: "idx_storage_commit_attempts_unresolved_data_set", table: "storage_commit_attempts", columns: []string{"storage_data_set_id", "created_at", "attempt_id"}, where: "resolved_at IS NULL"}, + {name: "idx_storage_commit_attempts_copy_history", table: "storage_commit_attempts", columns: []string{"content_id", "storage_data_set_id", "created_at DESC", "attempt_id"}}, + {name: "idx_storage_cleanup_copies_replica_slot", table: "storage_cleanup_copies", columns: []string{"bucket_id", "copy_index"}}, + {name: "idx_storage_copies_confirmed_attempt", table: "storage_copies", columns: []string{"confirmed_attempt_id", "confirmed_attempt_status"}, where: "confirmed_attempt_id IS NOT NULL"}, + {name: "idx_storage_copies_replica_slot", table: "storage_copies", columns: []string{"bucket_id", "copy_index"}}, + // One unresolved attempt per copy: a second source can only be tried + // after the first is abandoned. + {name: "idx_storage_pull_attempts_unresolved_copy", table: "storage_pull_attempts", columns: []string{"content_id", "storage_data_set_id"}, where: "resolved_at IS NULL", unique: true}, + {name: "idx_storage_pull_attempts_copy_history", table: "storage_pull_attempts", columns: []string{"content_id", "storage_data_set_id", "created_at DESC", "attempt_id"}}, + {name: "idx_storage_replacements_active_source", table: "storage_replacements", columns: []string{"source_data_set_id"}, where: "status NOT IN ('completed', 'superseded')", unique: true}, + {name: "idx_storage_replacements_active_target", table: "storage_replacements", columns: []string{"target_data_set_id"}, where: "status NOT IN ('completed', 'superseded')", unique: true}, + {name: "idx_storage_replacements_active_bucket_slot", table: "storage_replacements", columns: []string{"bucket_id", "copy_index"}, where: "status NOT IN ('completed', 'superseded')", unique: true}, + {name: "idx_storage_replacements_source_identity", table: "storage_replacements", columns: []string{"source_data_set_id", "bucket_id", "copy_index"}}, + {name: "idx_storage_replacements_target_identity", table: "storage_replacements", columns: []string{"target_data_set_id", "bucket_id", "copy_index"}}, + {name: "idx_storage_replacements_superseded_by", table: "storage_replacements", columns: []string{"superseded_by_id"}, where: "superseded_by_id IS NOT NULL"}, + {name: "idx_storage_replacements_bucket_slot", table: "storage_replacements", columns: []string{"bucket_id", "copy_index", "id"}}, + {name: "idx_storage_replacements_bucket_request", table: "storage_replacements", columns: []string{"bucket_id", "client_request_id"}, unique: true}, + {name: "idx_storage_replacements_task", table: "storage_replacements", columns: []string{"task_id"}, where: "task_id IS NOT NULL", unique: true}, + {name: "idx_storage_replacement_items_state", table: "storage_replacement_items", columns: []string{"replacement_id", "status", "id"}}, + {name: "idx_storage_replacement_items_content_id", table: "storage_replacement_items", columns: []string{"content_id"}}, + {name: "idx_storage_replacement_items_target_copy", table: "storage_replacement_items", columns: []string{"content_id", "target_data_set_id"}}, + {name: "idx_storage_cleanup_copies_content_status", table: "storage_cleanup_copies", columns: []string{"content_id", "status", "id"}}, + {name: "idx_storage_cleanup_copies_data_set_identity", table: "storage_cleanup_copies", columns: []string{"storage_data_set_id", "bucket_id", "copy_index", "provider_id"}}, + {name: "idx_storage_cleanup_copies_status_scheduled", table: "storage_cleanup_copies", columns: []string{"status", "scheduled_at", "id"}}, + } +} diff --git a/internal/db/migrations/schema_wallet_observability.go b/internal/db/migrations/schema_wallet_observability.go new file mode 100644 index 0000000..25116ab --- /dev/null +++ b/internal/db/migrations/schema_wallet_observability.go @@ -0,0 +1,148 @@ +package migrations + +import ( + "context" + "encoding/json" + "time" + + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" +) + +type walletOperation2026090101 struct { + bun.BaseModel `bun:"table:wallet_operations"` + + ID int64 `bun:",pk,autoincrement,identity"` + Type string `bun:"type:text,notnull"` + ClientRequestID string `bun:"type:text,notnull"` + Amount string `bun:"type:text,notnull"` + Status string `bun:"type:text,notnull,default:'pending'"` + TxHash *string `bun:"type:text"` + LastError *string `bun:"type:text"` + BroadcastAttemptedAt *time.Time + TaskID *int64 + StartedAt *time.Time + SubmittedAt *time.Time + CompletedAt *time.Time + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +func createWalletSchema(ctx context.Context, db bun.IDB) error { + amountCheck := `((type = 'approve' AND amount = '0') OR (type IN ('fund', 'withdraw') AND amount GLOB '[1-9]*' AND amount NOT GLOB '*[^0-9]*'))` + if db.Dialect().Name() == dialect.PG { + amountCheck = `((type = 'approve' AND amount = '0') OR (type IN ('fund', 'withdraw') AND amount ~ '^[1-9][0-9]*$'))` + } + if err := createInitialTable(ctx, db, initialTableSpec{ + name: "wallet_operations", + model: (*walletOperation2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_wallet_operations_identity CHECK (client_request_id <> '' AND amount <> '' AND (tx_hash IS NULL OR tx_hash <> ''))", + "CONSTRAINT chk_wallet_operations_type CHECK (type IN ('fund', 'withdraw', 'approve'))", + "CONSTRAINT chk_wallet_operations_status CHECK (status IN ('pending', 'submitted', 'confirmed', 'failed', 'unknown'))", + "CONSTRAINT chk_wallet_operations_submitted_shape CHECK (status <> 'submitted' OR (tx_hash IS NOT NULL AND submitted_at IS NOT NULL))", + "CONSTRAINT chk_wallet_operations_amount CHECK (" + amountCheck + ")", + // A uint256 in base 10 is at most 78 digits. The column stays text + // because the value is returned and compared verbatim. + "CONSTRAINT chk_wallet_operations_amount_length CHECK (length(amount) BETWEEN 1 AND 78)", + }, + foreignKeys: []string{ + "(task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }); err != nil { + return err + } + return createInitialIndexes(ctx, db, + initialIndexSpec{name: "idx_wallet_operations_request", table: "wallet_operations", columns: []string{"type", "client_request_id"}, unique: true}, + initialIndexSpec{name: "idx_wallet_operations_status_created", table: "wallet_operations", columns: []string{"status", "created_at", "id"}}, + initialIndexSpec{name: "idx_wallet_operations_recent", table: "wallet_operations", columns: []string{"created_at DESC", "id DESC"}}, + initialIndexSpec{name: "idx_wallet_operations_task", table: "wallet_operations", columns: []string{"task_id"}, where: "task_id IS NOT NULL", unique: true}, + ) +} + +type observabilityCollectionState2026090101 struct { + bun.BaseModel `bun:"table:observability_collection_states"` + + CollectionType string `bun:"type:text,pk"` + LastCheckedAt time.Time `bun:",notnull"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type observabilityProviderState2026090101 struct { + bun.BaseModel `bun:"table:observability_provider_states"` + + ProviderID string `bun:"type:text,pk"` + Status string `bun:"type:text,notnull"` + ReasonCodes json.RawMessage `bun:"type:jsonb,notnull"` + Active *bool + HasPDP *bool + ServiceURL *string `bun:"type:text"` + HealthStatus *string `bun:"type:text"` + LastCheckedAt time.Time `bun:",notnull"` + LastError *string `bun:"type:text"` + Evidence json.RawMessage `bun:"evidence_json,type:jsonb,notnull"` +} + +type observabilityDataSetState2026090101 struct { + bun.BaseModel `bun:"table:observability_data_set_states"` + + LocalDataSetID int64 `bun:",pk"` + BucketID int64 `bun:",notnull"` + CopyIndex int `bun:"type:integer,notnull"` + ProviderID string `bun:"type:text,notnull"` + ChainDataSetID *string `bun:"type:text"` + ClientDataSetID *string `bun:"type:text"` + // The bucket's name and the data set's own status are a join away and can + // disagree with the authorities that own them, so neither is copied here. + Status string `bun:"type:text,notnull"` + ReasonCodes json.RawMessage `bun:"type:jsonb,notnull"` + ActivePieceCount *int64 + LastCheckedAt time.Time `bun:",notnull"` + LastError *string `bun:"type:text"` + Evidence json.RawMessage `bun:"evidence_json,type:jsonb,notnull"` +} + +func createObservabilitySchema(ctx context.Context, db bun.IDB) error { + tables := []initialTableSpec{ + { + name: "observability_collection_states", + model: (*observabilityCollectionState2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_observability_collection_type CHECK (collection_type IN ('providers', 'data_sets'))", + }, + }, + { + name: "observability_provider_states", + model: (*observabilityProviderState2026090101)(nil), + jsonColumns: initialJSONColumns("observability_provider_states"), + constraints: []string{ + "CONSTRAINT chk_observability_provider_identity CHECK (provider_id <> '')", + "CONSTRAINT chk_observability_provider_status CHECK (status IN ('available', 'degraded', 'unavailable', 'unknown'))", + }, + }, + { + name: "observability_data_set_states", + model: (*observabilityDataSetState2026090101)(nil), + jsonColumns: initialJSONColumns("observability_data_set_states"), + constraints: []string{ + "CONSTRAINT chk_observability_data_set_identity CHECK (provider_id <> '' AND (chain_data_set_id IS NULL OR chain_data_set_id <> '') AND (client_data_set_id IS NULL OR client_data_set_id <> ''))", + "CONSTRAINT fk_observability_data_set_replica_slot FOREIGN KEY (bucket_id, copy_index) REFERENCES bucket_replica_slots (bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", + "CONSTRAINT chk_observability_data_set_status CHECK (status IN ('available', 'degraded', 'unavailable', 'unknown'))", + }, + foreignKeys: []string{ + "(local_data_set_id, bucket_id, copy_index, provider_id) REFERENCES storage_data_sets (id, bucket_id, copy_index, provider_id) ON UPDATE RESTRICT ON DELETE CASCADE", + }, + }, + } + for _, table := range tables { + if err := createInitialTable(ctx, db, table); err != nil { + return err + } + } + return createInitialIndexes(ctx, db, + initialIndexSpec{name: "idx_observability_provider_states_status", table: "observability_provider_states", columns: []string{"status", "last_checked_at"}}, + initialIndexSpec{name: "idx_observability_data_set_states_bucket_status", table: "observability_data_set_states", columns: []string{"bucket_id", "status", "last_checked_at"}}, + initialIndexSpec{name: "idx_observability_data_set_states_provider_status", table: "observability_data_set_states", columns: []string{"provider_id", "status", "last_checked_at"}}, + ) +} diff --git a/internal/db/migrations/storage_data_set_generations_test.go b/internal/db/migrations/storage_data_set_generations_test.go deleted file mode 100644 index e718a1f..0000000 --- a/internal/db/migrations/storage_data_set_generations_test.go +++ /dev/null @@ -1,417 +0,0 @@ -package migrations - -import ( - "context" - "database/sql" - "fmt" - "strings" - "testing" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect/sqlitedialect" - - _ "modernc.org/sqlite" -) - -func TestStorageDataSetGenerationsMigrationPreservesExistingSlots(t *testing.T) { - ctx := context.Background() - db := newGenerationsTestDB(t, "generations_preserve") - seedLegacyDataSet(t, db, 1, 1, 0, "101") - - if err := up2026082101StorageDataSetGenerations(ctx, db); err != nil { - t.Fatalf("up migration: %v", err) - } - - for _, column := range []string{"is_current", "generation"} { - if !sqliteColumnExists(t, db, "storage_data_sets", column) { - t.Fatalf("storage_data_sets.%s column missing", column) - } - } - var isCurrent bool - var generation int - if err := db.QueryRow("SELECT is_current, generation FROM storage_data_sets WHERE id = 1"). - Scan(&isCurrent, &generation); err != nil { - t.Fatalf("select migrated data set: %v", err) - } - if !isCurrent || generation != 1 { - t.Fatalf("migrated data set is_current=%v generation=%d, want true/1", isCurrent, generation) - } - - for _, index := range []string{ - "idx_storage_data_sets_bucket_copy_index", - "idx_storage_data_sets_bucket_provider", - "idx_storage_upload_copies_upload_index", - } { - if sqliteIndexExists(t, db, index) { - t.Fatalf("index %s should have been replaced", index) - } - } - for _, table := range []string{"storage_replacements", "storage_replacement_items"} { - if !sqliteTableExists(t, db, table) { - t.Fatalf("table %s missing", table) - } - } - for _, column := range []string{ - "client_request_id", "failure_reason", "abandoned_termination_tx_hash", - "abandoned_termination_epoch", "abandoned_termination_observed_at", - "state_version", "last_dispatched_at", - } { - if !sqliteColumnExists(t, db, "storage_replacements", column) { - t.Fatalf("storage_replacements.%s column missing", column) - } - } - for _, column := range []string{ - "scheduled_at", "retry_count", "max_retries", "claimed_at", "lease_until", - } { - if !sqliteColumnExists(t, db, "storage_replacement_items", column) { - t.Fatalf("storage_replacement_items.%s column missing", column) - } - } - if !sqliteIndexExists(t, db, "idx_storage_replacements_bucket_request") { - t.Fatal("bucket-scoped replacement idempotency index missing") - } - for _, index := range []string{ - "idx_storage_uploads_bucket_id", - "idx_storage_replacement_items_due", - "idx_storage_replacement_items_lease", - "idx_storage_replacement_items_state", - "idx_storage_replacement_items_upload_id", - "idx_storage_replacements_dispatch", - } { - if !sqliteIndexExists(t, db, index) { - t.Fatalf("persistent query index %s missing", index) - } - } - if sqliteIndexExists(t, db, "idx_storage_replacement_items_next") { - t.Fatal("obsolete replacement item index should not be created") - } -} - -func TestStorageDataSetGenerationsMigrationEnforcesSlotInvariants(t *testing.T) { - ctx := context.Background() - db := newGenerationsTestDB(t, "generations_invariants") - seedLegacyDataSet(t, db, 1, 1, 0, "101") - if err := up2026082101StorageDataSetGenerations(ctx, db); err != nil { - t.Fatalf("up migration: %v", err) - } - - // A slot keeps exactly one writable generation... - if err := insertDataSet(db, 2, 1, 0, "202", true, 2); err == nil { - t.Fatal("second current generation for one slot was accepted, want unique violation") - } - // ...but may retain historical ones. - if err := insertDataSet(db, 2, 1, 0, "202", false, 2); err != nil { - t.Fatalf("historical generation rejected: %v", err) - } - if err := insertDataSet(db, 3, 1, 0, "303", false, 2); err == nil { - t.Fatal("duplicate generation number for one slot was accepted, want unique violation") - } - - // A provider that only holds a historical generation can be selected again, - // which is what lets an operator reuse a previously used provider. - mustExecMigrationTest(t, db, "UPDATE storage_data_sets SET is_current = 0 WHERE id = 1") - if err := insertDataSet(db, 4, 1, 0, "101", true, 3); err != nil { - t.Fatalf("reusing a historical provider rejected: %v", err) - } - // Provider 101 now serves slot 0, so it cannot also take slot 1. - if err := insertDataSet(db, 5, 1, 1, "101", true, 1); err == nil { - t.Fatal("provider bound to two current slots was accepted, want unique violation") - } -} - -func TestStorageDataSetGenerationsMigrationClosesUnboundCopyHole(t *testing.T) { - ctx := context.Background() - db := newGenerationsTestDB(t, "generations_copy_hole") - seedLegacyDataSet(t, db, 1, 1, 0, "101") - mustExecMigrationTest(t, db, "INSERT INTO storage_uploads (id, bucket_id) VALUES (1, 1)") - if err := up2026082101StorageDataSetGenerations(ctx, db); err != nil { - t.Fatalf("up migration: %v", err) - } - - mustExecMigrationTest(t, db, - "INSERT INTO storage_upload_copies (id, upload_id, copy_index, storage_data_set_id) VALUES (1, 1, 0, 1)") - if err := insertUploadCopy(db, 2, 1, 0, sql.NullInt64{Int64: 1, Valid: true}); err == nil { - t.Fatal("duplicate copy for one data set was accepted, want unique violation") - } - - // NULL compares distinct in a unique index, so unbound copies need their own - // partial index or a slot could accumulate duplicates. - mustExecMigrationTest(t, db, - "INSERT INTO storage_upload_copies (id, upload_id, copy_index, storage_data_set_id) VALUES (3, 1, 1, NULL)") - if err := insertUploadCopy(db, 4, 1, 1, sql.NullInt64{}); err == nil { - t.Fatal("duplicate unbound copy for one slot was accepted, want unique violation") - } -} - -func TestStorageDataSetGenerationsMigrationLimitsActiveReplacements(t *testing.T) { - ctx := context.Background() - db := newGenerationsTestDB(t, "generations_active_replacement") - seedLegacyDataSet(t, db, 1, 1, 0, "101") - if err := up2026082101StorageDataSetGenerations(ctx, db); err != nil { - t.Fatalf("up migration: %v", err) - } - mustExecMigrationTest(t, db, "UPDATE storage_data_sets SET is_current = 0 WHERE id = 1") - if err := insertDataSet(db, 2, 1, 0, "202", true, 2); err != nil { - t.Fatalf("seed target generation: %v", err) - } - - if err := insertReplacement(db, 1, 1, 2, "migrating"); err != nil { - t.Fatalf("first replacement rejected: %v", err) - } - if err := insertReplacement(db, 2, 1, 2, "preparing_target"); err == nil { - t.Fatal("second active replacement for one source was accepted, want unique violation") - } - // A terminal replacement releases its source for a later confirmation. - mustExecMigrationTest(t, db, "UPDATE storage_replacements SET status = 'superseded' WHERE id = 1") - if err := insertReplacement(db, 2, 1, 2, "preparing_target"); err != nil { - t.Fatalf("replacement after supersede rejected: %v", err) - } - if err := insertReplacement(db, 3, 1, 1, "preparing_target"); err == nil { - t.Fatal("replacement onto itself was accepted, want check violation") - } -} - -func TestStorageDataSetGenerationsMigrationEnforcesReplacementQueueState(t *testing.T) { - ctx := context.Background() - db := newGenerationsTestDB(t, "generations_replacement_queue_constraints") - seedLegacyDataSet(t, db, 1, 1, 0, "101") - if err := up2026082101StorageDataSetGenerations(ctx, db); err != nil { - t.Fatalf("up migration: %v", err) - } - mustExecMigrationTest(t, db, "UPDATE storage_data_sets SET is_current = 0 WHERE id = 1") - if err := insertDataSet(db, 2, 1, 0, "202", true, 2); err != nil { - t.Fatalf("insert target: %v", err) - } - if err := insertReplacement(db, 1, 1, 2, "migrating"); err != nil { - t.Fatalf("insert replacement: %v", err) - } - mustExecMigrationTest(t, db, "INSERT INTO storage_uploads (id, bucket_id) VALUES (1, 1), (2, 1), (3, 1)") - - mustExecMigrationTest(t, db, `INSERT INTO storage_replacement_items - (replacement_id, upload_id, status, max_retries) VALUES (1, 1, 'retrying', 5)`) - mustExecMigrationTest(t, db, `INSERT INTO storage_replacement_items - (replacement_id, upload_id, status, max_retries) VALUES (1, 2, 'failed', 5)`) - if _, err := db.Exec(`INSERT INTO storage_replacement_items - (replacement_id, upload_id, status, max_retries) VALUES (1, 3, 'unknown', 5)`); err == nil { - t.Fatal("unknown item status accepted") - } - if _, err := db.Exec(`UPDATE storage_replacement_items SET claimed_at = CURRENT_TIMESTAMP WHERE upload_id = 1`); err == nil { - t.Fatal("half-populated item claim accepted") - } - if _, err := db.Exec(`UPDATE storage_replacement_items SET status = 'running' WHERE upload_id = 1`); err == nil { - t.Fatal("running item without a lease accepted") - } -} - -func TestStorageDataSetGenerationsMigrationRejectsIncompatibleData(t *testing.T) { - ctx := context.Background() - db := newGenerationsTestDB(t, "generations_incompatible") - seedLegacyDataSet(t, db, 1, 1, 0, "101") - mustExecMigrationTest(t, db, "INSERT INTO storage_uploads (id, bucket_id) VALUES (1, 1)") - // The old schema never enforced this, so a database damaged by an earlier - // bug must fail loudly instead of part way through the index rebuild. - mustExecMigrationTest(t, db, "DROP INDEX idx_storage_upload_copies_upload_index") - mustExecMigrationTest(t, db, - "INSERT INTO storage_upload_copies (id, upload_id, copy_index, storage_data_set_id) VALUES (1, 1, 0, 1), (2, 1, 1, 1)") - - err := up2026082101StorageDataSetGenerations(ctx, db) - if err == nil { - t.Fatal("migration accepted duplicate copies for one data set, want failure") - } - if !strings.Contains(err.Error(), "sharing one data set") { - t.Fatalf("error = %v, want it to name the offending invariant", err) - } - if sqliteColumnExists(t, db, "storage_data_sets", "is_current") { - t.Fatal("failed migration left is_current behind, want a rolled back transaction") - } -} - -func TestStorageDataSetGenerationsMigrationRejectsPartialSchema(t *testing.T) { - ctx := context.Background() - db := newGenerationsTestDB(t, "generations_partial_schema") - mustExecMigrationTest(t, db, "ALTER TABLE storage_data_sets ADD COLUMN generation INTEGER NOT NULL DEFAULT 1") - - err := up2026082101StorageDataSetGenerations(ctx, db) - if err == nil { - t.Fatal("migration accepted a partial schema") - } - if !strings.Contains(err.Error(), "partial schema state") { - t.Fatalf("partial schema error = %v", err) - } - if sqliteColumnExists(t, db, "storage_data_sets", "is_current") { - t.Fatal("partial schema failure continued applying DDL") - } -} - -func TestStorageDataSetGenerationsMigrationDown(t *testing.T) { - ctx := context.Background() - db := newGenerationsTestDB(t, "generations_down") - seedLegacyDataSet(t, db, 1, 1, 0, "101") - if err := up2026082101StorageDataSetGenerations(ctx, db); err != nil { - t.Fatalf("up migration: %v", err) - } - if err := down2026082101StorageDataSetGenerations(ctx, db); err != nil { - t.Fatalf("down migration: %v", err) - } - for _, column := range []string{"is_current", "generation"} { - if sqliteColumnExists(t, db, "storage_data_sets", column) { - t.Fatalf("storage_data_sets.%s survived rollback", column) - } - } - for _, index := range []string{ - "idx_storage_data_sets_bucket_copy_index", - "idx_storage_data_sets_bucket_provider", - "idx_storage_upload_copies_upload_index", - } { - if !sqliteIndexExists(t, db, index) { - t.Fatalf("index %s was not restored", index) - } - } - if sqliteTableExists(t, db, "storage_replacements") { - t.Fatal("storage_replacements survived rollback") - } - if sqliteIndexExists(t, db, "idx_storage_uploads_bucket_id") { - t.Fatal("idx_storage_uploads_bucket_id survived rollback") - } -} - -func TestStorageDataSetGenerationsMigrationDownDiagnosesConflicts(t *testing.T) { - tests := []struct { - name string - want string - setup func(*testing.T, *bun.DB) - }{ - { - name: "bucket copy index", - want: "duplicate (bucket_id, copy_index)", - setup: func(t *testing.T, db *bun.DB) { - if err := insertDataSet(db, 2, 1, 0, "202", false, 2); err != nil { - t.Fatalf("seed second generation: %v", err) - } - }, - }, - { - name: "bucket provider", - want: "duplicate (bucket_id, provider_id)", - setup: func(t *testing.T, db *bun.DB) { - if err := insertDataSet(db, 2, 1, 1, "101", false, 1); err != nil { - t.Fatalf("seed reused provider: %v", err) - } - }, - }, - { - name: "upload copy index", - want: "duplicate (upload_id, copy_index)", - setup: func(t *testing.T, db *bun.DB) { - if err := insertDataSet(db, 2, 1, 1, "202", true, 1); err != nil { - t.Fatalf("seed second data set: %v", err) - } - mustExecMigrationTest(t, db, "INSERT INTO storage_uploads (id, bucket_id) VALUES (1, 1)") - mustExecMigrationTest(t, db, `INSERT INTO storage_upload_copies - (id, upload_id, copy_index, storage_data_set_id) - VALUES (1, 1, 0, 1), (2, 1, 0, 2)`) - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - db := newGenerationsTestDB(t, "generations_down_"+strings.ReplaceAll(tt.name, " ", "_")) - seedLegacyDataSet(t, db, 1, 1, 0, "101") - if err := up2026082101StorageDataSetGenerations(ctx, db); err != nil { - t.Fatalf("up migration: %v", err) - } - tt.setup(t, db) - - err := down2026082101StorageDataSetGenerations(ctx, db) - if err == nil { - t.Fatal("rollback accepted incompatible data, want failure") - } - if !strings.Contains(err.Error(), tt.want) { - t.Fatalf("rollback error = %v, want %q", err, tt.want) - } - if !sqliteColumnExists(t, db, "storage_data_sets", "generation") { - t.Fatal("failed rollback dropped generation") - } - }) - } -} - -func newGenerationsTestDB(t *testing.T, name string) *bun.DB { - t.Helper() - sqldb, err := sql.Open("sqlite", "file:"+name+"?mode=memory&cache=shared&_pragma=foreign_keys(1)") - if err != nil { - t.Fatalf("open sqlite: %v", err) - } - sqldb.SetMaxOpenConns(1) - db := bun.NewDB(sqldb, sqlitedialect.New()) - t.Cleanup(func() { _ = db.Close() }) - - // Stand-ins carrying only what this migration reads or rewrites. - schema := []string{ - `CREATE TABLE buckets (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL UNIQUE)`, - `CREATE TABLE storage_uploads (id INTEGER PRIMARY KEY AUTOINCREMENT, bucket_id INTEGER NOT NULL REFERENCES buckets (id))`, - `CREATE TABLE storage_data_sets ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - bucket_id INTEGER NOT NULL REFERENCES buckets (id), - provider_id TEXT NOT NULL, - copy_index INTEGER NOT NULL, - status TEXT NOT NULL DEFAULT 'ready' - )`, - `CREATE TABLE storage_upload_copies ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - upload_id INTEGER NOT NULL REFERENCES storage_uploads (id), - copy_index INTEGER NOT NULL, - storage_data_set_id INTEGER REFERENCES storage_data_sets (id) - )`, - `CREATE UNIQUE INDEX idx_storage_data_sets_bucket_copy_index ON storage_data_sets (bucket_id, copy_index)`, - `CREATE UNIQUE INDEX idx_storage_data_sets_bucket_provider ON storage_data_sets (bucket_id, provider_id)`, - `CREATE UNIQUE INDEX idx_storage_upload_copies_upload_index ON storage_upload_copies (upload_id, copy_index)`, - `INSERT INTO buckets (id, name) VALUES (1, 'replacement-bucket')`, - } - for _, query := range schema { - mustExecMigrationTest(t, db, query) - } - return db -} - -func seedLegacyDataSet(t *testing.T, db *bun.DB, id, bucketID int64, copyIndex int, providerID string) { - t.Helper() - mustExecMigrationTest(t, db, fmt.Sprintf( - "INSERT INTO storage_data_sets (id, bucket_id, provider_id, copy_index) VALUES (%d, %d, '%s', %d)", - id, bucketID, providerID, copyIndex, - )) -} - -func insertDataSet(db *bun.DB, id, bucketID int64, copyIndex int, providerID string, isCurrent bool, generation int) error { - _, err := db.Exec( - "INSERT INTO storage_data_sets (id, bucket_id, provider_id, copy_index, is_current, generation) VALUES (?, ?, ?, ?, ?, ?)", - id, bucketID, providerID, copyIndex, isCurrent, generation, - ) - return err -} - -func insertUploadCopy(db *bun.DB, id, uploadID int64, copyIndex int, dataSetID sql.NullInt64) error { - _, err := db.Exec( - "INSERT INTO storage_upload_copies (id, upload_id, copy_index, storage_data_set_id) VALUES (?, ?, ?, ?)", - id, uploadID, copyIndex, dataSetID, - ) - return err -} - -func insertReplacement(db *bun.DB, id, sourceDataSetID, targetDataSetID int64, status string) error { - _, err := db.Exec( - `INSERT INTO storage_replacements - (id, bucket_id, copy_index, source_data_set_id, target_data_set_id, selection_mode, client_request_id, status) - VALUES (?, 1, 0, ?, ?, 'automatic', ?, ?)`, - id, sourceDataSetID, targetDataSetID, fmt.Sprintf("migration-request-%d", id), status, - ) - return err -} - -func mustExecMigrationTest(t *testing.T, db *bun.DB, query string) { - t.Helper() - if _, err := db.Exec(query); err != nil { - t.Fatalf("exec %q: %v", query, err) - } -} diff --git a/internal/db/repository/bucket_repo.go b/internal/db/repository/bucket_repo.go index 83d5556..ccf2365 100644 --- a/internal/db/repository/bucket_repo.go +++ b/internal/db/repository/bucket_repo.go @@ -17,15 +17,22 @@ type BunBucketRepo struct { var _ BucketRepository = (*BunBucketRepo)(nil) +// Create inserts a bucket together with the replica slots its durability policy +// asks for, so a bucket never exists without the slots its data sets bind to. func (r *BunBucketRepo) Create(ctx context.Context, bucket *model.Bucket) error { - _, err := r.db.NewInsert().Model(bucket).Exec(ctx) - if err != nil { - if isUniqueViolation(err) { - return fmt.Errorf("inserting bucket %q: %w", bucket.Name, ErrAlreadyExists) + if !model.ValidStorageCopies(bucket.DefaultCopies) || !model.ValidStorageCopies(bucket.MinimumDurableCopies) || + bucket.MinimumDurableCopies > bucket.DefaultCopies { + return fmt.Errorf("inserting bucket %q: %w", bucket.Name, ErrInvalidInput) + } + return runMaybeTx(ctx, r.db, func(db bun.IDB) error { + if _, err := db.NewInsert().Model(bucket).Exec(ctx); err != nil { + if isUniqueViolation(err) { + return fmt.Errorf("inserting bucket %q: %w", bucket.Name, ErrAlreadyExists) + } + return fmt.Errorf("inserting bucket: %w", err) } - return fmt.Errorf("inserting bucket: %w", err) - } - return nil + return openBucketReplicaSlots(ctx, db, bucket.ID, bucket.DefaultCopies) + }) } func (r *BunBucketRepo) GetByName(ctx context.Context, name string) (*model.Bucket, error) { @@ -81,7 +88,7 @@ func (r *BunBucketRepo) ListActive(ctx context.Context) ([]model.Bucket, error) var buckets []model.Bucket err := r.db.NewSelect(). Model(&buckets). - Where("status = ?", model.BucketStatusActive). + Where("status IN (?, ?)", model.BucketStatusProvisioning, model.BucketStatusReady). Scan(ctx) if err != nil { return nil, fmt.Errorf("listing active buckets: %w", err) @@ -90,14 +97,7 @@ func (r *BunBucketRepo) ListActive(ctx context.Context) ([]model.Bucket, error) } func (r *BunBucketRepo) SoftDelete(ctx context.Context, id int64) error { - _, err := r.db.NewDelete(). - Model((*model.Bucket)(nil)). - Where("id = ?", id). - Exec(ctx) - if err != nil { - return fmt.Errorf("deleting bucket: %w", err) - } - return nil + return deleteBucketRow(ctx, r.db, id, "deleting bucket") } func (r *BunBucketRepo) UpdateStatus(ctx context.Context, id int64, from, to model.BucketStatus) error { @@ -117,6 +117,26 @@ func (r *BunBucketRepo) UpdateStatus(ctx context.Context, id int64, from, to mod return nil } +func (r *BunBucketRepo) PromoteReadyIfProvisioned(ctx context.Context, id int64, requiredDataSets int) (bool, error) { + if id < 1 || !model.ValidStorageCopies(requiredDataSets) { + return false, fmt.Errorf("promoting bucket storage readiness: %w", ErrInvalidInput) + } + result, err := r.db.NewUpdate(). + Model((*model.Bucket)(nil)). + Set("status = ?", model.BucketStatusReady). + Set("updated_at = ?", time.Now()). + Where("id = ? AND status = ?", id, model.BucketStatusProvisioning). + Where(`? = (SELECT COUNT(*) FROM storage_data_sets + WHERE bucket_id = ? AND is_current = TRUE AND status = ? AND copy_index >= 0 AND copy_index < ?)`, + requiredDataSets, id, model.StorageDataSetStatusReady, requiredDataSets). + Exec(ctx) + if err != nil { + return false, fmt.Errorf("promoting bucket storage readiness: %w", err) + } + rows, _ := result.RowsAffected() + return rows == 1, nil +} + func (r *BunBucketRepo) SetACL(ctx context.Context, name string, acl []byte) error { res, err := r.db.NewUpdate(). Model((*model.Bucket)(nil)). @@ -162,20 +182,81 @@ func (r *BunBucketRepo) UpdateCopyPolicy(ctx context.Context, input UpdateBucket Model((*model.Bucket)(nil)). Set("updated_at = ?", time.Now().UTC()). Where("id = ?", bucket.ID) - if input.SetDefaultCopies { - bucket.DefaultCopies = input.DefaultCopies - update = update.Set("default_copies = ?", input.DefaultCopies) + if input.SetDefaultCopies && input.DefaultCopies != nil { + // Lowering the target would leave the replicas above it running and + // billed with nothing to retire them, so it is refused until slot + // retirement exists. An explicit null resolves to the configured + // default before it reaches here, so a reset that would lower is + // refused by this same rule. + if *input.DefaultCopies < bucket.DefaultCopies { + return nil, fmt.Errorf("replica target %d -> %d: %w", + bucket.DefaultCopies, *input.DefaultCopies, ErrReplicaTargetLowered) + } + bucket.DefaultCopies = *input.DefaultCopies + update = update.Set("default_copies = ?", *input.DefaultCopies) } if input.SetMinimumDurableCopies { - bucket.MinimumDurableCopies = input.MinimumDurableCopies - update = update.Set("minimum_durable_copies = ?", input.MinimumDurableCopies) + // A nil minimum means "every replica", which is now stored as the + // replica target itself instead of a null standing for it. + minimum := bucket.DefaultCopies + if input.MinimumDurableCopies != nil { + minimum = *input.MinimumDurableCopies + } + bucket.MinimumDurableCopies = minimum + update = update.Set("minimum_durable_copies = ?", minimum) + } + if bucket.MinimumDurableCopies > bucket.DefaultCopies { + return nil, fmt.Errorf("minimum durable copies exceeds explicit default copies: %w", ErrInvalidInput) } if _, err := update.Exec(ctx); err != nil { return nil, fmt.Errorf("updating bucket copy policy: %w", err) } + if err := openBucketReplicaSlots(ctx, r.db, bucket.ID, bucket.DefaultCopies); err != nil { + return nil, err + } return bucket, nil } +// ActiveReplicaSlots lists the copy indexes still open for new writes. +func (r *BunBucketRepo) ActiveReplicaSlots(ctx context.Context, bucketID int64) ([]int, error) { + var indexes []int + if err := r.db.NewSelect(). + Model((*model.BucketReplicaSlot)(nil)). + Column("copy_index"). + Where("bucket_id = ? AND status = ?", bucketID, model.BucketReplicaSlotStatusActive). + Order("copy_index ASC"). + Scan(ctx, &indexes); err != nil { + return nil, fmt.Errorf("listing active bucket replica slots: %w", err) + } + return indexes, nil +} + +// openBucketReplicaSlots makes slots 0..copies-1 active. Nothing closes a slot +// yet: the replica target can only grow, because lowering it would strand a paid +// storage service with no way to retire it. Closing slots belongs with that +// retirement path. +func openBucketReplicaSlots(ctx context.Context, db bun.IDB, bucketID int64, copies int) error { + slots := make([]model.BucketReplicaSlot, 0, copies) + now := time.Now().UTC() + for copyIndex := range copies { + slots = append(slots, model.BucketReplicaSlot{ + BucketID: bucketID, CopyIndex: copyIndex, + Status: model.BucketReplicaSlotStatusActive, CreatedAt: now, UpdatedAt: now, + }) + } + if len(slots) > 0 { + if _, err := db.NewInsert(). + Model(&slots). + On("CONFLICT (bucket_id, copy_index) DO UPDATE"). + Set("status = ?", model.BucketReplicaSlotStatusActive). + Set("updated_at = ?", now). + Exec(ctx); err != nil { + return fmt.Errorf("opening bucket replica slots: %w", err) + } + } + return nil +} + func (r *BunBucketRepo) SetDefaultCopies(ctx context.Context, name string, copies *int) error { bucket, err := r.UpdateCopyPolicy(ctx, UpdateBucketCopyPolicyInput{ Name: name, @@ -269,14 +350,28 @@ func (r *BunBucketRepo) AggregateCountsByOwner(ctx context.Context) (map[string] } func (r *BunBucketRepo) HardDelete(ctx context.Context, id int64) error { - _, err := r.db.NewDelete(). - Model((*model.Bucket)(nil)). - Where("id = ?", id). - Exec(ctx) - if err != nil { - return fmt.Errorf("hard-deleting bucket: %w", err) - } - return nil + return deleteBucketRow(ctx, r.db, id, "hard-deleting bucket") +} + +// deleteBucketRow removes a bucket together with the replica slots it owns. +// The slots are deleted explicitly rather than by cascade, so a slot a data set +// still references blocks the delete instead of silently disappearing. +func deleteBucketRow(ctx context.Context, db bun.IDB, id int64, action string) error { + return runMaybeTx(ctx, db, func(db bun.IDB) error { + if _, err := db.NewDelete(). + Model((*model.BucketReplicaSlot)(nil)). + Where("bucket_id = ?", id). + Exec(ctx); err != nil { + return fmt.Errorf("%s replica slots: %w", action, err) + } + if _, err := db.NewDelete(). + Model((*model.Bucket)(nil)). + Where("id = ?", id). + Exec(ctx); err != nil { + return fmt.Errorf("%s: %w", action, err) + } + return nil + }) } func (r *BunBucketRepo) List(ctx context.Context) ([]model.Bucket, error) { diff --git a/internal/db/repository/bucket_repo_test.go b/internal/db/repository/bucket_repo_test.go index 516d67b..4a37cfc 100644 --- a/internal/db/repository/bucket_repo_test.go +++ b/internal/db/repository/bucket_repo_test.go @@ -3,10 +3,14 @@ package repository_test import ( "context" "encoding/json" + "errors" + "strings" "testing" + "time" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/testutil" "github.com/versity/versitygw/auth" ) @@ -16,7 +20,7 @@ func TestBucketRepo_CreateAndGetByName(t *testing.T) { ctx := context.Background() - bucket := &model.Bucket{Name: "test-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "test-bucket", Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Create: %v", err) } @@ -63,8 +67,8 @@ func TestBucketRepo_GetNamesByIDs(t *testing.T) { t.Fatalf("GetNamesByIDs empty = %v, want empty map", names) } - first := &model.Bucket{Name: "names-first", Status: model.BucketStatusActive} - second := &model.Bucket{Name: "names-second", Status: model.BucketStatusActive} + first := &model.Bucket{Name: "names-first", Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8} + second := &model.Bucket{Name: "names-second", Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8} if err := repos.Buckets.Create(ctx, first); err != nil { t.Fatalf("Create first: %v", err) } @@ -86,7 +90,7 @@ func TestBucketRepo_GetByID(t *testing.T) { repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "by-id", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "by-id", Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Create: %v", err) } @@ -105,7 +109,7 @@ func TestBucketRepo_UpdateCopyPolicy(t *testing.T) { repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "copies-policy", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "copies-policy", Status: model.BucketStatusActive, DefaultCopies: 2, MinimumDurableCopies: 2} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Create: %v", err) } @@ -122,16 +126,15 @@ func TestBucketRepo_UpdateCopyPolicy(t *testing.T) { if err != nil { t.Fatalf("UpdateCopyPolicy set: %v", err) } - if updated == nil || updated.DefaultCopies == nil || *updated.DefaultCopies != copies || - updated.MinimumDurableCopies == nil || *updated.MinimumDurableCopies != minimum { + if updated == nil || updated.DefaultCopies != copies || updated.MinimumDurableCopies != minimum { t.Fatalf("UpdateCopyPolicy result = %#v, want target/minimum %d/%d", updated, copies, minimum) } + assertActiveReplicaSlots(t, ctx, repos, bucket.ID, copies) got, err := repos.Buckets.GetByName(ctx, bucket.Name) if err != nil { t.Fatalf("GetByName after set: %v", err) } - if got == nil || got.DefaultCopies == nil || *got.DefaultCopies != copies || - got.MinimumDurableCopies == nil || *got.MinimumDurableCopies != minimum { + if got == nil || got.DefaultCopies != copies || got.MinimumDurableCopies != minimum { t.Fatalf("copy policy after set = %#v, want target/minimum %d/%d", got, copies, minimum) } @@ -142,15 +145,85 @@ func TestBucketRepo_UpdateCopyPolicy(t *testing.T) { if err != nil { t.Fatalf("UpdateCopyPolicy clear minimum: %v", err) } - if updated == nil || updated.DefaultCopies == nil || *updated.DefaultCopies != copies || updated.MinimumDurableCopies != nil { - t.Fatalf("UpdateCopyPolicy clear result = %#v, want target %d and strict minimum", updated, copies) + if updated == nil || updated.DefaultCopies != copies || updated.MinimumDurableCopies != copies { + t.Fatalf("UpdateCopyPolicy clear result = %#v, want target %d and a minimum matching it", updated, copies) } got, err = repos.Buckets.GetByName(ctx, bucket.Name) if err != nil { t.Fatalf("GetByName after clear: %v", err) } - if got == nil || got.DefaultCopies == nil || *got.DefaultCopies != copies || got.MinimumDurableCopies != nil { - t.Fatalf("copy policy after minimum clear = %#v, want target %d and strict minimum", got, copies) + if got == nil || got.DefaultCopies != copies || got.MinimumDurableCopies != copies { + t.Fatalf("copy policy after minimum clear = %#v, want target %d and a minimum matching it", got, copies) + } + + grown := 6 + if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, SetDefaultCopies: true, DefaultCopies: &grown, + }); err != nil { + t.Fatalf("UpdateCopyPolicy grow: %v", err) + } + assertActiveReplicaSlots(t, ctx, repos, bucket.ID, grown) + // Setting the same target again is not a lowering. + if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, SetDefaultCopies: true, DefaultCopies: &grown, + }); err != nil { + t.Fatalf("UpdateCopyPolicy repeat target: %v", err) + } + assertActiveReplicaSlots(t, ctx, repos, bucket.ID, grown) +} + +// Lowering the replica target would leave the replicas above it running and +// billed with nothing to retire them, so it is refused outright and leaves both +// the policy and the slots untouched. +func TestBucketRepo_UpdateCopyPolicyRefusesLoweringTheTarget(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + + bucket := &model.Bucket{Name: "copies-no-lowering", Status: model.BucketStatusActive, DefaultCopies: 4, MinimumDurableCopies: 2} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Create: %v", err) + } + + lowered := 3 + if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, SetDefaultCopies: true, DefaultCopies: &lowered, + }); !errors.Is(err, repository.ErrInvalidInput) { + t.Fatalf("lowering the replica target error = %v, want ErrInvalidInput", err) + } + got, err := repos.Buckets.GetByName(ctx, bucket.Name) + if err != nil || got == nil || got.DefaultCopies != 4 || got.MinimumDurableCopies != 2 { + t.Fatalf("policy after refused lowering = %#v err=%v, want unchanged 4/2", got, err) + } + assertActiveReplicaSlots(t, ctx, repos, bucket.ID, 4) + + // Lowering only the minimum stays legal: it releases cache sooner, it does + // not strand a paid replica. + loweredMinimum := 1 + if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, SetMinimumDurableCopies: true, MinimumDurableCopies: &loweredMinimum, + }); err != nil { + t.Fatalf("lowering the minimum: %v", err) + } + got, err = repos.Buckets.GetByName(ctx, bucket.Name) + if err != nil || got == nil || got.DefaultCopies != 4 || got.MinimumDurableCopies != 1 { + t.Fatalf("policy after minimum lowering = %#v err=%v, want 4/1", got, err) + } +} + +func assertActiveReplicaSlots(t *testing.T, ctx context.Context, repos *repository.Repositories, bucketID int64, want int) { + t.Helper() + slots, err := repos.Buckets.ActiveReplicaSlots(ctx, bucketID) + if err != nil { + t.Fatalf("ActiveReplicaSlots: %v", err) + } + if len(slots) != want { + t.Fatalf("active replica slots = %v, want %d", slots, want) + } + for i, copyIndex := range slots { + if copyIndex != i { + t.Fatalf("active replica slots = %v, want contiguous 0..%d", slots, want-1) + } } } @@ -159,7 +232,7 @@ func TestBucketRepo_SetDefaultCopies(t *testing.T) { repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "set-default-copies", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "set-default-copies", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Create: %v", err) } @@ -172,7 +245,7 @@ func TestBucketRepo_SetDefaultCopies(t *testing.T) { if err != nil { t.Fatalf("GetByName: %v", err) } - if got == nil || got.DefaultCopies == nil || *got.DefaultCopies != copies { + if got == nil || got.DefaultCopies != copies { t.Fatalf("DefaultCopies = %#v, want %d", got, copies) } @@ -202,7 +275,7 @@ func TestBucketRepo_SetACL(t *testing.T) { repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "acl-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "acl-bucket", Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Create: %v", err) } @@ -233,7 +306,7 @@ func TestBucketRepo_SetOwnerAndACL(t *testing.T) { repos := repository.NewRepositories(db) ctx := context.Background() - owner := strptr("user2") + owner := new("user2") if err := repos.S3Accounts.Create(ctx, &model.S3Account{ AccessKey: *owner, SecretKey: "secret-" + *owner, @@ -242,7 +315,7 @@ func TestBucketRepo_SetOwnerAndACL(t *testing.T) { t.Fatalf("S3Accounts.Create: %v", err) } - bucket := &model.Bucket{Name: "owner-acl-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "owner-acl-bucket", Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Create: %v", err) } @@ -277,7 +350,7 @@ func TestBucketRepo_ListActive(t *testing.T) { ctx := context.Background() for _, name := range []string{"a", "b", "c"} { - b := &model.Bucket{Name: name, Status: model.BucketStatusActive} + b := &model.Bucket{Name: name, Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8} if err := repos.Buckets.Create(ctx, b); err != nil { t.Fatalf("Create(%s): %v", name, err) } @@ -296,7 +369,7 @@ func TestBucketRepo_UpdateStatus(t *testing.T) { repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "cas-status", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "cas-status", Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Create: %v", err) } @@ -316,7 +389,7 @@ func TestBucketRepo_HardDelete(t *testing.T) { repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "hard-del", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "hard-del", Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Create: %v", err) } @@ -339,7 +412,7 @@ func TestBucketRepo_SoftDelete(t *testing.T) { repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "to-delete", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "to-delete", Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Create: %v", err) } @@ -382,7 +455,7 @@ func TestBucketRepo_List(t *testing.T) { // Create multiple active buckets. for _, name := range []string{"alpha", "beta", "gamma", "delta"} { - b := &model.Bucket{Name: name, Status: model.BucketStatusActive} + b := &model.Bucket{Name: name, Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8} if err := repos.Buckets.Create(ctx, b); err != nil { t.Fatalf("Create(%s): %v", name, err) } @@ -421,7 +494,7 @@ func TestBucketRepo_CountByStatus(t *testing.T) { // Seed buckets: 3 active. for _, name := range []string{"a1", "a2", "a3"} { - b := &model.Bucket{Name: name, Status: model.BucketStatusActive} + b := &model.Bucket{Name: name, Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8} if err := repos.Buckets.Create(ctx, b); err != nil { t.Fatalf("Create(%s): %v", name, err) } @@ -460,12 +533,12 @@ func TestBucketRepo_AggregateCountsByOwner(t *testing.T) { name string owner *string }{ - {name: "owner-a-one", owner: strptr("owner-a")}, - {name: "owner-a-two", owner: strptr("owner-a")}, - {name: "owner-b-one", owner: strptr("owner-b")}, + {name: "owner-a-one", owner: new("owner-a")}, + {name: "owner-a-two", owner: new("owner-a")}, + {name: "owner-b-one", owner: new("owner-b")}, {name: "unassigned", owner: nil}, } { - b := &model.Bucket{Name: seed.name, Status: model.BucketStatusActive, OwnerAccessKey: seed.owner} + b := &model.Bucket{Name: seed.name, Status: model.BucketStatusActive, OwnerAccessKey: seed.owner, DefaultCopies: 8, MinimumDurableCopies: 8} if err := repos.Buckets.Create(ctx, b); err != nil { t.Fatalf("Create(%s): %v", seed.name, err) } @@ -499,8 +572,8 @@ func TestBucketRepo_ListACLsReturnsOnlyOwnershipFields(t *testing.T) { t.Fatalf("Marshal ACL: %v", err) } for _, bucket := range []*model.Bucket{ - {Name: "owned", Status: model.BucketStatusActive, ACL: acl}, - {Name: "unassigned", Status: model.BucketStatusActive}, + {Name: "owned", Status: model.BucketStatusActive, ACL: acl, DefaultCopies: 8, MinimumDurableCopies: 8}, + {Name: "unassigned", Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8}, } { if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Create(%s): %v", bucket.Name, err) @@ -527,23 +600,26 @@ func TestBucketRepo_CountStorageDataSets(t *testing.T) { repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "datasets-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "datasets-bucket", Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Create bucket: %v", err) } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ + origin := newObjectVersion(bucket.ID, "dataset-count", "dataset-count", 1) + if _, err := createVersion(t, repos, origin); err != nil { + t.Fatalf("Create origin version: %v", err) + } + upload, err := repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ BucketID: bucket.ID, - SourceVersionID: "dataset-count", ContentSize: 1, - Checksum: "sum", + Checksum: testutil.StorageChecksum("sum"), RequestedCopies: 3, }) if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) + t.Fatalf("EnsureContent: %v", err) } - seedCommittedUploadCopies(t, repos, bucket.ID, upload.ID, "bafk2bzacedatasetcount", []storageUploadCopySeed{ - {ProviderID: onChainIDPtr(t, "101"), DataSetID: onChainIDPtr(t, "1001"), PieceID: onChainIDPtr(t, "2001"), TransferMethod: model.StorageCopyTransferMethodIngress, RetrievalURL: strptr("https://provider.example/1")}, - {ProviderID: onChainIDPtr(t, "202"), DataSetID: onChainIDPtr(t, "2002"), PieceID: onChainIDPtr(t, "3001"), TransferMethod: model.StorageCopyTransferMethodPeerPull, RetrievalURL: strptr("https://provider.example/2")}, + seedCommittedUploadCopies(t, db, repos, bucket.ID, upload.ID, "bafk2bzacedatasetcount", []storageUploadCopySeed{ + {ProviderID: onChainIDPtr(t, "101"), DataSetID: onChainIDPtr(t, "1001"), PieceID: onChainIDPtr(t, "2001"), TransferMethod: model.StorageCopyTransferMethodIngress, RetrievalURL: new("https://provider.example/1")}, + {ProviderID: onChainIDPtr(t, "202"), DataSetID: onChainIDPtr(t, "2002"), PieceID: onChainIDPtr(t, "3001"), TransferMethod: model.StorageCopyTransferMethodPeerPull, RetrievalURL: new("https://provider.example/2")}, }) count, err := repos.Buckets.CountStorageDataSets(ctx) @@ -555,6 +631,80 @@ func TestBucketRepo_CountStorageDataSets(t *testing.T) { } } -func strptr(s string) *string { - return &s +func TestBucketRepo_PromoteReadyRequiresEveryConfiguredSlot(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := t.Context() + bucket := &model.Bucket{Name: "provisioning-bucket", Status: model.BucketStatusProvisioning, DefaultCopies: 8, MinimumDurableCopies: 8} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Create bucket: %v", err) + } + + markSlotReady := func(copyIndex int, provider, dataSet string) { + t.Helper() + providerID := onChainIDPtr(t, provider) + dataSetID := onChainIDPtr(t, dataSet) + clientDataSetID := onChainIDPtr(t, dataSet+"1") + binding, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: *providerID, CopyIndex: copyIndex, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding(%d): %v", copyIndex, err) + } + if err := repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: binding.ID, DataSetID: *dataSetID, ClientDataSetID: clientDataSetID, + }); err != nil { + t.Fatalf("MarkDataSetReady(%d): %v", copyIndex, err) + } + } + + markSlotReady(0, "101", "1001") + promoted, err := repos.Buckets.PromoteReadyIfProvisioned(ctx, bucket.ID, 2) + if err != nil || promoted { + t.Fatalf("promote with one slot = %v, err=%v; want false", promoted, err) + } + markSlotReady(2, "303", "3003") + promoted, err = repos.Buckets.PromoteReadyIfProvisioned(ctx, bucket.ID, 2) + if err != nil || promoted { + t.Fatalf("promote with wrong second slot = %v, err=%v; want false", promoted, err) + } + markSlotReady(1, "202", "2002") + promoted, err = repos.Buckets.PromoteReadyIfProvisioned(ctx, bucket.ID, 2) + if err != nil || !promoted { + t.Fatalf("promote with configured slots = %v, err=%v; want true", promoted, err) + } + stored, err := repos.Buckets.GetByID(ctx, bucket.ID) + if err != nil || stored == nil || stored.Status != model.BucketStatusReady { + t.Fatalf("ready bucket = %#v, err=%v", stored, err) + } +} + +// A row inserted without explicit timestamps must still carry them, and carry +// them in bun's encoding rather than the database's own clock format. +func TestModelInsertsStampAuditTimestamps(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + + before := time.Now().UTC().Add(-time.Second) + bucket := &model.Bucket{Name: "stamped-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Create: %v", err) + } + stored, err := repos.Buckets.GetByName(ctx, bucket.Name) + if err != nil || stored == nil { + t.Fatalf("GetByName: bucket=%#v err=%v", stored, err) + } + if stored.CreatedAt.Before(before) || stored.UpdatedAt.Before(before) { + t.Fatalf("stamped timestamps = created:%s updated:%s, want at or after %s", stored.CreatedAt, stored.UpdatedAt, before) + } + // Second granularity is what a database default would produce; the hook + // writes the same instant bun renders everywhere else. + var raw string + if err := db.NewRaw(`SELECT created_at FROM buckets WHERE id = ?`, bucket.ID).Scan(ctx, &raw); err != nil { + t.Fatalf("read raw created_at: %v", err) + } + if !strings.Contains(raw, ".") || strings.Contains(raw, " ") { + t.Fatalf("stored created_at = %q, want bun's fractional encoding rather than the database clock's", raw) + } } diff --git a/internal/db/repository/cache_eviction_repo.go b/internal/db/repository/cache_eviction_repo.go index 71489e8..1dd71d5 100644 --- a/internal/db/repository/cache_eviction_repo.go +++ b/internal/db/repository/cache_eviction_repo.go @@ -3,734 +3,436 @@ package repository import ( "context" "database/sql" + "encoding/json" "errors" "fmt" + "strconv" "time" "github.com/strahe/synaps3/internal/cacheeviction" "github.com/strahe/synaps3/internal/model" "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" ) -func (r *BunCacheEvictionRepo) AuthorizeDeletion( - ctx context.Context, - task *model.Task, - expectedAccess *time.Time, -) (*cacheeviction.AuthorizedDeletion, error) { - if task == nil || task.RefVersionID == "" { - return nil, fmt.Errorf("cache eviction task target is required: %w", ErrInvalidInput) +// Cache residency is keyed by content, not by object version: identical bytes +// written under several keys share one local file, so one eviction decision +// covers every version that references them. +type CacheEvictionRepository interface { + // GetCacheEntry returns the residency record for one content payload. + GetCacheEntry(ctx context.Context, contentID int64) (*model.ObjectCache, error) + NextEvictionGeneration(ctx context.Context, contentID int64) (int64, error) + PrepareEviction(ctx context.Context, contentID int64) (CacheEvictionReservation, error) + BindEvictionTask(ctx context.Context, contentID, generation, taskID int64) error + ListLRUCandidates(ctx context.Context, limit int) ([]cacheeviction.Candidate, error) + ActiveEvictionBytes(ctx context.Context) (int64, error) + AuthorizeDeletion(ctx context.Context, contentID, generation, taskID int64, expectedAccess *time.Time) (*cacheeviction.AuthorizedDeletion, error) + RecordDeletion(ctx context.Context, contentID, generation, taskID int64) error + DeletionRecorded(ctx context.Context, contentID, generation int64) (bool, error) + ReleaseEviction(ctx context.Context, contentID, generation, taskID int64) error + + NextDurabilityGeneration(ctx context.Context, bucketID int64) (int64, error) + BindDurabilityTask(ctx context.Context, bucketID, generation, taskID int64) error + // NextBucketDurabilityCandidate returns the next cached content in the + // bucket that now satisfies the bucket's minimum durability. It no longer + // promotes any lifecycle state: pipeline position is derived from the copy + // rows, so there is nothing to advance, only cache to reclaim. + NextBucketDurabilityCandidate(ctx context.Context, bucketID, generation, taskID int64) (*model.StorageContent, error) + CompleteBucketDurability(ctx context.Context, bucketID, generation, taskID int64) error +} + +type CacheEvictionReservation struct { + Generation int64 + ActiveTaskID *int64 +} + +type BunCacheEvictionRepo struct { + db bun.IDB +} + +var _ CacheEvictionRepository = (*BunCacheEvictionRepo)(nil) + +func (r *BunCacheEvictionRepo) GetCacheEntry(ctx context.Context, contentID int64) (*model.ObjectCache, error) { + entry := new(model.ObjectCache) + err := r.db.NewSelect().Model(entry).Where("content_id = ?", contentID).Scan(ctx) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil } - preflight, err := r.objectVersionByID(ctx, r.db, task.RefVersionID) if err != nil { - return nil, err + return nil, fmt.Errorf("selecting content cache entry: %w", err) } - if preflight.StorageUploadID == nil { - return nil, cacheeviction.ErrNoLongerEligible - } - - var authorized *cacheeviction.AuthorizedDeletion - err = r.runMaybeTx(ctx, func(db bun.IDB) error { - bucket, upload, version, lockedTask, err := lockCacheEvictionContext( - ctx, - db, - task, - preflight.BucketID, - *preflight.StorageUploadID, - preflight.VersionID, - ) - if err != nil { - return err - } - if lockedTask.RefVersionID != version.VersionID { - return fmt.Errorf("cache eviction task target changed: %w", ErrConflict) - } - alreadyAuthorized, err := cacheeviction.DeleteAuthorized(lockedTask) - if err != nil { - return err - } - if !alreadyAuthorized { - if lockedTask.RefType != "object" { - return fmt.Errorf("cache eviction task is not an object deletion: %w", ErrConflict) - } - if err := requireMinimumDurability(ctx, db, bucket, upload); err != nil { - return err - } - if !cacheDeletionStateEligible(version) { - return cacheeviction.ErrNoLongerEligible - } - if expectedAccess != nil && !cacheeviction.NormalizeAccessTime(cacheAccessTime(version)).Equal(cacheeviction.NormalizeAccessTime(*expectedAccess)) { - return cacheeviction.ErrAccessChanged - } - payload := cacheeviction.WithDeleteAuthorization(lockedTask.Payload) - if err := updateRunningEvictionTask(ctx, db, task, lockedTask.RefVersionID, payload); err != nil { - return err - } - task.Payload = payload - } - authorized = &cacheeviction.AuthorizedDeletion{Version: *version, BucketName: bucket.Name} - return nil - }) - return authorized, err + return entry, nil } -func (r *BunCacheEvictionRepo) NextBucketDurabilityCandidate( - ctx context.Context, - bucketID int64, -) (*model.ObjectVersion, error) { - return nextBucketDurabilityCandidate(ctx, r.db, bucketID) +func (r *BunCacheEvictionRepo) NextEvictionGeneration(ctx context.Context, contentID int64) (int64, error) { + var generation int64 + err := r.db.NewSelect(). + Model((*model.ObjectCache)(nil)). + ColumnExpr("cache_operation_generation + 1"). + Where("content_id = ?", contentID). + Scan(ctx, &generation) + if errors.Is(err, sql.ErrNoRows) { + return 0, ErrNotFound + } + if err != nil { + return 0, fmt.Errorf("reading cache operation generation: %w", err) + } + return generation, nil } -func (r *BunCacheEvictionRepo) PromoteBucketDurabilityCandidate( - ctx context.Context, - task *model.Task, - versionID string, - authorizeDelete bool, -) (*cacheeviction.AuthorizedDeletion, error) { - if task == nil || task.RefType != "bucket" || task.RefID <= 0 || versionID == "" { - return nil, fmt.Errorf("bucket durability task and candidate are required: %w", ErrInvalidInput) +// PrepareEviction locks one cache entry and either reuses its matching live +// eviction or reserves the next generation for the caller to enqueue and bind +// before its surrounding transaction commits. +func (r *BunCacheEvictionRepo) PrepareEviction(ctx context.Context, contentID int64) (CacheEvictionReservation, error) { + if contentID < 1 { + return CacheEvictionReservation{}, ErrInvalidInput } - preflight, err := r.objectVersionByID(ctx, r.db, versionID) + entry, err := lockCacheEntry(ctx, r.db, contentID) if err != nil { - return nil, err + return CacheEvictionReservation{}, err } - if preflight.BucketID != task.RefID || preflight.StorageUploadID == nil { - return nil, cacheeviction.ErrNoLongerEligible + if entry.CacheActiveTaskID == nil { + return CacheEvictionReservation{Generation: entry.CacheOperationGeneration + 1}, nil } - var deletion *cacheeviction.AuthorizedDeletion - err = r.runMaybeTx(ctx, func(db bun.IDB) error { - bucket, upload, version, lockedTask, err := lockCacheEvictionContext( - ctx, - db, - task, - preflight.BucketID, - *preflight.StorageUploadID, - versionID, - ) - if err != nil { - return err - } - if lockedTask.RefType != "bucket" || lockedTask.RefID != bucket.ID { - return fmt.Errorf("bucket durability task target changed: %w", ErrConflict) - } - if version.BucketID != bucket.ID || version.StorageUploadID == nil || *version.StorageUploadID != upload.ID || - version.State != model.ObjectStateReplicating || !version.InCache || version.IsDeleteMarker { - return cacheeviction.ErrNoLongerEligible - } - if err := requireMinimumDurability(ctx, db, bucket, upload); err != nil { - return err - } - now := time.Now() - res, err := db.NewUpdate(). - Model((*model.ObjectVersion)(nil)). - Set("state = ?", model.ObjectStateStored). - Set("updated_at = ?", now). - Where("version_id = ? AND state = ? AND in_cache = ?", version.VersionID, model.ObjectStateReplicating, true). - Exec(ctx) - if err != nil { - return fmt.Errorf("promoting bucket durability candidate: %w", err) - } - rows, _ := res.RowsAffected() - if rows != 1 { - return cacheeviction.ErrNoLongerEligible - } - version.State = model.ObjectStateStored - version.UpdatedAt = now - if authorizeDelete { - payload := cacheeviction.WithDeleteAuthorization(lockedTask.Payload) - if err := updateRunningEvictionTask(ctx, db, task, version.VersionID, payload); err != nil { - return err - } - task.RefVersionID = version.VersionID - task.Payload = payload - deletion = &cacheeviction.AuthorizedDeletion{Version: *version, BucketName: bucket.Name} - } - return nil - }) - return deletion, err + taskRow := new(model.Task) + err = withTaskPayload(r.db.NewSelect(). + Model(taskRow). + Where("task.id = ?", *entry.CacheActiveTaskID)). + Scan(ctx) + if errors.Is(err, sql.ErrNoRows) { + return CacheEvictionReservation{}, ErrConflict + } + if err != nil { + return CacheEvictionReservation{}, fmt.Errorf("loading active cache eviction task: %w", err) + } + if taskRow.Status == model.TaskStatusPending || taskRow.Status == model.TaskStatusRunning { + expectedSubjectKey := strconv.FormatInt(contentID, 10) + var taskInput cacheeviction.EvictInput + inputErr := json.Unmarshal(taskRow.Input, &taskInput) + if taskRow.Type != model.TaskTypeCacheEvict || + taskRow.InputVersion != 1 || inputErr != nil || + cacheeviction.ValidateEvictInput(&taskInput) != nil || + taskInput.ContentID != contentID || taskInput.Generation != entry.CacheOperationGeneration || + taskRow.IdempotencyKey != cacheeviction.EvictTaskKey(contentID, entry.CacheOperationGeneration) || + taskRow.SubjectType == nil || *taskRow.SubjectType != "storage_content" || + taskRow.SubjectKey == nil || *taskRow.SubjectKey != expectedSubjectKey { + return CacheEvictionReservation{}, ErrConflict + } + return CacheEvictionReservation{ + Generation: entry.CacheOperationGeneration, + ActiveTaskID: entry.CacheActiveTaskID, + }, nil + } + + result, err := r.db.NewUpdate(). + Model((*model.ObjectCache)(nil)). + Set("cache_active_task_id = NULL"). + Set("updated_at = ?", time.Now()). + Where("content_id = ?", contentID). + Where("cache_operation_generation = ?", entry.CacheOperationGeneration). + Where("cache_active_task_id = ?", *entry.CacheActiveTaskID). + Exec(ctx) + if err != nil { + return CacheEvictionReservation{}, fmt.Errorf("clearing terminal cache eviction owner: %w", err) + } + if rows, _ := result.RowsAffected(); rows != 1 { + return CacheEvictionReservation{}, ErrConflict + } + return CacheEvictionReservation{Generation: entry.CacheOperationGeneration + 1}, nil } -func (r *BunCacheEvictionRepo) CompleteBucketDurabilityReconciliation( - ctx context.Context, - task *model.Task, -) (bool, error) { - if task == nil || task.RefType != "bucket" || task.RefID <= 0 { - return false, fmt.Errorf("bucket durability task is required: %w", ErrInvalidInput) - } - completed := false - err := r.runMaybeTx(ctx, func(db bun.IDB) error { - bucket, err := lockBucketByID(ctx, db, task.RefID) - if err != nil { - return err - } - if bucket == nil { - return cacheeviction.ErrNoLongerEligible - } - candidate, err := nextBucketDurabilityCandidate(ctx, db, bucket.ID) - if err != nil { - return err - } - if candidate != nil { - return nil - } - if err := (&BunTaskRepo{db: db}).LockRunningClaim(ctx, task); err != nil { - return err - } - if err := (&BunTaskRepo{db: db}).Complete(ctx, task); err != nil { - return err - } - completed = true +func (r *BunCacheEvictionRepo) BindEvictionTask(ctx context.Context, contentID, generation, taskID int64) error { + if contentID < 1 || generation < 1 || taskID < 1 { + return ErrInvalidInput + } + result, err := r.db.NewUpdate(). + Model((*model.ObjectCache)(nil)). + Set("cache_operation_generation = ?", generation). + Set("cache_active_task_id = ?", taskID). + Set("updated_at = ?", time.Now()). + Where("content_id = ?", contentID). + Where("cache_operation_generation = ?", generation-1). + Where("cache_active_task_id IS NULL"). + Exec(ctx) + if err != nil { + return fmt.Errorf("binding cache eviction task: %w", err) + } + rows, _ := result.RowsAffected() + if rows == 1 { return nil - }) - return completed, err + } + var existing struct { + Generation int64 `bun:"cache_operation_generation"` + TaskID *int64 `bun:"cache_active_task_id"` + } + err = r.db.NewSelect(). + Model((*model.ObjectCache)(nil)). + Column("cache_operation_generation", "cache_active_task_id"). + Where("content_id = ?", contentID). + Scan(ctx, &existing) + if err != nil { + return fmt.Errorf("checking cache eviction binding: %w", err) + } + if existing.Generation == generation && existing.TaskID != nil && *existing.TaskID == taskID { + return nil + } + return ErrConflict } -func (r *BunCacheEvictionRepo) RecordAuthorizedDeletion(ctx context.Context, task *model.Task) error { - if task == nil || task.RefVersionID == "" { - return fmt.Errorf("authorized cache eviction task is required: %w", ErrInvalidInput) +func (r *BunCacheEvictionRepo) ListLRUCandidates(ctx context.Context, limit int) ([]cacheeviction.Candidate, error) { + var candidates []cacheeviction.Candidate + query := r.db.NewSelect(). + TableExpr("object_cache AS object_cache"). + ColumnExpr("object_cache.content_id"). + ColumnExpr("storage_content.bucket_id"). + ColumnExpr("storage_content.content_size"). + ColumnExpr("object_cache.cache_accessed_at"). + Join("JOIN storage_contents AS storage_content ON storage_content.id = object_cache.content_id"). + Join("JOIN buckets AS durability_bucket ON durability_bucket.id = storage_content.bucket_id"). + Where("object_cache.cache_active_task_id IS NULL"). + Where("object_cache.in_cache = ?", true). + Where("storage_content.content_size > 0"). + Where("object_cache.cache_accessed_at IS NOT NULL"). + Where(minimumDurabilityMetSQL("storage_content", "durability_bucket")). + OrderExpr("object_cache.cache_accessed_at, object_cache.content_id") + if limit > 0 { + query = query.Limit(limit) } - preflight, err := r.objectVersionByID(ctx, r.db, task.RefVersionID) + if err := query.Scan(ctx, &candidates); err != nil { + return nil, fmt.Errorf("listing LRU cache candidates: %w", err) + } + return candidates, nil +} + +func (r *BunCacheEvictionRepo) ActiveEvictionBytes(ctx context.Context) (int64, error) { + var total int64 + err := r.db.NewSelect(). + TableExpr("object_cache AS object_cache"). + ColumnExpr("COALESCE(SUM(storage_content.content_size), 0)"). + Join("JOIN storage_contents AS storage_content ON storage_content.id = object_cache.content_id"). + Where("object_cache.in_cache = ?", true). + Where("object_cache.cache_active_task_id IS NOT NULL"). + Scan(ctx, &total) if err != nil { - if errors.Is(err, ErrNotFound) && task.RefType == "bucket" { - return r.clearMissingBucketDurabilityAuthorization(ctx, task) - } - return err + return 0, fmt.Errorf("summing active cache eviction bytes: %w", err) } - if preflight.StorageUploadID == nil { - return cacheeviction.ErrNoLongerEligible - } - return r.runMaybeTx(ctx, func(db bun.IDB) error { - _, _, version, lockedTask, err := lockCacheEvictionContext( - ctx, - db, - task, - preflight.BucketID, - *preflight.StorageUploadID, - preflight.VersionID, - ) - if err != nil { - return err - } - authorized, err := cacheeviction.DeleteAuthorized(lockedTask) - if err != nil { - return err - } - if !authorized || lockedTask.RefVersionID != version.VersionID { - return fmt.Errorf("cache deletion was not authorized: %w", ErrConflict) - } - switch version.State { - case model.ObjectStateStored: - _, err = db.NewUpdate(). - Model((*model.ObjectVersion)(nil)). - Set("state = ?", model.ObjectStateCacheEvicted). - Set("in_cache = ?", false). - Set("updated_at = ?", time.Now()). - Where("version_id = ? AND state = ?", version.VersionID, model.ObjectStateStored). - Exec(ctx) - case model.ObjectStateCacheEvicted: - _, err = db.NewUpdate(). - Model((*model.ObjectVersion)(nil)). - Set("in_cache = ?", false). - Where("version_id = ?", version.VersionID). - Exec(ctx) - default: - return cacheeviction.ErrNoLongerEligible - } - if err != nil { - return fmt.Errorf("recording authorized cache deletion: %w", err) - } - if lockedTask.RefType == "bucket" { - if err := updateRunningEvictionTask(ctx, db, task, "", nil); err != nil { - return err - } - task.RefVersionID = "" - task.Payload = nil - } - return nil - }) + return total, nil } -func (r *BunCacheEvictionRepo) clearMissingBucketDurabilityAuthorization( +func (r *BunCacheEvictionRepo) AuthorizeDeletion( ctx context.Context, - task *model.Task, -) error { - if task == nil || task.RefType != "bucket" || task.RefID <= 0 || task.RefVersionID == "" { - return fmt.Errorf("bucket durability deletion authorization is required: %w", ErrInvalidInput) + contentID, generation, taskID int64, + expectedAccess *time.Time, +) (*cacheeviction.AuthorizedDeletion, error) { + if contentID < 1 || generation < 1 || taskID < 1 { + return nil, ErrInvalidInput } - return r.runMaybeTx(ctx, func(db bun.IDB) error { - bucket, err := lockBucketByID(ctx, db, task.RefID) + var authorized *cacheeviction.AuthorizedDeletion + err := runMaybeTx(ctx, r.db, func(db bun.IDB) error { + entry, err := lockCacheEntry(ctx, db, contentID) if err != nil { return err } - if bucket == nil { + if entry.CacheOperationGeneration != generation || entry.CacheActiveTaskID == nil || *entry.CacheActiveTaskID != taskID { + return ErrConflict + } + if !entry.InCache { return cacheeviction.ErrNoLongerEligible } - tasks := &BunTaskRepo{db: db} - if err := tasks.LockRunningClaim(ctx, task); err != nil { - return err + if expectedAccess != nil && !cacheeviction.NormalizeAccessTime(cacheAccessTime(entry)).Equal(cacheeviction.NormalizeAccessTime(*expectedAccess)) { + return cacheeviction.ErrAccessChanged } - lockedTask, err := tasks.GetByID(ctx, task.ID) + contents, err := lockStorageContentsByID(ctx, db, []int64{contentID}) if err != nil { return err } - if lockedTask == nil || lockedTask.RefType != "bucket" || lockedTask.RefID != bucket.ID || - lockedTask.RefVersionID != task.RefVersionID { - return fmt.Errorf("bucket durability deletion target changed: %w", ErrConflict) + content := contents[contentID] + if content == nil { + return cacheeviction.ErrNoLongerEligible } - authorized, err := cacheeviction.DeleteAuthorized(lockedTask) + bucket, err := lockBucketByID(ctx, db, content.BucketID) if err != nil { return err } - if !authorized { - return fmt.Errorf("bucket durability deletion was not authorized: %w", ErrConflict) + if bucket == nil { + return cacheeviction.ErrNoLongerEligible } - if err := updateRunningEvictionTask(ctx, db, task, "", nil); err != nil { + if err := requireMinimumDurability(ctx, db, bucket, content); err != nil { return err } - task.RefVersionID = "" - task.Payload = nil + authorized = &cacheeviction.AuthorizedDeletion{Content: *content, BucketName: bucket.Name} return nil }) + return authorized, err } -// CacheEvictionRepository owns persistence operations used only by cache -// eviction planning and policy reconciliation. -type CacheEvictionRepository interface { - EnsureAfterUploadTask(ctx context.Context, objectID int64, versionID string, maxRetries int) (bool, error) - EnsureBucketDurabilityReconciliation(ctx context.Context, bucketID int64, maxRetries int) (bool, error) - ListLRUCandidates(ctx context.Context, terminalSince time.Time, limit int) ([]cacheeviction.Candidate, error) - PlanLRU(ctx context.Context, candidate cacheeviction.Candidate, maxRetries int, terminalBefore time.Time) (bool, error) - ActiveLRUBytes(ctx context.Context) (int64, error) - CancelActiveTasksExcept(ctx context.Context, keepStage string, message string) (int, error) - AuthorizeDeletion(ctx context.Context, task *model.Task, expectedAccess *time.Time) (*cacheeviction.AuthorizedDeletion, error) - NextBucketDurabilityCandidate(ctx context.Context, bucketID int64) (*model.ObjectVersion, error) - PromoteBucketDurabilityCandidate(ctx context.Context, task *model.Task, versionID string, authorizeDelete bool) (*cacheeviction.AuthorizedDeletion, error) - CompleteBucketDurabilityReconciliation(ctx context.Context, task *model.Task) (bool, error) - RecordAuthorizedDeletion(ctx context.Context, task *model.Task) error -} - -// BunCacheEvictionRepo implements cache eviction planning and reconciliation -// persistence. -type BunCacheEvictionRepo struct { - db bun.IDB -} - -var _ CacheEvictionRepository = (*BunCacheEvictionRepo)(nil) - -func (r *BunCacheEvictionRepo) EnsureAfterUploadTask( - ctx context.Context, - objectID int64, - versionID string, - maxRetries int, -) (bool, error) { - task := cacheeviction.NewAfterUploadTask(objectID, versionID, maxRetries, time.Now()) - return r.createOrReactivate(ctx, task, taskReactivationRule{ - immediateStatuses: []model.TaskStatus{model.TaskStatusCancelled}, - errorAction: "reactivating after-upload eviction task", - }) -} - -func (r *BunCacheEvictionRepo) EnsureBucketDurabilityReconciliation( - ctx context.Context, - bucketID int64, - maxRetries int, -) (bool, error) { - task := cacheeviction.NewBucketDurabilityTask(bucketID, maxRetries, time.Now()) - requestedMaxRetries := task.MaxRetries - activated := false - err := r.runMaybeTx(ctx, func(db bun.IDB) error { - existing, err := loadAndLockTaskByIdempotencyKey(ctx, db, task.IdempotencyKey) - if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - return fmt.Errorf("loading bucket durability task: %w", err) - } - res, err := db.NewInsert().Model(task).On("CONFLICT (idempotency_key) DO NOTHING").Exec(ctx) - if err != nil { - return fmt.Errorf("creating bucket durability task: %w", err) - } - rows, _ := res.RowsAffected() - activated = rows == 1 - if activated && requestedMaxRetries == 0 { - // Bun otherwise substitutes the SQL default for this zero-valued field. - if _, err := db.NewUpdate(). - Model((*model.Task)(nil)). - Set("max_retries = ?", 0). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - return fmt.Errorf("preserving bucket durability task zero retries: %w", err) - } - task.MaxRetries = requestedMaxRetries - } - return nil - } - switch existing.Status { - case model.TaskStatusCompleted, - model.TaskStatusFailed, - model.TaskStatusExhausted, - model.TaskStatusCancelled: - default: - return nil - } - - preserveAuthorization, err := cacheeviction.DeleteAuthorized(existing) - if err != nil { - return fmt.Errorf("reading bucket durability task authorization: %w", err) - } - refVersionID := task.RefVersionID - payload := task.Payload - if preserveAuthorization && existing.Status != model.TaskStatusCompleted { - refVersionID = existing.RefVersionID - payload = existing.Payload - } - now := time.Now() - res, err := db.NewUpdate(). - Model((*model.Task)(nil)). - Set("stage = ?", task.Stage). - Set("ref_type = ?", task.RefType). - Set("ref_id = ?", task.RefID). - Set("ref_version_id = ?", refVersionID). - Set("payload = ?", payload). - Set("status = ?", model.TaskStatusQueued). - Set("retry_count = 0"). - Set("max_retries = ?", requestedMaxRetries). - Set("scheduled_at = ?", now). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("started_at = NULL"). - Set("completed_at = NULL"). - Set("last_error = NULL"). - Set("wait_reason = NULL"). - Set("status_message = NULL"). - Where("id = ? AND status = ?", existing.ID, existing.Status). - Exec(ctx) - if err != nil { - return fmt.Errorf("reactivating bucket durability task %d: %w", bucketID, err) - } - rows, _ := res.RowsAffected() - activated = rows == 1 +func (r *BunCacheEvictionRepo) RecordDeletion(ctx context.Context, contentID, generation, taskID int64) error { + result, err := r.db.NewUpdate(). + Model((*model.ObjectCache)(nil)). + Set("in_cache = ?", false). + Set("cache_presence_generation = cache_presence_generation + 1"). + Set("cache_active_task_id = NULL"). + Set("updated_at = ?", time.Now()). + Where("content_id = ?", contentID). + Where("cache_operation_generation = ?", generation). + Where("cache_active_task_id = ?", taskID). + Exec(ctx) + if err != nil { + return fmt.Errorf("recording cache deletion: %w", err) + } + rows, _ := result.RowsAffected() + if rows == 1 { return nil - }) - return activated, err -} - -func (r *BunCacheEvictionRepo) ListLRUCandidates( - ctx context.Context, - terminalSince time.Time, - limit int, -) ([]cacheeviction.Candidate, error) { - var candidates []cacheeviction.Candidate - q := r.db.NewSelect(). - TableExpr("object_versions AS object_version"). - ColumnExpr("object_version.object_id"). - ColumnExpr("object_version.version_id"). - ColumnExpr("object_version.size"). - ColumnExpr("object_version.cache_accessed_at"). - Join("JOIN storage_uploads AS storage_upload ON storage_upload.id = object_version.storage_upload_id"). - Join("JOIN buckets AS durability_bucket ON durability_bucket.id = storage_upload.bucket_id"). - Where("object_version.in_cache = ?", true). - Where("object_version.is_delete_marker = ?", false). - Where("object_version.size > 0"). - Where("object_version.cache_accessed_at IS NOT NULL"). - Where("object_version.state IN (?)", bun.List([]model.ObjectState{ - model.ObjectStateStored, - model.ObjectStateCacheEvicted, - })). - Where("storage_upload.status IN (?)", bun.List([]model.StorageUploadStatus{ - model.StorageUploadStatusReadable, - model.StorageUploadStatusComplete, - })). - Where(minimumDurabilityMetSQL("storage_upload", "durability_bucket")). - Where(`NOT EXISTS ( - SELECT 1 FROM tasks AS eviction_task - WHERE eviction_task.type = ? - AND eviction_task.ref_type = ? - AND eviction_task.ref_version_id = object_version.version_id - AND eviction_task.status IN (?) - )`, model.TaskTypeEvictCache, "object", bun.List(activeTaskStatuses())). - Where(`NOT EXISTS ( - SELECT 1 FROM tasks AS terminal_lru_task - WHERE terminal_lru_task.type = ? - AND terminal_lru_task.stage = ? - AND terminal_lru_task.ref_type = ? - AND terminal_lru_task.ref_version_id = object_version.version_id - AND terminal_lru_task.status IN (?) - AND (terminal_lru_task.completed_at IS NULL OR terminal_lru_task.completed_at > ?) - )`, - model.TaskTypeEvictCache, - cacheeviction.StageLRU, - "object", - bun.List([]model.TaskStatus{model.TaskStatusFailed, model.TaskStatusExhausted}), - terminalSince, - ). - OrderExpr("object_version.cache_accessed_at ASC"). - OrderExpr("object_version.created_at ASC"). - OrderExpr("object_version.version_id ASC") - if limit > 0 { - q = q.Limit(limit) } - if err := q.Scan(ctx, &candidates); err != nil { - return nil, fmt.Errorf("listing LRU cache eviction candidates: %w", err) + recorded, checkErr := r.DeletionRecorded(ctx, contentID, generation) + if checkErr != nil { + return checkErr } - return candidates, nil -} - -func (r *BunCacheEvictionRepo) PlanLRU( - ctx context.Context, - candidate cacheeviction.Candidate, - maxRetries int, - terminalBefore time.Time, -) (bool, error) { - task := cacheeviction.NewLRUTask(candidate, maxRetries, time.Now()) - return r.createOrReactivate(ctx, task, taskReactivationRule{ - immediateStatuses: []model.TaskStatus{ - model.TaskStatusCancelled, - model.TaskStatusCompleted, - }, - cooledStatuses: []model.TaskStatus{ - model.TaskStatusFailed, - model.TaskStatusExhausted, - }, - terminalBefore: &terminalBefore, - errorAction: "reactivating LRU eviction task", - }) -} - -type taskReactivationRule struct { - immediateStatuses []model.TaskStatus - cooledStatuses []model.TaskStatus - terminalBefore *time.Time - errorAction string + if recorded { + return nil + } + return ErrConflict } -func (r *BunCacheEvictionRepo) createOrReactivate( - ctx context.Context, - task *model.Task, - rule taskReactivationRule, -) (bool, error) { - activated := false - err := r.runMaybeTx(ctx, func(db bun.IDB) error { - existing, err := loadAndLockTaskByIdempotencyKey(ctx, db, task.IdempotencyKey) - if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - return err - } - res, err := db.NewInsert().Model(task).On("CONFLICT (idempotency_key) DO NOTHING").Exec(ctx) - if err != nil { - return err - } - rows, _ := res.RowsAffected() - activated = rows == 1 - return nil - } - eligible := taskStatusIn(existing.Status, rule.immediateStatuses) - if !eligible && taskStatusIn(existing.Status, rule.cooledStatuses) { - if rule.terminalBefore == nil { - return errors.New("reactivating task: terminal cutoff is required") - } - eligible = existing.CompletedAt != nil && !existing.CompletedAt.After(*rule.terminalBefore) - } - if !eligible { - return nil - } - - refVersionID := task.RefVersionID - payload := task.Payload - preserveAuthorization, err := cacheeviction.DeleteAuthorized(existing) - if err != nil { - return err - } - if preserveAuthorization && existing.Status != model.TaskStatusCompleted { - refVersionID = existing.RefVersionID - payload = existing.Payload - } - now := time.Now() - res, err := db.NewUpdate(). - Model((*model.Task)(nil)). - Set("type = ?", task.Type). - Set("stage = ?", task.Stage). - Set("ref_type = ?", task.RefType). - Set("ref_id = ?", task.RefID). - Set("ref_version_id = ?", refVersionID). - Set("payload = ?", payload). - Set("status = ?", model.TaskStatusQueued). - Set("retry_count = 0"). - Set("max_retries = ?", task.MaxRetries). - Set("scheduled_at = ?", now). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("started_at = NULL"). - Set("completed_at = NULL"). - Set("last_error = NULL"). - Set("wait_reason = NULL"). - Set("status_message = NULL"). - Where("id = ? AND status = ?", existing.ID, existing.Status). - Exec(ctx) - if err != nil { - return err - } - rows, _ := res.RowsAffected() - activated = rows == 1 - return nil - }) +func (r *BunCacheEvictionRepo) DeletionRecorded(ctx context.Context, contentID, generation int64) (bool, error) { + if contentID < 1 || generation < 1 { + return false, ErrInvalidInput + } + count, err := r.db.NewSelect(). + Model((*model.ObjectCache)(nil)). + Where("content_id = ?", contentID). + Where("cache_operation_generation = ?", generation). + Where("cache_active_task_id IS NULL"). + Where("in_cache = ?", false). + Count(ctx) if err != nil { - return false, fmt.Errorf("%s %q: %w", rule.errorAction, task.IdempotencyKey, err) + return false, fmt.Errorf("checking recorded cache deletion: %w", err) } - return activated, nil + return count == 1, nil } -func taskStatusIn(status model.TaskStatus, statuses []model.TaskStatus) bool { - for _, candidate := range statuses { - if status == candidate { - return true - } +func (r *BunCacheEvictionRepo) ReleaseEviction(ctx context.Context, contentID, generation, taskID int64) error { + result, err := r.db.NewUpdate(). + Model((*model.ObjectCache)(nil)). + Set("cache_active_task_id = NULL"). + Set("updated_at = ?", time.Now()). + Where("content_id = ?", contentID). + Where("cache_operation_generation = ?", generation). + Where("cache_active_task_id = ?", taskID). + Exec(ctx) + if err != nil { + return fmt.Errorf("releasing cache eviction: %w", err) + } + rows, _ := result.RowsAffected() + if rows != 1 { + return ErrConflict } - return false + return nil } -func (r *BunCacheEvictionRepo) ActiveLRUBytes(ctx context.Context) (int64, error) { - var total int64 +func (r *BunCacheEvictionRepo) NextDurabilityGeneration(ctx context.Context, bucketID int64) (int64, error) { + var generation int64 err := r.db.NewSelect(). - TableExpr("object_versions AS object_version"). - ColumnExpr("COALESCE(SUM(object_version.size), 0)"). - Where("object_version.in_cache = ?", true). - Where(`EXISTS ( - SELECT 1 FROM tasks AS eviction_task - WHERE eviction_task.type = ? - AND eviction_task.stage = ? - AND eviction_task.ref_type = ? - AND eviction_task.ref_version_id = object_version.version_id - AND eviction_task.status IN (?) - )`, - model.TaskTypeEvictCache, - cacheeviction.StageLRU, - "object", - bun.List(activeTaskStatuses()), - ). - Scan(ctx, &total) + Model((*model.Bucket)(nil)). + ColumnExpr("durability_generation + 1"). + Where("id = ?", bucketID). + Scan(ctx, &generation) + if errors.Is(err, sql.ErrNoRows) { + return 0, ErrNotFound + } if err != nil { - return 0, fmt.Errorf("summing active LRU eviction bytes: %w", err) + return 0, fmt.Errorf("reading bucket durability generation: %w", err) } - return total, nil + return generation, nil } -func (r *BunCacheEvictionRepo) CancelActiveTasksExcept( - ctx context.Context, - keepStage string, - message string, -) (int, error) { - now := time.Now() - q := r.db.NewUpdate(). - Model((*model.Task)(nil)). - Set("status = ?", model.TaskStatusCancelled). - Set("completed_at = ?", now). - Set("last_error = NULL"). - Set("wait_reason = NULL"). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("started_at = NULL"). - Where("type = ?", model.TaskTypeEvictCache). - Where("status IN (?)", bun.List(activeTaskStatuses())). - Where("stage IS NULL OR stage <> ?", cacheeviction.StageReconcileBucketDurability). - Where("NOT (" + cacheDeletionAuthorizedSQL(r.db.Dialect().Name()) + ")") - if keepStage != "" { - q = q.Where("(stage IS NULL OR stage <> ?)", keepStage) - } - if message == "" { - q = q.Set("status_message = NULL") - } else { - q = q.Set("status_message = ?", message) - } - res, err := q.Exec(ctx) +func (r *BunCacheEvictionRepo) BindDurabilityTask(ctx context.Context, bucketID, generation, taskID int64) error { + result, err := r.db.NewUpdate(). + Model((*model.Bucket)(nil)). + Set("durability_generation = ?", generation). + Set("durability_task_id = ?", taskID). + Set("updated_at = ?", time.Now()). + Where("id = ?", bucketID). + Where("durability_generation = ?", generation-1). + Exec(ctx) if err != nil { - return 0, fmt.Errorf("cancelling incompatible cache eviction tasks: %w", err) + return fmt.Errorf("binding bucket durability task: %w", err) + } + rows, _ := result.RowsAffected() + if rows != 1 { + return ErrConflict } - rows, _ := res.RowsAffected() - return int(rows), nil + return nil } -func (r *BunCacheEvictionRepo) objectVersionByID( - ctx context.Context, - db bun.IDB, - versionID string, -) (*model.ObjectVersion, error) { - version := new(model.ObjectVersion) - if err := db.NewSelect().Model(version).Where("version_id = ?", versionID).Scan(ctx); err != nil { +func (r *BunCacheEvictionRepo) NextBucketDurabilityCandidate(ctx context.Context, bucketID, generation, taskID int64) (*model.StorageContent, error) { + var current struct { + Generation int64 `bun:"durability_generation"` + TaskID *int64 `bun:"durability_task_id"` + } + if err := r.db.NewSelect(). + Model((*model.Bucket)(nil)). + Column("durability_generation", "durability_task_id"). + Where("id = ?", bucketID). + Scan(ctx, ¤t); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, ErrNotFound } - return nil, fmt.Errorf("loading cache eviction object version: %w", err) + return nil, fmt.Errorf("validating bucket durability task: %w", err) } - return version, nil + if current.Generation != generation || current.TaskID == nil || *current.TaskID != taskID { + return nil, ErrConflict + } + return nextBucketDurabilityCandidate(ctx, r.db, bucketID) } -func lockCacheEvictionContext( - ctx context.Context, - db bun.IDB, - claimedTask *model.Task, - bucketID int64, - uploadID int64, - versionID string, -) (*model.Bucket, *model.StorageUpload, *model.ObjectVersion, *model.Task, error) { - bucket, err := lockBucketByID(ctx, db, bucketID) +func (r *BunCacheEvictionRepo) CompleteBucketDurability(ctx context.Context, bucketID, generation, taskID int64) error { + result, err := r.db.NewUpdate(). + Model((*model.Bucket)(nil)). + Set("durability_task_id = NULL"). + Set("updated_at = ?", time.Now()). + Where("id = ? AND durability_generation = ? AND durability_task_id = ?", bucketID, generation, taskID). + Exec(ctx) if err != nil { - return nil, nil, nil, nil, err + return fmt.Errorf("completing bucket durability reconciliation: %w", err) } - if bucket == nil { - return nil, nil, nil, nil, cacheeviction.ErrNoLongerEligible + rows, _ := result.RowsAffected() + if rows != 1 { + return ErrConflict } - uploads, err := lockStorageUploadsByID(ctx, db, []int64{uploadID}) + return nil +} + +// lockCacheEntry takes the row lock the same way the other repositories do: a +// no-op update, which both dialects serialise without dialect-specific syntax. +func lockCacheEntry(ctx context.Context, db bun.IDB, contentID int64) (*model.ObjectCache, error) { + lockResult, err := db.NewUpdate(). + Model((*model.ObjectCache)(nil)). + Set("updated_at = updated_at"). + Where("content_id = ?", contentID). + Exec(ctx) if err != nil { - return nil, nil, nil, nil, err - } - upload := uploads[uploadID] - if upload == nil || upload.BucketID != bucket.ID { - return nil, nil, nil, nil, cacheeviction.ErrNoLongerEligible + return nil, fmt.Errorf("locking cache entry: %w", err) } - if err := lockObjectVersionsByID(ctx, db, []string{versionID}); err != nil { - if errors.Is(err, ErrNotFound) { - return nil, nil, nil, nil, cacheeviction.ErrNoLongerEligible - } - return nil, nil, nil, nil, err + if rows, _ := lockResult.RowsAffected(); rows == 0 { + return nil, ErrNotFound } - version := new(model.ObjectVersion) - if err := db.NewSelect().Model(version).Where("version_id = ?", versionID).Scan(ctx); err != nil { + entry := new(model.ObjectCache) + if err := db.NewSelect().Model(entry).Where("content_id = ?", contentID).Scan(ctx); err != nil { if errors.Is(err, sql.ErrNoRows) { - return nil, nil, nil, nil, cacheeviction.ErrNoLongerEligible + return nil, ErrNotFound } - return nil, nil, nil, nil, fmt.Errorf("loading locked cache eviction object version: %w", err) - } - if version.BucketID != bucket.ID || version.StorageUploadID == nil || *version.StorageUploadID != upload.ID { - return nil, nil, nil, nil, cacheeviction.ErrNoLongerEligible - } - tasks := &BunTaskRepo{db: db} - if err := tasks.LockRunningClaim(ctx, claimedTask); err != nil { - return nil, nil, nil, nil, err - } - lockedTask, err := tasks.GetByID(ctx, claimedTask.ID) - if err != nil { - return nil, nil, nil, nil, err - } - if lockedTask == nil || lockedTask.Type != model.TaskTypeEvictCache { - return nil, nil, nil, nil, fmt.Errorf("cache eviction task changed: %w", ErrConflict) + return nil, err } - return bucket, upload, version, lockedTask, nil + return entry, nil } -func requireMinimumDurability( - ctx context.Context, - db bun.IDB, - bucket *model.Bucket, - upload *model.StorageUpload, -) error { - if bucket == nil || upload == nil || upload.BucketID != bucket.ID { - return fmt.Errorf("cache eviction durability context is invalid: %w", ErrInvalidInput) - } - if upload.Status != model.StorageUploadStatusReadable && upload.Status != model.StorageUploadStatusComplete { - return cacheeviction.ErrDurabilityThreshold +func requireMinimumDurability(ctx context.Context, db bun.IDB, bucket *model.Bucket, content *model.StorageContent) error { + if bucket == nil || content == nil || content.BucketID != bucket.ID { + return ErrInvalidInput } - minimum := minimumDurableCopiesForUpload(bucket, upload.RequestedCopies) - readable, err := countReadableReplicaSlots(ctx, db, upload.ID) + minimum := minimumDurableCopiesForUpload(bucket, content.RequestedCopies) + readable, err := countReadableReplicaSlots(ctx, db, content.ID) if err != nil { return err } @@ -740,111 +442,47 @@ func requireMinimumDurability( return nil } -func updateRunningEvictionTask( - ctx context.Context, - db bun.IDB, - claimedTask *model.Task, - refVersionID string, - payload map[string]any, -) error { - taskID, claimedAt, err := runningTaskClaim(claimedTask) - if err != nil { - return err - } - now := time.Now() - res, err := db.NewUpdate(). - Model((*model.Task)(nil)). - Set("ref_version_id = ?", refVersionID). - Set("payload = ?", payload). - Where("id = ? AND status = ?", taskID, model.TaskStatusRunning). - Where("claimed_at = ?", claimedAt). - Where("lease_until IS NOT NULL AND lease_until > ?", now). - Exec(ctx) - if err != nil { - return fmt.Errorf("persisting cache deletion authorization: %w", err) - } - rows, _ := res.RowsAffected() - if rows != 1 { - return fmt.Errorf("persisting cache deletion authorization for task %d: not in active running claim", taskID) - } - return nil -} - -func nextBucketDurabilityCandidate( - ctx context.Context, - db bun.IDB, - bucketID int64, -) (*model.ObjectVersion, error) { - version := new(model.ObjectVersion) +// nextBucketDurabilityCandidate finds cached content that now satisfies the +// bucket's minimum durability. A copy-policy change is what schedules this +// scan, so the predicate reads durability from the copy rows rather than from +// any stored lifecycle value. +func nextBucketDurabilityCandidate(ctx context.Context, db bun.IDB, bucketID int64) (*model.StorageContent, error) { + content := new(model.StorageContent) err := db.NewSelect(). - Model(version). - Join("JOIN storage_uploads AS storage_upload ON storage_upload.id = object_version.storage_upload_id"). - Join("JOIN buckets AS durability_bucket ON durability_bucket.id = storage_upload.bucket_id"). - Where("object_version.bucket_id = ?", bucketID). - Where("object_version.state = ?", model.ObjectStateReplicating). - Where("object_version.in_cache = ?", true). - Where("object_version.is_delete_marker = ?", false). - Where("storage_upload.status IN (?)", bun.List([]model.StorageUploadStatus{ - model.StorageUploadStatusReadable, - model.StorageUploadStatusComplete, - })). - Where(minimumDurabilityMetSQL("storage_upload", "durability_bucket")). - OrderExpr("object_version.updated_at ASC"). - OrderExpr("object_version.version_id ASC"). + Model(content). + Join("JOIN object_cache AS cache_entry ON cache_entry.content_id = storage_content.id"). + Join("JOIN buckets AS durability_bucket ON durability_bucket.id = storage_content.bucket_id"). + Where("storage_content.bucket_id = ?", bucketID). + Where("cache_entry.in_cache = ?", true). + Where("cache_entry.cache_active_task_id IS NULL"). + Where(minimumDurabilityMetSQL("storage_content", "durability_bucket")). + OrderExpr("storage_content.updated_at, storage_content.id"). Limit(1). Scan(ctx) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } return nil, fmt.Errorf("selecting bucket durability candidate: %w", err) } - return version, nil + return content, nil } -// Keep this threshold aligned with minimumDurableCopiesForUpload; both compare -// readable replica slots, not physical data set generations, against the -// bucket's effective minimum. -func minimumDurabilityMetSQL(uploadAlias, bucketAlias string) string { +func minimumDurabilityMetSQL(contentAlias, bucketAlias string) string { return fmt.Sprintf(`%s >= CASE WHEN %s.minimum_durable_copies IS NULL OR %s.minimum_durable_copies >= %s.requested_copies THEN %s.requested_copies ELSE %s.minimum_durable_copies END`, - distinctReadableSlotCountSQL("durable_copy", "durable_data_set", uploadAlias+".id"), - bucketAlias, - bucketAlias, - uploadAlias, - uploadAlias, - bucketAlias, + distinctReadableSlotCountSQL("durable_copy", "durable_data_set", contentAlias+".id"), + bucketAlias, bucketAlias, contentAlias, contentAlias, bucketAlias, ) } -func cacheDeletionStateEligible(version *model.ObjectVersion) bool { - if version == nil || version.IsDeleteMarker || !version.InCache || version.StorageUploadID == nil { - return false +func cacheAccessTime(entry *model.ObjectCache) time.Time { + if entry.CacheAccessedAt != nil { + return *entry.CacheAccessedAt } - return version.State == model.ObjectStateStored || version.State == model.ObjectStateCacheEvicted -} - -func cacheAccessTime(version *model.ObjectVersion) time.Time { - if version == nil { - return time.Time{} - } - if version.CacheAccessedAt != nil { - return *version.CacheAccessedAt - } - return version.CreatedAt -} - -func cacheDeletionAuthorizedSQL(dialectName dialect.Name) string { - if dialectName == dialect.PG { - return "COALESCE(CAST(payload ->> 'delete_authorized' AS BOOLEAN), FALSE)" - } - return "COALESCE(CAST(json_extract(payload, '$.delete_authorized') AS INTEGER), 0) = 1" -} - -func (r *BunCacheEvictionRepo) runMaybeTx(ctx context.Context, fn func(bun.IDB) error) error { - return runMaybeTx(ctx, r.db, fn) + return entry.CreatedAt } diff --git a/internal/db/repository/cache_eviction_repo_test.go b/internal/db/repository/cache_eviction_repo_test.go index 6659fef..0f8f629 100644 --- a/internal/db/repository/cache_eviction_repo_test.go +++ b/internal/db/repository/cache_eviction_repo_test.go @@ -1,663 +1,151 @@ package repository_test import ( - "context" + "encoding/json" "errors" + "fmt" + "strconv" "testing" "time" "github.com/strahe/synaps3/internal/cacheeviction" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" - "github.com/uptrace/bun" ) -func TestCacheEvictionRepo_DurabilityAuthorizationUsesCommitOrder(t *testing.T) { - for _, tc := range []struct { - name string - raiseBeforePromote bool - wantAuthorized bool - }{ - {name: "policy_raise_first_blocks_deletion", raiseBeforePromote: true}, - {name: "authorization_first_survives_policy_raise", wantAuthorized: true}, - } { - t.Run(tc.name, func(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket, version, upload := seedMinimumDurabilityCandidate(t, repos, db, tc.name) - - if _, err := repos.CacheEvictions.EnsureBucketDurabilityReconciliation(ctx, bucket.ID, 4); err != nil { - t.Fatalf("EnsureBucketDurabilityReconciliation: %v", err) - } - task, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeEvictCache, time.Minute) - if err != nil || task == nil { - t.Fatalf("ClaimReady: task=%#v err=%v", task, err) - } - candidate, err := repos.CacheEvictions.NextBucketDurabilityCandidate(ctx, bucket.ID) - if err != nil || candidate == nil || candidate.VersionID != version.VersionID { - t.Fatalf("NextBucketDurabilityCandidate: candidate=%#v err=%v", candidate, err) - } - - minimumThree := 3 - if tc.raiseBeforePromote { - if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimumThree, - }); err != nil { - t.Fatalf("raise minimum before authorization: %v", err) - } - } - - deletion, err := repos.CacheEvictions.PromoteBucketDurabilityCandidate(ctx, task, version.VersionID, true) - if !tc.wantAuthorized { - if !errors.Is(err, cacheeviction.ErrDurabilityThreshold) || deletion != nil { - t.Fatalf("promotion after policy raise = deletion:%#v err:%v, want durability threshold", deletion, err) - } - got, getErr := repos.Objects.GetVersionByID(ctx, version.VersionID) - if getErr != nil || got == nil || got.State != model.ObjectStateReplicating || !got.InCache { - t.Fatalf("blocked candidate = %#v err=%v, want replicating in cache", got, getErr) - } - return - } - if err != nil || deletion == nil { - t.Fatalf("PromoteBucketDurabilityCandidate: deletion=%#v err=%v", deletion, err) - } - authorized, err := cacheeviction.DeleteAuthorized(task) - if err != nil || !authorized || task.RefVersionID != version.VersionID { - t.Fatalf("task authorization = %t ref=%q err=%v", authorized, task.RefVersionID, err) - } - if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimumThree, - }); err != nil { - t.Fatalf("raise minimum after authorization: %v", err) - } - if err := repos.CacheEvictions.RecordAuthorizedDeletion(ctx, task); err != nil { - t.Fatalf("RecordAuthorizedDeletion: %v", err) - } - got, err := repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil || got == nil || got.State != model.ObjectStateCacheEvicted || got.InCache { - t.Fatalf("authorized candidate = %#v err=%v, want cache evicted", got, err) - } - if task.RefVersionID != "" { - t.Fatalf("coordinator ref after deletion = %q, want cleared", task.RefVersionID) - } - completed, err := repos.CacheEvictions.CompleteBucketDurabilityReconciliation(ctx, task) - if err != nil || !completed { - t.Fatalf("CompleteBucketDurabilityReconciliation = %t err=%v", completed, err) - } - gotUpload, err := repos.Uploads.GetByID(ctx, upload.ID) - if err != nil || gotUpload == nil || gotUpload.Status != model.StorageUploadStatusReadable { - t.Fatalf("upload after cache deletion = %#v err=%v, want readable for repair", gotUpload, err) - } - }) - } -} - -func TestCacheEvictionRepo_RequeuesTerminalCoordinatorWithoutLosingAuthorization(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket, version, _ := seedMinimumDurabilityCandidate(t, repos, db, "preserve-authorization") - if _, err := repos.CacheEvictions.EnsureBucketDurabilityReconciliation(ctx, bucket.ID, 4); err != nil { - t.Fatalf("EnsureBucketDurabilityReconciliation: %v", err) - } - task, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeEvictCache, time.Minute) - if err != nil || task == nil { - t.Fatalf("ClaimReady: task=%#v err=%v", task, err) - } - if _, err := repos.CacheEvictions.PromoteBucketDurabilityCandidate(ctx, task, version.VersionID, true); err != nil { - t.Fatalf("PromoteBucketDurabilityCandidate: %v", err) - } - if err := repos.Tasks.FailRunning(ctx, task, "injected cache failure"); err != nil { - t.Fatalf("FailRunning: %v", err) - } - activated, err := repos.CacheEvictions.EnsureBucketDurabilityReconciliation(ctx, bucket.ID, 7) - if err != nil || !activated { - t.Fatalf("requeue terminal coordinator = %t err=%v", activated, err) - } - got, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil || got == nil { - t.Fatalf("GetByID: task=%#v err=%v", got, err) - } - authorized, err := cacheeviction.DeleteAuthorized(got) - if err != nil || !authorized || got.RefVersionID != version.VersionID || got.Status != model.TaskStatusQueued || got.MaxRetries != 7 { - t.Fatalf("requeued coordinator = %#v authorized=%t err=%v", got, authorized, err) - } -} - -func TestCacheEvictionRepo_BucketDurabilityCoordinatorPreservesZeroRetries(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "durability-zero-retries") - - created, err := repos.CacheEvictions.EnsureBucketDurabilityReconciliation(ctx, bucket.ID, 0) - if err != nil || !created { - t.Fatalf("EnsureBucketDurabilityReconciliation = %t err=%v", created, err) - } - tasks, total, err := repos.Tasks.List(ctx, string(model.TaskTypeEvictCache), cacheeviction.StageReconcileBucketDurability, "", 10, 0) - if err != nil || total != 1 || len(tasks) != 1 { - t.Fatalf("coordinator tasks total=%d tasks=%#v err=%v", total, tasks, err) - } - if tasks[0].MaxRetries != 0 { - t.Fatalf("coordinator max retries = %d, want 0", tasks[0].MaxRetries) - } -} - -func TestCacheEvictionRepo_ClearsCoordinatorAuthorizationWhenVersionWasDeleted(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket, version, _ := seedMinimumDurabilityCandidate(t, repos, db, "deleted-authorized-version") - if _, err := repos.CacheEvictions.EnsureBucketDurabilityReconciliation(ctx, bucket.ID, 4); err != nil { - t.Fatalf("EnsureBucketDurabilityReconciliation: %v", err) - } - task, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeEvictCache, time.Minute) - if err != nil || task == nil { - t.Fatalf("ClaimReady: task=%#v err=%v", task, err) - } - if _, err := repos.CacheEvictions.PromoteBucketDurabilityCandidate(ctx, task, version.VersionID, true); err != nil { - t.Fatalf("PromoteBucketDurabilityCandidate: %v", err) - } - if _, err := db.NewDelete().Model((*model.ObjectVersion)(nil)).Where("version_id = ?", version.VersionID).Exec(ctx); err != nil { - t.Fatalf("delete authorized version: %v", err) - } - if err := repos.CacheEvictions.RecordAuthorizedDeletion(ctx, task); err != nil { - t.Fatalf("RecordAuthorizedDeletion missing version: %v", err) - } - if task.RefVersionID != "" || task.Payload != nil { - t.Fatalf("cleared coordinator authorization = ref:%q payload:%#v", task.RefVersionID, task.Payload) - } - completed, err := repos.CacheEvictions.CompleteBucketDurabilityReconciliation(ctx, task) - if err != nil || !completed { - t.Fatalf("CompleteBucketDurabilityReconciliation = %t err=%v", completed, err) - } -} - -func TestCacheEvictionRepo_LRUCandidateRequiresCurrentMinimumDurability(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket, version, upload := seedMinimumDurabilityCandidate(t, repos, db, "lru-minimum") - - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateReplicating, model.ObjectStateStored); err != nil { - t.Fatalf("mark stored: %v", err) - } - if err := repos.Objects.RecordVersionCacheCommit(ctx, version.VersionID, time.Now().Add(-time.Hour)); err != nil { - t.Fatalf("RecordVersionCacheCommit: %v", err) - } - if _, err := db.NewDelete(). - Model((*model.StorageUploadCopy)(nil)). - Where("upload_id = ? AND copy_index = ?", upload.ID, 1). - Exec(ctx); err != nil { - t.Fatalf("remove second readable copy: %v", err) - } - - candidates, err := repos.CacheEvictions.ListLRUCandidates(ctx, time.Now().Add(-time.Hour), 10) - if err != nil { - t.Fatalf("ListLRUCandidates below minimum: %v", err) - } - if len(candidates) != 0 { - t.Fatalf("candidates below minimum = %#v, want none", candidates) - } - - minimumOne := 1 - if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimumOne, - }); err != nil { - t.Fatalf("lower minimum: %v", err) - } - candidates, err = repos.CacheEvictions.ListLRUCandidates(ctx, time.Now().Add(-time.Hour), 10) - if err != nil { - t.Fatalf("ListLRUCandidates at minimum: %v", err) - } - if len(candidates) != 1 || candidates[0].VersionID != version.VersionID { - t.Fatalf("candidates at minimum = %#v, want %s", candidates, version.VersionID) - } -} - -func TestCacheEvictionRepo_EnsureAfterUploadOnlyRequeuesCancelledTask(t *testing.T) { +func TestPrepareEvictionReusesOnlyMatchingLiveOwner(t *testing.T) { db := testDB(t) repos := repository.NewRepositories(db) - ctx := context.Background() - key := "evict_cache:01J0000000000000000000CR01" - original := &model.Task{ - Type: model.TaskTypeEvictCache, - RefType: "object", - RefID: 10, - RefVersionID: "01J0000000000000000000CR01", - IdempotencyKey: key, - Status: model.TaskStatusQueued, - MaxRetries: 2, - ScheduledAt: time.Now().Add(-time.Hour), - } - if err := repos.Tasks.Create(ctx, original); err != nil { - t.Fatalf("Create original task: %v", err) - } - mustExec( - t, - db, - `UPDATE tasks - SET status = ?, retry_count = 2, last_error = 'old error', - completed_at = ?, status_message = 'old message' - WHERE id = ?`, - model.TaskStatusCancelled, - time.Now(), - original.ID, - ) - - activated, err := repos.CacheEvictions.EnsureAfterUploadTask( - ctx, - 11, - original.RefVersionID, - 7, - ) - if err != nil { - t.Fatalf("EnsureAfterUploadTask(cancelled): %v", err) - } - if !activated { - t.Fatal("EnsureAfterUploadTask(cancelled) activated = false, want true") - } - got, err := repos.Tasks.GetByID(ctx, original.ID) - if err != nil || got == nil { - t.Fatalf("GetByID reactivated: task=%v err=%v", got, err) - } - if got.Status != model.TaskStatusQueued || - got.Stage == nil || - *got.Stage != cacheeviction.StageAfterUpload || - got.RefID != 11 || - got.MaxRetries != 7 || - got.RetryCount != 0 || - got.CompletedAt != nil || - got.LastError != nil || - got.StatusMessage != nil { - t.Fatalf("reactivated task = %#v, want reset queued replacement", got) - } - - mustExec(t, db, `UPDATE tasks SET status = ?, completed_at = ? WHERE id = ?`, model.TaskStatusCompleted, time.Now(), original.ID) - activated, err = repos.CacheEvictions.EnsureAfterUploadTask( - ctx, - 11, - original.RefVersionID, - 7, - ) - if err != nil { - t.Fatalf("EnsureAfterUploadTask(completed): %v", err) - } - if activated { - t.Fatal("EnsureAfterUploadTask(completed) activated = true, want false") - } - got, err = repos.Tasks.GetByID(ctx, original.ID) - if err != nil || got == nil || got.Status != model.TaskStatusCompleted { - t.Fatalf("completed task after reactivation attempt = %#v err=%v", got, err) - } -} - -func TestCacheEvictionRepo_PlanLRUHonorsTerminalCooldown(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - stage := cacheeviction.StageLRU - completedAt := time.Now() - lastError := "permission denied" - original := &model.Task{ - Type: model.TaskTypeEvictCache, - Stage: &stage, - RefType: "object", - RefID: 12, - RefVersionID: "01J0000000000000000000CR02", - IdempotencyKey: "evict_cache:lru:01J0000000000000000000CR02", - Status: model.TaskStatusExhausted, - RetryCount: 3, - MaxRetries: 3, - LastError: &lastError, - ScheduledAt: completedAt, - CompletedAt: &completedAt, - } - if err := repos.Tasks.Create(ctx, original); err != nil { - t.Fatalf("Create exhausted task: %v", err) - } - candidate := cacheeviction.Candidate{ - ObjectID: original.RefID, - VersionID: original.RefVersionID, - Size: 10, - AccessedAt: time.Now(), - } - - activated, err := repos.CacheEvictions.PlanLRU( - ctx, - candidate, - 5, - time.Now().Add(-time.Hour), - ) - if err != nil { - t.Fatalf("PlanLRU(recent): %v", err) - } - if activated { - t.Fatal("recent exhausted task activated before cooldown") - } - - mustExec(t, db, `UPDATE tasks SET completed_at = ? WHERE id = ?`, time.Now().Add(-2*time.Hour), original.ID) - activated, err = repos.CacheEvictions.PlanLRU( - ctx, - candidate, - 5, - time.Now().Add(-time.Hour), - ) - if err != nil { - t.Fatalf("PlanLRU(cooled): %v", err) - } - if !activated { - t.Fatal("cooled exhausted task activated = false, want true") - } - got, err := repos.Tasks.GetByID(ctx, original.ID) - if err != nil || got == nil { - t.Fatalf("GetByID reactivated recoverable: task=%v err=%v", got, err) - } - if got.Status != model.TaskStatusQueued || - got.RetryCount != 0 || - got.MaxRetries != 5 || - got.CompletedAt != nil || - got.LastError != nil { - t.Fatalf("reactivated recoverable task = %#v, want reset queued task", got) - } - - completedAt = time.Now() - mustExec( - t, - db, - `UPDATE tasks SET status = ?, completed_at = ? WHERE id = ?`, - model.TaskStatusCompleted, - completedAt, - original.ID, - ) - activated, err = repos.CacheEvictions.PlanLRU( - ctx, - candidate, - 5, - time.Now().Add(-time.Hour), - ) - if err != nil { - t.Fatalf("PlanLRU(completed): %v", err) - } - if !activated { - t.Fatal("completed LRU task activated = false, want true") - } -} - -func TestCacheEvictionRepo_CancelActiveTasksExceptPreservesMatchingStageAndTerminalHistory(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - lruStage := cacheeviction.StageLRU - afterUploadStage := cacheeviction.StageAfterUpload - reconcileStage := cacheeviction.StageReconcileBucketDurability - tasks := []*model.Task{ - { - Type: model.TaskTypeEvictCache, - Stage: &lruStage, - RefType: "object", - RefVersionID: "01J0000000000000000000CS01", - IdempotencyKey: "cancel-stage-lru", - Status: model.TaskStatusQueued, - }, - { - Type: model.TaskTypeEvictCache, - Stage: &afterUploadStage, - RefType: "object", - RefVersionID: "01J0000000000000000000CS02", - IdempotencyKey: "cancel-stage-after-upload", - Status: model.TaskStatusQueued, - }, - { - Type: model.TaskTypeEvictCache, - RefType: "object", - RefVersionID: "01J0000000000000000000CS03", - IdempotencyKey: "cancel-stage-legacy", - Status: model.TaskStatusWaiting, - }, - { - Type: model.TaskTypeEvictCache, - Stage: &lruStage, - RefType: "object", - RefVersionID: "01J0000000000000000000CS04", - IdempotencyKey: "cancel-stage-terminal", - Status: model.TaskStatusFailed, - }, - { - Type: model.TaskTypeEvictCache, - Stage: &reconcileStage, - RefType: "bucket", - RefID: 9, - IdempotencyKey: "cancel-stage-durability-coordinator", - Status: model.TaskStatusQueued, - }, - { - Type: model.TaskTypeEvictCache, - Stage: &afterUploadStage, - RefType: "object", - RefVersionID: "01J0000000000000000000CS05", - IdempotencyKey: "cancel-stage-authorized-after-upload", - Payload: cacheeviction.WithDeleteAuthorization(nil), - Status: model.TaskStatusQueued, - }, - } - for _, task := range tasks { - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create(%s): %v", task.IdempotencyKey, err) - } - } - - cancelled, err := repos.CacheEvictions.CancelActiveTasksExcept(ctx, lruStage, "policy changed") - if err != nil { - t.Fatalf("CancelActiveTasksExcept: %v", err) - } - if cancelled != 2 { - t.Fatalf("cancelled count = %d, want 2", cancelled) - } - wantStatuses := []model.TaskStatus{ - model.TaskStatusQueued, - model.TaskStatusCancelled, - model.TaskStatusCancelled, - model.TaskStatusFailed, - model.TaskStatusQueued, - model.TaskStatusQueued, - } - for index, task := range tasks { - got, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil || got == nil { - t.Fatalf("GetByID(%d): task=%v err=%v", task.ID, got, err) - } - if got.Status != wantStatuses[index] { - t.Fatalf("task %s status = %s, want %s", task.IdempotencyKey, got.Status, wantStatuses[index]) + bucket := seedBucket(t, db, "cache-eviction-reservation") + ctx := t.Context() + seedCachedContent := func(checksum string) int64 { + t.Helper() + contentID := seedContent(t, repos, bucket.ID, checksum, 10) + if err := repos.Objects.RecordContentCacheCommit(ctx, contentID, time.Now()); err != nil { + t.Fatalf("record cache presence: %v", err) } + return contentID } -} - -func seedMinimumDurabilityCandidate( - t *testing.T, - repos *repository.Repositories, - db *bun.DB, - suffix string, -) (*model.Bucket, *model.ObjectVersion, *model.StorageUpload) { - t.Helper() - ctx := context.Background() - bucket := seedBucket(t, db, "durability-candidate-"+suffix) - minimum := 2 - if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimum, - }); err != nil { - t.Fatalf("UpdateCopyPolicy: %v", err) - } - version := newObjectVersion(bucket.ID, "candidate.txt", model.NewVersionID(), 10) - version.Checksum = "durability-candidate-" + suffix - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - upload := startCopyHealthUpload(t, repos, bucket.ID, version.VersionID, version.Size, version.Checksum, 3) - commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 0, "101", "1001", "2001", "https://one.example/piece") - commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 1, "202", "2002", "2002", "https://two.example/piece") - bindStorageHealthVersion(t, repos, bucket.ID, upload.ID, version) - return bucket, version, upload -} -func TestCacheEvictionRepo_ListLRUCandidatesOrdersSafeCurrentAndHistoricalVersions(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "lru-candidates-bucket") - - type seededVersion struct { - version *model.ObjectVersion - objectID int64 - } - seedStored := func(versionID string, size int64) seededVersion { + enqueue := func(contentID, generation int64, taskType model.TaskType, subjectType, subjectKey string) *model.Task { t.Helper() - version := newObjectVersion(bucket.ID, "file.txt", versionID, size) - objectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version) + input, err := json.Marshal(cacheeviction.EvictInput{ContentID: contentID, Generation: generation}) if err != nil { - t.Fatalf("CreateVersionAndSetCurrent(%s): %v", versionID, err) + t.Fatalf("marshal task input: %v", err) } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("UpdateVersionState(%s): %v", versionID, err) + row, created, err := repos.Tasks.Enqueue(ctx, &model.Task{ + Type: taskType, IdempotencyKey: cacheeviction.EvictTaskKey(contentID, generation), + InputVersion: 1, InputHash: fmt.Sprintf("hash-%d-%d", contentID, generation), + Input: input, SubjectType: &subjectType, SubjectKey: &subjectKey, + }) + if err != nil || !created { + t.Fatalf("enqueue task = %#v created=%v err=%v", row, created, err) } - acceptTestStorageUploadForVersion(t, repos, bucket.ID, version, "piece-"+versionID) - return seededVersion{version: version, objectID: objectID} + return row } - oldest := seedStored("01J0000000000000000000LR01", 10) - middle := seedStored("01J0000000000000000000LR02", 20) - newest := seedStored("01J0000000000000000000LR03", 30) - unsafe := newObjectVersion(bucket.ID, "unsafe.txt", "01J0000000000000000000LR04", 40) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, unsafe); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(unsafe): %v", err) + matchingContentID := seedCachedContent("matching-cache-owner") + matching, err := repos.CacheEvictions.PrepareEviction(ctx, matchingContentID) + if err != nil || matching.Generation != 1 || matching.ActiveTaskID != nil { + t.Fatalf("initial matching reservation = %#v, err=%v", matching, err) } - - if err := repos.Objects.UpdateVersionState( - ctx, - newest.version.VersionID, - model.ObjectStateStored, - model.ObjectStateCacheEvicted, - ); err != nil { - t.Fatalf("mark newest cache_evicted: %v", err) + matchingTask := enqueue( + matchingContentID, + matching.Generation, + model.TaskTypeCacheEvict, + "storage_content", + strconv.FormatInt(matchingContentID, 10), + ) + if err := repos.CacheEvictions.BindEvictionTask(ctx, matchingContentID, matching.Generation, matchingTask.ID); err != nil { + t.Fatalf("bind matching owner: %v", err) } - base := time.Now().UTC().Add(time.Hour).Truncate(time.Microsecond) - for index, seeded := range []seededVersion{oldest, middle, newest} { - if err := repos.Objects.RecordVersionCacheCommit(ctx, seeded.version.VersionID, base.Add(time.Duration(index)*time.Hour)); err != nil { - t.Fatalf("RecordVersionCacheCommit(%s): %v", seeded.version.VersionID, err) - } + reused, err := repos.CacheEvictions.PrepareEviction(ctx, matchingContentID) + if err != nil || reused.Generation != matching.Generation || reused.ActiveTaskID == nil || *reused.ActiveTaskID != matchingTask.ID { + t.Fatalf("reused reservation = %#v, err=%v", reused, err) } - terminalSince := time.Now().Add(-time.Hour) - candidates, err := repos.CacheEvictions.ListLRUCandidates(ctx, terminalSince, 10) + terminalContentID := seedCachedContent("terminal-cache-owner") + terminal, err := repos.CacheEvictions.PrepareEviction(ctx, terminalContentID) if err != nil { - t.Fatalf("ListLRUCandidates: %v", err) - } - if len(candidates) != 3 { - t.Fatalf("candidate count = %d, want 3: %#v", len(candidates), candidates) - } - for index, want := range []string{oldest.version.VersionID, middle.version.VersionID, newest.version.VersionID} { - if candidates[index].VersionID != want { - t.Fatalf("candidate[%d] = %s, want %s; candidates=%#v", index, candidates[index].VersionID, want, candidates) - } + t.Fatalf("prepare terminal owner: %v", err) + } + terminalTask := enqueue( + terminalContentID, + terminal.Generation, + model.TaskTypeCacheEvict, + "storage_content", + strconv.FormatInt(terminalContentID, 10), + ) + if err := repos.CacheEvictions.BindEvictionTask(ctx, terminalContentID, terminal.Generation, terminalTask.ID); err != nil { + t.Fatalf("bind terminal owner: %v", err) + } + if _, err := db.NewUpdate().Model((*model.Task)(nil)). + Set("status = ?", model.TaskStatusFailed). + Set("failure_reason = ?", "test_terminal_owner"). + Set("finished_at = ?", time.Now()). + Where("id = ?", terminalTask.ID). + Exec(ctx); err != nil { + t.Fatalf("mark owner terminal: %v", err) } - - // A NULL access time is not eligible. The migration initializes all existing - // rows, while newly committed cache entries always record an access time. - mustExec(t, db, `UPDATE object_versions SET cache_accessed_at = NULL, created_at = ? WHERE version_id = ?`, base.Add(-time.Hour), oldest.version.VersionID) - stage := cacheeviction.StageLRU - task := &model.Task{ - Type: model.TaskTypeEvictCache, - Stage: &stage, - RefType: "object", - RefID: middle.objectID, - RefVersionID: middle.version.VersionID, - IdempotencyKey: "evict_cache:lru:test:" + middle.version.VersionID, - Status: model.TaskStatusQueued, - MaxRetries: 3, - ScheduledAt: time.Now(), + next, err := repos.CacheEvictions.PrepareEviction(ctx, terminalContentID) + if err != nil || next.Generation != terminal.Generation+1 || next.ActiveTaskID != nil { + t.Fatalf("next reservation after terminal owner = %#v, err=%v", next, err) } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create active eviction task: %v", err) + entry, err := repos.CacheEvictions.GetCacheEntry(ctx, terminalContentID) + if err != nil || entry == nil || entry.CacheActiveTaskID != nil { + t.Fatalf("cache entry after terminal owner = %#v, err=%v", entry, err) } - candidates, err = repos.CacheEvictions.ListLRUCandidates(ctx, terminalSince, 10) + conflictContentID := seedCachedContent("conflicting-cache-owner") + conflicting, err := repos.CacheEvictions.PrepareEviction(ctx, conflictContentID) if err != nil { - t.Fatalf("ListLRUCandidates after active task: %v", err) - } - if len(candidates) != 1 || candidates[0].VersionID != newest.version.VersionID { - t.Fatalf("candidates with active task = %#v, want newest only", candidates) - } - activeBytes, err := repos.CacheEvictions.ActiveLRUBytes(ctx) - if err != nil { - t.Fatalf("ActiveLRUBytes: %v", err) - } - if activeBytes != middle.version.Size { - t.Fatalf("active eviction bytes = %d, want %d", activeBytes, middle.version.Size) + t.Fatalf("prepare conflicting owner: %v", err) + } + conflictingTask := enqueue( + conflictContentID, + conflicting.Generation, + model.TaskTypeCacheEvict, + "object_version", + "wrong-subject", + ) + if err := repos.CacheEvictions.BindEvictionTask(ctx, conflictContentID, conflicting.Generation, conflictingTask.ID); err != nil { + t.Fatalf("bind conflicting owner: %v", err) } - if err := repos.Objects.RecordVersionCacheAccess(ctx, oldest.version.VersionID, base.Add(-time.Hour)); err != nil { - t.Fatalf("RecordVersionCacheAccess(restored): %v", err) + if _, err := repos.CacheEvictions.PrepareEviction(ctx, conflictContentID); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("prepare with mismatched live owner = %v, want ErrConflict", err) } - exhaustedAt := time.Now() - exhausted := &model.Task{ - Type: model.TaskTypeEvictCache, - Stage: &stage, - RefType: "object", - RefID: oldest.objectID, - RefVersionID: oldest.version.VersionID, - IdempotencyKey: "evict_cache:lru:" + oldest.version.VersionID, - Status: model.TaskStatusExhausted, - MaxRetries: 3, - ScheduledAt: exhaustedAt, - CompletedAt: &exhaustedAt, - } - if err := repos.Tasks.Create(ctx, exhausted); err != nil { - t.Fatalf("Create exhausted LRU task: %v", err) - } - candidates, err = repos.CacheEvictions.ListLRUCandidates(ctx, terminalSince, 10) + generationConflictContentID := seedCachedContent("generation-conflicting-cache-owner") + generationConflict, err := repos.CacheEvictions.PrepareEviction(ctx, generationConflictContentID) if err != nil { - t.Fatalf("ListLRUCandidates after exhausted task: %v", err) - } - if len(candidates) != 1 || candidates[0].VersionID != newest.version.VersionID { - t.Fatalf("candidates with active and exhausted tasks = %#v, want newest only", candidates) - } - - mustExec( - t, - db, - `UPDATE tasks SET completed_at = ? WHERE id = ?`, - terminalSince.Add(-time.Second), - exhausted.ID, + t.Fatalf("prepare generation-conflicting owner: %v", err) + } + generationConflictTask := enqueue( + generationConflictContentID, + generationConflict.Generation, + model.TaskTypeCacheEvict, + "storage_content", + strconv.FormatInt(generationConflictContentID, 10), ) - candidates, err = repos.CacheEvictions.ListLRUCandidates(ctx, terminalSince, 10) + badInput, err := json.Marshal(cacheeviction.EvictInput{ + ContentID: generationConflictContentID, + Generation: generationConflict.Generation + 1, + }) if err != nil { - t.Fatalf("ListLRUCandidates after exhausted cooldown: %v", err) + t.Fatalf("marshal mismatched generation input: %v", err) } - if len(candidates) != 2 || - candidates[0].VersionID != oldest.version.VersionID || - candidates[1].VersionID != newest.version.VersionID { - t.Fatalf("candidates after exhausted cooldown = %#v, want oldest/newest", candidates) + if _, err := db.NewUpdate().Model((*model.TaskPayload)(nil)).Set("input_json = ?", badInput).Where("task_id = ?", generationConflictTask.ID).Exec(ctx); err != nil { + t.Fatalf("corrupt generation input: %v", err) } - - mustExec(t, db, `UPDATE tasks SET completed_at = ? WHERE id = ?`, time.Now(), exhausted.ID) - newAccess := base.Add(4 * time.Hour) - if err := repos.Objects.RecordVersionCacheAccess(ctx, oldest.version.VersionID, newAccess); err != nil { - t.Fatalf("RecordVersionCacheAccess(newer access): %v", err) - } - candidates, err = repos.CacheEvictions.ListLRUCandidates(ctx, terminalSince, 10) - if err != nil { - t.Fatalf("ListLRUCandidates after newer access: %v", err) + if err := repos.CacheEvictions.BindEvictionTask( + ctx, + generationConflictContentID, + generationConflict.Generation, + generationConflictTask.ID, + ); err != nil { + t.Fatalf("bind generation-conflicting owner: %v", err) } - if len(candidates) != 1 || candidates[0].VersionID != newest.version.VersionID { - t.Fatalf("candidates after newer access = %#v, want recent exhausted task to remain cooling down", candidates) + if _, err := repos.CacheEvictions.PrepareEviction(ctx, generationConflictContentID); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("prepare with mismatched task generation = %v, want ErrConflict", err) } } diff --git a/internal/db/repository/errors.go b/internal/db/repository/errors.go index 8aa79e0..466399e 100644 --- a/internal/db/repository/errors.go +++ b/internal/db/repository/errors.go @@ -20,30 +20,19 @@ var ErrInvalidInput = errors.New("invalid input") // ErrConflict is returned when a compare-and-restore operation sees stale state. var ErrConflict = errors.New("conflict") +// ErrReplicaTargetLowered reports an attempt to reduce a bucket's replica +// target. It stays an ErrInvalidInput so existing handling still applies, but it +// is distinguishable because the reason a caller needs to hear is specific. +var ErrReplicaTargetLowered = fmt.Errorf("lowering the replica target is not supported: %w", ErrInvalidInput) + // ErrPermanentDeleteStorageBusy reports that a permanent delete cannot yet // cancel storage work without risking a remote write. It remains compatible // with ErrConflict for existing callers. var ErrPermanentDeleteStorageBusy = fmt.Errorf("permanent delete blocked by storage work: %w", ErrConflict) -// ErrReplicaRepairItemCancelled reports that the exact repair copy no longer -// has work the claimed coordinator may execute. -var ErrReplicaRepairItemCancelled = errors.New("replica repair item cancelled") - -// ErrUploadTaskCancelled reports that an ordinary upload task no longer has -// live storage work it may execute. -var ErrUploadTaskCancelled = errors.New("upload task cancelled") - -// ErrTaskClaimLost reports that a worker no longer owns the running task claim. -var ErrTaskClaimLost = errors.New("task claim lost") - -// ErrItemClaimLost reports that a provider replacement worker no longer owns -// the item lease identified by its fencing token. -var ErrItemClaimLost = errors.New("replacement item claim lost") - -// ErrReplacementRetryUnsupported means the task belongs to an operator-approved -// provider replacement, which resumes only through its own retry action so the -// replacement record and the task never disagree. It wraps ErrConflict. -var ErrReplacementRetryUnsupported = fmt.Errorf("provider replacement work cannot be retried from the task queue: %w", ErrConflict) +// ErrTaskLeaseLost reports that a claim generation is stale or its lease can +// no longer be proven valid. +var ErrTaskLeaseLost = errors.New("task lease lost") // ErrAlreadyCurrent is returned when a restore would not change the current object representation. var ErrAlreadyCurrent = errors.New("already current") diff --git a/internal/db/repository/helpers_test.go b/internal/db/repository/helpers_test.go index 1fef73d..7095096 100644 --- a/internal/db/repository/helpers_test.go +++ b/internal/db/repository/helpers_test.go @@ -3,11 +3,11 @@ package repository_test import ( "context" "database/sql" - "path/filepath" "testing" "github.com/strahe/synaps3/internal/db/migrations" "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/testutil" "github.com/strahe/synaps3/internal/types" "github.com/uptrace/bun" "github.com/uptrace/bun/dialect/sqlitedialect" @@ -40,38 +40,6 @@ func testDB(t *testing.T) *bun.DB { return db } -// concurrentTestDB creates a file-backed SQLite DB that allows overlapping connections. -func concurrentTestDB(t *testing.T) *bun.DB { - t.Helper() - - sqldb, err := sql.Open("sqlite", "file:"+filepath.Join(t.TempDir(), "repo.db")+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(1)") - if err != nil { - t.Fatalf("opening concurrent test db: %v", err) - } - sqldb.SetMaxOpenConns(8) - - db := bun.NewDB(sqldb, sqlitedialect.New()) - t.Cleanup(func() { _ = db.Close() }) - - ctx := context.Background() - migrator := migrations.NewMigrator(db) - if err := migrator.Init(ctx); err != nil { - t.Fatalf("init migrator: %v", err) - } - if _, err := migrator.Migrate(ctx); err != nil { - t.Fatalf("running migrations: %v", err) - } - - return db -} - -func mustExec(t *testing.T, db *bun.DB, query string, args ...interface{}) { - t.Helper() - if _, err := db.NewRaw(query, args...).Exec(context.Background()); err != nil { - t.Fatalf("exec %q: %v", query, err) - } -} - func onChainID(t *testing.T, value string) types.OnChainID { t.Helper() id, err := types.ParseOnChainID("test id", value) @@ -90,10 +58,11 @@ func onChainIDPtr(t *testing.T, value string) *types.OnChainID { // seedBucket inserts a bucket and returns it. func seedBucket(t *testing.T, db *bun.DB, name string) *model.Bucket { t.Helper() - bucket := &model.Bucket{Name: name, Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: name, Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8} _, err := db.NewInsert().Model(bucket).Exec(context.Background()) if err != nil { t.Fatalf("seeding bucket: %v", err) } + testutil.OpenBucketReplicaSlots(t, db, bucket.ID, bucket.DefaultCopies) return bucket } diff --git a/internal/db/repository/interfaces.go b/internal/db/repository/interfaces.go index 30431af..301b183 100644 --- a/internal/db/repository/interfaces.go +++ b/internal/db/repository/interfaces.go @@ -28,14 +28,21 @@ type BucketRepository interface { SoftDelete(ctx context.Context, id int64) error // UpdateStatus atomically transitions bucket status using CAS. UpdateStatus(ctx context.Context, id int64, from, to model.BucketStatus) error + // PromoteReadyIfProvisioned marks a provisioning bucket ready once enough current data sets are ready. + PromoteReadyIfProvisioned(ctx context.Context, id int64, requiredDataSets int) (bool, error) // SetACL stores the bucket ACL JSON blob used by VersityGW access control. SetACL(ctx context.Context, name string, acl []byte) error // SetOwnerAndACL stores both the authoritative owner and compatible ACL. SetOwnerAndACL(ctx context.Context, name string, ownerAccessKey *string, acl []byte) error // UpdateCopyPolicy locks and updates the independently optional bucket policy fields. UpdateCopyPolicy(ctx context.Context, input UpdateBucketCopyPolicyInput) (*model.Bucket, error) - // SetDefaultCopies stores the bucket target override. Nil means inherit. + // SetDefaultCopies stores the bucket replica target. Nil resets it to the + // caller-supplied configured default. SetDefaultCopies(ctx context.Context, name string, copies *int) error + // ActiveReplicaSlots returns the ascending copy indexes the bucket still + // accepts new writes on. Decommissioned slots keep their rows for history + // but never take a new data set. + ActiveReplicaSlots(ctx context.Context, bucketID int64) ([]int, error) // CountByOwner returns bucket count for the authoritative owner access key. CountByOwner(ctx context.Context, ownerAccessKey string) (int, error) // AggregateCountsByOwner returns bucket counts grouped by authoritative owner access key. @@ -97,32 +104,36 @@ type RecoverableDeleteMarker struct { type ObjectVersionRef struct { ObjectID int64 `bun:"object_id"` VersionID string `bun:"version_id"` + ContentID *int64 `bun:"content_id"` } type DeleteObjectVersionInput struct { - BucketID int64 - Key string - VersionID string - StorageCleanupMaxRetries *int + BucketID int64 + Key string + VersionID string +} + +type StorageCleanupReservation struct { + ContentID int64 + Generation int64 + TaskID *int64 } type DeleteObjectVersionResult struct { - DeletionID int64 - CacheKey string - StorageUploadID *int64 - StorageCleanupTaskID *int64 + DeletionID int64 + ContentID *int64 + StorageCleanup *StorageCleanupReservation } type DeleteDeletedObjectInput struct { - BucketID int64 - Key string - DeleteMarkerVersionID string - StorageCleanupMaxRetries *int + BucketID int64 + Key string + DeleteMarkerVersionID string } type DeletedObjectVersionSnapshot struct { VersionID string - CacheKey string + ContentID *int64 } type DeleteDeletedObjectResult struct { @@ -131,7 +142,7 @@ type DeleteDeletedObjectResult struct { DataVersionsDeleted int DeleteMarkersDeleted int DeletedVersions []DeletedObjectVersionSnapshot - StorageCleanupTaskIDs []int64 + StorageCleanups []StorageCleanupReservation } // ObjectRepository defines persistence operations for object identities and versions. @@ -145,7 +156,16 @@ type ObjectRepository interface { DeleteMarkerVersion(ctx context.Context, bucketID int64, key string, versionID string) error DeleteObjectVersionPermanently(ctx context.Context, input DeleteObjectVersionInput) (DeleteObjectVersionResult, error) DeleteDeletedObjectPermanently(ctx context.Context, input DeleteDeletedObjectInput) (DeleteDeletedObjectResult, error) - UpdateObjectDeletionCacheCleanup(ctx context.Context, versionID string, status model.CacheCleanupStatus, cacheError string) error + // ClearContentCachePresence records that a content payload no longer has + // cached bytes on this node. + ClearContentCachePresence(ctx context.Context, contentID int64) error + // ReleaseContentCacheIfUnreferenced locks the content row, rechecks live + // references, and invokes release before clearing cache presence in the + // same transaction. The callback must be idempotent. + ReleaseContentCacheIfUnreferenced(ctx context.Context, contentID int64, release func() error) (bool, error) + // ContentIsUnreferenced reports whether any live object version still + // points at the content. + ContentIsUnreferenced(ctx context.Context, contentID int64) (bool, error) // RestoreCurrentDeleteMarkerStack is the admin trash restore path: it removes // the current delete marker stack until the latest data version becomes current, // unlike S3 versioned delete which removes only one specified delete marker. @@ -156,28 +176,18 @@ type ObjectRepository interface { GetCurrentVersionByBucketAndKey(ctx context.Context, bucketID int64, key string) (*model.ObjectVersion, error) GetVersionByID(ctx context.Context, versionID string) (*model.ObjectVersion, error) GetVersionByBucketKeyAndID(ctx context.Context, bucketID int64, key string, versionID string) (*model.ObjectVersion, error) - FindReusableStoredVersion(ctx context.Context, bucketID int64, size int64, checksum string) (*model.ObjectVersion, error) - FindReusableReplicatingVersion(ctx context.Context, bucketID int64, size int64, checksum string) (*model.ObjectVersion, error) - FindReusableActiveUploadVersion(ctx context.Context, bucketID int64, size int64, checksum string) (*model.ObjectVersion, error) ListCurrentVersionsByBucket(ctx context.Context, bucketID int64, prefix string, afterKey string, maxKeys int) ([]model.ObjectVersion, error) ListCurrentVersionsByBucketAtOrAfter(ctx context.Context, bucketID int64, prefix string, fromKey string, maxKeys int) ([]model.ObjectVersion, error) ListVersionsByBucket(ctx context.Context, bucketID int64, prefix string, keyMarker string, versionIDMarker string, maxKeys int) ([]ObjectVersionListItem, error) ListVersionsByKey(ctx context.Context, bucketID int64, key string, afterVersionID string, maxKeys int) ([]ObjectVersionListItem, error) ListRecoverableDeleteMarkers(ctx context.Context, bucketID int64, prefix string, afterKey string, maxKeys int) ([]RecoverableDeleteMarker, error) - UpdateVersionState(ctx context.Context, versionID string, from, to model.ObjectState) error - UpdateVersionStateToFailed(ctx context.Context, versionID string, from model.ObjectState, lastError string) error SetVersionCachePresence(ctx context.Context, versionID string, inCache bool) error - // RecordVersionCacheAccess advances LRU recency without changing whether - // the local cache file is present. - RecordVersionCacheAccess(ctx context.Context, versionID string, accessedAt time.Time) error - // RecordVersionCacheCommit marks a newly committed local cache file present - // and initializes or advances its LRU recency. - RecordVersionCacheCommit(ctx context.Context, versionID string, accessedAt time.Time) error - SetVersionStorageUploadAndTransition(ctx context.Context, versionID string, storageUploadID int64, from, to model.ObjectState) error - FailUploadingContentFollowers(ctx context.Context, bucketID int64, size int64, checksum string, leaderVersionID string, lastError string) ([]ObjectVersionRef, error) - ListVersionsByState(ctx context.Context, state model.ObjectState, limit int) ([]model.ObjectVersion, error) - ListVersionsByStateAfter(ctx context.Context, state model.ObjectState, afterUpdatedAt time.Time, afterVersionID string, limit int) ([]model.ObjectVersion, error) - ResetStaleVersionStates(ctx context.Context, fromState, toState model.ObjectState, staleBefore time.Time) (int, error) + // RecordContentCacheAccess advances LRU recency for one content payload + // without changing whether its bytes are present. + RecordContentCacheAccess(ctx context.Context, contentID int64, accessedAt time.Time) error + // RecordContentCacheCommit marks a newly written cache file present and + // advances its recency in the same write. + RecordContentCacheCommit(ctx context.Context, contentID int64, accessedAt time.Time) error // CountByState returns object counts grouped by state. CountByState(ctx context.Context) ([]ObjectStateCount, error) // AggregateByState returns object counts and sizes grouped by state. @@ -193,35 +203,34 @@ type ObjectRepository interface { AggregateByBucket(ctx context.Context) (map[int64]BucketObjectStats, error) } -type StartObjectUploadAttemptInput struct { +type EnsureContentInput struct { BucketID int64 - SourceTaskID int64 - SourceVersionID string ContentSize int64 Checksum string RequestedCopies int } -type AppendUploadFailureInput struct { - UploadID int64 - CopyIndex int - ProviderID *types.OnChainID - TransferMethod string - Stage string - ErrorMessage string - Explicit bool +type BeginIngressStoreProgressInput struct { + CopyID int64 + Generation int64 + TaskID int64 + Attempt int } type RecordIngressStoreProgressInput struct { - UploadID int64 + CopyID int64 + Generation int64 + TaskID int64 Attempt int BytesUploaded int64 } -type StorageUploadProvenance struct { - Upload model.StorageUpload - Copies []model.StorageUploadCopy - Failures []model.StorageUploadFailure +type StorageContentProvenance struct { + Upload model.StorageContent + Copies []model.StorageCopy + // IngressCopy is the copy that performed the ingress transfer, when one + // exists. Progress belongs to that transfer rather than to the content. + IngressCopy *model.StorageCopy } type StorageDataSetSummary struct { @@ -229,14 +238,14 @@ type StorageDataSetSummary struct { BucketID int64 `bun:"bucket_id"` BucketName string `bun:"bucket_name"` CopyIndex int `bun:"copy_index"` - Generation int `bun:"generation"` + Generation int64 `bun:"generation"` IsCurrent bool `bun:"is_current"` ProviderID types.OnChainID `bun:"provider_id"` DataSetID *types.OnChainID `bun:"data_set_id"` ClientDataSetID *types.OnChainID `bun:"client_data_set_id"` Status model.StorageDataSetStatus `bun:"status"` - CreatedByUploadID *int64 `bun:"created_by_upload_id"` - LastUsedUploadID *int64 `bun:"last_used_upload_id"` + CreatedByContentID *int64 `bun:"created_by_content_id"` + LastUsedContentID *int64 `bun:"last_used_content_id"` CommittedCopies int64 `bun:"committed_copies"` ReadableCopies int64 `bun:"readable_copies"` PhysicalBytes int64 `bun:"physical_bytes"` @@ -247,7 +256,7 @@ type StorageDataSetSummary struct { } type ReadableStorageCopy struct { - UploadID int64 `bun:"upload_id"` + ContentID int64 `bun:"content_id"` PieceCID string `bun:"piece_cid"` CopyIndex int `bun:"copy_index"` ProviderID types.OnChainID `bun:"provider_id"` @@ -316,25 +325,27 @@ type BucketStorageHealthRiskDataSet struct { } type StorageCleanupRepository interface { - ListCopiesForTask(ctx context.Context, taskID int64) ([]model.StorageCleanupCopy, error) + BindTask(ctx context.Context, contentID, generation, taskID int64) error + AuthorizeTask(ctx context.Context, contentID, generation, taskID int64) ([]model.StorageCleanupCopy, error) MarkCopyRemoved(ctx context.Context, id int64) error MarkCopyDeleteScheduled(ctx context.Context, id int64, txHash string) error + MarkCopyFailed(ctx context.Context, id int64, message string) error MarkCopyUnsupported(ctx context.Context, id int64, message string) error - UploadHasObjectReferences(ctx context.Context, uploadID int64) (bool, error) - TaskHasObjectReferences(ctx context.Context, taskID int64, uploadID int64) (bool, error) - DeleteUploadProvenanceIfUnreferenced(ctx context.Context, uploadID int64) error + UploadHasObjectReferences(ctx context.Context, contentID int64) (bool, error) + CleanupHasObjectReferences(ctx context.Context, contentID int64) (bool, error) + CompleteTask(ctx context.Context, contentID, generation, taskID int64) error } type EnsureDataSetBindingInput struct { - BucketID int64 - ProviderID types.OnChainID - CopyIndex int - CreatedByUploadID int64 + BucketID int64 + ProviderID types.OnChainID + CopyIndex int + CreatedByContentID int64 } type MarkDataSetCreatingInput struct { ID int64 - UploadID int64 + ContentID int64 TransactionID string StatusURL string ClientDataSetID *types.OnChainID @@ -342,7 +353,7 @@ type MarkDataSetCreatingInput struct { type MarkDataSetReadyInput struct { ID int64 - UploadID int64 + ContentID int64 DataSetID types.OnChainID ClientDataSetID *types.OnChainID } @@ -360,7 +371,23 @@ type UploadCopyBindingInput struct { ProviderID types.OnChainID } -// StorageUploadCopyID names the exact copy row to write. A task that was +// ReservePullRequestInput records one provider-side copy request before it is +// sent. AttemptID is the ledger row's identity and SourcePieceCID names the +// piece being fetched, which is not the same as the content's own CID. +type ReservePullRequestInput struct { + CopyID int64 + Generation int64 + TaskID int64 + AttemptID string + SourceProviderID types.OnChainID + SourceDataSetID types.OnChainID + SourcePieceID types.OnChainID + SourcePieceCID string + SourceRetrievalURL string + CommitExtraDataHex string +} + +// StorageCopyID names the exact copy row to write. A task that was // queued before the current data set generation took over must still land on // the generation it actually stored to, so addressing is separate from the // eligibility guards below. @@ -369,14 +396,18 @@ type UploadCopyBindingInput struct { // or a copy that no longer matches, instead of reporting no rows. Coordinators // that own one specific copy set it; ordinary upload stages stay idempotent. type MarkUploadCopyPieceReadyInput struct { - StorageUploadCopyID int64 + StorageCopyID int64 RequireEligibleCopy bool - UploadID int64 + ContentID int64 CopyIndex int PieceCID string PieceID *types.OnChainID RetrievalURL string CommitExtraDataHex string + // PullAttemptID resolves the pull ledger row in the same transaction that + // records the piece. Leaving it empty is how an ingress store settles, since + // no request was sent to a source provider. + PullAttemptID string } // MarkUploadCopyFailedInput names the copy that failed. Without the id the @@ -385,16 +416,21 @@ type MarkUploadCopyPieceReadyInput struct { // either land on the replacement's copy or update nothing at all, leaving the // original stuck mid-transfer and holding retirement open. type MarkUploadCopyFailedInput struct { - StorageUploadCopyID int64 - UploadID int64 - CopyIndex int - LastError string + StorageCopyID int64 + ContentID int64 + CopyIndex int + LastError string + // PullAttemptID abandons the unresolved provider pull in the same + // transaction as the copy failure. Empty means this was not a pull failure. + PullAttemptID string } +// MarkUploadCopyCommittedInput requires CommitAttemptID: a committed copy is a +// projection of a confirmed ledger row, and the schema refuses one without it. type MarkUploadCopyCommittedInput struct { - StorageUploadCopyID int64 + StorageCopyID int64 RequireEligibleCopy bool - UploadID int64 + ContentID int64 CopyIndex int PieceCID string PieceID *types.OnChainID @@ -405,92 +441,47 @@ type MarkUploadCopyCommittedInput struct { CommitConfirmedTransactionID string } -// AcquireReplicaRepairItemInput identifies the exact repair work already -// parsed and claimed by the upload worker. -type AcquireReplicaRepairItemInput struct { - TaskID int64 - TaskClaimedAt time.Time - StorageDataSetID int64 - StorageUploadCopyID int64 - BucketID int64 -} - -// AcquireUploadTaskInput identifies one claimed ordinary upload task and its -// optional existing storage upload. -type AcquireUploadTaskInput struct { - TaskID int64 - TaskClaimedAt time.Time - UploadID int64 - VersionID string -} - -// ReplicaRepairItem is the consistent database snapshot authorized for one -// replica repair attempt. -type ReplicaRepairItem struct { - DataSet model.StorageDataSet - Copy model.StorageUploadCopy - Upload model.StorageUpload - Version model.ObjectVersion -} - // IncompleteReadableUpload identifies one durable upload that still needs // work to reach its frozen target copy count. type IncompleteReadableUpload struct { - Upload model.StorageUpload + Upload model.StorageContent Version model.ObjectVersion } type BindReadableUploadInput struct { - UploadID int64 - BucketID int64 - ContentSize int64 - Checksum string + ContentID int64 + BucketID int64 } type BindReadableUploadForVersionInput struct { - UploadID int64 - BucketID int64 - ContentSize int64 - Checksum string - VersionID string + ContentID int64 + BucketID int64 + VersionID string } type FinalizeUploadInput struct { - UploadID int64 - EnqueueAfterUploadEviction bool - EvictionMaxRetries int -} - -// NewFinalizeUploadInput builds the shared upload-finalization contract used -// by every upload completion path. -func NewFinalizeUploadInput( - uploadID int64, - enqueueAfterUploadEviction bool, - evictionMaxRetries int, -) FinalizeUploadInput { - return FinalizeUploadInput{ - UploadID: uploadID, - EnqueueAfterUploadEviction: enqueueAfterUploadEviction, - EvictionMaxRetries: evictionMaxRetries, - } -} - -type StorageUploadRepository interface { - StartObjectUploadAttempt(ctx context.Context, input StartObjectUploadAttemptInput) (*model.StorageUpload, error) - FindActiveUploadBySourceVersion(ctx context.Context, versionID string) (*model.StorageUpload, error) - FindLatestUploadBySourceVersion(ctx context.Context, versionID string) (*model.StorageUpload, error) - FindLatestUploadsBySourceVersions(ctx context.Context, versionIDs []string) (map[string]model.StorageUpload, error) - SetAcceptError(ctx context.Context, uploadID int64, message string) error - GetByID(ctx context.Context, uploadID int64) (*model.StorageUpload, error) - GetByIDs(ctx context.Context, uploadIDs []int64) (map[int64]model.StorageUpload, error) - BeginIngressStoreProgress(ctx context.Context, uploadID int64) (*model.StorageUpload, error) - RecordIngressStoreProgress(ctx context.Context, input RecordIngressStoreProgressInput) (*model.StorageUpload, error) - GetUploadProvenance(ctx context.Context, uploadID int64) (*StorageUploadProvenance, error) - AppendUploadFailure(ctx context.Context, input AppendUploadFailureInput) error - ListCopies(ctx context.Context, uploadID int64) ([]model.StorageUploadCopy, error) - CountCurrentGenerationCopySlots(ctx context.Context, uploadID int64) (int, error) - ListReadableCommittedCopies(ctx context.Context, uploadID int64) ([]ReadableStorageCopy, error) - HasReadableCommittedCopy(ctx context.Context, uploadID int64) (bool, error) + ContentID int64 +} + +func NewFinalizeUploadInput(contentID int64) FinalizeUploadInput { + return FinalizeUploadInput{ContentID: contentID} +} + +type StorageContentRepository interface { + EnsureContent(ctx context.Context, input EnsureContentInput) (*model.StorageContent, error) + GetByID(ctx context.Context, contentID int64) (*model.StorageContent, error) + GetByIDs(ctx context.Context, contentIDs []int64) (map[int64]model.StorageContent, error) + RecordContentFailure(ctx context.Context, contentID int64, message string) error + GetIngressCopy(ctx context.Context, contentID int64) (*model.StorageCopy, error) + // ContentPipelineState derives pipeline position from the copy rows. + ContentPipelineState(ctx context.Context, contentID int64) (model.ObjectState, error) + BeginIngressStoreProgress(ctx context.Context, input BeginIngressStoreProgressInput) (*model.StorageCopy, error) + RecordIngressStoreProgress(ctx context.Context, input RecordIngressStoreProgressInput) (*model.StorageCopy, error) + GetUploadProvenance(ctx context.Context, contentID int64) (*StorageContentProvenance, error) + ListCopies(ctx context.Context, contentID int64) ([]model.StorageCopy, error) + CountCurrentGenerationCopySlots(ctx context.Context, contentID int64) (int, error) + ListReadableCommittedCopies(ctx context.Context, contentID int64) ([]ReadableStorageCopy, error) + HasReadableCommittedCopy(ctx context.Context, contentID int64) (bool, error) ListBucketStorageHealthSummaries(ctx context.Context, bucketID int64, staleBefore time.Time, affectedVersionCap int) ([]BucketStorageHealthSummary, error) ListBucketStorageHealthAffectedVersions(ctx context.Context, input BucketStorageHealthAffectedVersionsInput) (BucketStorageHealthAffectedVersionPage, error) ListDataSetBindings(ctx context.Context, bucketID int64) ([]model.StorageDataSet, error) @@ -501,24 +492,36 @@ type StorageUploadRepository interface { MarkDataSetCreating(ctx context.Context, input MarkDataSetCreatingInput) error MarkDataSetReady(ctx context.Context, input MarkDataSetReadyInput) error BackfillClientDataSetID(ctx context.Context, input BackfillClientDataSetIDInput) error - RecoverDataSet(ctx context.Context, input MarkDataSetReadyInput) (bool, error) MarkDataSetDraining(ctx context.Context, id int64, lastError string) error MarkDataSetFailed(ctx context.Context, id int64, lastError string) error - MarkDataSetUnavailable(ctx context.Context, id int64, lastError string) error - DiscardFailedDataSetCandidate(ctx context.Context, uploadID int64, copyIndex int, storageDataSetID int64) (bool, error) - CreateUploadCopiesForBindings(ctx context.Context, uploadID int64, copies []UploadCopyBindingInput) error - GetUploadCopy(ctx context.Context, uploadID int64, copyIndex int) (*model.StorageUploadCopy, error) - GetUploadCopyByID(ctx context.Context, id int64) (*model.StorageUploadCopy, error) + // RetireRejectedDataSet ends a failed generation whose creation the chain + // refused, so its provider stops being reserved. The caller must have proof + // that no data set was created. Reports whether it was retired. + RetireRejectedDataSet(ctx context.Context, storageDataSetID int64) (bool, error) + CreateUploadCopiesForBindings(ctx context.Context, contentID int64, copies []UploadCopyBindingInput) error + GetUploadCopy(ctx context.Context, contentID int64, copyIndex int) (*model.StorageCopy, error) + GetUploadCopyByID(ctx context.Context, id int64) (*model.StorageCopy, error) + GetLiveVersionForUpload(ctx context.Context, contentID int64) (*model.ObjectVersion, error) + ListIncompleteCopiesForDataSet(ctx context.Context, storageDataSetID int64) ([]model.StorageCopy, error) + BindDataSetEnsureTask(ctx context.Context, dataSetID, taskID int64) error + AuthorizeDataSetEnsureTask(ctx context.Context, dataSetID, taskID int64) (*model.StorageDataSet, error) + CompleteDataSetEnsureTask(ctx context.Context, dataSetID, taskID int64) error + NextCopyWorkGeneration(ctx context.Context, copyID int64) (int64, error) + BindCopyTask(ctx context.Context, copyID, generation, taskID int64) error + AuthorizeCopyTask(ctx context.Context, copyID, generation, taskID, claimGeneration int64) (*model.StorageCopy, error) + ReservePullRequest(ctx context.Context, input ReservePullRequestInput) error + ReplaceCopyTask(ctx context.Context, copyID, generation, taskID, nextGeneration, nextTaskID int64) error + CompleteCopyTask(ctx context.Context, copyID, generation, taskID int64) error + NextDataSetRetirementGeneration(ctx context.Context, dataSetID int64) (int64, error) + BindDataSetRetirementTask(ctx context.Context, dataSetID, generation, taskID int64) error + AuthorizeDataSetRetirementTask(ctx context.Context, dataSetID, generation, taskID int64) (*model.StorageDataSet, error) + CompleteDataSetRetirementTask(ctx context.Context, dataSetID, generation, taskID int64) error // GetUploadCopyForDataSet addresses one concrete data set generation. - GetUploadCopyForDataSet(ctx context.Context, uploadID, storageDataSetID int64) (*model.StorageUploadCopy, error) - AcquireUploadTask(ctx context.Context, input AcquireUploadTaskInput) error - AcquireReplicaRepairItem(ctx context.Context, input AcquireReplicaRepairItemInput) (*ReplicaRepairItem, error) - NextIncompleteCopyForDataSet(ctx context.Context, storageDataSetID int64) (*model.StorageUploadCopy, error) - NextFinalizableCopyForDataSet(ctx context.Context, storageDataSetID int64) (*model.StorageUploadCopy, error) - ListUnavailableDataSetsWithIncompleteCopies(ctx context.Context, afterID int64, limit int) ([]model.StorageDataSet, error) + GetUploadCopyForDataSet(ctx context.Context, contentID, storageDataSetID int64) (*model.StorageCopy, error) + NextFinalizableCopyForDataSet(ctx context.Context, storageDataSetID int64) (*model.StorageCopy, error) ListIncompleteReadableUploads(ctx context.Context, afterID int64, limit int) ([]IncompleteReadableUpload, error) - ReassignIngressCopy(ctx context.Context, uploadID int64, unavailableCopyIndex int) (*model.StorageUploadCopy, error) MarkUploadCopyPieceReady(ctx context.Context, input MarkUploadCopyPieceReadyInput) error + ReopenFailedUploadCopy(ctx context.Context, copyID int64) error ReserveCommitAttempt(ctx context.Context, input storagecommit.ReserveInput) (storagecommit.ReserveResult, error) MarkCommitAttempted(ctx context.Context, input storagecommit.AttemptInput) (storagecommit.AttemptResult, error) RecordCommitTransaction(ctx context.Context, input storagecommit.EvidenceInput) error @@ -562,14 +565,18 @@ type StorageReplacementRepository interface { // Retry resumes failed or cleanup-attention work on the same approved // target. Choosing a different provider requires a new authorization. Retry(ctx context.Context, input RetryReplacementInput) (*storagereplacement.Replacement, error) + BindTask(ctx context.Context, replacementID, generation, taskID int64) error + AuthorizeTask(ctx context.Context, replacementID, generation, taskID int64) (*storagereplacement.Replacement, error) + CompleteTask(ctx context.Context, replacementID, generation, taskID int64) error GetByID(ctx context.Context, id int64) (*storagereplacement.Replacement, error) GetByClientRequestID(ctx context.Context, bucketID int64, clientRequestID string) (*storagereplacement.Replacement, error) ListForBucket(ctx context.Context, bucketID int64, limit int) ([]storagereplacement.Replacement, error) GetActiveForDataSet(ctx context.Context, dataSetID int64) (*storagereplacement.Replacement, error) // HasInProgressForDataSet reports whether recovery must leave this data set - // alone. Terminally failed work does not count, so a stuck slot can still - // repair in place. + // alone. Operator-paused failed or attention work does not count as actively + // progressing, so a stuck slot can still repair in place; its replacement + // identity remains reserved until retry, supersession, or completion. HasInProgressForDataSet(ctx context.Context, dataSetID int64) (bool, error) ListActive(ctx context.Context, afterID int64, limit int) ([]storagereplacement.Replacement, error) ListSupersededCleanupCandidates(ctx context.Context, afterID int64, limit int) ([]storagereplacement.Replacement, error) @@ -578,40 +585,24 @@ type StorageReplacementRepository interface { // in one transaction. It touches a fixed number of rows regardless of how // much history the bucket holds. Activate(ctx context.Context, replacementID int64) error - // SeedMigrationBatchWithBudget inserts one bounded batch of migration work, - // snapshots its retry budget, and advances the cursor. - SeedMigrationBatchWithBudget(ctx context.Context, replacementID int64, limit, maxRetries int) (inserted int, done bool, err error) - InitializeReplacementItemRetryBudgets(ctx context.Context, maxRetries int) (int, error) - ReleaseExpiredItemLeases(ctx context.Context) (int, error) - ClaimReadyReplacementItem(ctx context.Context, leaseTTL time.Duration) (*storagereplacement.Item, error) - RenewReplacementItemLease(ctx context.Context, token storagereplacement.ClaimToken, leaseTTL time.Duration) error - ReleaseReplacementItemClaim(ctx context.Context, token storagereplacement.ClaimToken) error - CancelReplacementItemClaim(ctx context.Context, token storagereplacement.ClaimToken) error - CompleteReplacementItemClaim(ctx context.Context, token storagereplacement.ClaimToken) error - WaitReplacementItemClaim(ctx context.Context, token storagereplacement.ClaimToken, nextCheck time.Time, lastError string) error - DeferReplacementItemClaim(ctx context.Context, token storagereplacement.ClaimToken, nextAttempt time.Time) error - RetryReplacementItemClaim(ctx context.Context, token storagereplacement.ClaimToken, nextAttempt time.Time, lastError string) (storagereplacement.ItemStatus, error) - PauseMigration(ctx context.Context, replacementID, expectedStateVersion int64, reason storagereplacement.WaitReason) error + SeedMigrationBatch(ctx context.Context, replacementID int64, limit int) (inserted int, done bool, err error) + NextPendingReplacementItem(ctx context.Context, replacementID int64) (*storagereplacement.Item, error) + MarkReplacementItemCopied(ctx context.Context, replacementID, itemID, targetCopyID int64) error + MarkReplacementItemCancelled(ctx context.Context, replacementID, itemID int64) error + MarkReplacementItemAttention(ctx context.Context, replacementID, itemID int64, lastError string) error ReplacementExecution(ctx context.Context, replacementID int64) (storagereplacement.ExecutionSnapshot, error) ReplacementProgresses(ctx context.Context, replacementIDs []int64) (map[int64]storagereplacement.ProgressSnapshot, error) - RunningReplacementItemClaimForUpload(ctx context.Context, replacementID, uploadID int64) (*storagereplacement.ClaimToken, error) - // AcquireItem re-derives a consistent snapshot and revalidates the worker - // claim. No provider call may start before it returns. AcquireItem(ctx context.Context, input AcquireReplacementItemInput) (*ReplacementItemSnapshot, error) - AttachTargetCopy(ctx context.Context, input AttachReplacementTargetCopyInput) (*model.StorageUploadCopy, error) + AttachTargetCopy(ctx context.Context, input AttachReplacementTargetCopyInput) (*model.StorageCopy, error) MarkMigrating(ctx context.Context, replacementID int64) error MarkWaiting(ctx context.Context, replacementID int64, reason storagereplacement.WaitReason) error MarkFailed(ctx context.Context, replacementID int64, reason *storagereplacement.FailureReason, lastError string) error - // FailCoordinator fails a replacement and its running coordinator atomically. - FailCoordinator(ctx context.Context, input ReplacementCoordinatorFailureInput) error - // ScheduleCoordinatorRetry updates the replacement and coordinator atomically. - ScheduleCoordinatorRetry(ctx context.Context, input ReplacementCoordinatorRetryInput) (model.TaskStatus, error) MarkCleanupAttention(ctx context.Context, replacementID int64, lastError string) error BeginRetirement(ctx context.Context, replacementID int64) error RecordTerminationEpoch(ctx context.Context, input RecordTerminationEpochInput) error RecordAbandonedTerminationEpoch(ctx context.Context, input RecordTerminationEpochInput) error - CompleteAbandonedTargetTermination(ctx context.Context, replacementID int64, observedAt time.Time) error + CompleteAbandonedTargetTermination(ctx context.Context, replacementID int64) error // EvaluateRetirementGate reports every blocker by name so the API and UI can // explain why a source is still held. EvaluateRetirementGate(ctx context.Context, replacementID int64, observedEpoch *int64) (RetirementGate, error) @@ -633,33 +624,15 @@ type AuthorizeReplacementInput struct { SelectionMode storagereplacement.SelectionMode TargetProviderID types.OnChainID ClientRequestID string - MaxRetries int } type RetryReplacementInput struct { - ReplacementID int64 - MaxRetries int - ItemMaxRetries int -} - -type ReplacementCoordinatorFailureInput struct { - ReplacementID int64 - Task *model.Task - FailureReason *storagereplacement.FailureReason - LastError string -} - -type ReplacementCoordinatorRetryInput struct { ReplacementID int64 - Task *model.Task - LastError string - Backoff time.Duration } type AcquireReplacementItemInput struct { ReplacementID int64 ItemID int64 - ItemClaimedAt time.Time } // ReplacementItemSnapshot is the consistent view one migration item needs. @@ -668,15 +641,14 @@ type ReplacementItemSnapshot struct { Item storagereplacement.Item Source model.StorageDataSet Target model.StorageDataSet - Upload model.StorageUpload + Upload model.StorageContent Version model.ObjectVersion } type AttachReplacementTargetCopyInput struct { ReplacementID int64 ItemID int64 - UploadID int64 - ItemClaimedAt time.Time + ContentID int64 } type RecordTerminationEpochInput struct { @@ -704,80 +676,66 @@ func (g RetirementGate) Passed() bool { return len(g.Blockers) == 0 } // TaskRepository defines persistence operations for Task entities. type TaskRepository interface { - Create(ctx context.Context, task *model.Task) error - // EnsureRecurring creates a singleton coordinator task or reactivates its - // completed row with the supplied payload. Active, failed, exhausted, and - // cancelled rows are left unchanged. - EnsureRecurring(ctx context.Context, task *model.Task) (bool, error) - // ResumeCoordinator revives a singleton coordinator on an operator's - // request, including one that exhausted its retries or failed. - ResumeCoordinator(ctx context.Context, task *model.Task) (bool, error) + Enqueue(ctx context.Context, task *model.Task) (*model.Task, bool, error) GetByID(ctx context.Context, id int64) (*model.Task, error) - GetByIdempotencyKey(ctx context.Context, idempotencyKey string) (*model.Task, error) - HasActiveByIdempotencyKey(ctx context.Context, idempotencyKey string) (bool, error) - HasEarlierRunningUploadCopyTask(ctx context.Context, claimedTask *model.Task, uploadID int64, copyIndex int) (bool, error) - HasEarlierRunningUploadCopyClaim(ctx context.Context, claimedAt time.Time, uploadID int64, copyIndex int) (bool, error) - - // ClaimReady atomically claims one ready task of the given type by - // transitioning it to running and setting a lease. Returns nil if no task is available. - ClaimReady(ctx context.Context, taskType model.TaskType, leaseDuration time.Duration) (*model.Task, error) - // RenewLease extends the same running task claim. - RenewLease(ctx context.Context, task *model.Task, leaseDuration time.Duration) error - // Complete marks the same running task claim as completed. - Complete(ctx context.Context, task *model.Task) error - // CompleteWithMessage marks the same running task claim as completed with a retained status message. - CompleteWithMessage(ctx context.Context, task *model.Task, message string) error - // FailRunning marks the same running task claim as non-retryably failed. - FailRunning(ctx context.Context, task *model.Task, lastError string) error - // ScheduleRetryRunning records a retryable failure for the same running claim - // and returns the resulting task status. - ScheduleRetryRunning(ctx context.Context, task *model.Task, lastError string, backoff time.Duration) (model.TaskStatus, error) - // WaitRunning records a non-error wait and releases the running task until scheduled_at. - WaitRunning(ctx context.Context, task *model.Task, reason model.TaskWaitReason, message string, delay time.Duration) error - // LockRunningClaim locks the same running task claim for a cross-repository - // transaction that must decide whether to continue or complete it. - LockRunningClaim(ctx context.Context, task *model.Task) error - // ContinueRunning completes one successful coordinator item by replacing - // its version reference and payload, then returning the same task row to the queue tail. - ContinueRunning(ctx context.Context, task *model.Task, refVersionID string, payload map[string]interface{}) error - // ReleaseRunning releases the same running task claim back to queued without recording an error. - ReleaseRunning(ctx context.Context, task *model.Task) error - // CancelRunning marks the same running task claim as cancelled. - CancelRunning(ctx context.Context, task *model.Task, message string) error - // ReleaseExpiredLeases resets running tasks whose lease has expired back to queued. - ReleaseExpiredLeases(ctx context.Context) (int, error) - // MarkRunningExhausted marks the same running task claim as exhausted. - MarkRunningExhausted(ctx context.Context, task *model.Task, lastError string) error - // ListExhausted returns exhausted tasks, ordered by most recent first. - ListExhausted(ctx context.Context, limit int) ([]model.Task, error) - // RetryExhausted resets an exhausted task back to queued for manual retry. - RetryExhausted(ctx context.Context, taskID int64) error - // CountByStatus returns task counts grouped by type and status. + GetByIdentity(ctx context.Context, taskType model.TaskType, idempotencyKey string) (*model.Task, error) + ClaimNext(ctx context.Context, leaseDuration time.Duration) (*model.Task, error) + RenewLease(ctx context.Context, id, generation int64, leaseDuration time.Duration) (time.Time, error) + WriteCheckpoint(ctx context.Context, id, generation int64, checkpoint []byte) error + ValidateClaim(ctx context.Context, id, generation int64) error + Settle(ctx context.Context, id, generation int64, transition TaskTransition) error + ShortenLease(ctx context.Context, id, generation int64, duration time.Duration) error + WakePending(ctx context.Context, ids []int64) (int, error) + RequestCancellation(ctx context.Context, id int64, reason string) error + RetryFailed(ctx context.Context, id int64) error + ReactivateTerminal(ctx context.Context, id int64) error + AcknowledgeFailed(ctx context.Context, id int64, retention time.Duration) error + DeleteRetained(ctx context.Context, now time.Time, limit int) (int, error) + List(ctx context.Context, filter TaskListFilter) (TaskPage, error) CountByStatus(ctx context.Context) ([]TaskStatusCount, error) + CountByPresentationStatus(ctx context.Context) ([]TaskStatusCount, error) + CountUnacknowledgedFailed(ctx context.Context) (int64, error) CountOverviewActivePipeline(ctx context.Context) ([]TaskPipelineCount, error) - // CountActiveObjectTasksByBucket returns active object tasks - // whose referenced current object belongs to the given bucket. CountActiveObjectTasksByBucket(ctx context.Context, bucketID int64) (int64, error) - // CountActiveBucketTasksByBucketID returns the number of active tasks - // that directly reference the given bucket (ref_type=bucket, ref_id=bucketID). CountActiveBucketTasksByBucketID(ctx context.Context, bucketID int64) (int64, error) - // CompleteByRef marks all active tasks matching the given ref as completed. - CompleteByRef(ctx context.Context, refType string, refID int64, taskType model.TaskType) error - // List returns tasks with optional filters, paginated by offset/limit. - // Returns the matching tasks and the total count (for pagination). - List(ctx context.Context, taskType string, stage string, status string, limit, offset int) ([]model.Task, int, error) +} + +type TaskTransition struct { + Status model.TaskStatus + ResumeMode model.TaskResumeMode + AvailableAt time.Time + WaitReason *string + FailureReason *string + LastError *string + StatusMessage *string + IncrementRetry bool + RetentionUntil *time.Time +} + +type TaskListFilter struct { + Type model.TaskType + Status model.TaskStatus + Acknowledged *bool + BeforeID int64 + Limit int + HideHealthyRecurringSystem bool +} + +type TaskPage struct { + Tasks []model.Task + NextBeforeID int64 } type WalletOperationRepository interface { CreateOrGet(ctx context.Context, input CreateWalletOperationInput) (*model.WalletOperation, bool, error) GetByID(ctx context.Context, id int64) (*model.WalletOperation, error) - ClaimPending(ctx context.Context, leaseDuration time.Duration) (*model.WalletOperation, error) - MarkSubmitted(ctx context.Context, id int64, txHash string) error - MarkConfirmed(ctx context.Context, id int64) error - MarkConfirmedWithoutTransaction(ctx context.Context, id int64) error - MarkFailed(ctx context.Context, id int64, lastError string) error - MarkExpiredRunningUnknown(ctx context.Context) ([]model.WalletOperation, error) - ListSubmitted(ctx context.Context, limit int) ([]model.WalletOperation, error) + BindTask(ctx context.Context, id, taskID int64) error + MarkBroadcastAttempted(ctx context.Context, id, taskID int64) error + MarkSubmitted(ctx context.Context, id, taskID int64, txHash string) error + MarkConfirmed(ctx context.Context, id, taskID int64, txHash string) error + MarkConfirmedWithoutTransaction(ctx context.Context, id, taskID int64) error + MarkFailed(ctx context.Context, id, taskID int64, lastError string) error + MarkUnknown(ctx context.Context, id, taskID int64, lastError string) error ListRecent(ctx context.Context, limit int) ([]model.WalletOperation, error) } @@ -836,16 +794,16 @@ type BucketStatusCount struct { type MultipartUploadRepository interface { Create(ctx context.Context, upload *model.MultipartUpload) error GetByUploadID(ctx context.Context, uploadID string) (*model.MultipartUpload, error) - ListByBucket(ctx context.Context, bucketID int64, prefix, keyMarker, uploadIDMarker string, maxUploads int) ([]model.MultipartUpload, error) + ListByBucket(ctx context.Context, bucketID int64, prefix, keyMarker, contentIDMarker string, maxUploads int) ([]model.MultipartUpload, error) // CountActiveByBucket returns initiated/completing multipart uploads for the given bucket. CountActiveByBucket(ctx context.Context, bucketID int64) (int64, error) // SetStatus atomically transitions status using CAS (compare-and-swap) to prevent races. - SetStatus(ctx context.Context, uploadID string, from, to model.MultipartStatus) error - Delete(ctx context.Context, uploadID string) error + SetStatus(ctx context.Context, contentID string, from, to model.MultipartStatus) error + Delete(ctx context.Context, contentID string) error // Part operations CreatePart(ctx context.Context, part *model.MultipartPart) error - GetParts(ctx context.Context, uploadID string, partNumberMarker, maxParts int) ([]model.MultipartPart, error) - GetPartsByNumbers(ctx context.Context, uploadID string, numbers []int) ([]model.MultipartPart, error) - DeleteParts(ctx context.Context, uploadID string) error + GetParts(ctx context.Context, contentID string, partNumberMarker, maxParts int) ([]model.MultipartPart, error) + GetPartsByNumbers(ctx context.Context, contentID string, numbers []int) ([]model.MultipartPart, error) + DeleteParts(ctx context.Context, contentID string) error } diff --git a/internal/db/repository/multipart_repo.go b/internal/db/repository/multipart_repo.go index 3c9d768..01e8f7a 100644 --- a/internal/db/repository/multipart_repo.go +++ b/internal/db/repository/multipart_repo.go @@ -18,6 +18,12 @@ type BunMultipartRepo struct { var _ MultipartUploadRepository = (*BunMultipartRepo)(nil) func (r *BunMultipartRepo) Create(ctx context.Context, upload *model.MultipartUpload) error { + if upload == nil { + return fmt.Errorf("inserting multipart upload: %w", ErrInvalidInput) + } + if upload.Metadata == nil { + upload.Metadata = map[string]string{} + } _, err := r.db.NewInsert().Model(upload).Exec(ctx) if err != nil { return fmt.Errorf("inserting multipart upload: %w", err) @@ -119,7 +125,9 @@ func (r *BunMultipartRepo) Delete(ctx context.Context, uploadID string) error { return nil } -// CreatePart upserts a part record. If the same (upload_id, part_number) exists, it is replaced. +// CreatePart upserts a part record. If the same (upload_id, part_number) exists, +// its bytes are replaced but created_at keeps the first upload's time: a +// re-uploaded part is the same part, not a new one. func (r *BunMultipartRepo) CreatePart(ctx context.Context, part *model.MultipartPart) error { _, err := r.db.NewInsert(). Model(part). @@ -127,7 +135,6 @@ func (r *BunMultipartRepo) CreatePart(ctx context.Context, part *model.Multipart Set("size = EXCLUDED.size"). Set("e_tag = EXCLUDED.e_tag"). Set("checksum = EXCLUDED.checksum"). - Set("created_at = EXCLUDED.created_at"). Exec(ctx) if err != nil { return fmt.Errorf("upserting multipart part: %w", err) diff --git a/internal/db/repository/multipart_repo_test.go b/internal/db/repository/multipart_repo_test.go index c849a1c..e41f17b 100644 --- a/internal/db/repository/multipart_repo_test.go +++ b/internal/db/repository/multipart_repo_test.go @@ -35,6 +35,9 @@ func TestMultipartRepo_CreateAndGet(t *testing.T) { if got.Key != "some/key" || got.Status != model.MultipartStatusInitiated { t.Errorf("unexpected upload: key=%s status=%s", got.Key, got.Status) } + if upload.Metadata == nil || got.Metadata == nil || len(got.Metadata) != 0 { + t.Fatalf("nil metadata was not normalized: input=%#v stored=%#v", upload.Metadata, got.Metadata) + } } func TestMultipartRepo_GetByUploadID_NotFound(t *testing.T) { diff --git a/internal/db/repository/object_deletion_postgres_test.go b/internal/db/repository/object_deletion_postgres_test.go index 640bd57..25d0e92 100644 --- a/internal/db/repository/object_deletion_postgres_test.go +++ b/internal/db/repository/object_deletion_postgres_test.go @@ -17,9 +17,9 @@ import ( "github.com/uptrace/bun" ) -type storageUploadLockContextKey struct{} +type storageContentLockContextKey struct{} -type storageUploadLockBarrier struct { +type storageContentLockBarrier struct { blockedOperation string locked chan struct{} release chan struct{} @@ -28,15 +28,15 @@ type storageUploadLockBarrier struct { attemptOnce sync.Once } -func (h *storageUploadLockBarrier) BeforeQuery(ctx context.Context, event *bun.QueryEvent) context.Context { - if storageUploadLockQuery(event.Query) && ctx.Value(storageUploadLockContextKey{}) != h.blockedOperation { +func (h *storageContentLockBarrier) BeforeQuery(ctx context.Context, event *bun.QueryEvent) context.Context { + if storageContentLockQuery(event.Query) && ctx.Value(storageContentLockContextKey{}) != h.blockedOperation { h.attemptOnce.Do(func() { close(h.attempted) }) } return ctx } -func (h *storageUploadLockBarrier) AfterQuery(ctx context.Context, event *bun.QueryEvent) { - if !storageUploadLockQuery(event.Query) || ctx.Value(storageUploadLockContextKey{}) != h.blockedOperation { +func (h *storageContentLockBarrier) AfterQuery(ctx context.Context, event *bun.QueryEvent) { + if !storageContentLockQuery(event.Query) || ctx.Value(storageContentLockContextKey{}) != h.blockedOperation { return } h.lockOnce.Do(func() { @@ -45,11 +45,84 @@ func (h *storageUploadLockBarrier) AfterQuery(ctx context.Context, event *bun.Qu }) } -func storageUploadLockQuery(query string) bool { +func storageContentLockQuery(query string) bool { query = strings.ToLower(query) - return strings.Contains(query, "update") && strings.Contains(query, "storage_uploads") && strings.Contains(query, "status = status") + return strings.Contains(query, "update") && strings.Contains(query, "storage_contents") && strings.Contains(query, "updated_at = updated_at") } +func permanentDeletePostgresDB(t *testing.T, dsn string) *bun.DB { + t.Helper() + ctx := context.Background() + adminDB, err := appdb.New(config.DatabaseConfig{Driver: "postgres", DSN: dsn, MaxOpenConns: 1, MaxIdleConns: 1}) + if err != nil { + t.Fatalf("opening postgres admin connection: %v", err) + } + schema := fmt.Sprintf("synaps3_permanent_delete_%d", time.Now().UnixNano()) + quotedSchema := quotePostgresIdentifier(schema) + if _, err := adminDB.ExecContext(ctx, "CREATE SCHEMA "+quotedSchema); err != nil { + _ = adminDB.Close() + t.Fatalf("creating postgres test schema: %v", err) + } + testDSN, err := postgresDSNWithSearchPath(dsn, schema) + if err != nil { + _ = adminDB.Close() + t.Fatalf("adding postgres test search path: %v", err) + } + db, err := appdb.New(config.DatabaseConfig{Driver: "postgres", DSN: testDSN, MaxOpenConns: 4, MaxIdleConns: 4}) + if err != nil { + _ = adminDB.Close() + t.Fatalf("opening postgres test connections: %v", err) + } + t.Cleanup(func() { + _ = db.Close() + _, _ = adminDB.ExecContext(context.Background(), "DROP SCHEMA IF EXISTS "+quotedSchema+" CASCADE") + _ = adminDB.Close() + }) + if err := appdb.RunMigrations(ctx, db); err != nil { + t.Fatalf("running postgres test migrations: %v", err) + } + return db +} + +func postgresDSNWithSearchPath(dsn, schema string) (string, error) { + if strings.Contains(dsn, "://") { + u, err := url.Parse(dsn) + if err != nil { + return "", err + } + query := u.Query() + query.Set("search_path", schema) + u.RawQuery = query.Encode() + return u.String(), nil + } + return strings.TrimSpace(dsn) + " search_path=" + schema, nil +} + +func waitPostgresDeleteSignal(t *testing.T, signal <-chan struct{}, name string) { + t.Helper() + select { + case <-signal: + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for %s", name) + } +} + +func waitPostgresDeleteResult(t *testing.T, result <-chan error, name string) error { + t.Helper() + select { + case err := <-result: + return err + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for %s", name) + return nil + } +} + +// TestPostgresPermanentDeleteSerializesContentReuse pins the race that content +// dedup creates: one writer permanently deletes the last version of a content +// while another writes a new version onto the same content. Both must serialize +// on the content row. Cache release rechecks after both operations commit, so +// the surviving follower must retain the shared file regardless of lock order. func TestPostgresPermanentDeleteSerializesContentReuse(t *testing.T) { dsn := os.Getenv("SYNAPS3_POSTGRES_TEST_DSN") if dsn == "" { @@ -59,32 +132,26 @@ func TestPostgresPermanentDeleteSerializesContentReuse(t *testing.T) { for _, tc := range []struct { name string blockedOperation string - wantFollowerBind bool - wantUploadStatus model.StorageUploadStatus }{ - {name: "delete wins", blockedOperation: "delete", wantUploadStatus: model.StorageUploadStatusSuperseded}, - {name: "reuse wins", blockedOperation: "reuse", wantFollowerBind: true, wantUploadStatus: model.StorageUploadStatusComplete}, + {name: "delete wins", blockedOperation: "delete"}, + {name: "reuse wins", blockedOperation: "reuse"}, } { t.Run(tc.name, func(t *testing.T) { db := permanentDeletePostgresDB(t, dsn) repos := repository.NewRepositories(db) ctx := context.Background() bucket := seedBucket(t, db, "postgres-permanent-delete-"+strings.ReplaceAll(tc.name, " ", "-")) + contentID := seedContent(t, repos, bucket.ID, "postgres-shared-content", 10) + source := newObjectVersion(bucket.ID, "source.txt", model.NewVersionID(), 10) + source.ContentID = &contentID if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, source); err != nil { t.Fatalf("CreateVersionAndSetCurrent(source): %v", err) } - uploadID := acceptTestStorageUploadForVersion(t, repos, bucket.ID, source, "bafk2bzacepostgresreuse") - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, source.VersionID, uploadID, model.ObjectStateCached, model.ObjectStateStored); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition(source): %v", err) - } - followerUploadID := uploadID follower := newObjectVersion(bucket.ID, "follower.txt", model.NewVersionID(), source.Size) - follower.Checksum = source.Checksum - follower.StorageUploadID = &followerUploadID - follower.State = model.ObjectStateStored + follower.ContentID = &contentID - barrier := &storageUploadLockBarrier{ + barrier := &storageContentLockBarrier{ blockedOperation: tc.blockedOperation, locked: make(chan struct{}), release: make(chan struct{}), @@ -98,8 +165,8 @@ func TestPostgresPermanentDeleteSerializesContentReuse(t *testing.T) { } }() db.AddQueryHook(barrier) - deleteCtx := context.WithValue(ctx, storageUploadLockContextKey{}, "delete") - reuseCtx := context.WithValue(ctx, storageUploadLockContextKey{}, "reuse") + deleteCtx := context.WithValue(ctx, storageContentLockContextKey{}, "delete") + reuseCtx := context.WithValue(ctx, storageContentLockContextKey{}, "reuse") deleteResult := make(chan error, 1) reuseResult := make(chan error, 1) startDelete := func() { @@ -121,13 +188,13 @@ func TestPostgresPermanentDeleteSerializesContentReuse(t *testing.T) { } else { startReuse() } - waitPostgresDeleteSignal(t, barrier.locked, "first storage upload lock") + waitPostgresDeleteSignal(t, barrier.locked, "first storage content lock") if tc.blockedOperation == "delete" { startReuse() } else { startDelete() } - waitPostgresDeleteSignal(t, barrier.attempted, "competing storage upload lock") + waitPostgresDeleteSignal(t, barrier.attempted, "competing storage content lock") close(barrier.release) if err := waitPostgresDeleteResult(t, deleteResult, "permanent delete"); err != nil { t.Fatalf("DeleteObjectVersionPermanently: %v", err) @@ -136,89 +203,22 @@ func TestPostgresPermanentDeleteSerializesContentReuse(t *testing.T) { t.Fatalf("CreateVersionAndSetCurrent(follower): %v", err) } - gotFollower, err := repos.Objects.GetVersionByID(ctx, follower.VersionID) - if err != nil || gotFollower == nil { - t.Fatalf("GetVersionByID(follower): version=%#v err=%v", gotFollower, err) + unreferenced, err := repos.Objects.ContentIsUnreferenced(ctx, contentID) + if err != nil { + t.Fatalf("ContentIsUnreferenced: %v", err) } - if tc.wantFollowerBind { - if gotFollower.StorageUploadID == nil || *gotFollower.StorageUploadID != uploadID || gotFollower.State != model.ObjectStateStored { - t.Fatalf("reuse-winner follower = upload:%#v state:%s, want stored upload %d", gotFollower.StorageUploadID, gotFollower.State, uploadID) - } - } else if gotFollower.StorageUploadID != nil || gotFollower.State != model.ObjectStateCached { - t.Fatalf("delete-winner follower = upload:%#v state:%s, want cached without stale upload", gotFollower.StorageUploadID, gotFollower.State) + // Whoever won, the follower survives, so the bytes are still named. + if unreferenced { + t.Fatal("content is unreferenced after the reuse committed") } - upload, err := repos.Uploads.GetByID(ctx, uploadID) - if err != nil || upload == nil || upload.Status != tc.wantUploadStatus { - t.Fatalf("upload after concurrent delete/reuse = %#v err=%v, want %s", upload, err, tc.wantUploadStatus) + deleteCalls := 0 + released, err := repos.Objects.ReleaseContentCacheIfUnreferenced(ctx, contentID, func() error { + deleteCalls++ + return nil + }) + if err != nil || released || deleteCalls != 0 { + t.Fatalf("cache release = %t, %v calls=%d, want retained", released, err, deleteCalls) } }) } } - -func permanentDeletePostgresDB(t *testing.T, dsn string) *bun.DB { - t.Helper() - ctx := context.Background() - adminDB, err := appdb.New(config.DatabaseConfig{Driver: "postgres", DSN: dsn, MaxOpenConns: 1, MaxIdleConns: 1}) - if err != nil { - t.Fatalf("opening postgres admin connection: %v", err) - } - schema := fmt.Sprintf("synaps3_permanent_delete_%d", time.Now().UnixNano()) - quotedSchema := quotePostgresIdentifier(schema) - if _, err := adminDB.ExecContext(ctx, "CREATE SCHEMA "+quotedSchema); err != nil { - _ = adminDB.Close() - t.Fatalf("creating postgres test schema: %v", err) - } - testDSN, err := postgresDSNWithSearchPath(dsn, schema) - if err != nil { - _ = adminDB.Close() - t.Fatalf("adding postgres test search path: %v", err) - } - db, err := appdb.New(config.DatabaseConfig{Driver: "postgres", DSN: testDSN, MaxOpenConns: 4, MaxIdleConns: 4}) - if err != nil { - _ = adminDB.Close() - t.Fatalf("opening postgres test connections: %v", err) - } - t.Cleanup(func() { - _ = db.Close() - _, _ = adminDB.ExecContext(context.Background(), "DROP SCHEMA IF EXISTS "+quotedSchema+" CASCADE") - _ = adminDB.Close() - }) - if err := appdb.RunMigrations(ctx, db); err != nil { - t.Fatalf("running postgres test migrations: %v", err) - } - return db -} - -func postgresDSNWithSearchPath(dsn, schema string) (string, error) { - if strings.Contains(dsn, "://") { - u, err := url.Parse(dsn) - if err != nil { - return "", err - } - query := u.Query() - query.Set("search_path", schema) - u.RawQuery = query.Encode() - return u.String(), nil - } - return strings.TrimSpace(dsn) + " search_path=" + schema, nil -} - -func waitPostgresDeleteSignal(t *testing.T, signal <-chan struct{}, name string) { - t.Helper() - select { - case <-signal: - case <-time.After(5 * time.Second): - t.Fatalf("timed out waiting for %s", name) - } -} - -func waitPostgresDeleteResult(t *testing.T, result <-chan error, name string) error { - t.Helper() - select { - case err := <-result: - return err - case <-time.After(5 * time.Second): - t.Fatalf("timed out waiting for %s", name) - return nil - } -} diff --git a/internal/db/repository/object_deletion_repo_test.go b/internal/db/repository/object_deletion_repo_test.go deleted file mode 100644 index 5703e84..0000000 --- a/internal/db/repository/object_deletion_repo_test.go +++ /dev/null @@ -1,1802 +0,0 @@ -package repository_test - -import ( - "context" - "errors" - "strconv" - "strings" - "testing" - "time" - - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagecommit" - "github.com/uptrace/bun" -) - -func TestObjectRepo_UpdateObjectDeletionCacheCleanup(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "deletion-cache-cleanup-bucket") - - deleteVersion := func(t *testing.T, key, versionID string) { - t.Helper() - version := newObjectVersion(bucket.ID, key, versionID, 100) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - if _, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: bucket.ID, - Key: key, - VersionID: versionID, - }); err != nil { - t.Fatalf("DeleteObjectVersionPermanently: %v", err) - } - } - - t.Run("deleted", func(t *testing.T) { - const versionID = "01J000000000000000CACHE01" - deleteVersion(t, "deleted.txt", versionID) - - if err := repos.Objects.UpdateObjectDeletionCacheCleanup(ctx, versionID, model.CacheCleanupStatusDeleted, ""); err != nil { - t.Fatalf("UpdateObjectDeletionCacheCleanup: %v", err) - } - - var deletion model.ObjectDeletion - if err := db.NewSelect().Model(&deletion).Where("version_id = ?", versionID).Scan(ctx); err != nil { - t.Fatalf("select deletion: %v", err) - } - if deletion.CacheCleanupStatus != model.CacheCleanupStatusDeleted { - t.Errorf("CacheCleanupStatus = %q, want %q", deletion.CacheCleanupStatus, model.CacheCleanupStatusDeleted) - } - if deletion.CacheError != nil { - t.Errorf("CacheError = %q, want nil", *deletion.CacheError) - } - if deletion.CacheCleanedAt == nil { - t.Error("CacheCleanedAt is nil") - } - }) - - t.Run("failed", func(t *testing.T) { - const ( - versionID = "01J000000000000000CACHE02" - cacheError = "disk I/O error" - ) - deleteVersion(t, "failed.txt", versionID) - - if err := repos.Objects.UpdateObjectDeletionCacheCleanup(ctx, versionID, model.CacheCleanupStatusFailed, cacheError); err != nil { - t.Fatalf("UpdateObjectDeletionCacheCleanup: %v", err) - } - - var deletion model.ObjectDeletion - if err := db.NewSelect().Model(&deletion).Where("version_id = ?", versionID).Scan(ctx); err != nil { - t.Fatalf("select deletion: %v", err) - } - if deletion.CacheCleanupStatus != model.CacheCleanupStatusFailed { - t.Errorf("CacheCleanupStatus = %q, want %q", deletion.CacheCleanupStatus, model.CacheCleanupStatusFailed) - } - if deletion.CacheError == nil || *deletion.CacheError != cacheError { - t.Errorf("CacheError = %v, want %q", deletion.CacheError, cacheError) - } - if deletion.CacheCleanedAt == nil { - t.Error("CacheCleanedAt is nil") - } - }) - - t.Run("invalid input", func(t *testing.T) { - for _, tc := range []struct { - name string - versionID string - status model.CacheCleanupStatus - }{ - {name: "empty version ID", status: model.CacheCleanupStatusDeleted}, - {name: "empty status", versionID: "01J000000000000000CACHE03"}, - } { - t.Run(tc.name, func(t *testing.T) { - err := repos.Objects.UpdateObjectDeletionCacheCleanup(ctx, tc.versionID, tc.status, "") - if !errors.Is(err, repository.ErrInvalidInput) { - t.Fatalf("UpdateObjectDeletionCacheCleanup error = %v, want ErrInvalidInput", err) - } - }) - } - }) - - t.Run("not found", func(t *testing.T) { - err := repos.Objects.UpdateObjectDeletionCacheCleanup(ctx, "01J000000000000000MISSING", model.CacheCleanupStatusDeleted, "") - if !errors.Is(err, repository.ErrNotFound) { - t.Fatalf("UpdateObjectDeletionCacheCleanup error = %v, want ErrNotFound", err) - } - }) -} - -func TestObjectRepo_DeleteObjectVersionPermanentlyRemovesVersionAndQueuesStorageCleanup(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "permanent-delete-bucket") - - oldVersion := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DEL01", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(old): %v", err) - } - uploadID := acceptTestStorageUploadForVersion(t, repos, bucket.ID, oldVersion, "bafk2bzacepermdelete") - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, oldVersion.VersionID, uploadID, model.ObjectStateCached, model.ObjectStateStored); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition(old): %v", err) - } - currentVersion := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DEL02", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, currentVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(current): %v", err) - } - - result, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: bucket.ID, - Key: oldVersion.Key, - VersionID: oldVersion.VersionID, - }) - if err != nil { - t.Fatalf("DeleteObjectVersionPermanently: %v", err) - } - if result.CacheKey != oldVersion.CacheKey { - t.Fatalf("cache key = %q, want %q", result.CacheKey, oldVersion.CacheKey) - } - if result.StorageCleanupTaskID == nil { - t.Fatal("expected storage cleanup task id") - } - - gotVersion, err := repos.Objects.GetVersionByID(ctx, oldVersion.VersionID) - if err != nil { - t.Fatalf("GetVersionByID(deleted): %v", err) - } - if gotVersion != nil { - t.Fatalf("deleted version still exists: %#v", gotVersion) - } - - var deletionCount int - if err := db.NewRaw(`SELECT COUNT(*) FROM object_deletions WHERE version_id = ? AND storage_upload_id = ?`, oldVersion.VersionID, uploadID).Scan(ctx, &deletionCount); err != nil { - t.Fatalf("count object_deletions: %v", err) - } - if deletionCount != 1 { - t.Fatalf("object_deletions count = %d, want 1", deletionCount) - } - - task, err := repos.Tasks.GetByID(ctx, *result.StorageCleanupTaskID) - if err != nil || task == nil { - t.Fatalf("GetByID(cleanup task): task=%v err=%v", task, err) - } - if task.Type != model.TaskTypeStorageCleanup || task.RefType != "storage_upload" || task.RefID != uploadID || task.RefVersionID != "" { - t.Fatalf("cleanup task ref = type:%s ref:%s/%d version:%q, want storage cleanup for upload %d", task.Type, task.RefType, task.RefID, task.RefVersionID, uploadID) - } - if task.MaxRetries != 5 { - t.Fatalf("cleanup task max retries = %d, want default 5", task.MaxRetries) - } - - var copyCount int - if err := db.NewRaw(`SELECT COUNT(*) FROM storage_cleanup_copies WHERE task_id = ? AND upload_id = ? AND piece_id IS NOT NULL`, task.ID, uploadID).Scan(ctx, ©Count); err != nil { - t.Fatalf("count storage_cleanup_copies: %v", err) - } - if copyCount != 1 { - t.Fatalf("storage_cleanup_copies count = %d, want 1", copyCount) - } - var deletedAt string - if err := db.NewRaw(`SELECT deleted_at FROM object_deletions WHERE version_id = ?`, oldVersion.VersionID).Scan(ctx, &deletedAt); err != nil { - t.Fatalf("select object deletion deleted_at: %v", err) - } - if deletedAt == "" { - t.Fatal("object deletion deleted_at is empty") - } -} - -func TestObjectRepo_DeleteObjectVersionPermanentlyAllowsStoppedInProgressState(t *testing.T) { - tests := []struct { - name string - state model.ObjectState - }{ - {name: "uploading", state: model.ObjectStateUploading}, - {name: "committing", state: model.ObjectStateCommitting}, - {name: "replicating", state: model.ObjectStateReplicating}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "permanent-delete-"+tt.name) - version := newObjectVersion(bucket.ID, "file.txt", model.NewVersionID(), 10) - if tt.state != model.ObjectStateReplicating { - version.State = tt.state - } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - if tt.state == model.ObjectStateReplicating { - uploadID := acceptTestStorageUploadForVersion(t, repos, bucket.ID, version, "bafk2bzacepermanentbusy") - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, version.VersionID, uploadID, model.ObjectStateCached, model.ObjectStateReplicating); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition: %v", err) - } - } - - _, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: bucket.ID, Key: version.Key, VersionID: version.VersionID, - }) - if err != nil { - t.Fatalf("DeleteObjectVersionPermanently: %v", err) - } - got, loadErr := repos.Objects.GetVersionByID(ctx, version.VersionID) - if loadErr != nil || got != nil { - t.Fatalf("version after permanent delete = %#v err=%v, want removed", got, loadErr) - } - }) - } -} - -func TestObjectRepo_DeleteObjectVersionPermanentlyCoordinatesReplicaRepairTaskStates(t *testing.T) { - tests := []struct { - name string - status model.TaskStatus - wantBusy bool - createTask bool - }{ - {name: "queued", status: model.TaskStatusQueued, wantBusy: true, createTask: true}, - {name: "scheduled", status: model.TaskStatusScheduled, wantBusy: true, createTask: true}, - {name: "waiting", status: model.TaskStatusWaiting, wantBusy: true, createTask: true}, - {name: "running", status: model.TaskStatusRunning, wantBusy: true, createTask: true}, - {name: "exhausted", status: model.TaskStatusExhausted, createTask: true}, - {name: "failed", status: model.TaskStatusFailed, createTask: true}, - {name: "cancelled", status: model.TaskStatusCancelled, createTask: true}, - {name: "completed", status: model.TaskStatusCompleted, createTask: true}, - {name: "no task"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - fixture := seedPermanentDeleteRepairFixture(t, db, repos, "repair-task-"+strings.ReplaceAll(tt.name, " ", "-")) - var taskID int64 - if tt.createTask { - stage := "repair_replica" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "bucket", RefID: fixture.bucket.ID, - RefVersionID: fixture.version.VersionID, IdempotencyKey: "upload:repair-data-set:permanent-delete", - Payload: map[string]interface{}{"storage_data_set_id": fixture.repair.ID, "storage_upload_copy_id": fixture.repairCopy.ID}, - Status: tt.status, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create repair task: %v", err) - } - taskID = task.ID - } - - result, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: fixture.bucket.ID, Key: fixture.version.Key, VersionID: fixture.version.VersionID, - }) - if tt.wantBusy { - if !errors.Is(err, repository.ErrPermanentDeleteStorageBusy) || !errors.Is(err, repository.ErrConflict) { - t.Fatalf("DeleteObjectVersionPermanently error = %v, want storage-work conflict", err) - } - gotVersion, loadErr := repos.Objects.GetVersionByID(ctx, fixture.version.VersionID) - if loadErr != nil || gotVersion == nil { - t.Fatalf("version after rejected delete = %#v err=%v, want retained", gotVersion, loadErr) - } - got, loadErr := repos.Uploads.GetUploadCopyByID(ctx, fixture.repairCopy.ID) - if loadErr != nil || got == nil || got.Status != model.StorageUploadCopyStatusPending { - t.Fatalf("repair copy after rejected delete = %#v err=%v, want pending", got, loadErr) - } - return - } - if err != nil { - t.Fatalf("DeleteObjectVersionPermanently: %v", err) - } - if result.StorageCleanupTaskID == nil { - t.Fatal("expected cleanup task for committed replica") - } - got, loadErr := repos.Uploads.GetUploadCopyByID(ctx, fixture.repairCopy.ID) - if loadErr != nil || got == nil || got.Status != model.StorageUploadCopyStatusFailed { - t.Fatalf("repair copy after accepted delete = %#v err=%v, want failed", got, loadErr) - } - gotUpload, loadErr := repos.Uploads.GetByID(ctx, fixture.upload.ID) - if loadErr != nil || gotUpload == nil || gotUpload.Status != model.StorageUploadStatusSuperseded { - t.Fatalf("upload after last-reference delete = %#v err=%v, want superseded", gotUpload, loadErr) - } - if taskID > 0 { - gotTask, taskErr := repos.Tasks.GetByID(ctx, taskID) - if taskErr != nil || gotTask == nil || gotTask.Status != tt.status { - t.Fatalf("repair coordinator after accepted delete = %#v err=%v, want preserved %s", gotTask, taskErr, tt.status) - } - } - }) - } -} - -func TestObjectRepo_DeleteObjectVersionPermanentlyKeepsSubmittedRepairCommit(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - fixture := seedPermanentDeleteRepairFixture(t, db, repos, "submitted-repair") - if err := repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: fixture.upload.ID, CopyIndex: fixture.repairCopy.CopyIndex, PieceCID: "bafk2bzacepermanentrepair", RetrievalURL: "https://repair.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - copyRow, err := repos.Uploads.GetUploadCopyByID(ctx, fixture.repairCopy.ID) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopyByID: copy=%#v err=%v", copyRow, err) - } - seedRepositoryCommitAttempt(t, repos, *copyRow, "submitted-repair", "abcd", "0xsubmitted") - - _, err = repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: fixture.bucket.ID, Key: fixture.version.Key, VersionID: fixture.version.VersionID, - }) - if !errors.Is(err, repository.ErrPermanentDeleteStorageBusy) { - t.Fatalf("DeleteObjectVersionPermanently error = %v, want submitted storage-work conflict", err) - } - got, err := repos.Uploads.GetUploadCopyByID(ctx, fixture.repairCopy.ID) - if err != nil || got == nil || got.Status != model.StorageUploadCopyStatusCommitting || got.CommitTransactionID == nil { - t.Fatalf("submitted repair copy = %#v err=%v, want retained committing copy", got, err) - } -} - -func TestObjectRepo_DeleteObjectVersionPermanentlyPreservesSharedRepairUntilLastReference(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - fixture := seedPermanentDeleteRepairFixture(t, db, repos, "shared-repair") - follower := newObjectVersion(fixture.bucket.ID, "follower.txt", "01J000000000000000000DEL0S", fixture.version.Size) - follower.Checksum = fixture.version.Checksum - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, follower); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(follower): %v", err) - } - bindPermanentDeleteFollower(t, repos, fixture.upload.ID, follower) - - first, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: fixture.bucket.ID, Key: fixture.version.Key, VersionID: fixture.version.VersionID, - }) - if err != nil || first.StorageCleanupTaskID == nil { - t.Fatalf("DeleteObjectVersionPermanently(source): result=%#v err=%v", first, err) - } - copyAfterSourceDelete, err := repos.Uploads.GetUploadCopyByID(ctx, fixture.repairCopy.ID) - if err != nil || copyAfterSourceDelete == nil || copyAfterSourceDelete.Status != model.StorageUploadCopyStatusPending { - t.Fatalf("shared repair copy after source delete = %#v err=%v, want pending", copyAfterSourceDelete, err) - } - if got, err := repos.Objects.GetVersionByID(ctx, follower.VersionID); err != nil || got == nil { - t.Fatalf("shared follower after source delete = %#v err=%v", got, err) - } - - second, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: fixture.bucket.ID, Key: follower.Key, VersionID: follower.VersionID, - }) - if err != nil { - t.Fatalf("DeleteObjectVersionPermanently(last reference): %v", err) - } - if second.StorageCleanupTaskID == nil || *second.StorageCleanupTaskID != *first.StorageCleanupTaskID { - t.Fatalf("cleanup task IDs = first:%v second:%v, want reused task", first.StorageCleanupTaskID, second.StorageCleanupTaskID) - } - copyAfterLastDelete, err := repos.Uploads.GetUploadCopyByID(ctx, fixture.repairCopy.ID) - if err != nil || copyAfterLastDelete == nil || copyAfterLastDelete.Status != model.StorageUploadCopyStatusFailed { - t.Fatalf("repair copy after last reference delete = %#v err=%v, want failed", copyAfterLastDelete, err) - } -} - -func TestObjectRepo_CreateVersionDoesNotAttachSupersededUploadAfterPermanentDelete(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "permanent-delete-stale-reuse") - source := newObjectVersion(bucket.ID, "source.txt", model.NewVersionID(), 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, source); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(source): %v", err) - } - uploadID := acceptTestStorageUploadForVersion(t, repos, bucket.ID, source, "bafk2bzacestalereuse") - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, source.VersionID, uploadID, model.ObjectStateCached, model.ObjectStateStored); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition(source): %v", err) - } - - staleUploadID := uploadID - follower := newObjectVersion(bucket.ID, "follower.txt", model.NewVersionID(), source.Size) - follower.Checksum = source.Checksum - follower.StorageUploadID = &staleUploadID - follower.State = model.ObjectStateStored - if _, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: bucket.ID, Key: source.Key, VersionID: source.VersionID, - }); err != nil { - t.Fatalf("DeleteObjectVersionPermanently(source): %v", err) - } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, follower); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(stale follower): %v", err) - } - - got, err := repos.Objects.GetVersionByID(ctx, follower.VersionID) - if err != nil || got == nil { - t.Fatalf("GetVersionByID(follower): version=%#v err=%v", got, err) - } - if got.StorageUploadID != nil || got.State != model.ObjectStateCached { - t.Fatalf("follower storage reference = upload:%#v state:%s, want cached without superseded upload", got.StorageUploadID, got.State) - } - upload, err := repos.Uploads.GetByID(ctx, uploadID) - if err != nil || upload == nil || upload.Status != model.StorageUploadStatusSuperseded { - t.Fatalf("deleted source upload = %#v err=%v, want superseded", upload, err) - } - if err := repos.Objects.UpdateVersionState(ctx, follower.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("UpdateVersionState follower cached to uploading: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForVersion(ctx, repository.BindReadableUploadForVersionInput{ - UploadID: uploadID, BucketID: bucket.ID, ContentSize: follower.Size, Checksum: follower.Checksum, VersionID: follower.VersionID, - }); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("BindReadableUploadForVersion error = %v, want superseded upload conflict", err) - } - if _, _, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: uploadID}); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("FinalizeUploadIfTargetCopiesMet error = %v, want superseded upload conflict", err) - } - copyRow, err := repos.Uploads.GetUploadCopy(ctx, uploadID, 0) - if err != nil || copyRow == nil || copyRow.StorageDataSetID == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - _, err = repos.Uploads.ReserveCommitAttempt(ctx, storagecommit.ReserveInput{ - Copy: storagecommit.CopyIdentity{ - StorageUploadCopyID: copyRow.ID, UploadID: uploadID, CopyIndex: 0, - StorageDataSetID: *copyRow.StorageDataSetID, RequireEligibleCopy: true, - }, - AttemptID: "late-superseded-attempt", - }) - if !errors.Is(err, repository.ErrConflict) { - t.Fatalf("ReserveCommitAttempt error = %v, want superseded upload conflict", err) - } -} - -func TestStorageCleanupRepo_SourceVersionIsAnObjectReferenceBeforeBinding(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "storage-cleanup-source-reference") - version := newObjectVersion(bucket.ID, "source.txt", model.NewVersionID(), 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: version.VersionID, ContentSize: version.Size, Checksum: version.Checksum, RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - - hasReferences, err := repos.StorageCleanup.UploadHasObjectReferences(ctx, upload.ID) - if err != nil { - t.Fatalf("UploadHasObjectReferences: %v", err) - } - if !hasReferences { - t.Fatal("UploadHasObjectReferences = false, want unbound source version to keep upload referenced") - } - if err := repos.StorageCleanup.DeleteUploadProvenanceIfUnreferenced(ctx, upload.ID); err != nil { - t.Fatalf("DeleteUploadProvenanceIfUnreferenced(referenced): %v", err) - } - if retained, err := repos.Uploads.GetByID(ctx, upload.ID); err != nil || retained == nil { - t.Fatalf("referenced upload after provenance cleanup = %#v err=%v, want retained", retained, err) - } - if _, err := db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("status = ?", model.StorageUploadStatusFailed). - Where("id = ?", upload.ID). - Exec(ctx); err != nil { - t.Fatalf("retire first upload fixture: %v", err) - } - currentUploadID := acceptTestStorageUploadForVersion(t, repos, bucket.ID, version, "bafk2bzacecurrentsource") - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, version.VersionID, currentUploadID, model.ObjectStateCached, model.ObjectStateStored); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition(current): %v", err) - } - hasReferences, err = repos.StorageCleanup.UploadHasObjectReferences(ctx, upload.ID) - if err != nil { - t.Fatalf("UploadHasObjectReferences(old): %v", err) - } - if hasReferences { - t.Fatal("UploadHasObjectReferences(old) = true, want rebound source to release historical upload") - } - if err := repos.StorageCleanup.DeleteUploadProvenanceIfUnreferenced(ctx, upload.ID); err != nil { - t.Fatalf("DeleteUploadProvenanceIfUnreferenced(unreferenced): %v", err) - } - if removed, err := repos.Uploads.GetByID(ctx, upload.ID); err != nil || removed != nil { - t.Fatalf("unreferenced upload after provenance cleanup = %#v err=%v, want removed", removed, err) - } -} - -func TestTaskRepo_RetryCannotReattachSupersededUpload(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - fixture := seedPermanentDeleteRepairFixture(t, db, repos, "retry-superseded-upload") - stage := "peer_pull" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "object", RefID: fixture.version.ObjectID, - RefVersionID: fixture.version.VersionID, IdempotencyKey: "upload:retry-superseded-upload", - Payload: map[string]interface{}{"upload_id": fixture.upload.ID}, Status: model.TaskStatusExhausted, - MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create exhausted upload task: %v", err) - } - if _, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: fixture.bucket.ID, Key: fixture.version.Key, VersionID: fixture.version.VersionID, - }); err != nil { - t.Fatalf("DeleteObjectVersionPermanently: %v", err) - } - if err := repos.Tasks.RetryExhausted(ctx, task.ID); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("RetryExhausted error = %v, want superseded upload conflict", err) - } - got, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil || got == nil || got.Status != model.TaskStatusExhausted { - t.Fatalf("task after rejected retry = %#v err=%v, want exhausted", got, err) - } -} - -func TestStorageUploadRepo_AcquireUploadTaskProtectsRunningWorkAndRejectsDeletedWork(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - fixture := seedPermanentDeleteRepairFixture(t, db, repos, "ordinary-upload-execution") - stage := "peer_pull" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "object", RefID: fixture.version.ObjectID, - RefVersionID: fixture.version.VersionID, IdempotencyKey: "upload:ordinary-execution-owner", - Payload: map[string]interface{}{"upload_id": fixture.upload.ID, "copy_index": fixture.repairCopy.CopyIndex}, - Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create ordinary upload task: %v", err) - } - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil || claimed == nil || claimed.ClaimedAt == nil { - t.Fatalf("ClaimReady(ordinary): task=%#v err=%v", claimed, err) - } - if err := repos.Uploads.AcquireUploadTask(ctx, repository.AcquireUploadTaskInput{ - TaskID: claimed.ID, TaskClaimedAt: *claimed.ClaimedAt, UploadID: fixture.upload.ID, VersionID: fixture.version.VersionID, - }); err != nil { - t.Fatalf("AcquireUploadTask: %v", err) - } - if _, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: fixture.bucket.ID, Key: fixture.version.Key, VersionID: fixture.version.VersionID, - }); !errors.Is(err, repository.ErrPermanentDeleteStorageBusy) { - t.Fatalf("DeleteObjectVersionPermanently while ordinary task owns execution = %v, want storage busy", err) - } - if err := repos.Tasks.Complete(ctx, claimed); err != nil { - t.Fatalf("Complete ordinary upload task: %v", err) - } - if _, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: fixture.bucket.ID, Key: fixture.version.Key, VersionID: fixture.version.VersionID, - }); err != nil { - t.Fatalf("DeleteObjectVersionPermanently after task stopped: %v", err) - } - - late := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "object", RefID: fixture.version.ObjectID, - RefVersionID: fixture.version.VersionID, IdempotencyKey: "upload:ordinary-execution-late", - Payload: map[string]interface{}{"upload_id": fixture.upload.ID, "copy_index": fixture.repairCopy.CopyIndex}, - Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, late); err != nil { - t.Fatalf("Create late ordinary upload task: %v", err) - } - lateClaim, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil || lateClaim == nil || lateClaim.ClaimedAt == nil { - t.Fatalf("ClaimReady(late ordinary): task=%#v err=%v", lateClaim, err) - } - if err := repos.Uploads.AcquireUploadTask(ctx, repository.AcquireUploadTaskInput{ - TaskID: lateClaim.ID, TaskClaimedAt: *lateClaim.ClaimedAt, UploadID: fixture.upload.ID, VersionID: fixture.version.VersionID, - }); !errors.Is(err, repository.ErrUploadTaskCancelled) { - t.Fatalf("AcquireUploadTask after permanent delete = %v, want cancelled", err) - } -} - -func TestObjectRepo_DeleteObjectVersionPermanentlyKeepsSharedVersionsWhileRepairIsRunning(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - fixture := seedPermanentDeleteRepairFixture(t, db, repos, "shared-running-repair") - follower := newObjectVersion(fixture.bucket.ID, "follower.txt", "01J000000000000000000DEL0X", fixture.version.Size) - follower.Checksum = fixture.version.Checksum - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, follower); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(follower): %v", err) - } - bindPermanentDeleteFollower(t, repos, fixture.upload.ID, follower) - stage := "repair_replica" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "bucket", RefID: fixture.bucket.ID, - RefVersionID: fixture.version.VersionID, IdempotencyKey: "upload:repair-data-set:shared-running-repair", - Payload: map[string]interface{}{"storage_data_set_id": fixture.repair.ID, "storage_upload_copy_id": fixture.repairCopy.ID}, - Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create repair task: %v", err) - } - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil || claimed == nil || claimed.ID != task.ID || claimed.Status != model.TaskStatusRunning || claimed.ClaimedAt == nil { - t.Fatalf("ClaimReady: task=%#v err=%v", claimed, err) - } - - for _, version := range []*model.ObjectVersion{fixture.version, follower} { - _, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: fixture.bucket.ID, Key: version.Key, VersionID: version.VersionID, - }) - if !errors.Is(err, repository.ErrPermanentDeleteStorageBusy) { - t.Fatalf("DeleteObjectVersionPermanently(%s) error = %v, want storage-work conflict", version.VersionID, err) - } - } - for _, versionID := range []string{fixture.version.VersionID, follower.VersionID} { - got, loadErr := repos.Objects.GetVersionByID(ctx, versionID) - if loadErr != nil || got == nil { - t.Fatalf("shared version %s after rejected delete = %#v err=%v, want retained", versionID, got, loadErr) - } - } - copyRow, err := repos.Uploads.GetUploadCopyByID(ctx, fixture.repairCopy.ID) - if err != nil || copyRow == nil || copyRow.Status != model.StorageUploadCopyStatusPending { - t.Fatalf("repair copy after rejected shared deletes = %#v err=%v, want pending", copyRow, err) - } -} - -func TestObjectRepo_DeleteObjectVersionPermanentlyAddsLateCommittedCleanupSnapshot(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - fixture := seedPermanentDeleteRepairFixture(t, db, repos, "late-cleanup-snapshot") - follower := newObjectVersion(fixture.bucket.ID, "follower.txt", "01J000000000000000000DEL0T", fixture.version.Size) - follower.Checksum = fixture.version.Checksum - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, follower); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(follower): %v", err) - } - bindPermanentDeleteFollower(t, repos, fixture.upload.ID, follower) - - first, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: fixture.bucket.ID, Key: fixture.version.Key, VersionID: fixture.version.VersionID, - }) - if err != nil || first.StorageCleanupTaskID == nil { - t.Fatalf("DeleteObjectVersionPermanently(source): result=%#v err=%v", first, err) - } - cleanupCopies, err := repos.StorageCleanup.ListCopiesForTask(ctx, *first.StorageCleanupTaskID) - if err != nil || len(cleanupCopies) != 1 || cleanupCopies[0].CopyIndex != 0 { - t.Fatalf("initial cleanup copies = %#v err=%v, want committed copy 0", cleanupCopies, err) - } - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeStorageCleanup, time.Minute) - if err != nil || claimed == nil || claimed.ID != *first.StorageCleanupTaskID { - t.Fatalf("ClaimReady(cleanup): task=%#v err=%v", claimed, err) - } - if err := repos.StorageCleanup.MarkCopyRemoved(ctx, cleanupCopies[0].ID); err != nil { - t.Fatalf("MarkCopyRemoved: %v", err) - } - if err := repos.Tasks.Complete(ctx, claimed); err != nil { - t.Fatalf("Complete(cleanup): %v", err) - } - - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - StorageUploadCopyID: fixture.repairCopy.ID, - UploadID: fixture.upload.ID, - CopyIndex: fixture.repairCopy.CopyIndex, - PieceCID: "bafk2bzacepermanentrepair", - PieceID: onChainIDPtr(t, "3002"), - RetrievalURL: "https://repair.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted(repair): %v", err) - } - second, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: fixture.bucket.ID, Key: follower.Key, VersionID: follower.VersionID, - }) - if err != nil { - t.Fatalf("DeleteObjectVersionPermanently(last reference): %v", err) - } - if second.StorageCleanupTaskID == nil || *second.StorageCleanupTaskID != *first.StorageCleanupTaskID { - t.Fatalf("cleanup task IDs = first:%v second:%v, want reused task", first.StorageCleanupTaskID, second.StorageCleanupTaskID) - } - cleanupTask, err := repos.Tasks.GetByID(ctx, *first.StorageCleanupTaskID) - if err != nil || cleanupTask == nil || cleanupTask.Status != model.TaskStatusQueued { - t.Fatalf("cleanup task after late snapshot = %#v err=%v, want requeued", cleanupTask, err) - } - cleanupCopies, err = repos.StorageCleanup.ListCopiesForTask(ctx, *first.StorageCleanupTaskID) - if err != nil || len(cleanupCopies) != 2 { - t.Fatalf("cleanup copies after late commit = %#v err=%v, want two snapshots", cleanupCopies, err) - } - if cleanupCopies[0].Status != model.StorageCleanupCopyStatusRemoved || cleanupCopies[1].CopyIndex != 1 || cleanupCopies[1].Status != model.StorageCleanupCopyStatusPending { - t.Fatalf("cleanup copy states after late commit = %#v, want removed copy 0 and pending copy 1", cleanupCopies) - } -} - -func TestStorageUploadRepo_ExactRepairWritesCannotRevivePermanentlyDeletedCopy(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - fixture := seedPermanentDeleteRepairFixture(t, db, repos, "exact-write-guard") - if _, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: fixture.bucket.ID, Key: fixture.version.Key, VersionID: fixture.version.VersionID, - }); err != nil { - t.Fatalf("DeleteObjectVersionPermanently: %v", err) - } - - tests := []struct { - name string - run func() error - }{ - { - name: "piece ready", - run: func() error { - return repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - StorageUploadCopyID: fixture.repairCopy.ID, UploadID: fixture.upload.ID, - CopyIndex: fixture.repairCopy.CopyIndex, PieceCID: "bafk2bzacepermanentrepair", RetrievalURL: "https://repair.example/piece", - }) - }, - }, - { - name: "commit reservation", - run: func() error { - _, err := repos.Uploads.ReserveCommitAttempt(ctx, storagecommit.ReserveInput{ - Copy: storagecommit.CopyIdentity{ - StorageUploadCopyID: fixture.repairCopy.ID, - UploadID: fixture.upload.ID, - CopyIndex: fixture.repairCopy.CopyIndex, - StorageDataSetID: *fixture.repairCopy.StorageDataSetID, - RequireEligibleCopy: true, - }, - AttemptID: "late-reservation", - }) - return err - }, - }, - { - name: "committed", - run: func() error { - return repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - StorageUploadCopyID: fixture.repairCopy.ID, UploadID: fixture.upload.ID, CopyIndex: fixture.repairCopy.CopyIndex, - PieceCID: "bafk2bzacepermanentrepair", PieceID: onChainIDPtr(t, "3002"), RetrievalURL: "https://repair.example/piece", - }) - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := tt.run(); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("exact repair write error = %v, want ErrConflict", err) - } - }) - } - copyRow, err := repos.Uploads.GetUploadCopyByID(ctx, fixture.repairCopy.ID) - if err != nil || copyRow == nil || copyRow.Status != model.StorageUploadCopyStatusFailed { - t.Fatalf("repair copy after rejected writes = %#v err=%v, want failed", copyRow, err) - } -} - -func TestStorageUploadRepo_AcquireReplicaRepairItemUsesSharedSurvivor(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - fixture := seedPermanentDeleteRepairFixture(t, db, repos, "acquire-shared-survivor") - follower := newObjectVersion(fixture.bucket.ID, "follower.txt", "01J000000000000000000DEL0U", fixture.version.Size) - follower.Checksum = fixture.version.Checksum - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, follower); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(follower): %v", err) - } - bindPermanentDeleteFollower(t, repos, fixture.upload.ID, follower) - stage := "repair_replica" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "bucket", RefID: fixture.bucket.ID, - RefVersionID: fixture.version.VersionID, IdempotencyKey: "upload:repair-data-set:acquire-shared-survivor", - Payload: map[string]interface{}{"storage_data_set_id": fixture.repair.ID, "storage_upload_copy_id": fixture.repairCopy.ID}, - Status: model.TaskStatusExhausted, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create repair task: %v", err) - } - if _, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: fixture.bucket.ID, Key: fixture.version.Key, VersionID: fixture.version.VersionID, - }); err != nil { - t.Fatalf("DeleteObjectVersionPermanently(source): %v", err) - } - if err := repos.Tasks.RetryExhausted(ctx, task.ID); err != nil { - t.Fatalf("RetryExhausted: %v", err) - } - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil || claimed == nil || claimed.ID != task.ID || claimed.ClaimedAt == nil { - t.Fatalf("ClaimReady(repair): task=%#v err=%v", claimed, err) - } - item, err := repos.Uploads.AcquireReplicaRepairItem(ctx, repository.AcquireReplicaRepairItemInput{ - TaskID: claimed.ID, TaskClaimedAt: *claimed.ClaimedAt, StorageDataSetID: fixture.repair.ID, - StorageUploadCopyID: fixture.repairCopy.ID, BucketID: fixture.bucket.ID, - }) - if err != nil { - t.Fatalf("AcquireReplicaRepairItem: %v", err) - } - if item.Version.VersionID != follower.VersionID || item.Copy.ID != fixture.repairCopy.ID || item.Upload.ID != fixture.upload.ID { - t.Fatalf("acquired repair item = %#v, want surviving version %s and exact copy/upload", item, follower.VersionID) - } - if _, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: fixture.bucket.ID, Key: follower.Key, VersionID: follower.VersionID, - }); !errors.Is(err, repository.ErrPermanentDeleteStorageBusy) { - t.Fatalf("DeleteObjectVersionPermanently while repair owns item error = %v, want storage-work conflict", err) - } -} - -func TestStorageUploadRepo_AcquireReplicaRepairItemRejectsDeletedWorkAndLostClaim(t *testing.T) { - t.Run("deleted work", func(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - fixture := seedPermanentDeleteRepairFixture(t, db, repos, "acquire-deleted-work") - stage := "repair_replica" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "bucket", RefID: fixture.bucket.ID, - RefVersionID: fixture.version.VersionID, IdempotencyKey: "upload:repair-data-set:acquire-deleted-work", - Payload: map[string]interface{}{"storage_data_set_id": fixture.repair.ID, "storage_upload_copy_id": fixture.repairCopy.ID}, - Status: model.TaskStatusExhausted, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create repair task: %v", err) - } - if _, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: fixture.bucket.ID, Key: fixture.version.Key, VersionID: fixture.version.VersionID, - }); err != nil { - t.Fatalf("DeleteObjectVersionPermanently: %v", err) - } - if err := repos.Tasks.RetryExhausted(ctx, task.ID); err != nil { - t.Fatalf("RetryExhausted: %v", err) - } - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil || claimed == nil || claimed.ClaimedAt == nil { - t.Fatalf("ClaimReady: task=%#v err=%v", claimed, err) - } - _, err = repos.Uploads.AcquireReplicaRepairItem(ctx, repository.AcquireReplicaRepairItemInput{ - TaskID: claimed.ID, TaskClaimedAt: *claimed.ClaimedAt, StorageDataSetID: fixture.repair.ID, - StorageUploadCopyID: fixture.repairCopy.ID, BucketID: fixture.bucket.ID, - }) - if !errors.Is(err, repository.ErrReplicaRepairItemCancelled) { - t.Fatalf("AcquireReplicaRepairItem error = %v, want cancelled item", err) - } - }) - - t.Run("lost claim", func(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - fixture := seedPermanentDeleteRepairFixture(t, db, repos, "acquire-lost-claim") - stage := "repair_replica" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "bucket", RefID: fixture.bucket.ID, - RefVersionID: fixture.version.VersionID, IdempotencyKey: "upload:repair-data-set:acquire-lost-claim", - Payload: map[string]interface{}{"storage_data_set_id": fixture.repair.ID, "storage_upload_copy_id": fixture.repairCopy.ID}, - Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create repair task: %v", err) - } - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil || claimed == nil || claimed.ClaimedAt == nil { - t.Fatalf("ClaimReady: task=%#v err=%v", claimed, err) - } - if err := repos.Tasks.Complete(ctx, claimed); err != nil { - t.Fatalf("Complete: %v", err) - } - _, err = repos.Uploads.AcquireReplicaRepairItem(ctx, repository.AcquireReplicaRepairItemInput{ - TaskID: claimed.ID, TaskClaimedAt: *claimed.ClaimedAt, StorageDataSetID: fixture.repair.ID, - StorageUploadCopyID: fixture.repairCopy.ID, BucketID: fixture.bucket.ID, - }) - if !errors.Is(err, repository.ErrTaskClaimLost) { - t.Fatalf("AcquireReplicaRepairItem error = %v, want lost claim", err) - } - }) -} - -type permanentDeleteRepairFixture struct { - bucket *model.Bucket - version *model.ObjectVersion - upload *model.StorageUpload - repair *model.StorageDataSet - repairCopy *model.StorageUploadCopy -} - -func seedPermanentDeleteRepairFixture(t *testing.T, db *bun.DB, repos *repository.Repositories, suffix string) permanentDeleteRepairFixture { - t.Helper() - ctx := context.Background() - bucket := seedBucket(t, db, "permanent-delete-"+suffix) - version := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DEL0R", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: version.VersionID, ContentSize: version.Size, Checksum: version.Checksum, RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("UpdateVersionState cached to uploading: %v", err) - } - primary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding primary: %v", err) - } - repair, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, "202"), CopyIndex: 1, CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding repair: %v", err) - } - for _, input := range []repository.MarkDataSetReadyInput{ - {ID: primary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001")}, - {ID: repair.ID, UploadID: upload.ID, DataSetID: onChainID(t, "2002")}, - } { - if err := repos.Uploads.MarkDataSetReady(ctx, input); err != nil { - t.Fatalf("MarkDataSetReady(%d): %v", input.ID, err) - } - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - {StorageDataSetID: repair.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, CopyIndex: 0, PieceCID: "bafk2bzacepermanentrepair", PieceID: onChainIDPtr(t, "3001"), RetrievalURL: "https://primary.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted primary: %v", err) - } - bindReadableUploadForContent(t, repos, upload.ID, bucket.ID, version.Size, version.Checksum) - gotVersion, err := repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil || gotVersion == nil || gotVersion.State != model.ObjectStateReplicating || gotVersion.StorageUploadID == nil || *gotVersion.StorageUploadID != upload.ID { - t.Fatalf("source after first readable replica = %#v err=%v, want replicating on upload %d", gotVersion, err, upload.ID) - } - if err := repos.Uploads.MarkDataSetUnavailable(ctx, repair.ID, "temporary outage"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - repairCopy, err := repos.Uploads.GetUploadCopy(ctx, upload.ID, 1) - if err != nil || repairCopy == nil { - t.Fatalf("GetUploadCopy(repair): copy=%#v err=%v", repairCopy, err) - } - return permanentDeleteRepairFixture{bucket: bucket, version: version, upload: upload, repair: repair, repairCopy: repairCopy} -} - -func bindPermanentDeleteFollower(t *testing.T, repos *repository.Repositories, uploadID int64, version *model.ObjectVersion) { - t.Helper() - ctx := context.Background() - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("UpdateVersionState follower cached to uploading: %v", err) - } - refs, err := repos.Uploads.BindReadableUploadForVersion(ctx, repository.BindReadableUploadForVersionInput{ - UploadID: uploadID, BucketID: version.BucketID, ContentSize: version.Size, Checksum: version.Checksum, VersionID: version.VersionID, - }) - if err != nil { - t.Fatalf("BindReadableUploadForVersion(follower): %v", err) - } - if len(refs) != 1 || refs[0].VersionID != version.VersionID { - t.Fatalf("BindReadableUploadForVersion refs = %#v, want follower %s", refs, version.VersionID) - } -} - -func TestObjectRepo_DeleteObjectVersionPermanentlyUsesConfiguredStorageCleanupMaxRetries(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "permanent-delete-retries-bucket") - - oldVersion := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DEL0A", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(old): %v", err) - } - uploadID := acceptTestStorageUploadForVersion(t, repos, bucket.ID, oldVersion, "bafk2bzaceretryzero") - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, oldVersion.VersionID, uploadID, model.ObjectStateCached, model.ObjectStateStored); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition(old): %v", err) - } - currentVersion := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DEL0B", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, currentVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(current): %v", err) - } - - maxRetries := 0 - result, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: bucket.ID, - Key: oldVersion.Key, - VersionID: oldVersion.VersionID, - StorageCleanupMaxRetries: &maxRetries, - }) - if err != nil { - t.Fatalf("DeleteObjectVersionPermanently: %v", err) - } - if result.StorageCleanupTaskID == nil { - t.Fatal("expected storage cleanup task id") - } - task, err := repos.Tasks.GetByID(ctx, *result.StorageCleanupTaskID) - if err != nil || task == nil { - t.Fatalf("GetByID(cleanup task): task=%v err=%v", task, err) - } - if task.MaxRetries != 0 { - t.Fatalf("cleanup task max retries = %d, want explicit 0", task.MaxRetries) - } -} - -func TestObjectRepo_DeleteObjectVersionPermanentlyQueuesCleanupForSharedStorageUpload(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "shared-permanent-delete-bucket") - - leader := newObjectVersion(bucket.ID, "leader.txt", "01J000000000000000000DEL03", 10) - leader.Checksum = "shared-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, leader); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(leader): %v", err) - } - uploadID := acceptTestStorageUploadForVersion(t, repos, bucket.ID, leader, "bafk2bzaceshareddelete") - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, leader.VersionID, uploadID, model.ObjectStateCached, model.ObjectStateStored); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition(leader): %v", err) - } - replacement := newObjectVersion(bucket.ID, "leader.txt", "01J000000000000000000DEL06", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, replacement); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(replacement): %v", err) - } - - follower := newObjectVersion(bucket.ID, "follower.txt", "01J000000000000000000DEL04", 10) - follower.Checksum = leader.Checksum - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, follower); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(follower): %v", err) - } - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, follower.VersionID, uploadID, model.ObjectStateCached, model.ObjectStateStored); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition(follower): %v", err) - } - - result, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: bucket.ID, - Key: leader.Key, - VersionID: leader.VersionID, - }) - if err != nil { - t.Fatalf("DeleteObjectVersionPermanently: %v", err) - } - if result.StorageCleanupTaskID == nil { - t.Fatal("expected storage cleanup task id for worker reference recheck") - } - - gotFollower, err := repos.Objects.GetVersionByID(ctx, follower.VersionID) - if err != nil || gotFollower == nil { - t.Fatalf("GetVersionByID(follower): version=%v err=%v", gotFollower, err) - } - if gotFollower.StorageUploadID == nil || *gotFollower.StorageUploadID != uploadID { - t.Fatalf("follower storage upload = %#v, want %d", gotFollower.StorageUploadID, uploadID) - } - - var cleanupTasks int - if err := db.NewRaw(`SELECT COUNT(*) FROM tasks WHERE type = ? AND ref_type = ? AND ref_id = ?`, model.TaskTypeStorageCleanup, "storage_upload", uploadID).Scan(ctx, &cleanupTasks); err != nil { - t.Fatalf("count cleanup tasks: %v", err) - } - if cleanupTasks != 1 { - t.Fatalf("cleanup task count = %d, want one idempotent task", cleanupTasks) - } -} - -func TestObjectRepo_DeleteObjectVersionPermanentlyQueuesCleanupForSharedPieceIdentity(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "shared-piece-delete-bucket") - - oldVersion := newObjectVersion(bucket.ID, "old.txt", "01J000000000000000000DEL07", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(old): %v", err) - } - uploadID := acceptTestStorageUploadForVersion(t, repos, bucket.ID, oldVersion, "bafk2bzacesharedpiece") - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, oldVersion.VersionID, uploadID, model.ObjectStateCached, model.ObjectStateStored); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition(old): %v", err) - } - replacement := newObjectVersion(bucket.ID, "old.txt", "01J000000000000000000DEL08", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, replacement); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(replacement): %v", err) - } - - var dataSetID string - if err := db.NewRaw(`SELECT storage_data_set.data_set_id - FROM storage_upload_copies AS storage_copy - JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id - WHERE storage_copy.upload_id = ?`, uploadID).Scan(ctx, &dataSetID); err != nil { - t.Fatalf("load storage data set id: %v", err) - } - otherVersion := newObjectVersion(bucket.ID, "other.txt", "01J000000000000000000DEL09", 30) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, otherVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(other): %v", err) - } - otherUpload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: otherVersion.VersionID, - ContentSize: otherVersion.Size, - Checksum: otherVersion.Checksum, - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt(other): %v", err) - } - seedCommittedUploadCopies(t, repos, bucket.ID, otherUpload.ID, "bafk2bzacesharedpiece", []storageUploadCopySeed{ - {ProviderID: onChainIDPtr(t, "101"), DataSetID: onChainIDPtr(t, dataSetID), PieceID: onChainIDPtr(t, "2001"), TransferMethod: model.StorageCopyTransferMethodPeerPull, RetrievalURL: strPtr("https://provider.example/" + otherVersion.VersionID)}, - }) - bindReadableUploadForContent(t, repos, otherUpload.ID, bucket.ID, otherVersion.Size, otherVersion.Checksum) - finalizeUploadForTest(t, repos, otherUpload.ID) - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, otherVersion.VersionID, otherUpload.ID, model.ObjectStateCached, model.ObjectStateStored); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition(other): %v", err) - } - - result, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: bucket.ID, - Key: oldVersion.Key, - VersionID: oldVersion.VersionID, - }) - if err != nil { - t.Fatalf("DeleteObjectVersionPermanently: %v", err) - } - if result.StorageCleanupTaskID == nil { - t.Fatal("expected storage cleanup task id for worker piece reference recheck") - } -} - -func TestObjectRepo_DeleteObjectVersionPermanentlyRequeuesRetainedCleanupTask(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "retained-cleanup-requeue-bucket") - - oldVersion := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DEL0C", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(old): %v", err) - } - uploadID := acceptTestStorageUploadForVersion(t, repos, bucket.ID, oldVersion, "bafk2bzaceretained") - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, oldVersion.VersionID, uploadID, model.ObjectStateCached, model.ObjectStateStored); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition(old): %v", err) - } - currentVersion := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DEL0D", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, currentVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(current): %v", err) - } - - now := time.Now() - statusMessage := "Remote replicas kept because another object version still uses them" - task := &model.Task{ - Type: model.TaskTypeStorageCleanup, - RefType: "storage_upload", - RefID: uploadID, - IdempotencyKey: "storage_cleanup:" + strconv.FormatInt(uploadID, 10), - Status: model.TaskStatusCompleted, - MaxRetries: 1, - StatusMessage: &statusMessage, - ScheduledAt: now.Add(-time.Hour), - CompletedAt: &now, - Payload: map[string]interface{}{ - "storage_upload_id": uploadID, - "deleted_source_version": "stale-version", - "deleted_source_versions": []string{"stale-version"}, - }, - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create retained cleanup task: %v", err) - } - if _, err := db.NewInsert().Model(&model.StorageCleanupCopy{ - TaskID: task.ID, - UploadID: uploadID, - CopyIndex: 0, - ProviderID: onChainIDPtr(t, "101"), - DataSetID: onChainIDPtr(t, "1001"), - ClientDataSetID: onChainIDPtr(t, "5001"), - PieceID: onChainIDPtr(t, "2001"), - PieceCID: "bafk2bzaceretained", - Status: model.StorageCleanupCopyStatusPending, - }).Exec(ctx); err != nil { - t.Fatalf("insert retained cleanup copy: %v", err) - } - - maxRetries := 9 - result, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: bucket.ID, - Key: oldVersion.Key, - VersionID: oldVersion.VersionID, - StorageCleanupMaxRetries: &maxRetries, - }) - if err != nil { - t.Fatalf("DeleteObjectVersionPermanently: %v", err) - } - if result.StorageCleanupTaskID == nil || *result.StorageCleanupTaskID != task.ID { - t.Fatalf("cleanup task id = %v, want retained task %d", result.StorageCleanupTaskID, task.ID) - } - got, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil || got == nil { - t.Fatalf("GetByID(retained cleanup): task=%v err=%v", got, err) - } - if got.Status != model.TaskStatusQueued || got.CompletedAt != nil || got.StatusMessage != nil || got.LastError != nil || got.WaitReason != nil || got.RetryCount != 0 { - t.Fatalf("requeued task diagnostics = status:%s completed:%v message:%v error:%v wait:%v retries:%d, want clean queued task", got.Status, got.CompletedAt, got.StatusMessage, got.LastError, got.WaitReason, got.RetryCount) - } - if got.MaxRetries != 9 { - t.Fatalf("requeued task max retries = %d, want 9", got.MaxRetries) - } - if got.Payload["deleted_source_version"] != "stale-version" { - t.Fatalf("deleted_source_version = %#v, want stale-version", got.Payload["deleted_source_version"]) - } - gotVersions := payloadStringSlice(got.Payload, "deleted_source_versions") - wantVersions := []string{"stale-version", oldVersion.VersionID} - if len(gotVersions) != len(wantVersions) { - t.Fatalf("deleted_source_versions = %#v, want %#v", gotVersions, wantVersions) - } - for i := range wantVersions { - if gotVersions[i] != wantVersions[i] { - t.Fatalf("deleted_source_versions = %#v, want %#v", gotVersions, wantVersions) - } - } -} - -func TestObjectRepo_DeleteObjectVersionPermanentlyReusesActiveCleanupTask(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "active-cleanup-reuse-bucket") - - oldVersion := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DEL0G", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(old): %v", err) - } - uploadID := acceptTestStorageUploadForVersion(t, repos, bucket.ID, oldVersion, "bafk2bzaceactivecleanup") - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, oldVersion.VersionID, uploadID, model.ObjectStateCached, model.ObjectStateStored); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition(old): %v", err) - } - currentVersion := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DEL0H", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, currentVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(current): %v", err) - } - - task := &model.Task{ - Type: model.TaskTypeStorageCleanup, - RefType: "storage_upload", - RefID: uploadID, - IdempotencyKey: "storage_cleanup:" + strconv.FormatInt(uploadID, 10), - Status: model.TaskStatusQueued, - MaxRetries: 1, - ScheduledAt: time.Now(), - Payload: map[string]interface{}{ - "storage_upload_id": uploadID, - "deleted_source_version": "existing-version", - }, - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create active cleanup task: %v", err) - } - - maxRetries := 9 - result, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: bucket.ID, - Key: oldVersion.Key, - VersionID: oldVersion.VersionID, - StorageCleanupMaxRetries: &maxRetries, - }) - if err != nil { - t.Fatalf("DeleteObjectVersionPermanently: %v", err) - } - if result.StorageCleanupTaskID == nil || *result.StorageCleanupTaskID != task.ID { - t.Fatalf("cleanup task id = %v, want active task %d", result.StorageCleanupTaskID, task.ID) - } - got, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil || got == nil { - t.Fatalf("GetByID(active cleanup): task=%v err=%v", got, err) - } - if got.Status != model.TaskStatusQueued || got.MaxRetries != 1 || got.Payload["deleted_source_version"] != "existing-version" { - t.Fatalf("active task changed = status:%s maxRetries:%d payload:%#v, want preserved status and retry config", got.Status, got.MaxRetries, got.Payload) - } - gotVersions := payloadStringSlice(got.Payload, "deleted_source_versions") - wantVersions := []string{"existing-version", oldVersion.VersionID} - if len(gotVersions) != len(wantVersions) { - t.Fatalf("deleted_source_versions = %#v, want %#v", gotVersions, wantVersions) - } - for i := range wantVersions { - if gotVersions[i] != wantVersions[i] { - t.Fatalf("deleted_source_versions = %#v, want %#v", gotVersions, wantVersions) - } - } -} - -func TestObjectRepo_DeleteObjectVersionPermanentlyDoesNotRequeueTerminalCleanupTask(t *testing.T) { - for _, status := range []model.TaskStatus{ - model.TaskStatusFailed, - model.TaskStatusExhausted, - model.TaskStatusCancelled, - } { - t.Run(string(status), func(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "terminal-cleanup-"+string(status)+"-bucket") - - oldVersion := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DEL0E", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(old): %v", err) - } - uploadID := acceptTestStorageUploadForVersion(t, repos, bucket.ID, oldVersion, "bafk2bzaceterminalcleanup") - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, oldVersion.VersionID, uploadID, model.ObjectStateCached, model.ObjectStateStored); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition(old): %v", err) - } - currentVersion := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DEL0F", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, currentVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(current): %v", err) - } - - lastError := "remote cleanup stopped" - task := &model.Task{ - Type: model.TaskTypeStorageCleanup, - RefType: "storage_upload", - RefID: uploadID, - IdempotencyKey: "storage_cleanup:" + strconv.FormatInt(uploadID, 10), - Status: status, - MaxRetries: 1, - LastError: &lastError, - ScheduledAt: time.Now(), - Payload: map[string]interface{}{"storage_upload_id": uploadID}, - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create terminal cleanup task: %v", err) - } - - result, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: bucket.ID, - Key: oldVersion.Key, - VersionID: oldVersion.VersionID, - }) - if err != nil { - t.Fatalf("DeleteObjectVersionPermanently: %v", err) - } - if result.StorageCleanupTaskID == nil || *result.StorageCleanupTaskID != task.ID { - t.Fatalf("cleanup task id = %v, want terminal task %d", result.StorageCleanupTaskID, task.ID) - } - got, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil || got == nil { - t.Fatalf("GetByID(terminal cleanup): task=%v err=%v", got, err) - } - if got.Status != status || got.LastError == nil || *got.LastError != lastError { - t.Fatalf("terminal task changed = status:%s error:%v, want preserved %s task", got.Status, got.LastError, status) - } - }) - } -} - -func TestObjectRepo_DeleteObjectVersionPermanentlyDeletesCurrentDataVersionAndPromotesPreviousVersion(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "current-permanent-delete-bucket") - - oldVersion := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DEL05", 10) - objectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion) - if err != nil { - t.Fatalf("CreateVersionAndSetCurrent(old): %v", err) - } - currentVersion := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DEL06", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, currentVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(current): %v", err) - } - - result, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: bucket.ID, - Key: currentVersion.Key, - VersionID: currentVersion.VersionID, - }) - if err != nil { - t.Fatalf("DeleteObjectVersionPermanently: %v", err) - } - if result.CacheKey != currentVersion.CacheKey { - t.Fatalf("cache key = %q, want %q", result.CacheKey, currentVersion.CacheKey) - } - - gotDeleted, err := repos.Objects.GetVersionByID(ctx, currentVersion.VersionID) - if err != nil { - t.Fatalf("GetVersionByID(deleted): %v", err) - } - if gotDeleted != nil { - t.Fatalf("deleted current version still exists: %#v", gotDeleted) - } - gotOld, err := repos.Objects.GetVersionByID(ctx, oldVersion.VersionID) - if err != nil || gotOld == nil { - t.Fatalf("GetVersionByID(promoted): version=%v err=%v", gotOld, err) - } - if !gotOld.IsCurrent { - t.Fatalf("old version is_current = false, want true") - } - gotObject, err := repos.Objects.GetObjectByID(ctx, objectID) - if err != nil || gotObject == nil { - t.Fatalf("GetObjectByID: object=%v err=%v", gotObject, err) - } -} - -func TestObjectRepo_DeleteObjectVersionPermanentlyDeletesObjectWhenOnlyVersionWasCurrent(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "single-current-permanent-delete-bucket") - - version := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DEL07", 10) - objectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version) - if err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - - if _, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: bucket.ID, - Key: version.Key, - VersionID: version.VersionID, - }); err != nil { - t.Fatalf("DeleteObjectVersionPermanently: %v", err) - } - - gotVersion, err := repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil { - t.Fatalf("GetVersionByID(deleted): %v", err) - } - if gotVersion != nil { - t.Fatalf("deleted only version still exists: %#v", gotVersion) - } - gotObject, err := repos.Objects.GetObjectByID(ctx, objectID) - if err != nil { - t.Fatalf("GetObjectByID: %v", err) - } - if gotObject != nil { - t.Fatalf("object row still exists after deleting only version: %#v", gotObject) - } -} - -func TestObjectRepo_DeleteDeletedObjectPermanentlyRemovesAllVersionsAndQueuesStorageCleanup(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "deleted-object-permanent-delete-bucket") - - first := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DOB01", 10) - first.Checksum = "deleted-object-shared-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, first); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(first): %v", err) - } - uploadID := acceptTestStorageUploadForVersion(t, repos, bucket.ID, first, "bafk2bzacedeletedobject") - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, first.VersionID, uploadID, model.ObjectStateCached, model.ObjectStateStored); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition(first): %v", err) - } - - second := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DOB02", 10) - second.Checksum = first.Checksum - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, second); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(second): %v", err) - } - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, second.VersionID, uploadID, model.ObjectStateCached, model.ObjectStateStored); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition(second): %v", err) - } - - marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "file.txt", "01J000000000000000000DOB03") - if err != nil { - t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) - } - - result, err := repos.Objects.DeleteDeletedObjectPermanently(ctx, repository.DeleteDeletedObjectInput{ - BucketID: bucket.ID, - Key: "file.txt", - DeleteMarkerVersionID: marker.VersionID, - }) - if err != nil { - t.Fatalf("DeleteDeletedObjectPermanently: %v", err) - } - if result.DataVersionsDeleted != 2 || result.DeleteMarkersDeleted != 1 { - t.Fatalf("deleted counts = data:%d markers:%d, want 2 data and 1 marker", result.DataVersionsDeleted, result.DeleteMarkersDeleted) - } - if len(result.DeletedVersions) != 2 { - t.Fatalf("deleted version snapshots len = %d, want 2", len(result.DeletedVersions)) - } - if len(result.StorageCleanupTaskIDs) != 1 { - t.Fatalf("storage cleanup task ids = %#v, want one task", result.StorageCleanupTaskIDs) - } - - gotObject, err := repos.Objects.GetObjectByBucketAndKey(ctx, bucket.ID, "file.txt") - if err != nil { - t.Fatalf("GetObjectByBucketAndKey: %v", err) - } - if gotObject != nil { - t.Fatalf("object still exists after deleted object permanent delete: %#v", gotObject) - } - for _, versionID := range []string{first.VersionID, second.VersionID, marker.VersionID} { - gotVersion, err := repos.Objects.GetVersionByID(ctx, versionID) - if err != nil { - t.Fatalf("GetVersionByID(%s): %v", versionID, err) - } - if gotVersion != nil { - t.Fatalf("version %s still exists after deleted object permanent delete", versionID) - } - } - - var deletionCount int - if err := db.NewRaw(`SELECT COUNT(*) FROM object_deletions WHERE key = ?`, "file.txt").Scan(ctx, &deletionCount); err != nil { - t.Fatalf("count object_deletions: %v", err) - } - if deletionCount != 2 { - t.Fatalf("object_deletions count = %d, want one row for each data version", deletionCount) - } - - var markerDeletionCount int - if err := db.NewRaw(`SELECT COUNT(*) FROM object_deletions WHERE version_id = ?`, marker.VersionID).Scan(ctx, &markerDeletionCount); err != nil { - t.Fatalf("count marker object_deletions: %v", err) - } - if markerDeletionCount != 0 { - t.Fatalf("delete marker audit rows = %d, want 0", markerDeletionCount) - } - - task, err := repos.Tasks.GetByID(ctx, result.StorageCleanupTaskIDs[0]) - if err != nil || task == nil { - t.Fatalf("GetByID(cleanup task): task=%v err=%v", task, err) - } - if task.RefID != uploadID || task.Type != model.TaskTypeStorageCleanup { - t.Fatalf("cleanup task = type:%s refID:%d, want storage cleanup for upload %d", task.Type, task.RefID, uploadID) - } -} - -func TestObjectRepo_DeleteDeletedObjectPermanentlyReportsActiveStorageWork(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "deleted-object-active-storage-work") - version := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DEL0V", 10) - objectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version) - if err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - stage := "prepare_upload" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "object", RefID: objectID, RefVersionID: version.VersionID, - IdempotencyKey: "upload:" + version.VersionID, Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create upload task: %v", err) - } - marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, version.Key, "01J000000000000000000DEL0W") - if err != nil { - t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) - } - - _, err = repos.Objects.DeleteDeletedObjectPermanently(ctx, repository.DeleteDeletedObjectInput{ - BucketID: bucket.ID, Key: version.Key, DeleteMarkerVersionID: marker.VersionID, - }) - if !errors.Is(err, repository.ErrPermanentDeleteStorageBusy) { - t.Fatalf("DeleteDeletedObjectPermanently error = %v, want storage-work conflict", err) - } - if got, loadErr := repos.Objects.GetVersionByID(ctx, version.VersionID); loadErr != nil || got == nil { - t.Fatalf("data version after rejected delete = %#v err=%v", got, loadErr) - } - if got, loadErr := repos.Objects.GetVersionByID(ctx, marker.VersionID); loadErr != nil || got == nil { - t.Fatalf("delete marker after rejected delete = %#v err=%v", got, loadErr) - } -} - -func TestObjectRepo_DeleteDeletedObjectPermanentlyCancelsStoppedReplicaRepair(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - fixture := seedPermanentDeleteRepairFixture(t, db, repos, "deleted-object-stopped-repair") - marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, fixture.bucket.ID, fixture.version.Key, model.NewVersionID()) - if err != nil { - t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) - } - - if _, err := repos.Objects.DeleteDeletedObjectPermanently(ctx, repository.DeleteDeletedObjectInput{ - BucketID: fixture.bucket.ID, Key: fixture.version.Key, DeleteMarkerVersionID: marker.VersionID, - }); err != nil { - t.Fatalf("DeleteDeletedObjectPermanently: %v", err) - } - copyRow, err := repos.Uploads.GetUploadCopyByID(ctx, fixture.repairCopy.ID) - if err != nil || copyRow == nil || copyRow.Status != model.StorageUploadCopyStatusFailed { - t.Fatalf("repair copy after deleted-object cleanup = %#v err=%v, want failed", copyRow, err) - } - upload, err := repos.Uploads.GetByID(ctx, fixture.upload.ID) - if err != nil || upload == nil || upload.Status != model.StorageUploadStatusSuperseded { - t.Fatalf("upload after deleted-object cleanup = %#v err=%v, want superseded", upload, err) - } -} - -func TestObjectRepo_DeleteDeletedObjectPermanentlyScopesStorageCleanupPayloadsByUpload(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "deleted-object-scoped-cleanup-payload-bucket") - - acceptUpload := func(version *model.ObjectVersion, pieceCID string) int64 { - t.Helper() - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt(%s): %v", version.VersionID, err) - } - uploadIDText := strconv.FormatInt(upload.ID, 10) - seedCommittedUploadCopies(t, repos, bucket.ID, upload.ID, pieceCID, []storageUploadCopySeed{{ - ProviderID: onChainIDPtr(t, "101"), - DataSetID: onChainIDPtr(t, "1001"+uploadIDText), - PieceID: onChainIDPtr(t, "2001"+uploadIDText), - TransferMethod: model.StorageCopyTransferMethodIngress, - RetrievalURL: strPtr("https://provider.example/" + version.VersionID), - IsNewDataSet: true, - }}) - bindReadableUploadForContent(t, repos, upload.ID, bucket.ID, version.Size, version.Checksum) - finalizeUploadForTest(t, repos, upload.ID) - return upload.ID - } - - first := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DOB08", 10) - first.Checksum = "deleted-object-first-upload-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, first); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(first): %v", err) - } - firstUploadID := acceptUpload(first, "bafk2bzacescopedpayloadone") - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, first.VersionID, firstUploadID, model.ObjectStateCached, model.ObjectStateStored); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition(first): %v", err) - } - - second := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DOB09", 20) - second.Checksum = "deleted-object-second-upload-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, second); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(second): %v", err) - } - secondUploadID := acceptUpload(second, "bafk2bzacescopedpayloadtwo") - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, second.VersionID, secondUploadID, model.ObjectStateCached, model.ObjectStateStored); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition(second): %v", err) - } - - marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "file.txt", "01J000000000000000000DOB0A") - if err != nil { - t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) - } - - result, err := repos.Objects.DeleteDeletedObjectPermanently(ctx, repository.DeleteDeletedObjectInput{ - BucketID: bucket.ID, - Key: "file.txt", - DeleteMarkerVersionID: marker.VersionID, - }) - if err != nil { - t.Fatalf("DeleteDeletedObjectPermanently: %v", err) - } - if len(result.StorageCleanupTaskIDs) != 2 { - t.Fatalf("storage cleanup task ids = %#v, want one task per storage upload", result.StorageCleanupTaskIDs) - } - - wantVersionByUpload := map[int64]string{ - firstUploadID: first.VersionID, - secondUploadID: second.VersionID, - } - seenUploads := make(map[int64]bool) - for _, taskID := range result.StorageCleanupTaskIDs { - task, err := repos.Tasks.GetByID(ctx, taskID) - if err != nil || task == nil { - t.Fatalf("GetByID(cleanup task %d): task=%v err=%v", taskID, task, err) - } - wantVersion, ok := wantVersionByUpload[task.RefID] - if !ok { - t.Fatalf("cleanup task %d ref upload = %d, want one of %#v", taskID, task.RefID, wantVersionByUpload) - } - gotVersions := payloadStringSlice(task.Payload, "deleted_source_versions") - if len(gotVersions) != 1 || gotVersions[0] != wantVersion { - t.Fatalf("cleanup task %d deleted_source_versions = %#v, want only %q", taskID, gotVersions, wantVersion) - } - if gotLegacy, ok := task.Payload["deleted_source_version"].(string); !ok || gotLegacy != wantVersion { - t.Fatalf("cleanup task %d deleted_source_version = %#v, want %q", taskID, task.Payload["deleted_source_version"], wantVersion) - } - seenUploads[task.RefID] = true - } - if len(seenUploads) != len(wantVersionByUpload) { - t.Fatalf("cleanup task uploads = %#v, want uploads %#v", seenUploads, wantVersionByUpload) - } -} - -func TestObjectRepo_DeleteDeletedObjectPermanentlyRejectsStaleMarker(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "deleted-object-stale-marker-bucket") - - version := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DOB04", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "file.txt", "01J000000000000000000DOB05") - if err != nil { - t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) - } - replacement := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000DOB06", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, replacement); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(replacement): %v", err) - } - - _, err = repos.Objects.DeleteDeletedObjectPermanently(ctx, repository.DeleteDeletedObjectInput{ - BucketID: bucket.ID, - Key: "file.txt", - DeleteMarkerVersionID: marker.VersionID, - }) - if !errors.Is(err, repository.ErrConflict) { - t.Fatalf("DeleteDeletedObjectPermanently error = %v, want ErrConflict", err) - } - - got, err := repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil || got == nil { - t.Fatalf("original data version should remain: version=%v err=%v", got, err) - } -} - -func TestObjectRepo_DeleteDeletedObjectPermanentlyAllowsMarkerOnlyObject(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "deleted-object-marker-only-bucket") - - marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "missing.txt", "01J000000000000000000DOB07") - if err != nil { - t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) - } - - result, err := repos.Objects.DeleteDeletedObjectPermanently(ctx, repository.DeleteDeletedObjectInput{ - BucketID: bucket.ID, - Key: "missing.txt", - DeleteMarkerVersionID: marker.VersionID, - }) - if err != nil { - t.Fatalf("DeleteDeletedObjectPermanently: %v", err) - } - if result.DataVersionsDeleted != 0 || result.DeleteMarkersDeleted != 1 || len(result.StorageCleanupTaskIDs) != 0 { - t.Fatalf("result = data:%d markers:%d cleanup:%#v, want marker-only delete", result.DataVersionsDeleted, result.DeleteMarkersDeleted, result.StorageCleanupTaskIDs) - } - gotObject, err := repos.Objects.GetObjectByBucketAndKey(ctx, bucket.ID, "missing.txt") - if err != nil { - t.Fatalf("GetObjectByBucketAndKey: %v", err) - } - if gotObject != nil { - t.Fatalf("marker-only object still exists: %#v", gotObject) - } -} - -func payloadStringSlice(payload map[string]interface{}, key string) []string { - values, ok := payload[key].([]interface{}) - if ok { - out := make([]string, 0, len(values)) - for _, value := range values { - if text, ok := value.(string); ok { - out = append(out, text) - } - } - return out - } - textValues, ok := payload[key].([]string) - if ok { - return textValues - } - return nil -} - -func TestObjectRepo_DeleteObjectVersionPermanentlyKeepsLegacyBareCommitTransaction(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - fixture := seedPermanentDeleteRepairFixture(t, db, repos, "legacy-bare-transaction") - if err := repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: fixture.upload.ID, CopyIndex: fixture.repairCopy.CopyIndex, - PieceCID: "bafk2bzacepermanentrepair", RetrievalURL: "https://repair.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - // Older code could send a submitted copy back to 'piece_ready' without clearing - // its transaction, leaving evidence the durable model never owned. - if _, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("commit_transaction_id = ?", "0xlegacybare"). - Where("id = ?", fixture.repairCopy.ID). - Exec(ctx); err != nil { - t.Fatalf("seed legacy bare transaction: %v", err) - } - - _, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: fixture.bucket.ID, Key: fixture.version.Key, VersionID: fixture.version.VersionID, - }) - if !errors.Is(err, repository.ErrPermanentDeleteStorageBusy) { - t.Fatalf("DeleteObjectVersionPermanently error = %v, want legacy storage-work conflict", err) - } - got, err := repos.Uploads.GetUploadCopyByID(ctx, fixture.repairCopy.ID) - if err != nil || got == nil || got.Status != model.StorageUploadCopyStatusPieceReady || - got.CommitTransactionID == nil || *got.CommitTransactionID != "0xlegacybare" { - t.Fatalf("legacy copy = %#v err=%v, want retained transaction evidence", got, err) - } -} diff --git a/internal/db/repository/object_repo.go b/internal/db/repository/object_repo.go index 1057e35..5ee21df 100644 --- a/internal/db/repository/object_repo.go +++ b/internal/db/repository/object_repo.go @@ -8,11 +8,14 @@ import ( "maps" "slices" "sort" + "strconv" "time" "github.com/strahe/synaps3/internal/cacheeviction" "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagecommit" "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" ) // BunObjectRepo implements ObjectRepository using Bun ORM. @@ -37,10 +40,7 @@ func (r *BunObjectRepo) CreateVersionAndSetCurrent(ctx context.Context, version if !shouldRetryObjectWrite(err, canRestartTx) || attempt >= 19 { return 0, err } - delay := time.Duration(attempt+1) * 25 * time.Millisecond - if delay > 200*time.Millisecond { - delay = 200 * time.Millisecond - } + delay := min(time.Duration(attempt+1)*25*time.Millisecond, 200*time.Millisecond) timer := time.NewTimer(delay) select { case <-ctx.Done(): @@ -66,10 +66,7 @@ func (r *BunObjectRepo) CreateVersionAndSetCurrentIfChanged(ctx context.Context, if !shouldRetryObjectWrite(err, canRestartTx) || attempt >= 19 { return ObjectVersionWriteResult{}, err } - delay := time.Duration(attempt+1) * 25 * time.Millisecond - if delay > 200*time.Millisecond { - delay = 200 * time.Millisecond - } + delay := min(time.Duration(attempt+1)*25*time.Millisecond, 200*time.Millisecond) timer := time.NewTimer(delay) select { case <-ctx.Done(): @@ -99,10 +96,7 @@ func (r *BunObjectRepo) CreateRestoredVersionAndSetCurrent(ctx context.Context, if !shouldRetryObjectWrite(err, canRestartTx) || attempt >= 19 { return 0, err } - delay := time.Duration(attempt+1) * 25 * time.Millisecond - if delay > 200*time.Millisecond { - delay = 200 * time.Millisecond - } + delay := min(time.Duration(attempt+1)*25*time.Millisecond, 200*time.Millisecond) timer := time.NewTimer(delay) select { case <-ctx.Done(): @@ -123,12 +117,9 @@ func (r *BunObjectRepo) CreateDeleteMarkerAndSetCurrent(ctx context.Context, buc Key: key, Size: 0, ETag: "", - Checksum: "", ContentType: "", - CacheKey: "", - InCache: false, + Metadata: map[string]string{}, IsDeleteMarker: true, - State: model.ObjectStateCached, } _, canRestartTx := r.db.(*bun.DB) @@ -142,10 +133,7 @@ func (r *BunObjectRepo) CreateDeleteMarkerAndSetCurrent(ctx context.Context, buc if !shouldRetryObjectWrite(err, canRestartTx) || attempt >= 19 { return nil, err } - delay := time.Duration(attempt+1) * 25 * time.Millisecond - if delay > 200*time.Millisecond { - delay = 200 * time.Millisecond - } + delay := min(time.Duration(attempt+1)*25*time.Millisecond, 200*time.Millisecond) timer := time.NewTimer(delay) select { case <-ctx.Done(): @@ -178,11 +166,11 @@ func (r *BunObjectRepo) DeleteObjectVersionPermanently(ctx context.Context, inpu if version == nil { return ErrNotFound } - preliminaryUploads, err := authoritativeStorageUploadsForVersions(ctx, db, []*model.ObjectVersion{version}) + preliminaryUploads, err := authoritativeStorageContentsForVersions(ctx, db, []*model.ObjectVersion{version}) if err != nil { return err } - lockedUploads, err := lockStorageUploadsByID(ctx, db, sortedStorageUploadIDs(preliminaryUploads)) + lockedUploads, err := lockStorageContentsByID(ctx, db, sortedContentIDs(preliminaryUploads)) if err != nil { if errors.Is(err, ErrNotFound) { return ErrPermanentDeleteStorageBusy @@ -208,11 +196,11 @@ func (r *BunObjectRepo) DeleteObjectVersionPermanently(ctx context.Context, inpu if err := objectVersionPermanentDeleteStateError(version.State); err != nil { return err } - currentUploads, err := authoritativeStorageUploadsForVersions(ctx, db, []*model.ObjectVersion{version}) + currentUploads, err := authoritativeStorageContentsForVersions(ctx, db, []*model.ObjectVersion{version}) if err != nil { return err } - if !sameStorageUploadIDs(preliminaryUploads, currentUploads) { + if !sameContentIDs(preliminaryUploads, currentUploads) { return ErrPermanentDeleteStorageBusy } if err := prepareObjectVersionsForPermanentDelete(ctx, db, []*model.ObjectVersion{version}, lockedUploads); err != nil { @@ -223,18 +211,13 @@ func (r *BunObjectRepo) DeleteObjectVersionPermanently(ctx context.Context, inpu now := time.Now() deletion := &model.ObjectDeletion{ - BucketID: version.BucketID, - ObjectID: version.ObjectID, - Key: version.Key, - VersionID: version.VersionID, - CacheKey: version.CacheKey, - StorageUploadID: version.StorageUploadID, - Size: version.Size, - Checksum: version.Checksum, - CacheCleanupStatus: model.CacheCleanupStatusPending, - CreatedAt: now, - UpdatedAt: now, - DeletedAt: now, + BucketID: version.BucketID, + ObjectID: version.ObjectID, + Key: version.Key, + VersionID: version.VersionID, + ContentID: version.ContentID, + Size: version.Size, + DeletedAt: now, } if _, err := db.NewInsert().Model(deletion).Exec(ctx); err != nil { if isUniqueViolation(err) { @@ -243,17 +226,21 @@ func (r *BunObjectRepo) DeleteObjectVersionPermanently(ctx context.Context, inpu return fmt.Errorf("recording object deletion: %w", err) } result.DeletionID = deletion.ID - result.CacheKey = version.CacheKey - result.StorageUploadID = version.StorageUploadID + result.ContentID = version.ContentID - if version.StorageUploadID != nil { - cleanupTaskID, err := createStorageCleanupTaskForDeletedVersion(ctx, db, *version.StorageUploadID, version.VersionID, input.StorageCleanupMaxRetries) + if version.ContentID != nil { + cleanup, err := reserveStorageCleanupForDeletedVersions(ctx, db, *version.ContentID, []string{version.VersionID}) if err != nil { return err } - result.StorageCleanupTaskID = cleanupTaskID + result.StorageCleanup = cleanup } + if wasCurrent { + if err := repointObjectAwayFromVersion(ctx, db, objectID, version.VersionID); err != nil { + return err + } + } res, err := db.NewDelete(). Model((*model.ObjectVersion)(nil)). Where("version_id = ?", version.VersionID). @@ -266,7 +253,7 @@ func (r *BunObjectRepo) DeleteObjectVersionPermanently(ctx context.Context, inpu return ErrNotFound } if wasCurrent { - if err := promoteLatestVersionOrDeleteObject(ctx, db, objectID); err != nil { + if err := deleteObjectIdentityIfEmpty(ctx, db, objectID); err != nil { return err } } @@ -310,11 +297,11 @@ func (r *BunObjectRepo) DeleteDeletedObjectPermanently(ctx context.Context, inpu versionIDs = append(versionIDs, versions[i].VersionID) } preliminaryDataVersions := dataObjectVersionPointers(versions) - preliminaryUploads, err := authoritativeStorageUploadsForVersions(ctx, db, preliminaryDataVersions) + preliminaryUploads, err := authoritativeStorageContentsForVersions(ctx, db, preliminaryDataVersions) if err != nil { return err } - lockedUploads, err := lockStorageUploadsByID(ctx, db, sortedStorageUploadIDs(preliminaryUploads)) + lockedUploads, err := lockStorageContentsByID(ctx, db, sortedContentIDs(preliminaryUploads)) if err != nil { if errors.Is(err, ErrNotFound) { return ErrPermanentDeleteStorageBusy @@ -354,11 +341,11 @@ func (r *BunObjectRepo) DeleteDeletedObjectPermanently(ctx context.Context, inpu } dataVersions = append(dataVersions, &versions[i]) } - currentUploads, err := authoritativeStorageUploadsForVersions(ctx, db, dataVersions) + currentUploads, err := authoritativeStorageContentsForVersions(ctx, db, dataVersions) if err != nil { return err } - if !sameStorageUploadIDs(preliminaryUploads, currentUploads) { + if !sameContentIDs(preliminaryUploads, currentUploads) { return ErrPermanentDeleteStorageBusy } if err := prepareObjectVersionsForPermanentDelete(ctx, db, dataVersions, lockedUploads); err != nil { @@ -375,27 +362,22 @@ func (r *BunObjectRepo) DeleteDeletedObjectPermanently(ctx context.Context, inpu continue } deletions = append(deletions, model.ObjectDeletion{ - BucketID: version.BucketID, - ObjectID: version.ObjectID, - Key: version.Key, - VersionID: version.VersionID, - CacheKey: version.CacheKey, - StorageUploadID: version.StorageUploadID, - Size: version.Size, - Checksum: version.Checksum, - CacheCleanupStatus: model.CacheCleanupStatusPending, - CreatedAt: now, - UpdatedAt: now, - DeletedAt: now, + BucketID: version.BucketID, + ObjectID: version.ObjectID, + Key: version.Key, + VersionID: version.VersionID, + ContentID: version.ContentID, + Size: version.Size, + DeletedAt: now, }) result.DeletedVersions = append(result.DeletedVersions, DeletedObjectVersionSnapshot{ VersionID: version.VersionID, - CacheKey: version.CacheKey, + ContentID: version.ContentID, }) result.DataVersionsDeleted++ - if version.StorageUploadID != nil { - uploadID := *version.StorageUploadID - deletedVersionIDsByUpload[uploadID] = append(deletedVersionIDsByUpload[uploadID], version.VersionID) + if version.ContentID != nil { + contentID := *version.ContentID + deletedVersionIDsByUpload[contentID] = append(deletedVersionIDsByUpload[contentID], version.VersionID) } } @@ -408,16 +390,26 @@ func (r *BunObjectRepo) DeleteDeletedObjectPermanently(ctx context.Context, inpu } } - for uploadID, deletedVersionIDs := range deletedVersionIDsByUpload { - cleanupTaskID, err := createStorageCleanupTaskForDeletedVersions(ctx, db, uploadID, deletedVersionIDs, input.StorageCleanupMaxRetries) + for contentID, deletedVersionIDs := range deletedVersionIDsByUpload { + cleanup, err := reserveStorageCleanupForDeletedVersions(ctx, db, contentID, deletedVersionIDs) if err != nil { return err } - if cleanupTaskID != nil { - result.StorageCleanupTaskIDs = append(result.StorageCleanupTaskIDs, *cleanupTaskID) + if cleanup != nil { + result.StorageCleanups = append(result.StorageCleanups, *cleanup) } } + // The object stops pointing at any version before the rows go; the + // pointer's foreign key would otherwise refuse the delete. + if _, err := db.NewUpdate(). + Model((*model.Object)(nil)). + Set("current_version_id = NULL"). + Set("updated_at = ?", now). + Where("id = ?", current.ObjectID). + Exec(ctx); err != nil { + return fmt.Errorf("clearing object current version: %w", err) + } if _, err := db.NewDelete(). Model((*model.ObjectVersion)(nil)). Where("object_id = ?", current.ObjectID). @@ -430,6 +422,7 @@ func (r *BunObjectRepo) DeleteDeletedObjectPermanently(ctx context.Context, inpu Exec(ctx); err != nil { return fmt.Errorf("deleting object identity: %w", err) } + return nil }) if err != nil { @@ -438,30 +431,90 @@ func (r *BunObjectRepo) DeleteDeletedObjectPermanently(ctx context.Context, inpu return result, nil } -func (r *BunObjectRepo) UpdateObjectDeletionCacheCleanup(ctx context.Context, versionID string, status model.CacheCleanupStatus, cacheError string) error { - if versionID == "" || status == "" { - return fmt.Errorf("updating object deletion cache cleanup: %w", ErrInvalidInput) +func (r *BunObjectRepo) ClearContentCachePresence(ctx context.Context, contentID int64) error { + if contentID < 1 { + return fmt.Errorf("clearing content cache presence: %w", ErrInvalidInput) } - now := time.Now() - q := r.db.NewUpdate(). - Model((*model.ObjectDeletion)(nil)). - Set("cache_cleanup_status = ?", status). - Set("cache_cleaned_at = ?", now). - Set("updated_at = ?", now) - if cacheError == "" { - q = q.Set("cache_error = NULL") - } else { - q = q.Set("cache_error = ?", cacheError) + _, err := r.db.NewUpdate(). + Model((*model.ObjectCache)(nil)). + Set("in_cache = ?", false). + Set("cache_presence_generation = cache_presence_generation + 1"). + Set("updated_at = ?", time.Now()). + Where("content_id = ?", contentID). + Exec(ctx) + if err != nil { + return fmt.Errorf("clearing content cache presence: %w", err) } - res, err := q.Where("version_id = ?", versionID).Exec(ctx) + return nil +} + +func (r *BunObjectRepo) ReleaseContentCacheIfUnreferenced( + ctx context.Context, + contentID int64, + release func() error, +) (bool, error) { + if contentID < 1 || release == nil { + return false, fmt.Errorf("releasing content cache: %w", ErrInvalidInput) + } + released := false + err := r.runMaybeTx(ctx, func(db bun.IDB) error { + contents, err := lockStorageContentsByID(ctx, db, []int64{contentID}) + if err != nil { + return fmt.Errorf("locking content for cache release: %w", err) + } + if contents[contentID] == nil { + return ErrNotFound + } + unreferenced, err := contentIsUnreferenced(ctx, db, contentID) + if err != nil { + return err + } + if !unreferenced { + return nil + } + if err := release(); err != nil { + return fmt.Errorf("deleting content cache file: %w", err) + } + if _, err := db.NewUpdate(). + Model((*model.ObjectCache)(nil)). + Set("in_cache = ?", false). + Set("cache_presence_generation = cache_presence_generation + 1"). + Set("updated_at = ?", time.Now()). + Where("content_id = ?", contentID). + Exec(ctx); err != nil { + return fmt.Errorf("clearing released content cache presence: %w", err) + } + released = true + return nil + }) if err != nil { - return fmt.Errorf("updating object deletion cache cleanup: %w", err) + return false, err } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("updating object deletion cache cleanup: %w", ErrNotFound) + return released, nil +} + +// contentIsUnreferenced reports whether any live object version still points at +// the content. Cache release and remote cleanup both hang off this answer, so it +// must be read inside the same transaction that removed the version. +func contentIsUnreferenced(ctx context.Context, db bun.IDB, contentID int64) (bool, error) { + count, err := db.NewSelect(). + Model((*model.ObjectVersion)(nil)). + Where("content_id = ?", contentID). + Count(ctx) + if err != nil { + return false, fmt.Errorf("counting live versions for content %d: %w", contentID, err) } - return nil + return count == 0, nil +} + +// ContentIsUnreferenced reports whether any live object version still points at +// the content, so callers outside a deletion transaction can decide whether +// releasing its cached bytes is safe. +func (r *BunObjectRepo) ContentIsUnreferenced(ctx context.Context, contentID int64) (bool, error) { + if contentID <= 0 { + return false, fmt.Errorf("checking content references: %w", ErrInvalidInput) + } + return contentIsUnreferenced(ctx, r.db, contentID) } func (r *BunObjectRepo) RestoreCurrentDeleteMarkerStack(ctx context.Context, bucketID int64, key string, currentMarkerVersionID string) (*model.ObjectVersion, error) { @@ -516,7 +569,8 @@ func (r *BunObjectRepo) GetCurrentVersionByObjectID(ctx context.Context, objectI Model(version). ModelTableExpr("object_versions AS object_version") q = withObjectVersionStorageColumns(q, "object_version") - err := q.Where("object_version.object_id = ? AND object_version.is_current = ?", objectID, true).Scan(ctx) + err := q.Where("object_version.object_id = ?", objectID). + Where("current_object.current_version_id = object_version.version_id").Scan(ctx) if err != nil { if err == sql.ErrNoRows { return nil, nil @@ -532,7 +586,8 @@ func (r *BunObjectRepo) GetCurrentVersionByBucketAndKey(ctx context.Context, buc Model(version). ModelTableExpr("object_versions AS object_version") q = withObjectVersionStorageColumns(q, "object_version") - err := q.Where("object_version.bucket_id = ? AND object_version.key = ? AND object_version.is_current = ?", bucketID, key, true).Scan(ctx) + err := q.Where("object_version.bucket_id = ? AND object_version.key = ?", bucketID, key). + Where("current_object.current_version_id = object_version.version_id").Scan(ctx) if err != nil { if err == sql.ErrNoRows { return nil, nil @@ -574,109 +629,6 @@ func (r *BunObjectRepo) GetVersionByBucketKeyAndID(ctx context.Context, bucketID return version, nil } -func (r *BunObjectRepo) FindReusableStoredVersion(ctx context.Context, bucketID int64, size int64, checksum string) (*model.ObjectVersion, error) { - version := new(model.ObjectVersion) - q := r.db.NewSelect(). - Model(version). - ModelTableExpr("object_versions AS object_version") - q = withObjectVersionStorageColumns(q, "object_version") - err := q.Where("object_version.bucket_id = ? AND object_version.size = ? AND object_version.checksum = ?", bucketID, size, checksum). - Where("object_version.is_delete_marker = ?", false). - Where("object_version.state IN (?)", bun.List([]model.ObjectState{model.ObjectStateStored, model.ObjectStateCacheEvicted})). - Where("storage_upload.status IN (?)", bun.List([]model.StorageUploadStatus{ - model.StorageUploadStatusReadable, - model.StorageUploadStatusComplete, - })). - Where(usableCopyExistsSQL("object_version.storage_upload_id")). - OrderExpr("object_version.created_at DESC"). - OrderExpr("object_version.version_id DESC"). - Limit(1). - Scan(ctx) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("finding reusable stored object version: %w", err) - } - return version, nil -} - -func (r *BunObjectRepo) FindReusableReplicatingVersion(ctx context.Context, bucketID int64, size int64, checksum string) (*model.ObjectVersion, error) { - version := new(model.ObjectVersion) - q := r.db.NewSelect(). - Model(version). - ModelTableExpr("object_versions AS object_version") - q = withObjectVersionStorageColumns(q, "object_version") - err := q.Where("object_version.bucket_id = ? AND object_version.size = ? AND object_version.checksum = ?", bucketID, size, checksum). - Where("object_version.is_delete_marker = ?", false). - Where("object_version.state = ?", model.ObjectStateReplicating). - Where(usableCopyExistsSQL("object_version.storage_upload_id")). - OrderExpr("object_version.created_at DESC"). - OrderExpr("object_version.version_id DESC"). - Limit(1). - Scan(ctx) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("finding reusable replicating object version: %w", err) - } - return version, nil -} - -func (r *BunObjectRepo) FindReusableActiveUploadVersion(ctx context.Context, bucketID int64, size int64, checksum string) (*model.ObjectVersion, error) { - version := new(model.ObjectVersion) - err := r.db.NewSelect(). - Model(version). - ModelTableExpr("object_versions AS object_version"). - ColumnExpr("object_version.*"). - Where("object_version.bucket_id = ? AND object_version.size = ? AND object_version.checksum = ?", bucketID, size, checksum). - Where("object_version.is_delete_marker = ?", false). - Where("object_version.state IN (?)", bun.List([]model.ObjectState{model.ObjectStateCached, model.ObjectStateUploading, model.ObjectStateCommitting})). - Where("object_version.in_cache = ?", true). - Where(`( - EXISTS ( - SELECT 1 FROM storage_uploads AS active_upload - WHERE active_upload.source_version_id = object_version.version_id - AND active_upload.status IN (?) - AND ( - (object_version.state = ? AND active_upload.status IN (?, ?)) - OR ( - object_version.state = ? - AND `+usableCopyExistsSQL("active_upload.id")+` - ) - ) - ) - OR ( - object_version.state IN (?, ?) - AND EXISTS ( - SELECT 1 FROM tasks AS active_task - WHERE active_task.ref_type = ? - AND active_task.ref_version_id = object_version.version_id - AND active_task.type = ? - AND active_task.status IN (?) - ) - ) - )`, - bun.List(activeUploadStatuses()), - model.ObjectStateUploading, model.StorageUploadStatusRunning, model.StorageUploadStatusIngressReady, - model.ObjectStateCommitting, - model.ObjectStateCached, model.ObjectStateUploading, - "object", model.TaskTypeUpload, bun.List(activeTaskStatuses()), - ). - OrderExpr("object_version.created_at DESC"). - OrderExpr("object_version.version_id DESC"). - Limit(1). - Scan(ctx) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("finding reusable active upload object version: %w", err) - } - return version, nil -} - func (r *BunObjectRepo) ListCurrentVersionsByBucket(ctx context.Context, bucketID int64, prefix string, afterKey string, maxKeys int) ([]model.ObjectVersion, error) { return r.listCurrentVersionsByBucket(ctx, bucketID, prefix, afterKey, false, maxKeys) } @@ -687,23 +639,27 @@ func (r *BunObjectRepo) ListCurrentVersionsByBucketAtOrAfter(ctx context.Context func (r *BunObjectRepo) listCurrentVersionsByBucket(ctx context.Context, bucketID int64, prefix string, keyBoundary string, includeBoundary bool, maxKeys int) ([]model.ObjectVersion, error) { var versions []model.ObjectVersion - keyExpr := keyOrderExpr(r.db, "object_version.key") + // Listing walks the objects unique key and follows each pointer, so the + // bucket+key ordering comes from objects rather than from a partial index + // over every version. + keyExpr := keyOrderExpr(r.db, "current_object.key") q := r.db.NewSelect(). Model(&versions). ModelTableExpr("object_versions AS object_version") q = withObjectVersionStorageColumns(q, "object_version"). - Where("object_version.bucket_id = ? AND object_version.is_current = ?", bucketID, true). + Where("current_object.bucket_id = ?", bucketID). + Where("current_object.current_version_id = object_version.version_id"). Where("object_version.is_delete_marker = ?", false). OrderExpr(keyExpr + " ASC") if prefix != "" { - q = applyCaseSensitivePrefixFilter(r.db, q, "object_version.key", prefix) + q = applyCaseSensitivePrefixFilter(r.db, q, "current_object.key", prefix) } if keyBoundary != "" { if includeBoundary { - q = q.Where(keyComparisonSQL(r.db, "object_version.key", ">="), keyBoundary) + q = q.Where(keyComparisonSQL(r.db, "current_object.key", ">="), keyBoundary) } else { - q = q.Where(keyComparisonSQL(r.db, "object_version.key", ">"), keyBoundary) + q = q.Where(keyComparisonSQL(r.db, "current_object.key", ">"), keyBoundary) } } if maxKeys > 0 { @@ -788,22 +744,22 @@ func (r *BunObjectRepo) ListVersionsByKey(ctx context.Context, bucketID int64, k func (r *BunObjectRepo) ListRecoverableDeleteMarkers(ctx context.Context, bucketID int64, prefix string, afterKey string, maxKeys int) ([]RecoverableDeleteMarker, error) { var markers []model.ObjectVersion - keyExpr := keyOrderExpr(r.db, "object_version.key") + keyExpr := keyOrderExpr(r.db, "current_object.key") q := r.db.NewSelect(). Model(&markers). ModelTableExpr("object_versions AS object_version") q = withObjectVersionStorageColumns(q, "object_version"). - Where("object_version.bucket_id = ?", bucketID). - Where("object_version.is_current = ?", true). + Where("current_object.bucket_id = ?", bucketID). + Where("current_object.current_version_id = object_version.version_id"). Where("object_version.is_delete_marker = ?", true). Where("EXISTS (SELECT 1 FROM object_versions AS data_version WHERE data_version.object_id = object_version.object_id AND data_version.is_delete_marker = ?)", false). OrderExpr(keyExpr + " ASC") if prefix != "" { - q = applyCaseSensitivePrefixFilter(r.db, q, "object_version.key", prefix) + q = applyCaseSensitivePrefixFilter(r.db, q, "current_object.key", prefix) } if afterKey != "" { - q = q.Where(keyComparisonSQL(r.db, "object_version.key", ">"), afterKey) + q = q.Where(keyComparisonSQL(r.db, "current_object.key", ">"), afterKey) } if maxKeys > 0 { q = q.Limit(maxKeys) @@ -834,224 +790,59 @@ func (r *BunObjectRepo) ListRecoverableDeleteMarkers(ctx context.Context, bucket return items, nil } -func (r *BunObjectRepo) UpdateVersionState(ctx context.Context, versionID string, from, to model.ObjectState) error { - return r.runMaybeTx(ctx, func(db bun.IDB) error { - return updateVersionState(ctx, db, versionID, from, to) - }) -} - -func (r *BunObjectRepo) UpdateVersionStateToFailed(ctx context.Context, versionID string, from model.ObjectState, lastError string) error { - return r.runMaybeTx(ctx, func(db bun.IDB) error { - now := time.Now() - res, err := db.NewUpdate(). - Model((*model.ObjectVersion)(nil)). - Set("state = ?", model.ObjectStateFailed). - Set("storage_upload_id = NULL"). - Set("failed_at_state = ?", from). - Set("last_error = ?", lastError). - Set("updated_at = ?", now). - Where("version_id = ? AND state = ?", versionID, from). - Exec(ctx) - if err != nil { - return fmt.Errorf("updating object version state to failed: %w", err) - } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("state transition %s→failed failed: version %s not in expected state", from, versionID) - } - return nil - }) -} - func (r *BunObjectRepo) SetVersionCachePresence(ctx context.Context, versionID string, inCache bool) error { return r.runMaybeTx(ctx, func(db bun.IDB) error { return setVersionCachePresence(ctx, db, versionID, inCache) }) } -func (r *BunObjectRepo) RecordVersionCacheAccess(ctx context.Context, versionID string, accessedAt time.Time) error { - return executeVersionCacheAccessUpdate( - ctx, - r.newVersionCacheAccessUpdate(versionID, accessedAt), - versionID, - ) +// RecordContentCacheAccess advances LRU recency for one content payload without +// asserting that its bytes are present. +func (r *BunObjectRepo) RecordContentCacheAccess(ctx context.Context, contentID int64, accessedAt time.Time) error { + return r.recordContentCacheAccess(ctx, contentID, accessedAt, false) } -func (r *BunObjectRepo) RecordVersionCacheCommit(ctx context.Context, versionID string, accessedAt time.Time) error { - query := r.newVersionCacheAccessUpdate(versionID, accessedAt). - Set("in_cache = ?", true) - return executeVersionCacheAccessUpdate(ctx, query, versionID) -} - -func (r *BunObjectRepo) newVersionCacheAccessUpdate( - versionID string, - accessedAt time.Time, -) *bun.UpdateQuery { - accessedAt = cacheeviction.NormalizeAccessTime(accessedAt) - return r.db.NewUpdate(). - Model((*model.ObjectVersion)(nil)). - Set( - `cache_accessed_at = CASE - WHEN cache_accessed_at IS NULL OR cache_accessed_at < ? THEN ? - ELSE cache_accessed_at - END`, - accessedAt, - accessedAt, - ). - Where("version_id = ? AND is_delete_marker = ?", versionID, false) +// RecordContentCacheCommit marks a freshly written cache file present and +// advances its recency in the same write. +func (r *BunObjectRepo) RecordContentCacheCommit(ctx context.Context, contentID int64, accessedAt time.Time) error { + return r.recordContentCacheAccess(ctx, contentID, accessedAt, true) } -func executeVersionCacheAccessUpdate( - ctx context.Context, - query *bun.UpdateQuery, - versionID string, -) error { - res, err := query.Exec(ctx) - if err != nil { - return fmt.Errorf("recording version cache access: %w", err) - } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("recording version cache access: version %s not found", versionID) - } - return nil -} - -func (r *BunObjectRepo) SetVersionStorageUploadAndTransition(ctx context.Context, versionID string, storageUploadID int64, from, to model.ObjectState) error { +func (r *BunObjectRepo) recordContentCacheAccess(ctx context.Context, contentID int64, accessedAt time.Time, present bool) error { return r.runMaybeTx(ctx, func(db bun.IDB) error { - upload, err := lockStorageUploadForObjectState(ctx, db, storageUploadID, to) - if err != nil { - return fmt.Errorf("locking storage upload for version transition: %w", err) - } - if to == model.ObjectStateStored || to == model.ObjectStateCacheEvicted { - if err := requireCurrentMinimumDurableCopies(ctx, db, upload); err != nil { - return fmt.Errorf("checking storage upload durability for version transition: %w", err) - } - } - now := time.Now() - res, err := db.NewUpdate(). - Model((*model.ObjectVersion)(nil)). - Set("storage_upload_id = ?", storageUploadID). - Set("state = ?", to). - Set("updated_at = ?", now). - Where("version_id = ? AND state = ?", versionID, from). - Exec(ctx) - if err != nil { - return fmt.Errorf("setting version storage upload and transitioning state: %w", err) - } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("SetVersionStorageUploadAndTransition %s→%s failed: version %s not in expected state or upload not usable", from, to, versionID) + if contentID < 1 { + return nil } - return nil - }) -} - -func (r *BunObjectRepo) FailUploadingContentFollowers(ctx context.Context, bucketID int64, size int64, checksum string, leaderVersionID string, lastError string) ([]ObjectVersionRef, error) { - var refs []ObjectVersionRef - err := r.runMaybeTx(ctx, func(db bun.IDB) error { - now := time.Now() - query := `UPDATE object_versions - SET state = ?, failed_at_state = state, last_error = ?, updated_at = ? - WHERE bucket_id = ? AND size = ? AND checksum = ? AND state IN (?, ?) - AND ( - version_id = ? - OR NOT EXISTS ( - SELECT 1 FROM tasks - WHERE tasks.ref_type = ? - AND tasks.ref_version_id = object_versions.version_id - AND tasks.type = ? - AND tasks.status IN (?) - ) - AND NOT EXISTS ( - SELECT 1 FROM storage_uploads AS active_upload - WHERE active_upload.source_version_id = object_versions.version_id - AND active_upload.status IN (?) - ) - ) - RETURNING object_id, version_id` - err := db.NewRaw(query, - model.ObjectStateFailed, - lastError, - now, - bucketID, - size, - checksum, - model.ObjectStateUploading, - model.ObjectStateCommitting, - leaderVersionID, - "object", - model.TaskTypeUpload, - bun.List(activeTaskStatuses()), - bun.List(activeUploadStatuses()), - ).Scan(ctx, &refs) - if err != nil { - if err == sql.ErrNoRows { - return fmt.Errorf("no active upload object versions matched content: %w", ErrNotFound) - } - return fmt.Errorf("failing active upload object versions: %w", err) - } - if len(refs) == 0 { - return fmt.Errorf("no active upload object versions matched content: %w", ErrNotFound) + normalized := cacheeviction.NormalizeAccessTime(accessedAt) + if present { + return upsertContentCachePresence(ctx, db, contentID, true, &normalized) } - return nil - }) - if err != nil { - return nil, err - } - return refs, nil -} - -func (r *BunObjectRepo) ListVersionsByState(ctx context.Context, state model.ObjectState, limit int) ([]model.ObjectVersion, error) { - return r.ListVersionsByStateAfter(ctx, state, time.Time{}, "", limit) -} - -func (r *BunObjectRepo) ListVersionsByStateAfter(ctx context.Context, state model.ObjectState, afterUpdatedAt time.Time, afterVersionID string, limit int) ([]model.ObjectVersion, error) { - var versions []model.ObjectVersion - q := r.db.NewSelect(). - Model(&versions). - ModelTableExpr("object_versions AS object_version") - q = withObjectVersionStorageColumns(q, "object_version"). - Where("object_version.state = ?", state). - Where("object_version.is_delete_marker = ?", false). - OrderExpr("object_version.updated_at ASC, object_version.version_id ASC") - if !afterUpdatedAt.IsZero() || afterVersionID != "" { - q = q.Where("(object_version.updated_at > ? OR (object_version.updated_at = ? AND object_version.version_id > ?))", afterUpdatedAt, afterUpdatedAt, afterVersionID) - } - if limit > 0 { - q = q.Limit(limit) - } - if err := q.Scan(ctx); err != nil { - return nil, fmt.Errorf("listing object versions by state: %w", err) - } - return versions, nil -} - -func (r *BunObjectRepo) ResetStaleVersionStates(ctx context.Context, fromState, toState model.ObjectState, staleBefore time.Time) (int, error) { - var reset int - err := r.runMaybeTx(ctx, func(db bun.IDB) error { - versionIDs, err := resetStaleVersions(ctx, db, fromState, toState, staleBefore) - if err != nil { - return err + // An access-only write must never claim the bytes are present: the + // reader may have served them from a provider. + if _, err := db.NewUpdate(). + Model((*model.ObjectCache)(nil)). + Set(`cache_accessed_at = CASE + WHEN cache_accessed_at IS NULL OR cache_accessed_at < ? THEN ? + ELSE cache_accessed_at END`, normalized, normalized). + Set("updated_at = ?", time.Now()). + Where("content_id = ?", contentID). + Exec(ctx); err != nil { + return fmt.Errorf("recording content cache access: %w", err) } - reset = len(versionIDs) return nil }) - if err != nil { - return reset, err - } - return reset, nil } -// CountByState returns current object counts grouped by state. func (r *BunObjectRepo) CountByState(ctx context.Context) ([]ObjectStateCount, error) { var counts []ObjectStateCount + state := objectVersionStateSQL("object_version") err := r.db.NewSelect(). Model((*model.ObjectVersion)(nil)). - ColumnExpr("state, COUNT(*) AS count"). - Where("is_current = ?", true). - Where("is_delete_marker = ?", false). - GroupExpr("state"). + ColumnExpr(state+" AS state, COUNT(*) AS count"). + Join("LEFT JOIN storage_contents AS storage_content ON storage_content.id = object_version.content_id"). + Where(currentVersionPredicateSQL("object_version")). + Where("object_version.is_delete_marker = ?", false). + GroupExpr(state). Scan(ctx, &counts) if err != nil { return nil, fmt.Errorf("counting current object versions by state: %w", err) @@ -1062,12 +853,14 @@ func (r *BunObjectRepo) CountByState(ctx context.Context) ([]ObjectStateCount, e // AggregateByState returns current object counts and sizes grouped by state. func (r *BunObjectRepo) AggregateByState(ctx context.Context) ([]ObjectStateAggregate, error) { var rows []ObjectStateAggregate + state := objectVersionStateSQL("object_version") err := r.db.NewSelect(). Model((*model.ObjectVersion)(nil)). - ColumnExpr("state, COUNT(*) AS count, COALESCE(SUM(size), 0) AS total_size"). - Where("is_current = ?", true). - Where("is_delete_marker = ?", false). - GroupExpr("state"). + ColumnExpr(state+" AS state, COUNT(*) AS count, COALESCE(SUM(object_version.size), 0) AS total_size"). + Join("LEFT JOIN storage_contents AS storage_content ON storage_content.id = object_version.content_id"). + Where(currentVersionPredicateSQL("object_version")). + Where("object_version.is_delete_marker = ?", false). + GroupExpr(state). Scan(ctx, &rows) if err != nil { return nil, fmt.Errorf("aggregating current object versions by state: %w", err) @@ -1077,39 +870,32 @@ func (r *BunObjectRepo) AggregateByState(ctx context.Context) ([]ObjectStateAggr func (r *BunObjectRepo) CountOverviewAttention(ctx context.Context) (ObjectAttentionCount, error) { var count ObjectAttentionCount + // Pipeline position and cache residency are derived from the content's + // copies and its cache entry, so the counts read those rather than columns + // object_versions no longer owns. query := `WITH current_versions AS ( - SELECT version_id, storage_upload_id, state, in_cache - FROM object_versions - WHERE is_current = ? - AND is_delete_marker = ? - ), - latest_upload_refs AS ( - SELECT source_upload.source_version_id, MAX(source_upload.id) AS latest_upload_id - FROM storage_uploads AS source_upload - JOIN current_versions AS current_version - ON current_version.version_id = source_upload.source_version_id - AND current_version.storage_upload_id IS NULL - WHERE source_upload.source_version_id <> '' - GROUP BY source_upload.source_version_id + SELECT object_version.version_id, + object_version.content_id, + ` + objectVersionStateSQL("object_version") + ` AS state, + COALESCE(cache_entry.in_cache, FALSE) AS in_cache + FROM object_versions AS object_version + LEFT JOIN storage_contents AS storage_content + ON storage_content.id = object_version.content_id + LEFT JOIN object_cache AS cache_entry + ON cache_entry.content_id = object_version.content_id + WHERE ` + currentVersionPredicateSQL("object_version") + ` + AND object_version.is_delete_marker = ? ) SELECT - COALESCE(SUM(CASE WHEN current_version.state = ? OR latest_upload.status IN (?, ?) THEN 1 ELSE 0 END), 0) AS needs_attention, - COALESCE(SUM(CASE WHEN current_version.in_cache = ? AND NOT (current_version.state IN (?, ?, ?) AND ` + usableCopyExistsSQL("current_version.storage_upload_id") + `) THEN 1 ELSE 0 END), 0) AS unavailable + COALESCE(SUM(CASE WHEN current_version.state = ? OR version_content.error_message IS NOT NULL THEN 1 ELSE 0 END), 0) AS needs_attention, + COALESCE(SUM(CASE WHEN current_version.in_cache = ? AND NOT ` + usableCopyExistsSQL("current_version.content_id") + ` THEN 1 ELSE 0 END), 0) AS unavailable FROM current_versions AS current_version - LEFT JOIN latest_upload_refs AS latest_upload_ref - ON latest_upload_ref.source_version_id = current_version.version_id - LEFT JOIN storage_uploads AS latest_upload - ON latest_upload.id = COALESCE(current_version.storage_upload_id, latest_upload_ref.latest_upload_id)` + LEFT JOIN storage_contents AS version_content + ON version_content.id = current_version.content_id` err := r.db.NewRaw(query, - true, false, model.ObjectStateFailed, - model.StorageUploadStatusFailed, - model.StorageUploadStatusRejected, false, - model.ObjectStateReplicating, - model.ObjectStateStored, - model.ObjectStateCacheEvicted, ).Scan(ctx, &count) if err != nil { return ObjectAttentionCount{}, fmt.Errorf("counting overview object attention: %w", err) @@ -1121,7 +907,8 @@ func (r *BunObjectRepo) CountOverviewAttention(ctx context.Context) (ObjectAtten func (r *BunObjectRepo) CountByBucket(ctx context.Context, bucketID int64) (int64, error) { count, err := r.db.NewSelect(). Model((*model.ObjectVersion)(nil)). - Where("bucket_id = ? AND is_current = ?", bucketID, true). + Where("bucket_id = ?", bucketID). + Where(currentVersionPredicateSQL("object_version")). Where("is_delete_marker = ?", false). Count(ctx) if err != nil { @@ -1136,7 +923,8 @@ func (r *BunObjectRepo) TotalSizeByBucket(ctx context.Context, bucketID int64) ( err := r.db.NewSelect(). Model((*model.ObjectVersion)(nil)). ColumnExpr("COALESCE(SUM(size), 0)"). - Where("bucket_id = ? AND is_current = ?", bucketID, true). + Where("bucket_id = ?", bucketID). + Where(currentVersionPredicateSQL("object_version")). Where("is_delete_marker = ?", false). Scan(ctx, &total) if err != nil { @@ -1152,7 +940,8 @@ func (r *BunObjectRepo) BucketStats(ctx context.Context, bucketID int64) (Bucket Model((*model.ObjectVersion)(nil)). ColumnExpr("COUNT(*) AS count"). ColumnExpr("COALESCE(SUM(size), 0) AS total_size"). - Where("bucket_id = ? AND is_current = ?", bucketID, true). + Where("bucket_id = ?", bucketID). + Where(currentVersionPredicateSQL("object_version")). Where("is_delete_marker = ?", false). Scan(ctx, &stats) if err != nil { @@ -1173,7 +962,7 @@ func (r *BunObjectRepo) AggregateByBucket(ctx context.Context) (map[int64]Bucket ColumnExpr("bucket_id"). ColumnExpr("COUNT(*) AS count"). ColumnExpr("COALESCE(SUM(size), 0) AS total_size"). - Where("is_current = ?", true). + Where(currentVersionPredicateSQL("object_version")). Where("is_delete_marker = ?", false). GroupExpr("bucket_id"). Scan(ctx, &rows) @@ -1196,16 +985,68 @@ func (r *BunObjectRepo) runMaybeTx(ctx context.Context, fn func(bun.IDB) error) return fn(r.db) } +// withObjectVersionStorageColumns projects everything an object version no +// longer stores. Content identity lives on storage_contents, cache residency on +// object_cache, and pipeline position is a function of the copy rows. Every read +// that returns a model.ObjectVersion must go through here, or those fields come +// back as zero values rather than facts. func withObjectVersionStorageColumns(q *bun.SelectQuery, alias string) *bun.SelectQuery { return q. ColumnExpr(alias + ".*"). - ColumnExpr("storage_upload.piece_cid AS piece_cid"). - ColumnExpr("CASE WHEN " + alias + ".state IN ('replicating', 'stored', 'cache_evicted') AND " + usableCopyExistsSQL(alias+".storage_upload_id") + " THEN TRUE ELSE FALSE END AS in_filecoin"). - Join("LEFT JOIN storage_uploads AS storage_upload ON storage_upload.id = " + alias + ".storage_upload_id") + ColumnExpr("CASE WHEN current_object.current_version_id = " + alias + ".version_id THEN TRUE ELSE FALSE END AS is_current"). + ColumnExpr("COALESCE(storage_content.checksum, '') AS checksum"). + ColumnExpr("storage_content.piece_cid AS piece_cid"). + ColumnExpr("COALESCE(object_cache_entry.in_cache, FALSE) AS in_cache"). + ColumnExpr("object_cache_entry.cache_accessed_at AS cache_accessed_at"). + ColumnExpr("CASE WHEN " + usableCopyExistsSQL(alias+".content_id") + " THEN TRUE ELSE FALSE END AS in_filecoin"). + ColumnExpr(objectVersionStateSQL(alias) + " AS state"). + Join("JOIN objects AS current_object ON current_object.id = " + alias + ".object_id"). + Join("LEFT JOIN storage_contents AS storage_content ON storage_content.id = " + alias + ".content_id"). + Join("LEFT JOIN object_cache AS object_cache_entry ON object_cache_entry.content_id = " + alias + ".content_id") +} + +// currentVersionPredicateSQL matches the one version an object currently serves. +// "Current" is the object's pointer, so it is asked of objects rather than of a +// flag repeated on every version. +func currentVersionPredicateSQL(alias string) string { + return "EXISTS (SELECT 1 FROM objects AS current_pointer WHERE current_pointer.id = " + alias + + ".object_id AND current_pointer.current_version_id = " + alias + ".version_id)" +} + +// objectVersionStateSQL derives how far a version's content has travelled. The +// value used to be a stored column that could disagree with the copies it +// summarised; deriving it makes that disagreement unrepresentable. A version +// with no content is a delete marker and has no pipeline to report. +func objectVersionStateSQL(alias string) string { + return fmt.Sprintf("CASE WHEN %s.content_id IS NULL THEN '%s' ELSE (%s) END", + alias, model.ObjectStateCached, contentPipelineStateSQL()) +} + +// contentPipelineStateSQL expects a storage_content alias in scope. +func contentPipelineStateSQL() string { + contentID := "storage_content.id" + readableSlots := distinctReadableSlotCountSQL("state_copy", "state_data_set", contentID) + anyCopy := "EXISTS (SELECT 1 FROM storage_copies AS any_copy WHERE any_copy.content_id = " + contentID + ")" + everyCopyFailed := "NOT EXISTS (SELECT 1 FROM storage_copies AS live_copy WHERE live_copy.content_id = " + contentID + + " AND live_copy.status <> '" + string(model.StorageCopyStatusFailed) + "')" + committing := "EXISTS (SELECT 1 FROM storage_copies AS committing_copy WHERE committing_copy.content_id = " + contentID + + " AND committing_copy.status IN ('" + string(model.StorageCopyStatusPieceReady) + "', '" + string(model.StorageCopyStatusCommitting) + "'))" + return fmt.Sprintf(`CASE + WHEN %[1]s >= COALESCE(storage_content.requested_copies, 1) THEN '%[5]s' + WHEN %[1]s >= 1 THEN '%[6]s' + WHEN %[2]s AND %[3]s THEN '%[7]s' + WHEN %[4]s THEN '%[8]s' + WHEN %[2]s THEN '%[9]s' + ELSE '%[10]s' + END`, + readableSlots, anyCopy, everyCopyFailed, committing, + model.ObjectStateStored, model.ObjectStateReplicating, model.ObjectStateFailed, + model.ObjectStateCommitting, model.ObjectStateUploading, model.ObjectStateCached, + ) } -func usableCopyExistsSQL(uploadIDExpr string) string { - return "EXISTS (SELECT 1 FROM storage_upload_copies AS storage_copy JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id WHERE storage_copy.upload_id = " + uploadIDExpr + " AND storage_copy.status = 'committed' AND storage_copy.storage_data_set_id IS NOT NULL AND storage_copy.provider_id IS NOT NULL AND storage_copy.provider_id <> '' AND storage_data_set.data_set_id IS NOT NULL AND storage_data_set.data_set_id <> '' AND storage_data_set.status IN ('ready', 'draining') AND storage_copy.piece_id IS NOT NULL AND storage_copy.piece_id <> '' AND storage_copy.retrieval_url IS NOT NULL AND storage_copy.retrieval_url <> '')" +func usableCopyExistsSQL(contentIDExpr string) string { + return "EXISTS (SELECT 1 FROM storage_copies AS storage_copy JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id WHERE storage_copy.content_id = " + contentIDExpr + " AND storage_copy.status = 'committed' AND storage_copy.provider_id <> '' AND storage_data_set.data_set_id IS NOT NULL AND storage_data_set.data_set_id <> '' AND storage_data_set.status IN ('ready', 'draining') AND storage_copy.piece_id IS NOT NULL AND storage_copy.piece_id <> '' AND storage_copy.retrieval_url IS NOT NULL AND storage_copy.retrieval_url <> '')" } func objectVersionPermanentDeleteStateError(state model.ObjectState) error { @@ -1215,7 +1056,6 @@ func objectVersionPermanentDeleteStateError(state model.ObjectState) error { model.ObjectStateCommitting, model.ObjectStateReplicating, model.ObjectStateStored, - model.ObjectStateCacheEvicted, model.ObjectStateFailed: return nil default: @@ -1227,37 +1067,36 @@ func prepareObjectVersionsForPermanentDelete( ctx context.Context, db bun.IDB, versions []*model.ObjectVersion, - uploadsByID map[int64]*model.StorageUpload, + uploadsByID map[int64]*model.StorageContent, ) error { if len(versions) == 0 { return nil } deletingVersionIDs := make([]string, 0, len(versions)) - bucketIDs := make(map[int64]struct{}) for _, version := range versions { if version == nil || version.VersionID == "" { return fmt.Errorf("preparing permanent delete storage work: %w", ErrInvalidInput) } deletingVersionIDs = append(deletingVersionIDs, version.VersionID) - bucketIDs[version.BucketID] = struct{}{} } sort.Strings(deletingVersionIDs) - uploadIDs := sortedStorageUploadIDs(uploadsByID) - var copies []model.StorageUploadCopy - if len(uploadIDs) > 0 { - if err := db.NewSelect(). - Model(&copies). - Where("upload_id IN (?)", bun.List(uploadIDs)). - OrderExpr("id ASC"). + contentIDs := sortedContentIDs(uploadsByID) + var copies []model.StorageCopy + if len(contentIDs) > 0 { + query := db.NewSelect().Model(&copies) + projectActiveCommitAttempt(query, "storage_copy") + if err := query. + Where("storage_copy.content_id IN (?)", bun.List(contentIDs)). + OrderExpr("storage_copy.id ASC"). Scan(ctx); err != nil { return fmt.Errorf("loading storage copies for permanent delete: %w", err) } } for i := range copies { res, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). + Model((*model.StorageCopy)(nil)). Set("updated_at = updated_at"). Where("id = ?", copies[i].ID). Exec(ctx) @@ -1271,15 +1110,12 @@ func prepareObjectVersionsForPermanentDelete( } relatedVersionIDs := append([]string(nil), deletingVersionIDs...) - for _, upload := range uploadsByID { - relatedVersionIDs = appendUniqueString(relatedVersionIDs, upload.SourceVersionID) - } var boundVersionIDs []string - if len(uploadIDs) > 0 { + if len(contentIDs) > 0 { if err := db.NewSelect(). Model((*model.ObjectVersion)(nil)). Column("version_id"). - Where("storage_upload_id IN (?)", bun.List(uploadIDs)). + Where("content_id IN (?)", bun.List(contentIDs)). Scan(ctx, &boundVersionIDs); err != nil { return fmt.Errorf("loading storage upload references for permanent delete: %w", err) } @@ -1289,91 +1125,98 @@ func prepareObjectVersionsForPermanentDelete( } sort.Strings(relatedVersionIDs) - bucketIDList := make([]int64, 0, len(bucketIDs)) - for bucketID := range bucketIDs { - bucketIDList = append(bucketIDList, bucketID) - } - sort.Slice(bucketIDList, func(i, j int) bool { return bucketIDList[i] < bucketIDList[j] }) - var taskIDs []int64 - if err := db.NewSelect(). - Model((*model.Task)(nil)). - Column("id"). - Where("type = ?", model.TaskTypeUpload). - Where(`( - (ref_type = ? AND ref_version_id IN (?)) - OR (ref_type = ? AND ref_id IN (?) AND ref_version_id IN (?)) - )`, "object", bun.List(deletingVersionIDs), "bucket", bun.List(bucketIDList), bun.List(relatedVersionIDs)). - OrderExpr("id ASC"). - Scan(ctx, &taskIDs); err != nil { - return fmt.Errorf("loading storage tasks for permanent delete: %w", err) + // Ingest is scheduled against the content, so a version cannot be removed + // while work is in flight for either the version or the bytes it names. + contentSubjectKeys := make([]string, 0, len(contentIDs)) + for _, contentID := range contentIDs { + contentSubjectKeys = append(contentSubjectKeys, strconv.FormatInt(contentID, 10)) + } + var relatedTasks []model.Task + taskQuery := db.NewSelect(). + Model(&relatedTasks). + Column("id", "status"). + OrderExpr("id ASC") + if len(contentSubjectKeys) > 0 { + taskQuery = taskQuery.Where( + "(subject_type = ? AND subject_key IN (?)) OR (subject_type = ? AND subject_key IN (?))", + "object_version", bun.List(relatedVersionIDs), + "storage_content", bun.List(contentSubjectKeys), + ) + } else { + taskQuery = taskQuery.Where("subject_type = ? AND subject_key IN (?)", "object_version", bun.List(relatedVersionIDs)) } - for _, taskID := range taskIDs { - if _, err := db.NewUpdate(). - Model((*model.Task)(nil)). - Set("status = status"). - Where("id = ?", taskID). - Exec(ctx); err != nil { - return fmt.Errorf("locking storage task for permanent delete: %w", err) - } + if db.Dialect().Name() == dialect.PG { + taskQuery = taskQuery.For("UPDATE") } - if len(taskIDs) > 0 { - activeTasksExist, err := db.NewSelect(). - Model((*model.Task)(nil)). - Where("id IN (?)", bun.List(taskIDs)). - Where("status IN (?)", bun.List(activeTaskStatuses())). - Exists(ctx) - if err != nil { - return fmt.Errorf("rechecking active storage tasks for permanent delete: %w", err) - } - if activeTasksExist { + if err := taskQuery.Scan(ctx); err != nil { + return fmt.Errorf("loading storage tasks for permanent delete: %w", err) + } + for i := range relatedTasks { + if relatedTasks[i].Status == model.TaskStatusPending || relatedTasks[i].Status == model.TaskStatusRunning { return ErrPermanentDeleteStorageBusy } } - if len(uploadIDs) == 0 { + if len(contentIDs) == 0 { return nil } - copiesByUploadID := make(map[int64][]model.StorageUploadCopy) + copiesByContentID := make(map[int64][]model.StorageCopy) for _, copyRow := range copies { - copiesByUploadID[copyRow.UploadID] = append(copiesByUploadID[copyRow.UploadID], copyRow) + copiesByContentID[copyRow.ContentID] = append(copiesByContentID[copyRow.ContentID], copyRow) } - for _, uploadID := range uploadIDs { - upload := uploadsByID[uploadID] - liveVersion, err := selectLiveObjectVersionForStorageUpload(ctx, db, upload, deletingVersionIDs) + for _, contentID := range contentIDs { + upload := uploadsByID[contentID] + liveVersion, err := selectLiveObjectVersionForStorageContent(ctx, db, upload, deletingVersionIDs) if err != nil { return err } if liveVersion != nil { continue } - for _, copyRow := range copiesByUploadID[uploadID] { + for _, copyRow := range copiesByContentID[contentID] { if storageUploadCopyHasAttemptedCommit(copyRow) { return ErrPermanentDeleteStorageBusy } if !storageUploadCopyCanBeCancelledForPermanentDelete(copyRow) { continue } + now := time.Now() + if _, err := db.NewUpdate(). + Model((*storagecommit.Attempt)(nil)). + Set("status = ?", storagecommit.AttemptStatusReleased). + Set("release_reason = ?", string(storagecommit.ReleaseOwnerTerminal)). + Set("resolved_at = ?", now). + Set("updated_at = ?", now). + Where("content_id = ? AND storage_data_set_id = ?", copyRow.ContentID, copyRow.StorageDataSetID). + Where("status = ? AND resolved_at IS NULL", storagecommit.AttemptStatusReserved). + Exec(ctx); err != nil { + return fmt.Errorf("releasing storage reservation for permanent delete: %w", err) + } res, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("status = ?", model.StorageUploadCopyStatusFailed). + Model((*model.StorageCopy)(nil)). + Set("status = ?", model.StorageCopyStatusFailed). + Set("active_task_id = NULL"). Set("commit_ready_at = NULL"). - Set("commit_attempt_id = NULL"). - Set("commit_attempted_at = NULL"). - Set("commit_submission_json = NULL"). Set("commit_extra_data_hex = NULL"). - Set("commit_transaction_id = NULL"). - Set("commit_confirmed_transaction_id = NULL"). - Set("commit_attention_code = NULL"). - Set("commit_attention_at = NULL"). Set("last_error = ?", "cancelled because the last object version was permanently deleted"). - Set("updated_at = ?", time.Now()). + Set("updated_at = ?", now). Where("id = ?", copyRow.ID). - Where("status IN (?)", bun.List([]model.StorageUploadCopyStatus{ - model.StorageUploadCopyStatusPending, - model.StorageUploadCopyStatusPieceReady, - model.StorageUploadCopyStatusCommitting, + Where("status IN (?)", bun.List([]model.StorageCopyStatus{ + model.StorageCopyStatusPending, + model.StorageCopyStatusPieceReady, + model.StorageCopyStatusCommitting, })). - Where("NOT " + attemptedStorageCommitSQL("storage_upload_copy")). + Where(`active_task_id IS NULL OR EXISTS ( + SELECT 1 FROM tasks AS terminal_task + WHERE terminal_task.id = storage_copy.active_task_id + AND terminal_task.status IN (?, ?, ?) + )`, model.TaskStatusCompleted, model.TaskStatusFailed, model.TaskStatusCancelled). + Where(`NOT EXISTS ( + SELECT 1 FROM storage_commit_attempts AS unresolved_attempt + WHERE unresolved_attempt.content_id = storage_copy.content_id + AND unresolved_attempt.storage_data_set_id = storage_copy.storage_data_set_id + AND unresolved_attempt.resolved_at IS NULL + )`). Exec(ctx) if err != nil { return fmt.Errorf("cancelling storage copy for permanent delete: %w", err) @@ -1384,38 +1227,36 @@ func prepareObjectVersionsForPermanentDelete( } } if _, err := db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("status = ?", model.StorageUploadStatusSuperseded). - Set("error_message = ?", "superseded because the last object version was permanently deleted"). + Model((*model.StorageContent)(nil)). + Set("error_message = ?", "the last object version referencing this content was permanently deleted"). Set("updated_at = ?", time.Now()). - Where("id = ?", uploadID). - Where("status <> ?", model.StorageUploadStatusSuperseded). + Where("id = ?", contentID). Exec(ctx); err != nil { - return fmt.Errorf("closing storage upload for permanent delete: %w", err) + return fmt.Errorf("closing storage content for permanent delete: %w", err) } } return nil } -func authoritativeStorageUploadsForVersions( +func authoritativeStorageContentsForVersions( ctx context.Context, db bun.IDB, versions []*model.ObjectVersion, -) (map[int64]*model.StorageUpload, error) { - uploadsByID := make(map[int64]*model.StorageUpload) +) (map[int64]*model.StorageContent, error) { + uploadsByID := make(map[int64]*model.StorageContent) for _, version := range versions { if version == nil || version.VersionID == "" { return nil, fmt.Errorf("loading permanent delete storage uploads: %w", ErrInvalidInput) } - upload, err := authoritativeStorageUploadForVersion(ctx, db, version) + upload, err := authoritativeStorageContentForVersion(ctx, db, version) if err != nil { return nil, err } if upload == nil { continue } - uploadID := upload.ID - version.StorageUploadID = &uploadID + contentID := upload.ID + version.ContentID = &contentID uploadsByID[upload.ID] = upload } return uploadsByID, nil @@ -1445,15 +1286,12 @@ func sameObjectVersionIDs(versions []model.ObjectVersion, expectedIDs []string) return slices.Equal(actualIDs, expected) } -func authoritativeStorageUploadForVersion(ctx context.Context, db bun.IDB, version *model.ObjectVersion) (*model.StorageUpload, error) { - upload := new(model.StorageUpload) - q := db.NewSelect().Model(upload) - if version.StorageUploadID != nil && *version.StorageUploadID > 0 { - q = q.Where("id = ?", *version.StorageUploadID) - } else { - q = q.Where("source_version_id = ?", version.VersionID).OrderExpr("id DESC").Limit(1) +func authoritativeStorageContentForVersion(ctx context.Context, db bun.IDB, version *model.ObjectVersion) (*model.StorageContent, error) { + if version.ContentID == nil || *version.ContentID <= 0 { + return nil, nil } - if err := q.Scan(ctx); err != nil { + upload := new(model.StorageContent) + if err := db.NewSelect().Model(upload).Where("id = ?", *version.ContentID).Scan(ctx); err != nil { if err == sql.ErrNoRows { return nil, nil } @@ -1462,25 +1300,25 @@ func authoritativeStorageUploadForVersion(ctx context.Context, db bun.IDB, versi return upload, nil } -func sortedStorageUploadIDs(uploadsByID map[int64]*model.StorageUpload) []int64 { +func sortedContentIDs(uploadsByID map[int64]*model.StorageContent) []int64 { ids := make([]int64, 0, len(uploadsByID)) for id := range uploadsByID { ids = append(ids, id) } - sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + slices.Sort(ids) return ids } -func storageUploadCopyCanBeCancelledForPermanentDelete(copyRow model.StorageUploadCopy) bool { +func storageUploadCopyCanBeCancelledForPermanentDelete(copyRow model.StorageCopy) bool { switch copyRow.Status { - case model.StorageUploadCopyStatusPending, model.StorageUploadCopyStatusPieceReady, model.StorageUploadCopyStatusCommitting: + case model.StorageCopyStatusPending, model.StorageCopyStatusPieceReady, model.StorageCopyStatusCommitting: return true default: return false } } -func storageUploadCopyHasAttemptedCommit(copyRow model.StorageUploadCopy) bool { +func storageUploadCopyHasAttemptedCommit(copyRow model.StorageCopy) bool { if copyRow.CommitAttemptID != nil && *copyRow.CommitAttemptID != "" && copyRow.CommitAttemptedAt != nil { return true } @@ -1488,286 +1326,77 @@ func storageUploadCopyHasAttemptedCommit(copyRow model.StorageUploadCopy) bool { // never resolved, including rows older code left behind when it sent a // submitted copy back to 'piece_ready'. Treat those as busy so a permanent // delete cannot discard a piece the provider may still be paid to keep. - return copyRow.Status != model.StorageUploadCopyStatusCommitted && + return copyRow.Status != model.StorageCopyStatusCommitted && copyRow.CommitTransactionID != nil && *copyRow.CommitTransactionID != "" } -const defaultStorageCleanupMaxRetries = 5 - -func createStorageCleanupTaskForDeletedVersion(ctx context.Context, db bun.IDB, uploadID int64, deletedVersionID string, maxRetries *int) (*int64, error) { - return createStorageCleanupTaskForDeletedVersions(ctx, db, uploadID, []string{deletedVersionID}, maxRetries) -} - -func createStorageCleanupTaskForDeletedVersions(ctx context.Context, db bun.IDB, uploadID int64, deletedVersionIDs []string, maxRetries *int) (*int64, error) { - if uploadID == 0 || len(deletedVersionIDs) == 0 { - return nil, fmt.Errorf("creating storage cleanup task: %w", ErrInvalidInput) +func reserveStorageCleanupForDeletedVersions(ctx context.Context, db bun.IDB, contentID int64, deletedVersionIDs []string) (*StorageCleanupReservation, error) { + if contentID == 0 || len(deletedVersionIDs) == 0 { + return nil, fmt.Errorf("preparing storage cleanup: %w", ErrInvalidInput) } - copies, err := storageCleanupCopySnapshots(ctx, db, uploadID) + copies, err := storageCleanupCopySnapshots(ctx, db, contentID) if err != nil { return nil, err } if len(copies) == 0 { return nil, nil } - - idempotencyKey := fmt.Sprintf("storage_cleanup:%d", uploadID) - task := new(model.Task) - err = db.NewSelect(). - Model(task). - Where("idempotency_key = ?", idempotencyKey). - Scan(ctx) - if err != nil && err != sql.ErrNoRows { - return nil, fmt.Errorf("selecting storage cleanup task: %w", err) - } - if err == nil { - return reuseStorageCleanupTask(ctx, db, task, uploadID, deletedVersionIDs, maxRetries) - } - - now := time.Now() - maxRetriesValue := storageCleanupMaxRetries(maxRetries) - task = &model.Task{ - Type: model.TaskTypeStorageCleanup, - RefType: "storage_upload", - RefID: uploadID, - RefVersionID: "", - IdempotencyKey: idempotencyKey, - Payload: storageCleanupTaskPayload(uploadID, deletedVersionIDs), - Status: model.TaskStatusQueued, - MaxRetries: maxRetriesValue, - ScheduledAt: now, - } - res, err := db.NewInsert(). - Model(task). - On("CONFLICT (idempotency_key) DO NOTHING"). - Exec(ctx) - if err != nil { - return nil, fmt.Errorf("creating storage cleanup task: %w", err) - } - rows, _ := res.RowsAffected() - if rows == 0 { - task = new(model.Task) - if err := db.NewSelect(). - Model(task). - Where("idempotency_key = ?", idempotencyKey). - Scan(ctx); err != nil { - return nil, fmt.Errorf("selecting concurrent storage cleanup task: %w", err) - } - return reuseStorageCleanupTask(ctx, db, task, uploadID, deletedVersionIDs, maxRetries) - } - if maxRetriesValue == 0 { - if _, err := db.NewUpdate(). - Model((*model.Task)(nil)). - Set("max_retries = 0"). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - return nil, fmt.Errorf("setting storage cleanup task max retries: %w", err) - } - } - if _, err := syncStorageCleanupCopySnapshots(ctx, db, task.ID, uploadID, copies); err != nil { - return nil, err - } - return &task.ID, nil -} - -func reuseStorageCleanupTask(ctx context.Context, db bun.IDB, task *model.Task, uploadID int64, deletedVersionIDs []string, maxRetries *int) (*int64, error) { - if task == nil { - return nil, fmt.Errorf("reusing storage cleanup task: %w", ErrInvalidInput) - } - copies, err := storageCleanupCopySnapshots(ctx, db, uploadID) - if err != nil { - return nil, err - } - if _, err := syncStorageCleanupCopySnapshots(ctx, db, task.ID, uploadID, copies); err != nil { - return nil, err - } - if task.Status != model.TaskStatusCompleted { - if taskStatusIsActive(task.Status) { - if err := updateStorageCleanupTaskPayload(ctx, db, task, uploadID, deletedVersionIDs); err != nil { - return nil, err - } - } - return &task.ID, nil - } - - remainingCopiesExist, err := db.NewSelect(). - Model((*model.StorageCleanupCopy)(nil)). - Where("task_id = ? AND status <> ?", task.ID, model.StorageCleanupCopyStatusRemoved). - Exists(ctx) - if err != nil { - return nil, fmt.Errorf("checking retained storage cleanup copies: %w", err) - } - if !remainingCopiesExist { - return &task.ID, nil - } - - now := time.Now() - res, err := db.NewUpdate(). - Model((*model.Task)(nil)). - Set("status = ?", model.TaskStatusQueued). - Set("retry_count = 0"). - Set("max_retries = ?", storageCleanupMaxRetries(maxRetries)). - Set("payload = ?", mergeStorageCleanupTaskPayload(task.Payload, uploadID, deletedVersionIDs)). - Set("scheduled_at = ?", now). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("started_at = NULL"). - Set("completed_at = NULL"). - Set("last_error = NULL"). - Set("wait_reason = NULL"). - Set("status_message = NULL"). - Where("id = ? AND status = ?", task.ID, model.TaskStatusCompleted). - Exec(ctx) - if err != nil { - return nil, fmt.Errorf("requeueing retained storage cleanup task: %w", err) - } - rows, _ := res.RowsAffected() - if rows == 0 { - return nil, fmt.Errorf("requeueing retained storage cleanup task %d: %w", task.ID, ErrConflict) - } - return &task.ID, nil -} - -func syncStorageCleanupCopySnapshots( - ctx context.Context, - db bun.IDB, - taskID int64, - uploadID int64, - copies []model.StorageCleanupCopy, -) (int, error) { - if taskID <= 0 || uploadID <= 0 { - return 0, fmt.Errorf("syncing storage cleanup copy snapshots: %w", ErrInvalidInput) - } - if len(copies) == 0 { - return 0, nil - } now := time.Now() for i := range copies { - copies[i].TaskID = taskID copies[i].CreatedAt = now copies[i].UpdatedAt = now } - res, err := db.NewInsert(). + _, err = db.NewInsert(). Model(&copies). - On("CONFLICT (task_id, copy_index) DO NOTHING"). + On("CONFLICT (content_id, storage_data_set_id, piece_id) DO NOTHING"). Exec(ctx) if err != nil { - return 0, fmt.Errorf("syncing storage cleanup copy snapshots: %w", err) - } - rows, _ := res.RowsAffected() - return int(rows), nil -} - -func updateStorageCleanupTaskPayload(ctx context.Context, db bun.IDB, task *model.Task, uploadID int64, deletedVersionIDs []string) error { - payload := mergeStorageCleanupTaskPayload(task.Payload, uploadID, deletedVersionIDs) - res, err := db.NewUpdate(). - Model((*model.Task)(nil)). - Set("payload = ?", payload). - Where("id = ?", task.ID). - Exec(ctx) + return nil, fmt.Errorf("persisting storage cleanup snapshots: %w", err) + } + reservation := new(StorageCleanupReservation) + err = db.NewRaw(`UPDATE storage_contents + SET cleanup_generation = cleanup_generation + CASE WHEN cleanup_task_id IS NULL THEN 1 ELSE 0 END, + updated_at = ? + WHERE id = ? + RETURNING id AS content_id, cleanup_generation AS generation, cleanup_task_id AS task_id`, now, contentID).Scan(ctx, reservation) if err != nil { - return fmt.Errorf("updating storage cleanup task payload: %w", err) + return nil, fmt.Errorf("reserving storage cleanup generation: %w", err) } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("updating storage cleanup task payload %d: %w", task.ID, ErrNotFound) - } - return nil -} - -func taskStatusIsActive(status model.TaskStatus) bool { - for _, active := range activeTaskStatuses() { - if status == active { - return true - } - } - return false -} - -func storageCleanupMaxRetries(maxRetries *int) int { - if maxRetries == nil { - return defaultStorageCleanupMaxRetries - } - return *maxRetries -} - -func storageCleanupTaskPayload(uploadID int64, deletedVersionIDs []string) map[string]interface{} { - payload := map[string]interface{}{ - "storage_upload_id": uploadID, - } - if len(deletedVersionIDs) > 0 { - payload["deleted_source_version"] = deletedVersionIDs[0] - } - payload["deleted_source_versions"] = deletedVersionIDs - return payload -} - -func mergeStorageCleanupTaskPayload(existing map[string]interface{}, uploadID int64, deletedVersionIDs []string) map[string]interface{} { - merged := storageCleanupPayloadVersionIDs(existing) - for _, versionID := range deletedVersionIDs { - merged = appendUniqueString(merged, versionID) - } - return storageCleanupTaskPayload(uploadID, merged) -} - -func storageCleanupPayloadVersionIDs(payload map[string]interface{}) []string { - var out []string - if payload == nil { - return out - } - - switch values := payload["deleted_source_versions"].(type) { - case []string: - for _, value := range values { - out = appendUniqueString(out, value) - } - case []interface{}: - for _, value := range values { - text, ok := value.(string) - if ok { - out = appendUniqueString(out, text) - } - } - } - legacy, ok := payload["deleted_source_version"].(string) - if ok { - out = appendUniqueString(out, legacy) - } - return out + return reservation, nil } func appendUniqueString(values []string, value string) []string { if value == "" { return values } - for _, existing := range values { - if existing == value { - return values - } + if slices.Contains(values, value) { + return values } return append(values, value) } -func storageCleanupCopySnapshots(ctx context.Context, db bun.IDB, uploadID int64) ([]model.StorageCleanupCopy, error) { +func storageCleanupCopySnapshots(ctx context.Context, db bun.IDB, contentID int64) ([]model.StorageCleanupCopy, error) { var copies []model.StorageCleanupCopy err := db.NewRaw(`SELECT - storage_copy.upload_id, + storage_copy.content_id, + storage_copy.bucket_id, storage_copy.copy_index, storage_copy.provider_id, storage_copy.storage_data_set_id, storage_data_set.data_set_id, storage_data_set.client_data_set_id, storage_copy.piece_id, - storage_upload.piece_cid, + storage_content.piece_cid, storage_copy.retrieval_url, ? AS status - FROM storage_upload_copies AS storage_copy - JOIN storage_uploads AS storage_upload ON storage_upload.id = storage_copy.upload_id - LEFT JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id - WHERE storage_copy.upload_id = ? AND storage_copy.status = ? + FROM storage_copies AS storage_copy + JOIN storage_contents AS storage_content ON storage_content.id = storage_copy.content_id + JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id + WHERE storage_copy.content_id = ? AND storage_copy.status = ? ORDER BY storage_copy.copy_index ASC`, model.StorageCleanupCopyStatusPending, - uploadID, - model.StorageUploadCopyStatusCommitted, + contentID, + model.StorageCopyStatusCommitted, ).Scan(ctx, &copies) if err != nil { return nil, fmt.Errorf("loading storage cleanup copy snapshots: %w", err) @@ -1863,7 +1492,6 @@ func createRestoredVersionAndSetCurrent(ctx context.Context, db bun.IDB, version now := time.Now() version.ObjectID = source.ObjectID - version.IsCurrent = true if version.CreatedAt.IsZero() { version.CreatedAt = now } @@ -1871,27 +1499,22 @@ func createRestoredVersionAndSetCurrent(ctx context.Context, db bun.IDB, version version.UpdatedAt = now } - res, err := db.NewUpdate(). - Model((*model.ObjectVersion)(nil)). - Set("is_current = ?", false). - Where("object_id = ? AND version_id = ? AND is_current = ?", source.ObjectID, expectedCurrentVersionID, true). - Exec(ctx) - if err != nil { - return 0, fmt.Errorf("clearing expected current version: %w", err) - } - rows, _ := res.RowsAffected() - if rows != 1 { - return 0, fmt.Errorf("creating restored object version: %w", ErrConflict) - } if _, err := db.NewInsert().Model(version).Exec(ctx); err != nil { return 0, fmt.Errorf("inserting restored object version: %w", err) } - if _, err := db.NewUpdate(). + // Moving the pointer is the compare-and-set: it succeeds only while the + // caller's expected version is still the current one. + res, err := db.NewUpdate(). Model((*model.Object)(nil)). + Set("current_version_id = ?", version.VersionID). Set("updated_at = ?", now). - Where("id = ?", source.ObjectID). - Exec(ctx); err != nil { - return 0, fmt.Errorf("updating restored object identity timestamp: %w", err) + Where("id = ? AND current_version_id = ?", source.ObjectID, expectedCurrentVersionID). + Exec(ctx) + if err != nil { + return 0, fmt.Errorf("pointing object at restored version: %w", err) + } + if rows, _ := res.RowsAffected(); rows != 1 { + return 0, fmt.Errorf("creating restored object version: %w", ErrConflict) } return source.ObjectID, nil } @@ -1912,7 +1535,6 @@ func createVersionAndSetCurrentFromExisting(ctx context.Context, db bun.IDB, ver } version.ObjectID = obj.ID - version.IsCurrent = true if version.CreatedAt.IsZero() { version.CreatedAt = now } @@ -1922,6 +1544,17 @@ func createVersionAndSetCurrentFromExisting(ctx context.Context, db bun.IDB, ver if _, insertErr := db.NewInsert().Model(version).Exec(ctx); insertErr != nil { return ObjectVersionWriteResult{}, fmt.Errorf("inserting object version: %w", insertErr) } + // The object is inserted with a null pointer because its version does + // not exist yet; one update completes the identity. + if _, updateErr := db.NewUpdate(). + Model((*model.Object)(nil)). + Set("current_version_id = ?", version.VersionID). + Set("updated_at = ?", now). + Where("id = ?", obj.ID). + Exec(ctx); updateErr != nil { + return ObjectVersionWriteResult{}, fmt.Errorf("pointing new object at its first version: %w", updateErr) + } + version.IsCurrent = true return ObjectVersionWriteResult{ ObjectID: obj.ID, VersionID: version.VersionID, @@ -1931,29 +1564,22 @@ func createVersionAndSetCurrentFromExisting(ctx context.Context, db bun.IDB, ver } version.ObjectID = existing.ID - version.IsCurrent = true if version.CreatedAt.IsZero() { version.CreatedAt = now } if version.UpdatedAt.IsZero() { version.UpdatedAt = now } - if _, err := db.NewUpdate(). - Model((*model.ObjectVersion)(nil)). - Set("is_current = ?", false). - Where("object_id = ? AND is_current = ?", existing.ID, true). - Exec(ctx); err != nil { - return ObjectVersionWriteResult{}, fmt.Errorf("clearing previous current version: %w", err) - } if _, insertErr := db.NewInsert().Model(version).Exec(ctx); insertErr != nil { return ObjectVersionWriteResult{}, fmt.Errorf("inserting object version: %w", insertErr) } if _, updateErr := db.NewUpdate(). Model((*model.Object)(nil)). + Set("current_version_id = ?", version.VersionID). Set("updated_at = ?", now). Where("id = ?", existing.ID). Exec(ctx); updateErr != nil { - return ObjectVersionWriteResult{}, fmt.Errorf("updating object identity timestamp: %w", updateErr) + return ObjectVersionWriteResult{}, fmt.Errorf("pointing object at new version: %w", updateErr) } return ObjectVersionWriteResult{ ObjectID: existing.ID, @@ -1994,18 +1620,11 @@ func createDeleteMarkerAndSetCurrent(ctx context.Context, db bun.IDB, marker *mo } } - marker.IsCurrent = true marker.IsDeleteMarker = true marker.Size = 0 marker.ETag = "" - marker.Checksum = "" marker.ContentType = "" - marker.CacheKey = "" - marker.StorageUploadID = nil - marker.InCache = false - marker.State = model.ObjectStateCached - marker.FailedAtState = nil - marker.LastError = nil + marker.ContentID = nil if marker.CreatedAt.IsZero() { marker.CreatedAt = now } @@ -2013,21 +1632,21 @@ func createDeleteMarkerAndSetCurrent(ctx context.Context, db bun.IDB, marker *mo marker.UpdatedAt = now } - if _, err := db.NewUpdate(). - Model((*model.ObjectVersion)(nil)). - Set("is_current = ?", false). - Where("object_id = ? AND is_current = ?", marker.ObjectID, true). - Exec(ctx); err != nil { - return fmt.Errorf("clearing previous current version: %w", err) - } if _, err := db.NewInsert(). Model(marker). - Column("version_id", "object_id", "bucket_id", "key", "size", "e_tag", "checksum", "content_type", "metadata", "cache_key", "storage_upload_id", "in_cache", "is_current", "is_delete_marker", "state", "failed_at_state", "last_error", "created_at", "updated_at"). + Column("version_id", "object_id", "bucket_id", "key", "content_id", "size", "e_tag", "content_type", "metadata", "is_delete_marker", "created_at", "updated_at"). Value("content_type", "?", ""). - Value("in_cache", "?", false). Exec(ctx); err != nil { return fmt.Errorf("inserting delete marker: %w", err) } + if _, err := db.NewUpdate(). + Model((*model.Object)(nil)). + Set("current_version_id = ?", marker.VersionID). + Set("updated_at = ?", now). + Where("id = ?", marker.ObjectID). + Exec(ctx); err != nil { + return fmt.Errorf("pointing object at delete marker: %w", err) + } return nil } @@ -2046,16 +1665,66 @@ func deleteMarkerVersion(ctx context.Context, db bun.IDB, bucketID int64, key st return fmt.Errorf("deleting marker version: %w", ErrInvalidInput) } + // The object's pointer has to leave this version before the row can go. + if version.IsCurrent { + if err := repointObjectAwayFromVersion(ctx, db, version.ObjectID, versionID); err != nil { + return err + } + } if _, err := db.NewDelete(). Model((*model.ObjectVersion)(nil)). Where("bucket_id = ? AND key = ? AND version_id = ?", bucketID, key, versionID). Exec(ctx); err != nil { return fmt.Errorf("deleting marker version: %w", err) } - if !version.IsCurrent { + if version.IsCurrent { + return deleteObjectIdentityIfEmpty(ctx, db, version.ObjectID) + } + return nil +} + +// repointObjectAwayFromVersion moves the object's current pointer to the newest +// version that is not the one about to be deleted, or clears it when that +// version was the last. A version can only be removed once nothing points at it. +func repointObjectAwayFromVersion(ctx context.Context, db bun.IDB, objectID int64, versionID string) error { + next, err := selectLatestVersionByObjectIDExcluding(ctx, db, objectID, versionID) + if err != nil { + return err + } + query := db.NewUpdate(). + Model((*model.Object)(nil)). + Set("updated_at = ?", time.Now()). + Where("id = ?", objectID) + if next == nil { + query = query.Set("current_version_id = NULL") + } else { + query = query.Set("current_version_id = ?", next.VersionID) + } + if _, err := query.Exec(ctx); err != nil { + return fmt.Errorf("repointing object away from version %s: %w", versionID, err) + } + return nil +} + +// deleteObjectIdentityIfEmpty removes an object that has no versions left. +func deleteObjectIdentityIfEmpty(ctx context.Context, db bun.IDB, objectID int64) error { + remaining, err := db.NewSelect(). + Model((*model.ObjectVersion)(nil)). + Where("object_id = ?", objectID). + Count(ctx) + if err != nil { + return fmt.Errorf("counting remaining object versions: %w", err) + } + if remaining > 0 { return nil } - return promoteLatestVersionOrDeleteObject(ctx, db, version.ObjectID) + if _, err := db.NewDelete(). + Model((*model.Object)(nil)). + Where("id = ?", objectID). + Exec(ctx); err != nil { + return fmt.Errorf("deleting empty object identity: %w", err) + } + return nil } func restoreCurrentDeleteMarkerStack(ctx context.Context, db bun.IDB, bucketID int64, key string, currentMarkerVersionID string) (*model.ObjectVersion, error) { @@ -2095,79 +1764,35 @@ func restoreCurrentDeleteMarkerStack(ctx context.Context, db bun.IDB, bucketID i return nil, fmt.Errorf("restoring delete marker stack: %w", ErrConflict) } - if _, err := db.NewDelete(). - Model((*model.ObjectVersion)(nil)). - Where("object_id = ? AND version_id IN (?)", current.ObjectID, bun.List(markerIDs)). - Exec(ctx); err != nil { - return nil, fmt.Errorf("deleting marker stack: %w", err) - } - if _, err := db.NewUpdate(). - Model((*model.ObjectVersion)(nil)). - Set("is_current = ?", true). - Where("version_id = ?", restoreTarget.VersionID). - Exec(ctx); err != nil { - return nil, fmt.Errorf("restoring data version current flag: %w", err) - } + // The pointer moves before the markers go: a version cannot be deleted while + // the object still points at it. if _, err := db.NewUpdate(). Model((*model.Object)(nil)). + Set("current_version_id = ?", restoreTarget.VersionID). Set("updated_at = ?", time.Now()). Where("id = ?", current.ObjectID). Exec(ctx); err != nil { - return nil, fmt.Errorf("updating object identity timestamp: %w", err) - } - restoreTarget.IsCurrent = true - return restoreTarget, nil -} - -func promoteLatestVersionOrDeleteObject(ctx context.Context, db bun.IDB, objectID int64) error { - latest, err := selectLatestVersionByObjectID(ctx, db, objectID) - if err != nil { - return err - } - if latest == nil { - if _, err := db.NewDelete(). - Model((*model.Object)(nil)). - Where("id = ?", objectID). - Exec(ctx); err != nil { - return fmt.Errorf("deleting empty object identity: %w", err) - } - return nil + return nil, fmt.Errorf("pointing object at restored version: %w", err) } - if _, err := db.NewUpdate(). + if _, err := db.NewDelete(). Model((*model.ObjectVersion)(nil)). - Set("is_current = (version_id = ?)", latest.VersionID). - Where("object_id = ?", objectID). - Exec(ctx); err != nil { - return fmt.Errorf("promoting latest version: %w", err) - } - if _, err := db.NewUpdate(). - Model((*model.Object)(nil)). - Set("updated_at = ?", time.Now()). - Where("id = ?", objectID). + Where("object_id = ? AND version_id IN (?)", current.ObjectID, bun.List(markerIDs)). Exec(ctx); err != nil { - return fmt.Errorf("updating object identity timestamp: %w", err) + return nil, fmt.Errorf("deleting marker stack: %w", err) } - return nil + restoreTarget.IsCurrent = true + return restoreTarget, nil } func normalizeObjectVersion(version *model.ObjectVersion) { - version.FailedAtState = nil - version.LastError = nil - if version.State == "" { - version.State = model.ObjectStateCached - } - if version.State != model.ObjectStateCacheEvicted { - version.InCache = true - } - if version.InCache && !version.IsDeleteMarker && version.CacheAccessedAt == nil { - accessedAt := version.CreatedAt - if accessedAt.IsZero() { - accessedAt = time.Now() - } - version.CacheAccessedAt = &accessedAt + if version.Metadata == nil { + version.Metadata = map[string]string{} } - if version.ContentType == "" { - version.ContentType = "application/octet-stream" + if version.IsDeleteMarker { + version.ContentID = nil + version.Size = 0 + version.ETag = "" + version.ContentType = "" } } @@ -2189,7 +1814,8 @@ func selectCurrentVersionByObjectID(ctx context.Context, db bun.IDB, objectID in Model(version). ModelTableExpr("object_versions AS object_version") q = withObjectVersionStorageColumns(q, "object_version") - err := q.Where("object_version.object_id = ? AND object_version.is_current = ?", objectID, true).Scan(ctx) + err := q.Where("object_version.object_id = ?", objectID). + Where("current_object.current_version_id = object_version.version_id").Scan(ctx) if err != nil { return nil, err } @@ -2202,7 +1828,8 @@ func selectCurrentVersionByBucketAndKey(ctx context.Context, db bun.IDB, bucketI Model(version). ModelTableExpr("object_versions AS object_version") q = withObjectVersionStorageColumns(q, "object_version") - err := q.Where("object_version.bucket_id = ? AND object_version.key = ? AND object_version.is_current = ?", bucketID, key, true).Scan(ctx) + err := q.Where("object_version.bucket_id = ? AND object_version.key = ?", bucketID, key). + Where("current_object.current_version_id = object_version.version_id").Scan(ctx) if err != nil { if err == sql.ErrNoRows { return nil, nil @@ -2228,13 +1855,16 @@ func selectVersionByBucketKeyAndID(ctx context.Context, db bun.IDB, bucketID int return version, nil } -func selectLatestVersionByObjectID(ctx context.Context, db bun.IDB, objectID int64) (*model.ObjectVersion, error) { +// selectLatestVersionByObjectIDExcluding finds the version that should take over +// when one version is about to be removed. +func selectLatestVersionByObjectIDExcluding(ctx context.Context, db bun.IDB, objectID int64, excludeVersionID string) (*model.ObjectVersion, error) { version := new(model.ObjectVersion) q := db.NewSelect(). Model(version). ModelTableExpr("object_versions AS object_version") q = withObjectVersionStorageColumns(q, "object_version") err := q.Where("object_version.object_id = ?", objectID). + Where("object_version.version_id <> ?", excludeVersionID). OrderExpr("object_version.created_at DESC"). OrderExpr("object_version.version_id DESC"). Limit(1). @@ -2243,7 +1873,7 @@ func selectLatestVersionByObjectID(ctx context.Context, db bun.IDB, objectID int if err == sql.ErrNoRows { return nil, nil } - return nil, fmt.Errorf("selecting latest object version: %w", err) + return nil, fmt.Errorf("selecting replacement object version: %w", err) } return version, nil } @@ -2338,9 +1968,12 @@ func objectVersionMatchesVersion(current *model.ObjectVersion, version *model.Ob if current == nil || current.State == model.ObjectStateFailed || current.IsDeleteMarker { return false } - return current.Size == version.Size && + // Content identity carries bucket, checksum and size, so it replaces the + // old size+checksum pair. The incoming version only ever carries a content + // id; checksum is a read projection and would compare empty here. + return current.ContentID != nil && version.ContentID != nil && + *current.ContentID == *version.ContentID && current.ETag == version.ETag && - current.Checksum == version.Checksum && current.ContentType == version.ContentType && maps.Equal(current.Metadata, version.Metadata) } @@ -2355,59 +1988,14 @@ func restoreSourceAlreadyCurrent(source, current *model.ObjectVersion) bool { if current.State == model.ObjectStateFailed || current.IsDeleteMarker || (!current.InCache && !current.InFilecoin) { return false } - return current.Size == source.Size && - current.Checksum == source.Checksum && + // Identical bytes are identical content rows, so content identity replaces + // the old size+checksum comparison and no longer depends on a projection. + return source.ContentID != nil && current.ContentID != nil && + *source.ContentID == *current.ContentID && current.ContentType == source.ContentType && maps.Equal(current.Metadata, source.Metadata) } -func updateVersionState(ctx context.Context, db bun.IDB, versionID string, from, to model.ObjectState) error { - now := time.Now() - q := db.NewUpdate(). - Model((*model.ObjectVersion)(nil)). - Set("state = ?", to). - Set("updated_at = ?", now). - Where("version_id = ? AND state = ?", versionID, from) - q = applyCacheLocationForState(q, to) - if from == model.ObjectStateFailed { - q = q.Set("failed_at_state = NULL") - q = q.Set("last_error = NULL") - } - res, err := q.Exec(ctx) - if err != nil { - return fmt.Errorf("updating object version state: %w", err) - } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("state transition %s→%s failed: version %s not in expected state", from, to, versionID) - } - return nil -} - -func resetStaleVersions(ctx context.Context, db bun.IDB, fromState, toState model.ObjectState, staleBefore time.Time) ([]string, error) { - now := time.Now() - var rows []struct { - VersionID string `bun:"version_id"` - } - cacheColumn := cacheLocationSQLForState(toState) - query := `UPDATE object_versions SET state = ?, updated_at = ?` + cacheColumn + ` WHERE state = ? AND updated_at < ? RETURNING version_id` - args := []interface{}{toState, now, fromState, staleBefore} - if fromState == model.ObjectStateFailed { - query = `UPDATE object_versions SET state = ?, failed_at_state = NULL, last_error = NULL, updated_at = ?` + cacheColumn + ` WHERE state = ? AND updated_at < ? RETURNING version_id` - } - if err := db.NewRaw(query, args...).Scan(ctx, &rows); err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("resetting stale object versions: %w", err) - } - versionIDs := make([]string, 0, len(rows)) - for _, row := range rows { - versionIDs = append(versionIDs, row.VersionID) - } - return versionIDs, nil -} - func objectIdentityFromVersion(version *model.ObjectVersion) *model.Object { return &model.Object{ BucketID: version.BucketID, @@ -2415,40 +2003,57 @@ func objectIdentityFromVersion(version *model.ObjectVersion) *model.Object { } } +// setVersionCachePresence records residency against the version's content. +// Several versions can share one cached file, so presence is stored once per +// content rather than once per version. func setVersionCachePresence(ctx context.Context, db bun.IDB, versionID string, inCache bool) error { - res, err := db.NewUpdate(). - Model((*model.ObjectVersion)(nil)). - Set("in_cache = ?", inCache). - Where("version_id = ?", versionID). - Exec(ctx) + contentID, err := contentIDForVersion(ctx, db, versionID) if err != nil { return fmt.Errorf("setting version cache presence: %w", err) } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("setting version cache presence: version %s not found", versionID) + if contentID == nil { + return nil } - return nil + return upsertContentCachePresence(ctx, db, *contentID, inCache, nil) } -func applyCacheLocationForState(q *bun.UpdateQuery, state model.ObjectState) *bun.UpdateQuery { - switch state { - case model.ObjectStateCached: - return q.Set("in_cache = ?", true) - case model.ObjectStateCacheEvicted: - return q.Set("in_cache = ?", false) - default: - return q +func contentIDForVersion(ctx context.Context, db bun.IDB, versionID string) (*int64, error) { + var contentID *int64 + err := db.NewSelect(). + Model((*model.ObjectVersion)(nil)). + Column("content_id"). + Where("version_id = ?", versionID). + Scan(ctx, &contentID) + if err != nil { + if err == sql.ErrNoRows { + return nil, fmt.Errorf("version %s not found", versionID) + } + return nil, err } + return contentID, nil } -func cacheLocationSQLForState(state model.ObjectState) string { - switch state { - case model.ObjectStateCached: - return ", in_cache = TRUE" - case model.ObjectStateCacheEvicted: - return ", in_cache = FALSE" - default: - return "" +// upsertContentCachePresence writes the residency row, creating it the first +// time a content is cached. +func upsertContentCachePresence(ctx context.Context, db bun.IDB, contentID int64, inCache bool, accessedAt *time.Time) error { + now := time.Now() + entry := &model.ObjectCache{ + ContentID: contentID, + InCache: inCache, + CacheAccessedAt: accessedAt, + CreatedAt: now, + UpdatedAt: now, + } + query := db.NewInsert().Model(entry).On("CONFLICT (content_id) DO UPDATE"). + Set("in_cache = EXCLUDED.in_cache"). + Set("updated_at = EXCLUDED.updated_at") + if accessedAt != nil { + query = query.Set(`cache_accessed_at = CASE + WHEN object_cache.cache_accessed_at IS NULL OR object_cache.cache_accessed_at < EXCLUDED.cache_accessed_at + THEN EXCLUDED.cache_accessed_at ELSE object_cache.cache_accessed_at END`) + } + if _, err := query.Exec(ctx); err != nil { + return fmt.Errorf("recording content cache presence: %w", err) } + return nil } diff --git a/internal/db/repository/object_repo_test.go b/internal/db/repository/object_repo_test.go index fd61467..e6387bd 100644 --- a/internal/db/repository/object_repo_test.go +++ b/internal/db/repository/object_repo_test.go @@ -12,10 +12,298 @@ import ( "github.com/strahe/synaps3/internal/db/migrations" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/testutil" "github.com/uptrace/bun" "github.com/uptrace/bun/dialect/sqlitedialect" ) +// seedContent creates the byte identity a data version needs before it exists. +func seedContent(t *testing.T, repos *repository.Repositories, bucketID int64, checksum string, size int64) int64 { + t.Helper() + content, err := repos.Contents.EnsureContent(t.Context(), repository.EnsureContentInput{ + BucketID: bucketID, ContentSize: size, Checksum: testutil.StorageChecksum(checksum), RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("EnsureContent(%s): %v", checksum, err) + } + return content.ID +} + +// createVersion gives a data version the content identity it now requires and +// then creates it. Tests used to insert versions directly; the bytes are their +// own row now, so content comes first. +func createVersion(t *testing.T, repos *repository.Repositories, version *model.ObjectVersion) (int64, error) { + t.Helper() + if !version.IsDeleteMarker && version.ContentID == nil { + id := seedContent(t, repos, version.BucketID, "checksum-"+version.VersionID, version.Size) + version.ContentID = &id + } + return repos.Objects.CreateVersionAndSetCurrent(t.Context(), version) +} + +// withContent attaches the content identity a data version now requires. +func withContent(t *testing.T, repos *repository.Repositories, version *model.ObjectVersion) *model.ObjectVersion { + t.Helper() + if !version.IsDeleteMarker && version.ContentID == nil { + id := seedContent(t, repos, version.BucketID, "checksum-"+version.VersionID, version.Size) + version.ContentID = &id + } + return version +} + +func TestObjectRepo_AggregateByStateIncludesTotalSize(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := seedBucket(t, db, "state-aggregate-bucket") + + cached := newObjectVersion(bucket.ID, "cached.txt", "01J00000000000000000000B01", 10) + if _, err := createVersion(t, repos, cached); err != nil { + t.Fatalf("seed cached version: %v", err) + } + // A content whose every copy failed reads as failed, so the aggregate has + // to derive the bucket instead of grouping a stored column. + failed := newObjectVersion(bucket.ID, "failed.txt", "01J00000000000000000000B02", 20) + if _, err := createVersion(t, repos, failed); err != nil { + t.Fatalf("seed failed version: %v", err) + } + seedFailedContentCopy(t, db, repos, bucket.ID, *failed.ContentID) + if _, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "deleted.txt", "01J00000000000000000000B03"); err != nil { + t.Fatalf("seed delete marker: %v", err) + } + + rows, err := repos.Objects.AggregateByState(ctx) + if err != nil { + t.Fatalf("AggregateByState: %v", err) + } + byState := make(map[string]repository.ObjectStateAggregate, len(rows)) + for _, row := range rows { + byState[row.State] = row + } + if got := byState[string(model.ObjectStateCached)]; got.Count != 1 || got.TotalSize != 10 { + t.Fatalf("cached aggregate = count:%d size:%d, want 1/10", got.Count, got.TotalSize) + } + if got := byState[string(model.ObjectStateFailed)]; got.Count != 1 || got.TotalSize != 20 { + t.Fatalf("failed aggregate = count:%d size:%d, want 1/20", got.Count, got.TotalSize) + } +} + +func TestObjectRepo_CountOverviewAttention(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := seedBucket(t, db, "overview-attention-bucket") + + healthy := newObjectVersion(bucket.ID, "healthy.txt", "01J00000000000000000000A01", 10) + if _, err := createVersion(t, repos, healthy); err != nil { + t.Fatalf("seed healthy version: %v", err) + } + + // Two distinct ways to need attention: every copy failed, and the content + // carries an ingest error. + failedCopies := newObjectVersion(bucket.ID, "failed-copies.txt", "01J00000000000000000000A02", 10) + if _, err := createVersion(t, repos, failedCopies); err != nil { + t.Fatalf("seed failed-copies version: %v", err) + } + seedFailedContentCopy(t, db, repos, bucket.ID, *failedCopies.ContentID) + + failedContent := newObjectVersion(bucket.ID, "failed-content.txt", "01J00000000000000000000A03", 10) + if _, err := createVersion(t, repos, failedContent); err != nil { + t.Fatalf("seed failed-content version: %v", err) + } + if err := repos.Contents.RecordContentFailure(ctx, *failedContent.ContentID, "provider failed"); err != nil { + t.Fatalf("record content failure: %v", err) + } + + // Unavailable is the absence of both a cached copy and a readable one. + unavailable := newObjectVersion(bucket.ID, "unavailable.txt", "01J00000000000000000000A04", 10) + if _, err := createVersion(t, repos, unavailable); err != nil { + t.Fatalf("seed unavailable version: %v", err) + } + if err := repos.Objects.ClearContentCachePresence(ctx, *unavailable.ContentID); err != nil { + t.Fatalf("clear unavailable cache presence: %v", err) + } + + if _, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "deleted.txt", "01J00000000000000000000A05"); err != nil { + t.Fatalf("seed deleted marker: %v", err) + } + + counts, err := repos.Objects.CountOverviewAttention(ctx) + if err != nil { + t.Fatalf("CountOverviewAttention: %v", err) + } + if counts.NeedsAttention != 2 { + t.Fatalf("NeedsAttention = %d, want 2", counts.NeedsAttention) + } + if counts.Unavailable != 1 { + t.Fatalf("Unavailable = %d, want 1", counts.Unavailable) + } +} + +// seedFailedContentCopy binds one copy to a content and fails it, which is how +// an ingest failure is expressed once state is derived from the copies. +func seedFailedContentCopy(t *testing.T, db *bun.DB, repos *repository.Repositories, bucketID, contentID int64) { + t.Helper() + ctx := context.Background() + binding, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucketID, ProviderID: onChainID(t, "808"), CopyIndex: 0, CreatedByContentID: contentID, + }) + if err != nil { + t.Fatalf("seed failed copy binding: %v", err) + } + if err := repos.Contents.CreateUploadCopiesForBindings(ctx, contentID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: binding.ID, CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: binding.ProviderID, + }}); err != nil { + t.Fatalf("seed failed copy: %v", err) + } + if _, err := db.NewUpdate(). + Model((*model.StorageCopy)(nil)). + Set("status = ?", model.StorageCopyStatusFailed). + Where("content_id = ?", contentID). + Exec(ctx); err != nil { + t.Fatalf("fail seeded copy: %v", err) + } +} + +func TestObjectRepo_CreateDeleteMarkerHidesCurrentObjectButKeepsVersionHistory(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := seedBucket(t, db, "delete-marker-bucket") + + data := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000001001", 10) + if _, err := createVersion(t, repos, data); err != nil { + t.Fatalf("create data version: %v", err) + } + marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "file.txt", "01J00000000000000000001002") + if err != nil { + t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) + } + // A marker names no bytes, so it has no content, no cache key and no size. + if !marker.IsDeleteMarker || marker.Size != 0 || marker.ContentID != nil || marker.CacheKey() != "" || marker.InCache { + t.Fatalf("marker = %#v, want metadata-only delete marker", marker) + } + if marker.Metadata == nil || len(marker.Metadata) != 0 { + t.Fatalf("delete marker metadata = %#v, want empty map", marker.Metadata) + } + + current, err := repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "file.txt") + if err != nil { + t.Fatalf("GetCurrentVersionByBucketAndKey: %v", err) + } + if current == nil || !current.IsDeleteMarker || current.VersionID != marker.VersionID { + t.Fatalf("current = %#v, want the delete marker", current) + } + + currentList, err := repos.Objects.ListCurrentVersionsByBucket(ctx, bucket.ID, "", "", 10) + if err != nil { + t.Fatalf("ListCurrentVersionsByBucket: %v", err) + } + if len(currentList) != 0 { + t.Fatalf("current list len = %d, want object hidden", len(currentList)) + } + + versions, err := repos.Objects.ListVersionsByBucket(ctx, bucket.ID, "", "", "", 10) + if err != nil { + t.Fatalf("ListVersionsByBucket: %v", err) + } + if len(versions) != 2 { + t.Fatalf("versions len = %d, want marker plus data version", len(versions)) + } + if !versions[0].IsDeleteMarker || !versions[0].IsCurrent { + t.Fatalf("first version = %#v, want current delete marker", versions[0].ObjectVersion) + } + if versions[1].VersionID != data.VersionID || versions[1].IsDeleteMarker { + t.Fatalf("second version = %#v, want data version %s", versions[1].ObjectVersion, data.VersionID) + } + // The hidden data version keeps its bytes and its derived position. + if versions[1].State != model.ObjectStateCached { + t.Fatalf("hidden data version state = %s, want cached", versions[1].State) + } +} + +func TestObjectRepo_CacheAccessAndCommitKeepPresenceSemanticsSeparate(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := seedBucket(t, db, "cache-access-bucket") + version := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000CA01", 10) + if _, err := createVersion(t, repos, version); err != nil { + t.Fatalf("CreateVersionAndSetCurrent: %v", err) + } + // Residency lives on the content's cache entry, so the version row is not + // what these writes touch. + lifecycleUpdatedAt := time.Date(2026, time.July, 1, 12, 0, 0, 0, time.UTC) + mustExecRaw(t, db, `UPDATE object_versions SET updated_at = ? WHERE version_id = ?`, lifecycleUpdatedAt, version.VersionID) + if err := repos.Objects.ClearContentCachePresence(ctx, *version.ContentID); err != nil { + t.Fatalf("clear cache presence: %v", err) + } + mustExecRaw(t, db, `UPDATE object_cache SET cache_accessed_at = NULL WHERE content_id = ?`, *version.ContentID) + + accessedAt := lifecycleUpdatedAt.Add(2 * time.Hour) + if err := repos.Objects.RecordContentCacheAccess(ctx, *version.ContentID, accessedAt); err != nil { + t.Fatalf("RecordVersionCacheAccess: %v", err) + } + got, err := repos.Objects.GetVersionByID(ctx, version.VersionID) + if err != nil || got == nil { + t.Fatalf("GetVersionByID: version=%v err=%v", got, err) + } + if got.InCache { + t.Fatal("in_cache = true after access-only timestamp update, want false") + } + if got.CacheAccessedAt == nil || !got.CacheAccessedAt.Equal(accessedAt) { + t.Fatalf("cache_accessed_at = %v, want %v", got.CacheAccessedAt, accessedAt) + } + if !got.UpdatedAt.Equal(lifecycleUpdatedAt) { + t.Fatalf("updated_at = %v, want unchanged %v", got.UpdatedAt, lifecycleUpdatedAt) + } + + committedAt := accessedAt.Add(time.Hour) + if err := repos.Objects.RecordContentCacheCommit(ctx, *version.ContentID, committedAt); err != nil { + t.Fatalf("RecordVersionCacheCommit: %v", err) + } + got, err = repos.Objects.GetVersionByID(ctx, version.VersionID) + if err != nil || got == nil { + t.Fatalf("GetVersionByID after cache commit: version=%v err=%v", got, err) + } + if !got.InCache { + t.Fatal("in_cache = false after cache commit, want true") + } + if got.CacheAccessedAt == nil || !got.CacheAccessedAt.Equal(committedAt) { + t.Fatalf("cache_accessed_at after commit = %v, want %v", got.CacheAccessedAt, committedAt) + } + if !got.UpdatedAt.Equal(lifecycleUpdatedAt) { + t.Fatalf("updated_at after cache commit = %v, want unchanged %v", got.UpdatedAt, lifecycleUpdatedAt) + } + + if err := repos.Objects.ClearContentCachePresence(ctx, *version.ContentID); err != nil { + t.Fatalf("ClearContentCachePresence: %v", err) + } + olderAccess := committedAt.Add(-time.Hour) + if err := repos.Objects.RecordContentCacheAccess(ctx, *version.ContentID, olderAccess); err != nil { + t.Fatalf("RecordVersionCacheAccess(older): %v", err) + } + got, err = repos.Objects.GetVersionByID(ctx, version.VersionID) + if err != nil || got == nil { + t.Fatalf("GetVersionByID after older access: version=%v err=%v", got, err) + } + if got.InCache { + t.Fatal("in_cache = true after access-only update of an absent cache entry") + } + if got.CacheAccessedAt == nil || !got.CacheAccessedAt.Equal(committedAt) { + t.Fatalf("cache_accessed_at after older write = %v, want monotonic %v", got.CacheAccessedAt, committedAt) + } +} + +// mustExecRaw runs one setup statement that has no repository entry point. +func mustExecRaw(t *testing.T, db *bun.DB, query string, args ...any) { + t.Helper() + if _, err := db.NewRaw(query, args...).Exec(context.Background()); err != nil { + t.Fatalf("exec %q: %v", query, err) + } +} + func newObjectVersion(bucketID int64, key, versionID string, size int64) *model.ObjectVersion { return &model.ObjectVersion{ VersionID: versionID, @@ -23,10 +311,7 @@ func newObjectVersion(bucketID int64, key, versionID string, size int64) *model. Key: key, Size: size, ETag: "etag-" + versionID, - Checksum: "checksum-" + versionID, ContentType: "text/plain", - CacheKey: ".versions/" + versionID, - State: model.ObjectStateCached, } } @@ -37,12 +322,12 @@ func TestObjectRepo_CreateVersionAndSetCurrent_SecondUploadKeepsVersionHistory(t bucket := seedBucket(t, db, "version-bucket") v1 := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000000001", 10) - objectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, v1) + objectID, err := createVersion(t, repos, v1) if err != nil { t.Fatalf("first CreateVersionAndSetCurrent: %v", err) } v2 := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000000002", 20) - objectID2, err := repos.Objects.CreateVersionAndSetCurrent(ctx, v2) + objectID2, err := createVersion(t, repos, v2) if err != nil { t.Fatalf("second CreateVersionAndSetCurrent: %v", err) } @@ -57,8 +342,8 @@ func TestObjectRepo_CreateVersionAndSetCurrent_SecondUploadKeepsVersionHistory(t if current.VersionID != v2.VersionID { t.Fatalf("current version = %s, want %s", current.VersionID, v2.VersionID) } - if current.Size != 20 || current.ETag != v2.ETag || current.CacheKey != v2.CacheKey { - t.Fatalf("current version not refreshed: size=%d etag=%s cache=%s", current.Size, current.ETag, current.CacheKey) + if current.Size != 20 || current.ETag != v2.ETag || current.CacheKey() != v2.CacheKey() { + t.Fatalf("current version not refreshed: size=%d etag=%s cache=%s", current.Size, current.ETag, current.CacheKey()) } gotV1, err := repos.Objects.GetVersionByID(ctx, v1.VersionID) @@ -72,6 +357,9 @@ func TestObjectRepo_CreateVersionAndSetCurrent_SecondUploadKeepsVersionHistory(t if gotV1 == nil || gotV2 == nil { t.Fatal("expected both versions to remain queryable") } + if gotV1.Metadata == nil || gotV2.Metadata == nil || len(gotV1.Metadata) != 0 || len(gotV2.Metadata) != 0 { + t.Fatalf("nil metadata was not normalized: v1=%#v v2=%#v", gotV1.Metadata, gotV2.Metadata) + } if gotV1.ObjectID != objectID || gotV2.ObjectID != objectID { t.Fatalf("version object ids = %d/%d, want %d", gotV1.ObjectID, gotV2.ObjectID, objectID) } @@ -89,7 +377,7 @@ func TestObjectRepo_CreateRestoredVersionAndSetCurrent(t *testing.T) { source := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000R001", 10) source.Metadata = map[string]string{"source": "old"} - objectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, source) + objectID, err := createVersion(t, repos, source) if err != nil { t.Fatalf("create source: %v", err) } @@ -97,7 +385,7 @@ func TestObjectRepo_CreateRestoredVersionAndSetCurrent(t *testing.T) { current.ETag = source.ETag current.Checksum = source.Checksum current.Metadata = map[string]string{"source": "new"} - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, current); err != nil { + if _, err := createVersion(t, repos, current); err != nil { t.Fatalf("create current: %v", err) } @@ -105,7 +393,7 @@ func TestObjectRepo_CreateRestoredVersionAndSetCurrent(t *testing.T) { restored.ETag = source.ETag restored.Checksum = source.Checksum restored.Metadata = map[string]string{"source": "old"} - gotObjectID, err := repos.Objects.CreateRestoredVersionAndSetCurrent(ctx, restored, source.VersionID, current.VersionID) + gotObjectID, err := repos.Objects.CreateRestoredVersionAndSetCurrent(ctx, withContent(t, repos, restored), source.VersionID, current.VersionID) if err != nil { t.Fatalf("CreateRestoredVersionAndSetCurrent: %v", err) } @@ -146,7 +434,7 @@ func TestObjectRepo_CreateRestoredVersionAndSetCurrent(t *testing.T) { ctx := context.Background() bucket := seedBucket(t, db, "restore-missing-source-bucket") current := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000R011", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, current); err != nil { + if _, err := createVersion(t, repos, current); err != nil { t.Fatalf("create current: %v", err) } @@ -167,12 +455,12 @@ func TestObjectRepo_CreateRestoredVersionAndSetCurrent(t *testing.T) { ctx := context.Background() bucket := seedBucket(t, db, "restore-current-source-bucket") current := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000R015", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, current); err != nil { + if _, err := createVersion(t, repos, current); err != nil { t.Fatalf("create current: %v", err) } restored := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000R016", 10) - _, err := repos.Objects.CreateRestoredVersionAndSetCurrent(ctx, restored, current.VersionID, current.VersionID) + _, err := repos.Objects.CreateRestoredVersionAndSetCurrent(ctx, withContent(t, repos, restored), current.VersionID, current.VersionID) if !errors.Is(err, repository.ErrAlreadyCurrent) { t.Fatalf("error = %v, want ErrAlreadyCurrent", err) } @@ -186,22 +474,24 @@ func TestObjectRepo_CreateRestoredVersionAndSetCurrent(t *testing.T) { repos := repository.NewRepositories(db) ctx := context.Background() bucket := seedBucket(t, db, "restore-matching-source-bucket") + sharedContentID := seedContent(t, repos, bucket.ID, "shared-restore-checksum", 10) source := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000R017", 10) + source.ContentID = &sharedContentID source.Metadata = map[string]string{"content": "same"} - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, source); err != nil { + if _, err := createVersion(t, repos, source); err != nil { t.Fatalf("create source: %v", err) } current := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000R018", source.Size) current.ETag = "different-etag-for-the-same-content" - current.Checksum = source.Checksum + current.ContentID = &sharedContentID current.ContentType = source.ContentType current.Metadata = map[string]string{"content": "same"} - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, current); err != nil { + if _, err := createVersion(t, repos, current); err != nil { t.Fatalf("create current: %v", err) } restored := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000R019", source.Size) - _, err := repos.Objects.CreateRestoredVersionAndSetCurrent(ctx, restored, source.VersionID, current.VersionID) + _, err := repos.Objects.CreateRestoredVersionAndSetCurrent(ctx, withContent(t, repos, restored), source.VersionID, current.VersionID) if !errors.Is(err, repository.ErrAlreadyCurrent) { t.Fatalf("error = %v, want ErrAlreadyCurrent", err) } @@ -220,16 +510,16 @@ func TestObjectRepo_CreateRestoredVersionAndSetCurrent(t *testing.T) { ctx := context.Background() bucket := seedBucket(t, db, "restore-stale-token-bucket") source := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000R021", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, source); err != nil { + if _, err := createVersion(t, repos, source); err != nil { t.Fatalf("create source: %v", err) } current := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000R022", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, current); err != nil { + if _, err := createVersion(t, repos, current); err != nil { t.Fatalf("create current: %v", err) } restored := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000R023", 10) - _, err := repos.Objects.CreateRestoredVersionAndSetCurrent(ctx, restored, source.VersionID, source.VersionID) + _, err := repos.Objects.CreateRestoredVersionAndSetCurrent(ctx, withContent(t, repos, restored), source.VersionID, source.VersionID) if !errors.Is(err, repository.ErrConflict) { t.Fatalf("error = %v, want ErrConflict", err) } @@ -264,11 +554,11 @@ func TestObjectRepo_CreateRestoredVersionAndSetCurrent_ConcurrentTokenHasOneWinn repos := repository.NewRepositories(db) bucket := seedBucket(t, db, "concurrent-restore-bucket") source := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000R031", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, source); err != nil { + if _, err := createVersion(t, repos, source); err != nil { t.Fatalf("create source: %v", err) } current := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000R032", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, current); err != nil { + if _, err := createVersion(t, repos, current); err != nil { t.Fatalf("create current: %v", err) } @@ -281,7 +571,7 @@ func TestObjectRepo_CreateRestoredVersionAndSetCurrent_ConcurrentTokenHasOneWinn versionID := "01J0000000000000000000R04" + string(rune('0'+i)) _, err := repos.Objects.CreateRestoredVersionAndSetCurrent( ctx, - newObjectVersion(bucket.ID, "file.txt", versionID, source.Size), + withContent(t, repos, newObjectVersion(bucket.ID, "file.txt", versionID, source.Size)), source.VersionID, current.VersionID, ) @@ -315,167 +605,50 @@ func TestObjectRepo_CreateRestoredVersionAndSetCurrent_ConcurrentTokenHasOneWinn } } -func TestObjectRepo_CountOverviewAttention(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "overview-attention-bucket") - - healthy := newObjectVersion(bucket.ID, "healthy.txt", "01J00000000000000000000A01", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, healthy); err != nil { - t.Fatalf("seed healthy version: %v", err) - } - - failedState := newObjectVersion(bucket.ID, "failed-state.txt", "01J00000000000000000000A02", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, failedState); err != nil { - t.Fatalf("seed failed-state version: %v", err) - } - if err := repos.Objects.UpdateVersionStateToFailed(ctx, failedState.VersionID, model.ObjectStateCached, "upload failed"); err != nil { - t.Fatalf("mark failed-state version failed: %v", err) +func TestObjectRepo_CreateVersionAndSetCurrent_ConcurrentFirstUpload(t *testing.T) { + sqldb, err := sql.Open("sqlite", "file:"+filepath.Join(t.TempDir(), "objects.db")+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(1)") + if err != nil { + t.Fatalf("opening sqlite db: %v", err) } + sqldb.SetMaxOpenConns(8) + db := bun.NewDB(sqldb, sqlitedialect.New()) + t.Cleanup(func() { _ = db.Close() }) - failedUpload := newObjectVersion(bucket.ID, "failed-upload.txt", "01J00000000000000000000A03", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, failedUpload); err != nil { - t.Fatalf("seed failed-upload version: %v", err) + ctx := context.Background() + migrator := migrations.NewMigrator(db) + if err := migrator.Init(ctx); err != nil { + t.Fatalf("init migrator: %v", err) } - mustExec(t, db, `INSERT INTO storage_uploads (bucket_id, source_version_id, content_size, checksum, status, error_message, requested_copies) VALUES (?, ?, ?, ?, ?, ?, ?)`, - bucket.ID, failedUpload.VersionID, failedUpload.Size, failedUpload.Checksum, model.StorageUploadStatusFailed, "provider failed", 3) - - resolvedUpload := newObjectVersion(bucket.ID, "resolved-upload.txt", "01J00000000000000000000A06", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, resolvedUpload); err != nil { - t.Fatalf("seed resolved-upload version: %v", err) + if _, err := migrator.Migrate(ctx); err != nil { + t.Fatalf("running migrations: %v", err) } - mustExec(t, db, `INSERT INTO storage_uploads (bucket_id, source_version_id, content_size, checksum, status, requested_copies) VALUES (?, ?, ?, ?, ?, ?)`, - bucket.ID, resolvedUpload.VersionID, resolvedUpload.Size, resolvedUpload.Checksum, model.StorageUploadStatusFailed, 3) - mustExec(t, db, `INSERT INTO storage_uploads (bucket_id, source_version_id, content_size, checksum, status, requested_copies) VALUES (?, ?, ?, ?, ?, ?)`, - bucket.ID, resolvedUpload.VersionID, resolvedUpload.Size, resolvedUpload.Checksum, model.StorageUploadStatusComplete, 3) - rejectedLatestUpload := newObjectVersion(bucket.ID, "rejected-latest-upload.txt", "01J00000000000000000000A07", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, rejectedLatestUpload); err != nil { - t.Fatalf("seed rejected-latest-upload version: %v", err) - } - mustExec(t, db, `INSERT INTO storage_uploads (bucket_id, source_version_id, content_size, checksum, status, requested_copies) VALUES (?, ?, ?, ?, ?, ?)`, - bucket.ID, rejectedLatestUpload.VersionID, rejectedLatestUpload.Size, rejectedLatestUpload.Checksum, model.StorageUploadStatusComplete, 3) - mustExec(t, db, `INSERT INTO storage_uploads (bucket_id, source_version_id, content_size, checksum, status, requested_copies) VALUES (?, ?, ?, ?, ?, ?)`, - bucket.ID, rejectedLatestUpload.VersionID, rejectedLatestUpload.Size, rejectedLatestUpload.Checksum, model.StorageUploadStatusRejected, 3) + repos := repository.NewRepositories(db) + bucket := seedBucket(t, db, "concurrent-version-bucket") - boundUpload := newObjectVersion(bucket.ID, "bound-upload.txt", "01J00000000000000000000A08", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, boundUpload); err != nil { - t.Fatalf("seed bound-upload version: %v", err) + const writers = 8 + var wg sync.WaitGroup + errs := make(chan error, writers) + objectIDs := make(chan int64, writers) + for i := range writers { + wg.Add(1) + go func(i int) { + defer wg.Done() + versionID := "01J00000000000000000000C" + string(rune('A'+i)) + objectID, err := createVersion(t, repos, newObjectVersion(bucket.ID, "same-key.txt", versionID, int64(i+1))) + if err != nil { + errs <- err + return + } + objectIDs <- objectID + }(i) } - mustExec(t, db, `INSERT INTO storage_uploads (bucket_id, content_size, checksum, status, requested_copies) VALUES (?, ?, ?, ?, ?)`, - bucket.ID, boundUpload.Size, boundUpload.Checksum, model.StorageUploadStatusComplete, 3) - mustExec(t, db, `UPDATE object_versions SET state = ?, storage_upload_id = (SELECT MAX(id) FROM storage_uploads) WHERE version_id = ?`, - model.ObjectStateStored, boundUpload.VersionID) - mustExec(t, db, `INSERT INTO storage_uploads (bucket_id, source_version_id, content_size, checksum, status, requested_copies) VALUES (?, ?, ?, ?, ?, ?)`, - bucket.ID, boundUpload.VersionID, boundUpload.Size, boundUpload.Checksum, model.StorageUploadStatusFailed, 3) + wg.Wait() + close(errs) + close(objectIDs) - unavailable := newObjectVersion(bucket.ID, "unavailable.txt", "01J00000000000000000000A04", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, unavailable); err != nil { - t.Fatalf("seed unavailable version: %v", err) - } - mustExec(t, db, `INSERT INTO storage_uploads (bucket_id, source_version_id, content_size, checksum, status, requested_copies) VALUES (?, ?, ?, ?, ?, ?)`, - bucket.ID, unavailable.VersionID, unavailable.Size, unavailable.Checksum, model.StorageUploadStatusComplete, 3) - mustExec(t, db, `UPDATE object_versions SET state = ?, storage_upload_id = (SELECT MAX(id) FROM storage_uploads), in_cache = FALSE WHERE version_id = ?`, - model.ObjectStateStored, unavailable.VersionID) - - if _, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "deleted.txt", "01J00000000000000000000A05"); err != nil { - t.Fatalf("seed deleted marker: %v", err) - } - - counts, err := repos.Objects.CountOverviewAttention(ctx) - if err != nil { - t.Fatalf("CountOverviewAttention: %v", err) - } - if counts.NeedsAttention != 3 { - t.Fatalf("NeedsAttention = %d, want 3", counts.NeedsAttention) - } - if counts.Unavailable != 1 { - t.Fatalf("Unavailable = %d, want 1", counts.Unavailable) - } -} - -func TestObjectRepo_AggregateByStateIncludesTotalSize(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "state-aggregate-bucket") - - cached := newObjectVersion(bucket.ID, "cached.txt", "01J00000000000000000000B01", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, cached); err != nil { - t.Fatalf("seed cached version: %v", err) - } - failed := newObjectVersion(bucket.ID, "failed.txt", "01J00000000000000000000B02", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, failed); err != nil { - t.Fatalf("seed failed version: %v", err) - } - if err := repos.Objects.UpdateVersionStateToFailed(ctx, failed.VersionID, model.ObjectStateCached, "upload failed"); err != nil { - t.Fatalf("mark failed version: %v", err) - } - if _, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "deleted.txt", "01J00000000000000000000B03"); err != nil { - t.Fatalf("seed delete marker: %v", err) - } - - rows, err := repos.Objects.AggregateByState(ctx) - if err != nil { - t.Fatalf("AggregateByState: %v", err) - } - byState := make(map[string]repository.ObjectStateAggregate, len(rows)) - for _, row := range rows { - byState[row.State] = row - } - if got := byState[string(model.ObjectStateCached)]; got.Count != 1 || got.TotalSize != 10 { - t.Fatalf("cached aggregate = count:%d size:%d, want 1/10", got.Count, got.TotalSize) - } - if got := byState[string(model.ObjectStateFailed)]; got.Count != 1 || got.TotalSize != 20 { - t.Fatalf("failed aggregate = count:%d size:%d, want 1/20", got.Count, got.TotalSize) - } -} - -func TestObjectRepo_CreateVersionAndSetCurrent_ConcurrentFirstUpload(t *testing.T) { - sqldb, err := sql.Open("sqlite", "file:"+filepath.Join(t.TempDir(), "objects.db")+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(1)") - if err != nil { - t.Fatalf("opening sqlite db: %v", err) - } - sqldb.SetMaxOpenConns(8) - db := bun.NewDB(sqldb, sqlitedialect.New()) - t.Cleanup(func() { _ = db.Close() }) - - ctx := context.Background() - migrator := migrations.NewMigrator(db) - if err := migrator.Init(ctx); err != nil { - t.Fatalf("init migrator: %v", err) - } - if _, err := migrator.Migrate(ctx); err != nil { - t.Fatalf("running migrations: %v", err) - } - - repos := repository.NewRepositories(db) - bucket := seedBucket(t, db, "concurrent-version-bucket") - - const writers = 8 - var wg sync.WaitGroup - errs := make(chan error, writers) - objectIDs := make(chan int64, writers) - for i := range writers { - wg.Add(1) - go func(i int) { - defer wg.Done() - versionID := "01J00000000000000000000C" + string(rune('A'+i)) - objectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, newObjectVersion(bucket.ID, "same-key.txt", versionID, int64(i+1))) - if err != nil { - errs <- err - return - } - objectIDs <- objectID - }(i) - } - wg.Wait() - close(errs) - close(objectIDs) - - for err := range errs { - t.Fatalf("CreateVersionAndSetCurrent concurrent error: %v", err) + for err := range errs { + t.Fatalf("CreateVersionAndSetCurrent concurrent error: %v", err) } var objectID int64 @@ -522,6 +695,10 @@ func TestObjectRepo_CreateVersionAndSetCurrentIfChanged_ConcurrentIdenticalWrite repos := repository.NewRepositories(db) bucket := seedBucket(t, db, "concurrent-dedupe-bucket") + // Identical bytes are one content row, so every writer points at the same + // content the way the backend would after a single EnsureContent. + sharedContentID := seedContent(t, repos, bucket.ID, "same-key-checksum", 100) + const writers = 8 var wg sync.WaitGroup results := make(chan repository.ObjectVersionWriteResult, writers) @@ -533,7 +710,7 @@ func TestObjectRepo_CreateVersionAndSetCurrentIfChanged_ConcurrentIdenticalWrite versionID := "01J00000000000000000000D" + string(rune('A'+i)) version := newObjectVersion(bucket.ID, "same-key.txt", versionID, 100) version.ETag = "same-etag" - version.Checksum = "same-checksum" + version.ContentID = &sharedContentID result, err := repos.Objects.CreateVersionAndSetCurrentIfChanged(ctx, version) if err != nil { errs <- err @@ -595,12 +772,12 @@ func TestObjectRepo_ListByBucketReadsCurrentVersionOnly(t *testing.T) { {"dir/c.txt", 3}, } { v := newObjectVersion(bucket.ID, tc.key, "01J0000000000000000000000"+string(rune('3'+tc.size)), tc.size) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, v); err != nil { + if _, err := createVersion(t, repos, v); err != nil { t.Fatalf("CreateVersionAndSetCurrent(%s): %v", tc.key, err) } } latestB := newObjectVersion(bucket.ID, "b.txt", "01J00000000000000000000009", 22) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, latestB); err != nil { + if _, err := createVersion(t, repos, latestB); err != nil { t.Fatalf("second b.txt upload: %v", err) } @@ -655,7 +832,7 @@ func TestObjectRepo_ListByBucketReadsCurrentVersionOnly(t *testing.T) { {"under_/literal.txt", "01J00000000000000000002003"}, {"underX/literal.txt", "01J00000000000000000002004"}, } { - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, newObjectVersion(bucket.ID, tc.key, tc.versionID, 10)); err != nil { + if _, err := createVersion(t, repos, newObjectVersion(bucket.ID, tc.key, tc.versionID, 10)); err != nil { t.Fatalf("CreateVersionAndSetCurrent(%s): %v", tc.key, err) } } @@ -712,7 +889,7 @@ func TestObjectRepo_GetVersionByBucketKeyAndIDScopesVersion(t *testing.T) { bucket := seedBucket(t, db, "version-scope-bucket") version := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000000071", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + if _, err := createVersion(t, repos, version); err != nil { t.Fatalf("CreateVersionAndSetCurrent: %v", err) } @@ -733,1060 +910,324 @@ func TestObjectRepo_GetVersionByBucketKeyAndIDScopesVersion(t *testing.T) { } } -func TestObjectRepo_FindReusableStoredVersionRequiresStoredChainInfo(t *testing.T) { +func TestObjectRepo_ListVersionsByBucketOrdersAndMarksCurrent(t *testing.T) { db := testDB(t) repos := repository.NewRepositories(db) ctx := context.Background() - bucket := seedBucket(t, db, "reuse-version-bucket") + bucket := seedBucket(t, db, "version-list-bucket") - cached := newObjectVersion(bucket.ID, "cached.txt", "01J00000000000000000000072", 10) - cached.Checksum = "same-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, cached); err != nil { - t.Fatalf("create cached version: %v", err) + oldVersion := newObjectVersion(bucket.ID, "a.txt", "01J00000000000000000000074", 10) + if _, err := createVersion(t, repos, oldVersion); err != nil { + t.Fatalf("create old version: %v", err) } - - stored := newObjectVersion(bucket.ID, "stored.txt", "01J00000000000000000000073", 10) - stored.Checksum = "same-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, stored); err != nil { - t.Fatalf("create stored version: %v", err) + currentVersion := newObjectVersion(bucket.ID, "a.txt", "01J00000000000000000000075", 20) + if _, err := createVersion(t, repos, currentVersion); err != nil { + t.Fatalf("create current version: %v", err) } - if err := repos.Objects.UpdateVersionState(ctx, stored.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("stored uploading: %v", err) + if _, err := createVersion(t, repos, newObjectVersion(bucket.ID, "b.txt", "01J00000000000000000000076", 30)); err != nil { + t.Fatalf("create b version: %v", err) } - acceptTestStorageUploadForVersion(t, repos, bucket.ID, stored, "piece-reuse") - got, err := repos.Objects.FindReusableStoredVersion(ctx, bucket.ID, 10, "same-checksum") + rows, err := repos.Objects.ListVersionsByBucket(ctx, bucket.ID, "", "", "", 10) if err != nil { - t.Fatalf("FindReusableStoredVersion: %v", err) + t.Fatalf("ListVersionsByBucket: %v", err) } - if got == nil || got.VersionID != stored.VersionID { - t.Fatalf("reusable version = %#v, want %s", got, stored.VersionID) + if len(rows) != 3 { + t.Fatalf("rows len = %d, want 3", len(rows)) } -} - -func TestObjectRepo_SetVersionStorageUploadAndTransitionUsesNewUpload(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "reuse-storage-upload-bucket") - - stored := newObjectVersion(bucket.ID, "stored.txt", "01J0000000000000000000007G", 10) - stored.Checksum = "same-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, stored); err != nil { - t.Fatalf("create stored version: %v", err) + if rows[0].Key != "a.txt" || rows[0].VersionID != currentVersion.VersionID { + t.Fatalf("first row = %s/%s, want current a.txt", rows[0].Key, rows[0].VersionID) } - if err := repos.Objects.UpdateVersionState(ctx, stored.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("stored uploading: %v", err) + if rows[0].VersionID != currentVersion.VersionID { + t.Fatalf("current marker = %s, want %s", rows[0].VersionID, currentVersion.VersionID) } - uploadID := acceptTestStorageUploadForVersion(t, repos, bucket.ID, stored, "piece-reuse") - follower := newObjectVersion(bucket.ID, "follower.txt", "01J0000000000000000000007H", 10) - follower.Checksum = "same-checksum" - follower.State = model.ObjectStateUploading - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, follower); err != nil { - t.Fatalf("create follower version: %v", err) + page, err := repos.Objects.ListVersionsByBucket(ctx, bucket.ID, "", rows[0].Key, rows[0].VersionID, 10) + if err != nil { + t.Fatalf("ListVersionsByBucket marker: %v", err) + } + if len(page) == 0 || page[0].VersionID != oldVersion.VersionID { + t.Fatalf("marker page first = %#v, want old version", page) } - if err := repos.Objects.SetVersionStorageUploadAndTransition(ctx, follower.VersionID, uploadID, model.ObjectStateUploading, model.ObjectStateStored); err != nil { - t.Fatalf("SetVersionStorageUploadAndTransition: %v", err) + if _, err := createVersion(t, repos, newObjectVersion(bucket.ID, "case/a.txt", "01J00000000000000000000077", 40)); err != nil { + t.Fatalf("create case version: %v", err) } - got, err := repos.Objects.GetVersionByID(ctx, follower.VersionID) - if err != nil || got == nil { - t.Fatalf("get follower: got=%v err=%v", got, err) + if _, err := createVersion(t, repos, newObjectVersion(bucket.ID, "Case/a.txt", "01J00000000000000000000078", 50)); err != nil { + t.Fatalf("create Case version: %v", err) + } + prefixed, err := repos.Objects.ListVersionsByBucket(ctx, bucket.ID, "case/", "", "", 10) + if err != nil { + t.Fatalf("ListVersionsByBucket prefix: %v", err) } - if got.StorageUploadID == nil || *got.StorageUploadID != uploadID || got.State != model.ObjectStateStored || !got.InFilecoin { - t.Fatalf("follower storage = state:%s upload:%v filecoin:%v, want stored with upload %d", got.State, got.StorageUploadID, got.InFilecoin, uploadID) + if len(prefixed) != 1 || prefixed[0].Key != "case/a.txt" { + t.Fatalf("case-sensitive prefix rows = %#v", prefixed) } } -func TestObjectRepo_NewStoredReferenceRechecksCurrentMinimumDurability(t *testing.T) { +func TestObjectRepo_ListVersionsByKey(t *testing.T) { db := testDB(t) repos := repository.NewRepositories(db) ctx := context.Background() - bucket := seedBucket(t, db, "reuse-current-minimum-bucket") - minimumOne := 1 - if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimumOne, - }); err != nil { - t.Fatalf("set minimum one: %v", err) - } - - source := newObjectVersion(bucket.ID, "source.txt", model.NewVersionID(), 10) - source.Checksum = "reuse-current-minimum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, source); err != nil { - t.Fatalf("create source: %v", err) + bucket := seedBucket(t, db, "version-list-key-bucket") + otherBucket := seedBucket(t, db, "version-list-key-other-bucket") + + oldVersion := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000002001", 10) + if _, err := createVersion(t, repos, oldVersion); err != nil { + t.Fatalf("create old version: %v", err) + } + middleVersion := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000002002", 20) + if _, err := createVersion(t, repos, middleVersion); err != nil { + t.Fatalf("create middle version: %v", err) + } + currentVersion := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000002003", 30) + if _, err := createVersion(t, repos, currentVersion); err != nil { + t.Fatalf("create current version: %v", err) + } + if _, err := createVersion(t, repos, newObjectVersion(bucket.ID, "other.txt", "01J00000000000000000002004", 40)); err != nil { + t.Fatalf("create other key version: %v", err) } - upload := startCopyHealthUpload(t, repos, bucket.ID, source.VersionID, source.Size, source.Checksum, 2) - commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 0, "101", "1001", "2001", "https://one.example/piece") - bindStorageHealthVersion(t, repos, bucket.ID, upload.ID, source) - if complete, _, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}); err != nil || complete { - t.Fatalf("FinalizeUploadIfTargetCopiesMet = complete:%t err:%v, want stored before target", complete, err) + if _, err := createVersion(t, repos, newObjectVersion(otherBucket.ID, "file.txt", "01J00000000000000000002005", 50)); err != nil { + t.Fatalf("create other bucket version: %v", err) } - reusable, err := repos.Objects.FindReusableStoredVersion(ctx, bucket.ID, source.Size, source.Checksum) + + rows, err := repos.Objects.ListVersionsByKey(ctx, bucket.ID, "file.txt", "", 10) if err != nil { - t.Fatalf("FindReusableStoredVersion before raising minimum: %v", err) + t.Fatalf("ListVersionsByKey: %v", err) + } + if len(rows) != 3 { + t.Fatalf("rows len = %d, want 3", len(rows)) + } + for i, row := range rows { + if row.BucketID != bucket.ID || row.Key != "file.txt" { + t.Fatalf("row %d = bucket:%d key:%q, want target bucket/file.txt", i, row.BucketID, row.Key) + } + } + if rows[0].VersionID != currentVersion.VersionID || !rows[0].IsCurrent { + t.Fatalf("first row = %#v, want current version %s", rows[0], currentVersion.VersionID) } - if reusable == nil || reusable.VersionID != source.VersionID { - t.Fatalf("reusable readable upload version = %#v, want %s", reusable, source.VersionID) + if rows[1].VersionID != middleVersion.VersionID || rows[2].VersionID != oldVersion.VersionID { + t.Fatalf("ordered rows = %s/%s/%s, want current/middle/old", rows[0].VersionID, rows[1].VersionID, rows[2].VersionID) } - minimumTwo := 2 - if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimumTwo, - }); err != nil { - t.Fatalf("raise minimum: %v", err) + page, err := repos.Objects.ListVersionsByKey(ctx, bucket.ID, "file.txt", currentVersion.VersionID, 10) + if err != nil { + t.Fatalf("ListVersionsByKey marker: %v", err) } - follower := newObjectVersion(bucket.ID, "follower.txt", model.NewVersionID(), source.Size) - follower.Checksum = source.Checksum - follower.State = model.ObjectStateStored - follower.InCache = true - follower.StorageUploadID = &upload.ID - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, follower); err != nil { - t.Fatalf("create follower: %v", err) + if len(page) != 2 || page[0].VersionID != middleVersion.VersionID || page[1].VersionID != oldVersion.VersionID { + t.Fatalf("marker page = %#v, want middle then old", page) } - got, err := repos.Objects.GetVersionByID(ctx, follower.VersionID) - if err != nil || got == nil { - t.Fatalf("get follower: version=%#v err=%v", got, err) + limited, err := repos.Objects.ListVersionsByKey(ctx, bucket.ID, "file.txt", "", 2) + if err != nil { + t.Fatalf("ListVersionsByKey limit: %v", err) } - if got.State != model.ObjectStateCached || got.StorageUploadID != nil || !got.InCache { - t.Fatalf("follower below current minimum = %#v, want retained cache without reused upload", got) + if len(limited) != 2 || limited[0].VersionID != currentVersion.VersionID || limited[1].VersionID != middleVersion.VersionID { + t.Fatalf("limited rows = %#v, want current then middle", limited) } } -func TestObjectRepo_FindReusableActiveUploadVersionRequiresActiveTask(t *testing.T) { +func TestObjectRepo_DeleteMarkerVersionRestoresPreviousCurrentVersion(t *testing.T) { db := testDB(t) repos := repository.NewRepositories(db) ctx := context.Background() - bucket := seedBucket(t, db, "active-reuse-bucket") + bucket := seedBucket(t, db, "delete-marker-restore-bucket") - version := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000007A", 10) - version.Checksum = "same-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("create version: %v", err) + first := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000001011", 10) + second := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000001012", 20) + if _, err := createVersion(t, repos, first); err != nil { + t.Fatalf("create first version: %v", err) } - - got, err := repos.Objects.FindReusableActiveUploadVersion(ctx, bucket.ID, 10, "same-checksum") - if err != nil { - t.Fatalf("FindReusableActiveUploadVersion without task: %v", err) + if _, err := createVersion(t, repos, second); err != nil { + t.Fatalf("create second version: %v", err) } - if got != nil { - t.Fatalf("active reusable without task = %#v, want nil", got) + marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "file.txt", "01J00000000000000000001013") + if err != nil { + t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) } - task := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "object", - RefID: version.ObjectID, - RefVersionID: version.VersionID, - IdempotencyKey: "upload:" + version.VersionID, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("create upload task: %v", err) + if err := repos.Objects.DeleteMarkerVersion(ctx, bucket.ID, "file.txt", marker.VersionID); err != nil { + t.Fatalf("DeleteMarkerVersion: %v", err) } - got, err = repos.Objects.FindReusableActiveUploadVersion(ctx, bucket.ID, 10, "same-checksum") + current, err := repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "file.txt") if err != nil { - t.Fatalf("FindReusableActiveUploadVersion: %v", err) - } - if got == nil || got.VersionID != version.VersionID { - t.Fatalf("active reusable = %#v, want %s", got, version.VersionID) + t.Fatalf("GetCurrentVersionByBucketAndKey: %v", err) } - - if err := repos.Objects.SetVersionCachePresence(ctx, version.VersionID, false); err != nil { - t.Fatalf("SetVersionCachePresence: %v", err) + if current == nil || current.VersionID != second.VersionID || current.IsDeleteMarker { + t.Fatalf("current = %#v, want restored data version %s", current, second.VersionID) } - got, err = repos.Objects.FindReusableActiveUploadVersion(ctx, bucket.ID, 10, "same-checksum") + deletedMarker, err := repos.Objects.GetVersionByID(ctx, marker.VersionID) if err != nil { - t.Fatalf("FindReusableActiveUploadVersion with missing cache: %v", err) + t.Fatalf("GetVersionByID(marker): %v", err) } - if got != nil { - t.Fatalf("active reusable with missing cache = %#v, want nil", got) + if deletedMarker != nil { + t.Fatalf("deleted marker still exists: %#v", deletedMarker) } } -func TestObjectRepo_FindReusableActiveUploadVersionUsesDurableUploadLeader(t *testing.T) { +func TestObjectRepo_DeleteOnlyMarkerVersionDeletesObjectIdentity(t *testing.T) { db := testDB(t) repos := repository.NewRepositories(db) ctx := context.Background() - bucket := seedBucket(t, db, "durable-active-reuse-bucket") - - version := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000007B", 10) - version.Checksum = "same-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("create version: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark uploading: %v", err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } + bucket := seedBucket(t, db, "delete-only-marker-bucket") - got, err := repos.Objects.FindReusableActiveUploadVersion(ctx, bucket.ID, 10, "same-checksum") + marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "deleted.txt", "01J00000000000000000001014") if err != nil { - t.Fatalf("FindReusableActiveUploadVersion(uploading): %v", err) - } - if got == nil || got.VersionID != version.VersionID { - t.Fatalf("uploading durable leader = %#v, want %s", got, version.VersionID) + t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) } - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("mark committing: %v", err) + if err := repos.Objects.DeleteMarkerVersion(ctx, bucket.ID, "deleted.txt", marker.VersionID); err != nil { + t.Fatalf("DeleteMarkerVersion: %v", err) } - got, err = repos.Objects.FindReusableActiveUploadVersion(ctx, bucket.ID, 10, "same-checksum") + + current, err := repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "deleted.txt") if err != nil { - t.Fatalf("FindReusableActiveUploadVersion(committing without readable copy): %v", err) + t.Fatalf("GetCurrentVersionByBucketAndKey: %v", err) } - if got != nil { - t.Fatalf("committing leader without readable copy = %#v, want nil", got) + if current != nil { + t.Fatalf("current = %#v, want none after deleting only marker", current) } - - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) + objectCount, err := db.NewSelect(). + Model((*model.Object)(nil)). + Where("bucket_id = ? AND key = ?", bucket.ID, "deleted.txt"). + Count(ctx) if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) + t.Fatalf("count object identities: %v", err) } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) + if objectCount != 0 { + t.Fatalf("object identity count = %d, want 0", objectCount) } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001")}); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) +} + +func TestObjectRepo_RestoreCurrentDeleteMarkerStackRemovesMarkersUntilDataVersion(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := seedBucket(t, db, "delete-marker-stack-bucket") + + data := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000001021", 10) + if _, err := createVersion(t, repos, data); err != nil { + t.Fatalf("create data version: %v", err) } - if err := repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "bafk2bzacefake", - RetrievalURL: "https://provider.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) + if _, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "file.txt", "01J00000000000000000001022"); err != nil { + t.Fatalf("create first marker: %v", err) } - got, err = repos.Objects.FindReusableActiveUploadVersion(ctx, bucket.ID, 10, "same-checksum") + currentMarker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "file.txt", "01J00000000000000000001023") if err != nil { - t.Fatalf("FindReusableActiveUploadVersion(committing with uncommitted copy): %v", err) + t.Fatalf("create second marker: %v", err) } - if got != nil { - t.Fatalf("committing leader with uncommitted copy = %#v, want nil", got) + + restored, err := repos.Objects.RestoreCurrentDeleteMarkerStack(ctx, bucket.ID, "file.txt", currentMarker.VersionID) + if err != nil { + t.Fatalf("RestoreCurrentDeleteMarkerStack: %v", err) } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "bafk2bzacefake", - PieceID: onChainIDPtr(t, "2001"), - RetrievalURL: "https://provider.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) + if restored.VersionID != data.VersionID || restored.IsDeleteMarker { + t.Fatalf("restored = %#v, want data version %s", restored, data.VersionID) } - got, err = repos.Objects.FindReusableActiveUploadVersion(ctx, bucket.ID, 10, "same-checksum") + versions, err := repos.Objects.ListVersionsByBucket(ctx, bucket.ID, "", "", "", 10) if err != nil { - t.Fatalf("FindReusableActiveUploadVersion(committing with readable copy): %v", err) + t.Fatalf("ListVersionsByBucket: %v", err) } - if got == nil || got.VersionID != version.VersionID { - t.Fatalf("committing durable leader = %#v, want %s", got, version.VersionID) + if len(versions) != 1 || versions[0].VersionID != data.VersionID || !versions[0].IsCurrent { + t.Fatalf("versions after restore = %#v, want only current data version", versions) } } -func TestObjectRepo_AcceptStorageUploadForContentUpdatesMatchingVersions(t *testing.T) { +func TestObjectRepo_DeleteMarkerStatsAndRecoverableListIgnoreUnrestorableMarkers(t *testing.T) { db := testDB(t) repos := repository.NewRepositories(db) ctx := context.Background() - bucket := seedBucket(t, db, "content-storage-bucket") + bucket := seedBucket(t, db, "delete-marker-stats-bucket") - oldVersion := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000007B", 10) - oldVersion.Checksum = "same-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion); err != nil { - t.Fatalf("old version: %v", err) - } - currentVersion := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000007C", 10) - currentVersion.Checksum = "same-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, currentVersion); err != nil { - t.Fatalf("current version: %v", err) - } - for _, versionID := range []string{oldVersion.VersionID, currentVersion.VersionID} { - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark %s uploading: %v", versionID, err) - } - } - - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: currentVersion.VersionID, - ContentSize: currentVersion.Size, - Checksum: currentVersion.Checksum, - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - seedCommittedUploadCopies(t, repos, bucket.ID, upload.ID, "piece-shared", []storageUploadCopySeed{ - {ProviderID: onChainIDPtr(t, "101"), DataSetID: onChainIDPtr(t, "1001001"), PieceID: onChainIDPtr(t, "2001"), TransferMethod: model.StorageCopyTransferMethodIngress, RetrievalURL: strPtr("https://provider.example/shared")}, - }) - refs := bindReadableUploadForContent(t, repos, upload.ID, bucket.ID, currentVersion.Size, currentVersion.Checksum) - finalizeUploadForTest(t, repos, upload.ID) - if len(refs) != 2 { - t.Fatalf("updated refs len = %d, want 2", len(refs)) - } - - for _, versionID := range []string{oldVersion.VersionID, currentVersion.VersionID} { - got, err := repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || got == nil { - t.Fatalf("version %s: got=%v err=%v", versionID, got, err) - } - if got.State != model.ObjectStateStored { - t.Fatalf("version %s state = %s, want stored", versionID, got.State) - } - if got.PieceCID == nil || *got.PieceCID != "piece-shared" { - t.Fatalf("version %s piece = %v, want piece-shared", versionID, got.PieceCID) - } - if !got.InFilecoin { - t.Fatalf("version %s in_filecoin = false, want true", versionID) - } - } - - current, err := repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "file.txt") - if err != nil { - t.Fatalf("GetByBucketAndKey: %v", err) - } - if current.VersionID != currentVersion.VersionID || current.State != model.ObjectStateStored { - t.Fatalf("current version = version:%s state:%s, want %s stored", current.VersionID, current.State, currentVersion.VersionID) - } - if current.PieceCID == nil || *current.PieceCID != "piece-shared" { - t.Fatalf("current piece = %v, want piece-shared", current.PieceCID) - } - if !current.InFilecoin { - t.Fatal("current in_filecoin = false, want true") - } -} - -func TestObjectRepo_FailUploadingContentFollowersKeepsIndependentActiveUpload(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "content-failure-bucket") - - leader := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000007D", 10) - leader.Checksum = "same-checksum" - objID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, leader) - if err != nil { - t.Fatalf("leader version: %v", err) - } - follower := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000007E", 10) - follower.Checksum = "same-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, follower); err != nil { - t.Fatalf("follower version: %v", err) - } - committingFollower := newObjectVersion(bucket.ID, "commit.txt", "01J0000000000000000000007G", 10) - committingFollower.Checksum = "same-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, committingFollower); err != nil { - t.Fatalf("committing follower version: %v", err) - } - independent := newObjectVersion(bucket.ID, "other.txt", "01J0000000000000000000007F", 10) - independent.Checksum = "same-checksum" - independentObjID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, independent) - if err != nil { - t.Fatalf("independent version: %v", err) - } - independentUpload := newObjectVersion(bucket.ID, "other-active.txt", "01J0000000000000000000007H", 10) - independentUpload.Checksum = "same-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, independentUpload); err != nil { - t.Fatalf("independent upload version: %v", err) - } - for _, versionID := range []string{leader.VersionID, follower.VersionID, committingFollower.VersionID, independent.VersionID, independentUpload.VersionID} { - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark %s uploading: %v", versionID, err) - } - } - if err := repos.Objects.UpdateVersionState(ctx, committingFollower.VersionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("mark committing follower committing: %v", err) - } - if _, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: independentUpload.VersionID, - ContentSize: independentUpload.Size, - Checksum: independentUpload.Checksum, - RequestedCopies: 3, - }); err != nil { - t.Fatalf("start independent active upload: %v", err) - } - - for _, task := range []*model.Task{ - { - Type: model.TaskTypeUpload, - RefType: "object", - RefID: objID, - RefVersionID: leader.VersionID, - IdempotencyKey: "upload:" + leader.VersionID, - Status: model.TaskStatusRunning, - MaxRetries: 1, - ScheduledAt: time.Now(), - }, - { - Type: model.TaskTypeUpload, - RefType: "object", - RefID: independentObjID, - RefVersionID: independent.VersionID, - IdempotencyKey: "upload:" + independent.VersionID, - Status: model.TaskStatusQueued, - MaxRetries: 1, - ScheduledAt: time.Now(), - }, - } { - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("create upload task for %s: %v", task.RefVersionID, err) - } - } - - refs, err := repos.Objects.FailUploadingContentFollowers(ctx, bucket.ID, 10, "same-checksum", leader.VersionID, "upload failed") - if err != nil { - t.Fatalf("FailUploadingContentFollowers: %v", err) - } - if len(refs) != 3 { - t.Fatalf("failed refs len = %d, want 3", len(refs)) - } - - wantFailedAtState := map[string]model.ObjectState{ - leader.VersionID: model.ObjectStateUploading, - follower.VersionID: model.ObjectStateUploading, - committingFollower.VersionID: model.ObjectStateCommitting, - } - for versionID, failedAtState := range wantFailedAtState { - got, err := repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || got == nil { - t.Fatalf("version %s: got=%v err=%v", versionID, got, err) - } - if got.State != model.ObjectStateFailed { - t.Fatalf("version %s state = %s, want failed", versionID, got.State) - } - if got.FailedAtState == nil || *got.FailedAtState != failedAtState { - t.Fatalf("version %s failed_at_state = %#v, want %s", versionID, got.FailedAtState, failedAtState) - } - if got.LastError == nil || *got.LastError != "upload failed" { - t.Fatalf("version %s last_error = %#v, want upload failed", versionID, got.LastError) - } - } - gotIndependent, err := repos.Objects.GetVersionByID(ctx, independent.VersionID) - if err != nil || gotIndependent == nil { - t.Fatalf("independent version: got=%v err=%v", gotIndependent, err) - } - if gotIndependent.State != model.ObjectStateUploading { - t.Fatalf("independent state = %s, want uploading", gotIndependent.State) - } - gotIndependentUpload, err := repos.Objects.GetVersionByID(ctx, independentUpload.VersionID) - if err != nil || gotIndependentUpload == nil { - t.Fatalf("independent upload version: got=%v err=%v", gotIndependentUpload, err) - } - if gotIndependentUpload.State != model.ObjectStateUploading { - t.Fatalf("independent upload state = %s, want uploading", gotIndependentUpload.State) - } -} - -func TestObjectRepo_ListVersionsByBucketOrdersAndMarksCurrent(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "version-list-bucket") - - oldVersion := newObjectVersion(bucket.ID, "a.txt", "01J00000000000000000000074", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion); err != nil { - t.Fatalf("create old version: %v", err) - } - currentVersion := newObjectVersion(bucket.ID, "a.txt", "01J00000000000000000000075", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, currentVersion); err != nil { - t.Fatalf("create current version: %v", err) - } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, newObjectVersion(bucket.ID, "b.txt", "01J00000000000000000000076", 30)); err != nil { - t.Fatalf("create b version: %v", err) - } - - rows, err := repos.Objects.ListVersionsByBucket(ctx, bucket.ID, "", "", "", 10) - if err != nil { - t.Fatalf("ListVersionsByBucket: %v", err) - } - if len(rows) != 3 { - t.Fatalf("rows len = %d, want 3", len(rows)) - } - if rows[0].Key != "a.txt" || rows[0].VersionID != currentVersion.VersionID { - t.Fatalf("first row = %s/%s, want current a.txt", rows[0].Key, rows[0].VersionID) - } - if rows[0].VersionID != currentVersion.VersionID { - t.Fatalf("current marker = %s, want %s", rows[0].VersionID, currentVersion.VersionID) - } - - page, err := repos.Objects.ListVersionsByBucket(ctx, bucket.ID, "", rows[0].Key, rows[0].VersionID, 10) - if err != nil { - t.Fatalf("ListVersionsByBucket marker: %v", err) - } - if len(page) == 0 || page[0].VersionID != oldVersion.VersionID { - t.Fatalf("marker page first = %#v, want old version", page) - } - - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, newObjectVersion(bucket.ID, "case/a.txt", "01J00000000000000000000077", 40)); err != nil { - t.Fatalf("create case version: %v", err) - } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, newObjectVersion(bucket.ID, "Case/a.txt", "01J00000000000000000000078", 50)); err != nil { - t.Fatalf("create Case version: %v", err) - } - prefixed, err := repos.Objects.ListVersionsByBucket(ctx, bucket.ID, "case/", "", "", 10) - if err != nil { - t.Fatalf("ListVersionsByBucket prefix: %v", err) - } - if len(prefixed) != 1 || prefixed[0].Key != "case/a.txt" { - t.Fatalf("case-sensitive prefix rows = %#v", prefixed) - } -} - -func TestObjectRepo_ListVersionsByKey(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "version-list-key-bucket") - otherBucket := seedBucket(t, db, "version-list-key-other-bucket") - - oldVersion := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000002001", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion); err != nil { - t.Fatalf("create old version: %v", err) - } - middleVersion := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000002002", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, middleVersion); err != nil { - t.Fatalf("create middle version: %v", err) - } - currentVersion := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000002003", 30) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, currentVersion); err != nil { - t.Fatalf("create current version: %v", err) - } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, newObjectVersion(bucket.ID, "other.txt", "01J00000000000000000002004", 40)); err != nil { - t.Fatalf("create other key version: %v", err) - } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, newObjectVersion(otherBucket.ID, "file.txt", "01J00000000000000000002005", 50)); err != nil { - t.Fatalf("create other bucket version: %v", err) - } - - rows, err := repos.Objects.ListVersionsByKey(ctx, bucket.ID, "file.txt", "", 10) - if err != nil { - t.Fatalf("ListVersionsByKey: %v", err) - } - if len(rows) != 3 { - t.Fatalf("rows len = %d, want 3", len(rows)) - } - for i, row := range rows { - if row.BucketID != bucket.ID || row.Key != "file.txt" { - t.Fatalf("row %d = bucket:%d key:%q, want target bucket/file.txt", i, row.BucketID, row.Key) - } - } - if rows[0].VersionID != currentVersion.VersionID || !rows[0].IsCurrent { - t.Fatalf("first row = %#v, want current version %s", rows[0], currentVersion.VersionID) - } - if rows[1].VersionID != middleVersion.VersionID || rows[2].VersionID != oldVersion.VersionID { - t.Fatalf("ordered rows = %s/%s/%s, want current/middle/old", rows[0].VersionID, rows[1].VersionID, rows[2].VersionID) - } - - page, err := repos.Objects.ListVersionsByKey(ctx, bucket.ID, "file.txt", currentVersion.VersionID, 10) - if err != nil { - t.Fatalf("ListVersionsByKey marker: %v", err) - } - if len(page) != 2 || page[0].VersionID != middleVersion.VersionID || page[1].VersionID != oldVersion.VersionID { - t.Fatalf("marker page = %#v, want middle then old", page) - } - - limited, err := repos.Objects.ListVersionsByKey(ctx, bucket.ID, "file.txt", "", 2) - if err != nil { - t.Fatalf("ListVersionsByKey limit: %v", err) - } - if len(limited) != 2 || limited[0].VersionID != currentVersion.VersionID || limited[1].VersionID != middleVersion.VersionID { - t.Fatalf("limited rows = %#v, want current then middle", limited) - } -} - -func TestObjectRepo_CreateDeleteMarkerHidesCurrentObjectButKeepsVersionHistory(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "delete-marker-bucket") - - data := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000001001", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, data); err != nil { - t.Fatalf("create data version: %v", err) - } - marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "file.txt", "01J00000000000000000001002") - if err != nil { - t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) - } - if !marker.IsDeleteMarker || marker.Size != 0 || marker.CacheKey != "" || marker.InCache { - t.Fatalf("marker = %#v, want metadata-only delete marker", marker) - } - - current, err := repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "file.txt") - if err != nil { - t.Fatalf("GetCurrentVersionByBucketAndKey: %v", err) - } - if current == nil || !current.IsDeleteMarker || current.VersionID != marker.VersionID { - t.Fatalf("current = %#v, want delete marker %s", current, marker.VersionID) - } - - currentList, err := repos.Objects.ListCurrentVersionsByBucket(ctx, bucket.ID, "", "", 10) - if err != nil { - t.Fatalf("ListCurrentVersionsByBucket: %v", err) - } - if len(currentList) != 0 { - t.Fatalf("current list len = %d, want object hidden", len(currentList)) - } - - versions, err := repos.Objects.ListVersionsByBucket(ctx, bucket.ID, "", "", "", 10) - if err != nil { - t.Fatalf("ListVersionsByBucket: %v", err) - } - if len(versions) != 2 { - t.Fatalf("versions len = %d, want marker plus data version", len(versions)) - } - if !versions[0].IsDeleteMarker || !versions[0].IsCurrent { - t.Fatalf("first version = %#v, want current delete marker", versions[0].ObjectVersion) - } - if versions[1].VersionID != data.VersionID || versions[1].IsDeleteMarker { - t.Fatalf("second version = %#v, want data version %s", versions[1].ObjectVersion, data.VersionID) - } - - cached, err := repos.Objects.ListVersionsByState(ctx, model.ObjectStateCached, 10) - if err != nil { - t.Fatalf("ListVersionsByState: %v", err) - } - if len(cached) != 1 || cached[0].VersionID != data.VersionID { - t.Fatalf("cached versions = %#v, want only data version", cached) - } -} - -func TestObjectRepo_DeleteMarkerVersionRestoresPreviousCurrentVersion(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "delete-marker-restore-bucket") - - first := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000001011", 10) - second := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000001012", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, first); err != nil { - t.Fatalf("create first version: %v", err) - } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, second); err != nil { - t.Fatalf("create second version: %v", err) - } - marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "file.txt", "01J00000000000000000001013") - if err != nil { - t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) - } - - if err := repos.Objects.DeleteMarkerVersion(ctx, bucket.ID, "file.txt", marker.VersionID); err != nil { - t.Fatalf("DeleteMarkerVersion: %v", err) - } - - current, err := repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "file.txt") - if err != nil { - t.Fatalf("GetCurrentVersionByBucketAndKey: %v", err) - } - if current == nil || current.VersionID != second.VersionID || current.IsDeleteMarker { - t.Fatalf("current = %#v, want restored data version %s", current, second.VersionID) - } - deletedMarker, err := repos.Objects.GetVersionByID(ctx, marker.VersionID) - if err != nil { - t.Fatalf("GetVersionByID(marker): %v", err) - } - if deletedMarker != nil { - t.Fatalf("deleted marker still exists: %#v", deletedMarker) - } -} - -func TestObjectRepo_DeleteOnlyMarkerVersionDeletesObjectIdentity(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "delete-only-marker-bucket") - - marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "deleted.txt", "01J00000000000000000001014") - if err != nil { - t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) - } - - if err := repos.Objects.DeleteMarkerVersion(ctx, bucket.ID, "deleted.txt", marker.VersionID); err != nil { - t.Fatalf("DeleteMarkerVersion: %v", err) - } - - current, err := repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "deleted.txt") - if err != nil { - t.Fatalf("GetCurrentVersionByBucketAndKey: %v", err) - } - if current != nil { - t.Fatalf("current = %#v, want none after deleting only marker", current) - } - objectCount, err := db.NewSelect(). - Model((*model.Object)(nil)). - Where("bucket_id = ? AND key = ?", bucket.ID, "deleted.txt"). - Count(ctx) - if err != nil { - t.Fatalf("count object identities: %v", err) - } - if objectCount != 0 { - t.Fatalf("object identity count = %d, want 0", objectCount) - } -} - -func TestObjectRepo_RestoreCurrentDeleteMarkerStackRemovesMarkersUntilDataVersion(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "delete-marker-stack-bucket") - - data := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000001021", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, data); err != nil { - t.Fatalf("create data version: %v", err) - } - if _, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "file.txt", "01J00000000000000000001022"); err != nil { - t.Fatalf("create first marker: %v", err) - } - currentMarker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "file.txt", "01J00000000000000000001023") - if err != nil { - t.Fatalf("create second marker: %v", err) - } - - restored, err := repos.Objects.RestoreCurrentDeleteMarkerStack(ctx, bucket.ID, "file.txt", currentMarker.VersionID) - if err != nil { - t.Fatalf("RestoreCurrentDeleteMarkerStack: %v", err) - } - if restored.VersionID != data.VersionID || restored.IsDeleteMarker { - t.Fatalf("restored = %#v, want data version %s", restored, data.VersionID) - } - - versions, err := repos.Objects.ListVersionsByBucket(ctx, bucket.ID, "", "", "", 10) - if err != nil { - t.Fatalf("ListVersionsByBucket: %v", err) - } - if len(versions) != 1 || versions[0].VersionID != data.VersionID || !versions[0].IsCurrent { - t.Fatalf("versions after restore = %#v, want only current data version", versions) - } -} - -func TestObjectRepo_DeleteMarkerStatsAndRecoverableListIgnoreUnrestorableMarkers(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "delete-marker-stats-bucket") - - data := newObjectVersion(bucket.ID, "restorable.txt", "01J00000000000000000001031", 25) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, data); err != nil { - t.Fatalf("create data version: %v", err) + data := newObjectVersion(bucket.ID, "restorable.txt", "01J00000000000000000001031", 25) + if _, err := createVersion(t, repos, data); err != nil { + t.Fatalf("create data version: %v", err) } marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "restorable.txt", "01J00000000000000000001032") if err != nil { t.Fatalf("create restorable marker: %v", err) } - if _, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "missing.txt", "01J00000000000000000001033"); err != nil { - t.Fatalf("create unrestorable marker: %v", err) - } - - count, err := repos.Objects.CountByBucket(ctx, bucket.ID) - if err != nil { - t.Fatalf("CountByBucket: %v", err) - } - if count != 0 { - t.Fatalf("current count = %d, want delete markers ignored", count) - } - total, err := repos.Objects.TotalSizeByBucket(ctx, bucket.ID) - if err != nil { - t.Fatalf("TotalSizeByBucket: %v", err) - } - if total != 0 { - t.Fatalf("current size = %d, want delete markers ignored", total) - } - bucketStats, err := repos.Objects.BucketStats(ctx, bucket.ID) - if err != nil { - t.Fatalf("BucketStats: %v", err) - } - if bucketStats.Count != 0 || bucketStats.TotalSize != 0 { - t.Fatalf("bucket stats = count:%d size:%d, want delete markers ignored", bucketStats.Count, bucketStats.TotalSize) - } - - deleted, err := repos.Objects.ListRecoverableDeleteMarkers(ctx, bucket.ID, "", "", 10) - if err != nil { - t.Fatalf("ListRecoverableDeleteMarkers: %v", err) - } - if len(deleted) != 1 { - t.Fatalf("recoverable markers len = %d, want 1", len(deleted)) - } - if deleted[0].Marker.VersionID != marker.VersionID || deleted[0].RestoreVersion.VersionID != data.VersionID { - t.Fatalf("recoverable marker = %#v, want marker %s restoring %s", deleted[0], marker.VersionID, data.VersionID) - } - - lowerData := newObjectVersion(bucket.ID, "trash/lower.txt", "01J00000000000000000001003", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, lowerData); err != nil { - t.Fatalf("create lower trash data: %v", err) - } - lowerMarker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "trash/lower.txt", "01J00000000000000000001004") - if err != nil { - t.Fatalf("create lower trash marker: %v", err) - } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, newObjectVersion(bucket.ID, "Trash/lower.txt", "01J00000000000000000001005", 10)); err != nil { - t.Fatalf("create upper trash data: %v", err) - } - if _, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "Trash/lower.txt", "01J00000000000000000001006"); err != nil { - t.Fatalf("create upper trash marker: %v", err) - } - prefixedDeleted, err := repos.Objects.ListRecoverableDeleteMarkers(ctx, bucket.ID, "trash/", "", 10) - if err != nil { - t.Fatalf("ListRecoverableDeleteMarkers prefix: %v", err) - } - if len(prefixedDeleted) != 1 || prefixedDeleted[0].Marker.VersionID != lowerMarker.VersionID { - t.Fatalf("case-sensitive recoverable markers = %#v", prefixedDeleted) - } -} - -func TestObjectRepo_RestoreCurrentDeleteMarkerStackRejectsStaleMarker(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "delete-marker-stale-bucket") - - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000001041", 10)); err != nil { - t.Fatalf("create data version: %v", err) - } - marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "file.txt", "01J00000000000000000001042") - if err != nil { - t.Fatalf("create marker: %v", err) - } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000001043", 20)); err != nil { - t.Fatalf("create newer data version: %v", err) - } - - if _, err := repos.Objects.RestoreCurrentDeleteMarkerStack(ctx, bucket.ID, "file.txt", marker.VersionID); err == nil { - t.Fatal("RestoreCurrentDeleteMarkerStack returned nil error for stale marker") - } -} - -func TestObjectRepo_VersionStateUpdatesOnlyMirrorCurrentVersion(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "state-bucket") - - v1 := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000000011", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, v1); err != nil { - t.Fatalf("first version: %v", err) - } - v2 := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000000012", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, v2); err != nil { - t.Fatalf("second version: %v", err) - } - - if err := repos.Objects.UpdateVersionState(ctx, v1.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("UpdateVersionState(old): %v", err) - } - current, err := repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "file.txt") - if err != nil { - t.Fatalf("GetByBucketAndKey: %v", err) - } - if current.VersionID != v2.VersionID || current.State != model.ObjectStateCached { - t.Fatalf("old version update polluted current version: version=%s state=%s", current.VersionID, current.State) + if _, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "missing.txt", "01J00000000000000000001033"); err != nil { + t.Fatalf("create unrestorable marker: %v", err) } - if err := repos.Objects.UpdateVersionState(ctx, v2.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("UpdateVersionState(current): %v", err) - } - current, err = repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "file.txt") + count, err := repos.Objects.CountByBucket(ctx, bucket.ID) if err != nil { - t.Fatalf("GetByBucketAndKey after current update: %v", err) + t.Fatalf("CountByBucket: %v", err) } - if current.State != model.ObjectStateUploading { - t.Fatalf("current state = %s, want uploading", current.State) + if count != 0 { + t.Fatalf("current count = %d, want delete markers ignored", count) } -} - -func TestObjectRepo_UpdateVersionStateFromFailedClearsFailureDetails(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "failed-retry-bucket") - - version := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000000013", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("version: %v", err) + total, err := repos.Objects.TotalSizeByBucket(ctx, bucket.ID) + if err != nil { + t.Fatalf("TotalSizeByBucket: %v", err) } - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) + if total != 0 { + t.Fatalf("current size = %d, want delete markers ignored", total) } - if err := repos.Objects.UpdateVersionStateToFailed(ctx, version.VersionID, model.ObjectStateUploading, "upload failed"); err != nil { - t.Fatalf("failed: %v", err) + bucketStats, err := repos.Objects.BucketStats(ctx, bucket.ID) + if err != nil { + t.Fatalf("BucketStats: %v", err) } - - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateFailed, model.ObjectStateUploading); err != nil { - t.Fatalf("retry uploading: %v", err) + if bucketStats.Count != 0 || bucketStats.TotalSize != 0 { + t.Fatalf("bucket stats = count:%d size:%d, want delete markers ignored", bucketStats.Count, bucketStats.TotalSize) } - got, err := repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil || got == nil { - t.Fatalf("version after retry: got=%v err=%v", got, err) - } - if got.FailedAtState != nil { - t.Fatalf("version failed_at_state = %#v, want nil", got.FailedAtState) - } - if got.LastError != nil { - t.Fatalf("version last_error = %#v, want nil", got.LastError) - } - current, err := repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "file.txt") - if err != nil || current == nil { - t.Fatalf("current after retry: got=%v err=%v", current, err) + deleted, err := repos.Objects.ListRecoverableDeleteMarkers(ctx, bucket.ID, "", "", 10) + if err != nil { + t.Fatalf("ListRecoverableDeleteMarkers: %v", err) } - if current.FailedAtState != nil { - t.Fatalf("current failed_at_state = %#v, want nil", current.FailedAtState) + if len(deleted) != 1 { + t.Fatalf("recoverable markers len = %d, want 1", len(deleted)) } - if current.LastError != nil { - t.Fatalf("current last_error = %#v, want nil", current.LastError) + if deleted[0].Marker.VersionID != marker.VersionID || deleted[0].RestoreVersion.VersionID != data.VersionID { + t.Fatalf("recoverable marker = %#v, want marker %s restoring %s", deleted[0], marker.VersionID, data.VersionID) } -} - -func TestObjectRepo_UpdateVersionStateToFailedClearsStorageUploadID(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "failed-upload-binding-bucket") - version := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000000120", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("committing: %v", err) + lowerData := newObjectVersion(bucket.ID, "trash/lower.txt", "01J00000000000000000001003", 10) + if _, err := createVersion(t, repos, lowerData); err != nil { + t.Fatalf("create lower trash data: %v", err) } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) + lowerMarker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "trash/lower.txt", "01J00000000000000000001004") if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) + t.Fatalf("create lower trash marker: %v", err) } - primary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: primary.ID, - UploadID: upload.ID, - DataSetID: onChainID(t, "1001"), - ClientDataSetID: onChainIDPtr(t, "9001"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "bafk2bzacefailedbinding", - PieceID: onChainIDPtr(t, "2001"), - RetrievalURL: "https://provider.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: version.Size, - Checksum: version.Checksum, - }); err != nil { - t.Fatalf("BindReadableUploadForContent: %v", err) - } - - if err := repos.Objects.UpdateVersionStateToFailed(ctx, version.VersionID, model.ObjectStateReplicating, "replication failed"); err != nil { - t.Fatalf("UpdateVersionStateToFailed: %v", err) + if _, err := createVersion(t, repos, newObjectVersion(bucket.ID, "Trash/lower.txt", "01J00000000000000000001005", 10)); err != nil { + t.Fatalf("create upper trash data: %v", err) } - got, err := repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil || got == nil { - t.Fatalf("GetVersionByID: got=%v err=%v", got, err) + if _, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "Trash/lower.txt", "01J00000000000000000001006"); err != nil { + t.Fatalf("create upper trash marker: %v", err) } - if got.State != model.ObjectStateFailed || got.StorageUploadID != nil { - t.Fatalf("failed version = state:%s upload:%#v, want failed without upload binding", got.State, got.StorageUploadID) + prefixedDeleted, err := repos.Objects.ListRecoverableDeleteMarkers(ctx, bucket.ID, "trash/", "", 10) + if err != nil { + t.Fatalf("ListRecoverableDeleteMarkers prefix: %v", err) } - if got.FailedAtState == nil || *got.FailedAtState != model.ObjectStateReplicating { - t.Fatalf("failed_at_state = %#v, want replicating", got.FailedAtState) + if len(prefixedDeleted) != 1 || prefixedDeleted[0].Marker.VersionID != lowerMarker.VersionID { + t.Fatalf("case-sensitive recoverable markers = %#v", prefixedDeleted) } } -func TestObjectRepo_AcceptStorageUploadMirrorsOnlyCurrentVersion(t *testing.T) { +func TestObjectRepo_RestoreCurrentDeleteMarkerStackRejectsStaleMarker(t *testing.T) { db := testDB(t) repos := repository.NewRepositories(db) ctx := context.Background() - bucket := seedBucket(t, db, "storage-bucket") - - oldVersion := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000000021", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion); err != nil { - t.Fatalf("old version: %v", err) - } - currentVersion := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000000022", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, currentVersion); err != nil { - t.Fatalf("current version: %v", err) - } + bucket := seedBucket(t, db, "delete-marker-stale-bucket") - if err := repos.Objects.UpdateVersionState(ctx, oldVersion.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("old version upload transition: %v", err) + if _, err := createVersion(t, repos, newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000001041", 10)); err != nil { + t.Fatalf("create data version: %v", err) } - acceptTestStorageUploadForVersion(t, repos, bucket.ID, oldVersion, "piece-old") - current, err := repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "file.txt") + marker, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "file.txt", "01J00000000000000000001042") if err != nil { - t.Fatalf("GetByBucketAndKey: %v", err) - } - if current.PieceCID != nil || current.State != model.ObjectStateCached { - t.Fatalf("old storage update polluted current version: piece=%v state=%s", current.PieceCID, current.State) + t.Fatalf("create marker: %v", err) } - if current.InFilecoin { - t.Fatal("old storage update polluted current in_filecoin") + if _, err := createVersion(t, repos, newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000001043", 20)); err != nil { + t.Fatalf("create newer data version: %v", err) } - if err := repos.Objects.UpdateVersionState(ctx, currentVersion.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("current version upload transition: %v", err) - } - acceptTestStorageUploadForVersion(t, repos, bucket.ID, currentVersion, "piece-current") - current, err = repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "file.txt") - if err != nil { - t.Fatalf("GetByBucketAndKey after current storage: %v", err) - } - if current.PieceCID == nil || *current.PieceCID != "piece-current" { - t.Fatalf("current storage info = piece:%v", current.PieceCID) - } - if current.State != model.ObjectStateStored { - t.Fatalf("current state = %s, want stored", current.State) - } - if !current.InFilecoin { - t.Fatal("current in_filecoin = false, want true") + if _, err := repos.Objects.RestoreCurrentDeleteMarkerStack(ctx, bucket.ID, "file.txt", marker.VersionID); err == nil { + t.Fatal("RestoreCurrentDeleteMarkerStack returned nil error for stale marker") } } @@ -1797,11 +1238,11 @@ func TestObjectRepo_SetVersionCachePresenceMirrorsOnlyCurrentVersion(t *testing. bucket := seedBucket(t, db, "cache-location-bucket") oldVersion := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000000023", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion); err != nil { + if _, err := createVersion(t, repos, oldVersion); err != nil { t.Fatalf("old version: %v", err) } currentVersion := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000000024", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, currentVersion); err != nil { + if _, err := createVersion(t, repos, currentVersion); err != nil { t.Fatalf("current version: %v", err) } @@ -1828,289 +1269,6 @@ func TestObjectRepo_SetVersionCachePresenceMirrorsOnlyCurrentVersion(t *testing. } } -func TestObjectRepo_CacheAccessAndCommitKeepPresenceSemanticsSeparate(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "cache-access-bucket") - version := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000000CA01", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - lifecycleUpdatedAt := time.Date(2026, time.July, 1, 12, 0, 0, 0, time.UTC) - mustExec( - t, - db, - `UPDATE object_versions SET in_cache = FALSE, cache_accessed_at = NULL, updated_at = ? WHERE version_id = ?`, - lifecycleUpdatedAt, - version.VersionID, - ) - - accessedAt := lifecycleUpdatedAt.Add(2 * time.Hour) - if err := repos.Objects.RecordVersionCacheAccess(ctx, version.VersionID, accessedAt); err != nil { - t.Fatalf("RecordVersionCacheAccess: %v", err) - } - got, err := repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil || got == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", got, err) - } - if got.InCache { - t.Fatal("in_cache = true after access-only timestamp update, want false") - } - if got.CacheAccessedAt == nil || !got.CacheAccessedAt.Equal(accessedAt) { - t.Fatalf("cache_accessed_at = %v, want %v", got.CacheAccessedAt, accessedAt) - } - if !got.UpdatedAt.Equal(lifecycleUpdatedAt) { - t.Fatalf("updated_at = %v, want unchanged %v", got.UpdatedAt, lifecycleUpdatedAt) - } - - committedAt := accessedAt.Add(time.Hour) - if err := repos.Objects.RecordVersionCacheCommit(ctx, version.VersionID, committedAt); err != nil { - t.Fatalf("RecordVersionCacheCommit: %v", err) - } - got, err = repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil || got == nil { - t.Fatalf("GetVersionByID after cache commit: version=%v err=%v", got, err) - } - if !got.InCache { - t.Fatal("in_cache = false after cache commit, want true") - } - if got.CacheAccessedAt == nil || !got.CacheAccessedAt.Equal(committedAt) { - t.Fatalf("cache_accessed_at after commit = %v, want %v", got.CacheAccessedAt, committedAt) - } - if !got.UpdatedAt.Equal(lifecycleUpdatedAt) { - t.Fatalf("updated_at after cache commit = %v, want unchanged %v", got.UpdatedAt, lifecycleUpdatedAt) - } - - if err := repos.Objects.SetVersionCachePresence(ctx, version.VersionID, false); err != nil { - t.Fatalf("SetVersionCachePresence(false): %v", err) - } - olderAccess := committedAt.Add(-time.Hour) - if err := repos.Objects.RecordVersionCacheAccess(ctx, version.VersionID, olderAccess); err != nil { - t.Fatalf("RecordVersionCacheAccess(older): %v", err) - } - got, err = repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil || got == nil { - t.Fatalf("GetVersionByID after older access: version=%v err=%v", got, err) - } - if got.InCache { - t.Fatal("in_cache = true after access-only update of an absent cache entry") - } - if got.CacheAccessedAt == nil || !got.CacheAccessedAt.Equal(committedAt) { - t.Fatalf("cache_accessed_at after older write = %v, want monotonic %v", got.CacheAccessedAt, committedAt) - } - if !got.UpdatedAt.Equal(lifecycleUpdatedAt) { - t.Fatalf("updated_at after older access = %v, want unchanged %v", got.UpdatedAt, lifecycleUpdatedAt) - } -} - -func TestObjectRepo_UpdateVersionStateMarksCacheEvictedLocation(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "cache-evicted-location-bucket") - - version := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000000025", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("version: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - acceptTestStorageUploadForVersion(t, repos, bucket.ID, version, "piece") - - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateStored, model.ObjectStateCacheEvicted); err != nil { - t.Fatalf("cache evicted: %v", err) - } - - got, err := repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil || got == nil { - t.Fatalf("version: got=%v err=%v", got, err) - } - if got.InCache { - t.Fatal("version in_cache = true, want false") - } - current, err := repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "file.txt") - if err != nil { - t.Fatalf("GetByBucketAndKey: %v", err) - } - if current.State != model.ObjectStateCacheEvicted || current.InCache { - t.Fatalf("current state/cache = %s/%v, want cache_evicted/false", current.State, current.InCache) - } -} - -func TestObjectRepo_ListAndResetVersionsByState(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "version-state-bucket") - - for _, versionID := range []string{"01J00000000000000000000031", "01J00000000000000000000032"} { - v := newObjectVersion(bucket.ID, "file-"+versionID+".txt", versionID, 1) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, v); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(%s): %v", versionID, err) - } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("UpdateVersionState(%s): %v", versionID, err) - } - } - - versions, err := repos.Objects.ListVersionsByState(ctx, model.ObjectStateUploading, 10) - if err != nil { - t.Fatalf("ListVersionsByState: %v", err) - } - if len(versions) != 2 { - t.Fatalf("uploading version count = %d, want 2", len(versions)) - } - - reset, err := repos.Objects.ResetStaleVersionStates(ctx, model.ObjectStateUploading, model.ObjectStateCached, time.Now().Add(time.Hour)) - if err != nil { - t.Fatalf("ResetStaleVersionStates: %v", err) - } - if reset != 2 { - t.Fatalf("reset count = %d, want 2", reset) - } -} - -func TestObjectRepo_ResetStaleVersionStatesFromFailedClearsFailureDetails(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "failed-stale-reset-bucket") - - version := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000000033", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("version: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - if err := repos.Objects.UpdateVersionStateToFailed(ctx, version.VersionID, model.ObjectStateUploading, "upload failed"); err != nil { - t.Fatalf("failed: %v", err) - } - - reset, err := repos.Objects.ResetStaleVersionStates(ctx, model.ObjectStateFailed, model.ObjectStateUploading, time.Now().Add(time.Hour)) - if err != nil { - t.Fatalf("ResetStaleVersionStates: %v", err) - } - if reset != 1 { - t.Fatalf("reset count = %d, want 1", reset) - } - - got, err := repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil || got == nil { - t.Fatalf("version after reset: got=%v err=%v", got, err) - } - if got.FailedAtState != nil || got.LastError != nil { - t.Fatalf("version failure details = failed_at_state:%#v last_error:%#v, want nil", got.FailedAtState, got.LastError) - } - current, err := repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "file.txt") - if err != nil || current == nil { - t.Fatalf("current after reset: got=%v err=%v", current, err) - } - if current.FailedAtState != nil || current.LastError != nil { - t.Fatalf("current failure details = failed_at_state:%#v last_error:%#v, want nil", current.FailedAtState, current.LastError) - } -} - -func TestObjectRepo_ResetStaleVersionStatesDoesNotMirrorOldVersion(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "stale-current-bucket") - - oldVersion := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000000051", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion); err != nil { - t.Fatalf("old version: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, oldVersion.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("old uploading: %v", err) - } - - currentVersion := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000000052", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, currentVersion); err != nil { - t.Fatalf("current version: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, currentVersion.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("current uploading: %v", err) - } - acceptTestStorageUploadForVersion(t, repos, bucket.ID, currentVersion, "piece-current") - - reset, err := repos.Objects.ResetStaleVersionStates(ctx, model.ObjectStateUploading, model.ObjectStateCached, time.Now().Add(time.Hour)) - if err != nil { - t.Fatalf("ResetStaleVersionStates: %v", err) - } - if reset != 1 { - t.Fatalf("reset count = %d, want 1", reset) - } - - current, err := repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "file.txt") - if err != nil { - t.Fatalf("GetByBucketAndKey: %v", err) - } - if current.VersionID != currentVersion.VersionID || current.State != model.ObjectStateStored { - t.Fatalf("current version polluted: version=%s state=%s", current.VersionID, current.State) - } -} - -func TestObjectRepo_ResetStaleVersionStatesLeavesSkippedVersionUnchanged(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "stale-skip-bucket") - - currentVersion := newObjectVersion(bucket.ID, "current.txt", "01J00000000000000000000061", 10) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, currentVersion); err != nil { - t.Fatalf("current version: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, currentVersion.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("current uploading: %v", err) - } - - otherVersion := newObjectVersion(bucket.ID, "other.txt", "01J00000000000000000000062", 20) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, otherVersion); err != nil { - t.Fatalf("other version: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, otherVersion.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("other uploading: %v", err) - } - - if _, err := db.ExecContext(ctx, ` - CREATE TRIGGER skip_current_stale_reset - BEFORE UPDATE OF state ON object_versions - WHEN OLD.version_id = '01J00000000000000000000061' - BEGIN - SELECT RAISE(IGNORE); - END; - `); err != nil { - t.Fatalf("creating skip trigger: %v", err) - } - - reset, err := repos.Objects.ResetStaleVersionStates(ctx, model.ObjectStateUploading, model.ObjectStateCached, time.Now().Add(time.Hour)) - if err != nil { - t.Fatalf("ResetStaleVersionStates: %v", err) - } - if reset != 1 { - t.Fatalf("reset count = %d, want 1", reset) - } - - current, err := repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "current.txt") - if err != nil { - t.Fatalf("GetByBucketAndKey(current): %v", err) - } - if current.State != model.ObjectStateUploading { - t.Fatalf("skipped current version state = %s, want uploading", current.State) - } - - other, err := repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "other.txt") - if err != nil { - t.Fatalf("GetByBucketAndKey(other): %v", err) - } - if other.State != model.ObjectStateCached { - t.Fatalf("updated current version state = %s, want cached", other.State) - } -} - func TestObjectRepo_CurrentStatsUseCurrentVersions(t *testing.T) { db := testDB(t) repos := repository.NewRepositories(db) @@ -2127,13 +1285,13 @@ func TestObjectRepo_CurrentStatsUseCurrentVersions(t *testing.T) { t.Fatalf("empty bucket stats = count:%d size:%d, want 0/0", emptyStats.Count, emptyStats.TotalSize) } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, newObjectVersion(bucketA.ID, "a.txt", "01J00000000000000000000041", 100)); err != nil { + if _, err := createVersion(t, repos, newObjectVersion(bucketA.ID, "a.txt", "01J00000000000000000000041", 100)); err != nil { t.Fatalf("create a v1: %v", err) } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, newObjectVersion(bucketA.ID, "a.txt", "01J00000000000000000000042", 250)); err != nil { + if _, err := createVersion(t, repos, newObjectVersion(bucketA.ID, "a.txt", "01J00000000000000000000042", 250)); err != nil { t.Fatalf("create a v2: %v", err) } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, newObjectVersion(bucketB.ID, "b.txt", "01J00000000000000000000043", 500)); err != nil { + if _, err := createVersion(t, repos, newObjectVersion(bucketB.ID, "b.txt", "01J00000000000000000000043", 500)); err != nil { t.Fatalf("create b: %v", err) } @@ -2178,7 +1336,7 @@ func TestRepos_WithTx(t *testing.T) { ctx := context.Background() err := repos.WithTx(ctx, func(txRepos *repository.Repositories) error { - b := &model.Bucket{Name: "tx-bucket", Status: model.BucketStatusActive} + b := &model.Bucket{Name: "tx-bucket", Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8} if err := txRepos.Buckets.Create(ctx, b); err != nil { return err } @@ -2197,7 +1355,7 @@ func TestRepos_WithTx(t *testing.T) { } err = repos.WithTx(ctx, func(txRepos *repository.Repositories) error { - b := &model.Bucket{Name: "tx-committed", Status: model.BucketStatusActive} + b := &model.Bucket{Name: "tx-committed", Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8} return txRepos.Buckets.Create(ctx, b) }) if err != nil { diff --git a/internal/db/repository/observability_repo.go b/internal/db/repository/observability_repo.go index 48ec409..f41e6bc 100644 --- a/internal/db/repository/observability_repo.go +++ b/internal/db/repository/observability_repo.go @@ -24,7 +24,7 @@ func (r *BunObservabilityRepo) ReplaceProviderStates(ctx context.Context, checke now := time.Now().UTC() checkedAt = normalizeCheckedAt(checkedAt, now) for i := range states { - prepareProviderState(&states[i], checkedAt, now) + prepareProviderState(&states[i], checkedAt) } if _, err := db.NewDelete().Model((*observability.ProviderState)(nil)).Where("1 = 1").Exec(ctx); err != nil { return err @@ -70,7 +70,7 @@ func (r *BunObservabilityRepo) ReplaceDataSetStates(ctx context.Context, checked now := time.Now().UTC() checkedAt = normalizeCheckedAt(checkedAt, now) for i := range states { - prepareDataSetState(&states[i], checkedAt, now) + prepareDataSetState(&states[i], checkedAt) } if _, err := db.NewDelete().Model((*observability.DataSetState)(nil)).Where("1 = 1").Exec(ctx); err != nil { return err @@ -85,8 +85,8 @@ func (r *BunObservabilityRepo) ReplaceDataSetStates(ctx context.Context, checked func (r *BunObservabilityRepo) ListDataSetStates(ctx context.Context, opts observability.ListOptions) (observability.DataSetStatePage, error) { limit, offset := normalizeObservabilityPagination(opts) var rows []observability.DataSetState - if err := applyDataSetObservabilityFilters(r.db.NewSelect().Model(&rows), opts). - OrderExpr("bucket_name ASC, local_data_set_id ASC"). + if err := applyDataSetObservabilityFilters(withDataSetStateJoins(r.db.NewSelect().Model(&rows)), opts). + OrderExpr("observed_bucket.name ASC, observability_data_set_state.local_data_set_id ASC"). Limit(limit). Offset(offset). Scan(ctx); err != nil { @@ -117,9 +117,8 @@ func (r *BunObservabilityRepo) GetDataSetStatesByLocalIDs(ctx context.Context, l return out, nil } var rows []observability.DataSetState - if err := r.db.NewSelect(). - Model(&rows). - Where("local_data_set_id IN (?)", bun.List(localIDs)). + if err := withDataSetStateJoins(r.db.NewSelect().Model(&rows)). + Where("observability_data_set_state.local_data_set_id IN (?)", bun.List(localIDs)). Scan(ctx); err != nil { return nil, err } @@ -138,7 +137,7 @@ func (r *BunObservabilityRepo) withTx(ctx context.Context, fn func(context.Conte return fn(ctx, r.db) } -func prepareProviderState(state *observability.ProviderState, checkedAt time.Time, now time.Time) { +func prepareProviderState(state *observability.ProviderState, checkedAt time.Time) { if state.ReasonCodes == nil { state.ReasonCodes = []observability.ReasonCode{} } @@ -148,13 +147,9 @@ func prepareProviderState(state *observability.ProviderState, checkedAt time.Tim if state.LastCheckedAt.IsZero() { state.LastCheckedAt = checkedAt } - if state.CreatedAt.IsZero() { - state.CreatedAt = now - } - state.UpdatedAt = now } -func prepareDataSetState(state *observability.DataSetState, checkedAt time.Time, now time.Time) { +func prepareDataSetState(state *observability.DataSetState, checkedAt time.Time) { if state.ReasonCodes == nil { state.ReasonCodes = []observability.ReasonCode{} } @@ -164,10 +159,6 @@ func prepareDataSetState(state *observability.DataSetState, checkedAt time.Time, if state.LastCheckedAt.IsZero() { state.LastCheckedAt = checkedAt } - if state.CreatedAt.IsZero() { - state.CreatedAt = now - } - state.UpdatedAt = now } func insertProviderStateRows(ctx context.Context, db bun.IDB, states []observability.ProviderState) error { @@ -223,10 +214,7 @@ func normalizeObservabilityPagination(opts observability.ListOptions) (int, int) if limit > maxObservabilityListLimit { limit = maxObservabilityListLimit } - offset := opts.Offset - if offset < 0 { - offset = 0 - } + offset := max(opts.Offset, 0) return limit, offset } @@ -242,17 +230,28 @@ func applyProviderObservabilityFilters(q *bun.SelectQuery, opts observability.Li func applyDataSetObservabilityFilters(q *bun.SelectQuery, opts observability.ListOptions) *bun.SelectQuery { if opts.Status != "" { - q.Where("status = ?", opts.Status) + q.Where("observability_data_set_state.status = ?", opts.Status) } if opts.BucketID > 0 { - q.Where("bucket_id = ?", opts.BucketID) + q.Where("observability_data_set_state.bucket_id = ?", opts.BucketID) } if opts.ProviderID != nil { - q.Where("provider_id = ?", opts.ProviderID.String()) + q.Where("observability_data_set_state.provider_id = ?", opts.ProviderID.String()) } return q } +// withDataSetStateJoins reads the bucket name and local status from the rows +// that own them instead of from a copy taken when the check ran. +func withDataSetStateJoins(q *bun.SelectQuery) *bun.SelectQuery { + return q. + ColumnExpr("observability_data_set_state.*"). + ColumnExpr("observed_bucket.name AS bucket_name"). + ColumnExpr("observed_data_set.status AS local_status"). + Join("JOIN storage_data_sets AS observed_data_set ON observed_data_set.id = observability_data_set_state.local_data_set_id"). + Join("JOIN buckets AS observed_bucket ON observed_bucket.id = observability_data_set_state.bucket_id") +} + type observabilityStateAggregate struct { Total int Available int diff --git a/internal/db/repository/observability_repo_test.go b/internal/db/repository/observability_repo_test.go index 3e1cea6..8ea2d9c 100644 --- a/internal/db/repository/observability_repo_test.go +++ b/internal/db/repository/observability_repo_test.go @@ -262,6 +262,8 @@ func seedStorageDataSet(t *testing.T, db *bun.DB, bucketID int64, providerID str BucketID: bucketID, ProviderID: onChainID(t, providerID), CopyIndex: int(bucketID), + Generation: 1, + IsCurrent: true, DataSetID: onChainIDPtr(t, dataSetID), Status: status, } diff --git a/internal/db/repository/prefix_postgres_test.go b/internal/db/repository/prefix_postgres_test.go index bf65edd..faae76a 100644 --- a/internal/db/repository/prefix_postgres_test.go +++ b/internal/db/repository/prefix_postgres_test.go @@ -31,7 +31,7 @@ func TestPostgresPrefixPlan(t *testing.T) { seedPostgresPrefixObjects(t, ctx, repos, bucket.ID) seedPostgresPrefixMultiparts(t, ctx, repos, bucket.ID) seedPostgresPrefixPlanRows(t, ctx, db, bucket.ID) - riskVersionID, staleBefore := seedPostgresStorageRisk(t, ctx, repos, bucket) + riskVersionID, staleBefore := seedPostgresStorageRisk(t, ctx, db, repos, bucket) if _, err := db.ExecContext(ctx, "ANALYZE object_versions, multipart_uploads"); err != nil { t.Fatalf("ANALYZE: %v", err) } @@ -42,7 +42,7 @@ func TestPostgresPrefixPlan(t *testing.T) { t.Fatalf("ListCurrentVersionsByBucket: %v", err) } requireObjectVersionKeys(t, current, []string{"prefix/00010000.txt"}) - assertPostgresPlanUsesIndex(t, db, "idx_object_versions_current_bucket_delete_key_c", capture.last(t)) + assertPostgresPlanUsesIndex(t, db, "idx_objects_bucket_key_c", capture.last(t)) capture.reset() versions, err := repos.Objects.ListVersionsByBucket(ctx, bucket.ID, "under", "underX/literal.txt", "01J000000000000000PG000003", 10) @@ -52,7 +52,7 @@ func TestPostgresPrefixPlan(t *testing.T) { if len(versions) == 0 || versions[0].Key != "under_/literal.txt" { t.Fatalf("version marker page = %#v, want under_/literal.txt first", versions) } - assertPostgresPlanUsesIndex(t, db, "idx_object_versions_bucket_key_created_c", capture.last(t)) + assertPostgresPlanUsesIndex(t, db, "idx_object_versions_bucket_key_created", capture.last(t)) capture.reset() uploads, err := repos.Multiparts.ListByBucket(ctx, bucket.ID, `back\slash/`, `back\slash/literal.txt`, "pg-prefix-upload-000002", 10) @@ -62,10 +62,10 @@ func TestPostgresPrefixPlan(t *testing.T) { if len(uploads) != 1 || uploads[0].UploadID != "pg-prefix-upload-000002-next" { t.Fatalf("multipart page = %#v, want next upload for the marker key", uploads) } - assertPostgresPlanUsesIndex(t, db, "idx_multipart_uploads_bucket_status_key_upload_c", capture.last(t)) + assertPostgresPlanUsesIndex(t, db, "idx_multipart_uploads_bucket_status_key_upload", capture.last(t)) capture.reset() - riskPage, err := repos.Uploads.ListBucketStorageHealthAffectedVersions(ctx, repository.BucketStorageHealthAffectedVersionsInput{ + riskPage, err := repos.Contents.ListBucketStorageHealthAffectedVersions(ctx, repository.BucketStorageHealthAffectedVersionsInput{ BucketID: bucket.ID, Prefix: "prefix/00010000", StaleBefore: staleBefore, @@ -77,7 +77,7 @@ func TestPostgresPrefixPlan(t *testing.T) { if len(riskPage.Versions) != 1 || riskPage.Versions[0].Version.VersionID != riskVersionID { t.Fatalf("storage risk versions = %#v, want %s", riskPage.Versions, riskVersionID) } - assertPostgresPlanUsesIndex(t, db, "idx_object_versions_bucket_key_created_c", capture.match(t, `ORDER BY object_version.key COLLATE "C" ASC`)) + assertPostgresPlanUsesIndex(t, db, "idx_object_versions_bucket_key_created", capture.match(t, `ORDER BY object_version.key COLLATE "C" ASC`)) } func newPostgresPrefixTestDB(t *testing.T, ctx context.Context, dsn string) *bun.DB { @@ -120,7 +120,7 @@ func seedPostgresPrefixObjects(t *testing.T, ctx context.Context, repos *reposit } for i, key := range keys { versionID := fmt.Sprintf("01J000000000000000PG%06d", i) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, newObjectVersion(bucketID, key, versionID, 10)); err != nil { + if _, err := createVersion(t, repos, newObjectVersion(bucketID, key, versionID, 10)); err != nil { t.Fatalf("CreateVersionAndSetCurrent(%s): %v", key, err) } } @@ -144,48 +144,74 @@ func seedPostgresPrefixMultiparts(t *testing.T, ctx context.Context, repos *repo func seedPostgresPrefixPlanRows(t *testing.T, ctx context.Context, db *bun.DB, bucketID int64) { t.Helper() if _, err := db.ExecContext(ctx, ` - INSERT INTO objects (bucket_id, key) - SELECT ?, 'prefix/' || lpad(value::text, 8, '0') || '.txt' + INSERT INTO objects (bucket_id, key, created_at, updated_at) + SELECT ?, 'prefix/' || lpad(value::text, 8, '0') || '.txt', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP FROM generate_series(0, 19999) AS series(value)`, bucketID); err != nil { t.Fatalf("seeding plan objects: %v", err) } + // Bytes own their identity, so each seeded version needs a content row and + // residency belongs to that content rather than to the version. + if _, err := db.ExecContext(ctx, ` + INSERT INTO storage_contents (bucket_id, checksum, content_size, requested_copies, created_at, updated_at) + SELECT object_row.bucket_id, lpad(to_hex(object_row.id), 64, '0'), 10, 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM objects AS object_row + WHERE object_row.bucket_id = ? AND object_row.key LIKE 'prefix/%'`, bucketID); err != nil { + t.Fatalf("seeding plan contents: %v", err) + } + // Only the plan contents need entries here; the objects seeded through the + // repository already have theirs, and their checksums do not collide. + if _, err := db.ExecContext(ctx, ` + INSERT INTO object_cache (content_id, in_cache, cache_accessed_at, created_at, updated_at) + SELECT content_row.id, TRUE, now(), CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM objects AS object_row + JOIN storage_contents AS content_row + ON content_row.checksum = lpad(to_hex(object_row.id), 64, '0') + AND content_row.bucket_id = object_row.bucket_id + WHERE object_row.bucket_id = ? AND object_row.key LIKE 'prefix/%'`, bucketID); err != nil { + t.Fatalf("seeding plan cache entries: %v", err) + } if _, err := db.ExecContext(ctx, ` INSERT INTO object_versions ( - version_id, object_id, bucket_id, key, size, e_tag, checksum, - content_type, cache_key, in_cache, is_current, is_delete_marker, state - ) + version_id, object_id, bucket_id, key, content_id, size, e_tag, + content_type, is_delete_marker + , created_at, updated_at) SELECT 'pg-prefix-version-' || object_row.id, object_row.id, object_row.bucket_id, object_row.key, + content_row.id, 10, 'etag', - 'checksum', 'text/plain', - '.versions/' || object_row.id, - TRUE, - TRUE, - FALSE, - 'cached' + FALSE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP FROM objects AS object_row + JOIN storage_contents AS content_row + ON content_row.checksum = lpad(to_hex(object_row.id), 64, '0') + AND content_row.bucket_id = object_row.bucket_id WHERE object_row.bucket_id = ? AND object_row.key LIKE 'prefix/%'`, bucketID); err != nil { t.Fatalf("seeding plan object versions: %v", err) } + // "Current" is the object's pointer now. if _, err := db.ExecContext(ctx, ` - INSERT INTO multipart_uploads (bucket_id, key, upload_id, content_type, status) + UPDATE objects SET current_version_id = 'pg-prefix-version-' || id + WHERE bucket_id = ? AND key LIKE 'prefix/%'`, bucketID); err != nil { + t.Fatalf("pointing plan objects at their versions: %v", err) + } + if _, err := db.ExecContext(ctx, ` + INSERT INTO multipart_uploads (bucket_id, key, upload_id, content_type, status, created_at, updated_at) SELECT ?, 'prefix/' || lpad(value::text, 8, '0') || '.txt', 'pg-prefix-plan-upload-' || value, 'application/octet-stream', - 'initiated' + 'initiated', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP FROM generate_series(0, 19999) AS series(value)`, bucketID); err != nil { t.Fatalf("seeding plan multipart uploads: %v", err) } } -func seedPostgresStorageRisk(t *testing.T, ctx context.Context, repos *repository.Repositories, bucket *model.Bucket) (string, time.Time) { +func seedPostgresStorageRisk(t *testing.T, ctx context.Context, db *bun.DB, repos *repository.Repositories, bucket *model.Bucket) (string, time.Time) { t.Helper() version, err := repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "prefix/00010000.txt") if err != nil { @@ -194,9 +220,12 @@ func seedPostgresStorageRisk(t *testing.T, ctx context.Context, repos *repositor if version == nil { t.Fatal("storage risk version is missing") } - upload := startCopyHealthUpload(t, repos, bucket.ID, version.VersionID, version.Size, version.Checksum, 1) - risk := commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 0, "501", "5501", "6501", "https://provider.example/prefix-risk") - bindStorageHealthVersion(t, repos, bucket.ID, upload.ID, version) + if version.ContentID == nil { + t.Fatal("storage risk version has no content") + } + contentID := *version.ContentID + risk := commitStorageHealthCopy(t, db, repos, bucket.ID, contentID, 0, "501", "5501", "6501", "https://provider.example/prefix-risk") + bindStorageHealthVersion(t, repos, bucket.ID, contentID, version) checkedAt := time.Now().UTC() if err := repos.Observability.ReplaceDataSetStates(ctx, checkedAt, []observability.DataSetState{{ LocalDataSetID: risk.ID, @@ -205,6 +234,7 @@ func seedPostgresStorageRisk(t *testing.T, ctx context.Context, repos *repositor CopyIndex: risk.CopyIndex, ProviderID: risk.ProviderID, ChainDataSetID: risk.DataSetID, + LocalStatus: risk.Status, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}, diff --git a/internal/db/repository/repos.go b/internal/db/repository/repos.go index 1d87dc2..91a1eef 100644 --- a/internal/db/repository/repos.go +++ b/internal/db/repository/repos.go @@ -15,7 +15,7 @@ type Repositories struct { Buckets BucketRepository S3Accounts S3AccountRepository Objects ObjectRepository - Uploads StorageUploadRepository + Contents StorageContentRepository Replacements StorageReplacementRepository StorageCleanup StorageCleanupRepository Tasks TaskRepository @@ -33,7 +33,7 @@ func NewRepositories(db bun.IDB) *Repositories { Buckets: &BunBucketRepo{db: db}, S3Accounts: &BunS3AccountRepo{db: db}, Objects: &BunObjectRepo{db: db}, - Uploads: &BunStorageUploadRepo{db: db}, + Contents: &BunStorageContentRepo{db: db}, Replacements: &BunStorageReplacementRepo{db: db}, StorageCleanup: &BunStorageCleanupRepo{db: db}, Tasks: &BunTaskRepo{db: db}, @@ -65,10 +65,7 @@ func (r *Repositories) WithTx(ctx context.Context, fn func(txRepos *Repositories if !shouldRetryRepositoryTx(err) || attempt >= 19 { return err } - delay := time.Duration(attempt+1) * 25 * time.Millisecond - if delay > 200*time.Millisecond { - delay = 200 * time.Millisecond - } + delay := min(time.Duration(attempt+1)*25*time.Millisecond, 200*time.Millisecond) timer := time.NewTimer(delay) select { case <-ctx.Done(): diff --git a/internal/db/repository/s3_account_repo_test.go b/internal/db/repository/s3_account_repo_test.go index 2baa9bc..bb991dc 100644 --- a/internal/db/repository/s3_account_repo_test.go +++ b/internal/db/repository/s3_account_repo_test.go @@ -128,10 +128,10 @@ func TestBucketRepo_OwnerAccessKeyCountAndACLUpdate(t *testing.T) { ownerA := "owner-a" root := "root-access" for _, bucket := range []*model.Bucket{ - {Name: "a-one", Status: model.BucketStatusActive, OwnerAccessKey: &ownerA}, - {Name: "a-two", Status: model.BucketStatusActive, OwnerAccessKey: &ownerA}, - {Name: "root-one", Status: model.BucketStatusActive, OwnerAccessKey: &root}, - {Name: "unassigned", Status: model.BucketStatusActive}, + {Name: "a-one", Status: model.BucketStatusActive, OwnerAccessKey: &ownerA, DefaultCopies: 8, MinimumDurableCopies: 8}, + {Name: "a-two", Status: model.BucketStatusActive, OwnerAccessKey: &ownerA, DefaultCopies: 8, MinimumDurableCopies: 8}, + {Name: "root-one", Status: model.BucketStatusActive, OwnerAccessKey: &root, DefaultCopies: 8, MinimumDurableCopies: 8}, + {Name: "unassigned", Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8}, } { if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Create bucket %s: %v", bucket.Name, err) diff --git a/internal/db/repository/storage_cleanup_repo.go b/internal/db/repository/storage_cleanup_repo.go index 2bcf5c9..2e685c2 100644 --- a/internal/db/repository/storage_cleanup_repo.go +++ b/internal/db/repository/storage_cleanup_repo.go @@ -5,11 +5,9 @@ import ( "database/sql" "errors" "fmt" - "slices" "time" "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagereplacement" "github.com/uptrace/bun" ) @@ -19,11 +17,49 @@ type BunStorageCleanupRepo struct { var _ StorageCleanupRepository = (*BunStorageCleanupRepo)(nil) -func (r *BunStorageCleanupRepo) ListCopiesForTask(ctx context.Context, taskID int64) ([]model.StorageCleanupCopy, error) { +func (r *BunStorageCleanupRepo) BindTask(ctx context.Context, contentID, generation, taskID int64) error { + if contentID < 1 || generation < 1 || taskID < 1 { + return ErrInvalidInput + } + result, err := r.db.NewUpdate(). + Model((*model.StorageContent)(nil)). + Set("cleanup_task_id = ?", taskID). + Set("updated_at = ?", time.Now()). + Where("id = ?", contentID). + Where("cleanup_generation = ?", generation). + Where("cleanup_task_id IS NULL OR cleanup_task_id = ?", taskID). + Exec(ctx) + if err != nil { + return fmt.Errorf("binding storage cleanup task: %w", err) + } + rows, _ := result.RowsAffected() + if rows != 1 { + return ErrConflict + } + return nil +} + +func (r *BunStorageCleanupRepo) AuthorizeTask(ctx context.Context, contentID, generation, taskID int64) ([]model.StorageCleanupCopy, error) { + var owner struct { + TaskID *int64 `bun:"cleanup_task_id"` + } + if err := r.db.NewSelect(). + Model((*model.StorageContent)(nil)). + Column("cleanup_task_id"). + Where("id = ? AND cleanup_generation = ?", contentID, generation). + Scan(ctx, &owner); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrConflict + } + return nil, fmt.Errorf("authorizing storage cleanup task: %w", err) + } + if owner.TaskID == nil || *owner.TaskID != taskID { + return nil, ErrConflict + } var copies []model.StorageCleanupCopy if err := r.db.NewSelect(). Model(&copies). - Where("task_id = ?", taskID). + Where("content_id = ?", contentID). OrderExpr("copy_index ASC"). Scan(ctx); err != nil { return nil, fmt.Errorf("listing storage cleanup copies: %w", err) @@ -40,8 +76,16 @@ func (r *BunStorageCleanupRepo) MarkCopyRemoved(ctx context.Context, id int64) e Set("last_error = NULL"). Set("updated_at = ?", now). Where("id = ?", id). + Where("status IN (?)", bun.List([]model.StorageCleanupCopyStatus{ + model.StorageCleanupCopyStatusPending, + model.StorageCleanupCopyStatusDeleteScheduled, + model.StorageCleanupCopyStatusFailed, + })). Exec(ctx) - return storageCleanupCopyUpdateResult(res, err, "marking storage cleanup copy removed") + if err := storageCleanupCopyTransitionResult(ctx, r.db, res, err, id, model.StorageCleanupCopyStatusRemoved, "", "marking storage cleanup copy removed"); err != nil { + return err + } + return nil } func (r *BunStorageCleanupRepo) MarkCopyDeleteScheduled(ctx context.Context, id int64, txHash string) error { @@ -54,8 +98,24 @@ func (r *BunStorageCleanupRepo) MarkCopyDeleteScheduled(ctx context.Context, id Set("last_error = NULL"). Set("updated_at = ?", now). Where("id = ?", id). + Where("status = ?", model.StorageCleanupCopyStatusPending). Exec(ctx) - return storageCleanupCopyUpdateResult(res, err, "marking storage cleanup copy scheduled") + return storageCleanupCopyTransitionResult( + ctx, r.db, res, err, id, model.StorageCleanupCopyStatusDeleteScheduled, txHash, + "marking storage cleanup copy scheduled", + ) +} + +func (r *BunStorageCleanupRepo) MarkCopyFailed(ctx context.Context, id int64, message string) error { + now := time.Now() + res, err := r.db.NewUpdate(). + Model((*model.StorageCleanupCopy)(nil)). + Set("status = ?", model.StorageCleanupCopyStatusFailed). + Set("last_error = ?", message). + Set("updated_at = ?", now). + Where("id = ?", id). + Exec(ctx) + return storageCleanupCopyUpdateResult(res, err, "marking storage cleanup copy failed") } func (r *BunStorageCleanupRepo) MarkCopyUnsupported(ctx context.Context, id int64, message string) error { @@ -70,20 +130,20 @@ func (r *BunStorageCleanupRepo) MarkCopyUnsupported(ctx context.Context, id int6 return storageCleanupCopyUpdateResult(res, err, "marking storage cleanup copy unsupported") } -func (r *BunStorageCleanupRepo) UploadHasObjectReferences(ctx context.Context, uploadID int64) (bool, error) { - return uploadHasObjectReferences(ctx, r.db, uploadID) +func (r *BunStorageCleanupRepo) UploadHasObjectReferences(ctx context.Context, contentID int64) (bool, error) { + return uploadHasObjectReferences(ctx, r.db, contentID) } -func uploadHasObjectReferences(ctx context.Context, db bun.IDB, uploadID int64) (bool, error) { +func uploadHasObjectReferences(ctx context.Context, db bun.IDB, contentID int64) (bool, error) { var row struct { Count int `bun:"count"` } err := db.NewRaw(`SELECT COUNT(DISTINCT object_version.version_id) AS count - FROM storage_uploads AS storage_upload + FROM storage_contents AS storage_content JOIN object_versions AS object_version - ON `+objectVersionReferencesStorageUploadSQL("object_version", "storage_upload")+` - WHERE storage_upload.id = ? - AND object_version.is_delete_marker = ?`, uploadID, false). + ON `+objectVersionReferencesStorageContentSQL("object_version", "storage_content")+` + WHERE storage_content.id = ? + AND object_version.is_delete_marker = ?`, contentID, false). Scan(ctx, &row) if err != nil { return false, fmt.Errorf("checking storage cleanup object references: %w", err) @@ -91,16 +151,16 @@ func uploadHasObjectReferences(ctx context.Context, db bun.IDB, uploadID int64) return row.Count > 0, nil } -func (r *BunStorageCleanupRepo) TaskHasObjectReferences(ctx context.Context, taskID int64, uploadID int64) (bool, error) { +func (r *BunStorageCleanupRepo) CleanupHasObjectReferences(ctx context.Context, contentID int64) (bool, error) { var row struct { Count int `bun:"count"` } err := r.db.NewRaw(`SELECT COUNT(DISTINCT object_version.version_id) AS count FROM storage_cleanup_copies AS cleanup_copy - JOIN storage_upload_copies AS storage_copy + JOIN storage_copies AS storage_copy ON storage_copy.status = ? AND ( - storage_copy.upload_id = ? + storage_copy.content_id = ? OR ( cleanup_copy.provider_id IS NOT NULL AND cleanup_copy.provider_id <> '' AND cleanup_copy.data_set_id IS NOT NULL AND cleanup_copy.data_set_id <> '' @@ -110,16 +170,16 @@ func (r *BunStorageCleanupRepo) TaskHasObjectReferences(ctx context.Context, tas ) ) LEFT JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id - JOIN storage_uploads AS referenced_upload ON referenced_upload.id = storage_copy.upload_id + JOIN storage_contents AS referenced_upload ON referenced_upload.id = storage_copy.content_id JOIN object_versions AS object_version - ON `+objectVersionReferencesStorageUploadSQL("object_version", "referenced_upload")+` - WHERE cleanup_copy.task_id = ? + ON `+objectVersionReferencesStorageContentSQL("object_version", "referenced_upload")+` + WHERE cleanup_copy.content_id = ? AND object_version.is_delete_marker = FALSE AND ( - storage_copy.upload_id = ? + storage_copy.content_id = ? OR storage_data_set.data_set_id = cleanup_copy.data_set_id )`, - model.StorageUploadCopyStatusCommitted, uploadID, taskID, uploadID, + model.StorageCopyStatusCommitted, contentID, contentID, contentID, ).Scan(ctx, &row) if err != nil { return false, fmt.Errorf("checking storage cleanup task references: %w", err) @@ -128,12 +188,12 @@ func (r *BunStorageCleanupRepo) TaskHasObjectReferences(ctx context.Context, tas return true, nil } row.Count = 0 - err = r.db.NewRaw(`SELECT COUNT(DISTINCT active_upload.id) AS count + err = r.db.NewRaw(`SELECT COUNT(DISTINCT active_content.id) AS count FROM storage_cleanup_copies AS cleanup_copy - JOIN storage_upload_copies AS storage_copy + JOIN storage_copies AS storage_copy ON storage_copy.status = ? AND ( - storage_copy.upload_id = ? + storage_copy.content_id = ? OR ( cleanup_copy.provider_id IS NOT NULL AND cleanup_copy.provider_id <> '' AND cleanup_copy.data_set_id IS NOT NULL AND cleanup_copy.data_set_id <> '' @@ -143,14 +203,14 @@ func (r *BunStorageCleanupRepo) TaskHasObjectReferences(ctx context.Context, tas ) ) LEFT JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id - JOIN storage_uploads AS active_upload ON active_upload.id = storage_copy.upload_id - WHERE cleanup_copy.task_id = ? - AND active_upload.status IN (?) + JOIN storage_contents AS active_content ON active_content.id = storage_copy.content_id + WHERE cleanup_copy.content_id = ? + AND active_content.accepted_at IS NULL AND ( - storage_copy.upload_id = ? + storage_copy.content_id = ? OR storage_data_set.data_set_id = cleanup_copy.data_set_id )`, - model.StorageUploadCopyStatusCommitted, uploadID, taskID, bun.List(activeUploadStatuses()), uploadID, + model.StorageCopyStatusCommitted, contentID, contentID, contentID, ).Scan(ctx, &row) if err != nil { return false, fmt.Errorf("checking storage cleanup active upload references: %w", err) @@ -158,82 +218,21 @@ func (r *BunStorageCleanupRepo) TaskHasObjectReferences(ctx context.Context, tas return row.Count > 0, nil } -func (r *BunStorageCleanupRepo) DeleteUploadProvenanceIfUnreferenced(ctx context.Context, uploadID int64) error { - if uploadID <= 0 { - return fmt.Errorf("deleting unreferenced storage upload provenance: %w", ErrInvalidInput) +func (r *BunStorageCleanupRepo) CompleteTask(ctx context.Context, contentID, generation, taskID int64) error { + result, err := r.db.NewUpdate(). + Model((*model.StorageContent)(nil)). + Set("cleanup_task_id = NULL"). + Set("updated_at = ?", time.Now()). + Where("id = ? AND cleanup_generation = ? AND cleanup_task_id = ?", contentID, generation, taskID). + Exec(ctx) + if err != nil { + return fmt.Errorf("completing storage cleanup task: %w", err) } - return r.runMaybeTx(ctx, func(db bun.IDB) error { - if _, err := lockStorageUploadsByID(ctx, db, []int64{uploadID}); err != nil { - if errors.Is(err, ErrNotFound) { - return nil - } - return fmt.Errorf("locking storage upload before deleting provenance: %w", err) - } - hasRefs, err := uploadHasObjectReferences(ctx, db, uploadID) - if err != nil { - return err - } - if hasRefs { - return nil - } - // Replacement items retain upload provenance while they are executable. - // Settle and remove them explicitly so the RESTRICT foreign key remains a - // guard against bypassing the replacement progress transaction. - var items []storagereplacement.Item - if err := db.NewRaw( - lockReplacementItemsByUploadSQL(), - uploadID, - ).Scan(ctx, &items); err != nil && err != sql.ErrNoRows { - return fmt.Errorf("locking replacement items before deleting provenance: %w", err) - } - lockedReplacements := make(map[int64]struct{}, len(items)) - replacementIDs := make([]int64, 0, len(items)) - for i := range items { - if _, ok := lockedReplacements[items[i].ReplacementID]; ok { - continue - } - lockedReplacements[items[i].ReplacementID] = struct{}{} - replacementIDs = append(replacementIDs, items[i].ReplacementID) - } - slices.Sort(replacementIDs) - for _, replacementID := range replacementIDs { - if _, err := lockReplacementByID(ctx, db, replacementID); err != nil { - return err - } - } - for i := range items { - item := &items[i] - if err := settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusCancelled); err != nil { - return err - } - } - if len(items) > 0 { - if _, err := db.NewDelete(). - Model((*storagereplacement.Item)(nil)). - Where("upload_id = ?", uploadID). - Exec(ctx); err != nil { - return fmt.Errorf("deleting settled replacement items: %w", err) - } - } - if _, err := db.NewDelete(). - Model((*model.StorageUpload)(nil)). - Where("id = ?", uploadID). - Exec(ctx); err != nil { - return fmt.Errorf("deleting unreferenced storage upload provenance: %w", err) - } - return nil - }) -} - -func lockReplacementItemsByUploadSQL() string { - return `UPDATE storage_replacement_items - SET updated_at = updated_at - WHERE upload_id = ? - RETURNING *` -} - -func (r *BunStorageCleanupRepo) runMaybeTx(ctx context.Context, fn func(bun.IDB) error) error { - return runMaybeTx(ctx, r.db, fn) + rows, _ := result.RowsAffected() + if rows != 1 { + return ErrConflict + } + return nil } func storageCleanupCopyUpdateResult(res sql.Result, err error, op string) error { @@ -246,3 +245,43 @@ func storageCleanupCopyUpdateResult(res sql.Result, err error, op string) error } return nil } + +func storageCleanupCopyTransitionResult( + ctx context.Context, + db bun.IDB, + res sql.Result, + err error, + id int64, + idempotentStatus model.StorageCleanupCopyStatus, + idempotentTxHash string, + op string, +) error { + if err != nil { + return fmt.Errorf("%s: %w", op, err) + } + rows, _ := res.RowsAffected() + if rows == 1 { + return nil + } + var current struct { + Status model.StorageCleanupCopyStatus `bun:"status"` + DeleteTxHash *string `bun:"delete_tx_hash"` + } + err = db.NewSelect(). + Model((*model.StorageCleanupCopy)(nil)). + Column("status", "delete_tx_hash"). + Where("id = ?", id). + Scan(ctx, ¤t) + if errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("%s: %w", op, ErrNotFound) + } + if err != nil { + return fmt.Errorf("%s: checking current state: %w", op, err) + } + if current.Status == idempotentStatus && + (idempotentStatus != model.StorageCleanupCopyStatusDeleteScheduled || + (current.DeleteTxHash != nil && *current.DeleteTxHash == idempotentTxHash)) { + return nil + } + return fmt.Errorf("%s: %w", op, ErrConflict) +} diff --git a/internal/db/repository/storage_cleanup_repo_test.go b/internal/db/repository/storage_cleanup_repo_test.go new file mode 100644 index 0000000..d2293f4 --- /dev/null +++ b/internal/db/repository/storage_cleanup_repo_test.go @@ -0,0 +1,83 @@ +package repository_test + +import ( + "errors" + "strconv" + "testing" + + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" +) + +func TestStorageCleanupCopyTransitionsAreGuardedAndIdempotent(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + bucket := seedBucket(t, db, "cleanup-transitions") + contentID := seedContent(t, repos, bucket.ID, "cleanup-transitions", 10) + providerID := onChainID(t, "701") + binding, err := repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: providerID, CopyIndex: 0, CreatedByContentID: contentID, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + insertCopy := func(piece int64) *model.StorageCleanupCopy { + t.Helper() + row := &model.StorageCleanupCopy{ + ContentID: contentID, BucketID: bucket.ID, CopyIndex: 0, ProviderID: providerID, + StorageDataSetID: binding.ID, PieceID: onChainID(t, strconv.FormatInt(piece, 10)), PieceCID: "piece-cid", + Status: model.StorageCleanupCopyStatusPending, + } + if _, err := db.NewInsert().Model(row).Exec(t.Context()); err != nil { + t.Fatalf("insert cleanup copy: %v", err) + } + return row + } + + scheduled := insertCopy(801) + if err := repos.StorageCleanup.MarkCopyDeleteScheduled(t.Context(), scheduled.ID, "0xtx"); err != nil { + t.Fatalf("MarkCopyDeleteScheduled: %v", err) + } + first := new(model.StorageCleanupCopy) + if err := db.NewSelect().Model(first).Where("id = ?", scheduled.ID).Scan(t.Context()); err != nil { + t.Fatalf("load scheduled copy: %v", err) + } + if err := repos.StorageCleanup.MarkCopyDeleteScheduled(t.Context(), scheduled.ID, "0xtx"); err != nil { + t.Fatalf("idempotent MarkCopyDeleteScheduled: %v", err) + } + replayed := new(model.StorageCleanupCopy) + if err := db.NewSelect().Model(replayed).Where("id = ?", scheduled.ID).Scan(t.Context()); err != nil { + t.Fatalf("reload scheduled copy: %v", err) + } + if first.ScheduledAt == nil || replayed.ScheduledAt == nil || !first.ScheduledAt.Equal(*replayed.ScheduledAt) { + t.Fatalf("scheduled_at changed across replay: %v -> %v", first.ScheduledAt, replayed.ScheduledAt) + } + if err := repos.StorageCleanup.MarkCopyDeleteScheduled(t.Context(), scheduled.ID, "0xother"); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("changed scheduling evidence = %v, want ErrConflict", err) + } + if err := repos.StorageCleanup.MarkCopyRemoved(t.Context(), scheduled.ID); err != nil { + t.Fatalf("MarkCopyRemoved(scheduled): %v", err) + } + if err := repos.StorageCleanup.MarkCopyRemoved(t.Context(), scheduled.ID); err != nil { + t.Fatalf("idempotent MarkCopyRemoved: %v", err) + } + + failed := insertCopy(802) + if err := repos.StorageCleanup.MarkCopyFailed(t.Context(), failed.ID, "unknown outcome"); err != nil { + t.Fatalf("MarkCopyFailed: %v", err) + } + if err := repos.StorageCleanup.MarkCopyRemoved(t.Context(), failed.ID); err != nil { + t.Fatalf("MarkCopyRemoved(failed): %v", err) + } + + unsupported := insertCopy(803) + if err := repos.StorageCleanup.MarkCopyUnsupported(t.Context(), unsupported.ID, "unsupported"); err != nil { + t.Fatalf("MarkCopyUnsupported: %v", err) + } + if err := repos.StorageCleanup.MarkCopyRemoved(t.Context(), unsupported.ID); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("MarkCopyRemoved(unsupported) = %v, want ErrConflict", err) + } + if err := repos.StorageCleanup.MarkCopyRemoved(t.Context(), 999999); !errors.Is(err, repository.ErrNotFound) { + t.Fatalf("MarkCopyRemoved(missing) = %v, want ErrNotFound", err) + } +} diff --git a/internal/db/repository/storage_commit_postgres_test.go b/internal/db/repository/storage_commit_postgres_test.go deleted file mode 100644 index c23c905..0000000 --- a/internal/db/repository/storage_commit_postgres_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package repository_test - -import ( - "context" - "os" - "strings" - "sync" - "testing" - "time" - - "github.com/strahe/synaps3/internal/storagecommit" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/uptrace/bun" -) - -type commitAttentionRaceContextKey struct{} - -type commitAttentionReleaseClaimBarrier struct { - releaseItemUpdated chan struct{} - claimUpdateStarted chan struct{} - allowRelease chan struct{} - releaseOnce sync.Once - claimOnce sync.Once -} - -func (h *commitAttentionReleaseClaimBarrier) BeforeQuery(ctx context.Context, event *bun.QueryEvent) context.Context { - operation, _ := ctx.Value(commitAttentionRaceContextKey{}).(string) - query := strings.ToLower(event.Query) - if operation == "claim" && - strings.Contains(query, "update storage_replacement_items") && - strings.Contains(query, "attempts = attempts + 1") && - strings.Contains(query, "returning *") { - h.claimOnce.Do(func() { close(h.claimUpdateStarted) }) - } - return ctx -} - -func (h *commitAttentionReleaseClaimBarrier) AfterQuery(ctx context.Context, event *bun.QueryEvent) { - operation, _ := ctx.Value(commitAttentionRaceContextKey{}).(string) - query := strings.ToLower(event.Query) - if operation == "release" && - strings.Contains(query, "storage_replacement_items") && - strings.Contains(query, "set \"status\" = 'failed'") && - strings.Contains(query, "claimed_at is null") { - h.releaseOnce.Do(func() { - close(h.releaseItemUpdated) - <-h.allowRelease - }) - } -} - -func waitPostgresCommitClaimBlocked(t *testing.T, ctx context.Context, db *bun.DB) { - t.Helper() - waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - ticker := time.NewTicker(10 * time.Millisecond) - defer ticker.Stop() - for { - var blocked bool - err := db.NewRaw(`SELECT EXISTS ( - SELECT 1 - FROM pg_stat_activity - WHERE datname = current_database() - AND pid <> pg_backend_pid() - AND state = 'active' - AND wait_event_type = 'Lock' - AND cardinality(pg_blocking_pids(pid)) > 0 - AND query ILIKE '%UPDATE storage_replacement_items%' - AND query ILIKE '%attempts = attempts + 1%' - )`).Scan(waitCtx, &blocked) - if err != nil { - t.Fatalf("observe blocked terminal replacement claim: %v", err) - } - if blocked { - return - } - select { - case <-waitCtx.Done(): - t.Fatal("timed out waiting for terminal replacement claim to block on the release transaction") - case <-ticker.C: - } - } -} - -func TestPostgresConcurrentStorageCommitReservationsRespectCapacity(t *testing.T) { - dsn := os.Getenv("SYNAPS3_POSTGRES_TEST_DSN") - if dsn == "" { - t.Skip("SYNAPS3_POSTGRES_TEST_DSN is not set") - } - assertConcurrentStorageCommitReservationsRespectCapacity( - t, - newPostgresReplacementDB(t, context.Background(), dsn), - ) -} - -func TestPostgresCommitAttentionReleaseDoesNotReviveTerminalReplacementClaim(t *testing.T) { - dsn := os.Getenv("SYNAPS3_POSTGRES_TEST_DSN") - if dsn == "" { - t.Skip("SYNAPS3_POSTGRES_TEST_DSN is not set") - } - ctx := context.Background() - db := newPostgresReplacementDB(t, ctx, dsn) - fixture := seedCommitAttentionReplacement( - t, - db, - "postgres-commit-release-claim-race", - storagereplacement.StatusFailed, - false, - ) - barrier := &commitAttentionReleaseClaimBarrier{ - releaseItemUpdated: make(chan struct{}), - claimUpdateStarted: make(chan struct{}), - allowRelease: make(chan struct{}), - } - db.AddQueryHook(barrier) - defer func() { - select { - case <-barrier.allowRelease: - default: - close(barrier.allowRelease) - } - }() - - releaseResult := make(chan error, 1) - go func() { - releaseCtx := context.WithValue(ctx, commitAttentionRaceContextKey{}, "release") - releaseResult <- fixture.repos.Uploads.ReleaseCommitAttention(releaseCtx, storagecommit.ManualReleaseInput{ - CopyID: fixture.copyRow.ID, - ExpectedAttemptID: fixture.attemptID, - AcknowledgePossibleDuplicate: true, - }) - }() - waitReplacementSignal(t, barrier.releaseItemUpdated, "terminal replacement release update") - - type claimResult struct { - item *storagereplacement.Item - err error - } - claimed := make(chan claimResult, 1) - go func() { - claimCtx := context.WithValue(ctx, commitAttentionRaceContextKey{}, "claim") - item, err := fixture.repos.Replacements.ClaimReadyReplacementItem(claimCtx, time.Minute) - claimed <- claimResult{item: item, err: err} - }() - waitReplacementSignal(t, barrier.claimUpdateStarted, "competing terminal replacement claim update") - waitPostgresCommitClaimBlocked(t, ctx, db) - close(barrier.allowRelease) - - if err := waitReplacementResult(t, releaseResult, "storage confirmation release"); err != nil { - t.Fatalf("ReleaseCommitAttention: %v", err) - } - var claim claimResult - select { - case claim = <-claimed: - case <-time.After(10 * time.Second): - t.Fatal("timed out waiting for terminal replacement claim") - } - if claim.err != nil || claim.item != nil { - t.Fatalf("terminal claim after release = %#v err=%v, want no work", claim.item, claim.err) - } - - copyRow, err := fixture.repos.Uploads.GetUploadCopyByID(ctx, fixture.copyRow.ID) - if err != nil || copyRow.CommitAttemptID != nil || copyRow.CommitAttentionAt != nil { - t.Fatalf("copy after release = %#v err=%v, want cleared attention fence", copyRow, err) - } - item := new(storagereplacement.Item) - if err := db.NewSelect().Model(item).Where("id = ?", fixture.item.ID).Scan(ctx); err != nil { - t.Fatalf("load replacement item: %v", err) - } - if item.Status != storagereplacement.ItemStatusFailed || item.ClaimedAt != nil || item.LeaseUntil != nil { - t.Fatalf("replacement item after race = %#v, want failed without a claim", item) - } -} diff --git a/internal/db/repository/storage_commit_repo.go b/internal/db/repository/storage_commit_repo.go index 891d451..277ef8a 100644 --- a/internal/db/repository/storage_commit_repo.go +++ b/internal/db/repository/storage_commit_repo.go @@ -9,11 +9,10 @@ import ( "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/storagecommit" - "github.com/strahe/synaps3/internal/storagereplacement" "github.com/uptrace/bun" ) -func (r *BunStorageUploadRepo) ReserveCommitAttempt( +func (r *BunStorageContentRepo) ReserveCommitAttempt( ctx context.Context, input storagecommit.ReserveInput, ) (storagecommit.ReserveResult, error) { @@ -27,8 +26,7 @@ func (r *BunStorageUploadRepo) ReserveCommitAttempt( if err != nil { return err } - currentGeneration := dataSet.IsCurrent && - (dataSet.Status == model.StorageDataSetStatusReady || dataSet.Status == model.StorageDataSetStatusUnavailable) + currentGeneration := dataSet.IsCurrent && dataSet.Status == model.StorageDataSetStatusReady if !currentGeneration && dataSet.Status != model.StorageDataSetStatusDraining { return fmt.Errorf("reserving storage commit attempt on unavailable generation: %w", ErrConflict) } @@ -36,12 +34,12 @@ func (r *BunStorageUploadRepo) ReserveCommitAttempt( if err != nil { return err } - if copyRow.Status != model.StorageUploadCopyStatusPieceReady { + if copyRow.Status != model.StorageCopyStatusPieceReady { return fmt.Errorf("reserving storage commit attempt for copy %d in status %s: %w", copyID, copyRow.Status, ErrConflict) } if copyRow.CommitReadyAt == nil { res, updateErr := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). + Model((*model.StorageCopy)(nil)). Set("commit_ready_at = ?", now). Set("updated_at = ?", now). Where("id = ?", copyID). @@ -58,7 +56,7 @@ func (r *BunStorageUploadRepo) ReserveCommitAttempt( return err } } - if copyRow.CommitAttemptID != nil && *copyRow.CommitAttemptID != "" { + if copyRow.CommitAttemptID != nil { out.State = storagecommit.ReservationAcquired out.Copy = *copyRow return nil @@ -68,12 +66,9 @@ func (r *BunStorageUploadRepo) ReserveCommitAttempt( return err } if active >= storagecommit.MaxActiveAttemptsPerDataSet { - // Say why capacity is gone. The count is diagnostic only, never an - // admission input, so it does not need to agree with the count above - // under a concurrent write. - held, err := countCommitAttentionAttemptsForDataSet(ctx, db, input.Copy.StorageDataSetID) - if err != nil { - return err + held, countErr := countCommitAttentionAttemptsForDataSet(ctx, db, input.Copy.StorageDataSetID) + if countErr != nil { + return countErr } out.State = storagecommit.ReservationWaiting out.Copy = *copyRow @@ -82,12 +77,17 @@ func (r *BunStorageUploadRepo) ReserveCommitAttempt( } var headID int64 err = db.NewSelect(). - Model((*model.StorageUploadCopy)(nil)). + Model((*model.StorageCopy)(nil)). Column("id"). Where("storage_data_set_id = ?", input.Copy.StorageDataSetID). - Where("status = ?", model.StorageUploadCopyStatusPieceReady). - Where("commit_attempt_id IS NULL"). + Where("status = ?", model.StorageCopyStatusPieceReady). Where("commit_ready_at IS NOT NULL"). + Where(`NOT EXISTS ( + SELECT 1 FROM storage_commit_attempts AS unresolved_attempt + WHERE unresolved_attempt.content_id = storage_copy.content_id + AND unresolved_attempt.storage_data_set_id = storage_copy.storage_data_set_id + AND unresolved_attempt.resolved_at IS NULL + )`). OrderExpr("commit_ready_at ASC"). OrderExpr("id ASC"). Limit(1). @@ -100,21 +100,20 @@ func (r *BunStorageUploadRepo) ReserveCommitAttempt( out.Copy = *copyRow return nil } - res, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("commit_attempt_id = ?", input.AttemptID). - Set("updated_at = ?", now). - Where("id = ?", copyID). - Where("status = ?", model.StorageUploadCopyStatusPieceReady). - Where("commit_attempt_id IS NULL"). - Exec(ctx) - if err != nil { + attempt := &storagecommit.Attempt{ + AttemptID: input.AttemptID, + ContentID: copyRow.ContentID, + StorageDataSetID: copyRow.StorageDataSetID, + Status: storagecommit.AttemptStatusReserved, + CreatedAt: now, + UpdatedAt: now, + } + if _, err := db.NewInsert().Model(attempt).Exec(ctx); err != nil { + if isUniqueViolation(err) { + return fmt.Errorf("reserving storage commit attempt: %w", ErrConflict) + } return fmt.Errorf("reserving storage commit attempt: %w", err) } - rows, _ := res.RowsAffected() - if rows != 1 { - return fmt.Errorf("reserving storage commit attempt: %w", ErrConflict) - } copyRow, err = loadCommitCopy(ctx, db, copyID, input.Copy.StorageDataSetID) if err != nil { return err @@ -126,7 +125,7 @@ func (r *BunStorageUploadRepo) ReserveCommitAttempt( return out, err } -func (r *BunStorageUploadRepo) MarkCommitAttempted( +func (r *BunStorageContentRepo) MarkCommitAttempted( ctx context.Context, input storagecommit.AttemptInput, ) (storagecommit.AttemptResult, error) { @@ -140,36 +139,63 @@ func (r *BunStorageUploadRepo) MarkCommitAttempted( if err != nil { return err } - copyRow, err := loadCommitCopy(ctx, db, copyID, input.Copy.StorageDataSetID) + attempt, err := loadCommitAttempt(ctx, db, input.Copy, input.AttemptID) if err != nil { return err } - if copyRow.CommitAttemptID == nil || *copyRow.CommitAttemptID != input.AttemptID { - return fmt.Errorf("marking storage commit attempted with stale token: %w", ErrConflict) - } - if copyRow.CommitAttemptedAt != nil { + if attempt.Status == storagecommit.AttemptStatusAttempted { + copyRow, loadErr := loadCommitCopy(ctx, db, copyID, input.Copy.StorageDataSetID) + if loadErr != nil { + return loadErr + } + if attempt.ExtraDataHex == nil || *attempt.ExtraDataHex != input.ExtraDataHex || + copyRow.Status != model.StorageCopyStatusCommitting || + copyRow.CommitExtraDataHex == nil || *copyRow.CommitExtraDataHex != input.ExtraDataHex { + return fmt.Errorf("marking storage commit attempted with conflicting evidence: %w", ErrConflict) + } out.Copy = *copyRow return nil } + if attempt.Status != storagecommit.AttemptStatusReserved { + return fmt.Errorf("marking storage commit attempted with resolved token: %w", ErrConflict) + } + copyRow, err := loadCommitCopy(ctx, db, copyID, input.Copy.StorageDataSetID) + if err != nil { + return err + } + if copyRow.Status != model.StorageCopyStatusPieceReady || + (copyRow.CommitExtraDataHex != nil && *copyRow.CommitExtraDataHex != input.ExtraDataHex) { + return fmt.Errorf("marking storage commit attempted for incompatible copy: %w", ErrConflict) + } res, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("status = ?", model.StorageUploadCopyStatusCommitting). - Set("commit_extra_data_hex = COALESCE(commit_extra_data_hex, ?)", input.ExtraDataHex). - Set("commit_attempted_at = ?", now). + Model((*storagecommit.Attempt)(nil)). + Set("status = ?", storagecommit.AttemptStatusAttempted). + Set("extra_data_hex = ?", input.ExtraDataHex). + Set("attempted_at = ?", now). Set("updated_at = ?", now). - Where("id = ?", copyID). - Where("status = ?", model.StorageUploadCopyStatusPieceReady). - Where("commit_attempt_id = ?", input.AttemptID). - Where("(commit_extra_data_hex IS NULL OR commit_extra_data_hex = '' OR commit_extra_data_hex = ?)", input.ExtraDataHex). - Where("commit_attempted_at IS NULL"). + Where("attempt_id = ?", input.AttemptID). + Where("content_id = ? AND storage_data_set_id = ?", input.Copy.ContentID, input.Copy.StorageDataSetID). + Where("status = ? AND resolved_at IS NULL", storagecommit.AttemptStatusReserved). Exec(ctx) if err != nil { return fmt.Errorf("marking storage commit attempted: %w", err) } - rows, _ := res.RowsAffected() - if rows != 1 { + if rows, _ := res.RowsAffected(); rows != 1 { return fmt.Errorf("marking storage commit attempted: %w", ErrConflict) } + res, err = db.NewUpdate(). + Model((*model.StorageCopy)(nil)). + Set("status = ?", model.StorageCopyStatusCommitting). + Set("commit_extra_data_hex = COALESCE(commit_extra_data_hex, ?)", input.ExtraDataHex). + Set("updated_at = ?", now). + Where("id = ? AND status = ?", copyID, model.StorageCopyStatusPieceReady). + Exec(ctx) + if err != nil { + return fmt.Errorf("projecting storage commit attempt: %w", err) + } + if rows, _ := res.RowsAffected(); rows != 1 { + return fmt.Errorf("projecting storage commit attempt: %w", ErrConflict) + } copyRow, err = loadCommitCopy(ctx, db, copyID, input.Copy.StorageDataSetID) if err != nil { return err @@ -181,20 +207,19 @@ func (r *BunStorageUploadRepo) MarkCommitAttempted( return out, err } -func (r *BunStorageUploadRepo) RecordCommitTransaction(ctx context.Context, input storagecommit.EvidenceInput) error { +func (r *BunStorageContentRepo) RecordCommitTransaction(ctx context.Context, input storagecommit.EvidenceInput) error { if err := validateCommitEvidenceInput(input, false); err != nil { return err } - return r.mutateAttempt(ctx, input.Copy, "recording storage commit transaction", func(db bun.IDB, copyID int64) error { + return r.mutateAttempt(ctx, input.Copy, "recording storage commit transaction", func(db bun.IDB, _ int64) error { res, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("commit_transaction_id = COALESCE(commit_transaction_id, ?)", input.TransactionID). + Model((*storagecommit.Attempt)(nil)). + Set("transaction_id = COALESCE(transaction_id, ?)", input.TransactionID). Set("updated_at = ?", commitInputTime(input.Now)). - Where("id = ?", copyID). - Where("status = ?", model.StorageUploadCopyStatusCommitting). - Where("commit_attempt_id = ?", input.AttemptID). - Where("commit_attempted_at IS NOT NULL"). - Where("(commit_transaction_id IS NULL OR commit_transaction_id = '' OR commit_transaction_id = ?)", input.TransactionID). + Where("attempt_id = ?", input.AttemptID). + Where("content_id = ? AND storage_data_set_id = ?", input.Copy.ContentID, input.Copy.StorageDataSetID). + Where("status = ? AND resolved_at IS NULL", storagecommit.AttemptStatusAttempted). + Where("(transaction_id IS NULL OR transaction_id = ?)", input.TransactionID). Exec(ctx) if err != nil { return err @@ -206,22 +231,21 @@ func (r *BunStorageUploadRepo) RecordCommitTransaction(ctx context.Context, inpu }) } -func (r *BunStorageUploadRepo) RecordCommitSubmission(ctx context.Context, input storagecommit.EvidenceInput) error { +func (r *BunStorageContentRepo) RecordCommitSubmission(ctx context.Context, input storagecommit.EvidenceInput) error { if err := validateCommitEvidenceInput(input, true); err != nil { return err } - return r.mutateAttempt(ctx, input.Copy, "recording storage commit submission", func(db bun.IDB, copyID int64) error { + return r.mutateAttempt(ctx, input.Copy, "recording storage commit submission", func(db bun.IDB, _ int64) error { res, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("commit_transaction_id = COALESCE(commit_transaction_id, ?)", input.TransactionID). - Set("commit_submission_json = ?", input.SubmissionJSON). + Model((*storagecommit.Attempt)(nil)). + Set("transaction_id = COALESCE(transaction_id, ?)", input.TransactionID). + Set("submission_json = COALESCE(submission_json, ?)", input.SubmissionJSON). Set("updated_at = ?", commitInputTime(input.Now)). - Where("id = ?", copyID). - Where("status = ?", model.StorageUploadCopyStatusCommitting). - Where("commit_attempt_id = ?", input.AttemptID). - Where("commit_attempted_at IS NOT NULL"). - Where("(commit_transaction_id IS NULL OR commit_transaction_id = '' OR commit_transaction_id = ?)", input.TransactionID). - Where("(commit_submission_json IS NULL OR commit_submission_json = '' OR commit_submission_json = ?)", input.SubmissionJSON). + Where("attempt_id = ?", input.AttemptID). + Where("content_id = ? AND storage_data_set_id = ?", input.Copy.ContentID, input.Copy.StorageDataSetID). + Where("status = ? AND resolved_at IS NULL", storagecommit.AttemptStatusAttempted). + Where("(transaction_id IS NULL OR transaction_id = ?)", input.TransactionID). + Where("(submission_json IS NULL OR submission_json = ?)", input.SubmissionJSON). Exec(ctx) if err != nil { return err @@ -233,20 +257,21 @@ func (r *BunStorageUploadRepo) RecordCommitSubmission(ctx context.Context, input }) } -func (r *BunStorageUploadRepo) MarkCommitAttention(ctx context.Context, input storagecommit.AttentionInput) error { +func (r *BunStorageContentRepo) MarkCommitAttention(ctx context.Context, input storagecommit.AttentionInput) error { if err := validateCommitCopyIdentity(input.Copy); err != nil || input.AttemptID == "" || !input.Code.Valid() { return fmt.Errorf("marking storage commit attention: %w", ErrInvalidInput) } - return r.mutateAttempt(ctx, input.Copy, "marking storage commit attention", func(db bun.IDB, copyID int64) error { + return r.mutateAttempt(ctx, input.Copy, "marking storage commit attention", func(db bun.IDB, _ int64) error { now := commitInputTime(input.Now) res, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("commit_attention_code = COALESCE(commit_attention_code, ?)", string(input.Code)). - Set("commit_attention_at = COALESCE(commit_attention_at, ?)", now). + Model((*storagecommit.Attempt)(nil)). + Set("attention_code = COALESCE(attention_code, ?)", string(input.Code)). + Set("attention_at = COALESCE(attention_at, ?)", now). Set("updated_at = ?", now). - Where("id = ?", copyID). - Where("commit_attempt_id = ?", input.AttemptID). - Where("commit_attempted_at IS NOT NULL"). + Where("attempt_id = ?", input.AttemptID). + Where("content_id = ? AND storage_data_set_id = ?", input.Copy.ContentID, input.Copy.StorageDataSetID). + Where("status = ? AND resolved_at IS NULL", storagecommit.AttemptStatusAttempted). + Where("(attention_code IS NULL OR attention_code = ?)", string(input.Code)). Exec(ctx) if err != nil { return err @@ -258,27 +283,21 @@ func (r *BunStorageUploadRepo) MarkCommitAttention(ctx context.Context, input st }) } -func (r *BunStorageUploadRepo) ResetCommitAttempt(ctx context.Context, input storagecommit.ResetInput) error { - if err := validateCommitCopyIdentity(input.Copy); err != nil || input.AttemptID == "" { +func (r *BunStorageContentRepo) ResetCommitAttempt(ctx context.Context, input storagecommit.ResetInput) error { + if err := validateCommitCopyIdentity(input.Copy); err != nil || input.AttemptID == "" || input.LastError == "" { return fmt.Errorf("resetting storage commit attempt: %w", ErrInvalidInput) } + now := commitInputTime(input.Now) return r.mutateAttempt(ctx, input.Copy, "resetting storage commit attempt", func(db bun.IDB, copyID int64) error { res, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("status = ?", model.StorageUploadCopyStatusPieceReady). - Set("commit_ready_at = NULL"). - Set("commit_attempt_id = NULL"). - Set("commit_attempted_at = NULL"). - Set("commit_submission_json = NULL"). - Set("commit_extra_data_hex = NULL"). - Set("commit_transaction_id = NULL"). - Set("commit_confirmed_transaction_id = NULL"). - Set("commit_attention_code = NULL"). - Set("commit_attention_at = NULL"). + Model((*storagecommit.Attempt)(nil)). + Set("status = ?", storagecommit.AttemptStatusRejected). Set("last_error = ?", nullableString(input.LastError)). - Set("updated_at = ?", commitInputTime(input.Now)). - Where("id = ?", copyID). - Where("commit_attempt_id = ?", input.AttemptID). + Set("resolved_at = ?", now). + Set("updated_at = ?", now). + Where("attempt_id = ?", input.AttemptID). + Where("content_id = ? AND storage_data_set_id = ?", input.Copy.ContentID, input.Copy.StorageDataSetID). + Where("status = ? AND resolved_at IS NULL", storagecommit.AttemptStatusAttempted). Exec(ctx) if err != nil { return err @@ -286,41 +305,34 @@ func (r *BunStorageUploadRepo) ResetCommitAttempt(ctx context.Context, input sto if rows, _ := res.RowsAffected(); rows != 1 { return ErrConflict } - return nil + return projectResolvedCommitAttempt(ctx, db, copyID, now, true, true, nullableString(input.LastError)) }) } -func (r *BunStorageUploadRepo) ReleaseCommitAttempt(ctx context.Context, input storagecommit.ReleaseInput) error { - if err := validateCommitCopyIdentity(input.Copy); err != nil || input.AttemptID == "" { +func (r *BunStorageContentRepo) ReleaseCommitAttempt(ctx context.Context, input storagecommit.ReleaseInput) error { + if err := validateCommitCopyIdentity(input.Copy); err != nil || input.AttemptID == "" || !input.Reason.Valid() { return fmt.Errorf("releasing storage commit attempt: %w", ErrInvalidInput) } + now := commitInputTime(input.Now) return r.mutateAttempt(ctx, input.Copy, "releasing storage commit attempt", func(db bun.IDB, copyID int64) error { q := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("status = CASE WHEN status = ? THEN ? ELSE status END", model.StorageUploadCopyStatusCommitting, model.StorageUploadCopyStatusPieceReady). - Set("commit_attempt_id = NULL"). - Set("commit_attempted_at = NULL"). - Set("commit_submission_json = NULL"). - Set("commit_transaction_id = NULL"). - Set("commit_confirmed_transaction_id = NULL"). - Set("commit_attention_code = NULL"). - Set("commit_attention_at = NULL"). - Set("last_error = NULL"). - Set("updated_at = ?", commitInputTime(input.Now)). - Where("id = ?", copyID). - Where("commit_attempt_id = ?", input.AttemptID) - if input.ClearReadyAt { - q = q.Set("commit_ready_at = NULL") - } - if input.ClearExtraData { - q = q.Set("commit_extra_data_hex = NULL") - } + Model((*storagecommit.Attempt)(nil)). + Set("status = ?", storagecommit.AttemptStatusReleased). + Set("release_reason = ?", string(input.Reason)). + Set("resolved_at = ?", now). + Set("updated_at = ?", now). + Where("attempt_id = ?", input.AttemptID). + Where("content_id = ? AND storage_data_set_id = ?", input.Copy.ContentID, input.Copy.StorageDataSetID). + Where("resolved_at IS NULL") if input.KnownNotSubmitted { q = q. - Where("commit_transaction_id IS NULL"). - Where("commit_submission_json IS NULL") + Where("status IN (?)", bun.List([]storagecommit.AttemptStatus{ + storagecommit.AttemptStatusReserved, + storagecommit.AttemptStatusAttempted, + })). + Where("transaction_id IS NULL AND submission_json IS NULL") } else { - q = q.Where("commit_attempted_at IS NULL") + q = q.Where("status = ?", storagecommit.AttemptStatusReserved) } res, err := q.Exec(ctx) if err != nil { @@ -329,11 +341,11 @@ func (r *BunStorageUploadRepo) ReleaseCommitAttempt(ctx context.Context, input s if rows, _ := res.RowsAffected(); rows != 1 { return ErrConflict } - return nil + return projectResolvedCommitAttempt(ctx, db, copyID, now, input.ClearReadyAt, input.ClearExtraData, nil) }) } -func (r *BunStorageUploadRepo) ReleaseCommitReservation( +func (r *BunStorageContentRepo) ReleaseCommitReservation( ctx context.Context, input storagecommit.ReservationReleaseInput, ) error { @@ -342,12 +354,16 @@ func (r *BunStorageUploadRepo) ReleaseCommitReservation( } return r.mutateAttempt(ctx, input.Copy, "releasing storage commit reservation", func(db bun.IDB, copyID int64) error { q := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). + Model((*model.StorageCopy)(nil)). Set("updated_at = ?", commitInputTime(input.Now)). Where("id = ?", copyID). - Where("status = ?", model.StorageUploadCopyStatusPieceReady). - Where("commit_attempt_id IS NULL"). - Where("commit_attempted_at IS NULL") + Where("status = ?", model.StorageCopyStatusPieceReady). + Where(`NOT EXISTS ( + SELECT 1 FROM storage_commit_attempts AS unresolved_attempt + WHERE unresolved_attempt.content_id = storage_copy.content_id + AND unresolved_attempt.storage_data_set_id = storage_copy.storage_data_set_id + AND unresolved_attempt.resolved_at IS NULL + )`) if input.ClearReadyAt { q = q.Set("commit_ready_at = NULL") } @@ -365,7 +381,7 @@ func (r *BunStorageUploadRepo) ReleaseCommitReservation( }) } -func (r *BunStorageUploadRepo) CountActiveCommitAttemptsForDataSet(ctx context.Context, storageDataSetID int64) (int, error) { +func (r *BunStorageContentRepo) CountActiveCommitAttemptsForDataSet(ctx context.Context, storageDataSetID int64) (int, error) { if storageDataSetID <= 0 { return 0, fmt.Errorf("counting active storage commit attempts: %w", ErrInvalidInput) } @@ -374,9 +390,8 @@ func (r *BunStorageUploadRepo) CountActiveCommitAttemptsForDataSet(ctx context.C func countActiveCommitAttemptsForDataSet(ctx context.Context, db bun.IDB, storageDataSetID int64) (int, error) { count, err := db.NewSelect(). - Model((*model.StorageUploadCopy)(nil)). - Where("storage_data_set_id = ?", storageDataSetID). - Where("commit_attempt_id IS NOT NULL AND commit_attempt_id <> ''"). + Model((*storagecommit.Attempt)(nil)). + Where("storage_data_set_id = ? AND resolved_at IS NULL", storageDataSetID). Count(ctx) if err != nil { return 0, fmt.Errorf("counting active storage commit attempts: %w", err) @@ -384,22 +399,11 @@ func countActiveCommitAttemptsForDataSet(ctx context.Context, db bun.IDB, storag return count, nil } -// countCommitAttentionAttemptsForDataSet counts the active attempts already -// flagged for operator attention, which is what stops the data set's commit -// queue from being able to move on its own. -// -// Every flagged attempt counts, whatever its code. Whether a given flag can -// still settle without a person is not a property of the row: the same code is -// written both as a terminal hold and as one the advancer keeps observing, and -// which one it behaves as depends on the path that next reaches it. Classifying -// by code would silently drop terminal holds, so this reports the flag and the -// caller words its message to match. func countCommitAttentionAttemptsForDataSet(ctx context.Context, db bun.IDB, storageDataSetID int64) (int, error) { count, err := db.NewSelect(). - Model((*model.StorageUploadCopy)(nil)). - Where("storage_data_set_id = ?", storageDataSetID). - Where("commit_attempt_id IS NOT NULL AND commit_attempt_id <> ''"). - Where("commit_attention_at IS NOT NULL"). + Model((*storagecommit.Attempt)(nil)). + Where("storage_data_set_id = ? AND resolved_at IS NULL", storageDataSetID). + Where("attention_at IS NOT NULL"). Count(ctx) if err != nil { return 0, fmt.Errorf("counting storage commit attempts held for attention: %w", err) @@ -407,10 +411,10 @@ func countCommitAttentionAttemptsForDataSet(ctx context.Context, db bun.IDB, sto return count, nil } -func (r *BunStorageUploadRepo) ListCommitAttention(ctx context.Context, limit int) ([]storagecommit.AttentionRecord, error) { +func (r *BunStorageContentRepo) ListCommitAttention(ctx context.Context, limit int) ([]storagecommit.AttentionRecord, error) { type attentionRow struct { CopyID int64 `bun:"copy_id"` - UploadID int64 `bun:"upload_id"` + ContentID int64 `bun:"content_id"` CopyIndex int `bun:"copy_index"` DataSetRowID int64 `bun:"data_set_row_id"` ProviderID string `bun:"provider_id"` @@ -424,24 +428,25 @@ func (r *BunStorageUploadRepo) ListCommitAttention(ctx context.Context, limit in } var rows []attentionRow q := r.db.NewSelect(). - TableExpr("storage_upload_copies AS storage_copy"). + TableExpr("storage_commit_attempts AS commit_attempt"). ColumnExpr("storage_copy.id AS copy_id"). - ColumnExpr("storage_copy.upload_id"). + ColumnExpr("commit_attempt.content_id"). ColumnExpr("storage_copy.copy_index"). - ColumnExpr("storage_copy.storage_data_set_id AS data_set_row_id"). + ColumnExpr("commit_attempt.storage_data_set_id AS data_set_row_id"). ColumnExpr("CAST(storage_data_set.provider_id AS TEXT) AS provider_id"). ColumnExpr("COALESCE(CAST(storage_data_set.data_set_id AS TEXT), '') AS data_set_id"). - ColumnExpr("COALESCE(storage_upload.piece_cid, '') AS piece_cid"). - ColumnExpr("storage_copy.commit_attempt_id AS attempt_id"). - ColumnExpr("COALESCE(storage_copy.commit_transaction_id, '') AS transaction_id"). - ColumnExpr("storage_copy.commit_attention_code AS attention_code"). - ColumnExpr("storage_copy.commit_attempted_at AS attempted_at"). - ColumnExpr("storage_copy.commit_attention_at AS attention_at"). - Join("JOIN storage_uploads AS storage_upload ON storage_upload.id = storage_copy.upload_id"). - Join("JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id"). - Where("storage_copy.commit_attention_at IS NOT NULL"). - Where("storage_copy.commit_attempt_id IS NOT NULL AND storage_copy.commit_attempt_id <> ''"). - OrderExpr("storage_copy.commit_attention_at ASC"). + ColumnExpr("COALESCE(storage_content.piece_cid, '') AS piece_cid"). + ColumnExpr("commit_attempt.attempt_id"). + ColumnExpr("COALESCE(commit_attempt.transaction_id, '') AS transaction_id"). + ColumnExpr("commit_attempt.attention_code"). + ColumnExpr("commit_attempt.attempted_at"). + ColumnExpr("commit_attempt.attention_at"). + Join("JOIN storage_copies AS storage_copy ON storage_copy.content_id = commit_attempt.content_id AND storage_copy.storage_data_set_id = commit_attempt.storage_data_set_id"). + Join("JOIN storage_contents AS storage_content ON storage_content.id = commit_attempt.content_id"). + Join("JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = commit_attempt.storage_data_set_id"). + Where("commit_attempt.status = ? AND commit_attempt.resolved_at IS NULL", storagecommit.AttemptStatusAttempted). + Where("commit_attempt.attention_at IS NOT NULL"). + OrderExpr("commit_attempt.attention_at ASC"). OrderExpr("storage_copy.id ASC") if limit > 0 { q = q.Limit(limit) @@ -451,69 +456,49 @@ func (r *BunStorageUploadRepo) ListCommitAttention(ctx context.Context, limit in } out := make([]storagecommit.AttentionRecord, 0, len(rows)) for _, row := range rows { - code, err := storagecommit.ParseAttentionCode(row.Code) - if err != nil { - code = storagecommit.AttentionCode(row.Code) - } out = append(out, storagecommit.AttentionRecord{ - CopyID: row.CopyID, UploadID: row.UploadID, CopyIndex: row.CopyIndex, + CopyID: row.CopyID, ContentID: row.ContentID, CopyIndex: row.CopyIndex, DataSetRowID: row.DataSetRowID, ProviderID: row.ProviderID, DataSetID: row.DataSetID, PieceCID: row.PieceCID, AttemptID: row.AttemptID, TransactionID: row.TransactionID, - Code: code, AttemptedAt: row.AttemptedAt, AttentionAt: row.AttentionAt, + Code: storagecommit.AttentionCode(row.Code), AttemptedAt: row.AttemptedAt, AttentionAt: row.AttentionAt, }) } return out, nil } -func (r *BunStorageUploadRepo) ReleaseCommitAttention(ctx context.Context, input storagecommit.ManualReleaseInput) error { +func (r *BunStorageContentRepo) ReleaseCommitAttention(ctx context.Context, input storagecommit.ManualReleaseInput) error { if input.CopyID <= 0 || input.ExpectedAttemptID == "" || !input.AcknowledgePossibleDuplicate { return fmt.Errorf("releasing storage confirmation attention: %w", ErrInvalidInput) } now := commitInputTime(input.Now) return r.runMaybeTx(ctx, func(db bun.IDB) error { - initial := new(model.StorageUploadCopy) + initial := new(model.StorageCopy) if err := db.NewSelect().Model(initial).Where("id = ?", input.CopyID).Scan(ctx); err != nil { if errors.Is(err, sql.ErrNoRows) { return ErrNotFound } return err } - if initial.StorageDataSetID == nil { - return ErrConflict - } identity := storagecommit.CopyIdentity{ - StorageUploadCopyID: initial.ID, UploadID: initial.UploadID, CopyIndex: initial.CopyIndex, - StorageDataSetID: *initial.StorageDataSetID, + StorageCopyID: initial.ID, + ContentID: initial.ContentID, + CopyIndex: initial.CopyIndex, + StorageDataSetID: initial.StorageDataSetID, } copyID, _, err := lockCommitCopyFamily(ctx, db, identity) if err != nil { return err } - copyRow, err := loadCommitCopy(ctx, db, copyID, *initial.StorageDataSetID) - if err != nil { - return err - } - if copyRow.CommitAttemptID == nil || *copyRow.CommitAttemptID != input.ExpectedAttemptID || copyRow.CommitAttentionAt == nil { - return ErrConflict - } - attemptID := input.ExpectedAttemptID res, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("status = ?", model.StorageUploadCopyStatusPieceReady). - Set("commit_ready_at = NULL"). - Set("commit_attempt_id = NULL"). - Set("commit_attempted_at = NULL"). - Set("commit_submission_json = NULL"). - Set("commit_extra_data_hex = NULL"). - Set("commit_transaction_id = NULL"). - Set("commit_confirmed_transaction_id = NULL"). - Set("commit_attention_code = NULL"). - Set("commit_attention_at = NULL"). - Set("last_error = NULL"). + Model((*storagecommit.Attempt)(nil)). + Set("status = ?", storagecommit.AttemptStatusReleased). + Set("release_reason = ?", string(storagecommit.ReleaseManualDuplicateAck)). + Set("resolved_at = ?", now). Set("updated_at = ?", now). - Where("id = ?", copyID). - Where("commit_attempt_id = ?", attemptID). - Where("commit_attention_at IS NOT NULL"). + Where("attempt_id = ?", input.ExpectedAttemptID). + Where("content_id = ? AND storage_data_set_id = ?", initial.ContentID, initial.StorageDataSetID). + Where("status = ? AND resolved_at IS NULL", storagecommit.AttemptStatusAttempted). + Where("attention_at IS NOT NULL"). Exec(ctx) if err != nil { return fmt.Errorf("releasing storage confirmation attention: %w", err) @@ -521,96 +506,39 @@ func (r *BunStorageUploadRepo) ReleaseCommitAttention(ctx context.Context, input if rows, _ := res.RowsAffected(); rows != 1 { return ErrConflict } - copyIDExpr := taskPayloadCopyIDSQL(db.Dialect().Name()) - uploadIDExpr, copyIndexExpr := runningUploadCopyTaskPayloadExpressions(db.Dialect().Name()) - copyTaskMatch := fmt.Sprintf( - "(%[1]s = ? OR ((%[1]s IS NULL OR %[1]s = 0) AND %[2]s = ? AND %[3]s = ?))", - copyIDExpr("task"), uploadIDExpr, copyIndexExpr, - ) - _, err = db.NewUpdate(). - Model((*model.Task)(nil)). - Set("status = ?", model.TaskStatusScheduled). - Set("scheduled_at = ?", now). - Set("last_error = NULL"). - Set("status_message = NULL"). - Set("wait_reason = NULL"). - Set("completed_at = NULL"). - Where("type = ?", model.TaskTypeUpload). - Where("status IN (?, ?, ?)", model.TaskStatusQueued, model.TaskStatusScheduled, model.TaskStatusWaiting). - Where(copyTaskMatch, copyID, copyRow.UploadID, copyRow.CopyIndex). - Exec(ctx) - if err != nil { - return fmt.Errorf("waking storage confirmation task: %w", err) - } - _, err = db.NewUpdate(). - Model((*storagereplacement.Item)(nil)). - Set("status = ?", storagereplacement.ItemStatusCancelled). - Set("scheduled_at = ?", now). - Set("last_error = NULL"). - Where("target_copy_id = ?", copyID). - Where("status <> ?", storagereplacement.ItemStatusCopied). - Where("claimed_at IS NULL"). - Where(`EXISTS ( - SELECT 1 FROM storage_replacements AS owner_replacement - WHERE owner_replacement.id = storage_replacement_item.replacement_id - AND owner_replacement.status IN (?, ?) - )`, storagereplacement.StatusCompleted, storagereplacement.StatusSuperseded). - Exec(ctx) - if err != nil { - return fmt.Errorf("settling terminal replacement confirmation item: %w", err) - } - _, err = db.NewUpdate(). - Model((*storagereplacement.Item)(nil)). - Set("status = ?", storagereplacement.ItemStatusFailed). - Set("scheduled_at = ?", now). - Set("last_error = NULL"). - Where("target_copy_id = ?", copyID). - Where("status <> ?", storagereplacement.ItemStatusCopied). - Where("claimed_at IS NULL"). - Where(`EXISTS ( - SELECT 1 FROM storage_replacements AS owner_replacement - WHERE owner_replacement.id = storage_replacement_item.replacement_id - AND owner_replacement.status IN (?, ?) - )`, storagereplacement.StatusFailed, storagereplacement.StatusCleanupAttention). - Exec(ctx) - if err != nil { - return fmt.Errorf("holding failed replacement confirmation item: %w", err) - } - _, err = db.NewUpdate(). - Model((*storagereplacement.Item)(nil)). - Set("status = ?", storagereplacement.ItemStatusPending). - Set("scheduled_at = ?", now). - Set("last_error = NULL"). - Where("target_copy_id = ?", copyID). - Where("status <> ?", storagereplacement.ItemStatusCopied). - Where("claimed_at IS NULL"). - Where(`EXISTS ( - SELECT 1 FROM storage_replacements AS owner_replacement - WHERE owner_replacement.id = storage_replacement_item.replacement_id - AND owner_replacement.status IN (?, ?, ?, ?) - )`, storagereplacement.StatusPreparingTarget, storagereplacement.StatusMigrating, - storagereplacement.StatusWaiting, storagereplacement.StatusRetiring). - Exec(ctx) - if err != nil { - return fmt.Errorf("waking replacement confirmation item: %w", err) - } - claimedItems, err := db.NewSelect(). - Model((*storagereplacement.Item)(nil)). - Where("target_copy_id = ?", copyID). - Where("status NOT IN (?, ?)", storagereplacement.ItemStatusCopied, storagereplacement.ItemStatusCancelled). - Where("claimed_at IS NOT NULL"). - Count(ctx) - if err != nil { - return fmt.Errorf("checking claimed replacement confirmation item: %w", err) + if err := projectResolvedCommitAttempt(ctx, db, copyID, now, true, true, nil); err != nil { + return fmt.Errorf("releasing storage confirmation attention: %w", err) } - if claimedItems > 0 { - return fmt.Errorf("releasing storage confirmation attention while replacement work is claimed: %w", ErrConflict) + return resumeCommitTaskAfterAttentionRelease(ctx, db, initial.ActiveTaskID) + }) +} + +func resumeCommitTaskAfterAttentionRelease(ctx context.Context, db bun.IDB, taskID *int64) error { + if taskID == nil { + return nil + } + tasks := &BunTaskRepo{db: db} + taskRow, err := tasks.GetByID(ctx, *taskID) + if err != nil { + return fmt.Errorf("loading released storage commit task: %w", err) + } + if taskRow == nil || taskRow.Type != model.TaskTypeStorageCommit { + return fmt.Errorf("released storage commit has no matching task: %w", ErrConflict) + } + switch taskRow.Status { + case model.TaskStatusPending, model.TaskStatusRunning: + return nil + case model.TaskStatusFailed: + if err := tasks.RetryFailed(ctx, taskRow.ID); err != nil { + return fmt.Errorf("resuming released storage commit task: %w", err) } return nil - }) + default: + return fmt.Errorf("released storage commit task is terminal: %w", ErrConflict) + } } -func (r *BunStorageUploadRepo) mutateAttempt( +func (r *BunStorageContentRepo) mutateAttempt( ctx context.Context, copyIdentity storagecommit.CopyIdentity, op string, @@ -630,11 +558,11 @@ func (r *BunStorageUploadRepo) mutateAttempt( } func lockCommitCopyFamily(ctx context.Context, db bun.IDB, identity storagecommit.CopyIdentity) (int64, *model.StorageDataSet, error) { - uploads, err := lockStorageUploadsByID(ctx, db, []int64{identity.UploadID}) + uploads, err := lockStorageContentsByID(ctx, db, []int64{identity.ContentID}) if err != nil { return 0, nil, fmt.Errorf("locking storage upload for commit: %w", err) } - if uploads[identity.UploadID] == nil { + if uploads[identity.ContentID] == nil { return 0, nil, fmt.Errorf("locking storage upload for commit: %w", ErrNotFound) } dataSet := new(model.StorageDataSet) @@ -648,14 +576,14 @@ func lockCommitCopyFamily(ctx context.Context, db bun.IDB, identity storagecommi } return 0, nil, fmt.Errorf("locking storage data set for commit: %w", err) } - copyID, err := slotCopyTarget(ctx, db, identity.StorageUploadCopyID, identity.UploadID, identity.CopyIndex) + copyID, err := slotCopyTarget(ctx, db, identity.StorageCopyID, identity.ContentID, identity.CopyIndex) if err != nil { return 0, nil, err } count, err := db.NewSelect(). - Model((*model.StorageUploadCopy)(nil)). + Model((*model.StorageCopy)(nil)). Where("id = ?", copyID). - Where("storage_data_set_id = ?", identity.StorageDataSetID). + Where("content_id = ? AND storage_data_set_id = ?", identity.ContentID, identity.StorageDataSetID). Count(ctx) if err != nil { return 0, nil, fmt.Errorf("validating storage commit copy data set: %w", err) @@ -665,10 +593,10 @@ func lockCommitCopyFamily(ctx context.Context, db bun.IDB, identity storagecommi } if identity.RequireEligibleCopy { count, err = db.NewSelect(). - Model((*model.StorageUploadCopy)(nil)). + Model((*model.StorageCopy)(nil)). Where("id = ?", copyID). - Where("status <> ?", model.StorageUploadCopyStatusFailed). - Where(liveObjectVersionExistsForUploadSQL(), identity.UploadID, false). + Where("status <> ?", model.StorageCopyStatusFailed). + Where(liveObjectVersionExistsForUploadSQL(), identity.ContentID, false). Count(ctx) if err != nil { return 0, nil, fmt.Errorf("checking storage commit copy eligibility: %w", err) @@ -680,12 +608,13 @@ func lockCommitCopyFamily(ctx context.Context, db bun.IDB, identity storagecommi return copyID, dataSet, nil } -func loadCommitCopy(ctx context.Context, db bun.IDB, copyID, storageDataSetID int64) (*model.StorageUploadCopy, error) { - copyRow := new(model.StorageUploadCopy) - err := db.NewSelect(). - Model(copyRow). - Where("id = ?", copyID). - Where("storage_data_set_id = ?", storageDataSetID). +func loadCommitCopy(ctx context.Context, db bun.IDB, copyID, storageDataSetID int64) (*model.StorageCopy, error) { + copyRow := new(model.StorageCopy) + q := db.NewSelect().Model(copyRow) + projectActiveCommitAttempt(q, "storage_copy") + err := q. + Where("storage_copy.id = ?", copyID). + Where("storage_copy.storage_data_set_id = ?", storageDataSetID). Scan(ctx) if err != nil { if errors.Is(err, sql.ErrNoRows) { @@ -696,8 +625,72 @@ func loadCommitCopy(ctx context.Context, db bun.IDB, copyID, storageDataSetID in return copyRow, nil } +func projectActiveCommitAttempt(q *bun.SelectQuery, copyAlias string) { + q.ColumnExpr(copyAlias + ".*"). + ColumnExpr("active_commit_attempt.attempt_id AS commit_attempt_id"). + ColumnExpr("active_commit_attempt.attempted_at AS commit_attempted_at"). + ColumnExpr("active_commit_attempt.transaction_id AS commit_transaction_id"). + ColumnExpr("active_commit_attempt.submission_json AS commit_submission_json"). + ColumnExpr("active_commit_attempt.confirmed_transaction_id AS commit_confirmed_transaction_id"). + ColumnExpr("active_commit_attempt.attention_code AS commit_attention_code"). + ColumnExpr("active_commit_attempt.attention_at AS commit_attention_at"). + Join("LEFT JOIN storage_commit_attempts AS active_commit_attempt ON active_commit_attempt.content_id = " + copyAlias + ".content_id AND active_commit_attempt.storage_data_set_id = " + copyAlias + ".storage_data_set_id AND active_commit_attempt.resolved_at IS NULL") +} + +func loadCommitAttempt( + ctx context.Context, + db bun.IDB, + identity storagecommit.CopyIdentity, + attemptID string, +) (*storagecommit.Attempt, error) { + attempt := new(storagecommit.Attempt) + err := db.NewSelect(). + Model(attempt). + Where("attempt_id = ?", attemptID). + Where("content_id = ? AND storage_data_set_id = ?", identity.ContentID, identity.StorageDataSetID). + Scan(ctx) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrConflict + } + return nil, fmt.Errorf("loading storage commit attempt: %w", err) + } + return attempt, nil +} + +func projectResolvedCommitAttempt( + ctx context.Context, + db bun.IDB, + copyID int64, + now time.Time, + clearReadyAt bool, + clearExtraData bool, + lastError *string, +) error { + q := db.NewUpdate(). + Model((*model.StorageCopy)(nil)). + Set("status = CASE WHEN status = ? THEN ? ELSE status END", model.StorageCopyStatusCommitting, model.StorageCopyStatusPieceReady). + Set("last_error = ?", lastError). + Set("updated_at = ?", now). + Where("id = ?", copyID) + if clearReadyAt { + q = q.Set("commit_ready_at = NULL") + } + if clearExtraData { + q = q.Set("commit_extra_data_hex = NULL") + } + res, err := q.Exec(ctx) + if err != nil { + return err + } + if rows, _ := res.RowsAffected(); rows != 1 { + return ErrConflict + } + return nil +} + func validateCommitCopyIdentity(identity storagecommit.CopyIdentity) error { - if identity.StorageUploadCopyID <= 0 || identity.UploadID <= 0 || identity.CopyIndex < 0 || identity.StorageDataSetID <= 0 { + if identity.StorageCopyID <= 0 || identity.ContentID <= 0 || identity.CopyIndex < 0 || identity.StorageDataSetID <= 0 { return ErrInvalidInput } return nil @@ -721,8 +714,11 @@ func commitInputTime(value time.Time) time.Time { } func attemptedStorageCommitSQL(alias string) string { - return fmt.Sprintf( - "(%[1]s.commit_attempt_id IS NOT NULL AND %[1]s.commit_attempt_id <> '' AND %[1]s.commit_attempted_at IS NOT NULL)", - alias, - ) + return fmt.Sprintf(`EXISTS ( + SELECT 1 FROM storage_commit_attempts AS attempted_commit + WHERE attempted_commit.content_id = %[1]s.content_id + AND attempted_commit.storage_data_set_id = %[1]s.storage_data_set_id + AND attempted_commit.status = 'attempted' + AND attempted_commit.resolved_at IS NULL + )`, alias) } diff --git a/internal/db/repository/storage_commit_repo_test.go b/internal/db/repository/storage_commit_repo_test.go deleted file mode 100644 index 8e47683..0000000 --- a/internal/db/repository/storage_commit_repo_test.go +++ /dev/null @@ -1,748 +0,0 @@ -package repository_test - -import ( - "errors" - "fmt" - "sync" - "testing" - "time" - - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagecommit" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/uptrace/bun" -) - -func TestStorageCommitReservationsEnforceCapacityAndFIFO(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - bucket := seedBucket(t, db, "commit-capacity-bucket") - dataSet := seedCommitDataSet(t, db, bucket.ID) - copies := seedCommitCopies(t, db, bucket.ID, dataSet.ID, 5) - base := time.Now().Add(-time.Minute) - - for i := range 4 { - result, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: commitCopyIdentity(copies[i]), AttemptID: fmt.Sprintf("attempt-%d", i), - Now: base.Add(time.Duration(i) * time.Second), - }) - if err != nil || result.State != storagecommit.ReservationAcquired { - t.Fatalf("reserve copy %d = %#v err=%v, want acquired", i, result, err) - } - } - active, err := repos.Uploads.CountActiveCommitAttemptsForDataSet(t.Context(), dataSet.ID) - if err != nil || active != storagecommit.MaxActiveAttemptsPerDataSet { - t.Fatalf("active reservations = %d err=%v, want %d", active, err, storagecommit.MaxActiveAttemptsPerDataSet) - } - fifth, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: commitCopyIdentity(copies[4]), AttemptID: "attempt-4", Now: base.Add(4 * time.Second), - }) - if err != nil || fifth.State != storagecommit.ReservationWaiting || fifth.Copy.CommitAttemptID != nil { - t.Fatalf("fifth reservation = %#v err=%v, want waiting without attempt", fifth, err) - } - if fifth.Copy.CommitReadyAt == nil { - t.Fatal("fifth reservation has no FIFO timestamp") - } - readyAt := *fifth.Copy.CommitReadyAt - updatedAt := fifth.Copy.UpdatedAt - fifth, err = repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: commitCopyIdentity(copies[4]), AttemptID: "attempt-4", Now: base.Add(time.Hour), - }) - if err != nil || fifth.State != storagecommit.ReservationWaiting || fifth.Copy.CommitAttemptID != nil || - fifth.Copy.CommitReadyAt == nil || - !fifth.Copy.CommitReadyAt.Equal(readyAt) || !fifth.Copy.UpdatedAt.Equal(updatedAt) { - t.Fatalf("repeated fifth reservation = %#v err=%v, want unchanged FIFO row", fifth, err) - } - - if err := repos.Uploads.ReleaseCommitAttempt(t.Context(), storagecommit.ReleaseInput{ - Copy: commitCopyIdentity(copies[0]), AttemptID: "attempt-0", ClearReadyAt: true, - }); err != nil { - t.Fatalf("release first reservation: %v", err) - } - fifth, err = repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: commitCopyIdentity(copies[4]), AttemptID: "attempt-4", Now: base.Add(10 * time.Second), - }) - if err != nil || fifth.State != storagecommit.ReservationAcquired { - t.Fatalf("fifth reservation after release = %#v err=%v, want acquired", fifth, err) - } -} - -func TestStorageCommitReservationAllowsBoundDrainingGeneration(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - bucket := seedBucket(t, db, "commit-draining-generation-bucket") - dataSet := seedCommitDataSet(t, db, bucket.ID) - copyRow := seedCommitCopies(t, db, bucket.ID, dataSet.ID, 1)[0] - if _, err := db.NewUpdate().Model((*model.StorageDataSet)(nil)). - Set("is_current = ?", false). - Set("status = ?", model.StorageDataSetStatusDraining). - Where("id = ?", dataSet.ID). - Exec(t.Context()); err != nil { - t.Fatalf("drain data set: %v", err) - } - - result, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: commitCopyIdentity(copyRow), AttemptID: "draining-attempt", - }) - if err != nil || result.State != storagecommit.ReservationAcquired { - t.Fatalf("draining reservation = %#v err=%v, want acquired", result, err) - } -} - -func TestStorageCommitReservationReleaseRejectsCommittedCopy(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - bucket := seedBucket(t, db, "commit-release-committed-bucket") - dataSet := seedCommitDataSet(t, db, bucket.ID) - copyRow := seedCommitCopies(t, db, bucket.ID, dataSet.ID, 1)[0] - pieceID := onChainID(t, "2001") - if err := repos.Uploads.MarkUploadCopyCommitted(t.Context(), repository.MarkUploadCopyCommittedInput{ - StorageUploadCopyID: copyRow.ID, - UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, - PieceCID: "bafkqaaa", - PieceID: &pieceID, - RetrievalURL: "https://provider.example/piece", - CommitExtraDataHex: "abcd", - CommitTransactionID: "0xcommitted", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - - err := repos.Uploads.ReleaseCommitReservation(t.Context(), storagecommit.ReservationReleaseInput{ - Copy: commitCopyIdentity(copyRow), ClearReadyAt: true, ClearExtraData: true, - }) - if !errors.Is(err, repository.ErrConflict) { - t.Fatalf("release committed reservation error = %v, want conflict", err) - } - persisted, err := repos.Uploads.GetUploadCopyByID(t.Context(), copyRow.ID) - if err != nil || persisted == nil || persisted.Status != model.StorageUploadCopyStatusCommitted || - persisted.CommitExtraDataHex == nil || *persisted.CommitExtraDataHex != "abcd" || - persisted.CommitTransactionID == nil || *persisted.CommitTransactionID != "0xcommitted" { - t.Fatalf("committed evidence after reservation release = %#v err=%v", persisted, err) - } -} - -func TestSQLiteConcurrentStorageCommitReservationsRespectCapacity(t *testing.T) { - assertConcurrentStorageCommitReservationsRespectCapacity(t, concurrentTestDB(t)) -} - -func assertConcurrentStorageCommitReservationsRespectCapacity(t *testing.T, db *bun.DB) { - t.Helper() - repos := repository.NewRepositories(db) - bucket := seedBucket(t, db, "concurrent-commit-capacity-bucket") - dataSet := seedCommitDataSet(t, db, bucket.ID) - copies := seedCommitCopies(t, db, bucket.ID, dataSet.ID, 8) - - start := make(chan struct{}) - results := make(chan storagecommit.ReserveResult, len(copies)) - errorsOut := make(chan error, len(copies)) - var workers sync.WaitGroup - for i := range copies { - workers.Add(1) - go func(index int) { - defer workers.Done() - <-start - result, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: commitCopyIdentity(copies[index]), AttemptID: fmt.Sprintf("concurrent-attempt-%d", index), - }) - if err != nil { - errorsOut <- err - return - } - results <- result - }(i) - } - close(start) - workers.Wait() - close(results) - close(errorsOut) - for err := range errorsOut { - t.Errorf("concurrent reservation: %v", err) - } - if t.Failed() { - return - } - acquired := 0 - waiting := 0 - for result := range results { - switch result.State { - case storagecommit.ReservationAcquired: - acquired++ - case storagecommit.ReservationWaiting: - waiting++ - default: - t.Fatalf("unexpected concurrent reservation state %q", result.State) - } - } - if acquired != storagecommit.MaxActiveAttemptsPerDataSet || waiting != len(copies)-storagecommit.MaxActiveAttemptsPerDataSet { - t.Fatalf("concurrent reservations acquired=%d waiting=%d, want %d/%d", - acquired, waiting, storagecommit.MaxActiveAttemptsPerDataSet, len(copies)-storagecommit.MaxActiveAttemptsPerDataSet) - } - count, err := db.NewSelect(). - Model((*model.StorageUploadCopy)(nil)). - Where("storage_data_set_id = ?", dataSet.ID). - Where("commit_attempt_id IS NOT NULL"). - Count(t.Context()) - if err != nil || count != storagecommit.MaxActiveAttemptsPerDataSet { - t.Fatalf("persisted active reservations = %d err=%v, want %d", count, err, storagecommit.MaxActiveAttemptsPerDataSet) - } -} - -func TestStorageCommitAttemptFenceBlocksAutomaticFailure(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - bucket := seedBucket(t, db, "commit-fence-bucket") - dataSet := seedCommitDataSet(t, db, bucket.ID) - copyRow := seedCommitCopies(t, db, bucket.ID, dataSet.ID, 1)[0] - identity := commitCopyIdentity(copyRow) - readyEvidence := repository.MarkUploadCopyPieceReadyInput{ - StorageUploadCopyID: copyRow.ID, - UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, - PieceCID: "bafkqaaa", - RetrievalURL: "https://provider.example/piece", - CommitExtraDataHex: "abcd", - } - if err := repos.Uploads.MarkUploadCopyPieceReady(t.Context(), readyEvidence); err != nil { - t.Fatalf("seed piece evidence: %v", err) - } - - reservation, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: identity, AttemptID: "attempt-fenced", - }) - if err != nil || reservation.State != storagecommit.ReservationAcquired { - t.Fatalf("reserve: %#v err=%v", reservation, err) - } - attempt, err := repos.Uploads.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ - Copy: identity, AttemptID: "attempt-fenced", ExtraDataHex: "abcd", - }) - if err != nil || !attempt.Entered { - t.Fatalf("mark attempted: %#v err=%v", attempt, err) - } - err = repos.Uploads.MarkUploadCopyFailed(t.Context(), repository.MarkUploadCopyFailedInput{ - StorageUploadCopyID: copyRow.ID, UploadID: copyRow.UploadID, CopyIndex: copyRow.CopyIndex, LastError: "injected", - }) - if !errors.Is(err, repository.ErrConflict) { - t.Fatalf("MarkUploadCopyFailed error = %v, want conflict", err) - } - if err := repos.Uploads.MarkUploadCopyPieceReady(t.Context(), readyEvidence); err != nil { - t.Fatalf("idempotent late MarkUploadCopyPieceReady: %v", err) - } - conflictingEvidence := readyEvidence - conflictingEvidence.RetrievalURL = "https://provider.example/different" - if err := repos.Uploads.MarkUploadCopyPieceReady(t.Context(), conflictingEvidence); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("conflicting late MarkUploadCopyPieceReady error = %v, want conflict", err) - } - persisted, err := repos.Uploads.GetUploadCopyByID(t.Context(), copyRow.ID) - if err != nil || persisted.Status != model.StorageUploadCopyStatusCommitting || - persisted.CommitAttemptID == nil || *persisted.CommitAttemptID != "attempt-fenced" || - persisted.CommitAttemptedAt == nil { - t.Fatalf("copy after late piece-ready = %#v err=%v, want unchanged attempted fence", persisted, err) - } - if err := repos.Uploads.ReleaseCommitAttempt(t.Context(), storagecommit.ReleaseInput{ - Copy: identity, AttemptID: "attempt-fenced", - }); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("automatic attempted release error = %v, want conflict", err) - } -} - -func TestStorageCommitAttentionRequiresAcknowledgedFencedRelease(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - bucket := seedBucket(t, db, "commit-attention-release-bucket") - dataSet := seedCommitDataSet(t, db, bucket.ID) - copyRow := seedCommitCopies(t, db, bucket.ID, dataSet.ID, 1)[0] - identity := commitCopyIdentity(copyRow) - - if _, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: identity, AttemptID: "attention-attempt", - }); err != nil { - t.Fatalf("reserve: %v", err) - } - if _, err := repos.Uploads.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ - Copy: identity, AttemptID: "attention-attempt", ExtraDataHex: "abcd", - }); err != nil { - t.Fatalf("mark attempted: %v", err) - } - if err := repos.Uploads.MarkCommitAttention(t.Context(), storagecommit.AttentionInput{ - Copy: identity, AttemptID: "attention-attempt", Code: storagecommit.AttentionAttemptOnlyAmbiguous, - }); err != nil { - t.Fatalf("mark attention: %v", err) - } - if err := repos.Uploads.MarkCommitAttention(t.Context(), storagecommit.AttentionInput{ - Copy: identity, AttemptID: "attention-attempt", Code: storagecommit.AttentionDataSetUnavailable, - }); err != nil { - t.Fatalf("repeat attention: %v", err) - } - stage := "peer_commit" - waitReason := model.TaskWaitReasonExternalConfirmation - legacyTask := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "object", RefID: 1, - RefVersionID: "legacy-confirmation", IdempotencyKey: "legacy-confirmation-task", - Payload: map[string]any{ - "upload_id": copyRow.UploadID, "copy_index": copyRow.CopyIndex, - }, - Status: model.TaskStatusWaiting, WaitReason: &waitReason, - ScheduledAt: time.Now().Add(24 * time.Hour), - } - if _, err := db.NewInsert().Model(legacyTask).Exec(t.Context()); err != nil { - t.Fatalf("insert legacy confirmation task: %v", err) - } - records, err := repos.Uploads.ListCommitAttention(t.Context(), 10) - if err != nil || len(records) != 1 || records[0].CopyID != copyRow.ID || - records[0].Code != storagecommit.AttentionAttemptOnlyAmbiguous { - t.Fatalf("attention records = %#v err=%v", records, err) - } - if _, err := db.NewUpdate().Model((*model.StorageUploadCopy)(nil)). - Set("commit_attention_code = ?", "future_attention_code"). - Where("id = ?", copyRow.ID). - Exec(t.Context()); err != nil { - t.Fatalf("set future attention code: %v", err) - } - records, err = repos.Uploads.ListCommitAttention(t.Context(), 10) - if err != nil || len(records) != 1 || records[0].Code != storagecommit.AttentionCode("future_attention_code") { - t.Fatalf("future attention records = %#v err=%v", records, err) - } - if err := repos.Uploads.ReleaseCommitAttention(t.Context(), storagecommit.ManualReleaseInput{ - CopyID: copyRow.ID, ExpectedAttemptID: "attention-attempt", - }); !errors.Is(err, repository.ErrInvalidInput) { - t.Fatalf("unacknowledged release error = %v, want invalid input", err) - } - if err := repos.Uploads.ReleaseCommitAttention(t.Context(), storagecommit.ManualReleaseInput{ - CopyID: copyRow.ID, ExpectedAttemptID: "stale-attempt", AcknowledgePossibleDuplicate: true, - }); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("stale attempt release error = %v, want conflict", err) - } - if err := repos.Uploads.ReleaseCommitAttention(t.Context(), storagecommit.ManualReleaseInput{ - CopyID: copyRow.ID, ExpectedAttemptID: "attention-attempt", AcknowledgePossibleDuplicate: true, - }); err != nil { - t.Fatalf("acknowledged release: %v", err) - } - persisted, err := repos.Uploads.GetUploadCopyByID(t.Context(), copyRow.ID) - if err != nil { - t.Fatalf("GetUploadCopyByID: %v", err) - } - if persisted.Status != model.StorageUploadCopyStatusPieceReady || persisted.CommitReadyAt != nil || - persisted.CommitAttemptID != nil || persisted.CommitAttemptedAt != nil || - persisted.CommitSubmissionJSON != nil || persisted.CommitAttentionAt != nil || - persisted.CommitAttentionCode != nil { - t.Fatalf("released copy = %#v", persisted) - } - persistedTask := new(model.Task) - if err := db.NewSelect().Model(persistedTask).Where("id = ?", legacyTask.ID).Scan(t.Context()); err != nil { - t.Fatalf("reload legacy confirmation task: %v", err) - } - if persistedTask.Status != model.TaskStatusScheduled || persistedTask.WaitReason != nil || persistedTask.ScheduledAt.After(time.Now().Add(time.Minute)) { - t.Fatalf("legacy confirmation task was not woken: %#v", persistedTask) - } -} - -func TestStorageCommitAttentionReleaseSucceedsWithoutRecoverableWork(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - bucket := seedBucket(t, db, "commit-attention-orphan-bucket") - dataSet := seedCommitDataSet(t, db, bucket.ID) - copyRow := seedCommitCopies(t, db, bucket.ID, dataSet.ID, 1)[0] - identity := commitCopyIdentity(copyRow) - if _, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: identity, AttemptID: "orphan-attention", - }); err != nil { - t.Fatalf("reserve: %v", err) - } - if _, err := repos.Uploads.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ - Copy: identity, AttemptID: "orphan-attention", ExtraDataHex: "abcd", - }); err != nil { - t.Fatalf("mark attempted: %v", err) - } - if err := repos.Uploads.MarkCommitAttention(t.Context(), storagecommit.AttentionInput{ - Copy: identity, AttemptID: "orphan-attention", Code: storagecommit.AttentionAttemptOnlyAmbiguous, - }); err != nil { - t.Fatalf("mark attention: %v", err) - } - - if err := repos.Uploads.ReleaseCommitAttention(t.Context(), storagecommit.ManualReleaseInput{ - CopyID: copyRow.ID, ExpectedAttemptID: "orphan-attention", AcknowledgePossibleDuplicate: true, - }); err != nil { - t.Fatalf("orphan release: %v", err) - } - persisted, loadErr := repos.Uploads.GetUploadCopyByID(t.Context(), copyRow.ID) - if loadErr != nil || persisted.CommitAttemptID != nil || persisted.CommitAttentionAt != nil || - persisted.Status != model.StorageUploadCopyStatusPieceReady { - t.Fatalf("orphan release did not clear the fence: copy=%#v err=%v", persisted, loadErr) - } - if err := repos.Uploads.ReleaseCommitAttention(t.Context(), storagecommit.ManualReleaseInput{ - CopyID: copyRow.ID, ExpectedAttemptID: "orphan-attention", AcknowledgePossibleDuplicate: true, - }); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("replayed orphan release error = %v, want conflict", err) - } -} - -func TestStorageCommitAttentionReleaseWaitsForReplacementClaim(t *testing.T) { - for _, tc := range []struct { - name string - owner storagereplacement.Status - wantStatus storagereplacement.ItemStatus - }{ - {name: "failed", owner: storagereplacement.StatusFailed, wantStatus: storagereplacement.ItemStatusFailed}, - {name: "superseded", owner: storagereplacement.StatusSuperseded, wantStatus: storagereplacement.ItemStatusCancelled}, - {name: "active", owner: storagereplacement.StatusMigrating, wantStatus: storagereplacement.ItemStatusPending}, - } { - t.Run(tc.name, func(t *testing.T) { - db := testDB(t) - fixture := seedCommitAttentionReplacement(t, db, "commit-release-claimed-"+tc.name, tc.owner, true) - input := storagecommit.ManualReleaseInput{ - CopyID: fixture.copyRow.ID, ExpectedAttemptID: fixture.attemptID, - AcknowledgePossibleDuplicate: true, - } - - if err := fixture.repos.Uploads.ReleaseCommitAttention(t.Context(), input); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("claimed release error = %v, want conflict", err) - } - persistedCopy, err := fixture.repos.Uploads.GetUploadCopyByID(t.Context(), fixture.copyRow.ID) - if err != nil || persistedCopy.CommitAttemptID == nil || *persistedCopy.CommitAttemptID != fixture.attemptID || - persistedCopy.CommitAttemptedAt == nil || persistedCopy.CommitAttentionAt == nil { - t.Fatalf("copy after claimed release = %#v err=%v, want intact attention fence", persistedCopy, err) - } - persistedItem := new(storagereplacement.Item) - if err := db.NewSelect().Model(persistedItem).Where("id = ?", fixture.item.ID).Scan(t.Context()); err != nil { - t.Fatalf("load claimed item: %v", err) - } - if persistedItem.Status != storagereplacement.ItemStatusRunning || persistedItem.ClaimedAt == nil || - !persistedItem.ClaimedAt.Equal(fixture.token.ClaimedAt) || persistedItem.LeaseUntil == nil { - t.Fatalf("item after claimed release = %#v, want unchanged claim", persistedItem) - } - - if err := fixture.repos.Replacements.ReleaseReplacementItemClaim(t.Context(), fixture.token); err != nil { - t.Fatalf("release replacement claim: %v", err) - } - if err := fixture.repos.Uploads.ReleaseCommitAttention(t.Context(), input); err != nil { - t.Fatalf("release after claim yielded: %v", err) - } - persistedItem = new(storagereplacement.Item) - if err := db.NewSelect().Model(persistedItem).Where("id = ?", fixture.item.ID).Scan(t.Context()); err != nil { - t.Fatalf("load settled item: %v", err) - } - if persistedItem.Status != tc.wantStatus || persistedItem.ClaimedAt != nil || persistedItem.LeaseUntil != nil { - t.Fatalf("settled item = %#v, want status %s without claim", persistedItem, tc.wantStatus) - } - }) - } -} - -func TestStorageCommitSettlementUsesConcreteDrainingGenerationOrigin(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - bucket := seedBucket(t, db, "commit-concrete-generation-bucket") - oldDataSet := seedCommitDataSet(t, db, bucket.ID) - copyRow := seedCommitCopies(t, db, bucket.ID, oldDataSet.ID, 1)[0] - if _, err := db.NewUpdate().Model((*model.StorageDataSet)(nil)). - Set("is_current = ?", false). - Set("status = ?", model.StorageDataSetStatusDraining). - Set("created_by_upload_id = ?", copyRow.UploadID). - Where("id = ?", oldDataSet.ID). - Exec(t.Context()); err != nil { - t.Fatalf("drain old data set: %v", err) - } - newDataSetID := onChainID(t, "1002") - newDataSet := &model.StorageDataSet{ - BucketID: bucket.ID, ProviderID: oldDataSet.ProviderID, CopyIndex: oldDataSet.CopyIndex, - Generation: 2, IsCurrent: true, DataSetID: &newDataSetID, Status: model.StorageDataSetStatusReady, - } - if _, err := db.NewInsert().Model(newDataSet).Exec(t.Context()); err != nil { - t.Fatalf("insert current generation: %v", err) - } - identity := commitCopyIdentity(copyRow) - if _, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: identity, AttemptID: "draining-settlement", - }); err != nil { - t.Fatalf("reserve draining copy: %v", err) - } - if _, err := repos.Uploads.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ - Copy: identity, AttemptID: "draining-settlement", ExtraDataHex: "abcd", - }); err != nil { - t.Fatalf("mark draining copy attempted: %v", err) - } - pieceID := onChainID(t, "5001") - if err := repos.Uploads.MarkUploadCopyCommitted(t.Context(), repository.MarkUploadCopyCommittedInput{ - StorageUploadCopyID: copyRow.ID, UploadID: copyRow.UploadID, CopyIndex: copyRow.CopyIndex, - PieceCID: "bafkqaaa", PieceID: &pieceID, RetrievalURL: "https://old.example/piece", - CommitAttemptID: "draining-settlement", - }); err != nil { - t.Fatalf("settle draining copy: %v", err) - } - persisted, err := repos.Uploads.GetUploadCopyByID(t.Context(), copyRow.ID) - if err != nil || !persisted.IsNewDataSet { - t.Fatalf("draining copy origin = %#v err=%v, want new-data-set provenance from concrete generation", persisted, err) - } -} - -func seedCommitDataSet(t *testing.T, db *bun.DB, bucketID int64) *model.StorageDataSet { - t.Helper() - providerID := onChainID(t, "101") - dataSetID := onChainID(t, "1001") - row := &model.StorageDataSet{ - BucketID: bucketID, ProviderID: providerID, CopyIndex: 0, Generation: 1, - IsCurrent: true, DataSetID: &dataSetID, Status: model.StorageDataSetStatusReady, - } - if _, err := db.NewInsert().Model(row).Exec(t.Context()); err != nil { - t.Fatalf("insert data set: %v", err) - } - return row -} - -func seedCommitCopies(t *testing.T, db *bun.DB, bucketID, dataSetID int64, count int) []model.StorageUploadCopy { - t.Helper() - providerID := onChainID(t, "101") - copies := make([]model.StorageUploadCopy, 0, count) - for i := range count { - upload := &model.StorageUpload{ - BucketID: bucketID, ContentSize: 1, Checksum: fmt.Sprintf("checksum-%d", i), - Status: model.StorageUploadStatusRunning, RequestedCopies: 1, - } - if _, err := db.NewInsert().Model(upload).Exec(t.Context()); err != nil { - t.Fatalf("insert upload %d: %v", i, err) - } - copyRow := model.StorageUploadCopy{ - UploadID: upload.ID, CopyIndex: 0, ProviderID: &providerID, - TransferMethod: model.StorageCopyTransferMethodPeerPull, - Status: model.StorageUploadCopyStatusPieceReady, StorageDataSetID: &dataSetID, - } - if _, err := db.NewInsert().Model(©Row).Exec(t.Context()); err != nil { - t.Fatalf("insert copy %d: %v", i, err) - } - copies = append(copies, copyRow) - } - return copies -} - -func commitCopyIdentity(copyRow model.StorageUploadCopy) storagecommit.CopyIdentity { - return storagecommit.CopyIdentity{ - StorageUploadCopyID: copyRow.ID, UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, StorageDataSetID: *copyRow.StorageDataSetID, - } -} - -func seedRepositoryCommitAttempt( - t *testing.T, - repos *repository.Repositories, - copyRow model.StorageUploadCopy, - attemptID string, - extraDataHex string, - transactionID string, -) { - t.Helper() - identity := commitCopyIdentity(copyRow) - if _, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: identity, AttemptID: attemptID, - }); err != nil { - t.Fatalf("ReserveCommitAttempt: %v", err) - } - if _, err := repos.Uploads.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ - Copy: identity, AttemptID: attemptID, ExtraDataHex: extraDataHex, - }); err != nil { - t.Fatalf("MarkCommitAttempted: %v", err) - } - if transactionID != "" { - if err := repos.Uploads.RecordCommitTransaction(t.Context(), storagecommit.EvidenceInput{ - Copy: identity, AttemptID: attemptID, TransactionID: transactionID, - }); err != nil { - t.Fatalf("RecordCommitTransaction: %v", err) - } - } -} - -type commitAttentionReplacementFixture struct { - repos *repository.Repositories - copyRow model.StorageUploadCopy - item storagereplacement.Item - token storagereplacement.ClaimToken - attemptID string -} - -func seedCommitAttentionReplacement( - t *testing.T, - db *bun.DB, - name string, - ownerStatus storagereplacement.Status, - claimed bool, -) commitAttentionReplacementFixture { - t.Helper() - repos := repository.NewRepositories(db) - bucket := seedBucket(t, db, name) - source := seedCommitDataSet(t, db, bucket.ID) - if _, err := db.NewUpdate().Model((*model.StorageDataSet)(nil)). - Set("is_current = ?", false). - Set("status = ?", model.StorageDataSetStatusDraining). - Where("id = ?", source.ID). - Exec(t.Context()); err != nil { - t.Fatalf("drain source data set: %v", err) - } - targetProviderID := onChainID(t, "202") - targetDataSetID := onChainID(t, "2002") - target := &model.StorageDataSet{ - BucketID: bucket.ID, ProviderID: targetProviderID, CopyIndex: source.CopyIndex, - Generation: 2, IsCurrent: true, DataSetID: &targetDataSetID, Status: model.StorageDataSetStatusReady, - } - if _, err := db.NewInsert().Model(target).Exec(t.Context()); err != nil { - t.Fatalf("insert target data set: %v", err) - } - copyRow := seedCommitCopies(t, db, bucket.ID, target.ID, 1)[0] - if _, err := db.NewUpdate().Model((*model.StorageUploadCopy)(nil)). - Set("provider_id = ?", targetProviderID). - Where("id = ?", copyRow.ID). - Exec(t.Context()); err != nil { - t.Fatalf("align target copy provider: %v", err) - } - copyRow.ProviderID = &targetProviderID - now := time.Now().Add(-time.Minute) - replacement := &storagereplacement.Replacement{ - BucketID: bucket.ID, CopyIndex: source.CopyIndex, - SourceDataSetID: source.ID, TargetDataSetID: target.ID, - SelectionMode: storagereplacement.SelectionModeManual, - ClientRequestID: name, Status: ownerStatus, - ItemsTotal: 1, ConfirmedAt: now, CreatedAt: now, UpdatedAt: now, - } - if _, err := db.NewInsert().Model(replacement).Exec(t.Context()); err != nil { - t.Fatalf("insert replacement: %v", err) - } - maxRetries := 5 - item := storagereplacement.Item{ - ReplacementID: replacement.ID, UploadID: copyRow.UploadID, TargetCopyID: ©Row.ID, - Status: storagereplacement.ItemStatusPending, ScheduledAt: now, - MaxRetries: &maxRetries, CreatedAt: now, UpdatedAt: now, - } - if claimed { - claimedAt := time.Now().Add(-time.Second) - leaseUntil := claimedAt.Add(time.Hour) - item.Status = storagereplacement.ItemStatusRunning - item.ClaimedAt = &claimedAt - item.LeaseUntil = &leaseUntil - } - if _, err := db.NewInsert().Model(&item).Exec(t.Context()); err != nil { - t.Fatalf("insert replacement item: %v", err) - } - // Claim tokens must carry the persisted timestamp. Timestamps round trip with - // microsecond precision, so an in-memory time.Now() never matches the stored - // value on a platform whose wall clock exposes nanoseconds. - token := storagereplacement.ClaimToken{} - if claimed { - if err := db.NewSelect().Model(&item).Where("id = ?", item.ID).Scan(t.Context()); err != nil { - t.Fatalf("reload claimed replacement item: %v", err) - } - token = storagereplacement.ClaimToken{ItemID: item.ID, ClaimedAt: *item.ClaimedAt} - } - attemptID := name + "-attempt" - seedRepositoryCommitAttempt(t, repos, copyRow, attemptID, "abcd", "0x"+name) - if err := repos.Uploads.MarkCommitAttention(t.Context(), storagecommit.AttentionInput{ - Copy: commitCopyIdentity(copyRow), AttemptID: attemptID, - Code: storagecommit.AttentionAttemptOnlyAmbiguous, - }); err != nil { - t.Fatalf("mark commit attention: %v", err) - } - return commitAttentionReplacementFixture{ - repos: repos, copyRow: copyRow, item: item, token: token, attemptID: attemptID, - } -} - -func TestStorageCommitCapacityReportsSlotsHeldForAttention(t *testing.T) { - cases := []struct { - name string - attention storagecommit.AttentionCode - wantAttentionHeld int - }{ - { - name: "attention only an operator can clear holds the slot", - attention: storagecommit.AttentionAttemptOnlyAmbiguous, - wantAttentionHeld: 4, - }, - { - // data_set_unavailable is raised both as a terminal hold and as one - // the advancer keeps observing. Classifying by code drops the terminal - // ones, which is a silent under-count of the very case that strands a - // data set, so every flagged attempt counts. - name: "data set unavailable attention holds the slot", - attention: storagecommit.AttentionDataSetUnavailable, - wantAttentionHeld: 4, - }, - { - name: "confirmation timeout attention holds the slot", - attention: storagecommit.AttentionConfirmationTimeout, - wantAttentionHeld: 4, - }, - { - name: "plain in-flight attempts are not flagged", - wantAttentionHeld: 0, - }, - } - for _, testCase := range cases { - t.Run(testCase.name, func(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - bucket := seedBucket(t, db, "commit-review-capacity-bucket") - dataSet := seedCommitDataSet(t, db, bucket.ID) - copies := seedCommitCopies(t, db, bucket.ID, dataSet.ID, storagecommit.MaxActiveAttemptsPerDataSet+1) - - for i := range storagecommit.MaxActiveAttemptsPerDataSet { - attemptID := fmt.Sprintf("review-attempt-%d", i) - seedRepositoryCommitAttempt(t, repos, copies[i], attemptID, "abcd", "") - if testCase.attention == "" { - continue - } - if err := repos.Uploads.MarkCommitAttention(t.Context(), storagecommit.AttentionInput{ - Copy: commitCopyIdentity(copies[i]), AttemptID: attemptID, Code: testCase.attention, - }); err != nil { - t.Fatalf("MarkCommitAttention(%d): %v", i, err) - } - } - - // Attention never frees the slot: those attempts may already have been - // accepted by the provider. - active, err := repos.Uploads.CountActiveCommitAttemptsForDataSet(t.Context(), dataSet.ID) - if err != nil || active != storagecommit.MaxActiveAttemptsPerDataSet { - t.Fatalf("active attempts = %d err=%v, want %d", active, err, storagecommit.MaxActiveAttemptsPerDataSet) - } - - blocked, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: commitCopyIdentity(copies[storagecommit.MaxActiveAttemptsPerDataSet]), - AttemptID: "review-blocked-attempt", - }) - if err != nil || blocked.State != storagecommit.ReservationWaiting { - t.Fatalf("blocked reservation = %#v err=%v, want waiting", blocked, err) - } - if blocked.AttentionHeld != testCase.wantAttentionHeld { - t.Fatalf("blocked reservation attention held = %d, want %d", blocked.AttentionHeld, testCase.wantAttentionHeld) - } - }) - } -} - -func TestStorageCommitQueuedReservationReportsNoAttentionHold(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - bucket := seedBucket(t, db, "commit-review-queued-bucket") - dataSet := seedCommitDataSet(t, db, bucket.ID) - copies := seedCommitCopies(t, db, bucket.ID, dataSet.ID, 2) - base := time.Now().Add(-time.Minute) - - // Give the head an earlier FIFO position without reserving it, so the second - // copy waits on order rather than on capacity. - if _, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: commitCopyIdentity(copies[0]), AttemptID: "queued-head", Now: base, - }); err != nil { - t.Fatalf("reserve head: %v", err) - } - if err := repos.Uploads.ReleaseCommitAttempt(t.Context(), storagecommit.ReleaseInput{ - Copy: commitCopyIdentity(copies[0]), AttemptID: "queued-head", - }); err != nil { - t.Fatalf("release head reservation: %v", err) - } - - queued, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: commitCopyIdentity(copies[1]), AttemptID: "queued-follower", Now: base.Add(time.Second), - }) - if err != nil || queued.State != storagecommit.ReservationWaiting || queued.AttentionHeld != 0 { - t.Fatalf("queued reservation = %#v err=%v, want waiting behind the FIFO head with no attention hold", queued, err) - } -} diff --git a/internal/db/repository/storage_upload_health_repo.go b/internal/db/repository/storage_content_health_repo.go similarity index 73% rename from internal/db/repository/storage_upload_health_repo.go rename to internal/db/repository/storage_content_health_repo.go index d122145..3068fbe 100644 --- a/internal/db/repository/storage_upload_health_repo.go +++ b/internal/db/repository/storage_content_health_repo.go @@ -41,23 +41,19 @@ func bucketStorageHealthDataSetCTE(db bun.IDB) string { } type affectedVersionRow struct { - VersionID string `bun:"version_id"` - ObjectID int64 `bun:"object_id"` - BucketID int64 `bun:"bucket_id"` - Key string `bun:"key"` - Size int64 `bun:"size"` - ETag string `bun:"e_tag"` - Checksum string `bun:"checksum"` - ContentType string `bun:"content_type"` - CacheKey string `bun:"cache_key"` - StorageUploadID *int64 `bun:"storage_upload_id"` - InCache bool `bun:"in_cache"` - IsCurrent bool `bun:"is_current"` - State model.ObjectState `bun:"state"` - FailedAtState *model.ObjectState `bun:"failed_at_state"` - LastError *string `bun:"last_error"` - CreatedAt time.Time `bun:"created_at"` - UpdatedAt time.Time `bun:"updated_at"` + VersionID string `bun:"version_id"` + ObjectID int64 `bun:"object_id"` + BucketID int64 `bun:"bucket_id"` + Key string `bun:"key"` + Size int64 `bun:"size"` + ETag string `bun:"e_tag"` + Checksum string `bun:"checksum"` + ContentType string `bun:"content_type"` + ContentID *int64 `bun:"content_id"` + InCache bool `bun:"in_cache"` + IsCurrent bool `bun:"is_current"` + CreatedAt time.Time `bun:"created_at"` + UpdatedAt time.Time `bun:"updated_at"` } type affectedVersionRiskDataSetRow struct { @@ -70,7 +66,7 @@ type affectedVersionReadableCountRow struct { Count int `bun:"readable_alternative_count"` } -func (r *BunStorageUploadRepo) ListBucketStorageHealthAffectedVersions(ctx context.Context, input BucketStorageHealthAffectedVersionsInput) (BucketStorageHealthAffectedVersionPage, error) { +func (r *BunStorageContentRepo) ListBucketStorageHealthAffectedVersions(ctx context.Context, input BucketStorageHealthAffectedVersionsInput) (BucketStorageHealthAffectedVersionPage, error) { if input.BucketID <= 0 { return BucketStorageHealthAffectedVersionPage{}, fmt.Errorf("bucket_id is required: %w", ErrInvalidInput) } @@ -127,9 +123,9 @@ func (r *BunStorageUploadRepo) ListBucketStorageHealthAffectedVersions(ctx conte return page, nil } -func (r *BunStorageUploadRepo) listBucketStorageHealthAffectedVersionRows(ctx context.Context, input BucketStorageHealthAffectedVersionsInput, limit int) ([]affectedVersionRow, error) { +func (r *BunStorageContentRepo) listBucketStorageHealthAffectedVersionRows(ctx context.Context, input BucketStorageHealthAffectedVersionsInput, limit int) ([]affectedVersionRow, error) { filters, args := bucketStorageHealthAffectedVersionObjectFilters(r.db, input) - args = append([]interface{}{input.StaleBefore, input.BucketID, input.BucketID}, args...) + args = append([]any{input.StaleBefore, input.BucketID, input.BucketID}, args...) dataSetFilter, dataSetArgs := bucketStorageHealthAffectedVersionDataSetFilter(input) args = append(args, dataSetArgs...) args = append(args, limit) @@ -142,27 +138,26 @@ func (r *BunStorageUploadRepo) listBucketStorageHealthAffectedVersionRows(ctx co object_version.key, object_version.size, object_version.e_tag, - object_version.checksum, + COALESCE(risk_content.checksum, '') AS checksum, object_version.content_type, - object_version.cache_key, - object_version.storage_upload_id, - object_version.in_cache, - object_version.is_current, - object_version.state, - object_version.failed_at_state, - object_version.last_error, + object_version.content_id, + COALESCE(risk_cache.in_cache, FALSE) AS in_cache, + CASE WHEN risk_object.current_version_id = object_version.version_id THEN TRUE ELSE FALSE END AS is_current, object_version.created_at, object_version.updated_at FROM object_versions AS object_version + JOIN objects AS risk_object ON risk_object.id = object_version.object_id + LEFT JOIN storage_contents AS risk_content ON risk_content.id = object_version.content_id + LEFT JOIN object_cache AS risk_cache ON risk_cache.content_id = object_version.content_id WHERE object_version.bucket_id = ? AND object_version.is_delete_marker = FALSE %s AND EXISTS ( SELECT 1 - FROM storage_upload_copies AS storage_copy + FROM storage_copies AS storage_copy JOIN abnormal_data_sets AS abnormal_data_set ON abnormal_data_set.local_data_set_id = storage_copy.storage_data_set_id - WHERE storage_copy.upload_id = object_version.storage_upload_id + WHERE storage_copy.content_id = object_version.content_id AND storage_copy.status = %s %s ) @@ -175,13 +170,13 @@ func (r *BunStorageUploadRepo) listBucketStorageHealthAffectedVersionRows(ctx co return rows, nil } -func (r *BunStorageUploadRepo) listBucketStorageHealthRiskDataSetsForVersions(ctx context.Context, input BucketStorageHealthAffectedVersionsInput, versionIDs []string) (map[string][]BucketStorageHealthRiskDataSet, error) { +func (r *BunStorageContentRepo) listBucketStorageHealthRiskDataSetsForVersions(ctx context.Context, input BucketStorageHealthAffectedVersionsInput, versionIDs []string) (map[string][]BucketStorageHealthRiskDataSet, error) { out := make(map[string][]BucketStorageHealthRiskDataSet, len(versionIDs)) if len(versionIDs) == 0 { return out, nil } dataSetFilter := "" - args := []interface{}{input.StaleBefore, input.BucketID, input.BucketID, bun.List(versionIDs)} + args := []any{input.StaleBefore, input.BucketID, input.BucketID, bun.List(versionIDs)} if input.LocalDataSetID > 0 { dataSetFilter = ` AND abnormal_data_set.local_data_set_id = ?` @@ -204,9 +199,9 @@ func (r *BunStorageUploadRepo) listBucketStorageHealthRiskDataSetsForVersions(ct abnormal_data_set.last_checked_at, abnormal_data_set.last_error FROM abnormal_data_sets AS abnormal_data_set - JOIN storage_upload_copies AS storage_copy ON storage_copy.storage_data_set_id = abnormal_data_set.local_data_set_id + JOIN storage_copies AS storage_copy ON storage_copy.storage_data_set_id = abnormal_data_set.local_data_set_id JOIN object_versions AS object_version - ON object_version.storage_upload_id = storage_copy.upload_id + ON object_version.content_id = storage_copy.content_id AND object_version.bucket_id = ? AND object_version.is_delete_marker = FALSE WHERE object_version.version_id IN (?) @@ -223,7 +218,7 @@ func (r *BunStorageUploadRepo) listBucketStorageHealthRiskDataSetsForVersions(ct return out, nil } -func (r *BunStorageUploadRepo) listBucketStorageHealthReadableAlternativeCounts(ctx context.Context, input BucketStorageHealthAffectedVersionsInput, versionIDs []string) (map[string]int, error) { +func (r *BunStorageContentRepo) listBucketStorageHealthReadableAlternativeCounts(ctx context.Context, input BucketStorageHealthAffectedVersionsInput, versionIDs []string) (map[string]int, error) { out := make(map[string]int, len(versionIDs)) if len(versionIDs) == 0 { return out, nil @@ -232,9 +227,9 @@ func (r *BunStorageUploadRepo) listBucketStorageHealthReadableAlternativeCounts( object_version.version_id, COUNT(DISTINCT readable_data_set.copy_index) AS readable_alternative_count FROM object_versions AS object_version - JOIN storage_upload_copies AS readable_copy ON readable_copy.upload_id = object_version.storage_upload_id - JOIN storage_uploads AS readable_upload - ON readable_upload.id = readable_copy.upload_id + JOIN storage_copies AS readable_copy ON readable_copy.content_id = object_version.content_id + JOIN storage_contents AS readable_upload + ON readable_upload.id = readable_copy.content_id AND readable_upload.bucket_id = object_version.bucket_id JOIN storage_data_sets AS readable_data_set ON readable_data_set.id = readable_copy.storage_data_set_id @@ -254,9 +249,9 @@ func (r *BunStorageUploadRepo) listBucketStorageHealthReadableAlternativeCounts( return out, nil } -func bucketStorageHealthAffectedVersionObjectFilters(db bun.IDB, input BucketStorageHealthAffectedVersionsInput) (string, []interface{}) { +func bucketStorageHealthAffectedVersionObjectFilters(db bun.IDB, input BucketStorageHealthAffectedVersionsInput) (string, []any) { var filters strings.Builder - args := make([]interface{}, 0, 8) + args := make([]any, 0, 8) if input.Key != "" { filters.WriteString(` AND object_version.key = ?`) @@ -292,12 +287,12 @@ func bucketStorageHealthAffectedVersionObjectFilters(db bun.IDB, input BucketSto return filters.String(), args } -func bucketStorageHealthAffectedVersionDataSetFilter(input BucketStorageHealthAffectedVersionsInput) (string, []interface{}) { +func bucketStorageHealthAffectedVersionDataSetFilter(input BucketStorageHealthAffectedVersionsInput) (string, []any) { if input.LocalDataSetID <= 0 { return "", nil } return ` - AND abnormal_data_set.local_data_set_id = ?`, []interface{}{input.LocalDataSetID} + AND abnormal_data_set.local_data_set_id = ?`, []any{input.LocalDataSetID} } func validateBucketStorageHealthAffectedVersionMarkers(input BucketStorageHealthAffectedVersionsInput) error { @@ -325,22 +320,18 @@ func readableCommittedStorageCopySQL() string { func affectedVersionRowObjectVersion(row affectedVersionRow) model.ObjectVersion { // Keep this mapper aligned with the affected version SELECT row. return model.ObjectVersion{ - VersionID: row.VersionID, - ObjectID: row.ObjectID, - BucketID: row.BucketID, - Key: row.Key, - Size: row.Size, - ETag: row.ETag, - Checksum: row.Checksum, - ContentType: row.ContentType, - CacheKey: row.CacheKey, - StorageUploadID: row.StorageUploadID, - InCache: row.InCache, - IsCurrent: row.IsCurrent, - State: row.State, - FailedAtState: row.FailedAtState, - LastError: row.LastError, - CreatedAt: row.CreatedAt, - UpdatedAt: row.UpdatedAt, + VersionID: row.VersionID, + ObjectID: row.ObjectID, + BucketID: row.BucketID, + Key: row.Key, + Size: row.Size, + ETag: row.ETag, + Checksum: row.Checksum, + ContentType: row.ContentType, + ContentID: row.ContentID, + InCache: row.InCache, + IsCurrent: row.IsCurrent, + CreatedAt: row.CreatedAt, + UpdatedAt: row.UpdatedAt, } } diff --git a/internal/db/repository/storage_content_reference.go b/internal/db/repository/storage_content_reference.go new file mode 100644 index 0000000..eea4f28 --- /dev/null +++ b/internal/db/repository/storage_content_reference.go @@ -0,0 +1,154 @@ +package repository + +import ( + "context" + "database/sql" + "fmt" + "slices" + "time" + + "github.com/strahe/synaps3/internal/model" + "github.com/uptrace/bun" +) + +// lockStorageContentsByID serializes object-version references with permanent +// deletion. Callers must acquire these locks before object or version locks. +func lockStorageContentsByID(ctx context.Context, db bun.IDB, contentIDs []int64) (map[int64]*model.StorageContent, error) { + ids := append([]int64(nil), contentIDs...) + slices.Sort(ids) + + uploads := make(map[int64]*model.StorageContent, len(ids)) + var previous int64 + for _, contentID := range ids { + if contentID <= 0 || contentID == previous { + continue + } + previous = contentID + + res, err := db.NewUpdate(). + Model((*model.StorageContent)(nil)). + Set("updated_at = updated_at"). + Where("id = ?", contentID). + Exec(ctx) + if err != nil { + return nil, fmt.Errorf("locking storage upload %d: %w", contentID, err) + } + rows, _ := res.RowsAffected() + if rows == 0 { + return nil, fmt.Errorf("locking storage upload %d: %w", contentID, ErrNotFound) + } + + upload := new(model.StorageContent) + if err := db.NewSelect().Model(upload).Where("id = ?", contentID).Scan(ctx); err != nil { + if err == sql.ErrNoRows { + return nil, fmt.Errorf("loading locked storage upload %d: %w", contentID, ErrNotFound) + } + return nil, fmt.Errorf("loading locked storage upload %d: %w", contentID, err) + } + uploads[contentID] = upload + } + return uploads, nil +} + +func lockStorageContentForObjectState( + ctx context.Context, + db bun.IDB, + contentID int64, + state model.ObjectState, +) (*model.StorageContent, error) { + var bucketID int64 + if err := db.NewSelect(). + Model((*model.StorageContent)(nil)). + Column("bucket_id"). + Where("id = ?", contentID). + Scan(ctx, &bucketID); err != nil { + if err == sql.ErrNoRows { + return nil, fmt.Errorf("loading storage upload %d: %w", contentID, ErrNotFound) + } + return nil, fmt.Errorf("loading storage upload %d bucket: %w", contentID, err) + } + if bucket, err := lockBucketByID(ctx, db, bucketID); err != nil { + return nil, err + } else if bucket == nil { + return nil, fmt.Errorf("locking storage upload %d bucket: %w", contentID, ErrNotFound) + } + uploads, err := lockStorageContentsByID(ctx, db, []int64{contentID}) + if err != nil { + return nil, err + } + upload := uploads[contentID] + if upload == nil { + return nil, fmt.Errorf("storage content %d cannot back object state %s: %w", contentID, state, ErrConflict) + } + // Whether the content can back a version is a fact about its copies, so it + // is asked of them rather than of a status column that mirrored them. + if err := requireReadableCommittedCopy(ctx, db, contentID); err != nil { + return nil, fmt.Errorf("storage content %d cannot back an object version: %w", contentID, ErrConflict) + } + return upload, nil +} + +func lockStorageContentForCopyMutation(ctx context.Context, db bun.IDB, contentID int64) error { + uploads, err := lockStorageContentsByID(ctx, db, []int64{contentID}) + if err != nil { + return err + } + upload := uploads[contentID] + if upload == nil { + return fmt.Errorf("storage content %d cannot accept copy updates: %w", contentID, ErrConflict) + } + return nil +} + +// prepareNewObjectVersionStorageReference locks the content a new version is +// about to name. +// +// It no longer demands a readable committed copy first. That requirement came +// from the old model, where a version only gained storage_upload_id once the +// upload was already readable, so the reference doubled as a durability claim. +// content_id is plain identity: it is set the moment the bytes are known, and +// durability is read from the copy rows instead. +func prepareNewObjectVersionStorageReference(ctx context.Context, db bun.IDB, version *model.ObjectVersion) error { + if version == nil || version.ContentID == nil || *version.ContentID <= 0 { + return nil + } + contents, err := lockStorageContentsByID(ctx, db, []int64{*version.ContentID}) + if err != nil { + return err + } + if contents[*version.ContentID] == nil { + return fmt.Errorf("storage content %d: %w", *version.ContentID, ErrNotFound) + } + // A data version is only created after its bytes are durably in the local + // cache, so the content is resident. Residency is per content: versions that + // share bytes share this row. + if !version.IsDeleteMarker { + accessedAt := version.CreatedAt + if accessedAt.IsZero() { + accessedAt = time.Now() + } + if err := upsertContentCachePresence(ctx, db, *version.ContentID, true, &accessedAt); err != nil { + return err + } + } + return nil +} + +func sameContentIDs(left, right map[int64]*model.StorageContent) bool { + if len(left) != len(right) { + return false + } + for contentID := range left { + if _, ok := right[contentID]; !ok { + return false + } + } + return true +} + +// objectVersionReferencesStorageContentSQL matches the versions backed by one +// content. A data version always carries content_id, so the reference is that +// column alone. +func objectVersionReferencesStorageContentSQL(versionAlias, uploadAlias string) string { + return fmt.Sprintf("%s.content_id = %s.id", versionAlias, uploadAlias) +} diff --git a/internal/db/repository/storage_content_repo.go b/internal/db/repository/storage_content_repo.go new file mode 100644 index 0000000..014586f --- /dev/null +++ b/internal/db/repository/storage_content_repo.go @@ -0,0 +1,2056 @@ +package repository + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/observability" + "github.com/strahe/synaps3/internal/storagecommit" + "github.com/strahe/synaps3/internal/storagepull" + "github.com/strahe/synaps3/internal/types" + "github.com/uptrace/bun" +) + +type BunStorageContentRepo struct { + db bun.IDB +} + +var _ StorageContentRepository = (*BunStorageContentRepo)(nil) + +// EnsureContent returns the content row for one byte payload in a bucket, +// creating it if this is the first time those bytes are written. +// A uniqueness conflict returns the existing row without changing its frozen +// requested-copy target. +func (r *BunStorageContentRepo) EnsureContent(ctx context.Context, input EnsureContentInput) (*model.StorageContent, error) { + if input.BucketID <= 0 || input.ContentSize < 0 || !validStorageContentChecksum(input.Checksum) { + return nil, fmt.Errorf("ensuring storage content: %w", ErrInvalidInput) + } + requestedCopies := input.RequestedCopies + if !model.ValidStorageCopies(requestedCopies) { + return nil, fmt.Errorf("requested copies must be between %d and %d, got %d", model.StorageCopiesMin, model.StorageCopiesMax, requestedCopies) + } + content := &model.StorageContent{ + BucketID: input.BucketID, + Checksum: input.Checksum, + ContentSize: input.ContentSize, + RequestedCopies: requestedCopies, + } + if _, err := r.db.NewInsert().Model(content).Exec(ctx); err != nil { + if isUniqueViolation(err) { + existing, selectErr := r.findContentByBytes(ctx, input.BucketID, input.Checksum, input.ContentSize) + if selectErr != nil { + return nil, selectErr + } + if existing != nil { + return existing, nil + } + } + return nil, fmt.Errorf("ensuring storage content: %w", err) + } + return content, nil +} + +func validStorageContentChecksum(checksum string) bool { + if len(checksum) != 64 { + return false + } + for i := range len(checksum) { + if (checksum[i] < '0' || checksum[i] > '9') && (checksum[i] < 'a' || checksum[i] > 'f') { + return false + } + } + return true +} + +// findContentByBytes resolves the content identity directly. Deduplication used +// to require scanning object versions by (bucket_id, size, checksum) and then +// joining uploads; the unique key makes it a single lookup. +func (r *BunStorageContentRepo) findContentByBytes(ctx context.Context, bucketID int64, checksum string, size int64) (*model.StorageContent, error) { + content := new(model.StorageContent) + err := r.db.NewSelect(). + Model(content). + Where("bucket_id = ? AND checksum = ? AND content_size = ?", bucketID, checksum, size). + Scan(ctx) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, fmt.Errorf("selecting storage content by bytes: %w", err) + } + return content, nil +} + +func (r *BunStorageContentRepo) GetByID(ctx context.Context, contentID int64) (*model.StorageContent, error) { + upload := new(model.StorageContent) + err := r.db.NewSelect(). + Model(upload). + Where("id = ?", contentID). + Scan(ctx) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, fmt.Errorf("selecting storage upload: %w", err) + } + return upload, nil +} + +func (r *BunStorageContentRepo) GetByIDs(ctx context.Context, contentIDs []int64) (map[int64]model.StorageContent, error) { + uploadsByID := make(map[int64]model.StorageContent, len(contentIDs)) + if len(contentIDs) == 0 { + return uploadsByID, nil + } + var uploads []model.StorageContent + if err := r.db.NewSelect(). + Model(&uploads). + Where("id IN (?)", bun.List(contentIDs)). + Scan(ctx); err != nil { + return nil, fmt.Errorf("selecting storage uploads by id: %w", err) + } + for _, upload := range uploads { + uploadsByID[upload.ID] = upload + } + return uploadsByID, nil +} + +// BeginIngressStoreProgress starts a fresh, fenced progress attempt on the +// ingress copy. The caller runs this in the same transaction as the task +// checkpoint that authorizes the provider request. +func (r *BunStorageContentRepo) BeginIngressStoreProgress(ctx context.Context, input BeginIngressStoreProgressInput) (*model.StorageCopy, error) { + if input.CopyID < 1 || input.Generation < 1 || input.TaskID < 1 || input.Attempt < 1 { + return nil, fmt.Errorf("beginning ingress store progress: %w", ErrInvalidInput) + } + now := time.Now() + copyRow := new(model.StorageCopy) + err := r.db.NewUpdate(). + Model(copyRow). + Set("ingress_store_attempt = ?", input.Attempt). + Set("ingress_bytes_transferred = 0"). + Set("progress_updated_at = ?", now). + Set("updated_at = ?", now). + Where("id = ? AND work_generation = ? AND active_task_id = ?", input.CopyID, input.Generation, input.TaskID). + Where("ingress_store_attempt = ?", input.Attempt-1). + Where("status = ? AND transfer_method = ?", model.StorageCopyStatusPending, model.StorageCopyTransferMethodIngress). + Returning("*"). + Scan(ctx) + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("beginning ingress store progress: %w", ErrConflict) + } + if err != nil { + return nil, fmt.Errorf("beginning ingress store progress: %w", err) + } + return copyRow, nil +} + +// GetIngressCopy returns the copy that performs the ingress transfer for this +// content, or nil when there is none. Progress lives there rather than on the +// content shared by every replica. +// ContentPipelineState derives how far a content has travelled from its copy +// rows. Nothing stores this: a stored column would be a second copy of facts the +// copies already own, and the two would drift. +func (r *BunStorageContentRepo) ContentPipelineState(ctx context.Context, contentID int64) (model.ObjectState, error) { + var state string + err := r.db.NewSelect(). + TableExpr("storage_contents AS storage_content"). + ColumnExpr(contentPipelineStateSQL()). + Where("storage_content.id = ?", contentID). + Scan(ctx, &state) + if errors.Is(err, sql.ErrNoRows) { + return model.ObjectStateCached, nil + } + if err != nil { + return "", fmt.Errorf("deriving content pipeline state: %w", err) + } + return model.ObjectState(state), nil +} + +// RecordContentFailure stores the message explaining why this content could not +// be placed. Whether it counts as failed is derived from its copies; only the +// human-readable reason is persisted. +func (r *BunStorageContentRepo) RecordContentFailure(ctx context.Context, contentID int64, message string) error { + if contentID < 1 { + return fmt.Errorf("recording content failure: %w", ErrInvalidInput) + } + _, err := r.db.NewUpdate(). + Model((*model.StorageContent)(nil)). + Set("error_message = ?", message). + Set("updated_at = ?", time.Now()). + Where("id = ?", contentID). + Where("accepted_at IS NULL"). + Exec(ctx) + if err != nil { + return fmt.Errorf("recording content failure: %w", err) + } + return nil +} + +func (r *BunStorageContentRepo) GetIngressCopy(ctx context.Context, contentID int64) (*model.StorageCopy, error) { + copyRow, err := r.ingressCopy(ctx, contentID) + if errors.Is(err, ErrNotFound) { + return nil, nil + } + return copyRow, err +} + +func (r *BunStorageContentRepo) ingressCopy(ctx context.Context, contentID int64) (*model.StorageCopy, error) { + copyRow := new(model.StorageCopy) + err := r.db.NewSelect(). + Model(copyRow). + Where("content_id = ? AND transfer_method = ?", contentID, model.StorageCopyTransferMethodIngress). + Scan(ctx) + if err != nil { + if err == sql.ErrNoRows { + return nil, fmt.Errorf("ingress copy for content %d: %w", contentID, ErrNotFound) + } + return nil, fmt.Errorf("selecting ingress copy: %w", err) + } + return copyRow, nil +} + +func (r *BunStorageContentRepo) RecordIngressStoreProgress(ctx context.Context, input RecordIngressStoreProgressInput) (*model.StorageCopy, error) { + if input.CopyID < 1 || input.Generation < 1 || input.TaskID < 1 || input.Attempt < 1 || input.BytesUploaded < 0 { + return nil, fmt.Errorf("recording ingress store progress: %w", ErrInvalidInput) + } + now := time.Now() + // content_size is repeated on the copy and pinned there by a composite + // foreign key, so the clamp stays a single-row update with no join. + copyRow := new(model.StorageCopy) + err := r.db.NewUpdate(). + Model(copyRow). + Set("progress_updated_at = CASE WHEN ingress_bytes_transferred < content_size AND ? > ingress_bytes_transferred THEN ? ELSE progress_updated_at END", input.BytesUploaded, now). + Set("ingress_bytes_transferred = CASE WHEN ? > content_size THEN content_size WHEN ? > ingress_bytes_transferred THEN ? ELSE ingress_bytes_transferred END", input.BytesUploaded, input.BytesUploaded, input.BytesUploaded). + Where("id = ? AND work_generation = ? AND active_task_id = ?", input.CopyID, input.Generation, input.TaskID). + Where("ingress_store_attempt = ?", input.Attempt). + Where("status = ? AND transfer_method = ?", model.StorageCopyStatusPending, model.StorageCopyTransferMethodIngress). + Returning("*"). + Scan(ctx) + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("recording ingress store progress: %w", ErrConflict) + } + if err != nil { + return nil, fmt.Errorf("recording ingress store progress: %w", err) + } + return copyRow, nil +} + +func (r *BunStorageContentRepo) GetUploadProvenance(ctx context.Context, contentID int64) (*StorageContentProvenance, error) { + upload, err := r.GetByID(ctx, contentID) + if err != nil || upload == nil { + return nil, err + } + copies, err := r.ListCopies(ctx, contentID) + if err != nil { + return nil, err + } + ingress, err := r.GetIngressCopy(ctx, contentID) + if err != nil { + return nil, err + } + return &StorageContentProvenance{ + Upload: *upload, + Copies: copies, IngressCopy: ingress, + }, nil +} + +func (r *BunStorageContentRepo) ListCopies(ctx context.Context, contentID int64) ([]model.StorageCopy, error) { + var copies []model.StorageCopy + query := `SELECT storage_copy.*, storage_data_set.data_set_id AS data_set_id, + CASE WHEN storage_data_set.created_by_content_id = storage_copy.content_id + THEN TRUE ELSE FALSE END AS is_new_data_set, + active_commit_attempt.attempt_id AS commit_attempt_id, + active_commit_attempt.attempted_at AS commit_attempted_at, + active_commit_attempt.transaction_id AS commit_transaction_id, + active_commit_attempt.submission_json AS commit_submission_json, + active_commit_attempt.confirmed_transaction_id AS commit_confirmed_transaction_id, + active_commit_attempt.attention_code AS commit_attention_code, + active_commit_attempt.attention_at AS commit_attention_at + FROM storage_copies AS storage_copy + LEFT JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id + LEFT JOIN storage_commit_attempts AS active_commit_attempt + ON active_commit_attempt.content_id = storage_copy.content_id + AND active_commit_attempt.storage_data_set_id = storage_copy.storage_data_set_id + AND active_commit_attempt.resolved_at IS NULL + WHERE storage_copy.content_id = ? + ORDER BY storage_copy.copy_index ASC` + if err := r.db.NewRaw(query, contentID).Scan(ctx, &copies); err != nil { + return nil, fmt.Errorf("listing storage upload copies: %w", err) + } + return copies, nil +} + +// CountCurrentGenerationCopySlots counts logical replica slots, not physical +// generation rows. +func (r *BunStorageContentRepo) CountCurrentGenerationCopySlots(ctx context.Context, contentID int64) (int, error) { + if contentID <= 0 { + return 0, fmt.Errorf("counting current upload copy slots: %w", ErrInvalidInput) + } + var count int + query := fmt.Sprintf(`SELECT COUNT(DISTINCT storage_copy.copy_index) + FROM storage_copies AS storage_copy + WHERE storage_copy.content_id = ? AND %s`, currentGenerationCopySQL("storage_copy")) + if err := r.db.NewRaw(query, contentID).Scan(ctx, &count); err != nil { + return 0, fmt.Errorf("counting current upload copy slots: %w", err) + } + return count, nil +} + +func (r *BunStorageContentRepo) ListReadableCommittedCopies(ctx context.Context, contentID int64) ([]ReadableStorageCopy, error) { + var copies []ReadableStorageCopy + query := fmt.Sprintf(`SELECT + storage_copy.content_id, + storage_content.piece_cid, + storage_copy.copy_index, + storage_copy.provider_id, + storage_data_set.data_set_id, + storage_copy.piece_id, + storage_copy.transfer_method, + storage_copy.retrieval_url + FROM storage_copies AS storage_copy + JOIN storage_contents AS storage_content ON storage_content.id = storage_copy.content_id + JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id + WHERE storage_copy.content_id = ? + AND storage_content.piece_cid IS NOT NULL AND storage_content.piece_cid <> '' + AND %s`, + readableCommittedCopyPredicateSQL("storage_copy", "storage_data_set"), + ) + args := []any{contentID} + query += " ORDER BY storage_copy.copy_index ASC" + if err := r.db.NewRaw(query, args...).Scan(ctx, &copies); err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, fmt.Errorf("listing readable storage copies: %w", err) + } + return copies, nil +} + +func (r *BunStorageContentRepo) HasReadableCommittedCopy(ctx context.Context, contentID int64) (bool, error) { + count, err := countReadableReplicaSlots(ctx, r.db, contentID) + if err != nil { + return false, err + } + return count > 0, nil +} + +type bucketStorageHealthSummaryRow struct { + BucketID int64 `bun:"bucket_id"` + AbnormalDataSets int `bun:"abnormal_data_sets"` + AffectedVersionsSeen int `bun:"affected_versions_seen"` + LocalStatusNotReady bool `bun:"local_status_not_ready"` + ObservationMissing bool `bun:"observation_missing"` + ObservationStale bool `bun:"observation_stale"` + ObservationUnavailable bool `bun:"observation_unavailable"` + ObservationDegraded bool `bun:"observation_degraded"` + ObservationUnknown bool `bun:"observation_unknown"` + LastCheckedAt *time.Time `bun:"last_checked_at"` +} + +// bucketStorageHealthAffectedVersionExistsSQL expects dataSetSourceAlias to expose bucket_id and data_set_id. +func bucketStorageHealthAffectedVersionExistsSQL(dataSetSourceAlias string) string { + return fmt.Sprintf(`EXISTS ( + SELECT 1 + FROM storage_copies AS storage_copy + JOIN object_versions AS object_version + ON object_version.content_id = storage_copy.content_id + AND object_version.bucket_id = %[1]s.bucket_id + AND object_version.is_delete_marker = FALSE + WHERE storage_copy.storage_data_set_id = %[1]s.data_set_id + AND storage_copy.status = %[2]s + )`, dataSetSourceAlias, storageHealthCommittedCopyStatusSQL()) +} + +func (r *BunStorageContentRepo) ListBucketStorageHealthSummaries(ctx context.Context, bucketID int64, staleBefore time.Time, affectedVersionCap int) ([]BucketStorageHealthSummary, error) { + if affectedVersionCap < 1 { + affectedVersionCap = 1 + } + var rows []bucketStorageHealthSummaryRow + dataSetBucketFilter := "" + args := []any{staleBefore} + if bucketID > 0 { + dataSetBucketFilter = ` + AND storage_data_set.bucket_id = ?` + args = append(args, bucketID) + } + query := fmt.Sprintf(`WITH bucket_data_sets AS ( + SELECT + storage_data_set.id AS data_set_id, + storage_data_set.bucket_id, + CASE WHEN storage_data_set.status NOT IN (%s) THEN 1 ELSE 0 END AS local_status_not_ready, + CASE WHEN observation.local_data_set_id IS NULL THEN 1 ELSE 0 END AS observation_missing, + CASE WHEN observation.local_data_set_id IS NOT NULL AND observation.last_checked_at < ? THEN 1 ELSE 0 END AS observation_stale, + CASE WHEN observation.status = %s THEN 1 ELSE 0 END AS observation_unavailable, + CASE WHEN observation.status = %s THEN 1 ELSE 0 END AS observation_degraded, + CASE WHEN observation.status = %s THEN 1 ELSE 0 END AS observation_unknown, + observation.last_checked_at + FROM storage_data_sets AS storage_data_set + LEFT JOIN observability_data_set_states AS observation ON observation.local_data_set_id = storage_data_set.id + WHERE 1 = 1 + %s + ), + bucket_observation AS ( + SELECT + bucket_id, + COALESCE(MAX(observation_stale), 0) AS observation_stale, + MIN(last_checked_at) AS last_checked_at + FROM bucket_data_sets + GROUP BY bucket_id + ), + abnormal_data_sets AS ( + SELECT * + FROM bucket_data_sets + WHERE local_status_not_ready = 1 + OR observation_missing = 1 + OR observation_stale = 1 + OR observation_unavailable = 1 + OR observation_degraded = 1 + OR observation_unknown = 1 + ), + bucket_abnormal AS ( + SELECT + bucket_id, + COUNT(*) AS abnormal_data_sets + FROM abnormal_data_sets + GROUP BY bucket_id + ), + affected_data_sets AS ( + SELECT + abnormal_data_set.* + FROM abnormal_data_sets AS abnormal_data_set + WHERE %s + ), + affected_bucket_summary AS ( + SELECT + bucket_id, + COUNT(*) AS affected_data_sets, + COALESCE(MAX(local_status_not_ready), 0) AS local_status_not_ready, + COALESCE(MAX(observation_missing), 0) AS observation_missing, + COALESCE(MAX(observation_stale), 0) AS observation_stale, + COALESCE(MAX(observation_unavailable), 0) AS observation_unavailable, + COALESCE(MAX(observation_degraded), 0) AS observation_degraded, + COALESCE(MAX(observation_unknown), 0) AS observation_unknown, + MIN(last_checked_at) AS last_checked_at + FROM affected_data_sets + GROUP BY bucket_id + ) + SELECT + bucket_observation.bucket_id, + COALESCE(bucket_abnormal.abnormal_data_sets, 0) AS abnormal_data_sets, + COALESCE(( + SELECT COUNT(*) + FROM ( + SELECT DISTINCT object_version.version_id + FROM affected_data_sets AS affected_data_set + JOIN storage_copies AS storage_copy + ON storage_copy.storage_data_set_id = affected_data_set.data_set_id + JOIN object_versions AS object_version + ON object_version.content_id = storage_copy.content_id + AND object_version.bucket_id = affected_data_set.bucket_id + AND object_version.is_delete_marker = FALSE + WHERE affected_data_set.bucket_id = bucket_observation.bucket_id + AND storage_copy.status = %s + LIMIT ? + ) AS capped_affected_versions + ), 0) AS affected_versions_seen, + COALESCE(affected_bucket_summary.local_status_not_ready, 0) > 0 AS local_status_not_ready, + COALESCE(affected_bucket_summary.observation_missing, 0) > 0 AS observation_missing, + (CASE + WHEN COALESCE(affected_bucket_summary.affected_data_sets, 0) > 0 + THEN COALESCE(affected_bucket_summary.observation_stale, 0) + ELSE bucket_observation.observation_stale + END) > 0 AS observation_stale, + COALESCE(affected_bucket_summary.observation_unavailable, 0) > 0 AS observation_unavailable, + COALESCE(affected_bucket_summary.observation_degraded, 0) > 0 AS observation_degraded, + COALESCE(affected_bucket_summary.observation_unknown, 0) > 0 AS observation_unknown, + CASE + WHEN COALESCE(affected_bucket_summary.affected_data_sets, 0) > 0 + THEN affected_bucket_summary.last_checked_at + ELSE bucket_observation.last_checked_at + END AS last_checked_at + FROM bucket_observation + LEFT JOIN bucket_abnormal ON bucket_abnormal.bucket_id = bucket_observation.bucket_id + LEFT JOIN affected_bucket_summary ON affected_bucket_summary.bucket_id = bucket_observation.bucket_id + ORDER BY bucket_observation.bucket_id ASC`, + storageHealthReadyDataSetStatusListSQL(), + storageHealthUnavailableObservationStatusSQL(), + storageHealthDegradedObservationStatusSQL(), + storageHealthUnknownObservationStatusSQL(), + dataSetBucketFilter, + bucketStorageHealthAffectedVersionExistsSQL("abnormal_data_set"), + storageHealthCommittedCopyStatusSQL(), + ) + args = append(args, affectedVersionCap+1) + if err := r.db.NewRaw(query, args...).Scan(ctx, &rows); err != nil { + return nil, fmt.Errorf("listing bucket storage health summaries: %w", err) + } + reasonCodes, err := r.listBucketStorageHealthReasonCodes(ctx, bucketID, staleBefore) + if err != nil { + return nil, err + } + summaries := make([]BucketStorageHealthSummary, 0, len(rows)) + for _, row := range rows { + affectedVersions := row.AffectedVersionsSeen + exceedsCap := affectedVersions > affectedVersionCap + if exceedsCap { + affectedVersions = affectedVersionCap + } + reasons := reasonCodes[row.BucketID] + if reasons == nil { + reasons = []observability.ReasonCode{} + } + summaries = append(summaries, BucketStorageHealthSummary{ + BucketID: row.BucketID, + AbnormalDataSets: row.AbnormalDataSets, + AffectedVersionsCapped: affectedVersions, + AffectedVersionsCap: affectedVersionCap, + AffectedVersionsExceedsCap: exceedsCap, + LocalStatusNotReady: row.LocalStatusNotReady, + ObservationMissing: row.ObservationMissing, + ObservationStale: row.ObservationStale, + ObservationUnavailable: row.ObservationUnavailable, + ObservationDegraded: row.ObservationDegraded, + ObservationUnknown: row.ObservationUnknown, + ReasonCodes: reasons, + LastCheckedAt: row.LastCheckedAt, + }) + } + return summaries, nil +} + +type bucketStorageHealthReasonCodeRow struct { + BucketID int64 `bun:"bucket_id"` + LocalStatus model.StorageDataSetStatus `bun:"local_status"` + ReasonCodes []observability.ReasonCode `bun:"reason_codes"` +} + +func (r *BunStorageContentRepo) listBucketStorageHealthReasonCodes(ctx context.Context, bucketID int64, staleBefore time.Time) (map[int64][]observability.ReasonCode, error) { + dataSetBucketFilter := "" + args := make([]any, 0, 2) + if bucketID > 0 { + dataSetBucketFilter = ` + AND storage_data_set.bucket_id = ?` + args = append(args, bucketID) + } + query := fmt.Sprintf(`WITH abnormal_data_sets AS ( + SELECT + storage_data_set.id AS data_set_id, + storage_data_set.bucket_id, + storage_data_set.status AS local_status, + COALESCE(observation.reason_codes, %s) AS reason_codes + FROM storage_data_sets AS storage_data_set + LEFT JOIN observability_data_set_states AS observation ON observation.local_data_set_id = storage_data_set.id + WHERE 1 = 1 +%s + AND ( + storage_data_set.status NOT IN (%s) + OR observation.local_data_set_id IS NULL + OR observation.status IN (%s) + OR observation.last_checked_at < ? + ) + ) + SELECT DISTINCT + abnormal_data_set.bucket_id, + abnormal_data_set.local_status, + abnormal_data_set.reason_codes + FROM abnormal_data_sets AS abnormal_data_set + WHERE %s + ORDER BY abnormal_data_set.bucket_id ASC`, + storageHealthEmptyJSONArraySQL(r.db), + dataSetBucketFilter, + storageHealthReadyDataSetStatusListSQL(), + storageHealthAbnormalObservationStatusListSQL(), + bucketStorageHealthAffectedVersionExistsSQL("abnormal_data_set"), + ) + args = append(args, staleBefore) + var rows []bucketStorageHealthReasonCodeRow + if err := r.db.NewRaw(query, args...).Scan(ctx, &rows); err != nil { + return nil, fmt.Errorf("listing bucket storage health reason codes: %w", err) + } + out := make(map[int64][]observability.ReasonCode) + for _, row := range rows { + if row.LocalStatus != model.StorageDataSetStatusReady && row.LocalStatus != model.StorageDataSetStatusDraining { + out[row.BucketID] = observability.AppendReasonCode(out[row.BucketID], observability.ReasonLocalStatusNotReady) + } + for _, reason := range row.ReasonCodes { + out[row.BucketID] = observability.AppendReasonCode(out[row.BucketID], reason) + } + } + return out, nil +} + +// ListDataSetBindings returns every generation, including retired ones, because +// callers need the full provider history as well as the current write targets. +func (r *BunStorageContentRepo) ListDataSetBindings(ctx context.Context, bucketID int64) ([]model.StorageDataSet, error) { + var bindings []model.StorageDataSet + if err := r.db.NewSelect(). + Model(&bindings). + Where("bucket_id = ?", bucketID). + OrderExpr("copy_index ASC, generation ASC"). + Scan(ctx); err != nil { + return nil, fmt.Errorf("listing storage data set bindings: %w", err) + } + return bindings, nil +} + +func (r *BunStorageContentRepo) ListDataSetSummaries(ctx context.Context, bucketID int64) ([]StorageDataSetSummary, error) { + var summaries []StorageDataSetSummary + query := fmt.Sprintf(`SELECT + storage_data_set.id, + storage_data_set.bucket_id, + bucket.name AS bucket_name, + storage_data_set.copy_index, + storage_data_set.generation, + storage_data_set.is_current, + storage_data_set.provider_id, + storage_data_set.data_set_id, + storage_data_set.client_data_set_id, + storage_data_set.status, + storage_data_set.created_by_content_id, + storage_data_set.last_used_content_id, + COALESCE(copy_stats.committed_copies, 0) AS committed_copies, + COALESCE(copy_stats.readable_copies, 0) AS readable_copies, + COALESCE(copy_stats.physical_bytes, 0) AS physical_bytes, + COALESCE(version_stats.referenced_versions, 0) AS referenced_versions, + COALESCE(version_stats.current_versions, 0) AS current_versions, + storage_data_set.created_at, + storage_data_set.updated_at + FROM storage_data_sets AS storage_data_set + JOIN buckets AS bucket ON bucket.id = storage_data_set.bucket_id + LEFT JOIN ( + SELECT + storage_copy.storage_data_set_id, + COUNT(*) AS committed_copies, + SUM(CASE + WHEN storage_data_set.status IN (%s) + AND storage_copy.provider_id <> '' + AND storage_data_set.data_set_id IS NOT NULL AND storage_data_set.data_set_id <> '' + AND storage_copy.piece_id IS NOT NULL AND storage_copy.piece_id <> '' + AND storage_copy.retrieval_url IS NOT NULL AND storage_copy.retrieval_url <> '' + THEN 1 ELSE 0 END) AS readable_copies, + SUM(storage_content.content_size) AS physical_bytes + FROM storage_copies AS storage_copy + JOIN storage_contents AS storage_content ON storage_content.id = storage_copy.content_id + JOIN storage_data_sets AS storage_data_set + ON storage_data_set.id = storage_copy.storage_data_set_id + AND storage_data_set.bucket_id = storage_content.bucket_id + WHERE storage_copy.status = %s + GROUP BY storage_copy.storage_data_set_id + ) AS copy_stats ON copy_stats.storage_data_set_id = storage_data_set.id + LEFT JOIN ( + SELECT + storage_copy.storage_data_set_id, + COUNT(DISTINCT object_version.version_id) AS referenced_versions, + COUNT(DISTINCT CASE WHEN referencing_object.current_version_id = object_version.version_id THEN object_version.version_id END) AS current_versions + FROM storage_copies AS storage_copy + JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id + JOIN object_versions AS object_version + ON object_version.content_id = storage_copy.content_id + AND object_version.bucket_id = storage_data_set.bucket_id + JOIN objects AS referencing_object ON referencing_object.id = object_version.object_id + WHERE storage_copy.status = %s + AND object_version.is_delete_marker = FALSE + GROUP BY storage_copy.storage_data_set_id + ) AS version_stats ON version_stats.storage_data_set_id = storage_data_set.id + WHERE (? = 0 OR storage_data_set.bucket_id = ?) + ORDER BY bucket.name ASC, storage_data_set.copy_index ASC, storage_data_set.generation ASC`, + storageHealthReadyDataSetStatusListSQL(), + storageHealthCommittedCopyStatusSQL(), + storageHealthCommittedCopyStatusSQL(), + ) + if err := r.db.NewRaw(query, bucketID, bucketID).Scan(ctx, &summaries); err != nil { + return nil, fmt.Errorf("listing storage data set summaries: %w", err) + } + return summaries, nil +} + +// GetDataSetBindingByCopyIndex returns the generation that currently owns the +// slot. Historical generations stay readable but never receive new writes. +func (r *BunStorageContentRepo) GetDataSetBindingByCopyIndex(ctx context.Context, bucketID int64, copyIndex int) (*model.StorageDataSet, error) { + binding := new(model.StorageDataSet) + err := r.db.NewSelect(). + Model(binding). + Where("bucket_id = ? AND copy_index = ? AND is_current", bucketID, copyIndex). + Scan(ctx) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, fmt.Errorf("selecting storage data set binding: %w", err) + } + return binding, nil +} + +func (r *BunStorageContentRepo) GetDataSetBindingByID(ctx context.Context, id int64) (*model.StorageDataSet, error) { + binding := new(model.StorageDataSet) + err := r.db.NewSelect(). + Model(binding). + Where("id = ?", id). + Scan(ctx) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, fmt.Errorf("selecting storage data set binding by id: %w", err) + } + return binding, nil +} + +func (r *BunStorageContentRepo) EnsureDataSetBinding(ctx context.Context, input EnsureDataSetBindingInput) (*model.StorageDataSet, error) { + var binding *model.StorageDataSet + err := r.runMaybeTx(ctx, func(db bun.IDB) error { + got, err := ensureDataSetBinding(ctx, db, input) + if err != nil { + return err + } + binding = got + return nil + }) + return binding, err +} + +func (r *BunStorageContentRepo) MarkDataSetCreating(ctx context.Context, input MarkDataSetCreatingInput) error { + now := time.Now() + _, err := r.db.NewUpdate(). + Model((*model.StorageDataSet)(nil)). + Set("status = ?", model.StorageDataSetStatusCreating). + Set("create_transaction_id = ?", nullableString(input.TransactionID)). + Set("create_status_url = ?", nullableString(input.StatusURL)). + Set("client_data_set_id = ?", input.ClientDataSetID). + Set("last_used_content_id = ?", nullableInt64(input.ContentID)). + Set("last_error = NULL"). + Set("updated_at = ?", now). + Where("id = ?", input.ID). + Exec(ctx) + if err != nil { + return fmt.Errorf("marking storage data set creating: %w", err) + } + return nil +} + +func (r *BunStorageContentRepo) MarkDataSetReady(ctx context.Context, input MarkDataSetReadyInput) error { + return r.runMaybeTx(ctx, func(db bun.IDB) error { + return markDataSetReady(ctx, db, input.ID, input.ContentID, input.DataSetID, input.ClientDataSetID) + }) +} + +func (r *BunStorageContentRepo) BackfillClientDataSetID(ctx context.Context, input BackfillClientDataSetIDInput) error { + if input.ID <= 0 || input.DataSetID.IsZero() { + return fmt.Errorf("backfilling storage client data set ID: %w", ErrInvalidInput) + } + res, err := r.db.NewUpdate(). + Model((*model.StorageDataSet)(nil)). + Set("client_data_set_id = ?", input.ClientDataSetID). + Set("updated_at = ?", time.Now()). + Where("id = ?", input.ID). + Where("data_set_id = ?", input.DataSetID). + Where("client_data_set_id IS NULL"). + Exec(ctx) + if err != nil { + return fmt.Errorf("backfilling storage client data set ID: %w", err) + } + rows, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("checking storage client data set ID backfill: %w", err) + } + if rows > 0 { + return nil + } + var current struct { + DataSetID *types.OnChainID `bun:"data_set_id"` + ClientDataSetID *types.OnChainID `bun:"client_data_set_id"` + } + err = r.db.NewSelect(). + Table("storage_data_sets"). + Column("data_set_id", "client_data_set_id"). + Where("id = ?", input.ID). + Scan(ctx, ¤t) + if err != nil { + if err == sql.ErrNoRows { + return fmt.Errorf("backfilling storage client data set ID: %w", ErrNotFound) + } + return fmt.Errorf("checking storage client data set ID: %w", err) + } + if current.DataSetID == nil || !current.DataSetID.Equal(input.DataSetID) || + current.ClientDataSetID == nil || !current.ClientDataSetID.Equal(input.ClientDataSetID) { + return fmt.Errorf("backfilling storage client data set ID: %w", ErrConflict) + } + return nil +} + +func (r *BunStorageContentRepo) MarkDataSetDraining(ctx context.Context, id int64, lastError string) error { + res, err := r.db.NewUpdate(). + Model((*model.StorageDataSet)(nil)). + Set("status = ?", model.StorageDataSetStatusDraining). + Set("is_current = ?", false). + Set("last_error = ?", lastError). + Set("updated_at = ?", time.Now()). + Where("id = ?", id). + Where("status IN (?)", bun.List([]model.StorageDataSetStatus{ + model.StorageDataSetStatusReady, + model.StorageDataSetStatusDraining, + })). + Exec(ctx) + if err != nil { + return fmt.Errorf("marking storage data set draining: %w", err) + } + return requireDataSetStatusUpdate(ctx, r.db, id, res, "marking storage data set draining") +} + +// MarkDataSetFailed records why a generation could not be created and gives up +// its slot. Keeping is_current would leave the failed row owning the bucket's +// replica slot with nothing able to release it, which stalls provisioning +// permanently; the slot has to be free for the next generation to take it. +func (r *BunStorageContentRepo) MarkDataSetFailed(ctx context.Context, id int64, lastError string) error { + res, err := r.db.NewUpdate(). + Model((*model.StorageDataSet)(nil)). + Set("status = ?", model.StorageDataSetStatusFailed). + Set("is_current = ?", false). + Set("last_error = ?", lastError). + Set("updated_at = ?", time.Now()). + Where("id = ?", id). + Where("status IN (?)", bun.List([]model.StorageDataSetStatus{ + model.StorageDataSetStatusPending, + model.StorageDataSetStatusCreating, + model.StorageDataSetStatusFailed, + })). + Exec(ctx) + if err != nil { + return fmt.Errorf("marking storage data set failed: %w", err) + } + return requireDataSetStatusUpdate(ctx, r.db, id, res, "marking storage data set failed") +} + +func requireDataSetStatusUpdate(ctx context.Context, db bun.IDB, id int64, result sql.Result, operation string) error { + rows, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("%s: reading affected rows: %w", operation, err) + } + if rows > 0 { + return nil + } + count, err := db.NewSelect(). + Model((*model.StorageDataSet)(nil)). + Where("id = ?", id). + Count(ctx) + if err != nil { + return fmt.Errorf("%s: checking data set: %w", operation, err) + } + if count == 0 { + return fmt.Errorf("%s: %w", operation, ErrNotFound) + } + return fmt.Errorf("%s: data set state changed: %w", operation, ErrConflict) +} + +// RetireRejectedDataSet ends a generation whose creation the chain refused, so +// the provider it had reserved becomes available to the bucket again. Without +// this a rejected creation leaks that provider permanently: the unique partial +// index on (bucket_id, provider_id) covers every non-retired row, and a failed +// generation can neither be retired by the replacement gate nor replaced. +// +// The row and its copies are kept. Retirement is the honest end state here — +// the generation holds nothing and owes nothing, and that is provable — while +// deleting it would erase that the bucket ever tried this provider. +// +// The caller must have proof that no data set was created. A rejected +// transaction is proof; an unknown outcome is not, because a creation can be +// submitted and the process can die before the callback records it. Missing +// identity columns are not proof on their own, so the only guard here is the +// one fact that settles it from this side: no data set identity was recorded. +func (r *BunStorageContentRepo) RetireRejectedDataSet(ctx context.Context, storageDataSetID int64) (bool, error) { + if storageDataSetID <= 0 { + return false, fmt.Errorf("retiring rejected storage data set: %w", ErrInvalidInput) + } + res, err := r.db.NewUpdate(). + Model((*model.StorageDataSet)(nil)). + Set("status = ?", model.StorageDataSetStatusRetired). + Set("updated_at = ?", time.Now()). + Where("id = ?", storageDataSetID). + Where("status = ?", model.StorageDataSetStatusFailed). + Where("is_current = ?", false). + Where("(data_set_id IS NULL OR data_set_id = '')"). + Exec(ctx) + if err != nil { + return false, fmt.Errorf("retiring rejected storage data set: %w", err) + } + rows, _ := res.RowsAffected() + return rows == 1, nil +} + +func (r *BunStorageContentRepo) CreateUploadCopiesForBindings(ctx context.Context, contentID int64, copies []UploadCopyBindingInput) error { + return r.runMaybeTx(ctx, func(db bun.IDB) error { + for _, input := range copies { + if input.ProviderID.IsZero() { + return fmt.Errorf("providerID is required: %w", ErrInvalidInput) + } + binding := new(model.StorageDataSet) + if err := db.NewSelect(). + Model(binding). + Where("id = ?", input.StorageDataSetID). + Scan(ctx); err != nil { + return fmt.Errorf("loading storage data set for bound copy: %w", err) + } + if binding.CopyIndex != input.CopyIndex || binding.ProviderID.String() != input.ProviderID.String() { + return fmt.Errorf("storage copy binding identity does not match data set: %w", ErrConflict) + } + var content struct { + BucketID int64 `bun:"bucket_id"` + ContentSize int64 `bun:"content_size"` + } + if err := db.NewSelect(). + Model((*model.StorageContent)(nil)). + Column("bucket_id", "content_size"). + Where("id = ?", contentID). + Scan(ctx, &content); err != nil { + return fmt.Errorf("loading storage content for bound copy: %w", err) + } + uploadBucketID := content.BucketID + if binding.BucketID != uploadBucketID { + return fmt.Errorf("storage copy upload and data set belong to different buckets: %w", ErrConflict) + } + copyRow := &model.StorageCopy{ + ContentID: contentID, + BucketID: binding.BucketID, + ContentSize: content.ContentSize, + CopyIndex: input.CopyIndex, + ProviderID: binding.ProviderID, + TransferMethod: input.TransferMethod, + Status: model.StorageCopyStatusPending, + StorageDataSetID: input.StorageDataSetID, + } + // Copies are unique per concrete data set so one upload can hold both + // generations of a slot while a replacement migrates. + if _, err := db.NewInsert(). + Model(copyRow). + On("CONFLICT (content_id, storage_data_set_id) DO NOTHING"). + Exec(ctx); err != nil { + return fmt.Errorf("creating storage upload copy row: %w", err) + } + } + return nil + }) +} + +// GetUploadCopy resolves a slot to the copy on its current generation, so a +// task that carries only (upload, slot) can never address a replaced or a +// not-yet-activated generation. +func (r *BunStorageContentRepo) GetUploadCopy(ctx context.Context, contentID int64, copyIndex int) (*model.StorageCopy, error) { + copyRow := new(model.StorageCopy) + q := r.db.NewSelect().Model(copyRow) + projectActiveCommitAttempt(q, "storage_copy") + err := q. + Where("storage_copy.content_id = ? AND storage_copy.copy_index = ?", contentID, copyIndex). + Where(currentGenerationCopySQL("storage_copy")). + Limit(1). + Scan(ctx) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, fmt.Errorf("selecting storage upload copy: %w", err) + } + return copyRow, nil +} + +// GetUploadCopyForDataSet addresses one concrete generation, which is how +// replacement work targets the new provider while the old one still exists. +func (r *BunStorageContentRepo) GetUploadCopyForDataSet(ctx context.Context, contentID, storageDataSetID int64) (*model.StorageCopy, error) { + if contentID <= 0 || storageDataSetID <= 0 { + return nil, fmt.Errorf("selecting storage upload copy for data set: %w", ErrInvalidInput) + } + copyRow := new(model.StorageCopy) + q := r.db.NewSelect().Model(copyRow) + projectActiveCommitAttempt(q, "storage_copy") + err := q. + Where("storage_copy.content_id = ? AND storage_copy.storage_data_set_id = ?", contentID, storageDataSetID). + Scan(ctx) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, fmt.Errorf("selecting storage upload copy for data set: %w", err) + } + return copyRow, nil +} + +func (r *BunStorageContentRepo) GetUploadCopyByID(ctx context.Context, id int64) (*model.StorageCopy, error) { + copyRow := new(model.StorageCopy) + q := r.db.NewSelect().Model(copyRow) + projectActiveCommitAttempt(q, "storage_copy") + err := q.Where("storage_copy.id = ?", id).Scan(ctx) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, fmt.Errorf("selecting storage upload copy by id: %w", err) + } + return copyRow, nil +} + +func selectLiveObjectVersionForStorageContent( + ctx context.Context, + db bun.IDB, + upload *model.StorageContent, + excludedVersionIDs []string, +) (*model.ObjectVersion, error) { + if upload == nil || upload.ID <= 0 { + return nil, fmt.Errorf("selecting live storage upload version: %w", ErrInvalidInput) + } + version := new(model.ObjectVersion) + q := db.NewSelect(). + Model(version). + Where("is_delete_marker = ?", false). + Where("content_id = ?", upload.ID) + if len(excludedVersionIDs) > 0 { + q = q.Where("version_id NOT IN (?)", bun.List(excludedVersionIDs)) + } + // Cache residency is a property of the content, not of any one version, so + // every candidate here shares it and it cannot break the tie. + err := q. + OrderExpr("CASE WHEN EXISTS (SELECT 1 FROM objects AS pointer WHERE pointer.id = object_version.object_id AND pointer.current_version_id = object_version.version_id) THEN 0 ELSE 1 END ASC"). + OrderExpr("created_at DESC"). + OrderExpr("version_id DESC"). + Limit(1). + Scan(ctx) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, fmt.Errorf("selecting live storage upload version: %w", err) + } + return version, nil +} + +func (r *BunStorageContentRepo) ListIncompleteCopiesForDataSet(ctx context.Context, storageDataSetID int64) ([]model.StorageCopy, error) { + if storageDataSetID < 1 { + return nil, ErrInvalidInput + } + var copies []model.StorageCopy + q := r.db.NewSelect().Model(&copies) + projectActiveCommitAttempt(q, "storage_copy") + err := q. + Join("JOIN storage_contents AS storage_content ON storage_content.id = storage_copy.content_id"). + Where("storage_copy.storage_data_set_id = ?", storageDataSetID). + Where("storage_copy.status IN (?)", bun.List([]model.StorageCopyStatus{ + model.StorageCopyStatusPending, + model.StorageCopyStatusPieceReady, + model.StorageCopyStatusCommitting, + })). + Where(`EXISTS ( + SELECT 1 FROM object_versions AS pending_version + WHERE pending_version.is_delete_marker = ? + AND `+objectVersionReferencesStorageContentSQL("pending_version", "storage_content")+` + )`, false). + OrderExpr("storage_copy.id ASC"). + Scan(ctx) + if err != nil { + return nil, fmt.Errorf("listing incomplete data set copies: %w", err) + } + return copies, nil +} + +func (r *BunStorageContentRepo) NextFinalizableCopyForDataSet(ctx context.Context, storageDataSetID int64) (*model.StorageCopy, error) { + copyRow := new(model.StorageCopy) + query := fmt.Sprintf(`SELECT storage_copy.* + FROM storage_copies AS storage_copy + JOIN storage_contents AS storage_content ON storage_content.id = storage_copy.content_id + WHERE storage_copy.storage_data_set_id = ? + AND storage_copy.status = ? + AND storage_content.accepted_at IS NULL + AND storage_content.requested_copies > 0 + AND EXISTS ( + SELECT 1 FROM object_versions AS pending_version + WHERE pending_version.content_id = storage_content.id + AND pending_version.is_delete_marker = FALSE + ) + AND ( + SELECT COUNT(DISTINCT readable_data_set.copy_index) + FROM storage_copies AS readable_copy + JOIN storage_data_sets AS readable_data_set ON readable_data_set.id = readable_copy.storage_data_set_id + WHERE readable_copy.content_id = storage_content.id + AND %s + ) >= storage_content.requested_copies + ORDER BY storage_copy.id ASC + LIMIT 1`, readableCommittedCopyPredicateWithDataSetStatusSQL( + "readable_copy", + "readable_data_set", + fmt.Sprintf("(readable_data_set.status IN (%s) OR readable_data_set.id = ?)", + storageHealthReadyDataSetStatusListSQL()), + )) + err := r.db.NewRaw( + query, + storageDataSetID, + model.StorageCopyStatusCommitted, + storageDataSetID, + ).Scan(ctx, copyRow) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, fmt.Errorf("selecting next finalizable data set copy: %w", err) + } + return copyRow, nil +} + +func (r *BunStorageContentRepo) ListIncompleteReadableUploads( + ctx context.Context, + afterID int64, + limit int, +) ([]IncompleteReadableUpload, error) { + var uploads []model.StorageContent + q := r.db.NewSelect(). + Model(&uploads). + Where("accepted_at IS NULL"). + Where("id > ?", afterID). + Where(`EXISTS ( + SELECT 1 FROM object_versions AS live_version + WHERE live_version.is_delete_marker = ? + AND live_version.state = ? + AND `+objectVersionReferencesStorageContentSQL("live_version", "storage_content")+` + )`, false, model.ObjectStateStored). + OrderExpr("id ASC") + if limit > 0 { + q = q.Limit(limit) + } + if err := q.Scan(ctx); err != nil { + return nil, fmt.Errorf("listing incomplete readable storage uploads: %w", err) + } + + items := make([]IncompleteReadableUpload, 0, len(uploads)) + for i := range uploads { + version := new(model.ObjectVersion) + err := r.db.NewSelect(). + Model(version). + Where("is_delete_marker = ?", false). + Where("state = ?", model.ObjectStateStored). + Where("content_id = ?", uploads[i].ID). + OrderExpr("in_cache DESC"). + OrderExpr("CASE WHEN EXISTS (SELECT 1 FROM objects AS pointer WHERE pointer.id = object_version.object_id AND pointer.current_version_id = object_version.version_id) THEN 0 ELSE 1 END ASC"). + OrderExpr("created_at DESC"). + OrderExpr("version_id DESC"). + Limit(1). + Scan(ctx) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + continue + } + return nil, fmt.Errorf("selecting durable version for incomplete readable upload %d: %w", uploads[i].ID, err) + } + items = append(items, IncompleteReadableUpload{ + Upload: uploads[i], + Version: *version, + }) + } + return items, nil +} + +func (r *BunStorageContentRepo) MarkUploadCopyPieceReady(ctx context.Context, input MarkUploadCopyPieceReadyInput) error { + return r.runMaybeTx(ctx, func(db bun.IDB) error { + if err := lockStorageContentForCopyMutation(ctx, db, input.ContentID); err != nil { + return fmt.Errorf("locking storage upload for piece-ready copy: %w", err) + } + copyID, err := slotCopyTarget(ctx, db, input.StorageCopyID, input.ContentID, input.CopyIndex) + if err != nil { + return err + } + now := time.Now() + q := db.NewUpdate(). + Model((*model.StorageCopy)(nil)). + Set("status = ?", model.StorageCopyStatusPieceReady). + Set("piece_id = COALESCE(?, piece_id)", input.PieceID). + Set("retrieval_url = COALESCE(?, retrieval_url)", nullableString(input.RetrievalURL)). + Set("commit_extra_data_hex = COALESCE(?, commit_extra_data_hex)", nullableString(input.CommitExtraDataHex)). + Set("last_error = NULL"). + Set("updated_at = ?", now). + Where("id = ?", copyID). + Where("status NOT IN (?, ?)", model.StorageCopyStatusCommitted, model.StorageCopyStatusCommitting) + if input.PieceCID != "" { + q = q.Where(`EXISTS ( + SELECT 1 FROM storage_contents AS evidence_upload + WHERE evidence_upload.id = ? + AND (evidence_upload.piece_cid IS NULL OR evidence_upload.piece_cid = '' OR evidence_upload.piece_cid = ?) + )`, input.ContentID, input.PieceCID) + } + if input.PieceID != nil { + q = q.Where("(piece_id IS NULL OR piece_id = ?)", input.PieceID) + } + if input.RetrievalURL != "" { + q = q.Where("(retrieval_url IS NULL OR retrieval_url = '' OR retrieval_url = ?)", input.RetrievalURL) + } + if input.CommitExtraDataHex != "" { + q = q.Where("(commit_extra_data_hex IS NULL OR commit_extra_data_hex = '' OR commit_extra_data_hex = ?)", input.CommitExtraDataHex) + } + if input.RequireEligibleCopy { + q = q. + Where("status <> ?", model.StorageCopyStatusFailed). + Where(liveObjectVersionExistsForUploadSQL(), input.ContentID, false) + } + res, err := q.Exec(ctx) + if err != nil { + return fmt.Errorf("marking storage upload copy piece ready: %w", err) + } + rows, _ := res.RowsAffected() + if rows == 0 { + var status model.StorageCopyStatus + if err := db.NewSelect().Model((*model.StorageCopy)(nil)). + Column("status").Where("id = ?", copyID).Scan(ctx, &status); err != nil { + return fmt.Errorf("loading storage upload copy after piece evidence conflict: %w", err) + } + if status == model.StorageCopyStatusCommitting { + compatible := db.NewSelect(). + Model((*model.StorageCopy)(nil)). + Where("id = ?", copyID). + Where("status = ?", model.StorageCopyStatusCommitting). + Where(attemptedStorageCommitSQL("storage_copy")) + if input.PieceCID != "" { + compatible = compatible.Where(`EXISTS ( + SELECT 1 FROM storage_contents AS evidence_upload + WHERE evidence_upload.id = ? + AND evidence_upload.piece_cid = ? + )`, input.ContentID, input.PieceCID) + } + if input.PieceID != nil { + compatible = compatible.Where("piece_id = ?", input.PieceID) + } + if input.RetrievalURL != "" { + compatible = compatible.Where("retrieval_url = ?", input.RetrievalURL) + } + if input.CommitExtraDataHex != "" { + compatible = compatible.Where("commit_extra_data_hex = ?", input.CommitExtraDataHex) + } + count, err := compatible.Count(ctx) + if err != nil { + return fmt.Errorf("checking idempotent piece evidence for committing copy: %w", err) + } + if count == 1 { + return nil + } + } + // A late piece-ready result cannot regress a committed copy. Treat that + // stale observation as a harmless no-op; every other zero-row result is + // conflicting monotonic evidence and must stop before Commit. + if status == model.StorageCopyStatusCommitted && !input.RequireEligibleCopy { + return nil + } + return fmt.Errorf("marking storage upload copy piece ready: %w", ErrConflict) + } + transferMethod, err := uploadCopyTransferMethod(ctx, db, input.ContentID, input.CopyIndex) + if err != nil { + return err + } + if transferMethod == model.StorageCopyTransferMethodIngress { + if err := updateUploadIngressReady(ctx, db, input.ContentID, input.PieceCID, now); err != nil { + return err + } + } else { + if _, err := db.NewUpdate(). + Model((*model.StorageContent)(nil)). + Set("piece_cid = COALESCE(?, piece_cid)", nullableString(input.PieceCID)). + Set("updated_at = ?", now). + Where("id = ?", input.ContentID). + Exec(ctx); err != nil { + return fmt.Errorf("recording storage upload piece CID: %w", err) + } + } + // A pull that produced this piece is finished. Resolving it here, in the + // same transaction, is what frees the copy's unresolved slot so a later + // retry can ask a different source. + if input.PullAttemptID != "" { + if err := resolvePullAttempt(ctx, db, input.PullAttemptID, now); err != nil { + return err + } + } + return nil + }) +} + +// resolvePullAttempt marks one request finished without changing its status: a +// resolved "attempted" row is the record of a request that succeeded. +func resolvePullAttempt(ctx context.Context, db bun.IDB, attemptID string, now time.Time) error { + res, err := db.NewUpdate(). + Model((*storagepull.Attempt)(nil)). + Set("resolved_at = COALESCE(resolved_at, ?)", now). + Set("updated_at = ?", now). + Where("attempt_id = ? AND status = ?", attemptID, storagepull.AttemptStatusAttempted). + Exec(ctx) + if err != nil { + return fmt.Errorf("resolving storage pull attempt: %w", err) + } + if rows, _ := res.RowsAffected(); rows != 1 { + return fmt.Errorf("resolving storage pull attempt %s: %w", attemptID, ErrConflict) + } + return nil +} + +func (r *BunStorageContentRepo) MarkUploadCopyCommitted(ctx context.Context, input MarkUploadCopyCommittedInput) error { + if input.ContentID <= 0 || input.CopyIndex < 0 || input.PieceCID == "" || input.PieceID == nil || input.RetrievalURL == "" || + input.CommitAttemptID == "" || input.StorageCopyID <= 0 || input.CommitExtraDataHex == "" || + input.CommitTransactionID == "" || input.CommitConfirmedTransactionID == "" { + return fmt.Errorf("marking storage upload copy committed: %w", ErrInvalidInput) + } + return r.runMaybeTx(ctx, func(db bun.IDB) error { + initial := new(model.StorageCopy) + if err := db.NewSelect(). + Model(initial). + Where("id = ?", input.StorageCopyID). + Where("content_id = ?", input.ContentID). + Where("copy_index = ?", input.CopyIndex). + Scan(ctx); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("loading storage commit copy: %w", ErrNotFound) + } + return fmt.Errorf("loading storage commit copy: %w", err) + } + copyID, _, err := lockCommitCopyFamily(ctx, db, storagecommit.CopyIdentity{ + StorageCopyID: input.StorageCopyID, + ContentID: input.ContentID, + CopyIndex: input.CopyIndex, + StorageDataSetID: initial.StorageDataSetID, + // Settling an attempted commit deliberately skips the owner + // eligibility check: the piece already reached the provider, so a + // deleted owner must not stop the copy from being recorded. + RequireEligibleCopy: false, + }) + if err != nil { + return err + } + now := time.Now() + { + copyIdentity := new(model.StorageCopy) + if err := db.NewSelect(). + Model(copyIdentity). + Where("id = ?", copyID). + Scan(ctx); err != nil { + return fmt.Errorf("loading confirmed storage commit identity: %w", err) + } + identity := storagecommit.CopyIdentity{ + StorageCopyID: copyID, + ContentID: input.ContentID, + CopyIndex: input.CopyIndex, + StorageDataSetID: copyIdentity.StorageDataSetID, + } + attempt, err := loadCommitAttempt(ctx, db, identity, input.CommitAttemptID) + if err != nil { + return err + } + if attempt.Status == storagecommit.AttemptStatusConfirmed { + if attempt.ResolvedAt == nil || derefString(attempt.ExtraDataHex) != input.CommitExtraDataHex || + derefString(attempt.TransactionID) != input.CommitTransactionID || + derefString(attempt.ConfirmedTransactionID) != input.CommitConfirmedTransactionID { + return fmt.Errorf("replaying confirmed storage commit with conflicting evidence: %w", ErrConflict) + } + matchingCopyCount, countErr := db.NewSelect(). + Model((*model.StorageCopy)(nil)). + Where("id = ?", copyID). + Where("status = ?", model.StorageCopyStatusCommitted). + Where("piece_id = ?", input.PieceID). + Where("retrieval_url = ?", input.RetrievalURL). + Where("commit_extra_data_hex = ?", input.CommitExtraDataHex). + Where(`EXISTS ( + SELECT 1 FROM storage_contents AS evidence_upload + WHERE evidence_upload.id = ? AND evidence_upload.piece_cid = ? + )`, input.ContentID, input.PieceCID). + Count(ctx) + if countErr != nil { + return fmt.Errorf("checking confirmed storage commit projection: %w", countErr) + } + if matchingCopyCount != 1 { + return fmt.Errorf("replaying confirmed storage commit with conflicting projection: %w", ErrConflict) + } + return updateUploadReadable(ctx, db, input.ContentID, input.PieceCID, now) + } + if attempt.Status != storagecommit.AttemptStatusAttempted || attempt.ResolvedAt != nil { + return fmt.Errorf("confirming resolved storage commit attempt: %w", ErrConflict) + } + attemptQuery := db.NewUpdate(). + Model((*storagecommit.Attempt)(nil)). + Set("status = ?", storagecommit.AttemptStatusConfirmed). + Set("transaction_id = COALESCE(transaction_id, ?)", input.CommitTransactionID). + Set("confirmed_transaction_id = ?", input.CommitConfirmedTransactionID). + Set("resolved_at = ?", now). + Set("updated_at = ?", now). + Where("attempt_id = ?", input.CommitAttemptID). + Where("content_id = ? AND storage_data_set_id = ?", input.ContentID, copyIdentity.StorageDataSetID). + Where("status = ? AND resolved_at IS NULL", storagecommit.AttemptStatusAttempted). + Where("(transaction_id IS NULL OR transaction_id = ?)", input.CommitTransactionID) + if input.CommitExtraDataHex != "" { + attemptQuery = attemptQuery.Where("extra_data_hex = ?", input.CommitExtraDataHex) + } + res, err := attemptQuery.Exec(ctx) + if err != nil { + return fmt.Errorf("confirming storage commit attempt: %w", err) + } + if rows, _ := res.RowsAffected(); rows != 1 { + return fmt.Errorf("confirming storage commit attempt: %w", ErrConflict) + } + } + // The attempt was set to confirmed just above, so the projection can + // name it; the composite foreign key refuses any other status. + q := db.NewUpdate(). + Model((*model.StorageCopy)(nil)). + Set("status = ?", model.StorageCopyStatusCommitted). + Set("piece_id = COALESCE(?, piece_id)", input.PieceID). + Set("retrieval_url = COALESCE(?, retrieval_url)", nullableString(input.RetrievalURL)). + Set("commit_extra_data_hex = COALESCE(?, commit_extra_data_hex)", nullableString(input.CommitExtraDataHex)). + Set("confirmed_attempt_id = ?", input.CommitAttemptID). + Set("confirmed_attempt_status = ?", string(storagecommit.AttemptStatusConfirmed)). + Set("last_error = NULL"). + Set("updated_at = ?", now). + Where("id = ?", copyID) + q = q.Where(`EXISTS ( + SELECT 1 FROM storage_contents AS evidence_upload + WHERE evidence_upload.id = ? + AND (evidence_upload.piece_cid IS NULL OR evidence_upload.piece_cid = '' OR evidence_upload.piece_cid = ?) + )`, input.ContentID, input.PieceCID) + if input.PieceID != nil { + q = q.Where("(piece_id IS NULL OR piece_id = ?)", input.PieceID) + } + if input.RetrievalURL != "" { + q = q.Where("(retrieval_url IS NULL OR retrieval_url = '' OR retrieval_url = ?)", input.RetrievalURL) + } + if input.CommitExtraDataHex != "" { + q = q.Where("(commit_extra_data_hex IS NULL OR commit_extra_data_hex = '' OR commit_extra_data_hex = ?)", input.CommitExtraDataHex) + } + q = q. + Set("commit_ready_at = NULL"). + Where("status = ?", model.StorageCopyStatusCommitting) + res, err := q.Exec(ctx) + if err != nil { + return fmt.Errorf("marking storage upload copy committed: %w", err) + } + rows, _ := res.RowsAffected() + if rows == 0 { + if input.RequireEligibleCopy { + return fmt.Errorf("marking storage upload copy committed: %w", ErrConflict) + } + return fmt.Errorf("marking storage upload copy committed: %w", ErrNotFound) + } + if err := updateUploadReadable(ctx, db, input.ContentID, input.PieceCID, now); err != nil { + return err + } + return nil + }) +} + +func liveObjectVersionExistsForUploadSQL() string { + return `EXISTS ( + SELECT 1 + FROM storage_contents AS guarded_content + JOIN object_versions AS live_version + ON ` + objectVersionReferencesStorageContentSQL("live_version", "guarded_content") + ` + WHERE guarded_content.id = ? + AND live_version.is_delete_marker = ? + )` +} + +func (r *BunStorageContentRepo) MarkUploadCopyFailed(ctx context.Context, input MarkUploadCopyFailedInput) error { + contentID, copyIndex, lastError := input.ContentID, input.CopyIndex, input.LastError + return r.runMaybeTx(ctx, func(db bun.IDB) error { + if err := lockStorageContentForCopyMutation(ctx, db, contentID); err != nil { + return fmt.Errorf("locking storage upload for failed copy: %w", err) + } + copyID, err := slotCopyTarget(ctx, db, input.StorageCopyID, contentID, copyIndex) + if err != nil { + return err + } + now := time.Now() + if input.PullAttemptID != "" { + result, err := db.NewUpdate(). + Model((*storagepull.Attempt)(nil)). + Set("status = ?", storagepull.AttemptStatusAbandoned). + Set("last_error = ?", lastError). + Set("resolved_at = ?", now). + Set("updated_at = ?", now). + Where("attempt_id = ?", input.PullAttemptID). + Where("content_id = ?", contentID). + Where(`storage_data_set_id = ( + SELECT storage_data_set_id FROM storage_copies WHERE id = ? + )`, copyID). + Where("status = ? AND resolved_at IS NULL", storagepull.AttemptStatusAttempted). + Exec(ctx) + if err != nil { + return fmt.Errorf("abandoning failed storage pull: %w", err) + } + rows, _ := result.RowsAffected() + if rows == 0 { + count, err := db.NewSelect(). + Model((*storagepull.Attempt)(nil)). + Where("attempt_id = ?", input.PullAttemptID). + Where("content_id = ?", contentID). + Where(`storage_data_set_id = ( + SELECT storage_data_set_id FROM storage_copies WHERE id = ? + )`, copyID). + Where("status = ? AND resolved_at IS NOT NULL", storagepull.AttemptStatusAbandoned). + Count(ctx) + if err != nil { + return fmt.Errorf("checking abandoned storage pull: %w", err) + } + if count != 1 { + return fmt.Errorf("abandoning failed storage pull: %w", ErrConflict) + } + } + } + if _, err := db.NewUpdate(). + Model((*storagecommit.Attempt)(nil)). + Set("status = ?", storagecommit.AttemptStatusReleased). + Set("release_reason = ?", string(storagecommit.ReleaseOwnerTerminal)). + Set("resolved_at = ?", now). + Set("updated_at = ?", now). + Where("content_id = ?", contentID). + Where(`storage_data_set_id = ( + SELECT storage_data_set_id FROM storage_copies WHERE id = ? + )`, copyID). + Where("status = ? AND resolved_at IS NULL", storagecommit.AttemptStatusReserved). + Exec(ctx); err != nil { + return fmt.Errorf("releasing failed storage copy reservation: %w", err) + } + res, err := db.NewUpdate(). + Model((*model.StorageCopy)(nil)). + Set("status = ?", model.StorageCopyStatusFailed). + Set("commit_ready_at = NULL"). + Set("commit_extra_data_hex = NULL"). + Set("last_error = ?", lastError). + Set("updated_at = ?", now). + Where("id = ?", copyID). + Where("status <> ?", model.StorageCopyStatusCommitted). + Where(`NOT EXISTS ( + SELECT 1 FROM storage_commit_attempts AS unresolved_attempt + WHERE unresolved_attempt.content_id = storage_copy.content_id + AND unresolved_attempt.storage_data_set_id = storage_copy.storage_data_set_id + AND unresolved_attempt.resolved_at IS NULL + )`). + Exec(ctx) + if err != nil { + return fmt.Errorf("marking storage upload copy failed: %w", err) + } + rows, rowsErr := res.RowsAffected() + if rowsErr != nil { + return fmt.Errorf("marking storage upload copy failed: reading affected rows: %w", rowsErr) + } + if rows == 0 { + submittedCount, countErr := db.NewSelect(). + Model((*model.StorageCopy)(nil)). + Where("id = ?", copyID). + Where(attemptedStorageCommitSQL("storage_copy")). + Count(ctx) + if countErr != nil { + return fmt.Errorf("checking submitted storage commit before failure: %w", countErr) + } + if submittedCount > 0 { + return fmt.Errorf("marking storage upload copy failed: submitted commit is still recoverable: %w", ErrConflict) + } + return nil + } + readableCount, err := countReadableReplicaSlots(ctx, db, contentID) + if err != nil { + return err + } + submittedCount, err := countSubmittedCommitCopies(ctx, db, contentID) + if err != nil { + return err + } + viableCount, err := db.NewSelect(). + Model((*model.StorageCopy)(nil)). + Where("content_id = ?", contentID). + Where("status <> ?", model.StorageCopyStatusFailed). + Count(ctx) + if err != nil { + return fmt.Errorf("counting viable storage upload copies: %w", err) + } + if readableCount == 0 && submittedCount == 0 && viableCount == 0 { + _, err = db.NewUpdate(). + Model((*model.StorageContent)(nil)). + Set("error_message = ?", lastError). + Set("updated_at = ?", now). + Where("id = ?", contentID). + Where("accepted_at IS NULL"). + Exec(ctx) + if err != nil { + return fmt.Errorf("marking storage upload failed: %w", err) + } + } else if readableCount > 0 { + _, err = db.NewUpdate(). + Model((*model.StorageContent)(nil)). + Set("error_message = ?", lastError). + Set("updated_at = ?", now). + Where("id = ?", contentID). + Exec(ctx) + if err != nil { + return fmt.Errorf("marking storage upload readable after copy failure: %w", err) + } + } + return nil + }) +} + +func (r *BunStorageContentRepo) ReopenFailedUploadCopy(ctx context.Context, copyID int64) error { + if copyID < 1 { + return ErrInvalidInput + } + return r.runMaybeTx(ctx, func(db bun.IDB) error { + return reopenFailedUploadCopy(ctx, db, copyID) + }) +} + +func reopenFailedUploadCopy(ctx context.Context, db bun.IDB, copyID int64) error { + now := time.Now() + result, err := db.NewUpdate(). + Model((*model.StorageCopy)(nil)). + Set("status = ?", model.StorageCopyStatusPending). + Set("piece_id = NULL"). + Set("retrieval_url = NULL"). + Set("commit_extra_data_hex = NULL"). + Set("commit_ready_at = NULL"). + Set("confirmed_attempt_id = NULL"). + Set("confirmed_attempt_status = NULL"). + Set("last_error = NULL"). + Set("updated_at = ?", now). + Where("id = ?", copyID). + Where("status = ?", model.StorageCopyStatusFailed). + Where("active_task_id IS NULL"). + Where(`NOT EXISTS ( + SELECT 1 FROM storage_commit_attempts AS unresolved_attempt + WHERE unresolved_attempt.content_id = storage_copy.content_id + AND unresolved_attempt.storage_data_set_id = storage_copy.storage_data_set_id + AND unresolved_attempt.resolved_at IS NULL + )`). + Exec(ctx) + if err := requireRows(result, err, "reopening failed storage upload copy"); err != nil { + return err + } + // The next attempt may choose a different source, so the request this copy + // already sent is abandoned rather than reused. Its unresolved slot has to + // be freed inside this transaction or the next reservation is refused. + if _, err := db.NewUpdate(). + Model((*storagepull.Attempt)(nil)). + Set("status = ?", storagepull.AttemptStatusAbandoned). + Set("resolved_at = ?", now). + Set("updated_at = ?", now). + Where("resolved_at IS NULL"). + Where(`(content_id, storage_data_set_id) IN ( + SELECT reopened_copy.content_id, reopened_copy.storage_data_set_id + FROM storage_copies AS reopened_copy WHERE reopened_copy.id = ? + )`, copyID). + Exec(ctx); err != nil { + return fmt.Errorf("abandoning storage pull attempt: %w", err) + } + return nil +} + +func countSubmittedCommitCopies(ctx context.Context, db bun.IDB, contentID int64) (int, error) { + count, err := db.NewSelect(). + Model((*model.StorageCopy)(nil)). + Where("content_id = ?", contentID). + Where(attemptedStorageCommitSQL("storage_copy")). + Count(ctx) + if err != nil { + return 0, fmt.Errorf("counting submitted storage commits: %w", err) + } + return count, nil +} + +// selectObjectVersionRefsForContent lists the live data versions backed by one +// content. Pipeline position is derived from the copies now, so callers that +// used to rewrite object_versions here only need to know which versions the +// change reached. +func selectObjectVersionRefsForContent(ctx context.Context, db bun.IDB, contentID int64, versionID string) ([]ObjectVersionRef, error) { + var refs []ObjectVersionRef + q := db.NewSelect(). + Table("object_versions"). + Column("object_id", "version_id", "content_id"). + Where("content_id = ?", contentID). + Where("is_delete_marker = ?", false) + if versionID != "" { + q = q.Where("version_id = ?", versionID) + } + if err := q.Scan(ctx, &refs); err != nil && err != sql.ErrNoRows { + return nil, err + } + return refs, nil +} + +// BindReadableUploadForContent records that one content became readable at a +// provider. Versions already point at the content from creation, so this only +// settles the content row and reports the versions it backs. +func (r *BunStorageContentRepo) BindReadableUploadForContent(ctx context.Context, input BindReadableUploadInput) ([]ObjectVersionRef, error) { + return r.bindReadableContent(ctx, input.ContentID, input.BucketID, "") +} + +// BindReadableUploadForVersion is BindReadableUploadForContent scoped to the +// refs of a single version. +func (r *BunStorageContentRepo) BindReadableUploadForVersion(ctx context.Context, input BindReadableUploadForVersionInput) ([]ObjectVersionRef, error) { + return r.bindReadableContent(ctx, input.ContentID, input.BucketID, input.VersionID) +} + +func (r *BunStorageContentRepo) bindReadableContent(ctx context.Context, contentID, bucketID int64, versionID string) ([]ObjectVersionRef, error) { + var refs []ObjectVersionRef + err := r.runMaybeTx(ctx, func(db bun.IDB) error { + content, err := lockStorageContentForObjectState(ctx, db, contentID, model.ObjectStateReplicating) + if err != nil { + return fmt.Errorf("locking storage content for readable bind: %w", err) + } + if content.BucketID != bucketID { + return fmt.Errorf("storage content %d belongs to bucket %d, not %d: %w", content.ID, content.BucketID, bucketID, ErrConflict) + } + if err := updateUploadReadable(ctx, db, contentID, derefString(content.PieceCID), time.Now()); err != nil { + return err + } + refs, err = selectObjectVersionRefsForContent(ctx, db, contentID, versionID) + if err != nil { + return fmt.Errorf("listing object versions for readable content: %w", err) + } + return nil + }) + return refs, err +} + +func (r *BunStorageContentRepo) FinalizeUploadIfTargetCopiesMet(ctx context.Context, input FinalizeUploadInput) (bool, []ObjectVersionRef, error) { + var refs []ObjectVersionRef + finalized := false + err := r.runMaybeTx(ctx, func(db bun.IDB) error { + var bucketID int64 + if err := db.NewSelect(). + Model((*model.StorageContent)(nil)). + Column("bucket_id"). + Where("id = ?", input.ContentID). + Scan(ctx, &bucketID); err != nil { + if err == sql.ErrNoRows { + return fmt.Errorf("loading storage upload %d: %w", input.ContentID, ErrNotFound) + } + return fmt.Errorf("loading storage upload bucket: %w", err) + } + bucket, err := lockBucketByID(ctx, db, bucketID) + if err != nil { + return err + } + if bucket == nil { + return fmt.Errorf("storage upload %d bucket not found: %w", input.ContentID, ErrNotFound) + } + locked, err := lockStorageContentsByID(ctx, db, []int64{input.ContentID}) + if err != nil { + return fmt.Errorf("locking storage upload for finalization: %w", err) + } + upload := locked[input.ContentID] + if upload == nil { + return fmt.Errorf("storage upload %d cannot be finalized: %w", input.ContentID, ErrConflict) + } + readable, err := countReadableReplicaSlots(ctx, db, input.ContentID) + if err != nil { + return err + } + minimum := minimumDurableCopiesForUpload(bucket, upload.RequestedCopies) + if minimum <= 0 || readable < minimum { + return nil + } + now := time.Now() + // Reaching the durability minimum is a fact about the copies, so no + // version row changes; the refs only tell callers who is affected. + refs, err = selectObjectVersionRefsForContent(ctx, db, input.ContentID, "") + if err != nil { + return fmt.Errorf("listing durable object versions: %w", err) + } + if upload.RequestedCopies <= 0 || readable < upload.RequestedCopies { + return nil + } + _, err = db.NewUpdate(). + Model((*model.StorageContent)(nil)). + Set("accepted_at = COALESCE(accepted_at, ?)", now). + Set("updated_at = ?", now). + Where("id = ?", input.ContentID). + Exec(ctx) + if err != nil { + return fmt.Errorf("marking storage upload complete: %w", err) + } + finalized = true + return nil + }) + return finalized, refs, err +} + +func minimumDurableCopiesForUpload(bucket *model.Bucket, requestedCopies int) int { + if requestedCopies <= 0 { + return 0 + } + if bucket == nil || bucket.MinimumDurableCopies >= requestedCopies { + return requestedCopies + } + return bucket.MinimumDurableCopies +} + +func (r *BunStorageContentRepo) runMaybeTx(ctx context.Context, fn func(bun.IDB) error) error { + return runMaybeTx(ctx, r.db, fn) +} + +// Generations are numbered per replica slot and never reused, so a retired +// generation stays distinguishable from the one that replaced it. +func nextDataSetGeneration(ctx context.Context, db bun.IDB, bucketID int64, copyIndex int) (int64, error) { + var generation int64 + err := db.NewRaw( + `SELECT COALESCE(MAX(generation), 0) + 1 FROM storage_data_sets WHERE bucket_id = ? AND copy_index = ?`, + bucketID, copyIndex, + ).Scan(ctx, &generation) + if err != nil { + return 0, fmt.Errorf("selecting next storage data set generation: %w", err) + } + return generation, nil +} + +func ensureDataSetBinding(ctx context.Context, db bun.IDB, input EnsureDataSetBindingInput) (*model.StorageDataSet, error) { + if input.BucketID == 0 || input.ProviderID.IsZero() || input.CopyIndex < 0 { + return nil, fmt.Errorf("invalid storage data set binding input: %w", ErrInvalidInput) + } + // A provider remains reserved until its old data set is verifiably retired. + // The current-slot rule is intentionally narrower so replacement can keep a + // draining source and a current target in the same logical slot. + existingByProvider := new(model.StorageDataSet) + err := db.NewSelect(). + Model(existingByProvider). + Where("bucket_id = ? AND provider_id = ? AND status <> ?", input.BucketID, input.ProviderID, model.StorageDataSetStatusRetired). + Scan(ctx) + if err == nil { + if !existingByProvider.IsCurrent { + return nil, fmt.Errorf("provider %s remains reserved by non-retired data set generation %d: %w", input.ProviderID, existingByProvider.ID, ErrAlreadyExists) + } + if existingByProvider.CopyIndex != input.CopyIndex { + return nil, fmt.Errorf("provider %s already bound to copy_index %d: %w", input.ProviderID, existingByProvider.CopyIndex, ErrAlreadyExists) + } + return existingByProvider, nil + } + if err != sql.ErrNoRows { + return nil, fmt.Errorf("selecting storage data set by provider: %w", err) + } + existingByIndex := new(model.StorageDataSet) + err = db.NewSelect(). + Model(existingByIndex). + Where("bucket_id = ? AND copy_index = ? AND is_current", input.BucketID, input.CopyIndex). + Scan(ctx) + if err == nil { + return nil, fmt.Errorf("copy_index %d already bound to provider %s: %w", input.CopyIndex, existingByIndex.ProviderID, ErrAlreadyExists) + } + if err != sql.ErrNoRows { + return nil, fmt.Errorf("selecting storage data set by copy index: %w", err) + } + generation, err := nextDataSetGeneration(ctx, db, input.BucketID, input.CopyIndex) + if err != nil { + return nil, err + } + now := time.Now() + binding := &model.StorageDataSet{ + BucketID: input.BucketID, + ProviderID: input.ProviderID, + CopyIndex: input.CopyIndex, + Generation: generation, + IsCurrent: true, + Status: model.StorageDataSetStatusPending, + CreatedByContentID: nullableInt64(input.CreatedByContentID), + LastUsedContentID: nullableInt64(input.CreatedByContentID), + CreatedAt: now, + UpdatedAt: now, + } + res, err := db.NewInsert(). + Model(binding). + On("CONFLICT DO NOTHING"). + Exec(ctx) + if err != nil { + return nil, fmt.Errorf("inserting storage data set binding: %w", err) + } + rows, _ := res.RowsAffected() + if rows == 0 { + existing := new(model.StorageDataSet) + selectErr := db.NewSelect(). + Model(existing). + Where("bucket_id = ? AND provider_id = ? AND status <> ?", input.BucketID, input.ProviderID, model.StorageDataSetStatusRetired). + Scan(ctx) + if selectErr == nil { + if !existing.IsCurrent { + return nil, fmt.Errorf("provider %s remains reserved by non-retired data set generation %d: %w", input.ProviderID, existing.ID, ErrAlreadyExists) + } + if existing.CopyIndex == input.CopyIndex { + return existing, nil + } + return nil, fmt.Errorf("provider %s already bound to copy_index %d: %w", input.ProviderID, existing.CopyIndex, ErrAlreadyExists) + } + if selectErr != nil && selectErr != sql.ErrNoRows { + return nil, fmt.Errorf("selecting storage data set after conflict: %w", selectErr) + } + return nil, fmt.Errorf("storage data set binding already exists: %w", ErrAlreadyExists) + } + return binding, nil +} + +func markDataSetReady(ctx context.Context, db bun.IDB, id int64, contentID int64, dataSetID types.OnChainID, clientDataSetID *types.OnChainID) error { + if dataSetID.IsZero() { + return fmt.Errorf("dataSetID is required: %w", ErrInvalidInput) + } + res, err := db.NewUpdate(). + Model((*model.StorageDataSet)(nil)). + Set("status = ?", model.StorageDataSetStatusReady). + Set("data_set_id = ?", dataSetID). + Set("client_data_set_id = COALESCE(?, client_data_set_id)", clientDataSetID). + Set("last_used_content_id = ?", nullableInt64(contentID)). + Set("last_error = NULL"). + Set("updated_at = ?", time.Now()). + Where("id = ?", id). + Where(`NOT EXISTS ( + SELECT 1 FROM storage_data_sets AS other + WHERE other.id <> ? + AND other.provider_id = (SELECT provider_id FROM storage_data_sets WHERE id = ?) + AND other.data_set_id = ? + )`, id, id, dataSetID). + Exec(ctx) + if err != nil { + return fmt.Errorf("marking storage data set ready: %w", err) + } + rows, _ := res.RowsAffected() + if rows > 0 { + return nil + } + count, err := db.NewSelect(). + Model((*model.StorageDataSet)(nil)). + Where("id = ?", id). + Count(ctx) + if err != nil { + return fmt.Errorf("checking storage data set ready result: %w", err) + } + if count == 0 { + return fmt.Errorf("storage data set %d not found: %w", id, ErrNotFound) + } + return fmt.Errorf("provider data set already bound to another bucket: %w", ErrAlreadyExists) +} + +// The result counts logical replica slots, so several data set generations of +// one slot never inflate an upload's durability. +func countReadableReplicaSlots(ctx context.Context, db bun.IDB, contentID int64) (int, error) { + var row struct { + Count int `bun:"count"` + } + err := db.NewRaw(fmt.Sprintf(`SELECT %s AS count`, + distinctReadableSlotCountSQL("storage_copy", "storage_data_set", "?"), + ), + contentID, + ).Scan(ctx, &row) + if err != nil { + return 0, fmt.Errorf("counting readable replica slots: %w", err) + } + return row.Count, nil +} + +func requireReadableCommittedCopy(ctx context.Context, db bun.IDB, contentID int64) error { + count, err := countReadableReplicaSlots(ctx, db, contentID) + if err != nil { + return err + } + if count == 0 { + return fmt.Errorf("storage upload %d has no readable committed copy: %w", contentID, ErrNotFound) + } + return nil +} + +// Addressing a copy by slot alone became ambiguous once a slot can own several +// generations, so every write resolves to one concrete row first. Returning +// zero means the slot has no copy yet; an ambiguous slot is a conflict rather +// than a silent multi-row update. +// slotCopyTarget picks the concrete copy a mutation must touch. Zero means the +// slot has no copy, which every caller already handles as "no rows updated". +func slotCopyTarget(ctx context.Context, db bun.IDB, copyID, contentID int64, copyIndex int) (int64, error) { + if copyID > 0 { + return copyID, nil + } + return resolveSlotCopyID(ctx, db, contentID, copyIndex) +} + +func resolveSlotCopyID(ctx context.Context, db bun.IDB, contentID int64, copyIndex int) (int64, error) { + var ids []int64 + err := db.NewSelect(). + Model((*model.StorageCopy)(nil)). + Column("id"). + Where("storage_copy.content_id = ? AND storage_copy.copy_index = ?", contentID, copyIndex). + Where(currentGenerationCopySQL("storage_copy")). + Scan(ctx, &ids) + if err != nil { + return 0, fmt.Errorf("resolving storage upload copy for slot: %w", err) + } + switch len(ids) { + case 0: + return 0, nil + case 1: + return ids[0], nil + default: + return 0, fmt.Errorf( + "storage upload %d replica slot %d matches %d copies: %w", + contentID, copyIndex, len(ids), ErrConflict, + ) + } +} + +func uploadCopyTransferMethod(ctx context.Context, db bun.IDB, contentID int64, copyIndex int) (model.StorageCopyTransferMethod, error) { + var row struct { + TransferMethod model.StorageCopyTransferMethod `bun:"transfer_method"` + } + err := db.NewSelect(). + Model((*model.StorageCopy)(nil)). + Column("transfer_method"). + Where("storage_copy.content_id = ? AND storage_copy.copy_index = ?", contentID, copyIndex). + Where(currentGenerationCopySQL("storage_copy")). + Limit(1). + Scan(ctx, &row) + if err != nil { + if err == sql.ErrNoRows { + return "", nil + } + return "", fmt.Errorf("loading storage upload copy transfer method: %w", err) + } + return row.TransferMethod, nil +} + +func updateUploadIngressReady(ctx context.Context, db bun.IDB, contentID int64, pieceCID string, now time.Time) error { + _, err := db.NewUpdate(). + Model((*model.StorageContent)(nil)). + Set("piece_cid = COALESCE(?, piece_cid)", nullableString(pieceCID)). + Set("error_message = NULL"). + Set("updated_at = ?", now). + Where("id = ?", contentID). + Where("accepted_at IS NULL"). + Exec(ctx) + if err != nil { + return fmt.Errorf("marking storage upload ingress ready: %w", err) + } + return nil +} + +func updateUploadReadable(ctx context.Context, db bun.IDB, contentID int64, pieceCID string, now time.Time) error { + _, err := db.NewUpdate(). + Model((*model.StorageContent)(nil)). + Set("piece_cid = COALESCE(?, piece_cid)", nullableString(pieceCID)). + Set("error_message = NULL"). + Set("updated_at = ?", now). + Where("id = ?", contentID). + Exec(ctx) + if err != nil { + return fmt.Errorf("marking storage upload readable: %w", err) + } + return nil +} + +func nullableString(value string) *string { + if value == "" { + return nil + } + return &value +} + +func nullableInt64(value int64) *int64 { + if value == 0 { + return nil + } + return &value +} + +func derefString(value *string) string { + if value == nil { + return "" + } + return *value +} diff --git a/internal/db/repository/storage_upload_seed_test.go b/internal/db/repository/storage_content_seed_test.go similarity index 51% rename from internal/db/repository/storage_upload_seed_test.go rename to internal/db/repository/storage_content_seed_test.go index ce02f49..7f394cf 100644 --- a/internal/db/repository/storage_upload_seed_test.go +++ b/internal/db/repository/storage_content_seed_test.go @@ -6,7 +6,9 @@ import ( "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/testutil" "github.com/strahe/synaps3/internal/types" + "github.com/uptrace/bun" ) type storageUploadCopySeed struct { @@ -19,7 +21,7 @@ type storageUploadCopySeed struct { IsNewDataSet bool } -func seedCommittedUploadCopies(t *testing.T, repos *repository.Repositories, bucketID int64, uploadID int64, pieceCID string, copies []storageUploadCopySeed) { +func seedCommittedUploadCopies(t *testing.T, db *bun.DB, repos *repository.Repositories, bucketID int64, contentID int64, pieceCID string, copies []storageUploadCopySeed) { t.Helper() ctx := context.Background() copyInputs := make([]repository.UploadCopyBindingInput, 0, len(copies)) @@ -32,23 +34,23 @@ func seedCommittedUploadCopies(t *testing.T, repos *repository.Repositories, buc if copySeed.ProviderID != nil { providerID = *copySeed.ProviderID } - createdByUploadID := int64(0) + createdByContentID := int64(0) if copySeed.IsNewDataSet { - createdByUploadID = uploadID + createdByContentID = contentID } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucketID, - ProviderID: providerID, - CopyIndex: copyIndex, - CreatedByUploadID: createdByUploadID, + binding, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucketID, + ProviderID: providerID, + CopyIndex: copyIndex, + CreatedByContentID: createdByContentID, }) if err != nil { t.Fatalf("EnsureDataSetBinding: %v", err) } if copySeed.DataSetID != nil { - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + if err := repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ ID: binding.ID, - UploadID: uploadID, + ContentID: contentID, DataSetID: *copySeed.DataSetID, }); err != nil { t.Fatalf("MarkDataSetReady: %v", err) @@ -70,7 +72,7 @@ func seedCommittedUploadCopies(t *testing.T, repos *repository.Repositories, buc copies[i].CopyIndex = copyIndex copies[i].TransferMethod = transferMethod } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, uploadID, copyInputs); err != nil { + if err := repos.Contents.CreateUploadCopiesForBindings(ctx, contentID, copyInputs); err != nil { t.Fatalf("CreateUploadCopiesForBindings: %v", err) } for _, copySeed := range copies { @@ -78,40 +80,12 @@ func seedCommittedUploadCopies(t *testing.T, repos *repository.Repositories, buc if copySeed.RetrievalURL != nil { retrievalURL = *copySeed.RetrievalURL } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: uploadID, + testutil.CommitStorageCopy(t, db, repos, repository.MarkUploadCopyCommittedInput{ + ContentID: contentID, CopyIndex: copySeed.CopyIndex, PieceCID: pieceCID, PieceID: copySeed.PieceID, RetrievalURL: retrievalURL, - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - } -} - -func bindReadableUploadForContent(t *testing.T, repos *repository.Repositories, uploadID int64, bucketID int64, size int64, checksum string) []repository.ObjectVersionRef { - t.Helper() - refs, err := repos.Uploads.BindReadableUploadForContent(context.Background(), repository.BindReadableUploadInput{ - UploadID: uploadID, - BucketID: bucketID, - ContentSize: size, - Checksum: checksum, - }) - if err != nil { - t.Fatalf("BindReadableUploadForContent: %v", err) - } - return refs -} - -func finalizeUploadForTest(t *testing.T, repos *repository.Repositories, uploadID int64) []repository.ObjectVersionRef { - t.Helper() - finalized, refs, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(context.Background(), repository.FinalizeUploadInput{UploadID: uploadID}) - if err != nil { - t.Fatalf("FinalizeUploadIfTargetCopiesMet: %v", err) - } - if !finalized { - t.Fatalf("FinalizeUploadIfTargetCopiesMet finalized = false, want true") + }) } - return refs } diff --git a/internal/db/repository/storage_domain_helpers_test.go b/internal/db/repository/storage_domain_helpers_test.go new file mode 100644 index 0000000..4a8bc00 --- /dev/null +++ b/internal/db/repository/storage_domain_helpers_test.go @@ -0,0 +1,680 @@ +package repository_test + +import ( + "context" + "errors" + "slices" + "strings" + "testing" + "time" + + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/observability" + "github.com/strahe/synaps3/internal/storagereplacement" + "github.com/strahe/synaps3/internal/testutil" + "github.com/uptrace/bun" +) + +func TestStartObjectUploadAttemptValidatesRequiredIdentity(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + bucket := seedBucket(t, db, "upload-identity") + valid := repository.EnsureContentInput{ + BucketID: bucket.ID, + ContentSize: 1, + Checksum: testutil.StorageChecksum("checksum-1"), + RequestedCopies: 1, + } + tests := []struct { + name string + mutate func(*repository.EnsureContentInput) + }{ + {name: "bucket", mutate: func(input *repository.EnsureContentInput) { input.BucketID = 0 }}, + {name: "content size", mutate: func(input *repository.EnsureContentInput) { input.ContentSize = -1 }}, + {name: "empty checksum", mutate: func(input *repository.EnsureContentInput) { input.Checksum = "" }}, + {name: "short checksum", mutate: func(input *repository.EnsureContentInput) { input.Checksum = strings.Repeat("a", 63) }}, + {name: "uppercase checksum", mutate: func(input *repository.EnsureContentInput) { input.Checksum = strings.Repeat("A", 64) }}, + {name: "prefixed checksum", mutate: func(input *repository.EnsureContentInput) { input.Checksum = "sha256:" + strings.Repeat("a", 64) }}, + {name: "non hex checksum", mutate: func(input *repository.EnsureContentInput) { input.Checksum = strings.Repeat("a", 63) + "g" }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + input := valid + tt.mutate(&input) + if _, err := repos.Contents.EnsureContent(t.Context(), input); !errors.Is(err, repository.ErrInvalidInput) { + t.Fatalf("EnsureContent error = %v, want ErrInvalidInput", err) + } + }) + } + count, err := db.NewSelect().Model((*model.StorageContent)(nil)).Count(t.Context()) + if err != nil { + t.Fatalf("count storage uploads: %v", err) + } + if count != 0 { + t.Fatalf("storage upload count = %d after invalid writes, want 0", count) + } +} + +func TestListCopiesDerivesNewDataSetFromCreatorContent(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + bucket := seedBucket(t, db, "copy-data-set-creator") + creator, err := repos.Contents.EnsureContent(t.Context(), repository.EnsureContentInput{ + BucketID: bucket.ID, ContentSize: 1, + Checksum: testutil.StorageChecksum("creator-content"), RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("EnsureContent(creator): %v", err) + } + reused, err := repos.Contents.EnsureContent(t.Context(), repository.EnsureContentInput{ + BucketID: bucket.ID, ContentSize: 2, + Checksum: testutil.StorageChecksum("reused-content"), RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("EnsureContent(reused): %v", err) + } + binding, err := repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, + CreatedByContentID: creator.ID, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + for _, tc := range []struct { + content *model.StorageContent + transferMethod model.StorageCopyTransferMethod + wantNew bool + }{ + {content: creator, transferMethod: model.StorageCopyTransferMethodIngress, wantNew: true}, + {content: reused, transferMethod: model.StorageCopyTransferMethodPeerPull, wantNew: false}, + } { + if err := repos.Contents.CreateUploadCopiesForBindings(t.Context(), tc.content.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: binding.ID, + CopyIndex: binding.CopyIndex, + ProviderID: binding.ProviderID, + TransferMethod: tc.transferMethod, + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings(%d): %v", tc.content.ID, err) + } + copies, err := repos.Contents.ListCopies(t.Context(), tc.content.ID) + if err != nil { + t.Fatalf("ListCopies(%d): %v", tc.content.ID, err) + } + if len(copies) != 1 || copies[0].IsNewDataSet != tc.wantNew { + t.Fatalf("ListCopies(%d) = %+v, want one copy with is_new_data_set=%t", tc.content.ID, copies, tc.wantNew) + } + } +} + +func TestAuthorizeReplacementPreservesCurrentSource(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + bucket := seedBucket(t, db, "replacement-current-source") + source, err := repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, + ProviderID: onChainID(t, "101"), + CopyIndex: 0, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + + replacement, created, err := repos.Replacements.Authorize(t.Context(), repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, + SourceDataSetID: source.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, "202"), + ClientRequestID: "replacement-current-source", + }) + if err != nil { + t.Fatalf("Authorize: %v", err) + } + if !created { + t.Fatal("Authorize created = false, want true") + } + target, err := repos.Contents.GetDataSetBindingByID(t.Context(), replacement.TargetDataSetID) + if err != nil { + t.Fatalf("GetDataSetBindingByID(target): %v", err) + } + if target == nil { + t.Fatal("replacement target is nil") + } + if target.IsCurrent { + t.Fatal("replacement target is current before activation") + } + currentSource, err := repos.Contents.GetDataSetBindingByID(t.Context(), source.ID) + if err != nil { + t.Fatalf("GetDataSetBindingByID(source): %v", err) + } + if currentSource == nil || !currentSource.IsCurrent { + t.Fatalf("replacement source = %#v, want current source", currentSource) + } +} + +func TestAttachReplacementTargetCopyRejectsMismatchedUpload(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + bucket := seedBucket(t, db, "replacement-item-upload-identity") + source, err := repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + replacement, _, err := repos.Replacements.Authorize(t.Context(), repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, "202"), ClientRequestID: "replacement-item-upload-identity", + }) + if err != nil { + t.Fatalf("Authorize: %v", err) + } + target, err := repos.Contents.GetDataSetBindingByID(t.Context(), replacement.TargetDataSetID) + if err != nil || target == nil { + t.Fatalf("GetDataSetBindingByID(target) = %#v, err=%v", target, err) + } + itemUpload := startCopyHealthUpload(t, repos, bucket.ID, "replacement-item-upload", 1, "replacement-item-checksum", 1) + wrongUpload := startCopyHealthUpload(t, repos, bucket.ID, "replacement-wrong-upload", 1, "replacement-wrong-checksum", 1) + if err := repos.Contents.CreateUploadCopiesForBindings(t.Context(), itemUpload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: target.ID, CopyIndex: target.CopyIndex, + ProviderID: target.ProviderID, TransferMethod: model.StorageCopyTransferMethodPeerPull, + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings(item upload): %v", err) + } + item := &storagereplacement.Item{ + ReplacementID: replacement.ID, + ContentID: itemUpload.ID, + TargetDataSetID: target.ID, + Status: storagereplacement.ItemStatusPending, + } + if _, err := db.NewInsert().Model(item).Exec(t.Context()); err != nil { + t.Fatalf("insert replacement item: %v", err) + } + + if _, err := repos.Replacements.AttachTargetCopy(t.Context(), repository.AttachReplacementTargetCopyInput{ + ReplacementID: replacement.ID, + ItemID: item.ID, + ContentID: wrongUpload.ID, + }); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("AttachTargetCopy error = %v, want ErrConflict", err) + } + count, err := db.NewSelect(). + Model((*model.StorageCopy)(nil)). + Where("content_id = ? AND storage_data_set_id = ?", wrongUpload.ID, target.ID). + Count(t.Context()) + if err != nil { + t.Fatalf("count rolled-back wrong target copies: %v", err) + } + if count != 0 { + t.Fatalf("wrong upload target copy count = %d, want 0", count) + } +} + +func TestEnsureDataSetBindingDoesNotReuseProviderBeforeRetirement(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + bucket := seedBucket(t, db, "provider-retirement-occupancy") + providerID := onChainID(t, "101") + source, err := repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: providerID, CopyIndex: 0, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding(source): %v", err) + } + if err := repos.Contents.MarkDataSetReady(t.Context(), repository.MarkDataSetReadyInput{ + ID: source.ID, DataSetID: onChainID(t, "1001"), + }); err != nil { + t.Fatalf("MarkDataSetReady: %v", err) + } + if err := repos.Contents.MarkDataSetDraining(t.Context(), source.ID, "replacement in progress"); err != nil { + t.Fatalf("MarkDataSetDraining: %v", err) + } + + if _, err := repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: providerID, CopyIndex: 0, + }); !errors.Is(err, repository.ErrAlreadyExists) { + t.Fatalf("reuse draining provider error = %v, want ErrAlreadyExists", err) + } + if _, err := db.NewUpdate().Model((*model.StorageDataSet)(nil)). + Set("status = ?", model.StorageDataSetStatusRetired). + Where("id = ?", source.ID). + Exec(t.Context()); err != nil { + t.Fatalf("retire source data set: %v", err) + } + + reused, err := repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: providerID, CopyIndex: 0, + }) + if err != nil { + t.Fatalf("reuse retired provider: %v", err) + } + if reused.ID == source.ID || reused.Generation <= source.Generation { + t.Fatalf("reused data set = %#v, want a newer generation than %#v", reused, source) + } +} + +func TestRetryReplacementRejectsUnknownFailureReason(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + bucket := seedBucket(t, db, "replacement-unknown-reason") + source, err := repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + replacement, _, err := repos.Replacements.Authorize(t.Context(), repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, "202"), ClientRequestID: "unknown-failure-reason", + }) + if err != nil { + t.Fatalf("Authorize: %v", err) + } + if _, err := db.NewUpdate().Model((*storagereplacement.Replacement)(nil)). + Set("status = ?", storagereplacement.StatusFailed). + Set("failure_reason = ?", "future_failure_reason"). + Set("last_error = ?", "written by a newer binary"). + Where("id = ?", replacement.ID). + Exec(t.Context()); err != nil { + t.Fatalf("store future failure reason: %v", err) + } + if _, err := repos.Replacements.Retry(t.Context(), repository.RetryReplacementInput{ + ReplacementID: replacement.ID, + }); !errors.Is(err, storagereplacement.ErrNotRetryable) { + t.Fatalf("Retry error = %v, want ErrNotRetryable", err) + } +} + +// Retiring a rejected generation frees its provider without erasing anything. +// Deleting it would have to refuse whenever a copy was bound to it, which is +// exactly when the provider is most worth getting back. +func TestRetireRejectedDataSetKeepsTheRowAndItsCopies(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := t.Context() + bucket := seedBucket(t, db, "retire-rejected-candidate") + upload := startCopyHealthUpload(t, repos, bucket.ID, "retire-version", 1, "retire-checksum", 1) + binding := ensureCopyHealthBinding(t, repos, bucket.ID, upload.ID, 0, "101") + if err := repos.Contents.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: binding.ID, CopyIndex: 0, ProviderID: binding.ProviderID, + TransferMethod: model.StorageCopyTransferMethodIngress, + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings: %v", err) + } + if _, err := db.NewUpdate().Model((*model.StorageCopy)(nil)). + Set("status = ?", model.StorageCopyStatusFailed). + Where("content_id = ? AND storage_data_set_id = ?", upload.ID, binding.ID). + Exec(ctx); err != nil { + t.Fatalf("mark candidate copy failed: %v", err) + } + if err := repos.Contents.MarkDataSetFailed(ctx, binding.ID, "creation refused"); err != nil { + t.Fatalf("MarkDataSetFailed: %v", err) + } + + retired, err := repos.Contents.RetireRejectedDataSet(ctx, binding.ID) + if err != nil || !retired { + t.Fatalf("RetireRejectedDataSet = %t, err=%v", retired, err) + } + kept, err := repos.Contents.GetDataSetBindingByID(ctx, binding.ID) + if err != nil || kept == nil || kept.Status != model.StorageDataSetStatusRetired { + t.Fatalf("retired generation = %#v err=%v, want the row kept as retired", kept, err) + } + // The copy is what records that ingest was attempted here. + copyRow, err := repos.Contents.GetUploadCopyForDataSet(ctx, upload.ID, binding.ID) + if err != nil || copyRow == nil { + t.Fatalf("retained copy = %#v err=%v", copyRow, err) + } + // And the provider is what retirement buys back. + reused, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: binding.ProviderID, CopyIndex: 0, + }) + if err != nil || reused == nil || reused.ID == binding.ID { + t.Fatalf("rebinding the freed provider = %#v err=%v", reused, err) + } +} + +func TestStorageContentBindingRejectsCrossBucketIdentity(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + sourceBucket := seedBucket(t, db, "upload-source-bucket") + targetBucket := seedBucket(t, db, "upload-target-bucket") + version := newObjectVersion(targetBucket.ID, "cross-bucket.txt", model.NewVersionID(), 11) + if _, err := createVersion(t, repos, version); err != nil { + t.Fatalf("CreateVersionAndSetCurrent: %v", err) + } + content := startCopyHealthUpload(t, repos, sourceBucket.ID, "source-version", version.Size, "cross-bucket-checksum", 1) + commitStorageHealthCopy(t, db, repos, sourceBucket.ID, content.ID, 0, "901", "1901", "2901", "https://provider.example/cross-bucket") + + if _, err := repos.Contents.BindReadableUploadForVersion(t.Context(), repository.BindReadableUploadForVersionInput{ + ContentID: content.ID, BucketID: targetBucket.ID, VersionID: version.VersionID, + }); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("BindReadableUploadForVersion error = %v, want ErrConflict", err) + } + + if _, err := db.NewRaw(`UPDATE object_versions + SET content_id = ? + WHERE version_id = ?`, content.ID, version.VersionID).Exec(t.Context()); err == nil { + t.Fatal("direct cross-bucket object version binding succeeded") + } + if _, err := db.NewRaw(`INSERT INTO object_deletions + (bucket_id, object_id, key, version_id, content_id, size, deleted_at) + VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)`, + targetBucket.ID, 1, "cross-bucket.txt", model.NewVersionID(), content.ID, 11).Exec(t.Context()); err == nil { + t.Fatal("direct cross-bucket object deletion binding succeeded") + } +} + +func startCopyHealthUpload( + t *testing.T, + repos *repository.Repositories, + bucketID int64, + versionID string, + size int64, + checksum string, + requestedCopies int, +) *model.StorageContent { + t.Helper() + origin, err := repos.Objects.GetVersionByID(t.Context(), versionID) + if err != nil { + t.Fatalf("GetVersionByID(origin): %v", err) + } + if origin == nil { + origin = newObjectVersion(bucketID, "origin-"+versionID, versionID, size) + origin.Checksum = checksum + if _, err := createVersion(t, repos, origin); err != nil { + t.Fatalf("CreateVersionAndSetCurrent(origin): %v", err) + } + } + upload, err := repos.Contents.EnsureContent(context.Background(), repository.EnsureContentInput{ + BucketID: bucketID, ContentSize: size, + Checksum: testutil.StorageChecksum(checksum), RequestedCopies: requestedCopies, + }) + if err != nil { + t.Fatalf("EnsureContent: %v", err) + } + return upload +} + +func ensureCopyHealthBinding( + t *testing.T, + repos *repository.Repositories, + bucketID, contentID int64, + copyIndex int, + providerID string, +) *model.StorageDataSet { + t.Helper() + binding, err := repos.Contents.EnsureDataSetBinding(context.Background(), repository.EnsureDataSetBindingInput{ + BucketID: bucketID, ProviderID: onChainID(t, providerID), CopyIndex: copyIndex, + CreatedByContentID: contentID, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + return binding +} + +func commitStorageHealthCopy( + t *testing.T, + db *bun.DB, + repos *repository.Repositories, + bucketID, contentID int64, + copyIndex int, + providerID, dataSetID, pieceID, retrievalURL string, +) *model.StorageDataSet { + t.Helper() + binding := ensureCopyHealthBinding(t, repos, bucketID, contentID, copyIndex, providerID) + if err := repos.Contents.MarkDataSetReady(context.Background(), repository.MarkDataSetReadyInput{ + ID: binding.ID, ContentID: contentID, DataSetID: onChainID(t, dataSetID), + }); err != nil { + t.Fatalf("MarkDataSetReady: %v", err) + } + if err := repos.Contents.CreateUploadCopiesForBindings(context.Background(), contentID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: binding.ID, CopyIndex: copyIndex, + TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, providerID), + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings: %v", err) + } + testutil.CommitStorageCopy(t, db, repos, repository.MarkUploadCopyCommittedInput{ + ContentID: contentID, CopyIndex: copyIndex, PieceCID: "bafk2bzacestorhealth", + PieceID: onChainIDPtr(t, pieceID), RetrievalURL: retrievalURL, + }) + got, err := repos.Contents.GetDataSetBindingByID(context.Background(), binding.ID) + if err != nil { + t.Fatalf("GetDataSetBindingByID: %v", err) + } + return got +} + +func bindStorageHealthVersion( + t *testing.T, + repos *repository.Repositories, + bucketID, contentID int64, + version *model.ObjectVersion, +) { + t.Helper() + if _, err := repos.Contents.BindReadableUploadForVersion(context.Background(), repository.BindReadableUploadForVersionInput{ + ContentID: contentID, BucketID: bucketID, VersionID: version.VersionID, + }); err != nil { + t.Fatalf("BindReadableUploadForVersion: %v", err) + } +} + +// A termination is a ledger row, so every path that asks "has the end of term +// been recorded?" has to read that row. The retirement gate blocks forever if it +// reads the replacement alone. +func TestRecordedTerminationEpochIsVisibleToTheRetirementGate(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + bucket := seedBucket(t, db, "replacement-termination-evidence") + source, err := repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + replacement, _, err := repos.Replacements.Authorize(t.Context(), repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, "202"), ClientRequestID: "termination-evidence", + }) + if err != nil { + t.Fatalf("Authorize: %v", err) + } + if _, err := db.NewUpdate().Model((*storagereplacement.Replacement)(nil)). + Set("status = ?", storagereplacement.StatusRetiring). + Where("id = ?", replacement.ID). + Exec(t.Context()); err != nil { + t.Fatalf("move replacement to retiring: %v", err) + } + if err := repos.Replacements.RecordTerminationEpoch(t.Context(), repository.RecordTerminationEpochInput{ + ReplacementID: replacement.ID, TxHash: "0xterminate", Epoch: 84, + }); err != nil { + t.Fatalf("RecordTerminationEpoch: %v", err) + } + // Recording twice would pay for a second termination transaction. + if err := repos.Replacements.RecordTerminationEpoch(t.Context(), repository.RecordTerminationEpochInput{ + ReplacementID: replacement.ID, TxHash: "0xterminate-again", Epoch: 85, + }); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("second RecordTerminationEpoch error = %v, want ErrConflict", err) + } + + stored, err := repos.Replacements.GetByID(t.Context(), replacement.ID) + if err != nil || stored == nil || stored.TerminationEpoch == nil || *stored.TerminationEpoch != 84 { + t.Fatalf("stored replacement = %#v err=%v, want termination epoch 84", stored, err) + } + observed := int64(90) + gate, err := repos.Replacements.EvaluateRetirementGate(t.Context(), replacement.ID, &observed) + if err != nil { + t.Fatalf("EvaluateRetirementGate: %v", err) + } + if gate.TerminationEpoch == nil || *gate.TerminationEpoch != 84 || !gate.EpochReached { + t.Fatalf("retirement gate = %#v, want the recorded epoch 84 reached at 90", gate) + } + if slices.Contains(gate.Blockers, "termination_epoch") { + t.Fatalf("retirement gate blockers = %v, want no epoch blocker", gate.Blockers) + } +} + +// A failed generation must give up the replica slot it was holding. While it +// kept is_current, bucket provisioning saw a current failed binding, suspended +// on it every minute, and nothing in the system could ever clear it. +func TestMarkDataSetFailedReleasesTheReplicaSlot(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + bucket := seedBucket(t, db, "data-set-failure-releases-slot") + binding, err := repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + if !binding.IsCurrent { + t.Fatalf("new binding = %#v, want is_current", binding) + } + + if err := repos.Contents.MarkDataSetFailed(t.Context(), binding.ID, "creation outcome unknown"); err != nil { + t.Fatalf("MarkDataSetFailed: %v", err) + } + failed, err := repos.Contents.GetDataSetBindingByID(t.Context(), binding.ID) + if err != nil || failed == nil { + t.Fatalf("GetDataSetBindingByID: binding=%#v err=%v", failed, err) + } + if failed.Status != model.StorageDataSetStatusFailed || failed.IsCurrent { + t.Fatalf("failed binding = status:%s is_current:%t, want failed and not current", failed.Status, failed.IsCurrent) + } + + // The slot is free, so the next generation can take it. + next, err := repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: onChainID(t, "202"), CopyIndex: 0, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding after failure: %v", err) + } + if !next.IsCurrent || next.Generation <= failed.Generation { + t.Fatalf("next generation = %#v, want current and newer than %d", next, failed.Generation) + } + + // And the database refuses to put the failed row back on the slot. + if _, err := db.NewUpdate().Model((*model.StorageDataSet)(nil)). + Set("is_current = ?", true). + Where("id = ?", failed.ID). + Exec(t.Context()); err == nil { + t.Fatal("restoring is_current on a failed data set was accepted") + } +} + +// Provisioning fails before any copy exists, so reclamation has to work with +// zero copies bound to the generation. A recorded data set identity is what +// keeps a generation: a rejected transaction hash is not, because it names a +// submission the chain refused rather than storage anyone is paying for. +func TestRetireRejectedDataSetFreesTheProviderWithoutADataSetIdentity(t *testing.T) { + for _, tc := range []struct { + name string + transactionID *string + dataSetID *string + wantDiscarded bool + }{ + {name: "rejected submission", transactionID: new("0xcreate"), wantDiscarded: true}, + {name: "no submission at all", wantDiscarded: true}, + {name: "recorded a data set identity", dataSetID: new("9001"), wantDiscarded: false}, + } { + t.Run(tc.name, func(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + bucket := seedBucket(t, db, "discard-"+strings.ReplaceAll(tc.name, " ", "-")) + provider := onChainID(t, "101") + binding, err := repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: provider, CopyIndex: 0, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + update := db.NewUpdate().Model((*model.StorageDataSet)(nil)).Where("id = ?", binding.ID) + if tc.transactionID != nil { + update = update.Set("create_transaction_id = ?", *tc.transactionID) + } + if tc.dataSetID != nil { + update = update.Set("data_set_id = ?", *tc.dataSetID) + } + if tc.transactionID != nil || tc.dataSetID != nil { + if _, err := update.Exec(t.Context()); err != nil { + t.Fatalf("record creation evidence: %v", err) + } + } + if err := repos.Contents.MarkDataSetFailed(t.Context(), binding.ID, "creation refused"); err != nil { + t.Fatalf("MarkDataSetFailed: %v", err) + } + + discarded, err := repos.Contents.RetireRejectedDataSet(t.Context(), binding.ID) + if err != nil { + t.Fatalf("RetireRejectedDataSet: %v", err) + } + if discarded != tc.wantDiscarded { + t.Fatalf("discarded = %t, want %t", discarded, tc.wantDiscarded) + } + + // The provider is reusable exactly when the row is gone. + reused, err := repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: provider, CopyIndex: 0, + }) + if tc.wantDiscarded { + if err != nil || reused == nil { + t.Fatalf("rebinding the freed provider = %#v err=%v", reused, err) + } + return + } + if !errors.Is(err, repository.ErrAlreadyExists) { + t.Fatalf("rebinding a retained provider error = %v, want ErrAlreadyExists", err) + } + kept, err := repos.Contents.GetDataSetBindingByID(t.Context(), binding.ID) + if err != nil || kept == nil || kept.DataSetID == nil { + t.Fatalf("retained generation = %#v err=%v, want its data set identity kept", kept, err) + } + }) + } +} + +// A health snapshot must not keep a rejected generation holding its provider. +// The observability table cascades with the data set and is rebuilt on every +// refresh, so it is a snapshot rather than evidence, and the refresh runs far +// more often than a creation takes to fail. +func TestRetireRejectedDataSetIgnoresObservabilitySnapshots(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := t.Context() + bucket := seedBucket(t, db, "discard-with-observation") + provider := onChainID(t, "101") + binding, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: provider, CopyIndex: 0, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + if err := repos.Contents.MarkDataSetFailed(ctx, binding.ID, "creation outcome unknown"); err != nil { + t.Fatalf("MarkDataSetFailed: %v", err) + } + // The periodic refresh snapshots every data set, including one that never + // reached the chain, and it runs far more often than a failure takes. + checkedAt := time.Now().UTC() + if err := repos.Observability.ReplaceDataSetStates(ctx, checkedAt, []observability.DataSetState{{ + LocalDataSetID: binding.ID, BucketID: bucket.ID, BucketName: bucket.Name, + CopyIndex: binding.CopyIndex, ProviderID: binding.ProviderID, + Status: observability.StatusUnavailable, LocalStatus: model.StorageDataSetStatusFailed, + ReasonCodes: []observability.ReasonCode{observability.ReasonChainDataSetMissing}, + LastCheckedAt: checkedAt, Evidence: map[string]any{}, + }}); err != nil { + t.Fatalf("ReplaceDataSetStates: %v", err) + } + + discarded, err := repos.Contents.RetireRejectedDataSet(ctx, binding.ID) + if err != nil || !discarded { + t.Fatalf("RetireRejectedDataSet = %t err=%v, want the snapshot ignored", discarded, err) + } + if _, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: provider, CopyIndex: 0, + }); err != nil { + t.Fatalf("rebinding the freed provider: %v", err) + } +} diff --git a/internal/db/repository/storage_health_sql.go b/internal/db/repository/storage_health_sql.go index 2a04f47..6fd18e1 100644 --- a/internal/db/repository/storage_health_sql.go +++ b/internal/db/repository/storage_health_sql.go @@ -32,7 +32,7 @@ func storageHealthAbnormalObservationStatusListSQL() string { } func storageHealthCommittedCopyStatusSQL() string { - return storageHealthSQLLiteral(string(model.StorageUploadCopyStatusCommitted)) + return storageHealthSQLLiteral(string(model.StorageCopyStatusCommitted)) } func storageHealthAvailableObservationStatusSQL() string { @@ -67,14 +67,13 @@ var storageHealthSQLLiterals = func() map[string]struct{} { string(model.StorageDataSetStatusCreating), string(model.StorageDataSetStatusReady), string(model.StorageDataSetStatusFailed), - string(model.StorageDataSetStatusUnavailable), string(model.StorageDataSetStatusDraining), string(model.StorageDataSetStatusRetired), - string(model.StorageUploadCopyStatusPending), - string(model.StorageUploadCopyStatusPieceReady), - string(model.StorageUploadCopyStatusCommitting), - string(model.StorageUploadCopyStatusCommitted), - string(model.StorageUploadCopyStatusFailed), + string(model.StorageCopyStatusPending), + string(model.StorageCopyStatusPieceReady), + string(model.StorageCopyStatusCommitting), + string(model.StorageCopyStatusCommitted), + string(model.StorageCopyStatusFailed), string(observability.StatusAvailable), string(observability.StatusDegraded), string(observability.StatusUnavailable), diff --git a/internal/db/repository/storage_health_sql_internal_test.go b/internal/db/repository/storage_health_sql_internal_test.go index 684dfe8..31eb186 100644 --- a/internal/db/repository/storage_health_sql_internal_test.go +++ b/internal/db/repository/storage_health_sql_internal_test.go @@ -16,16 +16,15 @@ func TestStorageHealthSQLLiteralAcceptsEveryDomainStatus(t *testing.T) { model.StorageDataSetStatusCreating, model.StorageDataSetStatusReady, model.StorageDataSetStatusFailed, - model.StorageDataSetStatusUnavailable, model.StorageDataSetStatusDraining, model.StorageDataSetStatusRetired, } - copyStatuses := []model.StorageUploadCopyStatus{ - model.StorageUploadCopyStatusPending, - model.StorageUploadCopyStatusPieceReady, - model.StorageUploadCopyStatusCommitting, - model.StorageUploadCopyStatusCommitted, - model.StorageUploadCopyStatusFailed, + copyStatuses := []model.StorageCopyStatus{ + model.StorageCopyStatusPending, + model.StorageCopyStatusPieceReady, + model.StorageCopyStatusCommitting, + model.StorageCopyStatusCommitted, + model.StorageCopyStatusFailed, } observationStatuses := []observability.Status{ observability.StatusAvailable, diff --git a/internal/db/repository/storage_progress_repo_test.go b/internal/db/repository/storage_progress_repo_test.go new file mode 100644 index 0000000..0dae1d7 --- /dev/null +++ b/internal/db/repository/storage_progress_repo_test.go @@ -0,0 +1,200 @@ +package repository_test + +import ( + "errors" + "testing" + "time" + + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/testutil" +) + +func TestIngressProgressRejectsStaleTransferWriters(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + bucket := seedBucket(t, db, "progress-fence") + content, err := repos.Contents.EnsureContent(t.Context(), repository.EnsureContentInput{ + BucketID: bucket.ID, ContentSize: 10, + Checksum: testutil.StorageChecksum("progress-fence"), RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("EnsureContent: %v", err) + } + binding, err := repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByContentID: content.ID, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + if err := repos.Contents.CreateUploadCopiesForBindings(t.Context(), content.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: binding.ID, CopyIndex: 0, ProviderID: binding.ProviderID, + TransferMethod: model.StorageCopyTransferMethodIngress, + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings: %v", err) + } + version := &model.ObjectVersion{ + VersionID: model.NewVersionID(), BucketID: bucket.ID, Key: "progress.bin", ContentID: &content.ID, + Size: content.ContentSize, ETag: "progress", ContentType: "application/octet-stream", + } + if _, err := repos.Objects.CreateVersionAndSetCurrent(t.Context(), version); err != nil { + t.Fatalf("CreateVersionAndSetCurrent: %v", err) + } + copies, err := repos.Contents.ListCopies(t.Context(), content.ID) + if err != nil || len(copies) != 1 { + t.Fatalf("ListCopies = %#v, err=%v", copies, err) + } + enqueueTask := func(key string) *model.Task { + taskRow, created, err := repos.Tasks.Enqueue(t.Context(), &model.Task{ + Type: model.TaskTypeStorageStore, IdempotencyKey: key, InputVersion: 1, + Input: []byte(`{}`), InputHash: key, Status: model.TaskStatusPending, + ResumeMode: model.TaskResumeModeExecute, AvailableAt: time.Now(), + }) + if err != nil || !created { + t.Fatalf("Enqueue(%s) = %#v, created=%v, err=%v", key, taskRow, created, err) + } + return taskRow + } + firstTask := enqueueTask("progress-first") + if err := repos.Contents.BindCopyTask(t.Context(), copies[0].ID, 1, firstTask.ID); err != nil { + t.Fatalf("BindCopyTask(first): %v", err) + } + staleClaim, err := repos.Tasks.ClaimNext(t.Context(), time.Minute) + if err != nil || staleClaim == nil || staleClaim.ID != firstTask.ID { + t.Fatalf("ClaimNext(first) = %#v, err=%v", staleClaim, err) + } + if _, err := repos.Contents.AuthorizeCopyTask(t.Context(), copies[0].ID, 1, firstTask.ID, staleClaim.ClaimGeneration); err != nil { + t.Fatalf("AuthorizeCopyTask(first): %v", err) + } + if _, err := db.NewRaw(`UPDATE tasks SET lease_until = ? WHERE id = ?`, time.Now().Add(-time.Second), firstTask.ID).Exec(t.Context()); err != nil { + t.Fatalf("expire first copy claim: %v", err) + } + freshClaim, err := repos.Tasks.ClaimNext(t.Context(), time.Minute) + if err != nil || freshClaim == nil || freshClaim.ID != firstTask.ID { + t.Fatalf("ClaimNext(fresh) = %#v, err=%v", freshClaim, err) + } + if _, err := repos.Contents.AuthorizeCopyTask(t.Context(), copies[0].ID, 1, firstTask.ID, staleClaim.ClaimGeneration); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("AuthorizeCopyTask(stale) error = %v, want conflict", err) + } + if _, err := repos.Contents.AuthorizeCopyTask(t.Context(), copies[0].ID, 1, firstTask.ID, freshClaim.ClaimGeneration); err != nil { + t.Fatalf("AuthorizeCopyTask(fresh): %v", err) + } + if _, err := repos.Contents.BeginIngressStoreProgress(t.Context(), repository.BeginIngressStoreProgressInput{ + CopyID: copies[0].ID, Generation: 1, TaskID: firstTask.ID, Attempt: 1, + }); err != nil { + t.Fatalf("BeginIngressStoreProgress(first): %v", err) + } + if _, err := repos.Contents.RecordIngressStoreProgress(t.Context(), repository.RecordIngressStoreProgressInput{ + CopyID: copies[0].ID, Generation: 1, TaskID: firstTask.ID, Attempt: 1, BytesUploaded: 4, + }); err != nil { + t.Fatalf("RecordIngressStoreProgress(first): %v", err) + } + + secondTask := enqueueTask("progress-second") + if err := repos.Contents.ReplaceCopyTask(t.Context(), copies[0].ID, 1, firstTask.ID, 2, secondTask.ID); err != nil { + t.Fatalf("ReplaceCopyTask: %v", err) + } + if err := repos.Tasks.Settle(t.Context(), freshClaim.ID, freshClaim.ClaimGeneration, repository.TaskTransition{ + Status: model.TaskStatusCompleted, ResumeMode: model.TaskResumeModeRecover, RetentionUntil: new(time.Now().Add(time.Hour)), + }); err != nil { + t.Fatalf("complete first task: %v", err) + } + if _, err := repos.Contents.BeginIngressStoreProgress(t.Context(), repository.BeginIngressStoreProgressInput{ + CopyID: copies[0].ID, Generation: 2, TaskID: secondTask.ID, Attempt: 2, + }); err != nil { + t.Fatalf("BeginIngressStoreProgress(second): %v", err) + } + for _, stale := range []repository.RecordIngressStoreProgressInput{ + {CopyID: copies[0].ID, Generation: 1, TaskID: firstTask.ID, Attempt: 1, BytesUploaded: 9}, + {CopyID: copies[0].ID, Generation: 2, TaskID: secondTask.ID, Attempt: 1, BytesUploaded: 9}, + } { + if _, err := repos.Contents.RecordIngressStoreProgress(t.Context(), stale); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("stale progress error = %v, want conflict", err) + } + } + if _, err := repos.Contents.RecordIngressStoreProgress(t.Context(), repository.RecordIngressStoreProgressInput{ + CopyID: copies[0].ID, Generation: 2, TaskID: secondTask.ID, Attempt: 2, BytesUploaded: 7, + }); err != nil { + t.Fatalf("RecordIngressStoreProgress(second): %v", err) + } + if err := repos.Contents.MarkUploadCopyFailed(t.Context(), repository.MarkUploadCopyFailedInput{ + StorageCopyID: copies[0].ID, ContentID: content.ID, CopyIndex: 0, LastError: "owner deleted", + }); err != nil { + t.Fatalf("MarkUploadCopyFailed: %v", err) + } + if _, err := repos.Contents.RecordIngressStoreProgress(t.Context(), repository.RecordIngressStoreProgressInput{ + CopyID: copies[0].ID, Generation: 2, TaskID: secondTask.ID, Attempt: 2, BytesUploaded: 10, + }); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("terminal progress error = %v, want conflict", err) + } + stored, err := repos.Contents.GetUploadCopyByID(t.Context(), copies[0].ID) + if err != nil || stored.IngressStoreAttempt != 2 || stored.IngressBytesTransferred != 7 { + t.Fatalf("stored progress = %#v, err=%v", stored, err) + } +} + +func TestPermanentDeleteClearsTerminalStoreFence(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + bucket := seedBucket(t, db, "terminal-store-delete") + content, err := repos.Contents.EnsureContent(t.Context(), repository.EnsureContentInput{ + BucketID: bucket.ID, ContentSize: 128, + Checksum: testutil.StorageChecksum("terminal-store-delete"), RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("EnsureContent: %v", err) + } + binding, err := repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: onChainID(t, "201"), CopyIndex: 0, CreatedByContentID: content.ID, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + if err := repos.Contents.CreateUploadCopiesForBindings(t.Context(), content.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: binding.ID, CopyIndex: 0, ProviderID: binding.ProviderID, + TransferMethod: model.StorageCopyTransferMethodIngress, + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings: %v", err) + } + version := &model.ObjectVersion{ + VersionID: model.NewVersionID(), BucketID: bucket.ID, Key: "unknown-store.bin", ContentID: &content.ID, + Size: content.ContentSize, ETag: "unknown-store", ContentType: "application/octet-stream", + } + if _, err := repos.Objects.CreateVersionAndSetCurrent(t.Context(), version); err != nil { + t.Fatalf("CreateVersionAndSetCurrent: %v", err) + } + copies, err := repos.Contents.ListCopies(t.Context(), content.ID) + if err != nil || len(copies) != 1 { + t.Fatalf("ListCopies = %#v, err=%v", copies, err) + } + taskRow, created, err := repos.Tasks.Enqueue(t.Context(), &model.Task{ + Type: model.TaskTypeStorageStore, IdempotencyKey: "terminal-store-delete", InputVersion: 1, + Input: []byte(`{}`), InputHash: "terminal-store-delete", Status: model.TaskStatusPending, + ResumeMode: model.TaskResumeModeExecute, AvailableAt: time.Now(), + }) + if err != nil || !created { + t.Fatalf("Enqueue = %#v, created=%v, err=%v", taskRow, created, err) + } + if err := repos.Contents.BindCopyTask(t.Context(), copies[0].ID, 1, taskRow.ID); err != nil { + t.Fatalf("BindCopyTask: %v", err) + } + claimed, err := repos.Tasks.ClaimNext(t.Context(), time.Minute) + if err != nil || claimed == nil || claimed.ID != taskRow.ID { + t.Fatalf("ClaimNext = %#v, err=%v", claimed, err) + } + if err := repos.Tasks.Settle(t.Context(), claimed.ID, claimed.ClaimGeneration, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, + FailureReason: new("store_outcome_unknown"), LastError: new("store outcome unknown"), + }); err != nil { + t.Fatalf("fail store task: %v", err) + } + if _, err := repos.Objects.DeleteObjectVersionPermanently(t.Context(), repository.DeleteObjectVersionInput{ + BucketID: bucket.ID, Key: version.Key, VersionID: version.VersionID, + }); err != nil { + t.Fatalf("DeleteObjectVersionPermanently: %v", err) + } + stored, err := repos.Contents.GetUploadCopyByID(t.Context(), copies[0].ID) + if err != nil || stored.Status != model.StorageCopyStatusFailed || stored.ActiveTaskID != nil { + t.Fatalf("copy after permanent delete = %#v, err=%v", stored, err) + } +} diff --git a/internal/db/repository/storage_readable_copy_sql.go b/internal/db/repository/storage_readable_copy_sql.go index d091ff2..a4844d8 100644 --- a/internal/db/repository/storage_readable_copy_sql.go +++ b/internal/db/repository/storage_readable_copy_sql.go @@ -17,8 +17,7 @@ func readableCommittedCopyPredicateSQL(copyAlias, dataSetAlias string) string { // also treat a data set as its own readable source while it is being finalized. func readableCommittedCopyPredicateWithDataSetStatusSQL(copyAlias, dataSetAlias, dataSetStatusCondition string) string { return fmt.Sprintf(`%[1]s.status = %[3]s - AND %[1]s.storage_data_set_id IS NOT NULL - AND %[1]s.provider_id IS NOT NULL AND %[1]s.provider_id <> '' + AND %[1]s.provider_id <> '' AND %[2]s.data_set_id IS NOT NULL AND %[2]s.data_set_id <> '' AND %[4]s AND %[1]s.piece_id IS NOT NULL AND %[1]s.piece_id <> '' @@ -31,34 +30,30 @@ func readableCommittedCopyPredicateWithDataSetStatusSQL(copyAlias, dataSetAlias, } // A replica slot can hold several data set generations during a provider -// replacement, so a lookup that knows only the upload and the slot must resolve -// to the generation that currently owns the slot. A copy with no data set yet -// belongs to the slot until one is assigned. +// replacement, so a lookup that knows only the upload and the slot resolves to +// the bound generation that currently owns the slot. func currentGenerationCopySQL(copyAlias string) string { - return fmt.Sprintf(`( - %[1]s.storage_data_set_id IS NULL - OR EXISTS ( + return fmt.Sprintf(`EXISTS ( SELECT 1 FROM storage_data_sets AS current_slot_data_set WHERE current_slot_data_set.id = %[1]s.storage_data_set_id AND current_slot_data_set.is_current - ) - )`, copyAlias) + )`, copyAlias) } // Durability is measured in logical replica slots. One slot can hold several // physical data set generations while a provider replacement is in flight, and // those generations must never count as separate replicas. -func distinctReadableSlotCountSQL(copyAlias, dataSetAlias, uploadIDExpr string) string { +func distinctReadableSlotCountSQL(copyAlias, dataSetAlias, contentIDExpr string) string { return fmt.Sprintf(`( SELECT COUNT(DISTINCT %[2]s.copy_index) - FROM storage_upload_copies AS %[1]s + FROM storage_copies AS %[1]s JOIN storage_data_sets AS %[2]s ON %[2]s.id = %[1]s.storage_data_set_id - WHERE %[1]s.upload_id = %[3]s + WHERE %[1]s.content_id = %[3]s AND %[4]s )`, copyAlias, dataSetAlias, - uploadIDExpr, + contentIDExpr, readableCommittedCopyPredicateSQL(copyAlias, dataSetAlias), ) } diff --git a/internal/db/repository/storage_replacement_gate.go b/internal/db/repository/storage_replacement_gate.go index b6f7d3f..d8d3e31 100644 --- a/internal/db/repository/storage_replacement_gate.go +++ b/internal/db/repository/storage_replacement_gate.go @@ -3,28 +3,14 @@ package repository import ( "context" "fmt" - "sort" + "slices" "time" "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/storagereplacement" "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" ) -// taskPayloadCopyIDSQL extracts the concrete copy a task targets. Tasks queued -// before copy ids were recorded yield NULL and are handled separately. -func taskPayloadCopyIDSQL(dialectName dialect.Name) func(alias string) string { - if dialectName == dialect.PG { - return func(alias string) string { - return "CAST(" + alias + ".payload ->> 'storage_upload_copy_id' AS BIGINT)" - } - } - return func(alias string) string { - return "CAST(json_extract(" + alias + ".payload, '$.storage_upload_copy_id') AS INTEGER)" - } -} - // Blocker names are stable so the API and UI can explain a held source without // inventing their own vocabulary. const ( @@ -46,12 +32,14 @@ func (r *BunStorageReplacementRepo) EvaluateRetirementGate(ctx context.Context, func evaluateRetirementGate(ctx context.Context, db bun.IDB, replacementID int64, observedEpoch *int64) (RetirementGate, error) { gate := RetirementGate{} row := new(storagereplacement.Replacement) - if err := db.NewSelect().Model(row).Where("id = ?", replacementID).Scan(ctx); err != nil { + if err := withReplacementTerminations(db.NewSelect().Model(row)). + Where("storage_replacement.id = ?", replacementID). + Scan(ctx); err != nil { return gate, fmt.Errorf("loading provider replacement for retirement: %w", err) } gate.TerminationEpoch = row.TerminationEpoch - uploads := &BunStorageUploadRepo{db: db} + uploads := &BunStorageContentRepo{db: db} source, err := uploads.GetDataSetBindingByID(ctx, row.SourceDataSetID) if err != nil { return gate, err @@ -97,12 +85,9 @@ func evaluateRetirementGate(ctx context.Context, db bun.IDB, replacementID int64 waiting, err := db.NewSelect(). Model((*storagereplacement.Item)(nil)). Where("replacement_id = ?", row.ID). - Where("status IN (?, ?, ?, ?, ?)", + Where("status IN (?, ?)", storagereplacement.ItemStatusPending, - storagereplacement.ItemStatusRunning, - storagereplacement.ItemStatusRetrying, - storagereplacement.ItemStatusWaitingSource, - storagereplacement.ItemStatusFailed). + storagereplacement.ItemStatusAttention). Count(ctx) if err != nil { return gate, fmt.Errorf("counting outstanding replacement items: %w", err) @@ -143,10 +128,10 @@ func countRetirementCoverageGaps(ctx context.Context, db bun.IDB, sourceDataSetI // retirementCoverageGapsSQL is shared with the query-plan regression test so // the tested plan cannot drift away from the production retirement gate. func retirementCoverageGapsSQL() string { - return fmt.Sprintf(`SELECT COUNT(*) FROM storage_uploads AS retiring_upload + return fmt.Sprintf(`SELECT COUNT(*) FROM storage_contents AS retiring_upload WHERE EXISTS ( - SELECT 1 FROM storage_upload_copies AS source_copy - WHERE source_copy.upload_id = retiring_upload.id + SELECT 1 FROM storage_copies AS source_copy + WHERE source_copy.content_id = retiring_upload.id AND source_copy.storage_data_set_id = ? AND source_copy.status = %[1]s ) @@ -156,67 +141,35 @@ func retirementCoverageGapsSQL() string { AND live_version.is_delete_marker = ? ) AND NOT EXISTS ( - SELECT 1 FROM storage_upload_copies AS target_copy + SELECT 1 FROM storage_copies AS target_copy JOIN storage_data_sets AS target_data_set ON target_data_set.id = target_copy.storage_data_set_id - WHERE target_copy.upload_id = retiring_upload.id + WHERE target_copy.content_id = retiring_upload.id AND target_copy.storage_data_set_id = ? AND %[3]s )`, storageHealthCommittedCopyStatusSQL(), - objectVersionReferencesStorageUploadSQL("live_version", "retiring_upload"), + objectVersionReferencesStorageContentSQL("live_version", "retiring_upload"), readableCommittedCopyPredicateSQL("target_copy", "target_data_set"), ) } -// Source writes are counted from both directions: copy rows that are still -// mid-transfer, and tasks that could still produce one. A running task with no -// recorded copy predates copy-id addressing, so it is treated as a possible -// source write; queued work is not, because it will resolve to the generation -// that owns the slot by the time it runs. -func countRetirementSourceWrites(ctx context.Context, db bun.IDB, bucketID, sourceDataSetID int64) (int, error) { +// A common storage task is bound to its copy before it can perform an external +// effect. Therefore the copy lifecycle is the complete source-write fence; the +// retirement gate never needs to inspect task input. +func countRetirementSourceWrites(ctx context.Context, db bun.IDB, _ int64, sourceDataSetID int64) (int, error) { inFlight, err := db.NewSelect(). - Model((*model.StorageUploadCopy)(nil)). + Model((*model.StorageCopy)(nil)). Where("storage_data_set_id = ?", sourceDataSetID). Where("status IN (?, ?, ?)", - model.StorageUploadCopyStatusPending, - model.StorageUploadCopyStatusPieceReady, - model.StorageUploadCopyStatusCommitting). + model.StorageCopyStatusPending, + model.StorageCopyStatusPieceReady, + model.StorageCopyStatusCommitting). Count(ctx) if err != nil { return 0, fmt.Errorf("counting in-flight source copies: %w", err) } - copyIDExpr := taskPayloadCopyIDSQL(db.Dialect().Name()) - var bound int - if err := db.NewRaw(`SELECT COUNT(*) FROM tasks AS active_task - JOIN storage_upload_copies AS bound_copy - ON bound_copy.id = `+copyIDExpr("active_task")+` - WHERE active_task.type = ? - AND active_task.status IN (?, ?, ?, ?) - AND bound_copy.storage_data_set_id = ?`, - model.TaskTypeUpload, - model.TaskStatusQueued, model.TaskStatusScheduled, model.TaskStatusWaiting, model.TaskStatusRunning, - sourceDataSetID, - ).Scan(ctx, &bound); err != nil { - return 0, fmt.Errorf("counting source-bound upload tasks: %w", err) - } - - var legacyRunning int - if err := db.NewRaw(`SELECT COUNT(*) FROM tasks AS legacy_task - WHERE legacy_task.type = ? - AND legacy_task.status = ? - AND legacy_task.ref_type = ? - AND `+copyIDExpr("legacy_task")+` IS NULL - AND EXISTS ( - SELECT 1 FROM object_versions AS task_version - WHERE task_version.version_id = legacy_task.ref_version_id - AND task_version.bucket_id = ? - )`, - model.TaskTypeUpload, model.TaskStatusRunning, "object", bucketID, - ).Scan(ctx, &legacyRunning); err != nil { - return 0, fmt.Errorf("counting legacy running upload tasks: %w", err) - } - return inFlight + bound + legacyRunning, nil + return inFlight, nil } // CountAbandonedTargetSoleCopies reports how many uploads would lose their only @@ -228,9 +181,9 @@ func (r *BunStorageReplacementRepo) CountAbandonedTargetSoleCopies(ctx context.C predicate := readableCommittedCopyPredicateSQL("abandoned_copy", "abandoned_data_set") elsewhere := readableCommittedCopyPredicateSQL("other_copy", "other_data_set") query := fmt.Sprintf(`SELECT COUNT(*) - FROM storage_upload_copies AS abandoned_copy + FROM storage_copies AS abandoned_copy JOIN storage_data_sets AS abandoned_data_set ON abandoned_data_set.id = abandoned_copy.storage_data_set_id - JOIN storage_uploads AS abandoned_upload ON abandoned_upload.id = abandoned_copy.upload_id + JOIN storage_contents AS abandoned_upload ON abandoned_upload.id = abandoned_copy.content_id WHERE abandoned_copy.storage_data_set_id = ? AND %[1]s AND EXISTS ( @@ -239,14 +192,14 @@ func (r *BunStorageReplacementRepo) CountAbandonedTargetSoleCopies(ctx context.C AND live_version.is_delete_marker = ? ) AND NOT EXISTS ( - SELECT 1 FROM storage_upload_copies AS other_copy + SELECT 1 FROM storage_copies AS other_copy JOIN storage_data_sets AS other_data_set ON other_data_set.id = other_copy.storage_data_set_id - WHERE other_copy.upload_id = abandoned_copy.upload_id + WHERE other_copy.content_id = abandoned_copy.content_id AND other_copy.storage_data_set_id <> ? AND %[3]s )`, predicate, - objectVersionReferencesStorageUploadSQL("live_version", "abandoned_upload"), + objectVersionReferencesStorageContentSQL("live_version", "abandoned_upload"), elsewhere, ) var count int @@ -260,26 +213,19 @@ func (r *BunStorageReplacementRepo) CountAbandonedTargetSoleCopies(ctx context.C // the replacement record, which stays superseded, and it refuses a generation // that still owns its slot. func (r *BunStorageReplacementRepo) RetireAbandonedTarget(ctx context.Context, replacementID int64) error { - return r.retireAbandonedTarget(ctx, replacementID, nil) + return r.retireAbandonedTarget(ctx, replacementID, false) } -// CompleteAbandonedTargetTermination retires the superseded target and records -// epoch observation in the same transaction. -func (r *BunStorageReplacementRepo) CompleteAbandonedTargetTermination( - ctx context.Context, - replacementID int64, - observedAt time.Time, -) error { - if observedAt.IsZero() { - return fmt.Errorf("completing abandoned target termination: %w", ErrInvalidInput) - } - return r.retireAbandonedTarget(ctx, replacementID, &observedAt) +// CompleteAbandonedTargetTermination retires the superseded target once its end +// of term has been recorded, refusing to retire one that has none. +func (r *BunStorageReplacementRepo) CompleteAbandonedTargetTermination(ctx context.Context, replacementID int64) error { + return r.retireAbandonedTarget(ctx, replacementID, true) } func (r *BunStorageReplacementRepo) retireAbandonedTarget( ctx context.Context, replacementID int64, - observedAt *time.Time, + requireRecordedTermination bool, ) error { return runMaybeTx(ctx, r.db, func(db bun.IDB) error { row, err := lockReplacementByID(ctx, db, replacementID) @@ -289,7 +235,7 @@ func (r *BunStorageReplacementRepo) retireAbandonedTarget( if err := lockReplacementDataSets(ctx, db, row.TargetDataSetID); err != nil { return err } - attempts, err := (&BunStorageUploadRepo{db: db}).CountActiveCommitAttemptsForDataSet(ctx, row.TargetDataSetID) + attempts, err := (&BunStorageContentRepo{db: db}).CountActiveCommitAttemptsForDataSet(ctx, row.TargetDataSetID) if err != nil { return err } @@ -305,10 +251,9 @@ func (r *BunStorageReplacementRepo) retireAbandonedTarget( return fmt.Errorf("retiring abandoned target of replacement %d holds %d sole copies: %w", replacementID, sole, storagereplacement.ErrPrematureComplete) } - if observedAt != nil { - if row.Status != storagereplacement.StatusSuperseded || row.AbandonedTerminationEpoch == nil { - return fmt.Errorf("completing abandoned target termination: %w", ErrConflict) - } + if requireRecordedTermination && + (row.Status != storagereplacement.StatusSuperseded || row.AbandonedTerminationEpoch == nil) { + return fmt.Errorf("completing abandoned target termination: %w", ErrConflict) } res, err := db.NewUpdate(). Model((*model.StorageDataSet)(nil)). @@ -323,21 +268,6 @@ func (r *BunStorageReplacementRepo) retireAbandonedTarget( if rows, _ := res.RowsAffected(); rows != 1 { return fmt.Errorf("retiring abandoned target: %w", ErrConflict) } - if observedAt != nil { - res, err = db.NewUpdate(). - Model((*storagereplacement.Replacement)(nil)). - Set("abandoned_termination_observed_at = ?", *observedAt). - Set("updated_at = ?", *observedAt). - Where("id = ? AND status = ?", replacementID, storagereplacement.StatusSuperseded). - Where("abandoned_termination_epoch IS NOT NULL"). - Exec(ctx) - if err != nil { - return fmt.Errorf("recording abandoned target termination observation: %w", err) - } - if rows, _ := res.RowsAffected(); rows != 1 { - return fmt.Errorf("recording abandoned target termination observation: %w", ErrConflict) - } - } return nil }) } @@ -386,18 +316,17 @@ func (r *BunStorageReplacementRepo) CompleteRetirement(ctx context.Context, repl storagereplacement.StatusCompleted, func(q *bun.UpdateQuery) *bun.UpdateQuery { return q.Set("wait_reason = NULL"). - Set("last_error = NULL"). - Set("termination_observed_at = ?", now) + Set("last_error = NULL") }, now) }) } func countReplacementActiveCommitAttempts(ctx context.Context, db bun.IDB, replacementID int64) (int, error) { count, err := db.NewSelect(). - Model((*model.StorageUploadCopy)(nil)). - Join("JOIN storage_replacement_items AS replacement_item ON replacement_item.target_copy_id = storage_upload_copy.id"). + Model((*model.StorageCopy)(nil)). + Join("JOIN storage_replacement_items AS replacement_item ON replacement_item.content_id = storage_copy.content_id AND replacement_item.target_data_set_id = storage_copy.storage_data_set_id"). Where("replacement_item.replacement_id = ?", replacementID). - Where(attemptedStorageCommitSQL("storage_upload_copy")). + Where(attemptedStorageCommitSQL("storage_copy")). Count(ctx) if err != nil { return 0, fmt.Errorf("counting replacement confirmation attempts: %w", err) @@ -407,7 +336,7 @@ func countReplacementActiveCommitAttempts(ctx context.Context, db bun.IDB, repla func lockReplacementDataSets(ctx context.Context, db bun.IDB, ids ...int64) error { ids = append([]int64(nil), ids...) - sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + slices.Sort(ids) for i, id := range ids { if id <= 0 || (i > 0 && id == ids[i-1]) { continue diff --git a/internal/db/repository/storage_replacement_gate_internal_test.go b/internal/db/repository/storage_replacement_gate_internal_test.go deleted file mode 100644 index ff0fc13..0000000 --- a/internal/db/repository/storage_replacement_gate_internal_test.go +++ /dev/null @@ -1,229 +0,0 @@ -package repository - -import ( - "context" - "database/sql" - "fmt" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/stdlib" - "github.com/strahe/synaps3/internal/config" - synaps3db "github.com/strahe/synaps3/internal/db" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" - "github.com/uptrace/bun/dialect/sqlitedialect" - _ "modernc.org/sqlite" -) - -func TestRetirementCoverageProductionSQLUsesIndexes(t *testing.T) { - sqldb, err := sql.Open("sqlite", "file:"+filepath.Join(t.TempDir(), "retirement-plan.db")+"?_pragma=foreign_keys(1)") - if err != nil { - t.Fatalf("open sqlite: %v", err) - } - db := bun.NewDB(sqldb, sqlitedialect.New()) - t.Cleanup(func() { _ = db.Close() }) - - ctx := context.Background() - if err := synaps3db.RunMigrations(ctx, db); err != nil { - t.Fatalf("RunMigrations: %v", err) - } - plan := sqliteProductionQueryPlan(t, sqldb, retirementCoverageGapsSQL(), int64(1), false, int64(2)) - for _, index := range []string{ - "idx_storage_upload_copies_status_data_set_upload", - "idx_object_versions_storage_upload", - } { - if !strings.Contains(plan, index) { - t.Fatalf("production retirement coverage plan =\n%s\nwant %s", plan, index) - } - } - - tests := []struct { - name string - query string - args []any - index string - }{ - { - name: "replacement upload window", - query: storageUploadMigrationWindowSQL(), - args: []any{int64(1), int64(0), 100}, - index: "idx_storage_uploads_bucket_id", - }, - { - name: "replacement item lock", - query: lockReplacementItemsByUploadSQL(), - args: []any{int64(1)}, - index: "idx_storage_replacement_items_upload_id", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - plan := sqliteProductionQueryPlan(t, sqldb, tt.query, tt.args...) - if !strings.Contains(plan, tt.index) { - t.Fatalf("production query plan =\n%s\nwant %s", plan, tt.index) - } - }) - } - - readyPlan := sqliteProductionQueryPlan(t, sqldb, readyReplacementSelectionSQL(dialect.SQLite), - storagereplacement.StatusMigrating, - storagereplacement.StatusWaiting, - storagereplacement.WaitReasonReadableSource, - time.Now(), - time.Now(), - storagereplacement.StatusFailed, - storagereplacement.StatusSuperseded, - time.Now(), - time.Now(), - ) - for _, index := range []string{"idx_storage_replacement_items_due", "idx_storage_replacement_items_lease"} { - if !strings.Contains(readyPlan, index) { - t.Fatalf("ready replacement plan =\n%s\nwant %s", readyPlan, index) - } - } - - candidateTests := []struct { - name string - query string - args []any - index string - }{ - { - name: "due replacement item", - query: readyReplacementDueItemSQL(false), - args: []any{int64(1), time.Now()}, - index: "idx_storage_replacement_items_due", - }, - { - name: "expired replacement item", - query: readyReplacementExpiredItemSQL(false), - args: []any{int64(1), time.Now()}, - index: "idx_storage_replacement_items_lease", - }, - } - for _, tt := range candidateTests { - t.Run(tt.name, func(t *testing.T) { - plan := sqliteProductionQueryPlan(t, sqldb, tt.query, tt.args...) - if !strings.Contains(plan, tt.index) || strings.Contains(plan, "USE TEMP B-TREE") { - t.Fatalf("replacement item candidate plan =\n%s\nwant %s without a temp sort", plan, tt.index) - } - }) - } -} - -func sqliteProductionQueryPlan(t *testing.T, db *sql.DB, query string, args ...any) string { - t.Helper() - rows, err := db.Query("EXPLAIN QUERY PLAN "+query, args...) - if err != nil { - t.Fatalf("EXPLAIN production query: %v", err) - } - defer func() { _ = rows.Close() }() - var details []string - for rows.Next() { - var id, parent, notUsed int - var detail string - if err := rows.Scan(&id, &parent, ¬Used, &detail); err != nil { - t.Fatalf("scan query plan: %v", err) - } - details = append(details, detail) - } - if err := rows.Err(); err != nil { - t.Fatalf("read query plan: %v", err) - } - return strings.Join(details, "\n") -} - -func TestPostgresReplacementProductionSQLUsesNewIndexes(t *testing.T) { - dsn := os.Getenv("SYNAPS3_POSTGRES_TEST_DSN") - if dsn == "" { - t.Skip("SYNAPS3_POSTGRES_TEST_DSN is not set") - } - ctx := context.Background() - adminDB, err := synaps3db.New(config.DatabaseConfig{ - Driver: "postgres", DSN: dsn, MaxOpenConns: 1, MaxIdleConns: 1, - }) - if err != nil { - t.Fatalf("open PostgreSQL admin connection: %v", err) - } - schema := fmt.Sprintf("replacement_query_plan_%d", time.Now().UnixNano()) - quoted := `"` + schema + `"` - if _, err := adminDB.ExecContext(ctx, "CREATE SCHEMA "+quoted); err != nil { - _ = adminDB.Close() - t.Fatalf("create PostgreSQL schema: %v", err) - } - pgConfig, err := pgx.ParseConfig(dsn) - if err != nil { - _, _ = adminDB.ExecContext(ctx, "DROP SCHEMA "+quoted+" CASCADE") - _ = adminDB.Close() - t.Fatalf("parse PostgreSQL DSN: %v", err) - } - pgConfig.RuntimeParams["search_path"] = schema - registeredDSN := stdlib.RegisterConnConfig(pgConfig) - db, err := synaps3db.New(config.DatabaseConfig{ - Driver: "postgres", DSN: registeredDSN, MaxOpenConns: 1, MaxIdleConns: 1, - }) - if err != nil { - stdlib.UnregisterConnConfig(registeredDSN) - _, _ = adminDB.ExecContext(ctx, "DROP SCHEMA "+quoted+" CASCADE") - _ = adminDB.Close() - t.Fatalf("open schema-scoped PostgreSQL connection: %v", err) - } - t.Cleanup(func() { - _ = db.Close() - stdlib.UnregisterConnConfig(registeredDSN) - _, _ = adminDB.ExecContext(context.Background(), "DROP SCHEMA "+quoted+" CASCADE") - _ = adminDB.Close() - }) - if err := synaps3db.RunMigrations(ctx, db); err != nil { - t.Fatalf("RunMigrations: %v", err) - } - statements := []string{ - `INSERT INTO buckets (id, name) VALUES (1, 'query-plan-target'), (2, 'query-plan-other')`, - `INSERT INTO storage_uploads (id, bucket_id, content_size, checksum, requested_copies) - SELECT n, CASE WHEN n = 1 THEN 1 ELSE 2 END, 1, 'checksum-' || n, 1 - FROM generate_series(1, 10000) AS n`, - `INSERT INTO storage_data_sets (id, bucket_id, provider_id, copy_index, status) - VALUES (1, 1, '101', 0, 'ready'), (2, 1, '202', 1, 'ready')`, - `INSERT INTO storage_replacements - (id, bucket_id, copy_index, source_data_set_id, target_data_set_id, - selection_mode, client_request_id, status) - SELECT n, 1, 0, 1, 2, 'automatic', 'query-plan-' || n, 'completed' - FROM generate_series(1, 10000) AS n`, - `INSERT INTO storage_replacement_items (id, replacement_id, upload_id, status) - SELECT n, n, n, 'pending' FROM generate_series(1, 10000) AS n`, - `ANALYZE storage_uploads`, - `ANALYZE storage_replacement_items`, - } - for _, statement := range statements { - if _, err := db.ExecContext(ctx, statement); err != nil { - t.Fatalf("seed PostgreSQL query plan data: %v", err) - } - } - tests := []struct { - name string - query string - args []any - index string - }{ - {"replacement upload window", storageUploadMigrationWindowSQL(), []any{int64(1), int64(0), 100}, "idx_storage_uploads_bucket_id"}, - {"replacement item lock", lockReplacementItemsByUploadSQL(), []any{int64(1)}, "idx_storage_replacement_items_upload_id"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var lines []string - if err := db.NewRaw("EXPLAIN "+tt.query, tt.args...).Scan(ctx, &lines); err != nil { - t.Fatalf("EXPLAIN production query: %v", err) - } - plan := strings.Join(lines, "\n") - if !strings.Contains(plan, tt.index) { - t.Fatalf("production query plan =\n%s\nwant %s", plan, tt.index) - } - }) - } -} diff --git a/internal/db/repository/storage_replacement_items.go b/internal/db/repository/storage_replacement_items.go index 56b3de6..50ea410 100644 --- a/internal/db/repository/storage_replacement_items.go +++ b/internal/db/repository/storage_replacement_items.go @@ -3,268 +3,210 @@ package repository import ( "context" "database/sql" + "errors" "fmt" "time" "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagecommit" "github.com/strahe/synaps3/internal/storagereplacement" "github.com/uptrace/bun" ) -// SeedMigrationBatchWithBudget snapshots the item retry budget at discovery. -func (r *BunStorageReplacementRepo) SeedMigrationBatchWithBudget( +func (r *BunStorageReplacementRepo) SeedMigrationBatch( ctx context.Context, replacementID int64, limit int, - maxRetries int, -) (int, bool, error) { - if maxRetries < 0 { - return 0, false, fmt.Errorf("seeding replacement migration: %w", ErrInvalidInput) - } - return r.seedMigrationBatch(ctx, replacementID, limit, maxRetries) -} - -func (r *BunStorageReplacementRepo) seedMigrationBatch( - ctx context.Context, - replacementID int64, - limit int, - maxRetries int, ) (int, bool, error) { if replacementID <= 0 || limit <= 0 { - return 0, false, fmt.Errorf("seeding replacement migration: %w", ErrInvalidInput) + return 0, false, ErrInvalidInput } row, err := r.GetByID(ctx, replacementID) if err != nil { return 0, false, err } if row == nil { - return 0, false, fmt.Errorf("provider replacement %d: %w", replacementID, ErrNotFound) + return 0, false, ErrNotFound } if row.SeedingComplete { return 0, true, nil } - - // Deciding which uploads need migrating is a read over bucket history. It - // runs outside the write transaction so a replacement never holds SQLite's - // single writer while it scans. eligible, cursor, scanned, err := r.scanMigrationCandidates(ctx, row, limit) if err != nil { return 0, false, err } if scanned == 0 { - return 0, true, markSeedingComplete(ctx, r.db, row.ID, row.SeedCursorUploadID) + return 0, true, markSeedingComplete(ctx, r.db, row.ID, row.SeedCursorContentID) } - inserted := 0 done := scanned < limit err = runMaybeTx(ctx, r.db, func(db bun.IDB) error { - // Re-lock and re-check the cursor: another pass may have advanced it - // while this one was reading. locked, err := lockReplacementByID(ctx, db, replacementID) if err != nil { return err } - if locked.SeedCursorUploadID != row.SeedCursorUploadID { - return fmt.Errorf("advancing replacement migration cursor: %w", ErrConflict) + if locked.SeedCursorContentID != row.SeedCursorContentID { + return ErrConflict } if len(eligible) > 0 { + // The target copy is created before the item that names it, so the + // item is born with a target and its composite foreign key to that + // copy is checked from the first write rather than skipped while the + // column is still null. + contents := &BunStorageContentRepo{db: db} + target, err := contents.GetDataSetBindingByID(ctx, row.TargetDataSetID) + if err != nil { + return err + } + if target == nil { + return fmt.Errorf("seeding replacement items: %w", ErrNotFound) + } items := make([]storagereplacement.Item, 0, len(eligible)) now := time.Now() - for _, uploadID := range eligible { + for _, contentID := range eligible { + if err := contents.CreateUploadCopiesForBindings(ctx, contentID, []UploadCopyBindingInput{{ + StorageDataSetID: target.ID, CopyIndex: target.CopyIndex, + TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: target.ProviderID, + }}); err != nil { + return err + } items = append(items, storagereplacement.Item{ - ReplacementID: row.ID, - UploadID: uploadID, - Status: storagereplacement.ItemStatusPending, - ScheduledAt: now, - MaxRetries: &maxRetries, - CreatedAt: now, - UpdatedAt: now, + ReplacementID: row.ID, ContentID: contentID, + TargetDataSetID: target.ID, + Status: storagereplacement.ItemStatusPending, + CreatedAt: now, UpdatedAt: now, }) } - res, err := db.NewInsert(). + result, err := db.NewInsert(). Model(&items). - On("CONFLICT (replacement_id, upload_id) DO NOTHING"). + On("CONFLICT (replacement_id, content_id) DO NOTHING"). Exec(ctx) if err != nil { - return fmt.Errorf("seeding replacement migration items: %w", err) + return fmt.Errorf("seeding replacement items: %w", err) } - affected, _ := res.RowsAffected() - inserted = int(affected) + rows, _ := result.RowsAffected() + inserted = int(rows) } - - q := db.NewUpdate(). + query := db.NewUpdate(). Model((*storagereplacement.Replacement)(nil)). - Set("seed_cursor_upload_id = ?", cursor). + Set("seed_cursor_content_id = ?", cursor). Set("items_total = items_total + ?", inserted). Set("updated_at = ?", time.Now()). - Where("id = ? AND seed_cursor_upload_id = ?", row.ID, row.SeedCursorUploadID) + Where("id = ? AND seed_cursor_content_id = ?", row.ID, row.SeedCursorContentID) if done { - q = q.Set("seeding_complete = ?", true) + query = query.Set("seeding_complete = ?", true) } - res, err := q.Exec(ctx) + result, err := query.Exec(ctx) if err != nil { - return fmt.Errorf("advancing replacement migration cursor: %w", err) + return fmt.Errorf("advancing replacement cursor: %w", err) } - if rows, _ := res.RowsAffected(); rows != 1 { - return fmt.Errorf("advancing replacement migration cursor: %w", ErrConflict) + if rows, _ := result.RowsAffected(); rows != 1 { + return ErrConflict } return nil }) - if err != nil { - return 0, false, err - } - return inserted, done, nil + return inserted, done, err } -// scanMigrationCandidates reads one bounded window of upload history and reports -// which uploads still need a copy on the new provider. The cursor advances over -// every upload examined, not only the eligible ones, so a window full of -// ineligible uploads still makes progress. func (r *BunStorageReplacementRepo) scanMigrationCandidates( ctx context.Context, row *storagereplacement.Replacement, limit int, ) (eligible []int64, cursor int64, scanned int, err error) { var candidates []int64 - if err := r.db.NewRaw(storageUploadMigrationWindowSQL(), row.BucketID, row.SeedCursorUploadID, limit). + if err := r.db.NewRaw(storageUploadMigrationWindowSQL(), row.BucketID, row.SeedCursorContentID, limit). Scan(ctx, &candidates); err != nil { - return nil, 0, 0, fmt.Errorf("scanning replacement migration candidates: %w", err) + return nil, 0, 0, fmt.Errorf("scanning replacement candidates: %w", err) } if len(candidates) == 0 { - return nil, row.SeedCursorUploadID, 0, nil + return nil, row.SeedCursorContentID, 0, nil } cursor = candidates[len(candidates)-1] - - // Migration is keyed by stored content, so content shared by many object - // versions is copied once. - // - // Seeding deliberately does not ask whether the retiring generation already - // holds a committed copy. An upload still in flight would answer "no" at this - // instant, commit to the source moments later, and never be revisited once - // the cursor moved past it, leaving the retirement coverage gate blocked - // forever. AcquireItem asks that question instead, at a point where it can - // settle the item either way. query := fmt.Sprintf(`SELECT candidate.id - FROM storage_uploads AS candidate + FROM storage_contents AS candidate WHERE candidate.id IN (?) AND EXISTS ( SELECT 1 FROM object_versions AS live_version - WHERE %s - AND live_version.is_delete_marker = ? + WHERE %s AND live_version.is_delete_marker = ? ) - ORDER BY candidate.id ASC`, - objectVersionReferencesStorageUploadSQL("live_version", "candidate"), - ) - if err := r.db.NewRaw(query, bun.List(candidates), false). - Scan(ctx, &eligible); err != nil { - return nil, 0, 0, fmt.Errorf("selecting replacement migration items: %w", err) + ORDER BY candidate.id`, objectVersionReferencesStorageContentSQL("live_version", "candidate")) + if err := r.db.NewRaw(query, bun.List(candidates), false).Scan(ctx, &eligible); err != nil { + return nil, 0, 0, fmt.Errorf("selecting replacement candidates: %w", err) } return eligible, cursor, len(candidates), nil } func storageUploadMigrationWindowSQL() string { - return `SELECT id FROM storage_uploads - WHERE bucket_id = ? AND id > ? - ORDER BY id ASC - LIMIT ?` + return `SELECT id FROM storage_contents WHERE bucket_id = ? AND id > ? ORDER BY id LIMIT ?` } func markSeedingComplete(ctx context.Context, db bun.IDB, replacementID, cursor int64) error { - _, err := db.NewUpdate(). + result, err := db.NewUpdate(). Model((*storagereplacement.Replacement)(nil)). Set("seeding_complete = ?", true). Set("updated_at = ?", time.Now()). - Where("id = ? AND seed_cursor_upload_id = ?", replacementID, cursor). + Where("id = ? AND seed_cursor_content_id = ?", replacementID, cursor). Exec(ctx) if err != nil { - return fmt.Errorf("completing replacement migration seeding: %w", err) + return fmt.Errorf("completing replacement seeding: %w", err) + } + if rows, _ := result.RowsAffected(); rows != 1 { + return ErrConflict } return nil } -// AcquireItem re-derives every identity the item depends on inside one -// transaction and revalidates the worker's claim, so no provider call can start -// from a stale snapshot. -// -// It also decides, at this moment rather than at seeding time, whether the item -// still needs migrating at all. An item that cannot or need not be migrated is -// settled to a terminal status here; leaving it executable would make the -// coordinator pick it up forever and hold retirement open. +func (r *BunStorageReplacementRepo) NextPendingReplacementItem(ctx context.Context, replacementID int64) (*storagereplacement.Item, error) { + item := new(storagereplacement.Item) + err := r.db.NewSelect(). + Model(item). + Where("replacement_id = ? AND status = ?", replacementID, storagereplacement.ItemStatusPending). + OrderExpr("CASE WHEN target_data_set_id IS NULL THEN 0 ELSE 1 END, id"). + Limit(1). + Scan(ctx) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("selecting pending replacement item: %w", err) + } + return item, nil +} + +// AcquireItem re-derives the current domain snapshot. Replacement items are +// ledgers, not workers; task claim and lease state never appears here. func (r *BunStorageReplacementRepo) AcquireItem(ctx context.Context, input AcquireReplacementItemInput) (*ReplacementItemSnapshot, error) { - if input.ReplacementID <= 0 || input.ItemID <= 0 || input.ItemClaimedAt.IsZero() { - return nil, fmt.Errorf("acquiring replacement item: %w", ErrInvalidInput) + if input.ReplacementID <= 0 || input.ItemID <= 0 { + return nil, ErrInvalidInput } var snapshot *ReplacementItemSnapshot - // settled and deferred are reported after the transaction commits, so the - // status this call writes survives; returning an error would roll it back. - settled := false - deferred := false + var terminalErr error err := runMaybeTx(ctx, r.db, func(db bun.IDB) error { item := new(storagereplacement.Item) - err := db.NewRaw( - `UPDATE storage_replacement_items SET updated_at = updated_at WHERE id = ? RETURNING *`, - input.ItemID, - ).Scan(ctx, item) + err := db.NewRaw(`UPDATE storage_replacement_items + SET updated_at = updated_at WHERE id = ? RETURNING *`, input.ItemID).Scan(ctx, item) + if errors.Is(err, sql.ErrNoRows) { + return ErrNotFound + } if err != nil { - if err == sql.ErrNoRows { - return fmt.Errorf("replacement item %d: %w", input.ItemID, ErrNotFound) - } - return fmt.Errorf("locking replacement item: %w", err) + return err } if item.ReplacementID != input.ReplacementID { - return fmt.Errorf("replacement item %d belongs to another replacement: %w", item.ID, ErrConflict) + return ErrConflict } - if !item.Status.Executable() { - settled = true + if item.Status != storagereplacement.ItemStatusPending { + terminalErr = storagereplacement.ErrItemCancelled return nil } - // The claim is revalidated before any provider call so a lost lease can - // never race a second worker into the same transfer. - if item.Status != storagereplacement.ItemStatusRunning || item.ClaimedAt == nil || - !item.ClaimedAt.Equal(input.ItemClaimedAt) || item.LeaseUntil == nil || !item.LeaseUntil.After(time.Now()) { - return ErrItemClaimLost - } - replacement, err := lockReplacementByID(ctx, db, input.ReplacementID) if err != nil { return err } - confirmationRecovery := false - readyOnlyReservation := false - if item.TargetCopyID != nil { - attempts, countErr := db.NewSelect().Model((*model.StorageUploadCopy)(nil)). - Where("id = ?", *item.TargetCopyID). - Where("commit_attempt_id IS NOT NULL AND commit_attempt_id <> ''"). - Count(ctx) - if countErr != nil { - return fmt.Errorf("checking replacement confirmation recovery: %w", countErr) - } - confirmationRecovery = attempts == 1 - readyReservations, countErr := db.NewSelect().Model((*model.StorageUploadCopy)(nil)). - Where("id = ?", *item.TargetCopyID). - Where("status = ?", model.StorageUploadCopyStatusPieceReady). - Where("commit_ready_at IS NOT NULL"). - Where("commit_attempt_id IS NULL"). - Where("commit_attempted_at IS NULL"). - Count(ctx) - if countErr != nil { - return fmt.Errorf("checking replacement ready-only reservation: %w", countErr) - } - readyOnlyReservation = readyReservations == 1 - } - if !replacement.Status.Active() && !confirmationRecovery { - switch { - case replacement.Status == storagereplacement.StatusFailed && readyOnlyReservation: - settled = true - return settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusFailed) - case replacement.Status.Terminal(): - settled = true - return settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusCancelled) - } - deferred = true - return releaseReplacementItemForRetry(ctx, db, item) + if replacement.Status.Terminal() { + terminalErr = storagereplacement.ErrItemCancelled + return settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusCancelled, "") } - uploads := &BunStorageUploadRepo{db: db} + uploads := &BunStorageContentRepo{db: db} source, err := uploads.GetDataSetBindingByID(ctx, replacement.SourceDataSetID) if err != nil { return err @@ -273,45 +215,20 @@ func (r *BunStorageReplacementRepo) AcquireItem(ctx context.Context, input Acqui if err != nil { return err } - if source == nil || target == nil { - return fmt.Errorf("acquiring replacement item: data set: %w", ErrNotFound) - } - if (!confirmationRecovery && !target.IsCurrent) || target.CopyIndex != replacement.CopyIndex || source.CopyIndex != replacement.CopyIndex { - return fmt.Errorf("acquiring replacement item: replica slot changed: %w", ErrConflict) - } - upload, err := uploads.GetByID(ctx, item.UploadID) + upload, err := uploads.GetByID(ctx, item.ContentID) if err != nil { return err } - if upload == nil || upload.BucketID != replacement.BucketID { - return fmt.Errorf("acquiring replacement item: upload %d: %w", item.UploadID, ErrNotFound) + if source == nil || target == nil || upload == nil || upload.BucketID != replacement.BucketID { + return ErrConflict } - version, err := selectLiveObjectVersionForStorageUpload(ctx, db, upload, nil) + version, err := selectLiveObjectVersionForStorageContent(ctx, db, upload, nil) if err != nil { return err } if version == nil { - if !confirmationRecovery { - // Nothing references this content any more, so the new provider does - // not need it. - settled = true - return settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusCancelled) - } - } - if confirmationRecovery { - var snapshotVersion model.ObjectVersion - if version != nil { - snapshotVersion = *version - } - snapshot = &ReplacementItemSnapshot{ - Replacement: *replacement, - Item: *item, - Source: *source, - Target: *target, - Upload: *upload, - Version: snapshotVersion, - } - return nil + terminalErr = storagereplacement.ErrItemCancelled + return settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusCancelled, "") } owed, inFlight, err := sourceCopyState(ctx, db, upload.ID, source.ID) if err != nil { @@ -319,250 +236,240 @@ func (r *BunStorageReplacementRepo) AcquireItem(ctx context.Context, input Acqui } if !owed { if inFlight { - // The retiring generation is still writing this content. It is not - // copyable yet and must not be cancelled: the write will commit, - // and the coverage gate would then block on content with no item - // behind it. Park it and revisit. - deferred = true - return r.parkItemWaitingSource(ctx, db, item, "the retiring provider has not finished storing this content") + _, err := db.NewUpdate(). + Model((*storagereplacement.Item)(nil)). + Set("last_error = ?", "the retiring provider has not finished storing this content"). + Set("updated_at = ?", time.Now()). + Where("id = ? AND status = ?", item.ID, storagereplacement.ItemStatusPending). + Exec(ctx) + terminalErr = storagereplacement.ErrItemDeferred + return err } - // The retiring generation never stored this content and never will, so - // the slot owes the target nothing for it. - settled = true - return settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusCancelled) + terminalErr = storagereplacement.ErrItemCancelled + return settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusCancelled, "") } covered, err := targetHoldsReadableCopy(ctx, db, upload.ID, target.ID) if err != nil { return err } if covered { - // Already migrated, most likely by an ordinary upload that landed on - // the target after activation. - settled = true - return settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusCopied) + terminalErr = storagereplacement.ErrItemCancelled + return settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusCopied, "") } snapshot = &ReplacementItemSnapshot{ - Replacement: *replacement, - Item: *item, - Source: *source, - Target: *target, - Upload: *upload, - Version: *version, + Replacement: *replacement, Item: *item, Source: *source, + Target: *target, Upload: *upload, Version: *version, } return nil }) if err != nil { return nil, err } - if deferred { - return nil, storagereplacement.ErrItemDeferred - } - if settled { - return nil, storagereplacement.ErrItemCancelled + if terminalErr != nil { + return nil, terminalErr } return snapshot, nil } -func releaseReplacementItemForRetry(ctx context.Context, db bun.IDB, item *storagereplacement.Item) error { - now := time.Now() - res, err := db.NewUpdate(). - Model((*storagereplacement.Item)(nil)). - Set("status = ?", storagereplacement.ItemStatusPending). - Set("scheduled_at = ?", now). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("updated_at = ?", now). - Where("id = ?", item.ID). - Where("status NOT IN (?, ?)", storagereplacement.ItemStatusCopied, storagereplacement.ItemStatusCancelled). - Exec(ctx) - if err != nil { - return fmt.Errorf("releasing inactive replacement item: %w", err) - } - if rows, _ := res.RowsAffected(); rows != 1 { - return ErrItemClaimLost - } - return nil +func (r *BunStorageReplacementRepo) MarkReplacementItemCopied(ctx context.Context, replacementID, itemID, targetCopyID int64) error { + return runMaybeTx(ctx, r.db, func(db bun.IDB) error { + item, err := lockReplacementItem(ctx, db, replacementID, itemID) + if err != nil { + return err + } + if targetCopyID > 0 { + + count, countErr := db.NewSelect(). + Model((*model.StorageCopy)(nil)). + Where("id = ? AND content_id = ? AND storage_data_set_id = ?", targetCopyID, item.ContentID, item.TargetDataSetID). + Count(ctx) + if countErr != nil { + return countErr + } + if count != 1 { + return ErrConflict + } + } + return settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusCopied, "") + }) } -func (r *BunStorageReplacementRepo) parkItemWaitingSource(ctx context.Context, db bun.IDB, item *storagereplacement.Item, reason string) error { - _, err := db.NewUpdate(). - Model((*storagereplacement.Item)(nil)). - Set("status = ?", storagereplacement.ItemStatusWaitingSource). - Set("scheduled_at = ?", time.Now().Add(time.Minute)). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("last_error = ?", reason). - Set("updated_at = ?", time.Now()). - Where("id = ?", item.ID). - Where("status NOT IN (?, ?)", storagereplacement.ItemStatusCopied, storagereplacement.ItemStatusCancelled). - Exec(ctx) - if err != nil { - return fmt.Errorf("parking replacement item: %w", err) +func (r *BunStorageReplacementRepo) MarkReplacementItemCancelled(ctx context.Context, replacementID, itemID int64) error { + return runMaybeTx(ctx, r.db, func(db bun.IDB) error { + item, err := lockReplacementItem(ctx, db, replacementID, itemID) + if err != nil { + return err + } + return settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusCancelled, "") + }) +} + +func (r *BunStorageReplacementRepo) MarkReplacementItemAttention(ctx context.Context, replacementID, itemID int64, lastError string) error { + return runMaybeTx(ctx, r.db, func(db bun.IDB) error { + item, err := lockReplacementItem(ctx, db, replacementID, itemID) + if err != nil { + return err + } + return settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusAttention, lastError) + }) +} + +func lockReplacementItem(ctx context.Context, db bun.IDB, replacementID, itemID int64) (*storagereplacement.Item, error) { + item := new(storagereplacement.Item) + err := db.NewRaw(`UPDATE storage_replacement_items + SET updated_at = updated_at + WHERE id = ? AND replacement_id = ? + RETURNING *`, itemID, replacementID).Scan(ctx, item) + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNotFound } - return nil + return item, err } -// settleReplacementItem moves an item to a terminal status and keeps the -// replacement's progress counter in step. -func settleReplacementItem(ctx context.Context, db bun.IDB, item *storagereplacement.Item, status storagereplacement.ItemStatus) error { +func settleReplacementItem( + ctx context.Context, + db bun.IDB, + item *storagereplacement.Item, + status storagereplacement.ItemStatus, + lastError string, +) error { + if item.Status == status { + return nil + } + if item.Status != storagereplacement.ItemStatusPending { + return ErrConflict + } now := time.Now() - res, err := db.NewUpdate(). + result, err := db.NewUpdate(). Model((*storagereplacement.Item)(nil)). Set("status = ?", status). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). + Set("last_error = ?", nullableText(lastError)). Set("updated_at = ?", now). - Where("id = ?", item.ID). - Where("status NOT IN (?, ?)", storagereplacement.ItemStatusCopied, storagereplacement.ItemStatusCancelled). + Where("id = ? AND status = ?", item.ID, storagereplacement.ItemStatusPending). Exec(ctx) if err != nil { return fmt.Errorf("settling replacement item: %w", err) } - if rows, _ := res.RowsAffected(); rows == 0 { - return nil - } - if status != storagereplacement.ItemStatusCopied { - return clearUnattemptedReplacementReservation(ctx, db, item.TargetCopyID, now) + if rows, _ := result.RowsAffected(); rows != 1 { + return ErrConflict } - if _, err := db.NewUpdate(). - Model((*storagereplacement.Replacement)(nil)). - Set("items_copied = items_copied + 1"). - Set("updated_at = ?", now). - Where("id = ? AND items_copied < items_total", item.ReplacementID). - Exec(ctx); err != nil { - return fmt.Errorf("recording replacement progress: %w", err) + if status == storagereplacement.ItemStatusCopied { + _, err = db.NewUpdate(). + Model((*storagereplacement.Replacement)(nil)). + Set("items_copied = items_copied + 1"). + Set("updated_at = ?", now). + Where("id = ? AND items_copied < items_total", item.ReplacementID). + Exec(ctx) + if err != nil { + return fmt.Errorf("recording replacement progress: %w", err) + } + _, err = db.NewUpdate(). + Model((*storagereplacement.Replacement)(nil)). + Set("status = ?", storagereplacement.StatusMigrating). + Set("wait_reason = NULL"). + Set("updated_at = ?", now). + Where("id = ? AND status = ? AND wait_reason = ?", item.ReplacementID, + storagereplacement.StatusWaiting, storagereplacement.WaitReasonReadableSource). + Exec(ctx) + return err } - return resumeReadableSourceMigration(ctx, db, item.ReplacementID, now) + return clearUnattemptedReplacementReservation(ctx, db, item.ContentID, item.TargetDataSetID, now) } func clearUnattemptedReplacementReservation( ctx context.Context, db bun.IDB, - targetCopyID *int64, + contentID int64, + targetDataSetID int64, now time.Time, ) error { - if targetCopyID == nil { + if targetDataSetID == 0 { return nil } if _, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("commit_attempt_id = NULL"). - Set("commit_attempted_at = NULL"). - Set("commit_submission_json = NULL"). - Set("commit_ready_at = NULL"). - Set("commit_extra_data_hex = NULL"). - Set("commit_transaction_id = NULL"). - Set("commit_confirmed_transaction_id = NULL"). - Set("commit_attention_code = NULL"). - Set("commit_attention_at = NULL"). + Model((*storagecommit.Attempt)(nil)). + Set("status = ?", storagecommit.AttemptStatusReleased). + Set("release_reason = ?", string(storagecommit.ReleaseOwnerTerminal)). + Set("resolved_at = ?", now). Set("updated_at = ?", now). - Where("id = ?", *targetCopyID). - Where("status = ?", model.StorageUploadCopyStatusPieceReady). - Where("commit_attempted_at IS NULL"). + Where("content_id = ? AND storage_data_set_id = ?", contentID, targetDataSetID). + Where("status = ? AND resolved_at IS NULL", storagecommit.AttemptStatusReserved). Exec(ctx); err != nil { - return fmt.Errorf("clearing cancelled replacement commit reservation: %w", err) + return err } - return nil + _, err := db.NewUpdate(). + Model((*model.StorageCopy)(nil)). + Set("commit_ready_at = NULL"). + Set("commit_extra_data_hex = NULL"). + Set("updated_at = ?", now). + Where("content_id = ? AND storage_data_set_id = ?", contentID, targetDataSetID). + Where("status = ?", model.StorageCopyStatusPieceReady). + Where(`NOT EXISTS ( + SELECT 1 FROM storage_commit_attempts AS unresolved_attempt + WHERE unresolved_attempt.content_id = storage_copy.content_id + AND unresolved_attempt.storage_data_set_id = storage_copy.storage_data_set_id + AND unresolved_attempt.resolved_at IS NULL + )`). + Exec(ctx) + return err } -// sourceCopyState answers two different questions that must not be collapsed: -// whether the retiring generation already stored this content, and whether it is -// still in the middle of storing it. Treating "not committed yet" as "never -// stored" cancels work that the coverage gate will later demand. -func sourceCopyState(ctx context.Context, db bun.IDB, uploadID, sourceDataSetID int64) (owed bool, inFlight bool, err error) { - var copies []model.StorageUploadCopy - if err := db.NewSelect(). - Model(&copies). - Where("upload_id = ? AND storage_data_set_id = ?", uploadID, sourceDataSetID). +func sourceCopyState(ctx context.Context, db bun.IDB, contentID, sourceDataSetID int64) (bool, bool, error) { + var copies []model.StorageCopy + if err := db.NewSelect().Model(&copies). + Where("content_id = ? AND storage_data_set_id = ?", contentID, sourceDataSetID). Scan(ctx); err != nil { - return false, false, fmt.Errorf("checking retiring generation copy: %w", err) + return false, false, err } + inFlight := false for i := range copies { switch copies[i].Status { - case model.StorageUploadCopyStatusCommitted: + case model.StorageCopyStatusCommitted: return true, false, nil - case model.StorageUploadCopyStatusPending, - model.StorageUploadCopyStatusPieceReady, - model.StorageUploadCopyStatusCommitting: + case model.StorageCopyStatusPending, model.StorageCopyStatusPieceReady, model.StorageCopyStatusCommitting: inFlight = true } } return false, inFlight, nil } -// targetHoldsReadableCopy answers the same question the retirement coverage gate -// asks, so an item is never left owing work the gate already considers done. -func targetHoldsReadableCopy(ctx context.Context, db bun.IDB, uploadID, targetDataSetID int64) (bool, error) { +func targetHoldsReadableCopy(ctx context.Context, db bun.IDB, contentID, targetDataSetID int64) (bool, error) { query := fmt.Sprintf(`SELECT COUNT(*) - FROM storage_upload_copies AS target_copy + FROM storage_copies AS target_copy JOIN storage_data_sets AS target_data_set ON target_data_set.id = target_copy.storage_data_set_id - WHERE target_copy.upload_id = ? + WHERE target_copy.content_id = ? AND target_copy.storage_data_set_id = ? AND %s`, readableCommittedCopyPredicateSQL("target_copy", "target_data_set")) var count int - if err := db.NewRaw(query, uploadID, targetDataSetID).Scan(ctx, &count); err != nil { - return false, fmt.Errorf("checking replacement target coverage: %w", err) + if err := db.NewRaw(query, contentID, targetDataSetID).Scan(ctx, &count); err != nil { + return false, err } return count > 0, nil } -// AttachTargetCopy creates the copy row on the target generation, or returns -// the existing one so a retried item reuses the same concrete row. -func (r *BunStorageReplacementRepo) AttachTargetCopy(ctx context.Context, input AttachReplacementTargetCopyInput) (*model.StorageUploadCopy, error) { - if input.ReplacementID <= 0 || input.ItemID <= 0 || input.UploadID <= 0 { - return nil, fmt.Errorf("attaching replacement target copy: %w", ErrInvalidInput) +// AttachTargetCopy resolves the copy an item already names. Seeding created +// both the item and its pending copy, so this only reads back the binding the +// item was born with and refuses an item that belongs to another content. +func (r *BunStorageReplacementRepo) AttachTargetCopy(ctx context.Context, input AttachReplacementTargetCopyInput) (*model.StorageCopy, error) { + if input.ReplacementID <= 0 || input.ItemID <= 0 || input.ContentID <= 0 { + return nil, ErrInvalidInput } - var attached *model.StorageUploadCopy + var attached *model.StorageCopy err := runMaybeTx(ctx, r.db, func(db bun.IDB) error { - replacement, err := lockReplacementByID(ctx, db, input.ReplacementID) + item, err := lockReplacementItem(ctx, db, input.ReplacementID, input.ItemID) if err != nil { return err } - uploads := &BunStorageUploadRepo{db: db} - target, err := uploads.GetDataSetBindingByID(ctx, replacement.TargetDataSetID) - if err != nil { - return err + if item.ContentID != input.ContentID || item.Status != storagereplacement.ItemStatusPending { + return ErrConflict } - if target == nil { - return fmt.Errorf("attaching replacement target copy: data set: %w", ErrNotFound) - } - if err := uploads.CreateUploadCopiesForBindings(ctx, input.UploadID, []UploadCopyBindingInput{{ - StorageDataSetID: target.ID, - CopyIndex: target.CopyIndex, - // Migration pulls from a remote replica whenever one is readable. - TransferMethod: model.StorageCopyTransferMethodPeerPull, - ProviderID: target.ProviderID, - }}); err != nil { - return err - } - copyRow, err := uploads.GetUploadCopyForDataSet(ctx, input.UploadID, target.ID) - if err != nil { - return err - } - if copyRow == nil { - return fmt.Errorf("attaching replacement target copy: %w", ErrNotFound) - } - query := db.NewUpdate(). - Model((*storagereplacement.Item)(nil)). - Set("target_copy_id = ?", copyRow.ID). - Set("updated_at = ?", time.Now()). - Where("id = ? AND replacement_id = ?", input.ItemID, input.ReplacementID) - if !input.ItemClaimedAt.IsZero() { - query = query.Where("status = ? AND claimed_at = ? AND lease_until > ?", - storagereplacement.ItemStatusRunning, input.ItemClaimedAt, time.Now()) - } - res, err := query.Exec(ctx) - if err != nil { - return fmt.Errorf("recording replacement target copy: %w", err) - } - if rows, _ := res.RowsAffected(); rows != 1 { - return ErrItemClaimLost + uploads := &BunStorageContentRepo{db: db} + copyRow, err := uploads.GetUploadCopyForDataSet(ctx, item.ContentID, item.TargetDataSetID) + if err != nil || copyRow == nil { + return errors.Join(err, ErrNotFound) } attached = copyRow return nil }) - if err != nil { - return nil, err - } - return attached, nil + return attached, err } diff --git a/internal/db/repository/storage_replacement_postgres_test.go b/internal/db/repository/storage_replacement_postgres_test.go deleted file mode 100644 index 660877b..0000000 --- a/internal/db/repository/storage_replacement_postgres_test.go +++ /dev/null @@ -1,587 +0,0 @@ -package repository_test - -import ( - "context" - "errors" - "fmt" - "os" - "strings" - "sync" - "testing" - "time" - - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/stdlib" - "github.com/strahe/synaps3/internal/config" - appdb "github.com/strahe/synaps3/internal/db" - "github.com/strahe/synaps3/internal/db/migrations" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/uptrace/bun" -) - -type replacementLockContextKey struct{} - -type replacementBucketLockBarrier struct { - winner string - locked chan struct{} - release chan struct{} - competitor chan struct{} - lockedOnce sync.Once - attemptOnce sync.Once -} - -type replacementItemClaimBarrier struct { - winner string - locked chan struct{} - release chan struct{} - lockedOnce sync.Once -} - -func (h *replacementItemClaimBarrier) BeforeQuery(ctx context.Context, event *bun.QueryEvent) context.Context { - return ctx -} - -func (h *replacementItemClaimBarrier) AfterQuery(ctx context.Context, event *bun.QueryEvent) { - if !replacementItemClaimQuery(event.Query) || ctx.Value(replacementLockContextKey{}) != h.winner { - return - } - h.lockedOnce.Do(func() { - close(h.locked) - <-h.release - }) -} - -func replacementItemClaimQuery(query string) bool { - query = strings.ToLower(query) - return strings.Contains(query, "from storage_replacements as replacement") && - strings.Contains(query, "for update of replacement skip locked") -} - -func (h *replacementBucketLockBarrier) BeforeQuery(ctx context.Context, event *bun.QueryEvent) context.Context { - if replacementBucketLockQuery(event.Query) && ctx.Value(replacementLockContextKey{}) != h.winner { - h.attemptOnce.Do(func() { close(h.competitor) }) - } - return ctx -} - -func (h *replacementBucketLockBarrier) AfterQuery(ctx context.Context, event *bun.QueryEvent) { - if !replacementBucketLockQuery(event.Query) || ctx.Value(replacementLockContextKey{}) != h.winner { - return - } - h.lockedOnce.Do(func() { - close(h.locked) - <-h.release - }) -} - -func replacementBucketLockQuery(query string) bool { - query = strings.ToLower(query) - return strings.Contains(query, "update") && strings.Contains(query, "buckets") && - strings.Contains(query, "updated_at = updated_at") -} - -// The generation indexes rely on partial-index semantics and on ON CONFLICT -// inferring a partial index. Those differ enough between SQLite and PostgreSQL -// that the guarantees have to be checked on both. -func TestPostgresStorageReplacementSchemaParity(t *testing.T) { - dsn := os.Getenv("SYNAPS3_POSTGRES_TEST_DSN") - if dsn == "" { - t.Skip("SYNAPS3_POSTGRES_TEST_DSN is not set") - } - ctx := context.Background() - db := newPostgresReplacementDB(t, ctx, dsn) - repos := repository.NewRepositories(db) - - bucket := &model.Bucket{Name: "pg-replacement", Status: model.BucketStatusActive} - if _, err := db.NewInsert().Model(bucket).Exec(ctx); err != nil { - t.Fatalf("seed bucket: %v", err) - } - source, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: source.ID, DataSetID: onChainID(t, "1001"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - - row, _, err := repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ - BucketID: bucket.ID, - SourceDataSetID: source.ID, - SelectionMode: storagereplacement.SelectionModeManual, - TargetProviderID: onChainID(t, "202"), - ClientRequestID: "postgres-replacement", - MaxRetries: 5, - }) - if err != nil { - t.Fatalf("Authorize: %v", err) - } - - // A second live replacement for one source must be impossible. - if _, err := db.NewInsert().Model(&storagereplacement.Replacement{ - BucketID: bucket.ID, CopyIndex: 0, - SourceDataSetID: source.ID, TargetDataSetID: row.TargetDataSetID, - SelectionMode: storagereplacement.SelectionModeManual, - Status: storagereplacement.StatusPreparingTarget, - ConfirmedAt: time.Now(), CreatedAt: time.Now(), UpdatedAt: time.Now(), - }).Exec(ctx); err == nil { - t.Fatal("PostgreSQL accepted a second active replacement for one source") - } - - // The slot may hold several generations, but only one current one. - if _, err := db.NewInsert().Model(&model.StorageDataSet{ - BucketID: bucket.ID, ProviderID: onChainID(t, "303"), CopyIndex: 0, - Generation: 5, IsCurrent: true, Status: model.StorageDataSetStatusPending, - CreatedAt: time.Now(), UpdatedAt: time.Now(), - }).Exec(ctx); err == nil { - t.Fatal("PostgreSQL accepted a second current generation for one slot") - } - if _, err := db.NewInsert().Model(&model.StorageDataSet{ - BucketID: bucket.ID, ProviderID: onChainID(t, "303"), CopyIndex: 0, - Generation: 5, IsCurrent: false, Status: model.StorageDataSetStatusPending, - CreatedAt: time.Now(), UpdatedAt: time.Now(), - }).Exec(ctx); err != nil { - t.Fatalf("PostgreSQL rejected a historical generation: %v", err) - } - - upload := &model.StorageUpload{ - BucketID: bucket.ID, SourceVersionID: "01J00000000000000000000PG1", - ContentSize: 10, Checksum: "pg-sum", Status: model.StorageUploadStatusRunning, RequestedCopies: 1, - CreatedAt: time.Now(), UpdatedAt: time.Now(), - } - if _, err := db.NewInsert().Model(upload).Exec(ctx); err != nil { - t.Fatalf("seed upload: %v", err) - } - // ON CONFLICT must infer the partial index, so a repeated bind is a no-op - // rather than an error or a duplicate. - for range 2 { - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: source.ID, CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: onChainID(t, "101"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - } - count, err := db.NewSelect().Model((*model.StorageUploadCopy)(nil)). - Where("upload_id = ?", upload.ID).Count(ctx) - if err != nil { - t.Fatalf("count copies: %v", err) - } - if count != 1 { - t.Fatalf("copies = %d, want the repeated bind to be a no-op", count) - } - - // Unbound copies are distinct under NULL, so they need their own guard. - if _, err := db.NewInsert().Model(&model.StorageUploadCopy{ - UploadID: upload.ID, CopyIndex: 3, - TransferMethod: model.StorageCopyTransferMethodPeerPull, - CreatedAt: time.Now(), UpdatedAt: time.Now(), - }).Exec(ctx); err != nil { - t.Fatalf("seed unbound copy: %v", err) - } - if _, err := db.NewInsert().Model(&model.StorageUploadCopy{ - UploadID: upload.ID, CopyIndex: 3, - TransferMethod: model.StorageCopyTransferMethodPeerPull, - CreatedAt: time.Now(), UpdatedAt: time.Now(), - }).Exec(ctx); err == nil { - t.Fatal("PostgreSQL accepted a duplicate unbound copy for one slot") - } - - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: row.TargetDataSetID, DataSetID: onChainID(t, "2002"), - }); err != nil { - t.Fatalf("MarkDataSetReady target: %v", err) - } - if err := repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate replacement: %v", err) - } - maxRetries := 5 - if _, err := db.NewInsert().Model(&storagereplacement.Item{ - ReplacementID: row.ID, UploadID: upload.ID, Status: storagereplacement.ItemStatusPending, - MaxRetries: &maxRetries, CreatedAt: time.Now(), UpdatedAt: time.Now(), - }).Exec(ctx); err != nil { - t.Fatalf("seed replacement item: %v", err) - } - claimed, err := repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || claimed == nil || claimed.ClaimedAt == nil || claimed.LeaseUntil == nil { - t.Fatalf("PostgreSQL durable item claim = %#v err=%v", claimed, err) - } - stale := storagereplacement.ClaimToken{ItemID: claimed.ID, ClaimedAt: claimed.ClaimedAt.Add(-time.Second)} - if err := repos.Replacements.RenewReplacementItemLease(ctx, stale, time.Minute); !errors.Is(err, repository.ErrItemClaimLost) { - t.Fatalf("PostgreSQL stale item renewal = %v, want ErrItemClaimLost", err) - } - if err := repos.Replacements.CancelReplacementItemClaim(ctx, stale); !errors.Is(err, repository.ErrItemClaimLost) { - t.Fatalf("PostgreSQL stale item cancellation = %v, want ErrItemClaimLost", err) - } - var preserved storagereplacement.Item - if err := db.NewSelect().Model(&preserved).Where("id = ?", claimed.ID).Scan(ctx); err != nil { - t.Fatalf("reload PostgreSQL item claim: %v", err) - } - if preserved.Status != storagereplacement.ItemStatusRunning || preserved.ClaimedAt == nil || - !preserved.ClaimedAt.Equal(*claimed.ClaimedAt) { - t.Fatalf("PostgreSQL stale cancellation changed the active claim: %#v", preserved) - } - if err := repos.Replacements.CancelReplacementItemClaim(ctx, storagereplacement.ClaimToken{ - ItemID: claimed.ID, ClaimedAt: *claimed.ClaimedAt, - }); err != nil { - t.Fatalf("PostgreSQL current item cancellation: %v", err) - } -} - -func TestPostgresConcurrentItemClaimsReserveReplacementFairnessSlot(t *testing.T) { - dsn := os.Getenv("SYNAPS3_POSTGRES_TEST_DSN") - if dsn == "" { - t.Skip("SYNAPS3_POSTGRES_TEST_DSN is not set") - } - ctx := context.Background() - db := newPostgresReplacementDB(t, ctx, dsn) - first := seedPostgresReplacement(t, db, "pg-claim-fairness-first") - second := seedPostgresReplacement(t, db, "pg-claim-fairness-second") - for _, fixture := range []*postgresReplacementFixture{first, second} { - if err := fixture.repos.Replacements.Activate(ctx, fixture.row.ID); err != nil { - t.Fatalf("activate replacement %d: %v", fixture.row.ID, err) - } - } - - maxRetries := 5 - seedItem := func(fixture *postgresReplacementFixture, versionID string) { - t.Helper() - upload := &model.StorageUpload{ - BucketID: fixture.bucket.ID, SourceVersionID: versionID, - ContentSize: 1, Checksum: versionID, Status: model.StorageUploadStatusRunning, RequestedCopies: 1, - CreatedAt: time.Now(), UpdatedAt: time.Now(), - } - if _, err := db.NewInsert().Model(upload).Exec(ctx); err != nil { - t.Fatalf("seed upload: %v", err) - } - if _, err := db.NewInsert().Model(&storagereplacement.Item{ - ReplacementID: fixture.row.ID, UploadID: upload.ID, - Status: storagereplacement.ItemStatusPending, ScheduledAt: time.Now().Add(-time.Second), - MaxRetries: &maxRetries, CreatedAt: time.Now(), UpdatedAt: time.Now(), - }).Exec(ctx); err != nil { - t.Fatalf("seed replacement item: %v", err) - } - } - seedItem(first, "pg-fairness-first-a") - seedItem(first, "pg-fairness-first-b") - seedItem(second, "pg-fairness-second-a") - - barrier := &replacementItemClaimBarrier{ - winner: "first-claim", locked: make(chan struct{}), release: make(chan struct{}), - } - db.AddQueryHook(barrier) - type claimResult struct { - item *storagereplacement.Item - err error - } - firstResult := make(chan claimResult, 1) - go func() { - claimCtx := context.WithValue(ctx, replacementLockContextKey{}, barrier.winner) - item, err := first.repos.Replacements.ClaimReadyReplacementItem(claimCtx, time.Minute) - firstResult <- claimResult{item: item, err: err} - }() - waitReplacementSignal(t, barrier.locked, "first replacement item claim") - released := false - defer func() { - if !released { - close(barrier.release) - } - }() - - secondCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - concurrent, err := second.repos.Replacements.ClaimReadyReplacementItem(secondCtx, time.Minute) - if err != nil || concurrent == nil { - t.Fatalf("concurrent item claim = %#v err=%v", concurrent, err) - } - close(barrier.release) - released = true - var initial claimResult - select { - case initial = <-firstResult: - case <-time.After(10 * time.Second): - t.Fatal("timed out waiting for initial replacement item claim") - } - if initial.err != nil || initial.item == nil { - t.Fatalf("initial item claim = %#v err=%v", initial.item, initial.err) - } - if initial.item.ReplacementID != first.row.ID || concurrent.ReplacementID != second.row.ID { - t.Fatalf("concurrent claims used replacements %d and %d, want %d then %d", - initial.item.ReplacementID, concurrent.ReplacementID, first.row.ID, second.row.ID) - } -} - -func TestPostgresAuthorizeAndActivateSerializeOnBucket(t *testing.T) { - dsn := os.Getenv("SYNAPS3_POSTGRES_TEST_DSN") - if dsn == "" { - t.Skip("SYNAPS3_POSTGRES_TEST_DSN is not set") - } - - for _, winner := range []string{"activate", "authorize"} { - t.Run(winner+" wins", func(t *testing.T) { - ctx := context.Background() - db := newPostgresReplacementDB(t, ctx, dsn) - fixture := seedPostgresReplacement(t, db, "pg-replacement-lock-"+winner) - barrier := &replacementBucketLockBarrier{ - winner: winner, locked: make(chan struct{}), release: make(chan struct{}), competitor: make(chan struct{}), - } - defer func() { - select { - case <-barrier.release: - default: - close(barrier.release) - } - }() - db.AddQueryHook(barrier) - - activateResult := make(chan error, 1) - authorizeResult := make(chan error, 1) - var successor *storagereplacement.Replacement - startActivate := func() { - go func() { - activateCtx := context.WithValue(ctx, replacementLockContextKey{}, "activate") - activateResult <- fixture.repos.Replacements.Activate(activateCtx, fixture.row.ID) - }() - } - startAuthorize := func() { - go func() { - authorizeCtx := context.WithValue(ctx, replacementLockContextKey{}, "authorize") - row, _, err := fixture.repos.Replacements.Authorize(authorizeCtx, repository.AuthorizeReplacementInput{ - BucketID: fixture.bucket.ID, SourceDataSetID: fixture.source.ID, - SelectionMode: storagereplacement.SelectionModeManual, - TargetProviderID: onChainID(t, "303"), ClientRequestID: "concurrent-successor", MaxRetries: 5, - }) - successor = row - authorizeResult <- err - }() - } - if winner == "activate" { - startActivate() - } else { - startAuthorize() - } - waitReplacementSignal(t, barrier.locked, "winning bucket lock") - if winner == "activate" { - startAuthorize() - } else { - startActivate() - } - waitReplacementSignal(t, barrier.competitor, "competing bucket lock") - close(barrier.release) - - activateErr := waitReplacementResult(t, activateResult, "Activate") - authorizeErr := waitReplacementResult(t, authorizeResult, "Authorize") - current, err := fixture.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, fixture.bucket.ID, fixture.source.CopyIndex) - if err != nil || current == nil { - t.Fatalf("current generation = %#v err=%v", current, err) - } - original, err := fixture.repos.Replacements.GetByID(ctx, fixture.row.ID) - if err != nil || original == nil { - t.Fatalf("original replacement = %#v err=%v", original, err) - } - - if winner == "activate" { - if activateErr != nil { - t.Fatalf("Activate winner: %v", activateErr) - } - if !errors.Is(authorizeErr, storagereplacement.ErrSourceNotCurrent) { - t.Fatalf("Authorize after activation = %v, want ErrSourceNotCurrent", authorizeErr) - } - if current.ID != fixture.target.ID || original.Status != storagereplacement.StatusMigrating { - t.Fatalf("winner state = current:%d replacement:%s, want target %d migrating", current.ID, original.Status, fixture.target.ID) - } - return - } - if authorizeErr != nil || successor == nil { - t.Fatalf("Authorize winner = %#v err=%v", successor, authorizeErr) - } - if !errors.Is(activateErr, repository.ErrConflict) { - t.Fatalf("Activate superseded replacement = %v, want ErrConflict", activateErr) - } - if current.ID != fixture.source.ID || original.Status != storagereplacement.StatusSuperseded { - t.Fatalf("winner state = current:%d replacement:%s, want source %d and superseded", current.ID, original.Status, fixture.source.ID) - } - if successor.TargetDataSetID == current.ID { - t.Fatalf("superseding target %d became current before activation", successor.TargetDataSetID) - } - }) - } -} - -func TestPostgresAuthorizeConcurrentIdempotentReplay(t *testing.T) { - dsn := os.Getenv("SYNAPS3_POSTGRES_TEST_DSN") - if dsn == "" { - t.Skip("SYNAPS3_POSTGRES_TEST_DSN is not set") - } - ctx := context.Background() - db := newPostgresReplacementDB(t, ctx, dsn) - repos := repository.NewRepositories(db) - bucket := &model.Bucket{Name: "pg-concurrent-idempotency", Status: model.BucketStatusActive} - if _, err := db.NewInsert().Model(bucket).Exec(ctx); err != nil { - t.Fatalf("seed bucket: %v", err) - } - source := &model.StorageDataSet{ - BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, - Generation: 1, IsCurrent: true, Status: model.StorageDataSetStatusReady, - DataSetID: onChainIDPtr(t, "1001"), CreatedAt: time.Now(), UpdatedAt: time.Now(), - } - if _, err := db.NewInsert().Model(source).Exec(ctx); err != nil { - t.Fatalf("seed source: %v", err) - } - input := repository.AuthorizeReplacementInput{ - BucketID: bucket.ID, SourceDataSetID: source.ID, - SelectionMode: storagereplacement.SelectionModeManual, - TargetProviderID: onChainID(t, "202"), ClientRequestID: "concurrent-confirmation", MaxRetries: 5, - } - type result struct { - row *storagereplacement.Replacement - created bool - err error - } - start := make(chan struct{}) - results := make(chan result, 2) - for range 2 { - go func() { - <-start - row, created, err := repos.Replacements.Authorize(ctx, input) - results <- result{row: row, created: created, err: err} - }() - } - close(start) - first := <-results - second := <-results - if first.err != nil || second.err != nil || first.row == nil || second.row == nil { - t.Fatalf("concurrent results = %#v and %#v", first, second) - } - if first.row.ID != second.row.ID || first.created == second.created { - t.Fatalf("concurrent results = ids %d/%d created %v/%v, want one shared row and one creator", - first.row.ID, second.row.ID, first.created, second.created) - } - count, err := db.NewSelect().Model((*storagereplacement.Replacement)(nil)). - Where("bucket_id = ? AND client_request_id = ?", bucket.ID, input.ClientRequestID).Count(ctx) - if err != nil || count != 1 { - t.Fatalf("persisted replacements = %d err=%v, want one", count, err) - } -} - -func waitReplacementSignal(t *testing.T, signal <-chan struct{}, name string) { - t.Helper() - select { - case <-signal: - case <-time.After(10 * time.Second): - t.Fatalf("timed out waiting for %s", name) - } -} - -func waitReplacementResult(t *testing.T, result <-chan error, name string) error { - t.Helper() - select { - case err := <-result: - return err - case <-time.After(10 * time.Second): - t.Fatalf("timed out waiting for %s", name) - return nil - } -} - -type postgresReplacementFixture struct { - repos *repository.Repositories - bucket *model.Bucket - source *model.StorageDataSet - target *model.StorageDataSet - row *storagereplacement.Replacement -} - -func seedPostgresReplacement(t *testing.T, db *bun.DB, name string) *postgresReplacementFixture { - t.Helper() - ctx := context.Background() - repos := repository.NewRepositories(db) - bucket := &model.Bucket{Name: name, Status: model.BucketStatusActive} - if _, err := db.NewInsert().Model(bucket).Exec(ctx); err != nil { - t.Fatalf("seed bucket: %v", err) - } - source, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, - }) - if err != nil { - t.Fatalf("seed source: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: source.ID, DataSetID: onChainID(t, "1001"), - }); err != nil { - t.Fatalf("ready source: %v", err) - } - row, _, err := repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ - BucketID: bucket.ID, SourceDataSetID: source.ID, - SelectionMode: storagereplacement.SelectionModeManual, - TargetProviderID: onChainID(t, "202"), ClientRequestID: "concurrent-original", MaxRetries: 5, - }) - if err != nil { - t.Fatalf("authorize replacement: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: row.TargetDataSetID, DataSetID: onChainID(t, "2002"), - }); err != nil { - t.Fatalf("ready target: %v", err) - } - target, err := repos.Uploads.GetDataSetBindingByID(ctx, row.TargetDataSetID) - if err != nil || target == nil { - t.Fatalf("target = %#v err=%v", target, err) - } - return &postgresReplacementFixture{repos: repos, bucket: bucket, source: source, target: target, row: row} -} - -func newPostgresReplacementDB(t *testing.T, ctx context.Context, dsn string) *bun.DB { - t.Helper() - adminDB, err := appdb.New(config.DatabaseConfig{ - Driver: "postgres", DSN: dsn, MaxOpenConns: 1, MaxIdleConns: 1, - }) - if err != nil { - t.Fatalf("opening postgres test db: %v", err) - } - - schema := fmt.Sprintf("synaps3_replacement_%d", time.Now().UnixNano()) - quoted := `"` + schema + `"` - if _, err := adminDB.ExecContext(ctx, "CREATE SCHEMA "+quoted); err != nil { - _ = adminDB.Close() - t.Fatalf("creating schema: %v", err) - } - t.Cleanup(func() { - dropCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - if _, err := adminDB.ExecContext(dropCtx, "DROP SCHEMA "+quoted+" CASCADE"); err != nil { - t.Logf("dropping schema %s: %v", schema, err) - } - _ = adminDB.Close() - }) - - pgConfig, err := pgx.ParseConfig(dsn) - if err != nil { - t.Fatalf("parsing postgres test DSN: %v", err) - } - pgConfig.RuntimeParams["search_path"] = schema - registeredDSN := stdlib.RegisterConnConfig(pgConfig) - t.Cleanup(func() { stdlib.UnregisterConnConfig(registeredDSN) }) - db, err := appdb.New(config.DatabaseConfig{ - Driver: "postgres", DSN: registeredDSN, MaxOpenConns: 4, MaxIdleConns: 2, - }) - if err != nil { - t.Fatalf("opening schema-scoped postgres test db: %v", err) - } - t.Cleanup(func() { _ = db.Close() }) - - migrator := migrations.NewMigrator(db) - if err := migrator.Init(ctx); err != nil { - t.Fatalf("migrator init: %v", err) - } - if _, err := migrator.Migrate(ctx); err != nil { - t.Fatalf("running postgres migrations: %v", err) - } - return db -} diff --git a/internal/db/repository/storage_replacement_progress.go b/internal/db/repository/storage_replacement_progress.go new file mode 100644 index 0000000..2699e66 --- /dev/null +++ b/internal/db/repository/storage_replacement_progress.go @@ -0,0 +1,165 @@ +package repository + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/strahe/synaps3/internal/storagereplacement" + "github.com/uptrace/bun" +) + +func (r *BunStorageReplacementRepo) ReplacementExecution( + ctx context.Context, + replacementID int64, +) (storagereplacement.ExecutionSnapshot, error) { + if replacementID <= 0 { + return storagereplacement.ExecutionSnapshot{}, fmt.Errorf("loading replacement execution: %w", ErrInvalidInput) + } + var snapshot storagereplacement.ExecutionSnapshot + err := r.db.NewRaw(`SELECT replacement.id AS replacement_id, + replacement.seeding_complete, + EXISTS (SELECT 1 FROM storage_replacement_items AS item + WHERE item.replacement_id = replacement.id AND item.status = 'pending') AS has_pending, + EXISTS (SELECT 1 FROM storage_replacement_items AS item + WHERE item.replacement_id = replacement.id AND item.status = 'attention') AS has_failed + FROM storage_replacements AS replacement + WHERE replacement.id = ?`, replacementID).Scan(ctx, &snapshot) + if err == sql.ErrNoRows { + return storagereplacement.ExecutionSnapshot{}, fmt.Errorf("provider replacement %d: %w", replacementID, ErrNotFound) + } + if err != nil { + return storagereplacement.ExecutionSnapshot{}, fmt.Errorf("loading replacement execution: %w", err) + } + return snapshot, nil +} + +func (r *BunStorageReplacementRepo) ReplacementProgresses( + ctx context.Context, + replacementIDs []int64, +) (map[int64]storagereplacement.ProgressSnapshot, error) { + out := make(map[int64]storagereplacement.ProgressSnapshot, len(replacementIDs)) + if len(replacementIDs) == 0 { + return out, nil + } + type progressRow struct { + ReplacementID int64 `bun:"replacement_id"` + Status storagereplacement.Status `bun:"replacement_status"` + TargetIsCurrent bool `bun:"target_is_current"` + SeedingComplete bool `bun:"seeding_complete"` + ItemsTotal int `bun:"items_total"` + ItemsCopied int `bun:"items_copied"` + ItemsPending int `bun:"items_pending"` + ItemsActive int `bun:"items_active"` + ItemsCancelled int `bun:"items_cancelled"` + ItemsAttention int `bun:"items_attention"` + ItemsRetrying int `bun:"items_retrying"` + ItemsWaitingSource int `bun:"items_waiting_source"` + ItemsFailed int `bun:"items_failed"` + NextRetryAt *time.Time `bun:"next_retry_at"` + } + var rows []progressRow + now := time.Now() + if err := r.db.NewRaw(`WITH item_progress AS ( + SELECT replacement.id AS replacement_id, + replacement.status AS replacement_status, + target.is_current AS target_is_current, + replacement.seeding_complete, + replacement.items_total, + replacement.items_copied, + item.status AS item_status, + CASE + WHEN item.id IS NULL THEN NULL + WHEN item.status <> 'pending' THEN item.status + WHEN target_copy.status = 'failed' OR copy_task.status = 'failed' THEN 'failed' + WHEN copy_task.status = 'pending' AND copy_task.wait_reason = 'source' THEN 'waiting_source' + WHEN coordinator.status = 'pending' AND coordinator.wait_reason = 'source' + AND item.id = (SELECT MIN(waiting_item.id) + FROM storage_replacement_items AS waiting_item + WHERE waiting_item.replacement_id = replacement.id + AND waiting_item.status = 'pending') THEN 'waiting_source' + WHEN copy_task.status = 'pending' AND copy_task.retry_count > 0 AND copy_task.available_at > ? THEN 'retrying' + WHEN copy_task.status IN ('pending', 'running') THEN 'active' + ELSE 'pending' + END AS progress_status, + CASE WHEN copy_task.status = 'pending' AND copy_task.retry_count > 0 AND copy_task.available_at > ? + THEN copy_task.available_at END AS next_retry_at + FROM storage_replacements AS replacement + JOIN storage_data_sets AS target ON target.id = replacement.target_data_set_id + LEFT JOIN storage_replacement_items AS item ON item.replacement_id = replacement.id + LEFT JOIN storage_copies AS target_copy + ON target_copy.content_id = item.content_id + AND target_copy.storage_data_set_id = item.target_data_set_id + LEFT JOIN tasks AS copy_task ON copy_task.id = target_copy.active_task_id + LEFT JOIN tasks AS coordinator ON coordinator.id = replacement.task_id + WHERE replacement.id IN (?) + ) + SELECT replacement_id, + replacement_status, + target_is_current, + seeding_complete, + items_total, + items_copied, + COALESCE(SUM(CASE WHEN progress_status = 'pending' THEN 1 ELSE 0 END), 0) AS items_pending, + COALESCE(SUM(CASE WHEN progress_status = 'active' THEN 1 ELSE 0 END), 0) AS items_active, + COALESCE(SUM(CASE WHEN item_status = 'cancelled' THEN 1 ELSE 0 END), 0) AS items_cancelled, + COALESCE(SUM(CASE WHEN item_status = 'attention' THEN 1 ELSE 0 END), 0) AS items_attention, + COALESCE(SUM(CASE WHEN progress_status = 'retrying' THEN 1 ELSE 0 END), 0) AS items_retrying, + COALESCE(SUM(CASE WHEN progress_status = 'waiting_source' THEN 1 ELSE 0 END), 0) AS items_waiting_source, + COALESCE(SUM(CASE WHEN progress_status = 'failed' THEN 1 ELSE 0 END), 0) AS items_failed, + MIN(next_retry_at) AS next_retry_at + FROM item_progress + GROUP BY replacement_id, replacement_status, target_is_current, seeding_complete, items_total, items_copied`, + now, now, bun.List(replacementIDs)).Scan(ctx, &rows); err != nil { + return nil, fmt.Errorf("loading replacement progress: %w", err) + } + for i := range rows { + row := rows[i] + processed := row.ItemsCopied + row.ItemsCancelled + var percent *int + if row.SeedingComplete { + value := 100 + if row.ItemsTotal > 0 { + value = min(100, processed*100/row.ItemsTotal) + } + percent = &value + } + out[row.ReplacementID] = storagereplacement.ProgressSnapshot{ + ReplacementID: row.ReplacementID, + Phase: replacementProgressPhase(row.Status, row.TargetIsCurrent), + SeedingComplete: row.SeedingComplete, + ItemsTotal: row.ItemsTotal, + ItemsProcessed: processed, + ItemsCopied: row.ItemsCopied, + ItemsNoLongerNeeded: row.ItemsCancelled, + ItemsPending: row.ItemsPending, + ItemsActive: row.ItemsActive, + ItemsAttention: row.ItemsAttention, + ItemsRetrying: row.ItemsRetrying, + ItemsWaitingSource: row.ItemsWaitingSource, + ItemsFailed: row.ItemsFailed, + Percent: percent, + NextRetryAt: row.NextRetryAt, + } + } + return out, nil +} + +func replacementProgressPhase(status storagereplacement.Status, targetIsCurrent bool) storagereplacement.Phase { + switch status { + case storagereplacement.StatusPreparingTarget: + return storagereplacement.PhasePrepare + case storagereplacement.StatusRetiring, storagereplacement.StatusCleanupAttention, storagereplacement.StatusCompleted: + return storagereplacement.PhaseRetire + case storagereplacement.StatusMigrating: + return storagereplacement.PhaseMigrate + case storagereplacement.StatusWaiting, storagereplacement.StatusFailed: + if targetIsCurrent { + return storagereplacement.PhaseMigrate + } + return storagereplacement.PhasePrepare + default: + return storagereplacement.PhaseNone + } +} diff --git a/internal/db/repository/storage_replacement_queue.go b/internal/db/repository/storage_replacement_queue.go deleted file mode 100644 index 50d031c..0000000 --- a/internal/db/repository/storage_replacement_queue.go +++ /dev/null @@ -1,867 +0,0 @@ -package repository - -import ( - "context" - "database/sql" - "fmt" - "time" - - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" -) - -// InitializeReplacementItemRetryBudgets snapshots the current configuration -// only for rows created before item-level retry budgets existed. -func (r *BunStorageReplacementRepo) InitializeReplacementItemRetryBudgets(ctx context.Context, maxRetries int) (int, error) { - if maxRetries < 0 { - return 0, fmt.Errorf("initializing replacement item retry budgets: %w", ErrInvalidInput) - } - res, err := r.db.NewUpdate(). - Model((*storagereplacement.Item)(nil)). - Set("max_retries = ?", maxRetries). - Where("max_retries IS NULL"). - Exec(ctx) - if err != nil { - return 0, fmt.Errorf("initializing replacement item retry budgets: %w", err) - } - rows, _ := res.RowsAffected() - return int(rows), nil -} - -// ReleaseExpiredItemLeases makes crash-interrupted transfers claimable again. -// Provider-side evidence stays on the target copy row and is deliberately not -// cleared here. -func (r *BunStorageReplacementRepo) ReleaseExpiredItemLeases(ctx context.Context) (int, error) { - now := time.Now() - res, err := r.db.NewUpdate(). - Model((*storagereplacement.Item)(nil)). - Set("status = ?", storagereplacement.ItemStatusPending). - Set("scheduled_at = ?", now). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("updated_at = ?", now). - Where("status = ?", storagereplacement.ItemStatusRunning). - Where("lease_until IS NULL OR lease_until <= ?", now). - Exec(ctx) - if err != nil { - return 0, fmt.Errorf("releasing expired replacement item leases: %w", err) - } - rows, _ := res.RowsAffected() - return int(rows), nil -} - -// ClaimReadyReplacementItem leases one item across every active replacement. -// Fairness is replacement-level first, then ready time and item id. Candidate -// selection stays bounded within the chosen replacement. PostgreSQL uses SKIP -// LOCKED; SQLite's write transaction serializes the selection. -func (r *BunStorageReplacementRepo) ClaimReadyReplacementItem(ctx context.Context, leaseTTL time.Duration) (*storagereplacement.Item, error) { - if leaseTTL <= 0 { - return nil, fmt.Errorf("claiming replacement item: %w", ErrInvalidInput) - } - var claimed *storagereplacement.Item - err := runMaybeTx(ctx, r.db, func(db bun.IDB) error { - now := time.Now() - replacementID, err := selectReadyReplacementID(ctx, db, now) - if err != nil { - return err - } - if replacementID == 0 { - return nil - } - var replacementStatus storagereplacement.Status - if err := db.NewSelect(). - Model((*storagereplacement.Replacement)(nil)). - Column("status"). - Where("id = ?", replacementID). - Scan(ctx, &replacementStatus); err != nil { - return fmt.Errorf("loading ready replacement status: %w", err) - } - terminalOnly := replacementStatus == storagereplacement.StatusFailed || - replacementStatus == storagereplacement.StatusSuperseded - itemID, err := selectReadyReplacementItemID(ctx, db, replacementID, now, terminalOnly) - if err != nil { - return err - } - if itemID == 0 { - return nil - } - item := new(storagereplacement.Item) - err = db.NewRaw(`UPDATE storage_replacement_items - SET status = ?, attempts = attempts + 1, claimed_at = ?, lease_until = ?, updated_at = ? - WHERE id = ? - AND max_retries IS NOT NULL - AND ((status IN (?, ?, ?) AND scheduled_at <= ? AND claimed_at IS NULL) - OR (status = ? AND ? AND scheduled_at <= ? AND claimed_at IS NULL - AND EXISTS ( - SELECT 1 FROM storage_upload_copies AS failed_copy - WHERE failed_copy.id = storage_replacement_items.target_copy_id - AND failed_copy.commit_attempt_id IS NOT NULL - AND failed_copy.commit_attempt_id <> '' - )) - OR (status = ? AND lease_until <= ?)) - RETURNING *`, - storagereplacement.ItemStatusRunning, now, now.Add(leaseTTL), now, - itemID, - storagereplacement.ItemStatusPending, - storagereplacement.ItemStatusRetrying, - storagereplacement.ItemStatusWaitingSource, - now, - storagereplacement.ItemStatusFailed, - terminalOnly, - now, - storagereplacement.ItemStatusRunning, - now, - ).Scan(ctx, item) - if err != nil { - if err == sql.ErrNoRows { - return nil - } - return fmt.Errorf("claiming replacement item: %w", err) - } - if terminalOnly { - settled, checkErr := settleClaimedTerminalReplacementItem(ctx, db, item, replacementStatus, now) - if checkErr != nil { - return checkErr - } - if settled { - return nil - } - } - dispatchUpdate := db.NewUpdate(). - Model((*storagereplacement.Replacement)(nil)). - Set("last_dispatched_at = ?", now). - Set("updated_at = ?", now). - Where("id = ?", item.ReplacementID) - if terminalOnly { - dispatchUpdate = dispatchUpdate.Where("status = ?", replacementStatus) - } else { - dispatchUpdate = dispatchUpdate.Where("status = ? OR (status = ? AND wait_reason = ?)", - storagereplacement.StatusMigrating, - storagereplacement.StatusWaiting, - storagereplacement.WaitReasonReadableSource) - } - if _, err := dispatchUpdate.Exec(ctx); err != nil { - return fmt.Errorf("recording replacement dispatch: %w", err) - } - claimed = item - return nil - }) - return claimed, err -} - -type replacementItemCommitWork uint8 - -const ( - replacementItemCommitWorkNone replacementItemCommitWork = iota - replacementItemCommitWorkDurable - replacementItemCommitWorkCommitted -) - -func settleClaimedTerminalReplacementItem( - ctx context.Context, - db bun.IDB, - item *storagereplacement.Item, - replacementStatus storagereplacement.Status, - now time.Time, -) (bool, error) { - commitWork, err := replacementItemCommitWorkState(ctx, db, item.TargetCopyID) - if err != nil { - return false, err - } - switch commitWork { - case replacementItemCommitWorkDurable: - return false, nil - case replacementItemCommitWorkCommitted: - if err := settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusCopied); err != nil { - return false, fmt.Errorf("settling replacement item with a committed target copy: %w", err) - } - return true, nil - case replacementItemCommitWorkNone: - settledStatus := storagereplacement.ItemStatusCancelled - if replacementStatus == storagereplacement.StatusFailed { - settledStatus = storagereplacement.ItemStatusFailed - } - res, err := db.NewUpdate(). - Model((*storagereplacement.Item)(nil)). - Set("status = ?", settledStatus). - Set("scheduled_at = ?", now). - Set("last_error = NULL"). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("updated_at = ?", now). - Where("id = ?", item.ID). - Where("status = ?", storagereplacement.ItemStatusRunning). - Where("claimed_at = ?", item.ClaimedAt). - Where("lease_until > ?", now). - Exec(ctx) - if err != nil { - return false, fmt.Errorf("settling replacement item without durable commit work: %w", err) - } - if rows, _ := res.RowsAffected(); rows != 1 { - return false, ErrItemClaimLost - } - return true, nil - default: - return false, fmt.Errorf("settling replacement item with unknown commit work: %w", ErrConflict) - } -} - -func replacementItemCommitWorkState( - ctx context.Context, - db bun.IDB, - targetCopyID *int64, -) (replacementItemCommitWork, error) { - if targetCopyID == nil { - return replacementItemCommitWorkNone, nil - } - type commitWorkSnapshot struct { - Status model.StorageUploadCopyStatus `bun:"status"` - CommitReadyAt *time.Time `bun:"commit_ready_at"` - CommitAttemptID *string `bun:"commit_attempt_id"` - CommitAttemptedAt *time.Time `bun:"commit_attempted_at"` - ReadableCommitted bool `bun:"readable_committed"` - } - snapshot := new(commitWorkSnapshot) - query := fmt.Sprintf(`SELECT target_copy.status, - target_copy.commit_ready_at, - target_copy.commit_attempt_id, - target_copy.commit_attempted_at, - COALESCE((%s), FALSE) AS readable_committed - FROM storage_upload_copies AS target_copy - LEFT JOIN storage_data_sets AS target_data_set ON target_data_set.id = target_copy.storage_data_set_id - WHERE target_copy.id = ?`, readableCommittedCopyPredicateSQL("target_copy", "target_data_set")) - err := db.NewRaw(query, *targetCopyID).Scan(ctx, snapshot) - if err == sql.ErrNoRows { - return replacementItemCommitWorkNone, nil - } - if err != nil { - return replacementItemCommitWorkNone, fmt.Errorf("checking replacement item durable commit work: %w", err) - } - if snapshot.ReadableCommitted { - return replacementItemCommitWorkCommitted, nil - } - if (snapshot.CommitAttemptID != nil && *snapshot.CommitAttemptID != "") || - (snapshot.Status == model.StorageUploadCopyStatusPieceReady && - snapshot.CommitReadyAt != nil && snapshot.CommitAttemptID == nil && snapshot.CommitAttemptedAt == nil) { - return replacementItemCommitWorkDurable, nil - } - return replacementItemCommitWorkNone, nil -} - -func selectReadyReplacementID(ctx context.Context, db bun.IDB, now time.Time) (int64, error) { - var id int64 - err := db.NewRaw(readyReplacementSelectionSQL(db.Dialect().Name()), - storagereplacement.StatusMigrating, - storagereplacement.StatusWaiting, - storagereplacement.WaitReasonReadableSource, - now, - now, - storagereplacement.StatusFailed, - storagereplacement.StatusSuperseded, - now, - now, - ).Scan(ctx, &id) - if err == sql.ErrNoRows { - return 0, nil - } - if err != nil { - return 0, fmt.Errorf("selecting ready replacement: %w", err) - } - return id, nil -} - -func readyReplacementSelectionSQL(dialectName dialect.Name) string { - query := `SELECT replacement.id - FROM storage_replacements AS replacement - WHERE (((replacement.status = ? OR (replacement.status = ? AND replacement.wait_reason = ?)) - AND (EXISTS ( - SELECT 1 - FROM storage_replacement_items AS due_item - WHERE due_item.replacement_id = replacement.id - AND due_item.status IN ('pending', 'retrying', 'waiting_source') - AND due_item.scheduled_at <= ? - AND due_item.claimed_at IS NULL - AND due_item.max_retries IS NOT NULL - ) OR EXISTS ( - SELECT 1 - FROM storage_replacement_items AS expired_item - WHERE expired_item.replacement_id = replacement.id - AND expired_item.status = 'running' - AND expired_item.lease_until <= ? - AND expired_item.max_retries IS NOT NULL - ))) - OR (replacement.status IN (?, ?) - AND (EXISTS ( - SELECT 1 - FROM storage_replacement_items AS due_item - JOIN storage_upload_copies AS due_copy ON due_copy.id = due_item.target_copy_id - WHERE due_item.replacement_id = replacement.id - AND (due_item.status IN ('pending', 'retrying', 'waiting_source') - OR (due_item.status = 'failed' - AND due_copy.commit_attempt_id IS NOT NULL - AND due_copy.commit_attempt_id <> '')) - AND due_item.scheduled_at <= ? - AND due_item.claimed_at IS NULL - AND due_item.max_retries IS NOT NULL - AND ((due_copy.commit_attempt_id IS NOT NULL AND due_copy.commit_attempt_id <> '') - OR (due_copy.status = 'piece_ready' - AND due_copy.commit_ready_at IS NOT NULL - AND due_copy.commit_attempt_id IS NULL - AND due_copy.commit_attempted_at IS NULL)) - ) OR EXISTS ( - SELECT 1 - FROM storage_replacement_items AS expired_item - JOIN storage_upload_copies AS expired_copy ON expired_copy.id = expired_item.target_copy_id - WHERE expired_item.replacement_id = replacement.id - AND expired_item.status = 'running' - AND expired_item.lease_until <= ? - AND expired_item.max_retries IS NOT NULL - AND ((expired_copy.commit_attempt_id IS NOT NULL AND expired_copy.commit_attempt_id <> '') - OR (expired_copy.status = 'piece_ready' - AND expired_copy.commit_ready_at IS NOT NULL - AND expired_copy.commit_attempt_id IS NULL - AND expired_copy.commit_attempted_at IS NULL)) - )))) - ORDER BY CASE WHEN replacement.last_dispatched_at IS NULL THEN 0 ELSE 1 END, - replacement.last_dispatched_at ASC, - replacement.id ASC - LIMIT 1` - if dialectName == dialect.PG { - // Serializing on the owning replacement keeps concurrent claimers from - // observing the same fairness timestamp. The item update below is fenced - // by its ready state inside the same transaction. - query += " FOR UPDATE OF replacement SKIP LOCKED" - } - return query -} - -type readyReplacementItemCandidate struct { - ID int64 `bun:"id"` - ReadyAt time.Time `bun:"ready_at"` -} - -func selectReadyReplacementItemID( - ctx context.Context, - db bun.IDB, - replacementID int64, - now time.Time, - terminalOnly bool, -) (int64, error) { - due, err := selectReadyReplacementItemCandidate(ctx, db, readyReplacementDueItemSQL(terminalOnly), - replacementID, - now, - ) - if err != nil { - return 0, err - } - expired, err := selectReadyReplacementItemCandidate(ctx, db, readyReplacementExpiredItemSQL(terminalOnly), - replacementID, - now, - ) - if err != nil { - return 0, err - } - if due == nil { - if expired == nil { - return 0, nil - } - return expired.ID, nil - } - if expired == nil || due.ReadyAt.Before(expired.ReadyAt) || (due.ReadyAt.Equal(expired.ReadyAt) && due.ID < expired.ID) { - return due.ID, nil - } - return expired.ID, nil -} - -func readyReplacementDueItemSQL(terminalOnly bool) string { - if terminalOnly { - return `SELECT item.id, item.scheduled_at AS ready_at - FROM storage_replacement_items AS item - JOIN storage_upload_copies AS storage_copy ON storage_copy.id = item.target_copy_id - WHERE item.replacement_id = ? - AND (item.status IN ('pending', 'retrying', 'waiting_source') - OR (item.status = 'failed' - AND storage_copy.commit_attempt_id IS NOT NULL - AND storage_copy.commit_attempt_id <> '')) - AND item.scheduled_at <= ? - AND item.claimed_at IS NULL - AND item.max_retries IS NOT NULL - AND ((storage_copy.commit_attempt_id IS NOT NULL AND storage_copy.commit_attempt_id <> '') - OR (storage_copy.status = 'piece_ready' - AND storage_copy.commit_ready_at IS NOT NULL - AND storage_copy.commit_attempt_id IS NULL - AND storage_copy.commit_attempted_at IS NULL)) - ORDER BY item.scheduled_at ASC, item.id ASC - LIMIT 1` - } - return `SELECT item.id, item.scheduled_at AS ready_at - FROM storage_replacement_items AS item - WHERE item.replacement_id = ? - AND item.status IN ('pending', 'retrying', 'waiting_source') - AND item.scheduled_at <= ? - AND item.claimed_at IS NULL - AND item.max_retries IS NOT NULL - ORDER BY item.scheduled_at ASC, item.id ASC - LIMIT 1` -} - -func readyReplacementExpiredItemSQL(terminalOnly bool) string { - if terminalOnly { - return `SELECT item.id, item.lease_until AS ready_at - FROM storage_replacement_items AS item - JOIN storage_upload_copies AS storage_copy ON storage_copy.id = item.target_copy_id - WHERE item.replacement_id = ? - AND item.status = 'running' - AND item.lease_until <= ? - AND item.max_retries IS NOT NULL - AND ((storage_copy.commit_attempt_id IS NOT NULL AND storage_copy.commit_attempt_id <> '') - OR (storage_copy.status = 'piece_ready' - AND storage_copy.commit_ready_at IS NOT NULL - AND storage_copy.commit_attempt_id IS NULL - AND storage_copy.commit_attempted_at IS NULL)) - ORDER BY item.lease_until ASC, item.id ASC - LIMIT 1` - } - return `SELECT item.id, item.lease_until AS ready_at - FROM storage_replacement_items AS item - WHERE item.replacement_id = ? - AND item.status = 'running' - AND item.lease_until <= ? - AND item.max_retries IS NOT NULL - ORDER BY item.lease_until ASC, item.id ASC - LIMIT 1` -} - -func selectReadyReplacementItemCandidate( - ctx context.Context, - db bun.IDB, - query string, - args ...any, -) (*readyReplacementItemCandidate, error) { - candidate := new(readyReplacementItemCandidate) - if err := db.NewRaw(query, args...).Scan(ctx, candidate); err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("selecting ready replacement item: %w", err) - } - return candidate, nil -} - -func (r *BunStorageReplacementRepo) RenewReplacementItemLease( - ctx context.Context, - token storagereplacement.ClaimToken, - leaseTTL time.Duration, -) error { - if token.ItemID <= 0 || token.ClaimedAt.IsZero() || leaseTTL <= 0 { - return fmt.Errorf("renewing replacement item lease: %w", ErrInvalidInput) - } - now := time.Now() - res, err := r.db.NewUpdate(). - Model((*storagereplacement.Item)(nil)). - Set("lease_until = ?", now.Add(leaseTTL)). - Where("id = ? AND status = ?", token.ItemID, storagereplacement.ItemStatusRunning). - Where("claimed_at = ?", token.ClaimedAt). - Where("lease_until > ?", now). - Exec(ctx) - if err != nil { - return fmt.Errorf("renewing replacement item lease: %w", err) - } - if rows, _ := res.RowsAffected(); rows != 1 { - return ErrItemClaimLost - } - return nil -} - -func (r *BunStorageReplacementRepo) ReleaseReplacementItemClaim(ctx context.Context, token storagereplacement.ClaimToken) error { - now := time.Now() - res, err := r.db.NewUpdate(). - Model((*storagereplacement.Item)(nil)). - Set("status = ?", storagereplacement.ItemStatusPending). - Set("scheduled_at = ?", now). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("updated_at = ?", now). - Where("id = ? AND status = ?", token.ItemID, storagereplacement.ItemStatusRunning). - Where("claimed_at = ?", token.ClaimedAt). - Exec(ctx) - if err != nil { - return fmt.Errorf("releasing replacement item claim: %w", err) - } - if rows, _ := res.RowsAffected(); rows != 1 { - return ErrItemClaimLost - } - return nil -} - -func (r *BunStorageReplacementRepo) CancelReplacementItemClaim(ctx context.Context, token storagereplacement.ClaimToken) error { - return runMaybeTx(ctx, r.db, func(db bun.IDB) error { - now := time.Now() - item := new(storagereplacement.Item) - err := db.NewRaw(`UPDATE storage_replacement_items - SET status = ?, claimed_at = NULL, lease_until = NULL, updated_at = ? - WHERE id = ? AND status = ? AND claimed_at = ? AND lease_until > ? - RETURNING *`, storagereplacement.ItemStatusCancelled, now, token.ItemID, - storagereplacement.ItemStatusRunning, token.ClaimedAt, now).Scan(ctx, item) - if err != nil { - if err == sql.ErrNoRows { - return ErrItemClaimLost - } - return fmt.Errorf("cancelling replacement item claim: %w", err) - } - return clearUnattemptedReplacementReservation(ctx, db, item.TargetCopyID, now) - }) -} - -func (r *BunStorageReplacementRepo) CompleteReplacementItemClaim(ctx context.Context, token storagereplacement.ClaimToken) error { - return runMaybeTx(ctx, r.db, func(db bun.IDB) error { - now := time.Now() - item := new(storagereplacement.Item) - err := db.NewRaw(`UPDATE storage_replacement_items - SET status = ?, last_error = NULL, claimed_at = NULL, lease_until = NULL, updated_at = ? - WHERE id = ? AND status = ? AND claimed_at = ? AND lease_until > ? - RETURNING *`, storagereplacement.ItemStatusCopied, now, token.ItemID, - storagereplacement.ItemStatusRunning, token.ClaimedAt, now).Scan(ctx, item) - if err != nil { - if err == sql.ErrNoRows { - return ErrItemClaimLost - } - return fmt.Errorf("settling replacement item claim: %w", err) - } - if _, err := db.NewUpdate(). - Model((*storagereplacement.Replacement)(nil)). - Set("items_copied = items_copied + 1"). - Set("updated_at = ?", now). - Where("id = ? AND items_copied < items_total", item.ReplacementID). - Exec(ctx); err != nil { - return fmt.Errorf("recording replacement progress: %w", err) - } - if err := resumeReadableSourceMigration(ctx, db, item.ReplacementID, now); err != nil { - return err - } - return nil - }) -} - -// WaitReplacementItemClaim parks an item without consuming its retry budget. -func (r *BunStorageReplacementRepo) WaitReplacementItemClaim( - ctx context.Context, - token storagereplacement.ClaimToken, - nextCheck time.Time, - lastError string, -) error { - return r.transitionReplacementItemClaim(ctx, token, storagereplacement.ItemStatusWaitingSource, nextCheck, lastError) -} - -// DeferReplacementItemClaim yields to an earlier upload without consuming a retry. -func (r *BunStorageReplacementRepo) DeferReplacementItemClaim( - ctx context.Context, - token storagereplacement.ClaimToken, - nextAttempt time.Time, -) error { - return r.transitionReplacementItemClaim(ctx, token, storagereplacement.ItemStatusPending, nextAttempt, "") -} - -// RetryReplacementItemClaim persists one transient failure. The item fails -// only after its fixed retry budget is exhausted. -func (r *BunStorageReplacementRepo) RetryReplacementItemClaim( - ctx context.Context, - token storagereplacement.ClaimToken, - nextAttempt time.Time, - lastError string, -) (storagereplacement.ItemStatus, error) { - status := storagereplacement.ItemStatusRetrying - err := runMaybeTx(ctx, r.db, func(db bun.IDB) error { - item := new(storagereplacement.Item) - if err := db.NewSelect().Model(item). - Where("id = ? AND status = ? AND claimed_at = ?", token.ItemID, storagereplacement.ItemStatusRunning, token.ClaimedAt). - Scan(ctx); err != nil { - if err == sql.ErrNoRows { - return ErrItemClaimLost - } - return fmt.Errorf("loading replacement item retry budget: %w", err) - } - if item.LeaseUntil == nil || !item.LeaseUntil.After(time.Now()) { - return ErrItemClaimLost - } - if item.MaxRetries == nil { - return fmt.Errorf("replacement item retry budget is not initialized: %w", ErrConflict) - } - if item.RetryCount >= *item.MaxRetries { - status = storagereplacement.ItemStatusFailed - } - now := time.Now() - res, err := db.NewUpdate().Model((*storagereplacement.Item)(nil)). - Set("status = ?", status). - Set("retry_count = retry_count + 1"). - Set("scheduled_at = ?", nextAttempt). - Set("last_error = ?", nullableString(lastError)). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("updated_at = ?", now). - Where("id = ? AND status = ? AND claimed_at = ?", token.ItemID, storagereplacement.ItemStatusRunning, token.ClaimedAt). - Where("lease_until > ?", now). - Exec(ctx) - if err != nil { - return fmt.Errorf("scheduling replacement item retry: %w", err) - } - if rows, _ := res.RowsAffected(); rows != 1 { - return ErrItemClaimLost - } - if status == storagereplacement.ItemStatusFailed { - return clearUnattemptedReplacementReservation(ctx, db, item.TargetCopyID, now) - } - return nil - }) - return status, err -} - -func (r *BunStorageReplacementRepo) transitionReplacementItemClaim( - ctx context.Context, - token storagereplacement.ClaimToken, - status storagereplacement.ItemStatus, - scheduledAt time.Time, - lastError string, -) error { - return runMaybeTx(ctx, r.db, func(db bun.IDB) error { - now := time.Now() - query := db.NewUpdate().Model((*storagereplacement.Item)(nil)). - Set("status = ?", status). - Set("scheduled_at = ?", scheduledAt). - Set("last_error = ?", nullableString(lastError)). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("updated_at = ?", now). - Where("id = ? AND status = ?", token.ItemID, storagereplacement.ItemStatusRunning). - Where("claimed_at = ?", token.ClaimedAt). - Where("lease_until > ?", now) - res, err := query.Exec(ctx) - if err != nil { - return fmt.Errorf("transitioning replacement item claim: %w", err) - } - if rows, _ := res.RowsAffected(); rows != 1 { - return ErrItemClaimLost - } - return nil - }) -} - -// PauseMigration rejects requests from an older replacement state version. -func (r *BunStorageReplacementRepo) PauseMigration( - ctx context.Context, - replacementID, expectedStateVersion int64, - reason storagereplacement.WaitReason, -) error { - if !reason.Valid() { - return fmt.Errorf("pausing replacement migration: %w", ErrInvalidInput) - } - now := time.Now() - res, err := r.db.NewUpdate().Model((*storagereplacement.Replacement)(nil)). - Set("status = ?", storagereplacement.StatusWaiting). - Set("wait_reason = ?", reason). - Set("updated_at = ?", now). - Where("id = ? AND state_version = ?", replacementID, expectedStateVersion). - Where("status = ? OR (status = ? AND wait_reason = ?)", - storagereplacement.StatusMigrating, - storagereplacement.StatusWaiting, - storagereplacement.WaitReasonReadableSource). - Exec(ctx) - if err != nil { - return fmt.Errorf("pausing replacement migration: %w", err) - } - if rows, _ := res.RowsAffected(); rows != 1 { - return ErrConflict - } - return nil -} - -func resumeReadableSourceMigration(ctx context.Context, db bun.IDB, replacementID int64, now time.Time) error { - _, err := db.NewUpdate().Model((*storagereplacement.Replacement)(nil)). - Set("status = ?", storagereplacement.StatusMigrating). - Set("wait_reason = NULL"). - Set("state_version = state_version + 1"). - Set("updated_at = ?", now). - Where("id = ? AND status = ? AND wait_reason = ?", - replacementID, - storagereplacement.StatusWaiting, - storagereplacement.WaitReasonReadableSource, - ). - Exec(ctx) - if err != nil { - return fmt.Errorf("resuming readable-source replacement migration: %w", err) - } - return nil -} - -// ReplacementExecution returns the bounded state the coordinator needs. The -// replacement-state index makes each item-state probe stop at its first match. -func (r *BunStorageReplacementRepo) ReplacementExecution( - ctx context.Context, - replacementID int64, -) (storagereplacement.ExecutionSnapshot, error) { - if replacementID <= 0 { - return storagereplacement.ExecutionSnapshot{}, fmt.Errorf("loading replacement execution: %w", ErrInvalidInput) - } - var snapshot storagereplacement.ExecutionSnapshot - err := r.db.NewRaw(`SELECT replacement.id AS replacement_id, - replacement.seeding_complete, - replacement.items_total, - replacement.items_copied, - EXISTS (SELECT 1 FROM storage_replacement_items AS item - WHERE item.replacement_id = replacement.id AND item.status = 'pending') AS has_pending, - EXISTS (SELECT 1 FROM storage_replacement_items AS item - WHERE item.replacement_id = replacement.id AND item.status = 'running') AS has_active, - EXISTS (SELECT 1 FROM storage_replacement_items AS item - WHERE item.replacement_id = replacement.id AND item.status = 'retrying') AS has_retrying, - EXISTS (SELECT 1 FROM storage_replacement_items AS item - WHERE item.replacement_id = replacement.id AND item.status = 'waiting_source') AS has_waiting_source, - EXISTS (SELECT 1 FROM storage_replacement_items AS item - WHERE item.replacement_id = replacement.id AND item.status = 'failed') AS has_failed - FROM storage_replacements AS replacement - WHERE replacement.id = ?`, replacementID).Scan(ctx, &snapshot) - if err != nil { - if err == sql.ErrNoRows { - return storagereplacement.ExecutionSnapshot{}, fmt.Errorf("provider replacement %d: %w", replacementID, ErrNotFound) - } - return storagereplacement.ExecutionSnapshot{}, fmt.Errorf("loading replacement execution: %w", err) - } - return snapshot, nil -} - -// ReplacementProgresses returns a batch of UI-neutral progress snapshots. -func (r *BunStorageReplacementRepo) ReplacementProgresses( - ctx context.Context, - replacementIDs []int64, -) (map[int64]storagereplacement.ProgressSnapshot, error) { - out := make(map[int64]storagereplacement.ProgressSnapshot, len(replacementIDs)) - if len(replacementIDs) == 0 { - return out, nil - } - type progressRow struct { - ReplacementID int64 `bun:"replacement_id"` - Status storagereplacement.Status `bun:"replacement_status"` - TargetIsCurrent bool `bun:"target_is_current"` - SeedingComplete bool `bun:"seeding_complete"` - ItemsTotal int `bun:"items_total"` - ItemsCopied int `bun:"items_copied"` - ItemsPending int `bun:"items_pending"` - ItemsActive int `bun:"items_active"` - ItemsAttention int `bun:"items_attention"` - ItemsRetrying int `bun:"items_retrying"` - ItemsWaitingSource int `bun:"items_waiting_source"` - ItemsFailed int `bun:"items_failed"` - NextRetryAt *time.Time `bun:"next_retry_at"` - } - var rows []progressRow - now := time.Now() - query := `SELECT replacement.id AS replacement_id, - replacement.status AS replacement_status, - target.is_current AS target_is_current, - replacement.seeding_complete, - replacement.items_total, - replacement.items_copied, - COALESCE(SUM(CASE WHEN target_copy.commit_attention_at IS NULL AND item.status = 'pending' - AND target_copy.commit_attempt_id IS NULL THEN 1 ELSE 0 END), 0) AS items_pending, - COALESCE(SUM(CASE WHEN target_copy.commit_attention_at IS NULL - AND (item.status = 'running' OR target_copy.commit_attempt_id IS NOT NULL) THEN 1 ELSE 0 END), 0) AS items_active, - COALESCE(SUM(CASE WHEN target_copy.commit_attention_at IS NOT NULL THEN 1 ELSE 0 END), 0) AS items_attention, - COALESCE(SUM(CASE WHEN target_copy.commit_attention_at IS NULL AND item.status = 'retrying' - AND target_copy.commit_attempt_id IS NULL THEN 1 ELSE 0 END), 0) AS items_retrying, - COALESCE(SUM(CASE WHEN target_copy.commit_attention_at IS NULL AND item.status = 'waiting_source' - AND target_copy.commit_attempt_id IS NULL THEN 1 ELSE 0 END), 0) AS items_waiting_source, - COALESCE(SUM(CASE WHEN target_copy.commit_attention_at IS NULL AND item.status = 'failed' - AND target_copy.commit_attempt_id IS NULL THEN 1 ELSE 0 END), 0) AS items_failed, - MIN(CASE WHEN target_copy.commit_attention_at IS NULL AND target_copy.commit_attempt_id IS NULL - AND item.status = 'retrying' AND item.scheduled_at > ? THEN item.scheduled_at END) AS next_retry_at - FROM storage_replacements AS replacement - JOIN storage_data_sets AS target ON target.id = replacement.target_data_set_id - LEFT JOIN storage_replacement_items AS item ON item.replacement_id = replacement.id - LEFT JOIN storage_upload_copies AS target_copy ON target_copy.id = item.target_copy_id - WHERE replacement.id IN (?) - GROUP BY replacement.id, replacement.status, target.is_current, replacement.seeding_complete, - replacement.items_total, replacement.items_copied` - if err := r.db.NewRaw(query, now, bun.List(replacementIDs)).Scan(ctx, &rows); err != nil { - return nil, fmt.Errorf("loading replacement progress: %w", err) - } - for i := range rows { - row := &rows[i] - outstanding := row.ItemsPending + row.ItemsActive + row.ItemsAttention + row.ItemsRetrying + row.ItemsWaitingSource + row.ItemsFailed - noLongerNeeded := max(0, row.ItemsTotal-row.ItemsCopied-outstanding) - processed := row.ItemsCopied + noLongerNeeded - var percent *int - if row.SeedingComplete { - value := 100 - if row.ItemsTotal > 0 { - value = processed * 100 / row.ItemsTotal - if value > 100 { - value = 100 - } - } - percent = &value - } - out[row.ReplacementID] = storagereplacement.ProgressSnapshot{ - ReplacementID: row.ReplacementID, - Phase: progressPhase(row.Status, row.TargetIsCurrent), - SeedingComplete: row.SeedingComplete, - ItemsTotal: row.ItemsTotal, - ItemsProcessed: processed, - ItemsCopied: row.ItemsCopied, - ItemsNoLongerNeeded: noLongerNeeded, - ItemsPending: row.ItemsPending, - ItemsActive: row.ItemsActive, - ItemsAttention: row.ItemsAttention, - ItemsRetrying: row.ItemsRetrying, - ItemsWaitingSource: row.ItemsWaitingSource, - ItemsFailed: row.ItemsFailed, - Percent: percent, - NextRetryAt: row.NextRetryAt, - } - } - return out, nil -} - -func progressPhase(status storagereplacement.Status, targetIsCurrent bool) storagereplacement.Phase { - switch status { - case storagereplacement.StatusPreparingTarget: - return storagereplacement.PhasePrepare - case storagereplacement.StatusRetiring, storagereplacement.StatusCleanupAttention, storagereplacement.StatusCompleted: - return storagereplacement.PhaseRetire - case storagereplacement.StatusMigrating: - return storagereplacement.PhaseMigrate - case storagereplacement.StatusWaiting, storagereplacement.StatusFailed: - if targetIsCurrent { - return storagereplacement.PhaseMigrate - } - return storagereplacement.PhasePrepare - default: - return storagereplacement.PhaseNone - } -} - -// RunningReplacementItemClaimForUpload exposes only the fencing identity needed -// for ordinary-upload mutual exclusion, including before a target copy is attached. -func (r *BunStorageReplacementRepo) RunningReplacementItemClaimForUpload( - ctx context.Context, - replacementID, uploadID int64, -) (*storagereplacement.ClaimToken, error) { - item := new(storagereplacement.Item) - err := r.db.NewSelect().Model(item). - Column("id", "claimed_at"). - Where("replacement_id = ? AND upload_id = ?", replacementID, uploadID). - Where("status = ?", storagereplacement.ItemStatusRunning). - Where("claimed_at IS NOT NULL AND lease_until > ?", time.Now()). - Limit(1). - Scan(ctx) - if err == sql.ErrNoRows { - return nil, nil - } - if err != nil { - return nil, fmt.Errorf("loading replacement item upload claim: %w", err) - } - if item.ClaimedAt == nil { - return nil, nil - } - return &storagereplacement.ClaimToken{ItemID: item.ID, ClaimedAt: *item.ClaimedAt}, nil -} diff --git a/internal/db/repository/storage_replacement_queue_internal_test.go b/internal/db/repository/storage_replacement_queue_internal_test.go deleted file mode 100644 index 636ac64..0000000 --- a/internal/db/repository/storage_replacement_queue_internal_test.go +++ /dev/null @@ -1,161 +0,0 @@ -package repository - -import ( - "context" - "database/sql" - "testing" - "time" - - synaps3db "github.com/strahe/synaps3/internal/db" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/strahe/synaps3/internal/types" - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect/sqlitedialect" - _ "modernc.org/sqlite" -) - -func TestSettleClaimedTerminalReplacementItemDistinguishesReadableCommittedCopy(t *testing.T) { - for _, tc := range []struct { - name string - targetStatus model.StorageDataSetStatus - wantStatus storagereplacement.ItemStatus - wantItemsCopied int - }{ - { - name: "readable", - targetStatus: model.StorageDataSetStatusReady, - wantStatus: storagereplacement.ItemStatusCopied, - wantItemsCopied: 1, - }, - { - name: "unavailable", - targetStatus: model.StorageDataSetStatusUnavailable, - wantStatus: storagereplacement.ItemStatusFailed, - wantItemsCopied: 0, - }, - } { - t.Run(tc.name, func(t *testing.T) { - db := terminalReplacementCommitTestDB(t) - ctx := t.Context() - now := time.Now() - bucket := &model.Bucket{Name: "terminal-commit-" + tc.name, Status: model.BucketStatusActive} - if _, err := db.NewInsert().Model(bucket).Exec(ctx); err != nil { - t.Fatalf("insert bucket: %v", err) - } - sourceDataSetID := types.NewOnChainID(1001) - targetDataSetID := types.NewOnChainID(2002) - source := &model.StorageDataSet{ - BucketID: bucket.ID, ProviderID: types.NewOnChainID(101), CopyIndex: 0, - Generation: 1, IsCurrent: false, DataSetID: &sourceDataSetID, - Status: model.StorageDataSetStatusDraining, - } - target := &model.StorageDataSet{ - BucketID: bucket.ID, ProviderID: types.NewOnChainID(202), CopyIndex: 0, - Generation: 2, IsCurrent: true, DataSetID: &targetDataSetID, - Status: tc.targetStatus, - } - if _, err := db.NewInsert().Model(source).Exec(ctx); err != nil { - t.Fatalf("insert source data set: %v", err) - } - if _, err := db.NewInsert().Model(target).Exec(ctx); err != nil { - t.Fatalf("insert target data set: %v", err) - } - pieceCID := "bafkqaaa" - upload := &model.StorageUpload{ - BucketID: bucket.ID, ContentSize: 1, Checksum: tc.name, - Status: model.StorageUploadStatusComplete, PieceCID: &pieceCID, RequestedCopies: 1, - } - if _, err := db.NewInsert().Model(upload).Exec(ctx); err != nil { - t.Fatalf("insert upload: %v", err) - } - providerID := target.ProviderID - pieceID := types.NewOnChainID(3003) - retrievalURL := "https://provider.example/piece" - copyRow := &model.StorageUploadCopy{ - UploadID: upload.ID, CopyIndex: target.CopyIndex, ProviderID: &providerID, - PieceID: &pieceID, TransferMethod: model.StorageCopyTransferMethodPeerPull, - Status: model.StorageUploadCopyStatusCommitted, RetrievalURL: &retrievalURL, - StorageDataSetID: &target.ID, - } - if _, err := db.NewInsert().Model(copyRow).Exec(ctx); err != nil { - t.Fatalf("insert target copy: %v", err) - } - replacement := &storagereplacement.Replacement{ - BucketID: bucket.ID, CopyIndex: source.CopyIndex, - SourceDataSetID: source.ID, TargetDataSetID: target.ID, - SelectionMode: storagereplacement.SelectionModeManual, - ClientRequestID: "terminal-commit-" + tc.name, - Status: storagereplacement.StatusFailed, ItemsTotal: 1, - ConfirmedAt: now, CreatedAt: now, UpdatedAt: now, - } - if _, err := db.NewInsert().Model(replacement).Exec(ctx); err != nil { - t.Fatalf("insert replacement: %v", err) - } - claimedAt := now.Add(-time.Second) - leaseUntil := now.Add(time.Minute) - maxRetries := 5 - item := &storagereplacement.Item{ - ReplacementID: replacement.ID, UploadID: upload.ID, TargetCopyID: ©Row.ID, - Status: storagereplacement.ItemStatusRunning, ScheduledAt: claimedAt, - MaxRetries: &maxRetries, ClaimedAt: &claimedAt, LeaseUntil: &leaseUntil, - CreatedAt: claimedAt, UpdatedAt: claimedAt, - } - if _, err := db.NewInsert().Model(item).Exec(ctx); err != nil { - t.Fatalf("insert replacement item: %v", err) - } - - var settled bool - err := db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { - var err error - settled, err = settleClaimedTerminalReplacementItem( - ctx, - tx, - item, - storagereplacement.StatusFailed, - now, - ) - return err - }) - if err != nil || !settled { - t.Fatalf("settle terminal item = %v err=%v, want settled", settled, err) - } - persistedItem := new(storagereplacement.Item) - if err := db.NewSelect().Model(persistedItem).Where("id = ?", item.ID).Scan(ctx); err != nil { - t.Fatalf("load replacement item: %v", err) - } - if persistedItem.Status != tc.wantStatus || persistedItem.ClaimedAt != nil || persistedItem.LeaseUntil != nil { - t.Fatalf("replacement item = %#v, want %s without a claim", persistedItem, tc.wantStatus) - } - persistedReplacement := new(storagereplacement.Replacement) - if err := db.NewSelect().Model(persistedReplacement).Where("id = ?", replacement.ID).Scan(ctx); err != nil { - t.Fatalf("load replacement: %v", err) - } - if persistedReplacement.ItemsCopied != tc.wantItemsCopied { - t.Fatalf("items copied = %d, want %d", persistedReplacement.ItemsCopied, tc.wantItemsCopied) - } - persistedCopy := new(model.StorageUploadCopy) - if err := db.NewSelect().Model(persistedCopy).Where("id = ?", copyRow.ID).Scan(ctx); err != nil { - t.Fatalf("load target copy: %v", err) - } - if persistedCopy.Status != model.StorageUploadCopyStatusCommitted { - t.Fatalf("target copy status = %s, want committed", persistedCopy.Status) - } - }) - } -} - -func terminalReplacementCommitTestDB(t *testing.T) *bun.DB { - t.Helper() - sqldb, err := sql.Open("sqlite", "file::memory:?cache=shared&_pragma=foreign_keys(1)") - if err != nil { - t.Fatalf("open SQLite: %v", err) - } - sqldb.SetMaxOpenConns(1) - db := bun.NewDB(sqldb, sqlitedialect.New()) - t.Cleanup(func() { _ = db.Close() }) - if err := synaps3db.RunMigrations(t.Context(), db); err != nil { - t.Fatalf("run migrations: %v", err) - } - return db -} diff --git a/internal/db/repository/storage_replacement_repo.go b/internal/db/repository/storage_replacement_repo.go index f2ebbe5..b0bcd05 100644 --- a/internal/db/repository/storage_replacement_repo.go +++ b/internal/db/repository/storage_replacement_repo.go @@ -3,6 +3,7 @@ package repository import ( "context" "database/sql" + "errors" "fmt" "strings" "time" @@ -19,9 +20,9 @@ type BunStorageReplacementRepo struct { db bun.IDB } -// Authorize is the only way a replacement comes into existence. It runs as one -// transaction so the superseded predecessor, the new target generation, the -// replacement record, and its coordinator either all exist or none do. +// Authorize is the only way a replacement comes into existence. It records +// domain authorization and topology state only; TaskService binds the +// coordinator in the caller's transaction. func (r *BunStorageReplacementRepo) Authorize(ctx context.Context, input AuthorizeReplacementInput) (*storagereplacement.Replacement, bool, error) { input.ClientRequestID = strings.TrimSpace(input.ClientRequestID) if input.BucketID <= 0 || input.SourceDataSetID <= 0 || input.TargetProviderID.IsZero() || @@ -52,7 +53,7 @@ func (r *BunStorageReplacementRepo) Authorize(ctx context.Context, input Authori result = existing return nil } - source, err := (&BunStorageUploadRepo{db: db}).GetDataSetBindingByID(ctx, input.SourceDataSetID) + source, err := (&BunStorageContentRepo{db: db}).GetDataSetBindingByID(ctx, input.SourceDataSetID) if err != nil { return err } @@ -138,7 +139,7 @@ func (r *BunStorageReplacementRepo) Authorize(ctx context.Context, input Authori SelectionMode: input.SelectionMode, ClientRequestID: input.ClientRequestID, Status: storagereplacement.StatusPreparingTarget, - ConfirmedAt: now, + TaskGeneration: 1, CreatedAt: now, UpdatedAt: now, } @@ -155,22 +156,6 @@ func (r *BunStorageReplacementRepo) Authorize(ctx context.Context, input Authori if err := linkSupersededReplacements(ctx, db, superseded, replacement.ID, now); err != nil { return err } - tasks := &BunTaskRepo{db: db} - for _, id := range superseded { - // Leftover targets keep costing money until their own coordinator - // ends them. Queue that work in this transaction so cleanup does - // not wait for the next process start. - if _, err := tasks.EnsureRecurring(ctx, storagereplacement.NewAbandonedTargetTask( - id, input.BucketID, input.MaxRetries, now, - )); err != nil { - return fmt.Errorf("queueing abandoned replacement cleanup: %w", err) - } - } - if _, err := tasks.EnsureRecurring(ctx, storagereplacement.NewMigrateTask( - replacement.ID, input.BucketID, "", input.MaxRetries, now, - )); err != nil { - return fmt.Errorf("queueing replacement migration coordinator: %w", err) - } result = replacement created = true return nil @@ -198,9 +183,8 @@ func getReplacementByClientRequestID( clientRequestID string, ) (*storagereplacement.Replacement, error) { row := new(storagereplacement.Replacement) - err := db.NewSelect(). - Model(row). - Where("bucket_id = ? AND client_request_id = ?", bucketID, clientRequestID). + err := withReplacementTerminations(db.NewSelect().Model(row)). + Where("storage_replacement.bucket_id = ? AND storage_replacement.client_request_id = ?", bucketID, clientRequestID). Scan(ctx) if err == sql.ErrNoRows { return nil, nil @@ -259,7 +243,7 @@ func linkSupersededReplacements(ctx context.Context, db bun.IDB, ids []int64, su // data rather than remembered, so a retry always restarts at the stage the // replacement actually reached. func (r *BunStorageReplacementRepo) Retry(ctx context.Context, input RetryReplacementInput) (*storagereplacement.Replacement, error) { - if input.ReplacementID <= 0 || input.MaxRetries < 0 || input.ItemMaxRetries < 0 { + if input.ReplacementID <= 0 { return nil, fmt.Errorf("retrying provider replacement: %w", ErrInvalidInput) } var resumed *storagereplacement.Replacement @@ -271,27 +255,16 @@ func (r *BunStorageReplacementRepo) Retry(ctx context.Context, input RetryReplac if row.Status == storagereplacement.StatusSuperseded { return fmt.Errorf("retrying provider replacement: %w", storagereplacement.ErrSuperseded) } + if row.FailureReason != nil && !row.FailureReason.Valid() { + return fmt.Errorf("retrying provider replacement with unknown failure reason %q: %w", *row.FailureReason, storagereplacement.ErrNotRetryable) + } if row.FailureReason != nil && *row.FailureReason == storagereplacement.FailureReasonTargetInUse { return fmt.Errorf("retrying provider replacement: %w", storagereplacement.ErrTargetInUse) } if !row.Status.Retryable() { return fmt.Errorf("retrying provider replacement: %w", storagereplacement.ErrNotRetryable) } - running, err := db.NewSelect(). - Model((*model.Task)(nil)). - Where("idempotency_key IN (?, ?)", - storagereplacement.MigrateTaskKey(row.ID), - storagereplacement.RetireTaskKey(row.ID)). - Where("status = ?", model.TaskStatusRunning). - Count(ctx) - if err != nil { - return fmt.Errorf("checking replacement coordinator tasks: %w", err) - } - if running > 0 { - return fmt.Errorf("retrying provider replacement: %w", storagereplacement.ErrTaskRunning) - } - - target, err := (&BunStorageUploadRepo{db: db}).GetDataSetBindingByID(ctx, row.TargetDataSetID) + target, err := (&BunStorageContentRepo{db: db}).GetDataSetBindingByID(ctx, row.TargetDataSetID) if err != nil { return err } @@ -306,38 +279,34 @@ func (r *BunStorageReplacementRepo) Retry(ctx context.Context, input RetryReplac next = storagereplacement.StatusMigrating } now := time.Now() + if _, err := db.NewUpdate().Model((*model.StorageDataSet)(nil)). + Set("retirement_task_id = NULL"). + Set("updated_at = ?", now). + Where("id = ?", row.SourceDataSetID). + Where("retirement_task_id IN (SELECT id FROM tasks WHERE status = ?)", model.TaskStatusFailed). + Exec(ctx); err != nil { + return fmt.Errorf("releasing failed retirement task: %w", err) + } if _, err := db.NewUpdate().Model((*storagereplacement.Item)(nil)). Set("status = ?", storagereplacement.ItemStatusPending). - Set("retry_count = 0"). - Set("max_retries = ?", input.ItemMaxRetries). - Set("scheduled_at = ?", now). Set("last_error = NULL"). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). Set("updated_at = ?", now). - Where("replacement_id = ? AND status = ?", row.ID, storagereplacement.ItemStatusFailed). + Where("replacement_id = ? AND status = ?", row.ID, storagereplacement.ItemStatusAttention). Exec(ctx); err != nil { - return fmt.Errorf("resetting failed replacement items: %w", err) + return fmt.Errorf("resetting replacement items needing attention: %w", err) } if err := transitionReplacement(ctx, db, row.ID, []storagereplacement.Status{row.Status}, next, func(q *bun.UpdateQuery) *bun.UpdateQuery { - return q.Set("last_error = NULL").Set("wait_reason = NULL").Set("failure_reason = NULL") + return q.Set("last_error = NULL").Set("wait_reason = NULL").Set("failure_reason = NULL"). + Set("task_generation = task_generation + 1").Set("task_id = NULL") }, now); err != nil { return err } - task := storagereplacement.NewMigrateTask(row.ID, row.BucketID, "", input.MaxRetries, now) - if next == storagereplacement.StatusRetiring { - task = storagereplacement.NewRetireTask(row.ID, row.BucketID, input.MaxRetries, now) - } - // The worker marks the coordinator exhausted or failed on its way into a - // retryable state, and automatic recurrence deliberately leaves those - // alone. Resuming is the operator's explicit request to undo that. - if _, err := (&BunTaskRepo{db: db}).ResumeCoordinator(ctx, task); err != nil { - return fmt.Errorf("requeueing replacement coordinator: %w", err) - } row.Status = next row.LastError = nil row.WaitReason = nil row.FailureReason = nil + row.TaskGeneration++ + row.TaskID = nil row.UpdatedAt = now resumed = row return nil @@ -348,6 +317,43 @@ func (r *BunStorageReplacementRepo) Retry(ctx context.Context, input RetryReplac return resumed, nil } +func (r *BunStorageReplacementRepo) BindTask(ctx context.Context, replacementID, generation, taskID int64) error { + if replacementID < 1 || generation < 1 || taskID < 1 { + return ErrInvalidInput + } + result, err := r.db.NewUpdate(). + Model((*storagereplacement.Replacement)(nil)). + Set("task_id = ?", taskID). + Set("updated_at = ?", time.Now()). + Where("id = ? AND task_generation = ? AND task_id IS NULL", replacementID, generation). + Exec(ctx) + return requireTaskFenceRows(result, err, "binding provider replacement task") +} + +func (r *BunStorageReplacementRepo) AuthorizeTask(ctx context.Context, replacementID, generation, taskID int64) (*storagereplacement.Replacement, error) { + row := new(storagereplacement.Replacement) + err := withReplacementTerminations(r.db.NewSelect().Model(row)). + Where("storage_replacement.id = ? AND storage_replacement.task_generation = ? AND storage_replacement.task_id = ?", replacementID, generation, taskID). + Scan(ctx) + if err == sql.ErrNoRows { + return nil, ErrConflict + } + if err != nil { + return nil, fmt.Errorf("authorizing provider replacement task: %w", err) + } + return row, nil +} + +func (r *BunStorageReplacementRepo) CompleteTask(ctx context.Context, replacementID, generation, taskID int64) error { + result, err := r.db.NewUpdate(). + Model((*storagereplacement.Replacement)(nil)). + Set("task_id = NULL"). + Set("updated_at = ?", time.Now()). + Where("id = ? AND task_generation = ? AND task_id = ?", replacementID, generation, taskID). + Exec(ctx) + return requireTaskFenceRows(result, err, "completing provider replacement task") +} + // Activate is the single atomic switch: the target starts receiving writes and // the source starts draining. It touches three rows whatever the bucket holds. func (r *BunStorageReplacementRepo) Activate(ctx context.Context, replacementID int64) error { @@ -459,62 +465,6 @@ func (r *BunStorageReplacementRepo) MarkFailed( }) } -func (r *BunStorageReplacementRepo) FailCoordinator( - ctx context.Context, - input ReplacementCoordinatorFailureInput, -) error { - if input.ReplacementID <= 0 || input.Task == nil || - (input.FailureReason != nil && !input.FailureReason.Valid()) { - return fmt.Errorf("failing provider replacement coordinator: %w", ErrInvalidInput) - } - return runMaybeTx(ctx, r.db, func(db bun.IDB) error { - replacements := &BunStorageReplacementRepo{db: db} - if err := replacements.MarkFailed(ctx, input.ReplacementID, input.FailureReason, input.LastError); err != nil { - return err - } - if err := (&BunTaskRepo{db: db}).FailRunning(ctx, input.Task, input.LastError); err != nil { - return fmt.Errorf("stopping provider replacement coordinator: %w", err) - } - return nil - }) -} - -func (r *BunStorageReplacementRepo) ScheduleCoordinatorRetry( - ctx context.Context, - input ReplacementCoordinatorRetryInput, -) (model.TaskStatus, error) { - if input.ReplacementID <= 0 || input.Task == nil { - return "", fmt.Errorf("scheduling provider replacement coordinator retry: %w", ErrInvalidInput) - } - var taskStatus model.TaskStatus - err := runMaybeTx(ctx, r.db, func(db bun.IDB) error { - replacement, err := lockReplacementByID(ctx, db, input.ReplacementID) - if err != nil { - return err - } - taskStatus, err = (&BunTaskRepo{db: db}).ScheduleRetryRunning(ctx, input.Task, input.LastError, input.Backoff) - if err != nil || taskStatus != model.TaskStatusExhausted { - return err - } - - next, changed := storagereplacement.OnTaskExhausted(replacement.Status) - if !changed { - return nil - } - replacementError := input.LastError + " (max retries reached)" - return transitionReplacement(ctx, db, replacement.ID, []storagereplacement.Status{replacement.Status}, next, - func(q *bun.UpdateQuery) *bun.UpdateQuery { - return q.Set("last_error = ?", replacementError). - Set("wait_reason = NULL"). - Set("failure_reason = NULL") - }, time.Now()) - }) - if err != nil { - return "", err - } - return taskStatus, nil -} - // MarkCleanupAttention is committed in the same transaction that stops the // coordinator task, so automatic retry can never resume suppressed cleanup. func (r *BunStorageReplacementRepo) MarkCleanupAttention(ctx context.Context, replacementID int64, lastError string) error { @@ -544,25 +494,29 @@ func (r *BunStorageReplacementRepo) RecordTerminationEpoch(ctx context.Context, if input.ReplacementID <= 0 { return fmt.Errorf("recording replacement termination epoch: %w", ErrInvalidInput) } - res, err := r.db.NewUpdate(). - Model((*storagereplacement.Replacement)(nil)). - Set("termination_epoch = ?", input.Epoch). - Set("termination_tx_hash = ?", nullableString(input.TxHash)). - Set("updated_at = ?", time.Now()). - Where("id = ?", input.ReplacementID). - Where("status IN (?, ?, ?)", + return runMaybeTx(ctx, r.db, func(db bun.IDB) error { + row, err := selectReplacementForTermination(ctx, db, input.ReplacementID, storagereplacement.StatusRetiring, storagereplacement.StatusWaiting, - storagereplacement.StatusCleanupAttention). - Where("termination_epoch IS NULL"). - Exec(ctx) - if err != nil { - return fmt.Errorf("recording replacement termination epoch: %w", err) - } - if rows, _ := res.RowsAffected(); rows != 1 { - return fmt.Errorf("recording replacement termination epoch: %w", ErrConflict) - } - return nil + storagereplacement.StatusCleanupAttention) + if err != nil { + return fmt.Errorf("recording replacement termination epoch: %w", err) + } + termination := &storagereplacement.Termination{ + ReplacementID: row.ID, + Role: storagereplacement.TerminationRoleSource, + SourceDataSetID: &row.SourceDataSetID, + TxHash: nullableString(input.TxHash), + Epoch: input.Epoch, + } + if _, err := db.NewInsert().Model(termination).Exec(ctx); err != nil { + if isUniqueViolation(err) { + return fmt.Errorf("recording replacement termination epoch: %w", ErrConflict) + } + return fmt.Errorf("recording replacement termination epoch: %w", err) + } + return nil + }) } // RecordAbandonedTerminationEpoch persists termination of a superseded target @@ -575,35 +529,82 @@ func (r *BunStorageReplacementRepo) RecordAbandonedTerminationEpoch( if input.ReplacementID <= 0 || input.Epoch < 0 { return fmt.Errorf("recording abandoned target termination epoch: %w", ErrInvalidInput) } - res, err := r.db.NewUpdate(). - Model((*storagereplacement.Replacement)(nil)). - Set("abandoned_termination_epoch = ?", input.Epoch). - Set("abandoned_termination_tx_hash = ?", nullableString(input.TxHash)). - Set("updated_at = ?", time.Now()). - Where("id = ?", input.ReplacementID). - Where("status = ?", storagereplacement.StatusSuperseded). - Where("abandoned_termination_epoch IS NULL"). - Exec(ctx) - if err != nil { - return fmt.Errorf("recording abandoned target termination epoch: %w", err) - } - if rows, _ := res.RowsAffected(); rows == 1 { - return nil - } - row, getErr := r.GetByID(ctx, input.ReplacementID) - if getErr != nil { - return getErr + return runMaybeTx(ctx, r.db, func(db bun.IDB) error { + row, err := selectReplacementForTermination(ctx, db, input.ReplacementID, storagereplacement.StatusSuperseded) + if err != nil { + return fmt.Errorf("recording abandoned target termination epoch: %w", err) + } + termination := &storagereplacement.Termination{ + ReplacementID: row.ID, + Role: storagereplacement.TerminationRoleAbandonedTarget, + AbandonedTargetDataSetID: &row.TargetDataSetID, + TxHash: nullableString(input.TxHash), + Epoch: input.Epoch, + } + if _, err := db.NewInsert().Model(termination).Exec(ctx); err == nil { + return nil + } else if !isUniqueViolation(err) { + return fmt.Errorf("recording abandoned target termination epoch: %w", err) + } + // A crash after the transaction was paid for replays with the same + // epoch; anything else is a genuine conflict. + existing := new(storagereplacement.Termination) + if err := db.NewSelect().Model(existing). + Where("replacement_id = ? AND role = ?", row.ID, storagereplacement.TerminationRoleAbandonedTarget). + Scan(ctx); err != nil { + return fmt.Errorf("recording abandoned target termination epoch: %w", err) + } + if existing.Epoch == input.Epoch { + return nil + } + return fmt.Errorf("recording abandoned target termination epoch: %w", ErrConflict) + }) +} + +// selectReplacementForTermination reads the replacement a termination is about +// to be recorded against, refusing any status that has no term to end. +func selectReplacementForTermination( + ctx context.Context, + db bun.IDB, + replacementID int64, + statuses ...storagereplacement.Status, +) (*storagereplacement.Replacement, error) { + row := new(storagereplacement.Replacement) + err := db.NewSelect(). + Model(row). + Where("id = ?", replacementID). + Where("status IN (?)", bun.List(statuses)). + Scan(ctx) + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrConflict } - if row != nil && row.Status == storagereplacement.StatusSuperseded && - row.AbandonedTerminationEpoch != nil && *row.AbandonedTerminationEpoch == input.Epoch { - return nil + if err != nil { + return nil, err } - return fmt.Errorf("recording abandoned target termination epoch: %w", ErrConflict) + return row, nil +} + +// withReplacementTerminations projects the end of term recorded for each of a +// replacement's two data sets. The rows live in storage_data_set_terminations, +// so a replacement carries no repeated column group of its own. +func withReplacementTerminations(q *bun.SelectQuery) *bun.SelectQuery { + return q. + ColumnExpr("storage_replacement.*"). + ColumnExpr("source_termination.tx_hash AS termination_tx_hash"). + ColumnExpr("source_termination.epoch AS termination_epoch"). + ColumnExpr("abandoned_termination.tx_hash AS abandoned_termination_tx_hash"). + ColumnExpr("abandoned_termination.epoch AS abandoned_termination_epoch"). + Join("LEFT JOIN storage_data_set_terminations AS source_termination"+ + " ON source_termination.replacement_id = storage_replacement.id AND source_termination.role = ?", + storagereplacement.TerminationRoleSource). + Join("LEFT JOIN storage_data_set_terminations AS abandoned_termination"+ + " ON abandoned_termination.replacement_id = storage_replacement.id AND abandoned_termination.role = ?", + storagereplacement.TerminationRoleAbandonedTarget) } func (r *BunStorageReplacementRepo) GetByID(ctx context.Context, id int64) (*storagereplacement.Replacement, error) { row := new(storagereplacement.Replacement) - err := r.db.NewSelect().Model(row).Where("id = ?", id).Scan(ctx) + err := withReplacementTerminations(r.db.NewSelect().Model(row)).Where("storage_replacement.id = ?", id).Scan(ctx) if err != nil { if err == sql.ErrNoRows { return nil, nil @@ -629,10 +630,9 @@ func (r *BunStorageReplacementRepo) GetByClientRequestID( // can present a stable order. func (r *BunStorageReplacementRepo) ListForBucket(ctx context.Context, bucketID int64, limit int) ([]storagereplacement.Replacement, error) { var rows []storagereplacement.Replacement - q := r.db.NewSelect(). - Model(&rows). - Where("bucket_id = ?", bucketID). - OrderExpr("id DESC") + q := withReplacementTerminations(r.db.NewSelect().Model(&rows)). + Where("storage_replacement.bucket_id = ?", bucketID). + OrderExpr("storage_replacement.id DESC") if limit > 0 { q = q.Limit(limit) } @@ -644,13 +644,12 @@ func (r *BunStorageReplacementRepo) ListForBucket(ctx context.Context, bucketID func (r *BunStorageReplacementRepo) GetActiveForDataSet(ctx context.Context, dataSetID int64) (*storagereplacement.Replacement, error) { row := new(storagereplacement.Replacement) - err := r.db.NewSelect(). - Model(row). + err := withReplacementTerminations(r.db.NewSelect().Model(row)). // Either generation is owned by the replacement. After activation the // coordinator writes the target, so a caller asking about the target has // to see the replacement too. - Where("source_data_set_id = ? OR target_data_set_id = ?", dataSetID, dataSetID). - Where("status NOT IN (?, ?)", storagereplacement.StatusCompleted, storagereplacement.StatusSuperseded). + Where("storage_replacement.source_data_set_id = ? OR storage_replacement.target_data_set_id = ?", dataSetID, dataSetID). + Where("storage_replacement.status NOT IN (?, ?)", storagereplacement.StatusCompleted, storagereplacement.StatusSuperseded). Limit(1). Scan(ctx) if err != nil { @@ -665,9 +664,9 @@ func (r *BunStorageReplacementRepo) GetActiveForDataSet(ctx context.Context, dat // HasInProgressForDataSet covers both generations a replacement owns. The // target matters as much as the source: in-place recovery would otherwise queue // repair work on the generation the coordinator is actively writing, and the -// two would race over the same copy row. It excludes terminally failed and -// attention states so a generation whose replacement gave up can still recover -// and repair in place. +// two would race over the same copy row. It excludes operator-paused failed and +// attention states so that generation can still recover and repair in place; +// the replacement nevertheless retains its source, target, and slot identity. func (r *BunStorageReplacementRepo) HasInProgressForDataSet(ctx context.Context, dataSetID int64) (bool, error) { exists, err := r.db.NewSelect(). Model((*storagereplacement.Replacement)(nil)). @@ -686,15 +685,14 @@ func (r *BunStorageReplacementRepo) HasInProgressForDataSet(ctx context.Context, func (r *BunStorageReplacementRepo) ListActive(ctx context.Context, afterID int64, limit int) ([]storagereplacement.Replacement, error) { var rows []storagereplacement.Replacement - if err := r.db.NewSelect(). - Model(&rows). - Where("id > ?", afterID). - Where("status IN (?, ?, ?, ?)", + if err := withReplacementTerminations(r.db.NewSelect().Model(&rows)). + Where("storage_replacement.id > ?", afterID). + Where("storage_replacement.status IN (?, ?, ?, ?)", storagereplacement.StatusPreparingTarget, storagereplacement.StatusMigrating, storagereplacement.StatusWaiting, storagereplacement.StatusRetiring). - OrderExpr("id ASC"). + OrderExpr("storage_replacement.id ASC"). Limit(limit). Scan(ctx); err != nil { return nil, fmt.Errorf("listing active provider replacements: %w", err) @@ -707,17 +705,16 @@ func (r *BunStorageReplacementRepo) ListActive(ctx context.Context, afterID int6 // so the abandoned service does not keep costing money. func (r *BunStorageReplacementRepo) ListSupersededCleanupCandidates(ctx context.Context, afterID int64, limit int) ([]storagereplacement.Replacement, error) { var rows []storagereplacement.Replacement - if err := r.db.NewSelect(). - Model(&rows). - Where("id > ?", afterID). - Where("status = ?", storagereplacement.StatusSuperseded). + if err := withReplacementTerminations(r.db.NewSelect().Model(&rows)). + Where("storage_replacement.id > ?", afterID). + Where("storage_replacement.status = ?", storagereplacement.StatusSuperseded). Where(`EXISTS ( SELECT 1 FROM storage_data_sets AS abandoned_target WHERE abandoned_target.id = storage_replacement.target_data_set_id AND abandoned_target.is_current = ? AND abandoned_target.status <> ? )`, false, model.StorageDataSetStatusRetired). - OrderExpr("id ASC"). + OrderExpr("storage_replacement.id ASC"). Limit(limit). Scan(ctx); err != nil { return nil, fmt.Errorf("listing superseded replacement cleanup candidates: %w", err) @@ -763,7 +760,6 @@ func transitionReplacement( q := db.NewUpdate(). Model((*storagereplacement.Replacement)(nil)). Set("status = ?", to). - Set("state_version = state_version + 1"). Set("updated_at = ?", now). Where("id = ?", replacementID). Where("status IN (?)", bun.List(allowed)) @@ -802,5 +798,33 @@ func lockReplacementByID(ctx context.Context, db bun.IDB, replacementID int64) ( } return nil, fmt.Errorf("locking provider replacement: %w", err) } + // The lock statement returns the replacement's own columns; the terminations + // it records live beside it and callers of this lock read them. + if err := loadReplacementTerminations(ctx, db, row); err != nil { + return nil, err + } return row, nil } + +// loadReplacementTerminations fills in the termination projection for a +// replacement read without the join. +func loadReplacementTerminations(ctx context.Context, db bun.IDB, row *storagereplacement.Replacement) error { + var terminations []storagereplacement.Termination + if err := db.NewSelect(). + Model(&terminations). + Where("replacement_id = ?", row.ID). + Scan(ctx); err != nil { + return fmt.Errorf("loading provider replacement terminations: %w", err) + } + for i := range terminations { + termination := &terminations[i] + epoch := termination.Epoch + switch termination.Role { + case storagereplacement.TerminationRoleSource: + row.TerminationTxHash, row.TerminationEpoch = termination.TxHash, &epoch + case storagereplacement.TerminationRoleAbandonedTarget: + row.AbandonedTerminationTxHash, row.AbandonedTerminationEpoch = termination.TxHash, &epoch + } + } + return nil +} diff --git a/internal/db/repository/storage_replacement_repo_test.go b/internal/db/repository/storage_replacement_repo_test.go deleted file mode 100644 index bf3a4ba..0000000 --- a/internal/db/repository/storage_replacement_repo_test.go +++ /dev/null @@ -1,2298 +0,0 @@ -package repository_test - -import ( - "context" - "errors" - "fmt" - "slices" - "strings" - "testing" - "time" - - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagecommit" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/uptrace/bun" -) - -type replacementFixture struct { - db *bun.DB - repos *repository.Repositories - bucket *model.Bucket - upload *model.StorageUpload - version *model.ObjectVersion - source *model.StorageDataSet - request int -} - -func newReplacementFixture(t *testing.T, name, versionID string) *replacementFixture { - t.Helper() - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, name) - - version := newObjectVersion(bucket.ID, "file.txt", versionID, 10) - version.Checksum = name + "-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - upload := startCopyHealthUpload(t, repos, bucket.ID, version.VersionID, version.Size, version.Checksum, 1) - source := commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 0, "101", "1001", "2001", "https://source.example/piece") - bindStorageHealthVersion(t, repos, bucket.ID, upload.ID, version) - return &replacementFixture{db: db, repos: repos, bucket: bucket, upload: upload, version: version, source: source} -} - -func (f *replacementFixture) authorize(t *testing.T, provider string) *storagereplacement.Replacement { - t.Helper() - f.request++ - row, _, err := f.repos.Replacements.Authorize(context.Background(), repository.AuthorizeReplacementInput{ - BucketID: f.bucket.ID, - SourceDataSetID: f.source.ID, - SelectionMode: storagereplacement.SelectionModeManual, - TargetProviderID: onChainID(t, provider), - ClientRequestID: fmt.Sprintf("fixture-%d", f.request), - MaxRetries: 5, - }) - if err != nil { - t.Fatalf("Authorize: %v", err) - } - return row -} - -// readyTarget brings the approved target to the point where it can take over. -func (f *replacementFixture) readyTarget(t *testing.T, row *storagereplacement.Replacement, dataSetID string) *model.StorageDataSet { - t.Helper() - ctx := context.Background() - if err := f.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: row.TargetDataSetID, - UploadID: f.upload.ID, - DataSetID: onChainID(t, dataSetID), - }); err != nil { - t.Fatalf("MarkDataSetReady target: %v", err) - } - target, err := f.repos.Uploads.GetDataSetBindingByID(ctx, row.TargetDataSetID) - if err != nil || target == nil { - t.Fatalf("GetDataSetBindingByID target = %#v err=%v", target, err) - } - return target -} - -func TestStorageReplacementRepo_AuthorizeCreatesTargetGenerationWithoutMovingWrites(t *testing.T) { - f := newReplacementFixture(t, "replacement-authorize", "01J000000000000000000RPL01") - ctx := context.Background() - - row := f.authorize(t, "202") - if row.Status != storagereplacement.StatusPreparingTarget || row.CopyIndex != 0 { - t.Fatalf("replacement = %#v, want preparing_target on slot 0", row) - } - - // Writes must keep going to the source until the target is actually usable. - current, err := f.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, f.bucket.ID, 0) - if err != nil || current == nil || current.ID != f.source.ID { - t.Fatalf("current binding = %#v err=%v, want the source until activation", current, err) - } - target, err := f.repos.Uploads.GetDataSetBindingByID(ctx, row.TargetDataSetID) - if err != nil || target == nil { - t.Fatalf("target binding = %#v err=%v", target, err) - } - if target.IsCurrent || target.Generation != f.source.Generation+1 || target.Status != model.StorageDataSetStatusPending { - t.Fatalf("target = %#v, want a pending next generation that is not current", target) - } - - task, err := f.repos.Tasks.GetByIdempotencyKey(ctx, storagereplacement.MigrateTaskKey(row.ID)) - if err != nil || task == nil { - t.Fatalf("migration coordinator = %#v err=%v, want it queued with the confirmation", task, err) - } -} - -func TestStorageReplacementRepo_AuthorizeRejections(t *testing.T) { - f := newReplacementFixture(t, "replacement-reject", "01J000000000000000000RPL02") - ctx := context.Background() - base := repository.AuthorizeReplacementInput{ - BucketID: f.bucket.ID, - SourceDataSetID: f.source.ID, - SelectionMode: storagereplacement.SelectionModeManual, - TargetProviderID: onChainID(t, "202"), - MaxRetries: 5, - ClientRequestID: "reject-request", - } - - t.Run("target is the source", func(t *testing.T) { - input := base - input.TargetProviderID = f.source.ProviderID - if _, _, err := f.repos.Replacements.Authorize(ctx, input); !errors.Is(err, storagereplacement.ErrInvalidTarget) { - t.Fatalf("error = %v, want ErrInvalidTarget", err) - } - }) - - t.Run("target already serves the bucket", func(t *testing.T) { - other := commitStorageHealthCopy(t, f.repos, f.bucket.ID, f.upload.ID, 1, "303", "3003", "3003", "https://other.example/piece") - input := base - input.TargetProviderID = other.ProviderID - if _, _, err := f.repos.Replacements.Authorize(ctx, input); !errors.Is(err, storagereplacement.ErrTargetInUse) { - t.Fatalf("error = %v, want ErrTargetInUse", err) - } - }) - - // A provider whose earlier generation is draining still owns its data set for - // this bucket, so preparing a second one would collide on the provider/data - // set uniqueness inside the worker. - t.Run("target still holds a draining generation", func(t *testing.T) { - drainingProvider := commitStorageHealthCopy(t, f.repos, f.bucket.ID, f.upload.ID, 2, "404", "4004", "4004", "https://draining.example/piece") - mustExec(t, f.db, `UPDATE storage_data_sets SET is_current = FALSE, status = ? WHERE id = ?`, - model.StorageDataSetStatusDraining, drainingProvider.ID) - input := base - input.TargetProviderID = drainingProvider.ProviderID - if _, _, err := f.repos.Replacements.Authorize(ctx, input); !errors.Is(err, storagereplacement.ErrTargetInUse) { - t.Fatalf("error = %v, want ErrTargetInUse", err) - } - }) - - t.Run("unknown data set", func(t *testing.T) { - input := base - input.SourceDataSetID = 9999 - if _, _, err := f.repos.Replacements.Authorize(ctx, input); !errors.Is(err, repository.ErrNotFound) { - t.Fatalf("error = %v, want ErrNotFound", err) - } - }) - - t.Run("unknown selection mode", func(t *testing.T) { - input := base - input.SelectionMode = "guess" - if _, _, err := f.repos.Replacements.Authorize(ctx, input); !errors.Is(err, repository.ErrInvalidInput) { - t.Fatalf("error = %v, want ErrInvalidInput", err) - } - }) -} - -// A later confirmation takes over in the same transaction, so a source never -// holds two live replacements. -func TestStorageReplacementRepo_AuthorizeSupersedesEarlierConfirmation(t *testing.T) { - f := newReplacementFixture(t, "replacement-supersede", "01J000000000000000000RPL03") - ctx := context.Background() - - first := f.authorize(t, "202") - second := f.authorize(t, "303") - if second.ID == first.ID { - t.Fatal("second confirmation reused the first replacement") - } - - got, err := f.repos.Replacements.GetByID(ctx, first.ID) - if err != nil || got == nil { - t.Fatalf("GetByID first = %#v err=%v", got, err) - } - if got.Status != storagereplacement.StatusSuperseded { - t.Fatalf("first replacement status = %s, want superseded", got.Status) - } - if got.SupersededByID == nil || *got.SupersededByID != second.ID { - t.Fatalf("first.SupersededByID = %v, want %d", got.SupersededByID, second.ID) - } - active, err := f.repos.Replacements.GetActiveForDataSet(ctx, f.source.ID) - if err != nil || active == nil || active.ID != second.ID { - t.Fatalf("active replacement = %#v err=%v, want the newest confirmation", active, err) - } - - abandoned, err := f.repos.Tasks.GetByIdempotencyKey(ctx, storagereplacement.AbandonedTargetTaskKey(first.ID)) - if err != nil || abandoned == nil { - t.Fatalf("abandoned-target cleanup = %#v err=%v, want it queued with the later confirmation", abandoned, err) - } - claimed, err := f.repos.Tasks.ClaimReady(ctx, model.TaskTypeStorageCleanup, time.Minute) - if err != nil || claimed == nil || claimed.ID != abandoned.ID { - t.Fatalf("ClaimReady abandoned cleanup = %#v err=%v, want the leftover terminator", claimed, err) - } -} - -func TestStorageReplacementRepo_ActivateSwitchesTheSlotAtomically(t *testing.T) { - f := newReplacementFixture(t, "replacement-activate", "01J000000000000000000RPL04") - ctx := context.Background() - row := f.authorize(t, "202") - - // A target that is not writable yet cannot take the slot. - if err := f.repos.Replacements.Activate(ctx, row.ID); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("activate before ready = %v, want conflict", err) - } - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - - current, err := f.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, f.bucket.ID, 0) - if err != nil || current == nil || current.ID != row.TargetDataSetID { - t.Fatalf("current binding = %#v err=%v, want the target", current, err) - } - source, err := f.repos.Uploads.GetDataSetBindingByID(ctx, f.source.ID) - if err != nil || source == nil || source.IsCurrent || source.Status != model.StorageDataSetStatusDraining { - t.Fatalf("source after activation = %#v err=%v, want draining and not current", source, err) - } - got, err := f.repos.Replacements.GetByID(ctx, row.ID) - if err != nil || got == nil || got.Status != storagereplacement.StatusMigrating { - t.Fatalf("replacement after activation = %#v err=%v, want migrating", got, err) - } - if err := f.repos.Replacements.Activate(ctx, row.ID); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("second activate = %v, want conflict", err) - } -} - -func TestStorageReplacementRepo_DurableItemClaimFencesRetriesAndRecovery(t *testing.T) { - f := newReplacementFixture(t, "replacement-item-claim", "01J000000000000000ITEMQ01") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, done, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 200, 1); err != nil || !done { - t.Fatalf("SeedMigrationBatchWithBudget done=%v err=%v", done, err) - } - - item, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || item == nil || item.ClaimedAt == nil || item.MaxRetries == nil || *item.MaxRetries != 1 { - t.Fatalf("first item claim = %#v err=%v", item, err) - } - firstToken := storagereplacement.ClaimToken{ItemID: item.ID, ClaimedAt: *item.ClaimedAt} - staleToken := firstToken - staleToken.ClaimedAt = staleToken.ClaimedAt.Add(-time.Second) - if err := f.repos.Replacements.RenewReplacementItemLease(ctx, staleToken, time.Minute); !errors.Is(err, repository.ErrItemClaimLost) { - t.Fatalf("stale renewal error = %v, want ErrItemClaimLost", err) - } - - status, err := f.repos.Replacements.RetryReplacementItemClaim(ctx, firstToken, time.Now(), "temporary failure") - if err != nil || status != storagereplacement.ItemStatusRetrying { - t.Fatalf("first retry status=%s err=%v", status, err) - } - item, err = f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || item == nil || item.ClaimedAt == nil || item.RetryCount != 1 { - t.Fatalf("retry item claim = %#v err=%v", item, err) - } - secondToken := storagereplacement.ClaimToken{ItemID: item.ID, ClaimedAt: *item.ClaimedAt} - status, err = f.repos.Replacements.RetryReplacementItemClaim(ctx, secondToken, time.Now(), "still failing") - if err != nil || status != storagereplacement.ItemStatusFailed { - t.Fatalf("exhausted retry status=%s err=%v", status, err) - } - if err := f.repos.Replacements.CompleteReplacementItemClaim(ctx, firstToken); !errors.Is(err, repository.ErrItemClaimLost) { - t.Fatalf("stale completion error = %v, want ErrItemClaimLost", err) - } - - progresses, err := f.repos.Replacements.ReplacementProgresses(ctx, []int64{row.ID}) - if err != nil { - t.Fatalf("ReplacementProgresses: %v", err) - } - if progress := progresses[row.ID]; progress.ItemsFailed != 1 || progress.ItemsProcessed != 0 || progress.Percent == nil || *progress.Percent != 0 { - t.Fatalf("progress after exhaustion = %#v", progress) - } - if err := f.repos.Replacements.MarkFailed(ctx, row.ID, nil, "item retries exhausted"); err != nil { - t.Fatalf("MarkFailed: %v", err) - } - if _, err := f.repos.Replacements.Retry(ctx, repository.RetryReplacementInput{ - ReplacementID: row.ID, - MaxRetries: 5, - ItemMaxRetries: 3, - }); err != nil { - t.Fatalf("Retry: %v", err) - } - item, err = f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || item == nil || item.MaxRetries == nil || *item.MaxRetries != 3 || item.RetryCount != 0 { - t.Fatalf("operator-retried item = %#v err=%v", item, err) - } -} - -func TestStorageReplacementRepo_FailedReplacementReleasesClaimWithoutCancellingWork(t *testing.T) { - f := newReplacementFixture(t, "replacement-item-failed-release", "01J000000000000000ITEMQ16") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, done, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 200, 3); err != nil || !done { - t.Fatalf("SeedMigrationBatchWithBudget done=%v err=%v", done, err) - } - - item, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || item == nil || item.ClaimedAt == nil { - t.Fatalf("ClaimReadyReplacementItem = %#v err=%v", item, err) - } - if err := f.repos.Replacements.MarkFailed(ctx, row.ID, nil, "coordinator exhausted"); err != nil { - t.Fatalf("MarkFailed: %v", err) - } - if _, err := f.repos.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ - ReplacementID: row.ID, - ItemID: item.ID, - ItemClaimedAt: *item.ClaimedAt, - }); !errors.Is(err, storagereplacement.ErrItemDeferred) { - t.Fatalf("AcquireItem after replacement failure = %v, want ErrItemDeferred", err) - } - - var released storagereplacement.Item - if err := f.db.NewSelect().Model(&released).Where("id = ?", item.ID).Scan(ctx); err != nil { - t.Fatalf("load released item: %v", err) - } - if released.Status != storagereplacement.ItemStatusPending || released.ClaimedAt != nil || released.LeaseUntil != nil { - t.Fatalf("released item = %#v, want pending without a claim", released) - } - if _, err := f.repos.Replacements.Retry(ctx, repository.RetryReplacementInput{ - ReplacementID: row.ID, - MaxRetries: 5, - ItemMaxRetries: 3, - }); err != nil { - t.Fatalf("Retry: %v", err) - } - reclaimed, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || reclaimed == nil || reclaimed.ID != item.ID { - t.Fatalf("reclaimed item = %#v err=%v, want item %d", reclaimed, err, item.ID) - } -} - -func TestStorageReplacementRepo_GlobalClaimsAreFairAcrossReplacements(t *testing.T) { - f := newReplacementFixture(t, "replacement-item-fairness", "01J000000000000000ITEMQ06") - ctx := context.Background() - first := f.authorize(t, "202") - f.readyTarget(t, first, "2002") - if err := f.repos.Replacements.Activate(ctx, first.ID); err != nil { - t.Fatalf("Activate first: %v", err) - } - - secondSource, err := f.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: f.bucket.ID, ProviderID: onChainID(t, "303"), CopyIndex: 1, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding second source: %v", err) - } - if err := f.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: secondSource.ID, DataSetID: onChainID(t, "3003"), - }); err != nil { - t.Fatalf("MarkDataSetReady second source: %v", err) - } - second, _, err := f.repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ - BucketID: f.bucket.ID, SourceDataSetID: secondSource.ID, - SelectionMode: storagereplacement.SelectionModeManual, - TargetProviderID: onChainID(t, "404"), ClientRequestID: "fairness-second", MaxRetries: 5, - }) - if err != nil { - t.Fatalf("Authorize second: %v", err) - } - if err := f.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: second.TargetDataSetID, DataSetID: onChainID(t, "4004"), - }); err != nil { - t.Fatalf("MarkDataSetReady second target: %v", err) - } - if err := f.repos.Replacements.Activate(ctx, second.ID); err != nil { - t.Fatalf("Activate second: %v", err) - } - - extraUpload, err := f.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: f.bucket.ID, ContentSize: 1, Checksum: "fairness-extra", RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt extra: %v", err) - } - maxRetries := 5 - now := time.Now().Add(-time.Second) - for _, replacementID := range []int64{first.ID, second.ID} { - for _, uploadID := range []int64{f.upload.ID, extraUpload.ID} { - if _, err := f.db.NewInsert().Model(&storagereplacement.Item{ - ReplacementID: replacementID, UploadID: uploadID, - Status: storagereplacement.ItemStatusPending, ScheduledAt: now, - MaxRetries: &maxRetries, CreatedAt: now, UpdatedAt: now, - }).Exec(ctx); err != nil { - t.Fatalf("insert fairness item: %v", err) - } - } - } - - want := []int64{first.ID, second.ID, first.ID, second.ID} - for i, replacementID := range want { - item, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || item == nil { - t.Fatalf("claim %d = %#v err=%v", i, item, err) - } - if item.ReplacementID != replacementID { - t.Fatalf("claim %d replacement = %d, want %d; sequence must alternate while both have work", i, item.ReplacementID, replacementID) - } - } -} - -func TestStorageReplacementRepo_WaitingSourcePreservesRetryBudgetAcrossLease(t *testing.T) { - f := newReplacementFixture(t, "replacement-item-wait", "01J000000000000000ITEMQ02") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, _, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 200, 2); err != nil { - t.Fatalf("SeedMigrationBatchWithBudget: %v", err) - } - item, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || item == nil || item.ClaimedAt == nil { - t.Fatalf("ClaimReadyReplacementItem = %#v err=%v", item, err) - } - token := storagereplacement.ClaimToken{ItemID: item.ID, ClaimedAt: *item.ClaimedAt} - if err := f.repos.Replacements.WaitReplacementItemClaim(ctx, token, time.Now(), "no readable source"); err != nil { - t.Fatalf("WaitReplacementItemClaim: %v", err) - } - item, err = f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || item == nil || item.RetryCount != 0 { - t.Fatalf("waiting-source reclaim = %#v err=%v, want retry count unchanged", item, err) - } -} - -func TestStorageReplacementRepo_CopiedItemResumesReadableSourceWaitWithoutWakingCoordinator(t *testing.T) { - f := newReplacementFixture(t, "replacement-item-readable-again", "01J000000000000000ITEMQ20") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, done, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 200, 2); err != nil || !done { - t.Fatalf("SeedMigrationBatchWithBudget done=%v err=%v", done, err) - } - item, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || item == nil || item.ClaimedAt == nil { - t.Fatalf("ClaimReadyReplacementItem = %#v err=%v", item, err) - } - if err := f.repos.Replacements.MarkWaiting(ctx, row.ID, storagereplacement.WaitReasonReadableSource); err != nil { - t.Fatalf("MarkWaiting: %v", err) - } - waiting, err := f.repos.Replacements.GetByID(ctx, row.ID) - if err != nil || waiting == nil { - t.Fatalf("GetByID waiting = %#v err=%v", waiting, err) - } - - task, err := f.repos.Tasks.GetByIdempotencyKey(ctx, storagereplacement.MigrateTaskKey(row.ID)) - if err != nil || task == nil { - t.Fatalf("GetByIdempotencyKey = %#v err=%v", task, err) - } - nextPoll := time.Now().Add(time.Hour).Truncate(time.Second) - if _, err := f.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", nextPoll). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - t.Fatalf("delay coordinator: %v", err) - } - - token := storagereplacement.ClaimToken{ItemID: item.ID, ClaimedAt: *item.ClaimedAt} - if err := f.repos.Replacements.CompleteReplacementItemClaim(ctx, token); err != nil { - t.Fatalf("CompleteReplacementItemClaim: %v", err) - } - resumed, err := f.repos.Replacements.GetByID(ctx, row.ID) - if err != nil || resumed == nil || resumed.Status != storagereplacement.StatusMigrating || resumed.WaitReason != nil { - t.Fatalf("replacement after readable copy = %#v err=%v, want migrating without a wait reason", resumed, err) - } - if resumed.StateVersion != waiting.StateVersion+1 { - t.Fatalf("state version after readable copy = %d, want %d", resumed.StateVersion, waiting.StateVersion+1) - } - unchangedTask, err := f.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || unchangedTask == nil || !unchangedTask.ScheduledAt.Equal(nextPoll) { - t.Fatalf("coordinator after item completion = %#v err=%v, want poll time %s unchanged", unchangedTask, err, nextPoll) - } - execution, err := f.repos.Replacements.ReplacementExecution(ctx, row.ID) - if err != nil || !execution.SeedingComplete || execution.ItemsTotal != 1 || execution.ItemsCopied != 1 || - execution.HasPending || execution.HasActive || execution.HasRetrying || execution.HasWaitingSource || execution.HasFailed { - t.Fatalf("replacement execution after copy = %#v err=%v", execution, err) - } -} - -func TestStorageReplacementRepo_ExpiredLeaseIsRecoveredAndFenced(t *testing.T) { - f := newReplacementFixture(t, "replacement-item-expiry", "01J000000000000000ITEMQ03") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, _, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 200, 2); err != nil { - t.Fatalf("SeedMigrationBatchWithBudget: %v", err) - } - item, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || item == nil || item.ClaimedAt == nil { - t.Fatalf("first claim = %#v err=%v", item, err) - } - stale := storagereplacement.ClaimToken{ItemID: item.ID, ClaimedAt: *item.ClaimedAt} - mustExec(t, f.db, `UPDATE storage_replacement_items SET lease_until = ? WHERE id = ?`, time.Now().Add(-time.Second), item.ID) - if err := f.repos.Replacements.CancelReplacementItemClaim(ctx, stale); !errors.Is(err, repository.ErrItemClaimLost) { - t.Fatalf("expired cancellation error = %v, want ErrItemClaimLost", err) - } - if _, err := f.repos.Replacements.RetryReplacementItemClaim( - ctx, stale, time.Now(), "late retry", - ); !errors.Is(err, repository.ErrItemClaimLost) { - t.Fatalf("expired retry transition error = %v, want ErrItemClaimLost", err) - } - - reclaimed, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || reclaimed == nil || reclaimed.ClaimedAt == nil { - t.Fatalf("directly reclaimed expired item = %#v err=%v", reclaimed, err) - } - if reclaimed.ClaimedAt.Equal(stale.ClaimedAt) { - t.Fatal("reclaimed item reused the expired fencing token") - } - if err := f.repos.Replacements.CancelReplacementItemClaim(ctx, stale); !errors.Is(err, repository.ErrItemClaimLost) { - t.Fatalf("stale cancellation error = %v, want ErrItemClaimLost", err) - } - var stillClaimed storagereplacement.Item - if err := f.db.NewSelect().Model(&stillClaimed).Where("id = ?", reclaimed.ID).Scan(ctx); err != nil { - t.Fatalf("reload reclaimed item: %v", err) - } - if stillClaimed.Status != storagereplacement.ItemStatusRunning || stillClaimed.ClaimedAt == nil || - !stillClaimed.ClaimedAt.Equal(*reclaimed.ClaimedAt) { - t.Fatalf("stale cancellation changed the new claim: %#v", stillClaimed) - } - if err := f.repos.Replacements.CompleteReplacementItemClaim(ctx, stale); !errors.Is(err, repository.ErrItemClaimLost) { - t.Fatalf("stale completion error = %v, want ErrItemClaimLost", err) - } - mustExec(t, f.db, `UPDATE storage_replacement_items SET lease_until = ? WHERE id = ?`, time.Now().Add(-time.Second), reclaimed.ID) - released, err := f.repos.Replacements.ReleaseExpiredItemLeases(ctx) - if err != nil || released != 1 { - t.Fatalf("ReleaseExpiredItemLeases count=%d err=%v", released, err) - } -} - -func TestStorageReplacementRepo_RunningClaimIsVisibleBeforeTargetCopyAttach(t *testing.T) { - f := newReplacementFixture(t, "replacement-pre-attach-claim", "01J000000000000000ITEMQ19") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, _, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 200, 2); err != nil { - t.Fatalf("SeedMigrationBatchWithBudget: %v", err) - } - item, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || item == nil || item.ClaimedAt == nil || item.TargetCopyID != nil { - t.Fatalf("pre-attach claim = %#v err=%v", item, err) - } - - claim, err := f.repos.Replacements.RunningReplacementItemClaimForUpload(ctx, row.ID, f.upload.ID) - if err != nil || claim == nil { - t.Fatalf("RunningReplacementItemClaimForUpload = %#v err=%v", claim, err) - } - if claim.ItemID != item.ID || !claim.ClaimedAt.Equal(*item.ClaimedAt) { - t.Fatalf("claim = %#v, want item %d claimed at %s", claim, item.ID, item.ClaimedAt) - } -} - -func TestStorageReplacementRepo_StaleWorkerCannotPauseResumedMigration(t *testing.T) { - f := newReplacementFixture(t, "replacement-pause-version", "01J000000000000000ITEMQ04") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - migrating, err := f.repos.Replacements.GetByID(ctx, row.ID) - if err != nil || migrating == nil { - t.Fatalf("GetByID migrating = %#v err=%v", migrating, err) - } - staleVersion := migrating.StateVersion - if err := f.repos.Replacements.PauseMigration(ctx, row.ID, staleVersion, storagereplacement.WaitReasonTarget); err != nil { - t.Fatalf("PauseMigration: %v", err) - } - if err := f.repos.Replacements.MarkMigrating(ctx, row.ID); err != nil { - t.Fatalf("MarkMigrating: %v", err) - } - if err := f.repos.Replacements.PauseMigration(ctx, row.ID, staleVersion, storagereplacement.WaitReasonTarget); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("stale PauseMigration error = %v, want ErrConflict", err) - } - current, err := f.repos.Replacements.GetByID(ctx, row.ID) - if err != nil || current == nil || current.Status != storagereplacement.StatusMigrating { - t.Fatalf("replacement after stale pause = %#v err=%v, want migrating", current, err) - } -} - -func TestStorageReplacementRepo_TargetPauseSupersedesReadableSourceWait(t *testing.T) { - f := newReplacementFixture(t, "replacement-pause-source-wait", "01J000000000000000ITEMQ14") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if err := f.repos.Replacements.MarkWaiting(ctx, row.ID, storagereplacement.WaitReasonReadableSource); err != nil { - t.Fatalf("MarkWaiting: %v", err) - } - waiting, err := f.repos.Replacements.GetByID(ctx, row.ID) - if err != nil || waiting == nil { - t.Fatalf("GetByID waiting = %#v err=%v", waiting, err) - } - if err := f.repos.Replacements.PauseMigration( - ctx, row.ID, waiting.StateVersion, storagereplacement.WaitReasonTarget, - ); err != nil { - t.Fatalf("PauseMigration from readable-source wait: %v", err) - } - paused, err := f.repos.Replacements.GetByID(ctx, row.ID) - if err != nil || paused == nil || paused.Status != storagereplacement.StatusWaiting || - paused.WaitReason == nil || *paused.WaitReason != storagereplacement.WaitReasonTarget { - t.Fatalf("paused replacement = %#v err=%v, want waiting/target", paused, err) - } -} - -func TestStorageReplacementRepo_ProgressPhaseFollowsTargetGenerationOwnership(t *testing.T) { - f := newReplacementFixture(t, "replacement-progress-phase", "01J000000000000000ITEMQ15") - ctx := context.Background() - row := f.authorize(t, "202") - if err := f.repos.Replacements.MarkWaiting(ctx, row.ID, storagereplacement.WaitReasonFunding); err != nil { - t.Fatalf("MarkWaiting before activation: %v", err) - } - progresses, err := f.repos.Replacements.ReplacementProgresses(ctx, []int64{row.ID}) - if err != nil { - t.Fatalf("ReplacementProgresses before activation: %v", err) - } - if progress := progresses[row.ID]; progress.Phase != storagereplacement.PhasePrepare { - t.Fatalf("progress before activation = %#v, want prepare phase", progress) - } - - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if err := f.repos.Replacements.MarkWaiting(ctx, row.ID, storagereplacement.WaitReasonReadableSource); err != nil { - t.Fatalf("MarkWaiting after activation: %v", err) - } - progresses, err = f.repos.Replacements.ReplacementProgresses(ctx, []int64{row.ID}) - if err != nil { - t.Fatalf("ReplacementProgresses after activation: %v", err) - } - if progress := progresses[row.ID]; progress.Phase != storagereplacement.PhaseMigrate { - t.Fatalf("progress after activation = %#v, want migrate phase", progress) - } -} - -func TestStorageReplacementRepo_NextRetryAtReportsOnlyFutureRetries(t *testing.T) { - f := newReplacementFixture(t, "replacement-progress-retry", "01J000000000000000ITEMQ21") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, done, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 200, 2); err != nil || !done { - t.Fatalf("SeedMigrationBatchWithBudget done=%v err=%v", done, err) - } - - var waitingItemID int64 - if err := f.db.NewRaw(`SELECT id FROM storage_replacement_items WHERE replacement_id = ?`, row.ID). - Scan(ctx, &waitingItemID); err != nil { - t.Fatalf("select waiting item: %v", err) - } - extraUpload, err := f.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: f.bucket.ID, ContentSize: 1, Checksum: "replacement-progress-retry-extra", RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - waitingCheck := time.Now().Add(time.Minute).Truncate(time.Second) - retryAt := waitingCheck.Add(time.Hour) - maxRetries := 2 - mustExec(t, f.db, `UPDATE storage_replacement_items SET status = ?, scheduled_at = ? WHERE id = ?`, - storagereplacement.ItemStatusWaitingSource, waitingCheck, waitingItemID) - if _, err := f.db.NewInsert().Model(&storagereplacement.Item{ - ReplacementID: row.ID, - UploadID: extraUpload.ID, - Status: storagereplacement.ItemStatusRetrying, - ScheduledAt: retryAt, - MaxRetries: &maxRetries, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }).Exec(ctx); err != nil { - t.Fatalf("insert retrying item: %v", err) - } - mustExec(t, f.db, `UPDATE storage_replacements SET items_total = 2 WHERE id = ?`, row.ID) - - progresses, err := f.repos.Replacements.ReplacementProgresses(ctx, []int64{row.ID}) - if err != nil { - t.Fatalf("ReplacementProgresses: %v", err) - } - progress := progresses[row.ID] - if progress.NextRetryAt == nil || !progress.NextRetryAt.Equal(retryAt) { - t.Fatalf("next retry = %v, want retrying item at %s rather than readable-source check at %s", progress.NextRetryAt, retryAt, waitingCheck) - } - execution, err := f.repos.Replacements.ReplacementExecution(ctx, row.ID) - if err != nil || !execution.HasRetrying || !execution.HasWaitingSource { - t.Fatalf("replacement execution = %#v err=%v, want retrying and readable-source work", execution, err) - } - pastRetry := time.Now().Add(-time.Minute).Truncate(time.Second) - mustExec(t, f.db, `UPDATE storage_replacement_items SET status = ?, scheduled_at = ? WHERE id = ?`, - storagereplacement.ItemStatusRetrying, pastRetry, waitingItemID) - progresses, err = f.repos.Replacements.ReplacementProgresses(ctx, []int64{row.ID}) - if err != nil { - t.Fatalf("ReplacementProgresses with overdue retry: %v", err) - } - if progress := progresses[row.ID]; progress.NextRetryAt == nil || !progress.NextRetryAt.Equal(retryAt) { - t.Fatalf("next retry = %v, want future retry at %s rather than overdue retry at %s", progress.NextRetryAt, retryAt, pastRetry) - } - mustExec(t, f.db, `UPDATE storage_replacement_items SET scheduled_at = ? WHERE upload_id = ?`, - pastRetry, extraUpload.ID) - progresses, err = f.repos.Replacements.ReplacementProgresses(ctx, []int64{row.ID}) - if err != nil { - t.Fatalf("ReplacementProgresses with only overdue retries: %v", err) - } - if progress := progresses[row.ID]; progress.NextRetryAt != nil { - t.Fatalf("next retry = %v, want no future retry", progress.NextRetryAt) - } -} - -func TestStorageReplacementRepo_ProgressBucketsCommitAttentionExclusively(t *testing.T) { - f := newReplacementFixture(t, "replacement-progress-attention", "01J000000000000000ITEMQ22") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, done, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 10, 2); err != nil || !done { - t.Fatalf("SeedMigrationBatchWithBudget done=%v err=%v", done, err) - } - item, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || item == nil || item.ClaimedAt == nil { - t.Fatalf("ClaimReadyReplacementItem = %#v err=%v", item, err) - } - snapshot, err := f.repos.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ - ReplacementID: row.ID, ItemID: item.ID, ItemClaimedAt: *item.ClaimedAt, - }) - if err != nil || snapshot == nil { - t.Fatalf("AcquireItem = %#v err=%v", snapshot, err) - } - copyRow, err := f.repos.Replacements.AttachTargetCopy(ctx, repository.AttachReplacementTargetCopyInput{ - ReplacementID: row.ID, ItemID: item.ID, UploadID: snapshot.Upload.ID, ItemClaimedAt: *item.ClaimedAt, - }) - if err != nil || copyRow == nil { - t.Fatalf("AttachTargetCopy = %#v err=%v", copyRow, err) - } - mustExec(t, f.db, `UPDATE storage_replacement_items - SET status = ?, claimed_at = NULL, lease_until = NULL WHERE id = ?`, - storagereplacement.ItemStatusPending, item.ID) - mustExec(t, f.db, `UPDATE storage_upload_copies - SET status = ?, commit_attempt_id = 'attempt-progress', commit_attempted_at = CURRENT_TIMESTAMP - WHERE id = ?`, model.StorageUploadCopyStatusCommitting, copyRow.ID) - - progresses, err := f.repos.Replacements.ReplacementProgresses(ctx, []int64{row.ID}) - if err != nil { - t.Fatalf("ReplacementProgresses active: %v", err) - } - active := progresses[row.ID] - if active.ItemsActive != 1 || active.ItemsPending != 0 || active.ItemsAttention != 0 { - t.Fatalf("active progress = %#v, want one exclusive active item", active) - } - - mustExec(t, f.db, `UPDATE storage_replacement_items SET status = ? WHERE id = ?`, - storagereplacement.ItemStatusFailed, item.ID) - mustExec(t, f.db, `UPDATE storage_upload_copies - SET commit_attention_code = 'attempt_only_ambiguous', commit_attention_at = CURRENT_TIMESTAMP - WHERE id = ?`, copyRow.ID) - progresses, err = f.repos.Replacements.ReplacementProgresses(ctx, []int64{row.ID}) - if err != nil { - t.Fatalf("ReplacementProgresses attention: %v", err) - } - attention := progresses[row.ID] - if attention.ItemsAttention != 1 || attention.ItemsActive != 0 || attention.ItemsPending != 0 || - attention.ItemsRetrying != 0 || attention.ItemsWaitingSource != 0 || attention.ItemsFailed != 0 { - t.Fatalf("attention progress = %#v, want one mutually exclusive attention item", attention) - } - if attention.ItemsProcessed != 0 || attention.ItemsNoLongerNeeded != 0 || - attention.Percent == nil || *attention.Percent != 0 { - t.Fatalf("attention progress accounting = %#v, want outstanding item at 0%%", attention) - } -} - -func TestStorageReplacementRepo_TerminalReplacementClaimsOnlyDurableCommitWork(t *testing.T) { - for _, tc := range []struct { - name string - status storagereplacement.Status - capacityWait bool - attempted bool - confirm bool - }{ - {name: "failed reservation is released", status: storagereplacement.StatusFailed}, - {name: "failed ready-only reservation is released", status: storagereplacement.StatusFailed, capacityWait: true}, - {name: "failed submission is recoverable", status: storagereplacement.StatusFailed, attempted: true}, - {name: "superseded submission is confirmed", status: storagereplacement.StatusSuperseded, attempted: true}, - {name: "superseded submission settles without live owner", status: storagereplacement.StatusSuperseded, attempted: true, confirm: true}, - } { - t.Run(tc.name, func(t *testing.T) { - f := newReplacementFixture(t, "replacement-terminal-"+strings.ReplaceAll(tc.name, " ", "-"), "01J000000000000000ITEMQ23") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, done, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 10, 2); err != nil || !done { - t.Fatalf("SeedMigrationBatchWithBudget done=%v err=%v", done, err) - } - item, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || item == nil || item.ClaimedAt == nil { - t.Fatalf("initial claim = %#v err=%v", item, err) - } - snapshot, err := f.repos.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ - ReplacementID: row.ID, ItemID: item.ID, ItemClaimedAt: *item.ClaimedAt, - }) - if err != nil || snapshot == nil { - t.Fatalf("AcquireItem = %#v err=%v", snapshot, err) - } - copyRow, err := f.repos.Replacements.AttachTargetCopy(ctx, repository.AttachReplacementTargetCopyInput{ - ReplacementID: row.ID, ItemID: item.ID, UploadID: snapshot.Upload.ID, ItemClaimedAt: *item.ClaimedAt, - }) - if err != nil || copyRow == nil { - t.Fatalf("AttachTargetCopy = %#v err=%v", copyRow, err) - } - if snapshot.Upload.PieceCID == nil { - t.Fatal("fixture upload has no piece CID") - } - if err := f.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - StorageUploadCopyID: copyRow.ID, RequireEligibleCopy: true, - UploadID: snapshot.Upload.ID, CopyIndex: copyRow.CopyIndex, - PieceCID: *snapshot.Upload.PieceCID, RetrievalURL: "https://target.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - identity := storagecommit.CopyIdentity{ - StorageUploadCopyID: copyRow.ID, UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, StorageDataSetID: *copyRow.StorageDataSetID, - RequireEligibleCopy: true, - } - if tc.capacityWait { - capacityCopies := seedCommitCopies(t, f.db, f.bucket.ID, row.TargetDataSetID, storagecommit.MaxActiveAttemptsPerDataSet) - for i := range capacityCopies { - if _, err := f.repos.Uploads.ReserveCommitAttempt(ctx, storagecommit.ReserveInput{ - Copy: commitCopyIdentity(capacityCopies[i]), AttemptID: fmt.Sprintf("terminal-capacity-%d", i), - }); err != nil { - t.Fatalf("reserve terminal capacity %d: %v", i, err) - } - } - } - reservation, err := f.repos.Uploads.ReserveCommitAttempt(ctx, storagecommit.ReserveInput{ - Copy: identity, AttemptID: "terminal-attempt", - }) - if err != nil { - t.Fatalf("ReserveCommitAttempt: %v", err) - } - if tc.capacityWait && (reservation.State != storagecommit.ReservationWaiting || reservation.Copy.CommitAttemptID != nil) { - t.Fatalf("terminal capacity reservation = %#v, want ready-only wait", reservation) - } - if tc.attempted { - if _, err := f.repos.Uploads.MarkCommitAttempted(ctx, storagecommit.AttemptInput{ - Copy: identity, AttemptID: "terminal-attempt", ExtraDataHex: "abcd", - }); err != nil { - t.Fatalf("MarkCommitAttempted: %v", err) - } - mustExec(t, f.db, `DELETE FROM object_versions WHERE version_id = ?`, f.version.VersionID) - } - token := storagereplacement.ClaimToken{ItemID: item.ID, ClaimedAt: *item.ClaimedAt} - if err := f.repos.Replacements.ReleaseReplacementItemClaim(ctx, token); err != nil { - t.Fatalf("ReleaseReplacementItemClaim: %v", err) - } - oldDispatch := time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC) - mustExec(t, f.db, `UPDATE storage_replacements - SET status = ?, wait_reason = NULL, last_dispatched_at = ? WHERE id = ?`, tc.status, oldDispatch, row.ID) - - claimed, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || claimed == nil || claimed.ID != item.ID || claimed.ClaimedAt == nil { - t.Fatalf("terminal claim = %#v err=%v", claimed, err) - } - dispatched, err := f.repos.Replacements.GetByID(ctx, row.ID) - if err != nil || dispatched.LastDispatchedAt == nil || !dispatched.LastDispatchedAt.After(oldDispatch) { - t.Fatalf("terminal dispatch = %#v err=%v, want fairness timestamp after %s", dispatched, err, oldDispatch) - } - recovered, err := f.repos.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ - ReplacementID: row.ID, ItemID: claimed.ID, ItemClaimedAt: *claimed.ClaimedAt, - }) - if tc.capacityWait { - if !errors.Is(err, storagereplacement.ErrItemCancelled) || recovered != nil { - t.Fatalf("terminal ready-only settlement = %#v err=%v, want cancellation", recovered, err) - } - persisted, loadErr := f.repos.Uploads.GetUploadCopyByID(ctx, copyRow.ID) - if loadErr != nil || persisted.CommitReadyAt != nil || persisted.CommitExtraDataHex != nil || persisted.CommitAttemptID != nil { - t.Fatalf("terminal ready-only copy = %#v err=%v, want cleared reservation", persisted, loadErr) - } - return - } - if err != nil || recovered == nil || recovered.Replacement.Status != tc.status { - t.Fatalf("terminal recovery snapshot = %#v err=%v", recovered, err) - } - if tc.confirm { - pieceID := onChainID(t, "7001") - if err := f.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - StorageUploadCopyID: copyRow.ID, RequireEligibleCopy: true, - UploadID: copyRow.UploadID, CopyIndex: copyRow.CopyIndex, - PieceCID: *snapshot.Upload.PieceCID, PieceID: &pieceID, - RetrievalURL: "https://target.example/piece", CommitAttemptID: "terminal-attempt", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted without live owner: %v", err) - } - return - } - if tc.attempted { - recoveryToken := storagereplacement.ClaimToken{ItemID: claimed.ID, ClaimedAt: *claimed.ClaimedAt} - if err := f.repos.Replacements.ReleaseReplacementItemClaim(ctx, recoveryToken); err != nil { - t.Fatalf("release recovery claim: %v", err) - } - if err := f.repos.Uploads.MarkCommitAttention(ctx, storagecommit.AttentionInput{ - Copy: storagecommit.CopyIdentity{ - StorageUploadCopyID: copyRow.ID, UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, StorageDataSetID: *copyRow.StorageDataSetID, - }, - AttemptID: "terminal-attempt", Code: storagecommit.AttentionAttemptOnlyAmbiguous, - }); err != nil { - t.Fatalf("MarkCommitAttention without live owner: %v", err) - } - if err := f.repos.Uploads.ReleaseCommitAttention(ctx, storagecommit.ManualReleaseInput{ - CopyID: copyRow.ID, ExpectedAttemptID: "terminal-attempt", AcknowledgePossibleDuplicate: true, - }); err != nil { - t.Fatalf("ReleaseCommitAttention: %v", err) - } - var releasedStatus storagereplacement.ItemStatus - if err := f.db.NewSelect().Model((*storagereplacement.Item)(nil)). - Column("status").Where("id = ?", item.ID).Scan(ctx, &releasedStatus); err != nil { - t.Fatalf("load released item: %v", err) - } - wantStatus := storagereplacement.ItemStatusFailed - if tc.status == storagereplacement.StatusSuperseded { - wantStatus = storagereplacement.ItemStatusCancelled - } - if releasedStatus != wantStatus { - t.Fatalf("released item status = %s, want %s", releasedStatus, wantStatus) - } - } - }) - } -} - -func TestStorageReplacementRepo_FinalRetryClearsUnattemptedReservation(t *testing.T) { - f := newReplacementFixture(t, "replacement-final-retry-fifo", "01J000000000000000ITEMQ24") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, _, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 10, 0); err != nil { - t.Fatalf("SeedMigrationBatchWithBudget: %v", err) - } - item, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || item == nil || item.ClaimedAt == nil { - t.Fatalf("claim = %#v err=%v", item, err) - } - snapshot, err := f.repos.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ - ReplacementID: row.ID, ItemID: item.ID, ItemClaimedAt: *item.ClaimedAt, - }) - if err != nil || snapshot == nil || snapshot.Upload.PieceCID == nil { - t.Fatalf("AcquireItem = %#v err=%v", snapshot, err) - } - copyRow, err := f.repos.Replacements.AttachTargetCopy(ctx, repository.AttachReplacementTargetCopyInput{ - ReplacementID: row.ID, ItemID: item.ID, UploadID: snapshot.Upload.ID, ItemClaimedAt: *item.ClaimedAt, - }) - if err != nil || copyRow == nil { - t.Fatalf("AttachTargetCopy = %#v err=%v", copyRow, err) - } - if err := f.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - StorageUploadCopyID: copyRow.ID, RequireEligibleCopy: true, - UploadID: copyRow.UploadID, CopyIndex: copyRow.CopyIndex, - PieceCID: *snapshot.Upload.PieceCID, RetrievalURL: "https://target.example/piece", CommitExtraDataHex: "abcd", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - targetIdentity := storagecommit.CopyIdentity{ - StorageUploadCopyID: copyRow.ID, UploadID: copyRow.UploadID, CopyIndex: copyRow.CopyIndex, - StorageDataSetID: row.TargetDataSetID, RequireEligibleCopy: true, - } - targetReservation, err := f.repos.Uploads.ReserveCommitAttempt(ctx, storagecommit.ReserveInput{ - Copy: targetIdentity, AttemptID: "target-unattempted", - }) - if err != nil || targetReservation.State != storagecommit.ReservationAcquired || - targetReservation.Copy.CommitAttemptID == nil || targetReservation.Copy.CommitAttemptedAt != nil { - t.Fatalf("target reservation = %#v err=%v, want unattempted token", targetReservation, err) - } - capacityCopies := seedCommitCopies(t, f.db, f.bucket.ID, row.TargetDataSetID, storagecommit.MaxActiveAttemptsPerDataSet-1) - for i := range capacityCopies { - if _, err := f.repos.Uploads.ReserveCommitAttempt(ctx, storagecommit.ReserveInput{ - Copy: commitCopyIdentity(capacityCopies[i]), AttemptID: fmt.Sprintf("retry-capacity-%d", i), - }); err != nil { - t.Fatalf("reserve capacity %d: %v", i, err) - } - } - follower := seedCommitCopies(t, f.db, f.bucket.ID, row.TargetDataSetID, 1)[0] - if followerReservation, err := f.repos.Uploads.ReserveCommitAttempt(ctx, storagecommit.ReserveInput{ - Copy: commitCopyIdentity(follower), AttemptID: "follower-waiting", - }); err != nil || followerReservation.State != storagecommit.ReservationWaiting { - t.Fatalf("follower reservation = %#v err=%v, want waiting", followerReservation, err) - } - status, err := f.repos.Replacements.RetryReplacementItemClaim(ctx, storagereplacement.ClaimToken{ - ItemID: item.ID, ClaimedAt: *item.ClaimedAt, - }, time.Now(), "presign failed") - if err != nil || status != storagereplacement.ItemStatusFailed { - t.Fatalf("final retry status=%s err=%v, want failed", status, err) - } - persisted, err := f.repos.Uploads.GetUploadCopyByID(ctx, copyRow.ID) - if err != nil || persisted.CommitReadyAt != nil || persisted.CommitExtraDataHex != nil || persisted.CommitAttemptID != nil { - t.Fatalf("failed target reservation = %#v err=%v, want cleared", persisted, err) - } - admitted, err := f.repos.Uploads.ReserveCommitAttempt(ctx, storagecommit.ReserveInput{ - Copy: commitCopyIdentity(follower), AttemptID: "follower-admitted", - }) - if err != nil || admitted.State != storagecommit.ReservationAcquired { - t.Fatalf("follower after terminal cleanup = %#v err=%v, want acquired", admitted, err) - } -} - -func TestStorageReplacementRepo_FailedAttemptedItemRemainsConfirmable(t *testing.T) { - f := newReplacementFixture(t, "replacement-failed-attempt-confirmation", "01J000000000000000ITEMQ25") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, _, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 10, 0); err != nil { - t.Fatalf("SeedMigrationBatchWithBudget: %v", err) - } - item, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || item == nil || item.ClaimedAt == nil { - t.Fatalf("claim = %#v err=%v", item, err) - } - snapshot, err := f.repos.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ - ReplacementID: row.ID, ItemID: item.ID, ItemClaimedAt: *item.ClaimedAt, - }) - if err != nil || snapshot == nil || snapshot.Upload.PieceCID == nil { - t.Fatalf("AcquireItem = %#v err=%v", snapshot, err) - } - copyRow, err := f.repos.Replacements.AttachTargetCopy(ctx, repository.AttachReplacementTargetCopyInput{ - ReplacementID: row.ID, ItemID: item.ID, UploadID: snapshot.Upload.ID, ItemClaimedAt: *item.ClaimedAt, - }) - if err != nil || copyRow == nil { - t.Fatalf("AttachTargetCopy = %#v err=%v", copyRow, err) - } - const ( - attemptID = "failed-item-attempt" - transactionID = "0xfailed-item-attempt" - ) - if err := f.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - StorageUploadCopyID: copyRow.ID, RequireEligibleCopy: true, - UploadID: copyRow.UploadID, CopyIndex: copyRow.CopyIndex, - PieceCID: *snapshot.Upload.PieceCID, RetrievalURL: "https://target.example/piece", CommitExtraDataHex: "abcd", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - identity := storagecommit.CopyIdentity{ - StorageUploadCopyID: copyRow.ID, UploadID: copyRow.UploadID, CopyIndex: copyRow.CopyIndex, - StorageDataSetID: row.TargetDataSetID, RequireEligibleCopy: true, - } - if _, err := f.repos.Uploads.ReserveCommitAttempt(ctx, storagecommit.ReserveInput{ - Copy: identity, AttemptID: attemptID, - }); err != nil { - t.Fatalf("ReserveCommitAttempt: %v", err) - } - if _, err := f.repos.Uploads.MarkCommitAttempted(ctx, storagecommit.AttemptInput{ - Copy: identity, AttemptID: attemptID, ExtraDataHex: "abcd", - }); err != nil { - t.Fatalf("MarkCommitAttempted: %v", err) - } - if err := f.repos.Uploads.RecordCommitTransaction(ctx, storagecommit.EvidenceInput{ - Copy: identity, AttemptID: attemptID, TransactionID: transactionID, - }); err != nil { - t.Fatalf("RecordCommitTransaction: %v", err) - } - status, err := f.repos.Replacements.RetryReplacementItemClaim(ctx, storagereplacement.ClaimToken{ - ItemID: item.ID, ClaimedAt: *item.ClaimedAt, - }, time.Now(), "settlement failed") - if err != nil || status != storagereplacement.ItemStatusFailed { - t.Fatalf("final retry status=%s err=%v, want failed", status, err) - } - persisted, err := f.repos.Uploads.GetUploadCopyByID(ctx, copyRow.ID) - if err != nil || persisted.CommitAttemptID == nil || *persisted.CommitAttemptID != attemptID || - persisted.CommitAttemptedAt == nil { - t.Fatalf("attempted fence after item exhaustion = %#v err=%v", persisted, err) - } - if claimed, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute); err != nil || claimed != nil { - t.Fatalf("active replacement claimed failed item = %#v err=%v", claimed, err) - } - if err := f.repos.Replacements.MarkFailed(ctx, row.ID, nil, "item retries exhausted"); err != nil { - t.Fatalf("MarkFailed: %v", err) - } - claimed, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || claimed == nil || claimed.ID != item.ID || claimed.ClaimedAt == nil { - t.Fatalf("terminal confirmation claim = %#v err=%v", claimed, err) - } - recovered, err := f.repos.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ - ReplacementID: row.ID, ItemID: claimed.ID, ItemClaimedAt: *claimed.ClaimedAt, - }) - if err != nil || recovered == nil || recovered.Replacement.Status != storagereplacement.StatusFailed { - t.Fatalf("terminal confirmation snapshot = %#v err=%v", recovered, err) - } - pieceID := onChainID(t, "3002") - if err := f.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - StorageUploadCopyID: copyRow.ID, RequireEligibleCopy: true, - UploadID: copyRow.UploadID, CopyIndex: copyRow.CopyIndex, - PieceCID: *snapshot.Upload.PieceCID, PieceID: &pieceID, - RetrievalURL: "https://target.example/piece", CommitExtraDataHex: "abcd", - CommitTransactionID: transactionID, CommitAttemptID: attemptID, - CommitConfirmedTransactionID: transactionID, - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - if err := f.repos.Replacements.CompleteReplacementItemClaim(ctx, storagereplacement.ClaimToken{ - ItemID: claimed.ID, ClaimedAt: *claimed.ClaimedAt, - }); err != nil { - t.Fatalf("CompleteReplacementItemClaim: %v", err) - } - persisted, err = f.repos.Uploads.GetUploadCopyByID(ctx, copyRow.ID) - if err != nil || persisted.Status != model.StorageUploadCopyStatusCommitted || persisted.CommitAttemptID != nil { - t.Fatalf("confirmed failed-item copy = %#v err=%v", persisted, err) - } - var completed storagereplacement.Item - if err := f.db.NewSelect().Model(&completed).Where("id = ?", item.ID).Scan(ctx); err != nil { - t.Fatalf("reload completed item: %v", err) - } - if completed.Status != storagereplacement.ItemStatusCopied { - t.Fatalf("completed item status = %s, want copied", completed.Status) - } -} - -func TestStorageReplacementRepo_ProviderEvidenceIsMonotonic(t *testing.T) { - f := newReplacementFixture(t, "replacement-item-evidence", "01J000000000000000ITEMQ05") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, _, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 200, 2); err != nil { - t.Fatalf("SeedMigrationBatchWithBudget: %v", err) - } - item, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || item == nil || item.ClaimedAt == nil { - t.Fatalf("claim = %#v err=%v", item, err) - } - snapshot, err := f.repos.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ - ReplacementID: row.ID, ItemID: item.ID, ItemClaimedAt: *item.ClaimedAt, - }) - if err != nil || snapshot == nil { - t.Fatalf("AcquireItem = %#v err=%v", snapshot, err) - } - copyRow, err := f.repos.Replacements.AttachTargetCopy(ctx, repository.AttachReplacementTargetCopyInput{ - ReplacementID: row.ID, ItemID: item.ID, UploadID: snapshot.Upload.ID, ItemClaimedAt: *item.ClaimedAt, - }) - if err != nil || copyRow == nil { - t.Fatalf("AttachTargetCopy = %#v err=%v", copyRow, err) - } - pieceCID := "bafk2bzaceproviderreplacement" - if snapshot.Upload.PieceCID != nil { - pieceCID = *snapshot.Upload.PieceCID - } - ready := repository.MarkUploadCopyPieceReadyInput{ - StorageUploadCopyID: copyRow.ID, RequireEligibleCopy: true, - UploadID: snapshot.Upload.ID, CopyIndex: copyRow.CopyIndex, - PieceCID: pieceCID, RetrievalURL: "https://target.example/piece", - } - if err := f.repos.Uploads.MarkUploadCopyPieceReady(ctx, ready); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - ready.RetrievalURL = "https://stale.example/different" - if err := f.repos.Uploads.MarkUploadCopyPieceReady(ctx, ready); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("different retrieval evidence error = %v, want ErrConflict", err) - } - identity := storagecommit.CopyIdentity{ - StorageUploadCopyID: copyRow.ID, RequireEligibleCopy: true, - UploadID: snapshot.Upload.ID, CopyIndex: copyRow.CopyIndex, StorageDataSetID: row.TargetDataSetID, - } - if _, err := f.repos.Uploads.ReserveCommitAttempt(ctx, storagecommit.ReserveInput{ - Copy: identity, AttemptID: "provider-evidence", - }); err != nil { - t.Fatalf("ReserveCommitAttempt: %v", err) - } - if _, err := f.repos.Uploads.MarkCommitAttempted(ctx, storagecommit.AttemptInput{ - Copy: identity, AttemptID: "provider-evidence", ExtraDataHex: "01", - }); err != nil { - t.Fatalf("MarkCommitAttempted: %v", err) - } - if err := f.repos.Uploads.RecordCommitTransaction(ctx, storagecommit.EvidenceInput{ - Copy: identity, AttemptID: "provider-evidence", TransactionID: "0xsubmitted", - }); err != nil { - t.Fatalf("RecordCommitTransaction: %v", err) - } - if err := f.repos.Uploads.RecordCommitTransaction(ctx, storagecommit.EvidenceInput{ - Copy: identity, AttemptID: "provider-evidence", TransactionID: "0xsubmitted", - }); err != nil { - t.Fatalf("idempotent RecordCommitTransaction: %v", err) - } - if err := f.repos.Uploads.RecordCommitTransaction(ctx, storagecommit.EvidenceInput{ - Copy: identity, AttemptID: "provider-evidence", TransactionID: "0xdifferent", - }); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("different commit evidence error = %v, want ErrConflict", err) - } -} - -// Recovery must not revive a generation an operator is actively replacing, but -// a replacement that has given up should not hold the slot hostage. -func TestStorageReplacementRepo_RecoveryYieldsToInProgressReplacementOnly(t *testing.T) { - f := newReplacementFixture(t, "replacement-recovery", "01J000000000000000000RPL05") - ctx := context.Background() - row := f.authorize(t, "202") - - if err := f.repos.Uploads.MarkDataSetUnavailable(ctx, f.source.ID, "provider unreachable"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - recovered, err := f.repos.Uploads.RecoverDataSet(ctx, repository.MarkDataSetReadyInput{ - ID: f.source.ID, - UploadID: f.upload.ID, - DataSetID: onChainID(t, "1001"), - }) - if err != nil { - t.Fatalf("RecoverDataSet during replacement: %v", err) - } - if recovered { - t.Fatal("recovery revived a generation with an in-progress replacement") - } - - if err := f.repos.Replacements.MarkFailed(ctx, row.ID, nil, "target creation exhausted"); err != nil { - t.Fatalf("MarkFailed: %v", err) - } - recovered, err = f.repos.Uploads.RecoverDataSet(ctx, repository.MarkDataSetReadyInput{ - ID: f.source.ID, - UploadID: f.upload.ID, - DataSetID: onChainID(t, "1001"), - }) - if err != nil { - t.Fatalf("RecoverDataSet after failure: %v", err) - } - if !recovered { - t.Fatal("recovery stayed blocked after the replacement terminally failed") - } -} - -func TestStorageReplacementRepo_SeedMigrationBatchIsBounded(t *testing.T) { - f := newReplacementFixture(t, "replacement-seed", "01J000000000000000000RPL06") - ctx := context.Background() - - // Extra stored content on the same source, so seeding has to page. - for i, versionID := range []string{ - "01J000000000000000000RPL07", - "01J000000000000000000RPL08", - "01J000000000000000000RPL09", - } { - version := newObjectVersion(f.bucket.ID, "file.txt", versionID, 10) - version.Checksum = versionID - if _, err := f.repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent %d: %v", i, err) - } - upload := startCopyHealthUpload(t, f.repos, f.bucket.ID, version.VersionID, version.Size, version.Checksum, 1) - if err := f.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: f.source.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: f.source.ProviderID, - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings %d: %v", i, err) - } - if err := f.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "bafk2bzacestorhealth", - PieceID: onChainIDPtr(t, "700"+versionID[len(versionID)-1:]), - RetrievalURL: "https://source.example/piece-" + versionID, - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted %d: %v", i, err) - } - bindStorageHealthVersion(t, f.repos, f.bucket.ID, upload.ID, version) - } - - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - - total := 0 - passes := 0 - for { - inserted, done, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 2, 5) - if err != nil { - t.Fatalf("SeedMigrationBatch: %v", err) - } - if inserted > 2 { - t.Fatalf("seeded %d items in one pass, want at most the batch limit", inserted) - } - total += inserted - passes++ - if done { - break - } - if passes > 10 { - t.Fatal("seeding never completed, cursor is not advancing") - } - } - if total != 4 { - t.Fatalf("seeded %d items, want one per stored upload", total) - } - if passes < 2 { - t.Fatalf("seeding finished in %d pass, want it to page", passes) - } - - got, err := f.repos.Replacements.GetByID(ctx, row.ID) - if err != nil || got == nil || !got.SeedingComplete || got.ItemsTotal != 4 { - t.Fatalf("replacement after seeding = %#v err=%v, want complete with 4 items", got, err) - } - // Re-running is a no-op rather than a duplicate. - inserted, done, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 2, 5) - if err != nil || inserted != 0 || !done { - t.Fatalf("re-seed = (%d, %v, %v), want no new work", inserted, done, err) - } -} - -func TestStorageReplacementRepo_RetryOnlyResumesOperatorAttentionStates(t *testing.T) { - f := newReplacementFixture(t, "replacement-retry", "01J000000000000000000RPL11") - ctx := context.Background() - row := f.authorize(t, "202") - - if _, err := f.repos.Replacements.Retry(ctx, repository.RetryReplacementInput{ReplacementID: row.ID, MaxRetries: 5, ItemMaxRetries: 5}); !errors.Is(err, storagereplacement.ErrNotRetryable) { - t.Fatalf("retry while preparing = %v, want ErrNotRetryable", err) - } - if err := f.repos.Replacements.MarkFailed(ctx, row.ID, nil, "creation exhausted"); err != nil { - t.Fatalf("MarkFailed: %v", err) - } - resumed, err := f.repos.Replacements.Retry(ctx, repository.RetryReplacementInput{ReplacementID: row.ID, MaxRetries: 5, ItemMaxRetries: 5}) - if err != nil { - t.Fatalf("Retry: %v", err) - } - // The target never activated, so the retry resumes preparation. - if resumed.Status != storagereplacement.StatusPreparingTarget || resumed.LastError != nil { - t.Fatalf("resumed = %#v, want preparing_target with the error cleared", resumed) - } - - superseded := f.authorize(t, "303") - if _, err := f.repos.Replacements.Retry(ctx, repository.RetryReplacementInput{ReplacementID: row.ID, MaxRetries: 5, ItemMaxRetries: 5}); !errors.Is(err, storagereplacement.ErrSuperseded) { - t.Fatalf("retry after supersede = %v, want ErrSuperseded", err) - } - if superseded.ID == row.ID { - t.Fatal("supersede reused the replacement row") - } -} - -func TestStorageReplacementRepo_FailCoordinatorRollsBackBothRecords(t *testing.T) { - f := newReplacementFixture(t, "replacement-coordinator-failure", "01J000000000000000ITEMQ17") - ctx := context.Background() - row := f.authorize(t, "202") - claimed, err := f.repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil || claimed == nil || claimed.ClaimedAt == nil { - t.Fatalf("ClaimReady coordinator = %#v err=%v", claimed, err) - } - - stale := *claimed - staleClaimedAt := claimed.ClaimedAt.Add(-time.Second) - stale.ClaimedAt = &staleClaimedAt - if err := f.repos.Replacements.FailCoordinator(ctx, repository.ReplacementCoordinatorFailureInput{ - ReplacementID: row.ID, - Task: &stale, - LastError: "stored items need attention", - }); err == nil { - t.Fatal("FailCoordinator with stale task claim succeeded") - } - unchanged, err := f.repos.Replacements.GetByID(ctx, row.ID) - if err != nil || unchanged == nil || unchanged.Status != storagereplacement.StatusPreparingTarget { - t.Fatalf("replacement after rollback = %#v err=%v, want preparing_target", unchanged, err) - } - activeTask, err := f.repos.Tasks.GetByID(ctx, claimed.ID) - if err != nil || activeTask == nil || activeTask.Status != model.TaskStatusRunning { - t.Fatalf("task after rollback = %#v err=%v, want running", activeTask, err) - } - - if err := f.repos.Replacements.FailCoordinator(ctx, repository.ReplacementCoordinatorFailureInput{ - ReplacementID: row.ID, - Task: claimed, - LastError: "stored items need attention", - }); err != nil { - t.Fatalf("FailCoordinator: %v", err) - } - failed, err := f.repos.Replacements.GetByID(ctx, row.ID) - if err != nil || failed == nil || failed.Status != storagereplacement.StatusFailed { - t.Fatalf("failed replacement = %#v err=%v", failed, err) - } - failedTask, err := f.repos.Tasks.GetByID(ctx, claimed.ID) - if err != nil || failedTask == nil || failedTask.Status != model.TaskStatusFailed { - t.Fatalf("failed coordinator = %#v err=%v", failedTask, err) - } -} - -func TestStorageReplacementRepo_ExhaustedCoordinatorFailsAtomically(t *testing.T) { - f := newReplacementFixture(t, "replacement-coordinator-exhausted", "01J000000000000000ITEMQ18") - ctx := context.Background() - row := f.authorize(t, "202") - task, err := f.repos.Tasks.GetByIdempotencyKey(ctx, storagereplacement.MigrateTaskKey(row.ID)) - if err != nil || task == nil { - t.Fatalf("GetByIdempotencyKey coordinator = %#v err=%v", task, err) - } - if _, err := f.db.NewUpdate().Model((*model.Task)(nil)). - Set("max_retries = ?", 1). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - t.Fatalf("set coordinator retry budget: %v", err) - } - claimed, err := f.repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil || claimed == nil { - t.Fatalf("ClaimReady coordinator = %#v err=%v", claimed, err) - } - - status, err := f.repos.Replacements.ScheduleCoordinatorRetry(ctx, repository.ReplacementCoordinatorRetryInput{ - ReplacementID: row.ID, - Task: claimed, - LastError: "database unavailable", - Backoff: time.Second, - }) - if err != nil || status != model.TaskStatusExhausted { - t.Fatalf("ScheduleCoordinatorRetry status=%s err=%v", status, err) - } - failed, err := f.repos.Replacements.GetByID(ctx, row.ID) - if err != nil || failed == nil || failed.Status != storagereplacement.StatusFailed { - t.Fatalf("replacement after exhaustion = %#v err=%v, want failed", failed, err) - } - if failed.LastError == nil || !strings.Contains(*failed.LastError, "max retries reached") { - t.Fatalf("replacement last error = %v, want exhausted retry context", failed.LastError) - } - exhausted, err := f.repos.Tasks.GetByID(ctx, claimed.ID) - if err != nil || exhausted == nil || exhausted.Status != model.TaskStatusExhausted { - t.Fatalf("coordinator after exhaustion = %#v err=%v, want exhausted", exhausted, err) - } -} - -func TestStorageCleanupRepo_SettlesReplacementItemBeforeDeletingUpload(t *testing.T) { - f := newReplacementFixture(t, "replacement-provenance-cleanup", model.NewVersionID()) - ctx := context.Background() - replacement := f.authorize(t, "202") - orphan, err := f.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: f.bucket.ID, SourceVersionID: model.NewVersionID(), ContentSize: 10, - Checksum: "orphaned-replacement-content", RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - claimedAt := time.Now() - leaseUntil := claimedAt.Add(time.Minute) - item := &storagereplacement.Item{ - ReplacementID: replacement.ID, UploadID: orphan.ID, - Status: storagereplacement.ItemStatusRunning, ClaimedAt: &claimedAt, LeaseUntil: &leaseUntil, - CreatedAt: time.Now(), UpdatedAt: time.Now(), - } - if _, err := f.db.NewInsert().Model(item).Exec(ctx); err != nil { - t.Fatalf("insert replacement item: %v", err) - } - if _, err := f.db.NewUpdate().Model((*storagereplacement.Replacement)(nil)). - Set("items_total = ?", 1).Set("seeding_complete = ?", true). - Where("id = ?", replacement.ID).Exec(ctx); err != nil { - t.Fatalf("set replacement total: %v", err) - } - - if err := f.repos.StorageCleanup.DeleteUploadProvenanceIfUnreferenced(ctx, orphan.ID); err != nil { - t.Fatalf("DeleteUploadProvenanceIfUnreferenced: %v", err) - } - if got, err := f.repos.Uploads.GetByID(ctx, orphan.ID); err != nil || got != nil { - t.Fatalf("upload after cleanup = %#v err=%v, want deleted", got, err) - } - itemCount, err := f.db.NewSelect().Model((*storagereplacement.Item)(nil)). - Where("id = ?", item.ID).Count(ctx) - if err != nil { - t.Fatalf("count replacement item: %v", err) - } - if itemCount != 0 { - t.Fatalf("replacement item count = %d, want deleted after settlement", itemCount) - } - got, err := f.repos.Replacements.GetByID(ctx, replacement.ID) - if err != nil || got == nil || got.ItemsCopied != 0 || got.ItemsTotal != 1 { - t.Fatalf("replacement progress after late completion = %#v err=%v, want copied 0 of historical total 1", got, err) - } - progresses, err := f.repos.Replacements.ReplacementProgresses(ctx, []int64{replacement.ID}) - if err != nil { - t.Fatalf("ReplacementProgresses: %v", err) - } - progress := progresses[replacement.ID] - if progress.ItemsNoLongerNeeded != 1 || progress.ItemsProcessed != 1 || progress.Percent == nil || *progress.Percent != 100 { - t.Fatalf("progress after provenance deletion = %#v, want one no-longer-needed item at 100%%", progress) - } -} - -// The retirement gate is the last thing standing between a replacement and -// permanent data loss, so every predicate is checked independently and -// CompleteRetirement refuses on its own, whoever calls it. -func TestStorageReplacementRepo_RetirementGateBlocksEachUnsafeCondition(t *testing.T) { - f := newReplacementFixture(t, "replacement-gate", "01J000000000000000000RPL12") - ctx := context.Background() - row := f.authorize(t, "202") - target := f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, _, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 10, 5); err != nil { - t.Fatalf("SeedMigrationBatch: %v", err) - } - - // Outstanding migration work blocks retirement. - gate, err := f.repos.Replacements.EvaluateRetirementGate(ctx, row.ID, nil) - if err != nil { - t.Fatalf("EvaluateRetirementGate: %v", err) - } - if gate.Passed() || gate.WaitingItems != 1 { - t.Fatalf("gate = %#v, want it blocked by one outstanding item", gate) - } - if err := f.repos.Replacements.CompleteRetirement(ctx, row.ID, 9999); !errors.Is(err, storagereplacement.ErrPrematureComplete) { - t.Fatalf("CompleteRetirement with outstanding work = %v, want ErrPrematureComplete", err) - } - - var itemID int64 - if err := f.db.NewRaw(`SELECT id FROM storage_replacement_items WHERE replacement_id = ? LIMIT 1`, row.ID).Scan(ctx, &itemID); err != nil { - t.Fatalf("select replacement item: %v", err) - } - for _, status := range []storagereplacement.ItemStatus{ - storagereplacement.ItemStatusRetrying, - storagereplacement.ItemStatusWaitingSource, - storagereplacement.ItemStatusFailed, - } { - mustExec(t, f.db, `UPDATE storage_replacement_items SET status = ? WHERE id = ?`, status, itemID) - gate, err = f.repos.Replacements.EvaluateRetirementGate(ctx, row.ID, nil) - if err != nil { - t.Fatalf("EvaluateRetirementGate for %s: %v", status, err) - } - if gate.WaitingItems != 1 { - t.Fatalf("gate for %s = %#v, want the item to block retirement", status, gate) - } - } - mustExec(t, f.db, `UPDATE storage_replacement_items SET status = ?, claimed_at = NULL, lease_until = NULL WHERE id = ?`, - storagereplacement.ItemStatusCopied, itemID) - - // The content is not actually on the new provider yet. - gate, err = f.repos.Replacements.EvaluateRetirementGate(ctx, row.ID, nil) - if err != nil { - t.Fatalf("EvaluateRetirementGate: %v", err) - } - if gate.Passed() || gate.CoverageGaps != 1 { - t.Fatalf("gate = %#v, want it blocked by a coverage gap", gate) - } - - mustExec(t, f.db, `INSERT INTO storage_upload_copies (upload_id, copy_index, provider_id, piece_id, transfer_method, status, retrieval_url, storage_data_set_id, created_at, updated_at) - VALUES (?, 0, '202', '3002', ?, ?, 'https://target.example/piece', ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, - f.upload.ID, model.StorageCopyTransferMethodPeerPull, model.StorageUploadCopyStatusCommitted, target.ID) - var targetCopyID int64 - if err := f.db.NewRaw(`SELECT id FROM storage_upload_copies WHERE upload_id = ? AND storage_data_set_id = ?`, - f.upload.ID, target.ID).Scan(ctx, &targetCopyID); err != nil { - t.Fatalf("select target copy: %v", err) - } - mustExec(t, f.db, `UPDATE storage_replacement_items SET target_copy_id = ? WHERE id = ?`, targetCopyID, itemID) - mustExec(t, f.db, `UPDATE storage_upload_copies - SET status = ?, commit_attempt_id = 'attempt-retirement', commit_attempted_at = CURRENT_TIMESTAMP - WHERE id = ?`, model.StorageUploadCopyStatusCommitting, targetCopyID) - gate, err = f.repos.Replacements.EvaluateRetirementGate(ctx, row.ID, nil) - if err != nil { - t.Fatalf("EvaluateRetirementGate with confirmation attempt: %v", err) - } - if gate.ActiveAttempts != 1 || !slices.Contains(gate.Blockers, "confirmation_attempts") { - t.Fatalf("gate = %#v, want active replacement confirmation blocker", gate) - } - mustExec(t, f.db, `UPDATE storage_upload_copies - SET status = ?, commit_attempt_id = NULL, commit_attempted_at = NULL WHERE id = ?`, - model.StorageUploadCopyStatusCommitted, targetCopyID) - - // Everything is covered, but the service has not been terminated yet. - gate, err = f.repos.Replacements.EvaluateRetirementGate(ctx, row.ID, nil) - if err != nil { - t.Fatalf("EvaluateRetirementGate: %v", err) - } - if !gate.Passed() { - t.Fatalf("gate = %#v, want everything except the epoch satisfied", gate) - } - if err := f.repos.Replacements.CompleteRetirement(ctx, row.ID, 9999); !errors.Is(err, storagereplacement.ErrPrematureComplete) { - t.Fatalf("CompleteRetirement before termination = %v, want ErrPrematureComplete", err) - } - - if err := f.repos.Replacements.BeginRetirement(ctx, row.ID); err != nil { - t.Fatalf("BeginRetirement: %v", err) - } - if err := f.repos.Replacements.RecordTerminationEpoch(ctx, repository.RecordTerminationEpochInput{ - ReplacementID: row.ID, TxHash: "0xterminate", Epoch: 5000, - }); err != nil { - t.Fatalf("RecordTerminationEpoch: %v", err) - } - - // The chain has not reached the end of term. - if err := f.repos.Replacements.CompleteRetirement(ctx, row.ID, 4999); !errors.Is(err, storagereplacement.ErrPrematureComplete) { - t.Fatalf("CompleteRetirement before the epoch = %v, want ErrPrematureComplete", err) - } - - // Another upload still writing to the source blocks retirement even now. - inFlight := startCopyHealthUpload(t, f.repos, f.bucket.ID, "01J000000000000000000RPL13", 10, "in-flight-checksum", 1) - mustExec(t, f.db, `INSERT INTO storage_upload_copies (upload_id, copy_index, provider_id, transfer_method, status, storage_data_set_id, created_at, updated_at) - VALUES (?, 0, '101', ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, - inFlight.ID, model.StorageCopyTransferMethodPeerPull, model.StorageUploadCopyStatusCommitting, f.source.ID) - if err := f.repos.Replacements.CompleteRetirement(ctx, row.ID, 5000); !errors.Is(err, storagereplacement.ErrPrematureComplete) { - t.Fatalf("CompleteRetirement with an in-flight source write = %v, want ErrPrematureComplete", err) - } - mustExec(t, f.db, `DELETE FROM storage_upload_copies WHERE upload_id = ?`, inFlight.ID) - - if err := f.repos.Replacements.CompleteRetirement(ctx, row.ID, 5000); err != nil { - t.Fatalf("CompleteRetirement: %v", err) - } - source, err := f.repos.Uploads.GetDataSetBindingByID(ctx, f.source.ID) - if err != nil || source == nil || source.Status != model.StorageDataSetStatusRetired { - t.Fatalf("source = %#v err=%v, want retired", source, err) - } - done, err := f.repos.Replacements.GetByID(ctx, row.ID) - if err != nil || done == nil || done.Status != storagereplacement.StatusCompleted { - t.Fatalf("replacement = %#v err=%v, want completed", done, err) - } - // Completing twice is harmless, which keeps a retried task safe. - if err := f.repos.Replacements.CompleteRetirement(ctx, row.ID, 5000); err != nil { - t.Fatalf("second CompleteRetirement: %v", err) - } -} - -// An upload that is still in flight when seeding runs must still get migrated. -// Judging eligibility at seeding time skipped it forever, and the retirement -// coverage gate then blocked on content no item was ever created for. -func TestStorageReplacementRepo_SeedingCoversUploadsStillInFlight(t *testing.T) { - f := newReplacementFixture(t, "replacement-inflight", "01J000000000000000000RPL14") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - - // A second upload starts before activation and has not committed yet. - inFlightVersion := newObjectVersion(f.bucket.ID, "later.txt", "01J000000000000000000RPL15", 10) - inFlightVersion.Checksum = "in-flight-checksum" - if _, err := f.repos.Objects.CreateVersionAndSetCurrent(ctx, inFlightVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - inFlight := startCopyHealthUpload(t, f.repos, f.bucket.ID, inFlightVersion.VersionID, 10, inFlightVersion.Checksum, 1) - if err := f.repos.Uploads.CreateUploadCopiesForBindings(ctx, inFlight.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: f.source.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: f.source.ProviderID, - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, _, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 10, 5); err != nil { - t.Fatalf("SeedMigrationBatch: %v", err) - } - - // The in-flight upload commits to the retiring generation afterwards. - inFlightCopy, err := f.repos.Uploads.GetUploadCopyForDataSet(ctx, inFlight.ID, f.source.ID) - if err != nil || inFlightCopy == nil { - t.Fatalf("GetUploadCopyForDataSet = %#v err=%v", inFlightCopy, err) - } - if err := f.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - StorageUploadCopyID: inFlightCopy.ID, - UploadID: inFlight.ID, - CopyIndex: 0, - PieceCID: "bafk2bzacestorhealth", - PieceID: onChainIDPtr(t, "7001"), - RetrievalURL: "https://source.example/in-flight", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - - var seeded int - if err = f.db.NewRaw(`SELECT COUNT(*) FROM storage_replacement_items WHERE replacement_id = ? AND upload_id = ?`, - row.ID, inFlight.ID).Scan(ctx, &seeded); err != nil { - t.Fatalf("count seeded items: %v", err) - } - if seeded != 1 { - t.Fatal("the upload that was still in flight during seeding was never given migration work") - } - - // The retirement gate must therefore see it as owed work, not as a - // permanent coverage gap with no item behind it. - gate, err := f.repos.Replacements.EvaluateRetirementGate(ctx, row.ID, nil) - if err != nil { - t.Fatalf("EvaluateRetirementGate: %v", err) - } - if gate.WaitingItems == 0 { - t.Fatalf("gate = %#v, want the in-flight upload counted as outstanding work", gate) - } -} - -// An item that can never be satisfied must reach a terminal status. Leaving it -// executable made the coordinator pick the same item forever and held the -// retirement gate open. -func TestStorageReplacementRepo_UnsatisfiableItemsSettleTerminally(t *testing.T) { - f := newReplacementFixture(t, "replacement-settle", "01J000000000000000000RPL16") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, _, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 10, 5); err != nil { - t.Fatalf("SeedMigrationBatch: %v", err) - } - var itemID int64 - if err := f.db.NewRaw(`SELECT id FROM storage_replacement_items WHERE replacement_id = ? LIMIT 1`, row.ID).Scan(ctx, &itemID); err != nil { - t.Fatalf("select replacement item: %v", err) - } - item := claimSpecificReplacementItem(t, f, row.ID, itemID) - - // The content stops being referenced before the item runs. - mustExec(t, f.db, `DELETE FROM object_versions WHERE version_id = ?`, f.version.VersionID) - - if _, err := f.repos.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ - ReplacementID: row.ID, - ItemID: item.ID, - ItemClaimedAt: *item.ClaimedAt, - }); !errors.Is(err, storagereplacement.ErrItemCancelled) { - t.Fatalf("AcquireItem = %v, want ErrItemCancelled", err) - } - - // The decisive part: the item must not come back. - next, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil { - t.Fatalf("ClaimReadyReplacementItem: %v", err) - } - if next != nil { - t.Fatalf("item %d is still executable after being cancelled, so the coordinator would loop on it", next.ID) - } - gate, err := f.repos.Replacements.EvaluateRetirementGate(ctx, row.ID, nil) - if err != nil { - t.Fatalf("EvaluateRetirementGate: %v", err) - } - if gate.WaitingItems != 0 { - t.Fatalf("gate = %#v, want no outstanding items once they are settled", gate) - } -} - -func TestStorageReplacementRepo_CancelClaimOnlyClearsUnattemptedReservation(t *testing.T) { - f := newReplacementFixture(t, "replacement-cancelled-fifo", "01J000000000000000000RPL30") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, _, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 10, 5); err != nil { - t.Fatalf("SeedMigrationBatch: %v", err) - } - item, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || item == nil || item.ClaimedAt == nil { - t.Fatalf("claim = %#v err=%v", item, err) - } - snapshot, err := f.repos.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ - ReplacementID: row.ID, ItemID: item.ID, ItemClaimedAt: *item.ClaimedAt, - }) - if err != nil || snapshot == nil { - t.Fatalf("AcquireItem: snapshot=%#v err=%v", snapshot, err) - } - copyRow, err := f.repos.Replacements.AttachTargetCopy(ctx, repository.AttachReplacementTargetCopyInput{ - ReplacementID: row.ID, ItemID: item.ID, UploadID: snapshot.Upload.ID, ItemClaimedAt: *item.ClaimedAt, - }) - if err != nil || copyRow == nil || snapshot.Upload.PieceCID == nil { - t.Fatalf("AttachTargetCopy: copy=%#v err=%v", copyRow, err) - } - if err := f.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - StorageUploadCopyID: copyRow.ID, RequireEligibleCopy: true, - UploadID: copyRow.UploadID, CopyIndex: copyRow.CopyIndex, - PieceCID: *snapshot.Upload.PieceCID, RetrievalURL: "https://target.example/piece", CommitExtraDataHex: "abcd", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - reservation, err := f.repos.Uploads.ReserveCommitAttempt(ctx, storagecommit.ReserveInput{ - Copy: storagecommit.CopyIdentity{ - StorageUploadCopyID: copyRow.ID, UploadID: copyRow.UploadID, CopyIndex: copyRow.CopyIndex, - StorageDataSetID: row.TargetDataSetID, RequireEligibleCopy: true, - }, - AttemptID: "cancelled-fifo", - }) - if err != nil || reservation.State != storagecommit.ReservationAcquired || reservation.Copy.CommitAttemptID == nil || - reservation.Copy.CommitAttemptedAt != nil { - t.Fatalf("unattempted reservation = %#v err=%v", reservation, err) - } - if err := f.repos.Replacements.CancelReplacementItemClaim(ctx, storagereplacement.ClaimToken{ - ItemID: item.ID, ClaimedAt: *item.ClaimedAt, - }); err != nil { - t.Fatalf("CancelReplacementItemClaim: %v", err) - } - var cancelled storagereplacement.Item - if err := f.db.NewSelect().Model(&cancelled).Where("id = ?", item.ID).Scan(ctx); err != nil { - t.Fatalf("reload cancelled item: %v", err) - } - if cancelled.Status != storagereplacement.ItemStatusCancelled || cancelled.ClaimedAt != nil || cancelled.LeaseUntil != nil { - t.Fatalf("cancelled item = %#v", cancelled) - } - persisted, err := f.repos.Uploads.GetUploadCopyByID(ctx, copyRow.ID) - if err != nil || persisted == nil || persisted.CommitReadyAt != nil || persisted.CommitExtraDataHex != nil || - persisted.CommitAttemptID != nil || persisted.CommitAttemptedAt != nil { - t.Fatalf("cancelled FIFO copy = %#v err=%v", persisted, err) - } - - pieceID := onChainID(t, "3002") - if err := f.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - StorageUploadCopyID: copyRow.ID, - UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, - PieceCID: *snapshot.Upload.PieceCID, - PieceID: &pieceID, - RetrievalURL: "https://target.example/piece", - CommitExtraDataHex: "abcd", - CommitTransactionID: "0xcommitted", - CommitConfirmedTransactionID: "0xcommitted", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - claimedAt := time.Now() - mustExec(t, f.db, `UPDATE storage_replacement_items - SET status = ?, claimed_at = ?, lease_until = ?, updated_at = ? - WHERE id = ?`, storagereplacement.ItemStatusRunning, claimedAt, claimedAt.Add(time.Minute), claimedAt, item.ID) - if err := f.repos.Replacements.CancelReplacementItemClaim(ctx, storagereplacement.ClaimToken{ - ItemID: item.ID, ClaimedAt: claimedAt, - }); err != nil { - t.Fatalf("CancelReplacementItemClaim for committed copy: %v", err) - } - persisted, err = f.repos.Uploads.GetUploadCopyByID(ctx, copyRow.ID) - if err != nil || persisted == nil || persisted.Status != model.StorageUploadCopyStatusCommitted || - persisted.CommitExtraDataHex == nil || *persisted.CommitExtraDataHex != "abcd" || - persisted.CommitTransactionID == nil || *persisted.CommitTransactionID != "0xcommitted" || - persisted.CommitConfirmedTransactionID == nil || *persisted.CommitConfirmedTransactionID != "0xcommitted" { - t.Fatalf("committed copy evidence after claim cancellation = %#v err=%v", persisted, err) - } -} - -// A rejected attempt on the retiring generation must reset that generation, -// not whichever one currently owns the slot. -func TestStorageReplacementRepo_ResetCommitAttemptTargetsTheRecordedCopy(t *testing.T) { - f := newReplacementFixture(t, "replacement-reset", "01J000000000000000000RPL17") - ctx := context.Background() - sourceCopy, err := f.repos.Uploads.GetUploadCopyForDataSet(ctx, f.upload.ID, f.source.ID) - if err != nil || sourceCopy == nil { - t.Fatalf("GetUploadCopyForDataSet = %#v err=%v", sourceCopy, err) - } - mustExec(t, f.db, `UPDATE storage_upload_copies SET status = ? WHERE id = ?`, - model.StorageUploadCopyStatusPieceReady, sourceCopy.ID) - identity := storagecommit.CopyIdentity{ - StorageUploadCopyID: sourceCopy.ID, UploadID: sourceCopy.UploadID, - CopyIndex: sourceCopy.CopyIndex, StorageDataSetID: f.source.ID, - } - seedRepositoryCommitAttempt(t, f.repos, *sourceCopy, "retiring-rejected", "abcd", "0xrejected") - - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - - if err := f.repos.Uploads.ResetCommitAttempt(ctx, storagecommit.ResetInput{ - Copy: identity, AttemptID: "retiring-rejected", LastError: "provider rejected the commit", - }); err != nil { - t.Fatalf("ResetCommitAttempt: %v", err) - } - got, err := f.repos.Uploads.GetUploadCopyByID(ctx, sourceCopy.ID) - if err != nil || got == nil || got.Status != model.StorageUploadCopyStatusPieceReady { - t.Fatalf("retiring copy = %#v err=%v, want it reset to piece_ready", got, err) - } -} - -// A target a later confirmation abandoned keeps costing money until its own -// service ends. Retiring it must never touch the source, which the successor -// still depends on. -func TestStorageReplacementRepo_AbandonedTargetRetiresWithoutTouchingTheSource(t *testing.T) { - f := newReplacementFixture(t, "replacement-abandoned", "01J000000000000000000RPL18") - ctx := context.Background() - first := f.authorize(t, "202") - f.readyTarget(t, first, "2002") - second := f.authorize(t, "303") - - superseded, err := f.repos.Replacements.GetByID(ctx, first.ID) - if err != nil || superseded == nil || superseded.Status != storagereplacement.StatusSuperseded { - t.Fatalf("first replacement = %#v err=%v, want superseded", superseded, err) - } - candidates, err := f.repos.Replacements.ListSupersededCleanupCandidates(ctx, 0, 10) - if err != nil { - t.Fatalf("ListSupersededCleanupCandidates: %v", err) - } - if len(candidates) != 1 || candidates[0].ID != first.ID { - t.Fatalf("cleanup candidates = %#v, want the superseded replacement", candidates) - } - - // The abandoned target holds nothing anyone depends on. - sole, err := f.repos.Replacements.CountAbandonedTargetSoleCopies(ctx, first.TargetDataSetID) - if err != nil { - t.Fatalf("CountAbandonedTargetSoleCopies: %v", err) - } - if sole != 0 { - t.Fatalf("sole copies = %d, want none on an unused target", sole) - } - reservedCopy := seedCommitCopies(t, f.db, f.bucket.ID, first.TargetDataSetID, 1)[0] - mustExec(t, f.db, `UPDATE storage_upload_copies SET commit_attempt_id = ? WHERE id = ?`, "abandoned-reservation", reservedCopy.ID) - if err := f.repos.Replacements.RetireAbandonedTarget(ctx, first.ID); !errors.Is(err, storagereplacement.ErrPrematureComplete) { - t.Fatalf("RetireAbandonedTarget with reservation = %v, want premature-complete", err) - } - mustExec(t, f.db, `UPDATE storage_upload_copies SET commit_attempt_id = NULL WHERE id = ?`, reservedCopy.ID) - if err := f.repos.Replacements.RetireAbandonedTarget(ctx, first.ID); err != nil { - t.Fatalf("RetireAbandonedTarget: %v", err) - } - - abandoned, err := f.repos.Uploads.GetDataSetBindingByID(ctx, first.TargetDataSetID) - if err != nil || abandoned == nil || abandoned.Status != model.StorageDataSetStatusRetired { - t.Fatalf("abandoned target = %#v err=%v, want retired", abandoned, err) - } - source, err := f.repos.Uploads.GetDataSetBindingByID(ctx, f.source.ID) - if err != nil || source == nil || source.Status == model.StorageDataSetStatusRetired || !source.IsCurrent { - t.Fatalf("source = %#v err=%v, want it untouched and still current", source, err) - } - // The replacement record stays superseded; cleanup never rewrites it. - got, err := f.repos.Replacements.GetByID(ctx, first.ID) - if err != nil || got == nil || got.Status != storagereplacement.StatusSuperseded { - t.Fatalf("replacement = %#v err=%v, want it left superseded", got, err) - } - if second.ID == first.ID { - t.Fatal("the later confirmation reused the superseded record") - } -} - -func claimSpecificReplacementItem( - t *testing.T, - f *replacementFixture, - replacementID int64, - itemID int64, -) *storagereplacement.Item { - t.Helper() - ctx := context.Background() - now := time.Now() - mustExec(t, f.db, `UPDATE storage_replacement_items SET scheduled_at = ? WHERE replacement_id = ? AND id <> ?`, - now.Add(time.Hour), replacementID, itemID) - mustExec(t, f.db, `UPDATE storage_replacement_items SET scheduled_at = ? WHERE id = ?`, now, itemID) - item, err := f.repos.Replacements.ClaimReadyReplacementItem(ctx, time.Minute) - if err != nil || item == nil || item.ID != itemID || item.ClaimedAt == nil { - t.Fatalf("ClaimReadyReplacementItem = %#v err=%v, want item %d", item, err, itemID) - } - return item -} - -// Content the retiring generation is still writing is not "never stored". A -// cancelled item never migrates, the write then commits, and the coverage gate -// blocks forever on content no item exists for. -func TestStorageReplacementRepo_InFlightSourceCopyIsParkedNotCancelled(t *testing.T) { - f := newReplacementFixture(t, "replacement-inflight-acquire", "01J000000000000000000RPL19") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - - inFlightVersion := newObjectVersion(f.bucket.ID, "pending.txt", "01J000000000000000000RPL20", 10) - inFlightVersion.Checksum = "pending-checksum" - if _, err := f.repos.Objects.CreateVersionAndSetCurrent(ctx, inFlightVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - inFlight := startCopyHealthUpload(t, f.repos, f.bucket.ID, inFlightVersion.VersionID, 10, inFlightVersion.Checksum, 1) - if err := f.repos.Uploads.CreateUploadCopiesForBindings(ctx, inFlight.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: f.source.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: f.source.ProviderID, - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, _, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 10, 5); err != nil { - t.Fatalf("SeedMigrationBatch: %v", err) - } - - var itemID int64 - if err := f.db.NewRaw(`SELECT id FROM storage_replacement_items WHERE replacement_id = ? AND upload_id = ?`, - row.ID, inFlight.ID).Scan(ctx, &itemID); err != nil { - t.Fatalf("select in-flight item: %v", err) - } - item := claimSpecificReplacementItem(t, f, row.ID, itemID) - if _, err := f.repos.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ - ReplacementID: row.ID, - ItemID: itemID, - ItemClaimedAt: *item.ClaimedAt, - }); !errors.Is(err, storagereplacement.ErrItemDeferred) { - t.Fatalf("AcquireItem while the source is still writing = %v, want ErrItemDeferred", err) - } - - // Parked, not cancelled: it must come back once the source finishes. - var status storagereplacement.ItemStatus - if err := f.db.NewRaw(`SELECT status FROM storage_replacement_items WHERE id = ?`, itemID).Scan(ctx, &status); err != nil { - t.Fatalf("select item status: %v", err) - } - if status != storagereplacement.ItemStatusWaitingSource { - t.Fatalf("item status = %s, want waiting_source so the coordinator revisits it", status) - } - gate, err := f.repos.Replacements.EvaluateRetirementGate(ctx, row.ID, nil) - if err != nil { - t.Fatalf("EvaluateRetirementGate: %v", err) - } - if gate.WaitingItems == 0 { - t.Fatalf("gate = %#v, want the parked item to keep retirement open", gate) - } -} - -// An upload the retiring generation never stored, and never will, owes the -// target nothing. -func TestStorageReplacementRepo_ItemWithNoSourceCopyIsCancelled(t *testing.T) { - f := newReplacementFixture(t, "replacement-nosource", "01J000000000000000000RPL21") - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - - elsewhereVersion := newObjectVersion(f.bucket.ID, "elsewhere.txt", "01J000000000000000000RPL22", 10) - elsewhereVersion.Checksum = "elsewhere-checksum" - if _, err := f.repos.Objects.CreateVersionAndSetCurrent(ctx, elsewhereVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - elsewhere := startCopyHealthUpload(t, f.repos, f.bucket.ID, elsewhereVersion.VersionID, 10, elsewhereVersion.Checksum, 1) - - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if _, _, err := f.repos.Replacements.SeedMigrationBatchWithBudget(ctx, row.ID, 10, 5); err != nil { - t.Fatalf("SeedMigrationBatch: %v", err) - } - var itemID int64 - if err := f.db.NewRaw(`SELECT id FROM storage_replacement_items WHERE replacement_id = ? AND upload_id = ?`, - row.ID, elsewhere.ID).Scan(ctx, &itemID); err != nil { - t.Fatalf("select item: %v", err) - } - item := claimSpecificReplacementItem(t, f, row.ID, itemID) - if _, err := f.repos.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ - ReplacementID: row.ID, - ItemID: itemID, - ItemClaimedAt: *item.ClaimedAt, - }); !errors.Is(err, storagereplacement.ErrItemCancelled) { - t.Fatalf("AcquireItem = %v, want ErrItemCancelled", err) - } - var status storagereplacement.ItemStatus - if err := f.db.NewRaw(`SELECT status FROM storage_replacement_items WHERE id = ?`, itemID).Scan(ctx, &status); err != nil { - t.Fatalf("select item status: %v", err) - } - if status != storagereplacement.ItemStatusCancelled { - t.Fatalf("item status = %s, want cancelled", status) - } -} - -// A generation some unfinished replacement is migrating into cannot become a -// source: that replacement needs it to keep owning the slot. -func TestStorageReplacementRepo_AuthorizeRejectsReplacingALiveTarget(t *testing.T) { - f := newReplacementFixture(t, "replacement-chain", "01J000000000000000000RPL23") - ctx := context.Background() - first := f.authorize(t, "202") - target := f.readyTarget(t, first, "2002") - if err := f.repos.Replacements.Activate(ctx, first.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if err := f.repos.Replacements.MarkFailed(ctx, first.ID, nil, "migration exhausted"); err != nil { - t.Fatalf("MarkFailed: %v", err) - } - - if _, _, err := f.repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ - BucketID: f.bucket.ID, - SourceDataSetID: target.ID, - SelectionMode: storagereplacement.SelectionModeManual, - TargetProviderID: onChainID(t, "404"), - ClientRequestID: "replace-live-target", - MaxRetries: 5, - }); !errors.Is(err, storagereplacement.ErrActiveReplacement) { - t.Fatalf("replacing a live replacement's target = %v, want ErrActiveReplacement", err) - } -} - -// After an activation a failure bound to the retiring generation must land on -// that generation. Resolving through the slot would either mark the replacement -// copy failed or update nothing, leaving the original stuck mid-transfer and -// holding retirement open. -func TestStorageUploadRepo_CopyFailureFollowsTheRecordedCopy(t *testing.T) { - f := newReplacementFixture(t, "replacement-failure", "01J000000000000000000RPL24") - ctx := context.Background() - sourceCopy, err := f.repos.Uploads.GetUploadCopyForDataSet(ctx, f.upload.ID, f.source.ID) - if err != nil || sourceCopy == nil { - t.Fatalf("GetUploadCopyForDataSet = %#v err=%v", sourceCopy, err) - } - mustExec(t, f.db, `UPDATE storage_upload_copies SET status = ? WHERE id = ?`, - model.StorageUploadCopyStatusPieceReady, sourceCopy.ID) - - row := f.authorize(t, "202") - target := f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - if err := f.repos.Uploads.CreateUploadCopiesForBindings(ctx, f.upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: target.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodPeerPull, - ProviderID: target.ProviderID, - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings target: %v", err) - } - targetCopy, err := f.repos.Uploads.GetUploadCopyForDataSet(ctx, f.upload.ID, target.ID) - if err != nil || targetCopy == nil { - t.Fatalf("GetUploadCopyForDataSet target = %#v err=%v", targetCopy, err) - } - - if err := f.repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{ - StorageUploadCopyID: sourceCopy.ID, - UploadID: f.upload.ID, - CopyIndex: 0, - LastError: "ingress store: provider rejected the piece", - }); err != nil { - t.Fatalf("MarkUploadCopyFailed: %v", err) - } - gotSource, err := f.repos.Uploads.GetUploadCopyByID(ctx, sourceCopy.ID) - if err != nil || gotSource == nil || gotSource.Status != model.StorageUploadCopyStatusFailed { - t.Fatalf("retiring copy = %#v err=%v, want failed", gotSource, err) - } - gotTarget, err := f.repos.Uploads.GetUploadCopyByID(ctx, targetCopy.ID) - if err != nil || gotTarget == nil || gotTarget.Status != model.StorageUploadCopyStatusPending { - t.Fatalf("replacement copy = %#v err=%v, want it untouched", gotTarget, err) - } -} - -// The dedicated retry exists to resume work the worker gave up on, and the -// worker gives up by driving the coordinator task to a terminal status: -// migration exhausts its retries, cleanup fails outright. Moving the record -// back to a working status without reviving that task leaves the replacement -// permanently stuck — displayed as in progress, with the retry hidden because -// the status is no longer retryable, and nothing queued to make progress. -func TestStorageReplacementRepo_RetryMakesTheCoordinatorClaimableAgain(t *testing.T) { - for _, tc := range []struct { - name string - markStuck func(t *testing.T, f *replacementFixture, row *storagereplacement.Replacement) - taskStatus model.TaskStatus - taskType model.TaskType - key func(int64) string - }{ - { - name: "migration exhausted its retries", - markStuck: func(t *testing.T, f *replacementFixture, row *storagereplacement.Replacement) { - if err := f.repos.Replacements.MarkFailed(context.Background(), row.ID, nil, "copy replacement item: exhausted"); err != nil { - t.Fatalf("MarkFailed: %v", err) - } - }, - taskStatus: model.TaskStatusExhausted, - taskType: model.TaskTypeUpload, - key: storagereplacement.MigrateTaskKey, - }, - { - name: "cleanup could not end the service", - markStuck: func(t *testing.T, f *replacementFixture, row *storagereplacement.Replacement) { - if err := f.repos.Replacements.BeginRetirement(context.Background(), row.ID); err != nil { - t.Fatalf("BeginRetirement: %v", err) - } - if err := f.repos.Replacements.MarkCleanupAttention(context.Background(), row.ID, "payment debt"); err != nil { - t.Fatalf("MarkCleanupAttention: %v", err) - } - }, - taskStatus: model.TaskStatusFailed, - taskType: model.TaskTypeStorageCleanup, - key: storagereplacement.RetireTaskKey, - }, - } { - t.Run(tc.name, func(t *testing.T) { - f := newReplacementFixture(t, "retry-revives-"+model.NewVersionID()[:8], model.NewVersionID()) - ctx := context.Background() - row := f.authorize(t, "202") - f.readyTarget(t, row, "2002") - if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - tc.markStuck(t, f, row) - - // Put the coordinator where the worker leaves it on the way in. - key := tc.key(row.ID) - if _, err := f.repos.Tasks.EnsureRecurring(ctx, storagereplacement.NewRetireTask(row.ID, f.bucket.ID, 5, time.Now())); err != nil { - t.Fatalf("seed retire coordinator: %v", err) - } - if _, err := f.db.NewUpdate(). - Model((*model.Task)(nil)). - Set("status = ?", tc.taskStatus). - Set("retry_count = 5"). - Where("idempotency_key = ?", key). - Exec(ctx); err != nil { - t.Fatalf("mark coordinator %s: %v", tc.taskStatus, err) - } - - if _, err := f.repos.Replacements.Retry(ctx, repository.RetryReplacementInput{ReplacementID: row.ID, MaxRetries: 5, ItemMaxRetries: 5}); err != nil { - t.Fatalf("Retry: %v", err) - } - - // The only assertion that matters: a worker can pick the work up. - claimed, err := f.repos.Tasks.ClaimReady(ctx, tc.taskType, time.Minute) - if err != nil { - t.Fatalf("ClaimReady %s: %v", tc.taskType, err) - } - if claimed == nil || claimed.IdempotencyKey != key { - t.Fatalf("claimed = %#v, want the resumed coordinator %s", claimed, key) - } - if claimed.RetryCount != 0 { - t.Fatalf("retry count = %d, want the operator's retry to restore the budget", claimed.RetryCount) - } - }) - } -} - -// Automatic recurrence must keep its own rule: a coordinator that gave up is -// not restarted just because recovery ran again. -func TestTaskRepo_StartupRecoveryDoesNotRestartAbandonedCoordinators(t *testing.T) { - f := newReplacementFixture(t, "startup-vs-exhausted", model.NewVersionID()) - ctx := context.Background() - row := f.authorize(t, "202") - key := storagereplacement.MigrateTaskKey(row.ID) - if _, err := f.db.NewUpdate(). - Model((*model.Task)(nil)). - Set("status = ?", model.TaskStatusExhausted). - Where("idempotency_key = ?", key). - Exec(ctx); err != nil { - t.Fatalf("mark coordinator exhausted: %v", err) - } - if _, err := f.repos.Tasks.EnsureRecurring(ctx, storagereplacement.NewMigrateTask(row.ID, f.bucket.ID, "", 5, time.Now())); err != nil { - t.Fatalf("EnsureRecurring: %v", err) - } - claimed, err := f.repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil { - t.Fatalf("ClaimReady: %v", err) - } - if claimed != nil { - t.Fatal("startup recovery restarted a coordinator that had given up and needs an operator") - } -} - -// Abandoned-target cleanup cannot use the Data Sets retry: the replacement is -// superseded. ResumeCoordinator is the only way a failed leftover terminator -// becomes claimable again after a restart. -func TestTaskRepo_ResumeCoordinatorRevivesAbandonedTargetCleanup(t *testing.T) { - f := newReplacementFixture(t, "resume-abandoned-cleanup", model.NewVersionID()) - ctx := context.Background() - task := storagereplacement.NewAbandonedTargetTask(11, f.bucket.ID, 5, time.Now()) - if err := f.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create: %v", err) - } - if _, err := f.db.NewUpdate(). - Model((*model.Task)(nil)). - Set("status = ?", model.TaskStatusFailed). - Set("retry_count = 5"). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - t.Fatalf("mark failed: %v", err) - } - if _, err := f.repos.Tasks.EnsureRecurring(ctx, storagereplacement.NewAbandonedTargetTask(11, f.bucket.ID, 5, time.Now())); err != nil { - t.Fatalf("EnsureRecurring: %v", err) - } - claimed, err := f.repos.Tasks.ClaimReady(ctx, model.TaskTypeStorageCleanup, time.Minute) - if err != nil { - t.Fatalf("ClaimReady after EnsureRecurring: %v", err) - } - if claimed != nil { - t.Fatal("automatic recurrence revived abandoned cleanup that had failed") - } - if _, err := f.repos.Tasks.ResumeCoordinator(ctx, storagereplacement.NewAbandonedTargetTask(11, f.bucket.ID, 5, time.Now())); err != nil { - t.Fatalf("ResumeCoordinator: %v", err) - } - claimed, err = f.repos.Tasks.ClaimReady(ctx, model.TaskTypeStorageCleanup, time.Minute) - if err != nil { - t.Fatalf("ClaimReady after ResumeCoordinator: %v", err) - } - if claimed == nil || claimed.IdempotencyKey != storagereplacement.AbandonedTargetTaskKey(11) { - t.Fatalf("claimed = %#v, want the abandoned-target coordinator", claimed) - } -} diff --git a/internal/db/repository/storage_task_fence_repo.go b/internal/db/repository/storage_task_fence_repo.go new file mode 100644 index 0000000..086c430 --- /dev/null +++ b/internal/db/repository/storage_task_fence_repo.go @@ -0,0 +1,310 @@ +package repository + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagepull" + "github.com/uptrace/bun" +) + +func (r *BunStorageContentRepo) GetLiveVersionForUpload(ctx context.Context, contentID int64) (*model.ObjectVersion, error) { + upload, err := r.GetByID(ctx, contentID) + if err != nil || upload == nil { + return nil, err + } + return selectLiveObjectVersionForStorageContent(ctx, r.db, upload, nil) +} + +func (r *BunStorageContentRepo) BindDataSetEnsureTask(ctx context.Context, dataSetID, taskID int64) error { + return r.bindDataSetTask(ctx, dataSetID, dataSetEnsureFence, 0, taskID) +} + +func (r *BunStorageContentRepo) AuthorizeDataSetEnsureTask(ctx context.Context, dataSetID, taskID int64) (*model.StorageDataSet, error) { + return r.authorizeDataSetTask(ctx, dataSetID, dataSetEnsureFence, 0, taskID) +} + +func (r *BunStorageContentRepo) CompleteDataSetEnsureTask(ctx context.Context, dataSetID, taskID int64) error { + return r.completeDataSetTask(ctx, dataSetID, dataSetEnsureFence, 0, taskID) +} + +func (r *BunStorageContentRepo) NextCopyWorkGeneration(ctx context.Context, copyID int64) (int64, error) { + var generation int64 + err := r.db.NewSelect(). + Model((*model.StorageCopy)(nil)). + ColumnExpr("work_generation + 1"). + Where("id = ? AND active_task_id IS NULL", copyID). + Scan(ctx, &generation) + if errors.Is(err, sql.ErrNoRows) { + return 0, ErrConflict + } + if err != nil { + return 0, fmt.Errorf("reading copy work generation: %w", err) + } + return generation, nil +} + +func (r *BunStorageContentRepo) BindCopyTask(ctx context.Context, copyID, generation, taskID int64) error { + if copyID < 1 || generation < 1 || taskID < 1 { + return ErrInvalidInput + } + result, err := r.db.NewUpdate(). + Model((*model.StorageCopy)(nil)). + Set("work_generation = ?", generation). + Set("active_task_id = ?", taskID). + Set("updated_at = ?", time.Now()). + Where("id = ? AND work_generation = ? AND active_task_id IS NULL", copyID, generation-1). + Exec(ctx) + return requireTaskFenceRows(result, err, "binding storage copy task") +} + +func (r *BunStorageContentRepo) AuthorizeCopyTask(ctx context.Context, copyID, generation, taskID, claimGeneration int64) (*model.StorageCopy, error) { + copyRow := new(model.StorageCopy) + q := r.db.NewSelect().Model(copyRow) + projectActiveCommitAttempt(q, "storage_copy") + err := q. + Join("JOIN tasks AS copy_task ON copy_task.id = storage_copy.active_task_id"). + Where("storage_copy.id = ? AND storage_copy.work_generation = ? AND storage_copy.active_task_id = ?", copyID, generation, taskID). + Where("copy_task.status = ? AND copy_task.claim_generation = ? AND copy_task.lease_until > ?", model.TaskStatusRunning, claimGeneration, time.Now()). + Scan(ctx) + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrConflict + } + if err != nil { + return nil, fmt.Errorf("authorizing storage copy task: %w", err) + } + return copyRow, nil +} + +// ReservePullRequest records a provider-side copy request before it is sent. +// The attempt lives in its own ledger row, so a crash between writing it and +// sending it leaves a fully identified operation recovery can observe instead +// of nullable columns that may be half written. +func (r *BunStorageContentRepo) ReservePullRequest(ctx context.Context, input ReservePullRequestInput) error { + if input.CopyID < 1 || input.Generation < 1 || input.TaskID < 1 || + input.AttemptID == "" || input.SourcePieceCID == "" || + input.SourceProviderID.IsZero() || input.SourceDataSetID.IsZero() || input.SourcePieceID.IsZero() || + input.SourceRetrievalURL == "" || input.CommitExtraDataHex == "" { + return ErrInvalidInput + } + return r.runMaybeTx(ctx, func(db bun.IDB) error { + now := time.Now() + copyRow := new(model.StorageCopy) + if err := db.NewSelect(). + Model(copyRow). + Column("content_id", "storage_data_set_id"). + Where("id = ? AND work_generation = ? AND active_task_id = ?", input.CopyID, input.Generation, input.TaskID). + Scan(ctx); err != nil { + if err == sql.ErrNoRows { + return fmt.Errorf("reserving storage pull request: %w", ErrConflict) + } + return fmt.Errorf("reserving storage pull request: %w", err) + } + // The extra data is presigned for this target and is reused across + // recovery, so it stays on the copy where the commit path reads it. + if _, err := db.NewUpdate(). + Model((*model.StorageCopy)(nil)). + Set("commit_extra_data_hex = COALESCE(commit_extra_data_hex, ?)", input.CommitExtraDataHex). + Set("updated_at = ?", now). + Where("id = ? AND work_generation = ? AND active_task_id = ?", input.CopyID, input.Generation, input.TaskID). + Where("commit_extra_data_hex IS NULL OR commit_extra_data_hex = ?", input.CommitExtraDataHex). + Exec(ctx); err != nil { + return fmt.Errorf("reserving storage pull commit evidence: %w", err) + } + + existing := new(storagepull.Attempt) + err := db.NewSelect(). + Model(existing). + Where("content_id = ? AND storage_data_set_id = ?", copyRow.ContentID, copyRow.StorageDataSetID). + Where("resolved_at IS NULL"). + Scan(ctx) + switch { + case err == nil: + // Recovery re-reserving the same attempt is a no-op; a different one + // would mean two live requests for one copy. + if existing.AttemptID != input.AttemptID { + return fmt.Errorf("reserving storage pull request: %w", ErrConflict) + } + return nil + case err != sql.ErrNoRows: + return fmt.Errorf("loading unresolved storage pull attempt: %w", err) + } + + attempt := &storagepull.Attempt{ + AttemptID: input.AttemptID, ContentID: copyRow.ContentID, + StorageDataSetID: copyRow.StorageDataSetID, Status: storagepull.AttemptStatusAttempted, + SourceProviderID: input.SourceProviderID, + SourceDataSetID: input.SourceDataSetID, SourcePieceID: input.SourcePieceID, + SourcePieceCID: input.SourcePieceCID, SourceRetrievalURL: input.SourceRetrievalURL, + AttemptedAt: now, CreatedAt: now, UpdatedAt: now, + } + if _, err := db.NewInsert().Model(attempt).Exec(ctx); err != nil { + if isUniqueViolation(err) { + return fmt.Errorf("reserving storage pull request: %w", ErrConflict) + } + return fmt.Errorf("reserving storage pull request: %w", err) + } + return nil + }) +} + +func (r *BunStorageContentRepo) ReplaceCopyTask(ctx context.Context, copyID, generation, taskID, nextGeneration, nextTaskID int64) error { + if copyID < 1 || generation < 1 || taskID < 1 || nextGeneration != generation+1 || nextTaskID < 1 { + return ErrInvalidInput + } + result, err := r.db.NewUpdate(). + Model((*model.StorageCopy)(nil)). + Set("work_generation = ?", nextGeneration). + Set("active_task_id = ?", nextTaskID). + Set("updated_at = ?", time.Now()). + Where("id = ? AND work_generation = ? AND active_task_id = ?", copyID, generation, taskID). + Exec(ctx) + return requireTaskFenceRows(result, err, "advancing storage copy task") +} + +func (r *BunStorageContentRepo) CompleteCopyTask(ctx context.Context, copyID, generation, taskID int64) error { + result, err := r.db.NewUpdate(). + Model((*model.StorageCopy)(nil)). + Set("active_task_id = NULL"). + Set("updated_at = ?", time.Now()). + Where("id = ? AND work_generation = ? AND active_task_id = ?", copyID, generation, taskID). + Exec(ctx) + return requireTaskFenceRows(result, err, "completing storage copy task") +} + +func (r *BunStorageContentRepo) NextDataSetRetirementGeneration(ctx context.Context, dataSetID int64) (int64, error) { + return r.nextDataSetTaskGeneration(ctx, dataSetID, dataSetRetirementFence) +} + +func (r *BunStorageContentRepo) BindDataSetRetirementTask(ctx context.Context, dataSetID, generation, taskID int64) error { + return r.bindDataSetTask(ctx, dataSetID, dataSetRetirementFence, generation, taskID) +} + +func (r *BunStorageContentRepo) AuthorizeDataSetRetirementTask(ctx context.Context, dataSetID, generation, taskID int64) (*model.StorageDataSet, error) { + return r.authorizeDataSetTask(ctx, dataSetID, dataSetRetirementFence, generation, taskID) +} + +func (r *BunStorageContentRepo) CompleteDataSetRetirementTask(ctx context.Context, dataSetID, generation, taskID int64) error { + return r.completeDataSetTask(ctx, dataSetID, dataSetRetirementFence, generation, taskID) +} + +// dataSetFence names the column pair guarding one kind of data-set task. The +// three fences below are the only valid pairs, so passing a fence value instead +// of raw column names keeps an invalid pair unrepresentable and removes the +// string whitelists this file used to need before building SQL. +type dataSetFence struct { + taskColumn string + // generationColumn is empty for fences that carry no generation. + generationColumn string +} + +var ( + dataSetEnsureFence = dataSetFence{taskColumn: "ensure_task_id"} + dataSetRetirementFence = dataSetFence{taskColumn: "retirement_task_id", generationColumn: "retirement_generation"} +) + +// usesGeneration reports whether this call should fence on a generation, and +// rejects a generation supplied for a fence that has no generation column. +func (f dataSetFence) usesGeneration(generation int64) (bool, error) { + if generation <= 0 { + return false, nil + } + if f.generationColumn == "" { + return false, ErrInvalidInput + } + return true, nil +} + +func (r *BunStorageContentRepo) nextDataSetTaskGeneration(ctx context.Context, dataSetID int64, fence dataSetFence) (int64, error) { + if fence.generationColumn == "" { + return 0, ErrInvalidInput + } + var generation int64 + err := r.db.NewSelect(). + Model((*model.StorageDataSet)(nil)). + ColumnExpr(fence.generationColumn+" + 1"). + Where("id = ?", dataSetID). + Where(fence.taskColumn+" IS NULL"). + Scan(ctx, &generation) + if errors.Is(err, sql.ErrNoRows) { + return 0, ErrConflict + } + if err != nil { + return 0, fmt.Errorf("reading data set task generation: %w", err) + } + return generation, nil +} + +func (r *BunStorageContentRepo) bindDataSetTask(ctx context.Context, dataSetID int64, fence dataSetFence, generation, taskID int64) error { + if dataSetID < 1 || taskID < 1 { + return ErrInvalidInput + } + fenced, err := fence.usesGeneration(generation) + if err != nil { + return err + } + query := r.db.NewUpdate(). + Model((*model.StorageDataSet)(nil)). + Set(fence.taskColumn+" = ?", taskID). + Set("updated_at = ?", time.Now()). + Where("id = ?", dataSetID). + Where(fence.taskColumn + " IS NULL") + if fenced { + query = query.Set(fence.generationColumn+" = ?", generation). + Where(fence.generationColumn+" = ?", generation-1) + } + result, err := query.Exec(ctx) + return requireTaskFenceRows(result, err, "binding data set task") +} + +func (r *BunStorageContentRepo) authorizeDataSetTask(ctx context.Context, dataSetID int64, fence dataSetFence, generation, taskID int64) (*model.StorageDataSet, error) { + fenced, err := fence.usesGeneration(generation) + if err != nil { + return nil, err + } + dataSet := new(model.StorageDataSet) + query := r.db.NewSelect().Model(dataSet).Where("id = ?", dataSetID).Where(fence.taskColumn+" = ?", taskID) + if fenced { + query = query.Where(fence.generationColumn+" = ?", generation) + } + if err := query.Scan(ctx); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrConflict + } + return nil, fmt.Errorf("authorizing data set task: %w", err) + } + return dataSet, nil +} + +func (r *BunStorageContentRepo) completeDataSetTask(ctx context.Context, dataSetID int64, fence dataSetFence, generation, taskID int64) error { + fenced, err := fence.usesGeneration(generation) + if err != nil { + return err + } + query := r.db.NewUpdate(). + Model((*model.StorageDataSet)(nil)). + Set(fence.taskColumn+" = NULL"). + Set("updated_at = ?", time.Now()). + Where("id = ?", dataSetID). + Where(fence.taskColumn+" = ?", taskID) + if fenced { + query = query.Where(fence.generationColumn+" = ?", generation) + } + result, err := query.Exec(ctx) + return requireTaskFenceRows(result, err, "completing data set task") +} + +func requireTaskFenceRows(result sql.Result, err error, operation string) error { + if err != nil { + return fmt.Errorf("%s: %w", operation, err) + } + rows, _ := result.RowsAffected() + if rows != 1 { + return ErrConflict + } + return nil +} diff --git a/internal/db/repository/storage_upload_reference.go b/internal/db/repository/storage_upload_reference.go deleted file mode 100644 index 72a858a..0000000 --- a/internal/db/repository/storage_upload_reference.go +++ /dev/null @@ -1,170 +0,0 @@ -package repository - -import ( - "context" - "database/sql" - "errors" - "fmt" - "sort" - - "github.com/strahe/synaps3/internal/model" - "github.com/uptrace/bun" -) - -const objectVersionReferencesStorageUploadIDSQL = "(storage_upload_id = ? OR (version_id = ? AND storage_upload_id IS NULL))" - -// lockStorageUploadsByID serializes object-version references with permanent -// deletion. Callers must acquire these locks before object or version locks. -func lockStorageUploadsByID(ctx context.Context, db bun.IDB, uploadIDs []int64) (map[int64]*model.StorageUpload, error) { - ids := append([]int64(nil), uploadIDs...) - sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) - - uploads := make(map[int64]*model.StorageUpload, len(ids)) - var previous int64 - for _, uploadID := range ids { - if uploadID <= 0 || uploadID == previous { - continue - } - previous = uploadID - - res, err := db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("status = status"). - Where("id = ?", uploadID). - Exec(ctx) - if err != nil { - return nil, fmt.Errorf("locking storage upload %d: %w", uploadID, err) - } - rows, _ := res.RowsAffected() - if rows == 0 { - return nil, fmt.Errorf("locking storage upload %d: %w", uploadID, ErrNotFound) - } - - upload := new(model.StorageUpload) - if err := db.NewSelect().Model(upload).Where("id = ?", uploadID).Scan(ctx); err != nil { - if err == sql.ErrNoRows { - return nil, fmt.Errorf("loading locked storage upload %d: %w", uploadID, ErrNotFound) - } - return nil, fmt.Errorf("loading locked storage upload %d: %w", uploadID, err) - } - uploads[uploadID] = upload - } - return uploads, nil -} - -func lockStorageUploadForObjectState( - ctx context.Context, - db bun.IDB, - uploadID int64, - state model.ObjectState, -) (*model.StorageUpload, error) { - var bucketID int64 - if err := db.NewSelect(). - Model((*model.StorageUpload)(nil)). - Column("bucket_id"). - Where("id = ?", uploadID). - Scan(ctx, &bucketID); err != nil { - if err == sql.ErrNoRows { - return nil, fmt.Errorf("loading storage upload %d: %w", uploadID, ErrNotFound) - } - return nil, fmt.Errorf("loading storage upload %d bucket: %w", uploadID, err) - } - if bucket, err := lockBucketByID(ctx, db, bucketID); err != nil { - return nil, err - } else if bucket == nil { - return nil, fmt.Errorf("locking storage upload %d bucket: %w", uploadID, ErrNotFound) - } - uploads, err := lockStorageUploadsByID(ctx, db, []int64{uploadID}) - if err != nil { - return nil, err - } - upload := uploads[uploadID] - if upload == nil || !storageUploadSupportsObjectState(upload.Status, state) { - return nil, fmt.Errorf("storage upload %d cannot back object state %s: %w", uploadID, state, ErrConflict) - } - if err := requireReadableCommittedCopy(ctx, db, uploadID); err != nil { - return nil, fmt.Errorf("storage upload %d cannot back an object version: %w", uploadID, ErrConflict) - } - return upload, nil -} - -func lockStorageUploadForCopyMutation(ctx context.Context, db bun.IDB, uploadID int64) error { - uploads, err := lockStorageUploadsByID(ctx, db, []int64{uploadID}) - if err != nil { - return err - } - upload := uploads[uploadID] - if upload == nil || upload.Status == model.StorageUploadStatusSuperseded { - return fmt.Errorf("storage upload %d cannot accept copy updates: %w", uploadID, ErrConflict) - } - return nil -} - -func storageUploadSupportsObjectState(status model.StorageUploadStatus, state model.ObjectState) bool { - switch state { - case model.ObjectStateStored, model.ObjectStateCacheEvicted: - return status == model.StorageUploadStatusReadable || status == model.StorageUploadStatusComplete - case model.ObjectStateReplicating: - return status != model.StorageUploadStatusRejected && status != model.StorageUploadStatusSuperseded - default: - return false - } -} - -func prepareNewObjectVersionStorageReference(ctx context.Context, db bun.IDB, version *model.ObjectVersion) error { - if version == nil || version.StorageUploadID == nil || *version.StorageUploadID <= 0 { - return nil - } - upload, err := lockStorageUploadForObjectState(ctx, db, *version.StorageUploadID, version.State) - if err == nil && (version.State == model.ObjectStateStored || version.State == model.ObjectStateCacheEvicted) { - err = requireCurrentMinimumDurableCopies(ctx, db, upload) - } - if err == nil { - return nil - } - if !errors.Is(err, ErrConflict) && !errors.Is(err, ErrNotFound) { - return err - } - if !version.InCache { - return err - } - - version.StorageUploadID = nil - version.State = model.ObjectStateCached - version.FailedAtState = nil - version.LastError = nil - return nil -} - -func sameStorageUploadIDs(left, right map[int64]*model.StorageUpload) bool { - if len(left) != len(right) { - return false - } - for uploadID := range left { - if _, ok := right[uploadID]; !ok { - return false - } - } - return true -} - -func objectVersionReferencesStorageUpload(version *model.ObjectVersion, upload *model.StorageUpload) bool { - if version == nil || upload == nil || version.IsDeleteMarker { - return false - } - if version.StorageUploadID != nil { - return *version.StorageUploadID == upload.ID - } - return version.VersionID == upload.SourceVersionID -} - -func objectVersionReferencesStorageUploadSQL(versionAlias, uploadAlias string) string { - return fmt.Sprintf( - `(%s.storage_upload_id = %s.id OR (%s.version_id = %s.source_version_id AND %s.storage_upload_id IS NULL))`, - versionAlias, - uploadAlias, - versionAlias, - uploadAlias, - versionAlias, - ) -} diff --git a/internal/db/repository/storage_upload_repo.go b/internal/db/repository/storage_upload_repo.go deleted file mode 100644 index e6c691d..0000000 --- a/internal/db/repository/storage_upload_repo.go +++ /dev/null @@ -1,2473 +0,0 @@ -package repository - -import ( - "context" - "database/sql" - "errors" - "fmt" - "time" - - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/observability" - "github.com/strahe/synaps3/internal/storagecommit" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/strahe/synaps3/internal/types" - "github.com/uptrace/bun" -) - -type BunStorageUploadRepo struct { - db bun.IDB -} - -var _ StorageUploadRepository = (*BunStorageUploadRepo)(nil) - -func (r *BunStorageUploadRepo) StartObjectUploadAttempt(ctx context.Context, input StartObjectUploadAttemptInput) (*model.StorageUpload, error) { - requestedCopies := input.RequestedCopies - if !model.ValidStorageCopies(requestedCopies) { - return nil, fmt.Errorf("requested copies must be between %d and %d, got %d", model.StorageCopiesMin, model.StorageCopiesMax, requestedCopies) - } - upload := &model.StorageUpload{ - BucketID: input.BucketID, - SourceVersionID: input.SourceVersionID, - ContentSize: input.ContentSize, - Checksum: input.Checksum, - Status: model.StorageUploadStatusRunning, - RequestedCopies: requestedCopies, - } - if input.SourceTaskID != 0 { - upload.SourceTaskID = &input.SourceTaskID - } - if _, err := r.db.NewInsert().Model(upload).Exec(ctx); err != nil { - if isUniqueViolation(err) && input.SourceVersionID != "" { - existing, selectErr := r.findActiveUploadBySourceVersion(ctx, input.SourceVersionID) - if selectErr != nil { - return nil, selectErr - } - if existing != nil { - return existing, nil - } - } - return nil, fmt.Errorf("starting storage upload attempt: %w", err) - } - return upload, nil -} - -func (r *BunStorageUploadRepo) GetByID(ctx context.Context, uploadID int64) (*model.StorageUpload, error) { - upload := new(model.StorageUpload) - err := r.db.NewSelect(). - Model(upload). - Where("id = ?", uploadID). - Scan(ctx) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("selecting storage upload: %w", err) - } - return upload, nil -} - -func (r *BunStorageUploadRepo) GetByIDs(ctx context.Context, uploadIDs []int64) (map[int64]model.StorageUpload, error) { - uploadsByID := make(map[int64]model.StorageUpload, len(uploadIDs)) - if len(uploadIDs) == 0 { - return uploadsByID, nil - } - var uploads []model.StorageUpload - if err := r.db.NewSelect(). - Model(&uploads). - Where("id IN (?)", bun.List(uploadIDs)). - Scan(ctx); err != nil { - return nil, fmt.Errorf("selecting storage uploads by id: %w", err) - } - for _, upload := range uploads { - uploadsByID[upload.ID] = upload - } - return uploadsByID, nil -} - -func (r *BunStorageUploadRepo) BeginIngressStoreProgress(ctx context.Context, uploadID int64) (*model.StorageUpload, error) { - if uploadID == 0 { - return nil, fmt.Errorf("uploadID is required: %w", ErrInvalidInput) - } - now := time.Now() - _, err := r.db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("ingress_store_attempt = ingress_store_attempt + 1"). - Set("ingress_bytes_transferred = 0"). - Set("progress_updated_at = ?", now). - Where("id = ?", uploadID). - Exec(ctx) - if err != nil { - return nil, fmt.Errorf("beginning ingress store progress: %w", err) - } - upload, err := r.GetByID(ctx, uploadID) - if err != nil { - return nil, err - } - if upload == nil { - return nil, fmt.Errorf("beginning ingress store progress: %w", ErrNotFound) - } - return upload, nil -} - -func (r *BunStorageUploadRepo) RecordIngressStoreProgress(ctx context.Context, input RecordIngressStoreProgressInput) (*model.StorageUpload, error) { - if input.UploadID == 0 { - return nil, fmt.Errorf("uploadID is required: %w", ErrInvalidInput) - } - if input.Attempt <= 0 { - return nil, fmt.Errorf("attempt is required: %w", ErrInvalidInput) - } - now := time.Now() - _, err := r.db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("progress_updated_at = CASE WHEN ingress_bytes_transferred < content_size AND ? > ingress_bytes_transferred THEN ? ELSE progress_updated_at END", input.BytesUploaded, now). - Set("ingress_bytes_transferred = CASE WHEN ? > content_size THEN content_size WHEN ? > ingress_bytes_transferred THEN ? ELSE ingress_bytes_transferred END", input.BytesUploaded, input.BytesUploaded, input.BytesUploaded). - Where("id = ?", input.UploadID). - Where("ingress_store_attempt = ?", input.Attempt). - Exec(ctx) - if err != nil { - return nil, fmt.Errorf("recording ingress store progress: %w", err) - } - upload, err := r.GetByID(ctx, input.UploadID) - if err != nil { - return nil, err - } - if upload == nil { - return nil, fmt.Errorf("recording ingress store progress: %w", ErrNotFound) - } - return upload, nil -} - -func (r *BunStorageUploadRepo) GetUploadProvenance(ctx context.Context, uploadID int64) (*StorageUploadProvenance, error) { - upload, err := r.GetByID(ctx, uploadID) - if err != nil || upload == nil { - return nil, err - } - copies, err := r.ListCopies(ctx, uploadID) - if err != nil { - return nil, err - } - failures, err := r.listFailures(ctx, uploadID) - if err != nil { - return nil, err - } - return &StorageUploadProvenance{ - Upload: *upload, - Copies: copies, - Failures: failures, - }, nil -} - -func (r *BunStorageUploadRepo) AppendUploadFailure(ctx context.Context, input AppendUploadFailureInput) error { - if input.UploadID == 0 { - return fmt.Errorf("uploadID is required: %w", ErrInvalidInput) - } - for { - err := r.appendUploadFailureOnce(ctx, input) - if err == nil { - return nil - } - if !shouldRetryUploadFailureAppend(err) { - return err - } - if err := waitUploadFailureAppendRetry(ctx); err != nil { - return fmt.Errorf("retrying storage upload failure append: %w", err) - } - } -} - -func (r *BunStorageUploadRepo) appendUploadFailureOnce(ctx context.Context, input AppendUploadFailureInput) error { - return r.runMaybeTx(ctx, func(db bun.IDB) error { - providerID := input.ProviderID - transferMethod := input.TransferMethod - if providerID == nil || transferMethod == "" { - copyRow := new(model.StorageUploadCopy) - err := db.NewSelect(). - Model(copyRow). - Where("upload_id = ? AND copy_index = ?", input.UploadID, input.CopyIndex). - Scan(ctx) - if err != nil && err != sql.ErrNoRows { - return fmt.Errorf("loading upload copy for failure: %w", err) - } - if err == nil { - if providerID == nil { - providerID = copyRow.ProviderID - } - if transferMethod == "" { - transferMethod = string(copyRow.TransferMethod) - } - } - } - var next struct { - AttemptIndex int `bun:"attempt_index"` - } - if err := db.NewRaw(`SELECT COALESCE(MAX(attempt_index), -1) + 1 AS attempt_index FROM storage_upload_failures WHERE upload_id = ?`, input.UploadID).Scan(ctx, &next); err != nil { - return fmt.Errorf("selecting next upload failure index: %w", err) - } - failure := &model.StorageUploadFailure{ - UploadID: input.UploadID, - AttemptIndex: next.AttemptIndex, - ProviderID: providerID, - TransferMethod: transferMethod, - Stage: nullableString(input.Stage), - ErrorMessage: nullableString(input.ErrorMessage), - Explicit: input.Explicit, - } - if _, err := db.NewInsert().Model(failure).Exec(ctx); err != nil { - return fmt.Errorf("appending storage upload failure: %w", err) - } - return nil - }) -} - -func (r *BunStorageUploadRepo) ListCopies(ctx context.Context, uploadID int64) ([]model.StorageUploadCopy, error) { - var copies []model.StorageUploadCopy - query := `SELECT storage_copy.*, storage_data_set.data_set_id AS data_set_id - FROM storage_upload_copies AS storage_copy - LEFT JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id - WHERE storage_copy.upload_id = ? - ORDER BY storage_copy.copy_index ASC` - if err := r.db.NewRaw(query, uploadID).Scan(ctx, &copies); err != nil { - return nil, fmt.Errorf("listing storage upload copies: %w", err) - } - return copies, nil -} - -// CountCurrentGenerationCopySlots counts logical replica slots, not physical -// generation rows. An unbound copy belongs to its slot until it is assigned. -func (r *BunStorageUploadRepo) CountCurrentGenerationCopySlots(ctx context.Context, uploadID int64) (int, error) { - if uploadID <= 0 { - return 0, fmt.Errorf("counting current upload copy slots: %w", ErrInvalidInput) - } - var count int - query := fmt.Sprintf(`SELECT COUNT(DISTINCT storage_copy.copy_index) - FROM storage_upload_copies AS storage_copy - WHERE storage_copy.upload_id = ? AND %s`, currentGenerationCopySQL("storage_copy")) - if err := r.db.NewRaw(query, uploadID).Scan(ctx, &count); err != nil { - return 0, fmt.Errorf("counting current upload copy slots: %w", err) - } - return count, nil -} - -func shouldRetryUploadFailureAppend(err error) bool { - return isUniqueViolation(err) || isSQLiteBusy(err) -} - -func waitUploadFailureAppendRetry(ctx context.Context) error { - const retryDelay = 5 * time.Millisecond - timer := time.NewTimer(retryDelay) - defer timer.Stop() - select { - case <-ctx.Done(): - return ctx.Err() - case <-timer.C: - return nil - } -} - -func (r *BunStorageUploadRepo) listFailures(ctx context.Context, uploadID int64) ([]model.StorageUploadFailure, error) { - var failures []model.StorageUploadFailure - if err := r.db.NewSelect(). - Model(&failures). - Where("upload_id = ?", uploadID). - OrderExpr("attempt_index ASC"). - Scan(ctx); err != nil { - return nil, fmt.Errorf("listing storage upload failures: %w", err) - } - return failures, nil -} - -func (r *BunStorageUploadRepo) ListReadableCommittedCopies(ctx context.Context, uploadID int64) ([]ReadableStorageCopy, error) { - var copies []ReadableStorageCopy - query := fmt.Sprintf(`SELECT - storage_copy.upload_id, - storage_upload.piece_cid, - storage_copy.copy_index, - storage_copy.provider_id, - storage_data_set.data_set_id, - storage_copy.piece_id, - storage_copy.transfer_method, - storage_copy.retrieval_url - FROM storage_upload_copies AS storage_copy - JOIN storage_uploads AS storage_upload ON storage_upload.id = storage_copy.upload_id - JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id - WHERE storage_copy.upload_id = ? - AND storage_upload.piece_cid IS NOT NULL AND storage_upload.piece_cid <> '' - AND %s`, - readableCommittedCopyPredicateSQL("storage_copy", "storage_data_set"), - ) - args := []interface{}{uploadID} - query += " ORDER BY storage_copy.copy_index ASC" - if err := r.db.NewRaw(query, args...).Scan(ctx, &copies); err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("listing readable storage copies: %w", err) - } - return copies, nil -} - -func (r *BunStorageUploadRepo) HasReadableCommittedCopy(ctx context.Context, uploadID int64) (bool, error) { - count, err := countReadableReplicaSlots(ctx, r.db, uploadID) - if err != nil { - return false, err - } - return count > 0, nil -} - -type bucketStorageHealthSummaryRow struct { - BucketID int64 `bun:"bucket_id"` - AbnormalDataSets int `bun:"abnormal_data_sets"` - AffectedVersionsSeen int `bun:"affected_versions_seen"` - LocalStatusNotReady bool `bun:"local_status_not_ready"` - ObservationMissing bool `bun:"observation_missing"` - ObservationStale bool `bun:"observation_stale"` - ObservationUnavailable bool `bun:"observation_unavailable"` - ObservationDegraded bool `bun:"observation_degraded"` - ObservationUnknown bool `bun:"observation_unknown"` - LastCheckedAt *time.Time `bun:"last_checked_at"` -} - -// bucketStorageHealthAffectedVersionExistsSQL expects dataSetSourceAlias to expose bucket_id and data_set_id. -func bucketStorageHealthAffectedVersionExistsSQL(dataSetSourceAlias string) string { - return fmt.Sprintf(`EXISTS ( - SELECT 1 - FROM storage_upload_copies AS storage_copy - JOIN object_versions AS object_version - ON object_version.storage_upload_id = storage_copy.upload_id - AND object_version.bucket_id = %[1]s.bucket_id - AND object_version.is_delete_marker = FALSE - WHERE storage_copy.storage_data_set_id = %[1]s.data_set_id - AND storage_copy.status = %[2]s - AND storage_copy.storage_data_set_id IS NOT NULL - )`, dataSetSourceAlias, storageHealthCommittedCopyStatusSQL()) -} - -func (r *BunStorageUploadRepo) ListBucketStorageHealthSummaries(ctx context.Context, bucketID int64, staleBefore time.Time, affectedVersionCap int) ([]BucketStorageHealthSummary, error) { - if affectedVersionCap < 1 { - affectedVersionCap = 1 - } - var rows []bucketStorageHealthSummaryRow - dataSetBucketFilter := "" - args := []interface{}{staleBefore} - if bucketID > 0 { - dataSetBucketFilter = ` - AND storage_data_set.bucket_id = ?` - args = append(args, bucketID) - } - query := fmt.Sprintf(`WITH bucket_data_sets AS ( - SELECT - storage_data_set.id AS data_set_id, - storage_data_set.bucket_id, - CASE WHEN storage_data_set.status NOT IN (%s) THEN 1 ELSE 0 END AS local_status_not_ready, - CASE WHEN observation.local_data_set_id IS NULL THEN 1 ELSE 0 END AS observation_missing, - CASE WHEN observation.local_data_set_id IS NOT NULL AND observation.last_checked_at < ? THEN 1 ELSE 0 END AS observation_stale, - CASE WHEN observation.status = %s THEN 1 ELSE 0 END AS observation_unavailable, - CASE WHEN observation.status = %s THEN 1 ELSE 0 END AS observation_degraded, - CASE WHEN observation.status = %s THEN 1 ELSE 0 END AS observation_unknown, - observation.last_checked_at - FROM storage_data_sets AS storage_data_set - LEFT JOIN observability_data_set_states AS observation ON observation.local_data_set_id = storage_data_set.id - WHERE 1 = 1 - %s - ), - bucket_observation AS ( - SELECT - bucket_id, - COALESCE(MAX(observation_stale), 0) AS observation_stale, - MIN(last_checked_at) AS last_checked_at - FROM bucket_data_sets - GROUP BY bucket_id - ), - abnormal_data_sets AS ( - SELECT * - FROM bucket_data_sets - WHERE local_status_not_ready = 1 - OR observation_missing = 1 - OR observation_stale = 1 - OR observation_unavailable = 1 - OR observation_degraded = 1 - OR observation_unknown = 1 - ), - bucket_abnormal AS ( - SELECT - bucket_id, - COUNT(*) AS abnormal_data_sets - FROM abnormal_data_sets - GROUP BY bucket_id - ), - affected_data_sets AS ( - SELECT - abnormal_data_set.* - FROM abnormal_data_sets AS abnormal_data_set - WHERE %s - ), - affected_bucket_summary AS ( - SELECT - bucket_id, - COUNT(*) AS affected_data_sets, - COALESCE(MAX(local_status_not_ready), 0) AS local_status_not_ready, - COALESCE(MAX(observation_missing), 0) AS observation_missing, - COALESCE(MAX(observation_stale), 0) AS observation_stale, - COALESCE(MAX(observation_unavailable), 0) AS observation_unavailable, - COALESCE(MAX(observation_degraded), 0) AS observation_degraded, - COALESCE(MAX(observation_unknown), 0) AS observation_unknown, - MIN(last_checked_at) AS last_checked_at - FROM affected_data_sets - GROUP BY bucket_id - ) - SELECT - bucket_observation.bucket_id, - COALESCE(bucket_abnormal.abnormal_data_sets, 0) AS abnormal_data_sets, - COALESCE(( - SELECT COUNT(*) - FROM ( - SELECT DISTINCT object_version.version_id - FROM affected_data_sets AS affected_data_set - JOIN storage_upload_copies AS storage_copy - ON storage_copy.storage_data_set_id = affected_data_set.data_set_id - JOIN object_versions AS object_version - ON object_version.storage_upload_id = storage_copy.upload_id - AND object_version.bucket_id = affected_data_set.bucket_id - AND object_version.is_delete_marker = FALSE - WHERE affected_data_set.bucket_id = bucket_observation.bucket_id - AND storage_copy.status = %s - AND storage_copy.storage_data_set_id IS NOT NULL - LIMIT ? - ) AS capped_affected_versions - ), 0) AS affected_versions_seen, - COALESCE(affected_bucket_summary.local_status_not_ready, 0) > 0 AS local_status_not_ready, - COALESCE(affected_bucket_summary.observation_missing, 0) > 0 AS observation_missing, - (CASE - WHEN COALESCE(affected_bucket_summary.affected_data_sets, 0) > 0 - THEN COALESCE(affected_bucket_summary.observation_stale, 0) - ELSE bucket_observation.observation_stale - END) > 0 AS observation_stale, - COALESCE(affected_bucket_summary.observation_unavailable, 0) > 0 AS observation_unavailable, - COALESCE(affected_bucket_summary.observation_degraded, 0) > 0 AS observation_degraded, - COALESCE(affected_bucket_summary.observation_unknown, 0) > 0 AS observation_unknown, - CASE - WHEN COALESCE(affected_bucket_summary.affected_data_sets, 0) > 0 - THEN affected_bucket_summary.last_checked_at - ELSE bucket_observation.last_checked_at - END AS last_checked_at - FROM bucket_observation - LEFT JOIN bucket_abnormal ON bucket_abnormal.bucket_id = bucket_observation.bucket_id - LEFT JOIN affected_bucket_summary ON affected_bucket_summary.bucket_id = bucket_observation.bucket_id - ORDER BY bucket_observation.bucket_id ASC`, - storageHealthReadyDataSetStatusListSQL(), - storageHealthUnavailableObservationStatusSQL(), - storageHealthDegradedObservationStatusSQL(), - storageHealthUnknownObservationStatusSQL(), - dataSetBucketFilter, - bucketStorageHealthAffectedVersionExistsSQL("abnormal_data_set"), - storageHealthCommittedCopyStatusSQL(), - ) - args = append(args, affectedVersionCap+1) - if err := r.db.NewRaw(query, args...).Scan(ctx, &rows); err != nil { - return nil, fmt.Errorf("listing bucket storage health summaries: %w", err) - } - reasonCodes, err := r.listBucketStorageHealthReasonCodes(ctx, bucketID, staleBefore) - if err != nil { - return nil, err - } - summaries := make([]BucketStorageHealthSummary, 0, len(rows)) - for _, row := range rows { - affectedVersions := row.AffectedVersionsSeen - exceedsCap := affectedVersions > affectedVersionCap - if exceedsCap { - affectedVersions = affectedVersionCap - } - reasons := reasonCodes[row.BucketID] - if reasons == nil { - reasons = []observability.ReasonCode{} - } - summaries = append(summaries, BucketStorageHealthSummary{ - BucketID: row.BucketID, - AbnormalDataSets: row.AbnormalDataSets, - AffectedVersionsCapped: affectedVersions, - AffectedVersionsCap: affectedVersionCap, - AffectedVersionsExceedsCap: exceedsCap, - LocalStatusNotReady: row.LocalStatusNotReady, - ObservationMissing: row.ObservationMissing, - ObservationStale: row.ObservationStale, - ObservationUnavailable: row.ObservationUnavailable, - ObservationDegraded: row.ObservationDegraded, - ObservationUnknown: row.ObservationUnknown, - ReasonCodes: reasons, - LastCheckedAt: row.LastCheckedAt, - }) - } - return summaries, nil -} - -type bucketStorageHealthReasonCodeRow struct { - BucketID int64 `bun:"bucket_id"` - LocalStatus model.StorageDataSetStatus `bun:"local_status"` - ReasonCodes []observability.ReasonCode `bun:"reason_codes"` -} - -func (r *BunStorageUploadRepo) listBucketStorageHealthReasonCodes(ctx context.Context, bucketID int64, staleBefore time.Time) (map[int64][]observability.ReasonCode, error) { - dataSetBucketFilter := "" - args := make([]interface{}, 0, 2) - if bucketID > 0 { - dataSetBucketFilter = ` - AND storage_data_set.bucket_id = ?` - args = append(args, bucketID) - } - query := fmt.Sprintf(`WITH abnormal_data_sets AS ( - SELECT - storage_data_set.id AS data_set_id, - storage_data_set.bucket_id, - storage_data_set.status AS local_status, - COALESCE(observation.reason_codes, %s) AS reason_codes - FROM storage_data_sets AS storage_data_set - LEFT JOIN observability_data_set_states AS observation ON observation.local_data_set_id = storage_data_set.id - WHERE 1 = 1 -%s - AND ( - storage_data_set.status NOT IN (%s) - OR observation.local_data_set_id IS NULL - OR observation.status IN (%s) - OR observation.last_checked_at < ? - ) - ) - SELECT DISTINCT - abnormal_data_set.bucket_id, - abnormal_data_set.local_status, - abnormal_data_set.reason_codes - FROM abnormal_data_sets AS abnormal_data_set - WHERE %s - ORDER BY abnormal_data_set.bucket_id ASC`, - storageHealthEmptyJSONArraySQL(r.db), - dataSetBucketFilter, - storageHealthReadyDataSetStatusListSQL(), - storageHealthAbnormalObservationStatusListSQL(), - bucketStorageHealthAffectedVersionExistsSQL("abnormal_data_set"), - ) - args = append(args, staleBefore) - var rows []bucketStorageHealthReasonCodeRow - if err := r.db.NewRaw(query, args...).Scan(ctx, &rows); err != nil { - return nil, fmt.Errorf("listing bucket storage health reason codes: %w", err) - } - out := make(map[int64][]observability.ReasonCode) - for _, row := range rows { - if row.LocalStatus != model.StorageDataSetStatusReady && row.LocalStatus != model.StorageDataSetStatusDraining { - out[row.BucketID] = observability.AppendReasonCode(out[row.BucketID], observability.ReasonLocalStatusNotReady) - } - for _, reason := range row.ReasonCodes { - out[row.BucketID] = observability.AppendReasonCode(out[row.BucketID], reason) - } - } - return out, nil -} - -// ListDataSetBindings returns every generation, including retired ones, because -// callers need the full provider history as well as the current write targets. -func (r *BunStorageUploadRepo) ListDataSetBindings(ctx context.Context, bucketID int64) ([]model.StorageDataSet, error) { - var bindings []model.StorageDataSet - if err := r.db.NewSelect(). - Model(&bindings). - Where("bucket_id = ?", bucketID). - OrderExpr("copy_index ASC, generation ASC"). - Scan(ctx); err != nil { - return nil, fmt.Errorf("listing storage data set bindings: %w", err) - } - return bindings, nil -} - -func (r *BunStorageUploadRepo) ListDataSetSummaries(ctx context.Context, bucketID int64) ([]StorageDataSetSummary, error) { - var summaries []StorageDataSetSummary - query := fmt.Sprintf(`SELECT - storage_data_set.id, - storage_data_set.bucket_id, - bucket.name AS bucket_name, - storage_data_set.copy_index, - storage_data_set.generation, - storage_data_set.is_current, - storage_data_set.provider_id, - storage_data_set.data_set_id, - storage_data_set.client_data_set_id, - storage_data_set.status, - storage_data_set.created_by_upload_id, - storage_data_set.last_used_upload_id, - COALESCE(copy_stats.committed_copies, 0) AS committed_copies, - COALESCE(copy_stats.readable_copies, 0) AS readable_copies, - COALESCE(copy_stats.physical_bytes, 0) AS physical_bytes, - COALESCE(version_stats.referenced_versions, 0) AS referenced_versions, - COALESCE(version_stats.current_versions, 0) AS current_versions, - storage_data_set.created_at, - storage_data_set.updated_at - FROM storage_data_sets AS storage_data_set - JOIN buckets AS bucket ON bucket.id = storage_data_set.bucket_id - LEFT JOIN ( - SELECT - storage_copy.storage_data_set_id, - COUNT(*) AS committed_copies, - SUM(CASE - WHEN storage_data_set.status IN (%s) - AND storage_copy.provider_id IS NOT NULL AND storage_copy.provider_id <> '' - AND storage_data_set.data_set_id IS NOT NULL AND storage_data_set.data_set_id <> '' - AND storage_copy.piece_id IS NOT NULL AND storage_copy.piece_id <> '' - AND storage_copy.retrieval_url IS NOT NULL AND storage_copy.retrieval_url <> '' - THEN 1 ELSE 0 END) AS readable_copies, - SUM(storage_upload.content_size) AS physical_bytes - FROM storage_upload_copies AS storage_copy - JOIN storage_uploads AS storage_upload ON storage_upload.id = storage_copy.upload_id - JOIN storage_data_sets AS storage_data_set - ON storage_data_set.id = storage_copy.storage_data_set_id - AND storage_data_set.bucket_id = storage_upload.bucket_id - WHERE storage_copy.status = %s - GROUP BY storage_copy.storage_data_set_id - ) AS copy_stats ON copy_stats.storage_data_set_id = storage_data_set.id - LEFT JOIN ( - SELECT - storage_copy.storage_data_set_id, - COUNT(DISTINCT object_version.version_id) AS referenced_versions, - COUNT(DISTINCT CASE WHEN object_version.is_current THEN object_version.version_id END) AS current_versions - FROM storage_upload_copies AS storage_copy - JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id - JOIN object_versions AS object_version - ON object_version.storage_upload_id = storage_copy.upload_id - AND object_version.bucket_id = storage_data_set.bucket_id - WHERE storage_copy.status = %s - AND object_version.is_delete_marker = FALSE - GROUP BY storage_copy.storage_data_set_id - ) AS version_stats ON version_stats.storage_data_set_id = storage_data_set.id - WHERE (? = 0 OR storage_data_set.bucket_id = ?) - ORDER BY bucket.name ASC, storage_data_set.copy_index ASC, storage_data_set.generation ASC`, - storageHealthReadyDataSetStatusListSQL(), - storageHealthCommittedCopyStatusSQL(), - storageHealthCommittedCopyStatusSQL(), - ) - if err := r.db.NewRaw(query, bucketID, bucketID).Scan(ctx, &summaries); err != nil { - return nil, fmt.Errorf("listing storage data set summaries: %w", err) - } - return summaries, nil -} - -// GetDataSetBindingByCopyIndex returns the generation that currently owns the -// slot. Historical generations stay readable but never receive new writes. -func (r *BunStorageUploadRepo) GetDataSetBindingByCopyIndex(ctx context.Context, bucketID int64, copyIndex int) (*model.StorageDataSet, error) { - binding := new(model.StorageDataSet) - err := r.db.NewSelect(). - Model(binding). - Where("bucket_id = ? AND copy_index = ? AND is_current", bucketID, copyIndex). - Scan(ctx) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("selecting storage data set binding: %w", err) - } - return binding, nil -} - -func (r *BunStorageUploadRepo) GetDataSetBindingByID(ctx context.Context, id int64) (*model.StorageDataSet, error) { - binding := new(model.StorageDataSet) - err := r.db.NewSelect(). - Model(binding). - Where("id = ?", id). - Scan(ctx) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("selecting storage data set binding by id: %w", err) - } - return binding, nil -} - -func (r *BunStorageUploadRepo) EnsureDataSetBinding(ctx context.Context, input EnsureDataSetBindingInput) (*model.StorageDataSet, error) { - var binding *model.StorageDataSet - err := r.runMaybeTx(ctx, func(db bun.IDB) error { - got, err := ensureDataSetBinding(ctx, db, input) - if err != nil { - return err - } - binding = got - return nil - }) - return binding, err -} - -func (r *BunStorageUploadRepo) MarkDataSetCreating(ctx context.Context, input MarkDataSetCreatingInput) error { - now := time.Now() - _, err := r.db.NewUpdate(). - Model((*model.StorageDataSet)(nil)). - Set("status = ?", model.StorageDataSetStatusCreating). - Set("create_transaction_id = ?", nullableString(input.TransactionID)). - Set("create_status_url = ?", nullableString(input.StatusURL)). - Set("client_data_set_id = ?", input.ClientDataSetID). - Set("last_used_upload_id = ?", nullableInt64(input.UploadID)). - Set("last_error = NULL"). - Set("updated_at = ?", now). - Where("id = ?", input.ID). - Exec(ctx) - if err != nil { - return fmt.Errorf("marking storage data set creating: %w", err) - } - return nil -} - -func (r *BunStorageUploadRepo) MarkDataSetReady(ctx context.Context, input MarkDataSetReadyInput) error { - return r.runMaybeTx(ctx, func(db bun.IDB) error { - return markDataSetReady(ctx, db, input.ID, input.UploadID, input.DataSetID, input.ClientDataSetID) - }) -} - -func (r *BunStorageUploadRepo) BackfillClientDataSetID(ctx context.Context, input BackfillClientDataSetIDInput) error { - if input.ID <= 0 || input.DataSetID.IsZero() { - return fmt.Errorf("backfilling storage client data set ID: %w", ErrInvalidInput) - } - res, err := r.db.NewUpdate(). - Model((*model.StorageDataSet)(nil)). - Set("client_data_set_id = ?", input.ClientDataSetID). - Set("updated_at = ?", time.Now()). - Where("id = ?", input.ID). - Where("data_set_id = ?", input.DataSetID). - Where("client_data_set_id IS NULL"). - Exec(ctx) - if err != nil { - return fmt.Errorf("backfilling storage client data set ID: %w", err) - } - rows, err := res.RowsAffected() - if err != nil { - return fmt.Errorf("checking storage client data set ID backfill: %w", err) - } - if rows > 0 { - return nil - } - var current struct { - DataSetID *types.OnChainID `bun:"data_set_id"` - ClientDataSetID *types.OnChainID `bun:"client_data_set_id"` - } - err = r.db.NewSelect(). - Table("storage_data_sets"). - Column("data_set_id", "client_data_set_id"). - Where("id = ?", input.ID). - Scan(ctx, ¤t) - if err != nil { - if err == sql.ErrNoRows { - return fmt.Errorf("backfilling storage client data set ID: %w", ErrNotFound) - } - return fmt.Errorf("checking storage client data set ID: %w", err) - } - if current.DataSetID == nil || !current.DataSetID.Equal(input.DataSetID) || - current.ClientDataSetID == nil || !current.ClientDataSetID.Equal(input.ClientDataSetID) { - return fmt.Errorf("backfilling storage client data set ID: %w", ErrConflict) - } - return nil -} - -// RecoverDataSet restores a quarantined binding once storage confirms it is -// usable again. A generation an operator is actively replacing is never -// revived, because bringing it back would fight the approved migration. -// Terminally failed or attention-holding replacements do not block recovery, so -// a slot whose replacement gave up can still repair in place. -func (r *BunStorageUploadRepo) RecoverDataSet(ctx context.Context, input MarkDataSetReadyInput) (bool, error) { - if input.ID <= 0 || input.DataSetID.IsZero() { - return false, fmt.Errorf("recovering storage data set: %w", ErrInvalidInput) - } - now := time.Now() - res, err := r.db.NewUpdate(). - Model((*model.StorageDataSet)(nil)). - Set("status = ?", model.StorageDataSetStatusReady). - Set("client_data_set_id = COALESCE(?, client_data_set_id)", input.ClientDataSetID). - Set("last_used_upload_id = COALESCE(?, last_used_upload_id)", nullableInt64(input.UploadID)). - Set("last_error = NULL"). - Set("updated_at = ?", now). - Where("id = ?", input.ID). - Where("data_set_id = ?", input.DataSetID). - Where("status IN (?)", bun.List([]model.StorageDataSetStatus{ - model.StorageDataSetStatusUnavailable, - model.StorageDataSetStatusReady, - })). - Where(`NOT EXISTS ( - SELECT 1 FROM storage_replacements AS blocking_replacement - WHERE blocking_replacement.source_data_set_id = storage_data_set.id - AND blocking_replacement.status IN (?, ?, ?, ?) - )`, - storagereplacement.StatusPreparingTarget, - storagereplacement.StatusMigrating, - storagereplacement.StatusWaiting, - storagereplacement.StatusRetiring). - Exec(ctx) - if err != nil { - return false, fmt.Errorf("recovering storage data set: %w", err) - } - rows, _ := res.RowsAffected() - if rows > 0 { - return true, nil - } - count, err := r.db.NewSelect(). - Model((*model.StorageDataSet)(nil)). - Where("id = ?", input.ID). - Count(ctx) - if err != nil { - return false, fmt.Errorf("checking storage data set recovery: %w", err) - } - if count == 0 { - return false, fmt.Errorf("recovering storage data set %d: %w", input.ID, ErrNotFound) - } - return false, nil -} - -func (r *BunStorageUploadRepo) MarkDataSetDraining(ctx context.Context, id int64, lastError string) error { - res, err := r.db.NewUpdate(). - Model((*model.StorageDataSet)(nil)). - Set("status = ?", model.StorageDataSetStatusDraining). - Set("last_error = ?", lastError). - Set("updated_at = ?", time.Now()). - Where("id = ?", id). - Where("status IN (?)", bun.List([]model.StorageDataSetStatus{ - model.StorageDataSetStatusReady, - model.StorageDataSetStatusUnavailable, - model.StorageDataSetStatusDraining, - })). - Exec(ctx) - if err != nil { - return fmt.Errorf("marking storage data set draining: %w", err) - } - return requireDataSetStatusUpdate(ctx, r.db, id, res, "marking storage data set draining") -} - -func (r *BunStorageUploadRepo) MarkDataSetFailed(ctx context.Context, id int64, lastError string) error { - res, err := r.db.NewUpdate(). - Model((*model.StorageDataSet)(nil)). - Set("status = ?", model.StorageDataSetStatusFailed). - Set("last_error = ?", lastError). - Set("updated_at = ?", time.Now()). - Where("id = ?", id). - Where("status IN (?)", bun.List([]model.StorageDataSetStatus{ - model.StorageDataSetStatusPending, - model.StorageDataSetStatusCreating, - model.StorageDataSetStatusFailed, - })). - Exec(ctx) - if err != nil { - return fmt.Errorf("marking storage data set failed: %w", err) - } - return requireDataSetStatusUpdate(ctx, r.db, id, res, "marking storage data set failed") -} - -func (r *BunStorageUploadRepo) MarkDataSetUnavailable(ctx context.Context, id int64, lastError string) error { - res, err := r.db.NewUpdate(). - Model((*model.StorageDataSet)(nil)). - Set("status = ?", model.StorageDataSetStatusUnavailable). - Set("last_error = ?", lastError). - Set("updated_at = ?", time.Now()). - Where("id = ?", id). - Where("status IN (?)", bun.List([]model.StorageDataSetStatus{ - model.StorageDataSetStatusReady, - model.StorageDataSetStatusUnavailable, - })). - Exec(ctx) - if err != nil { - return fmt.Errorf("marking storage data set unavailable: %w", err) - } - return requireDataSetStatusUpdate(ctx, r.db, id, res, "marking storage data set unavailable") -} - -func requireDataSetStatusUpdate(ctx context.Context, db bun.IDB, id int64, result sql.Result, operation string) error { - rows, err := result.RowsAffected() - if err != nil { - return fmt.Errorf("%s: reading affected rows: %w", operation, err) - } - if rows > 0 { - return nil - } - count, err := db.NewSelect(). - Model((*model.StorageDataSet)(nil)). - Where("id = ?", id). - Count(ctx) - if err != nil { - return fmt.Errorf("%s: checking data set: %w", operation, err) - } - if count == 0 { - return fmt.Errorf("%s: %w", operation, ErrNotFound) - } - return fmt.Errorf("%s: data set state changed: %w", operation, ErrConflict) -} - -func (r *BunStorageUploadRepo) DiscardFailedDataSetCandidate(ctx context.Context, uploadID int64, copyIndex int, storageDataSetID int64) (bool, error) { - if uploadID <= 0 || copyIndex < 0 || storageDataSetID <= 0 { - return false, fmt.Errorf("invalid failed storage data set candidate: %w", ErrInvalidInput) - } - discarded := false - err := r.runMaybeTx(ctx, func(db bun.IDB) error { - candidates, err := db.NewSelect(). - Model((*model.StorageDataSet)(nil)). - Where("id = ?", storageDataSetID). - Where("created_by_upload_id = ?", uploadID). - Where("status = ?", model.StorageDataSetStatusFailed). - Where("(data_set_id IS NULL OR data_set_id = '')"). - Where("(client_data_set_id IS NULL OR client_data_set_id = '')"). - Where("(create_transaction_id IS NULL OR create_transaction_id = '')"). - Where("(create_status_url IS NULL OR create_status_url = '')"). - Count(ctx) - if err != nil { - return fmt.Errorf("checking failed storage data set candidate: %w", err) - } - if candidates == 0 { - return nil - } - refs, err := db.NewSelect(). - Model((*model.StorageUploadCopy)(nil)). - Where("storage_data_set_id = ?", storageDataSetID). - Count(ctx) - if err != nil { - return fmt.Errorf("checking failed storage data set candidate references: %w", err) - } - currentRefs, err := db.NewSelect(). - Model((*model.StorageUploadCopy)(nil)). - Where("upload_id = ?", uploadID). - Where("copy_index = ?", copyIndex). - Where("storage_data_set_id = ?", storageDataSetID). - Where("status = ?", model.StorageUploadCopyStatusFailed). - Count(ctx) - if err != nil { - return fmt.Errorf("checking failed storage upload copy candidate: %w", err) - } - if refs != 1 || currentRefs != 1 { - return nil - } - if _, err := db.NewDelete(). - Model((*model.StorageUploadCopy)(nil)). - Where("upload_id = ?", uploadID). - Where("copy_index = ?", copyIndex). - Where("storage_data_set_id = ?", storageDataSetID). - Where("status = ?", model.StorageUploadCopyStatusFailed). - Exec(ctx); err != nil { - return fmt.Errorf("deleting failed storage upload copy candidate: %w", err) - } - res, err := db.NewDelete(). - Model((*model.StorageDataSet)(nil)). - Where("id = ?", storageDataSetID). - Where("created_by_upload_id = ?", uploadID). - Where("status = ?", model.StorageDataSetStatusFailed). - Where("(data_set_id IS NULL OR data_set_id = '')"). - Where("(client_data_set_id IS NULL OR client_data_set_id = '')"). - Where("(create_transaction_id IS NULL OR create_transaction_id = '')"). - Where("(create_status_url IS NULL OR create_status_url = '')"). - Exec(ctx) - if err != nil { - return fmt.Errorf("deleting failed storage data set candidate: %w", err) - } - rows, _ := res.RowsAffected() - if rows != 1 { - return fmt.Errorf("deleting failed storage data set candidate: %w", ErrConflict) - } - discarded = true - return nil - }) - return discarded, err -} - -func (r *BunStorageUploadRepo) CreateUploadCopiesForBindings(ctx context.Context, uploadID int64, copies []UploadCopyBindingInput) error { - return r.runMaybeTx(ctx, func(db bun.IDB) error { - for _, input := range copies { - if input.ProviderID.IsZero() { - return fmt.Errorf("providerID is required: %w", ErrInvalidInput) - } - providerID := input.ProviderID - isNewDataSet, err := storageDataSetCreatedByUpload(ctx, db, input.StorageDataSetID, uploadID) - if err != nil { - return err - } - copyRow := &model.StorageUploadCopy{ - UploadID: uploadID, - CopyIndex: input.CopyIndex, - ProviderID: &providerID, - TransferMethod: input.TransferMethod, - Status: model.StorageUploadCopyStatusPending, - StorageDataSetID: &input.StorageDataSetID, - IsNewDataSet: isNewDataSet, - } - // Copies are unique per concrete data set so one upload can hold both - // generations of a slot while a replacement migrates. - if _, err := db.NewInsert(). - Model(copyRow). - On("CONFLICT (upload_id, storage_data_set_id) WHERE storage_data_set_id IS NOT NULL DO NOTHING"). - Exec(ctx); err != nil { - return fmt.Errorf("creating storage upload copy row: %w", err) - } - } - return nil - }) -} - -// GetUploadCopy resolves a slot to the copy on its current generation, so a -// task that carries only (upload, slot) can never address a replaced or a -// not-yet-activated generation. A bound copy wins over an unbound one. -func (r *BunStorageUploadRepo) GetUploadCopy(ctx context.Context, uploadID int64, copyIndex int) (*model.StorageUploadCopy, error) { - copyRow := new(model.StorageUploadCopy) - err := r.db.NewSelect(). - Model(copyRow). - Where("storage_upload_copy.upload_id = ? AND storage_upload_copy.copy_index = ?", uploadID, copyIndex). - Where(currentGenerationCopySQL("storage_upload_copy")). - OrderExpr("(storage_upload_copy.storage_data_set_id IS NULL) ASC"). - Limit(1). - Scan(ctx) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("selecting storage upload copy: %w", err) - } - return copyRow, nil -} - -// GetUploadCopyForDataSet addresses one concrete generation, which is how -// replacement work targets the new provider while the old one still exists. -func (r *BunStorageUploadRepo) GetUploadCopyForDataSet(ctx context.Context, uploadID, storageDataSetID int64) (*model.StorageUploadCopy, error) { - if uploadID <= 0 || storageDataSetID <= 0 { - return nil, fmt.Errorf("selecting storage upload copy for data set: %w", ErrInvalidInput) - } - copyRow := new(model.StorageUploadCopy) - err := r.db.NewSelect(). - Model(copyRow). - Where("upload_id = ? AND storage_data_set_id = ?", uploadID, storageDataSetID). - Scan(ctx) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("selecting storage upload copy for data set: %w", err) - } - return copyRow, nil -} - -func (r *BunStorageUploadRepo) GetUploadCopyByID(ctx context.Context, id int64) (*model.StorageUploadCopy, error) { - copyRow := new(model.StorageUploadCopy) - err := r.db.NewSelect().Model(copyRow).Where("id = ?", id).Scan(ctx) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("selecting storage upload copy by id: %w", err) - } - return copyRow, nil -} - -func (r *BunStorageUploadRepo) AcquireUploadTask(ctx context.Context, input AcquireUploadTaskInput) error { - if input.TaskID <= 0 || input.TaskClaimedAt.IsZero() || input.UploadID < 0 || input.VersionID == "" { - return fmt.Errorf("acquiring upload task: %w", ErrInvalidInput) - } - return r.runMaybeTx(ctx, func(db bun.IDB) error { - var upload *model.StorageUpload - if input.UploadID > 0 { - uploads, err := lockStorageUploadsByID(ctx, db, []int64{input.UploadID}) - if err != nil { - if errors.Is(err, ErrNotFound) { - return ErrUploadTaskCancelled - } - return fmt.Errorf("locking upload task storage upload: %w", err) - } - upload = uploads[input.UploadID] - if upload == nil || upload.Status == model.StorageUploadStatusSuperseded { - return ErrUploadTaskCancelled - } - } - - version := new(model.ObjectVersion) - err := db.NewRaw(`UPDATE object_versions - SET updated_at = updated_at - WHERE version_id = ? - RETURNING *`, input.VersionID).Scan(ctx, version) - if err != nil { - if err == sql.ErrNoRows { - return ErrUploadTaskCancelled - } - return fmt.Errorf("locking upload task object version: %w", err) - } - if version.IsDeleteMarker { - return ErrUploadTaskCancelled - } - if upload != nil && (version.BucketID != upload.BucketID || !objectVersionReferencesStorageUpload(version, upload)) { - return ErrUploadTaskCancelled - } - - now := time.Now() - task := new(model.Task) - err = db.NewRaw(`UPDATE tasks - SET status = status - WHERE id = ? - AND status = ? - AND claimed_at = ? - AND lease_until IS NOT NULL - AND lease_until > ? - RETURNING *`, input.TaskID, model.TaskStatusRunning, input.TaskClaimedAt, now).Scan(ctx, task) - if err != nil { - if err == sql.ErrNoRows { - return ErrTaskClaimLost - } - return fmt.Errorf("locking upload task claim: %w", err) - } - if task.Type != model.TaskTypeUpload || task.RefType != "object" || task.RefID != version.ObjectID || task.RefVersionID != version.VersionID { - return fmt.Errorf("upload task identity mismatch: %w", ErrConflict) - } - return nil - }) -} - -func (r *BunStorageUploadRepo) AcquireReplicaRepairItem(ctx context.Context, input AcquireReplicaRepairItemInput) (*ReplicaRepairItem, error) { - if input.TaskID <= 0 || input.TaskClaimedAt.IsZero() || input.StorageDataSetID <= 0 || input.StorageUploadCopyID <= 0 || input.BucketID <= 0 { - return nil, fmt.Errorf("acquiring replica repair item: %w", ErrInvalidInput) - } - - var item ReplicaRepairItem - err := r.runMaybeTx(ctx, func(db bun.IDB) error { - copyRow := new(model.StorageUploadCopy) - err := db.NewRaw(`UPDATE storage_upload_copies - SET updated_at = updated_at - WHERE id = ? - RETURNING *`, input.StorageUploadCopyID).Scan(ctx, copyRow) - if err != nil { - if err == sql.ErrNoRows { - return ErrReplicaRepairItemCancelled - } - return fmt.Errorf("locking replica repair copy: %w", err) - } - - now := time.Now() - task := new(model.Task) - err = db.NewRaw(`UPDATE tasks - SET status = status - WHERE id = ? - AND status = ? - AND claimed_at = ? - AND lease_until IS NOT NULL - AND lease_until > ? - RETURNING *`, input.TaskID, model.TaskStatusRunning, input.TaskClaimedAt, now).Scan(ctx, task) - if err != nil { - if err == sql.ErrNoRows { - return ErrTaskClaimLost - } - return fmt.Errorf("locking replica repair task claim: %w", err) - } - - if copyRow.Status == model.StorageUploadCopyStatusFailed { - return ErrReplicaRepairItemCancelled - } - if copyRow.StorageDataSetID == nil || *copyRow.StorageDataSetID != input.StorageDataSetID { - return fmt.Errorf("replica repair copy data set mismatch: %w", ErrConflict) - } - - dataSet := new(model.StorageDataSet) - if err := db.NewSelect().Model(dataSet).Where("id = ?", input.StorageDataSetID).Scan(ctx); err != nil { - if err == sql.ErrNoRows { - return ErrReplicaRepairItemCancelled - } - return fmt.Errorf("loading replica repair data set: %w", err) - } - upload := new(model.StorageUpload) - if err := db.NewSelect().Model(upload).Where("id = ?", copyRow.UploadID).Scan(ctx); err != nil { - if err == sql.ErrNoRows { - return ErrReplicaRepairItemCancelled - } - return fmt.Errorf("loading replica repair upload: %w", err) - } - if dataSet.BucketID != input.BucketID || upload.BucketID != input.BucketID || task.Type != model.TaskTypeUpload || task.RefType != "bucket" || task.RefID != input.BucketID { - return fmt.Errorf("replica repair bucket identity mismatch: %w", ErrConflict) - } - if copyRow.CopyIndex != dataSet.CopyIndex { - return fmt.Errorf("replica repair copy slot mismatch: %w", ErrConflict) - } - if copyRow.ProviderID == nil || !copyRow.ProviderID.Equal(dataSet.ProviderID) { - return fmt.Errorf("replica repair copy provider mismatch: %w", ErrConflict) - } - - version, err := selectLiveObjectVersionForStorageUpload(ctx, db, upload, nil) - if err != nil { - return err - } - if version == nil { - return ErrReplicaRepairItemCancelled - } - if version.BucketID != input.BucketID { - return fmt.Errorf("replica repair object version bucket mismatch: %w", ErrConflict) - } - - item = ReplicaRepairItem{ - DataSet: *dataSet, - Copy: *copyRow, - Upload: *upload, - Version: *version, - } - return nil - }) - if err != nil { - return nil, fmt.Errorf("acquiring replica repair item: %w", err) - } - return &item, nil -} - -func selectLiveObjectVersionForStorageUpload( - ctx context.Context, - db bun.IDB, - upload *model.StorageUpload, - excludedVersionIDs []string, -) (*model.ObjectVersion, error) { - if upload == nil || upload.ID <= 0 { - return nil, fmt.Errorf("selecting live storage upload version: %w", ErrInvalidInput) - } - version := new(model.ObjectVersion) - q := db.NewSelect(). - Model(version). - Where("is_delete_marker = ?", false). - Where(objectVersionReferencesStorageUploadIDSQL, upload.ID, upload.SourceVersionID) - if len(excludedVersionIDs) > 0 { - q = q.Where("version_id NOT IN (?)", bun.List(excludedVersionIDs)) - } - err := q. - OrderExpr("in_cache DESC"). - OrderExpr("CASE WHEN version_id = ? THEN 0 ELSE 1 END ASC", upload.SourceVersionID). - OrderExpr("is_current DESC"). - OrderExpr("created_at DESC"). - OrderExpr("version_id DESC"). - Limit(1). - Scan(ctx) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("selecting live storage upload version: %w", err) - } - return version, nil -} - -func (r *BunStorageUploadRepo) NextIncompleteCopyForDataSet(ctx context.Context, storageDataSetID int64) (*model.StorageUploadCopy, error) { - copyRow := new(model.StorageUploadCopy) - err := r.db.NewSelect(). - Model(copyRow). - Join("JOIN storage_uploads AS storage_upload ON storage_upload.id = storage_upload_copy.upload_id"). - Where("storage_upload_copy.storage_data_set_id = ?", storageDataSetID). - Where("storage_upload_copy.status IN (?)", bun.List([]model.StorageUploadCopyStatus{ - model.StorageUploadCopyStatusPending, - model.StorageUploadCopyStatusPieceReady, - model.StorageUploadCopyStatusCommitting, - })). - Where(`EXISTS ( - SELECT 1 FROM object_versions AS repair_version - WHERE repair_version.is_delete_marker = ? - AND `+objectVersionReferencesStorageUploadSQL("repair_version", "storage_upload")+` - )`, false). - OrderExpr("storage_upload_copy.id ASC"). - Limit(1). - Scan(ctx) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("selecting next incomplete data set copy: %w", err) - } - return copyRow, nil -} - -func (r *BunStorageUploadRepo) NextFinalizableCopyForDataSet(ctx context.Context, storageDataSetID int64) (*model.StorageUploadCopy, error) { - copyRow := new(model.StorageUploadCopy) - query := fmt.Sprintf(`SELECT storage_copy.* - FROM storage_upload_copies AS storage_copy - JOIN storage_uploads AS storage_upload ON storage_upload.id = storage_copy.upload_id - WHERE storage_copy.storage_data_set_id = ? - AND storage_copy.status = ? - AND storage_upload.status IN (?, ?, ?) - AND storage_upload.requested_copies > 0 - AND EXISTS ( - SELECT 1 FROM object_versions AS pending_version - WHERE pending_version.storage_upload_id = storage_upload.id - AND pending_version.state = ? - ) - AND ( - SELECT COUNT(DISTINCT readable_data_set.copy_index) - FROM storage_upload_copies AS readable_copy - JOIN storage_data_sets AS readable_data_set ON readable_data_set.id = readable_copy.storage_data_set_id - WHERE readable_copy.upload_id = storage_upload.id - AND %s - ) >= storage_upload.requested_copies - ORDER BY storage_copy.id ASC - LIMIT 1`, readableCommittedCopyPredicateWithDataSetStatusSQL( - "readable_copy", - "readable_data_set", - fmt.Sprintf("(readable_data_set.status IN (%s) OR readable_data_set.id = ?)", - storageHealthReadyDataSetStatusListSQL()), - )) - err := r.db.NewRaw( - query, - storageDataSetID, - model.StorageUploadCopyStatusCommitted, - model.StorageUploadStatusRunning, - model.StorageUploadStatusIngressReady, - model.StorageUploadStatusReadable, - model.ObjectStateReplicating, - storageDataSetID, - ).Scan(ctx, copyRow) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("selecting next finalizable data set copy: %w", err) - } - return copyRow, nil -} - -func (r *BunStorageUploadRepo) ListUnavailableDataSetsWithIncompleteCopies(ctx context.Context, afterID int64, limit int) ([]model.StorageDataSet, error) { - var bindings []model.StorageDataSet - q := r.db.NewSelect(). - Model(&bindings). - Where("status = ?", model.StorageDataSetStatusUnavailable). - Where("id > ?", afterID). - Where(`EXISTS ( - SELECT 1 FROM storage_upload_copies AS storage_copy - JOIN storage_uploads AS storage_upload ON storage_upload.id = storage_copy.upload_id - WHERE storage_copy.storage_data_set_id = storage_data_set.id - AND storage_copy.status IN (?, ?, ?) - AND EXISTS ( - SELECT 1 FROM object_versions AS repair_version - WHERE repair_version.is_delete_marker = ? - AND `+objectVersionReferencesStorageUploadSQL("repair_version", "storage_upload")+` - ) - )`, - model.StorageUploadCopyStatusPending, - model.StorageUploadCopyStatusPieceReady, - model.StorageUploadCopyStatusCommitting, - false, - ). - OrderExpr("id ASC") - if limit > 0 { - q = q.Limit(limit) - } - if err := q.Scan(ctx); err != nil { - return nil, fmt.Errorf("listing unavailable data sets with incomplete copies: %w", err) - } - return bindings, nil -} - -func (r *BunStorageUploadRepo) ListIncompleteReadableUploads( - ctx context.Context, - afterID int64, - limit int, -) ([]IncompleteReadableUpload, error) { - var uploads []model.StorageUpload - q := r.db.NewSelect(). - Model(&uploads). - Where("status = ?", model.StorageUploadStatusReadable). - Where("id > ?", afterID). - Where(`EXISTS ( - SELECT 1 FROM object_versions AS live_version - WHERE live_version.is_delete_marker = ? - AND live_version.state IN (?, ?) - AND `+objectVersionReferencesStorageUploadSQL("live_version", "storage_upload")+` - )`, false, model.ObjectStateStored, model.ObjectStateCacheEvicted). - OrderExpr("id ASC") - if limit > 0 { - q = q.Limit(limit) - } - if err := q.Scan(ctx); err != nil { - return nil, fmt.Errorf("listing incomplete readable storage uploads: %w", err) - } - - items := make([]IncompleteReadableUpload, 0, len(uploads)) - for i := range uploads { - version := new(model.ObjectVersion) - err := r.db.NewSelect(). - Model(version). - Where("is_delete_marker = ?", false). - Where("state IN (?, ?)", model.ObjectStateStored, model.ObjectStateCacheEvicted). - Where(objectVersionReferencesStorageUploadIDSQL, uploads[i].ID, uploads[i].SourceVersionID). - OrderExpr("in_cache DESC"). - OrderExpr("is_current DESC"). - OrderExpr("created_at DESC"). - OrderExpr("version_id DESC"). - Limit(1). - Scan(ctx) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - continue - } - return nil, fmt.Errorf("selecting durable version for incomplete readable upload %d: %w", uploads[i].ID, err) - } - items = append(items, IncompleteReadableUpload{ - Upload: uploads[i], - Version: *version, - }) - } - return items, nil -} - -func (r *BunStorageUploadRepo) ReassignIngressCopy(ctx context.Context, uploadID int64, unavailableCopyIndex int) (*model.StorageUploadCopy, error) { - var selected *model.StorageUploadCopy - err := r.runMaybeTx(ctx, func(db bun.IDB) error { - candidate := new(model.StorageUploadCopy) - err := db.NewSelect(). - Model(candidate). - Join("JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_upload_copy.storage_data_set_id"). - Where("storage_upload_copy.upload_id = ?", uploadID). - Where("storage_upload_copy.copy_index <> ?", unavailableCopyIndex). - Where("storage_upload_copy.status = ?", model.StorageUploadCopyStatusPending). - Where("storage_upload_copy.transfer_method = ?", model.StorageCopyTransferMethodPeerPull). - Where("storage_data_set.status = ?", model.StorageDataSetStatusReady). - Where("storage_data_set.is_current"). - OrderExpr("storage_upload_copy.copy_index ASC"). - Limit(1). - Scan(ctx) - if err == sql.ErrNoRows { - return nil - } - if err != nil { - return fmt.Errorf("selecting alternate ingress copy: %w", err) - } - unavailableCopyID, err := resolveSlotCopyID(ctx, db, uploadID, unavailableCopyIndex) - if err != nil { - return err - } - res, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("transfer_method = ?", model.StorageCopyTransferMethodPeerPull). - Set("commit_ready_at = NULL"). - Set("commit_attempt_id = NULL"). - Set("commit_attempted_at = NULL"). - Set("commit_submission_json = NULL"). - Set("commit_extra_data_hex = NULL"). - Set("commit_transaction_id = NULL"). - Set("commit_confirmed_transaction_id = NULL"). - Set("commit_attention_code = NULL"). - Set("commit_attention_at = NULL"). - Where("id = ?", unavailableCopyID). - Where("transfer_method = ?", model.StorageCopyTransferMethodIngress). - Where("status <> ?", model.StorageUploadCopyStatusCommitted). - Where("NOT " + attemptedStorageCommitSQL("storage_upload_copy")). - Exec(ctx) - if err != nil { - return fmt.Errorf("releasing unavailable ingress copy: %w", err) - } - rows, err := res.RowsAffected() - if err != nil { - return fmt.Errorf("releasing unavailable ingress copy: reading affected rows: %w", err) - } - if rows != 1 { - return fmt.Errorf("releasing unavailable ingress copy: %w", ErrConflict) - } - res, err = db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("transfer_method = ?", model.StorageCopyTransferMethodIngress). - Set("updated_at = ?", time.Now()). - Where("id = ?", candidate.ID). - Where("status = ?", model.StorageUploadCopyStatusPending). - Where("transfer_method = ?", model.StorageCopyTransferMethodPeerPull). - Exec(ctx) - if err != nil { - return fmt.Errorf("assigning alternate ingress copy: %w", err) - } - rows, err = res.RowsAffected() - if err != nil { - return fmt.Errorf("assigning alternate ingress copy: reading affected rows: %w", err) - } - if rows != 1 { - return fmt.Errorf("assigning alternate ingress copy: %w", ErrConflict) - } - candidate.TransferMethod = model.StorageCopyTransferMethodIngress - selected = candidate - return nil - }) - return selected, err -} - -func (r *BunStorageUploadRepo) MarkUploadCopyPieceReady(ctx context.Context, input MarkUploadCopyPieceReadyInput) error { - return r.runMaybeTx(ctx, func(db bun.IDB) error { - if err := lockStorageUploadForCopyMutation(ctx, db, input.UploadID); err != nil { - return fmt.Errorf("locking storage upload for piece-ready copy: %w", err) - } - copyID, err := slotCopyTarget(ctx, db, input.StorageUploadCopyID, input.UploadID, input.CopyIndex) - if err != nil { - return err - } - now := time.Now() - q := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("status = ?", model.StorageUploadCopyStatusPieceReady). - Set("piece_id = COALESCE(?, piece_id)", input.PieceID). - Set("retrieval_url = COALESCE(?, retrieval_url)", nullableString(input.RetrievalURL)). - Set("commit_extra_data_hex = COALESCE(?, commit_extra_data_hex)", nullableString(input.CommitExtraDataHex)). - Set("last_error = NULL"). - Set("updated_at = ?", now). - Where("id = ?", copyID). - Where("status NOT IN (?, ?)", model.StorageUploadCopyStatusCommitted, model.StorageUploadCopyStatusCommitting) - if input.PieceCID != "" { - q = q.Where(`EXISTS ( - SELECT 1 FROM storage_uploads AS evidence_upload - WHERE evidence_upload.id = ? - AND (evidence_upload.piece_cid IS NULL OR evidence_upload.piece_cid = '' OR evidence_upload.piece_cid = ?) - )`, input.UploadID, input.PieceCID) - } - if input.PieceID != nil { - q = q.Where("(piece_id IS NULL OR piece_id = ?)", input.PieceID) - } - if input.RetrievalURL != "" { - q = q.Where("(retrieval_url IS NULL OR retrieval_url = '' OR retrieval_url = ?)", input.RetrievalURL) - } - if input.CommitExtraDataHex != "" { - q = q.Where("(commit_extra_data_hex IS NULL OR commit_extra_data_hex = '' OR commit_extra_data_hex = ?)", input.CommitExtraDataHex) - } - if input.RequireEligibleCopy { - q = q. - Where("status <> ?", model.StorageUploadCopyStatusFailed). - Where(liveObjectVersionExistsForUploadSQL(), input.UploadID, false) - } - res, err := q.Exec(ctx) - if err != nil { - return fmt.Errorf("marking storage upload copy piece ready: %w", err) - } - rows, _ := res.RowsAffected() - if rows == 0 { - var status model.StorageUploadCopyStatus - if err := db.NewSelect().Model((*model.StorageUploadCopy)(nil)). - Column("status").Where("id = ?", copyID).Scan(ctx, &status); err != nil { - return fmt.Errorf("loading storage upload copy after piece evidence conflict: %w", err) - } - if status == model.StorageUploadCopyStatusCommitting { - compatible := db.NewSelect(). - Model((*model.StorageUploadCopy)(nil)). - Where("id = ?", copyID). - Where("status = ?", model.StorageUploadCopyStatusCommitting). - Where("commit_attempt_id IS NOT NULL AND commit_attempt_id <> ''"). - Where("commit_attempted_at IS NOT NULL") - if input.PieceCID != "" { - compatible = compatible.Where(`EXISTS ( - SELECT 1 FROM storage_uploads AS evidence_upload - WHERE evidence_upload.id = ? - AND evidence_upload.piece_cid = ? - )`, input.UploadID, input.PieceCID) - } - if input.PieceID != nil { - compatible = compatible.Where("piece_id = ?", input.PieceID) - } - if input.RetrievalURL != "" { - compatible = compatible.Where("retrieval_url = ?", input.RetrievalURL) - } - if input.CommitExtraDataHex != "" { - compatible = compatible.Where("commit_extra_data_hex = ?", input.CommitExtraDataHex) - } - count, err := compatible.Count(ctx) - if err != nil { - return fmt.Errorf("checking idempotent piece evidence for committing copy: %w", err) - } - if count == 1 { - return nil - } - } - // A late piece-ready result cannot regress a committed copy. Treat that - // stale observation as a harmless no-op; every other zero-row result is - // conflicting monotonic evidence and must stop before Commit. - if status == model.StorageUploadCopyStatusCommitted && !input.RequireEligibleCopy { - return nil - } - return fmt.Errorf("marking storage upload copy piece ready: %w", ErrConflict) - } - transferMethod, err := uploadCopyTransferMethod(ctx, db, input.UploadID, input.CopyIndex) - if err != nil { - return err - } - if transferMethod == model.StorageCopyTransferMethodIngress { - if err := updateUploadIngressReady(ctx, db, input.UploadID, input.PieceCID, now); err != nil { - return err - } - } else { - if _, err := db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("piece_cid = COALESCE(?, piece_cid)", nullableString(input.PieceCID)). - Set("updated_at = ?", now). - Where("id = ?", input.UploadID). - Exec(ctx); err != nil { - return fmt.Errorf("recording storage upload piece CID: %w", err) - } - } - return nil - }) -} - -func (r *BunStorageUploadRepo) MarkUploadCopyCommitted(ctx context.Context, input MarkUploadCopyCommittedInput) error { - if input.UploadID <= 0 || input.CopyIndex < 0 || input.PieceCID == "" || input.PieceID == nil || input.RetrievalURL == "" || - (input.CommitAttemptID != "" && input.StorageUploadCopyID <= 0) { - return fmt.Errorf("marking storage upload copy committed: %w", ErrInvalidInput) - } - return r.runMaybeTx(ctx, func(db bun.IDB) error { - var copyID int64 - if input.CommitAttemptID != "" { - initial := new(model.StorageUploadCopy) - if err := db.NewSelect(). - Model(initial). - Where("id = ?", input.StorageUploadCopyID). - Where("upload_id = ?", input.UploadID). - Where("copy_index = ?", input.CopyIndex). - Scan(ctx); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return fmt.Errorf("loading storage commit copy: %w", ErrNotFound) - } - return fmt.Errorf("loading storage commit copy: %w", err) - } - if initial.StorageDataSetID == nil { - return fmt.Errorf("locking storage commit copy without data set: %w", ErrConflict) - } - lockedCopyID, _, err := lockCommitCopyFamily(ctx, db, storagecommit.CopyIdentity{ - StorageUploadCopyID: input.StorageUploadCopyID, - UploadID: input.UploadID, - CopyIndex: input.CopyIndex, - StorageDataSetID: *initial.StorageDataSetID, - // Settling an attempted commit deliberately skips the owner - // eligibility check: the piece already reached the provider, so a - // deleted owner must not stop the copy from being recorded. - RequireEligibleCopy: false, - }) - if err != nil { - return err - } - copyID = lockedCopyID - } else { - if err := lockStorageUploadForCopyMutation(ctx, db, input.UploadID); err != nil { - return fmt.Errorf("locking storage upload for committed copy: %w", err) - } - var err error - copyID, err = slotCopyTarget(ctx, db, input.StorageUploadCopyID, input.UploadID, input.CopyIndex) - if err != nil { - return err - } - } - now := time.Now() - isNewDataSet, err := uploadCopyDataSetCreatedByUpload(ctx, db, copyID) - if err != nil { - return err - } - q := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("status = ?", model.StorageUploadCopyStatusCommitted). - Set("piece_id = COALESCE(?, piece_id)", input.PieceID). - Set("retrieval_url = COALESCE(?, retrieval_url)", nullableString(input.RetrievalURL)). - Set("is_new_data_set = ?", isNewDataSet). - Set("commit_extra_data_hex = COALESCE(?, commit_extra_data_hex)", nullableString(input.CommitExtraDataHex)). - Set("commit_transaction_id = COALESCE(?, commit_transaction_id)", nullableString(input.CommitTransactionID)). - Set("commit_confirmed_transaction_id = COALESCE(?, commit_confirmed_transaction_id)", nullableString(input.CommitConfirmedTransactionID)). - Set("last_error = NULL"). - Set("updated_at = ?", now). - Where("id = ?", copyID) - q = q.Where(`EXISTS ( - SELECT 1 FROM storage_uploads AS evidence_upload - WHERE evidence_upload.id = ? - AND (evidence_upload.piece_cid IS NULL OR evidence_upload.piece_cid = '' OR evidence_upload.piece_cid = ?) - )`, input.UploadID, input.PieceCID) - if input.PieceID != nil { - q = q.Where("(piece_id IS NULL OR piece_id = ?)", input.PieceID) - } - if input.RetrievalURL != "" { - q = q.Where("(retrieval_url IS NULL OR retrieval_url = '' OR retrieval_url = ?)", input.RetrievalURL) - } - if input.CommitExtraDataHex != "" { - q = q.Where("(commit_extra_data_hex IS NULL OR commit_extra_data_hex = '' OR commit_extra_data_hex = ?)", input.CommitExtraDataHex) - } - if input.CommitTransactionID != "" { - q = q.Where("(commit_transaction_id IS NULL OR commit_transaction_id = '' OR commit_transaction_id = ?)", input.CommitTransactionID) - } - if input.CommitAttemptID != "" { - q = q. - Set("commit_ready_at = NULL"). - Set("commit_attempt_id = NULL"). - Set("commit_attempted_at = NULL"). - Set("commit_submission_json = NULL"). - Set("commit_attention_code = NULL"). - Set("commit_attention_at = NULL"). - Where("commit_attempt_id = ?", input.CommitAttemptID) - } - if input.RequireEligibleCopy && input.CommitAttemptID == "" { - q = q. - Where("status <> ?", model.StorageUploadCopyStatusFailed). - Where(liveObjectVersionExistsForUploadSQL(), input.UploadID, false) - } - res, err := q.Exec(ctx) - if err != nil { - return fmt.Errorf("marking storage upload copy committed: %w", err) - } - rows, _ := res.RowsAffected() - if rows == 0 { - if input.RequireEligibleCopy { - return fmt.Errorf("marking storage upload copy committed: %w", ErrConflict) - } - return fmt.Errorf("marking storage upload copy committed: %w", ErrNotFound) - } - if err := updateUploadReadable(ctx, db, input.UploadID, input.PieceCID, now); err != nil { - return err - } - return nil - }) -} - -func liveObjectVersionExistsForUploadSQL() string { - return `EXISTS ( - SELECT 1 - FROM storage_uploads AS guarded_upload - JOIN object_versions AS live_version - ON ` + objectVersionReferencesStorageUploadSQL("live_version", "guarded_upload") + ` - WHERE guarded_upload.id = ? - AND live_version.is_delete_marker = ? - )` -} - -func (r *BunStorageUploadRepo) MarkUploadCopyFailed(ctx context.Context, input MarkUploadCopyFailedInput) error { - uploadID, copyIndex, lastError := input.UploadID, input.CopyIndex, input.LastError - return r.runMaybeTx(ctx, func(db bun.IDB) error { - if err := lockStorageUploadForCopyMutation(ctx, db, uploadID); err != nil { - return fmt.Errorf("locking storage upload for failed copy: %w", err) - } - copyID, err := slotCopyTarget(ctx, db, input.StorageUploadCopyID, uploadID, copyIndex) - if err != nil { - return err - } - now := time.Now() - res, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("status = ?", model.StorageUploadCopyStatusFailed). - Set("commit_ready_at = NULL"). - Set("commit_attempt_id = NULL"). - Set("commit_attempted_at = NULL"). - Set("commit_submission_json = NULL"). - Set("commit_extra_data_hex = NULL"). - Set("commit_transaction_id = NULL"). - Set("commit_confirmed_transaction_id = NULL"). - Set("commit_attention_code = NULL"). - Set("commit_attention_at = NULL"). - Set("last_error = ?", lastError). - Set("updated_at = ?", now). - Where("id = ?", copyID). - Where("status <> ?", model.StorageUploadCopyStatusCommitted). - Where("NOT " + attemptedStorageCommitSQL("storage_upload_copy")). - Exec(ctx) - if err != nil { - return fmt.Errorf("marking storage upload copy failed: %w", err) - } - rows, rowsErr := res.RowsAffected() - if rowsErr != nil { - return fmt.Errorf("marking storage upload copy failed: reading affected rows: %w", rowsErr) - } - if rows == 0 { - submittedCount, countErr := db.NewSelect(). - Model((*model.StorageUploadCopy)(nil)). - Where("id = ?", copyID). - Where(attemptedStorageCommitSQL("storage_upload_copy")). - Count(ctx) - if countErr != nil { - return fmt.Errorf("checking submitted storage commit before failure: %w", countErr) - } - if submittedCount > 0 { - return fmt.Errorf("marking storage upload copy failed: submitted commit is still recoverable: %w", ErrConflict) - } - return nil - } - readableCount, err := countReadableReplicaSlots(ctx, db, uploadID) - if err != nil { - return err - } - submittedCount, err := countSubmittedCommitCopies(ctx, db, uploadID) - if err != nil { - return err - } - if readableCount == 0 && submittedCount == 0 { - _, err = db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("status = ?", model.StorageUploadStatusFailed). - Set("error_message = ?", lastError). - Set("updated_at = ?", now). - Where("id = ?", uploadID). - Where("status IN (?)", bun.List(activeUploadStatuses())). - Exec(ctx) - if err != nil { - return fmt.Errorf("marking storage upload failed: %w", err) - } - } else if readableCount > 0 { - _, err = db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("status = ?", model.StorageUploadStatusReadable). - Set("error_message = ?", lastError). - Set("updated_at = ?", now). - Where("id = ?", uploadID). - Where("status IN (?)", bun.List([]model.StorageUploadStatus{ - model.StorageUploadStatusReadable, - })). - Exec(ctx) - if err != nil { - return fmt.Errorf("marking storage upload readable after copy failure: %w", err) - } - } - return nil - }) -} - -func countSubmittedCommitCopies(ctx context.Context, db bun.IDB, uploadID int64) (int, error) { - count, err := db.NewSelect(). - Model((*model.StorageUploadCopy)(nil)). - Where("upload_id = ?", uploadID). - Where(attemptedStorageCommitSQL("storage_upload_copy")). - Count(ctx) - if err != nil { - return 0, fmt.Errorf("counting submitted storage commits: %w", err) - } - return count, nil -} - -func (r *BunStorageUploadRepo) BindReadableUploadForContent(ctx context.Context, input BindReadableUploadInput) ([]ObjectVersionRef, error) { - var refs []ObjectVersionRef - err := r.runMaybeTx(ctx, func(db bun.IDB) error { - upload, err := lockStorageUploadForObjectState(ctx, db, input.UploadID, model.ObjectStateReplicating) - if err != nil { - return fmt.Errorf("locking storage upload for readable bind: %w", err) - } - now := time.Now() - if err := updateUploadReadable(ctx, db, input.UploadID, derefString(upload.PieceCID), now); err != nil { - return err - } - // Bind the source committing version and matching waiting followers. - // Followers with their own active upload/task are left untouched. - query := `UPDATE object_versions - SET storage_upload_id = ?, state = ?, failed_at_state = NULL, last_error = NULL, updated_at = ? - WHERE bucket_id = ? AND size = ? AND checksum = ? - AND ( - (version_id = ? AND state IN (?, ?, ?)) - OR ( - state = ? - AND NOT EXISTS ( - SELECT 1 FROM storage_uploads AS active_upload - WHERE active_upload.source_version_id = object_versions.version_id - AND active_upload.id <> ? - AND active_upload.status IN ('running', 'ingress_ready', 'readable') - ) - AND NOT EXISTS ( - SELECT 1 FROM tasks AS active_task - WHERE active_task.ref_type = 'object' - AND active_task.ref_version_id = object_versions.version_id - AND active_task.type = ? - AND active_task.status IN (?) - ) - ) - ) - RETURNING object_id, version_id` - err = db.NewRaw(query, - input.UploadID, model.ObjectStateReplicating, now, - input.BucketID, input.ContentSize, input.Checksum, - upload.SourceVersionID, model.ObjectStateUploading, model.ObjectStateCommitting, model.ObjectStateFailed, - model.ObjectStateUploading, - input.UploadID, - model.TaskTypeUpload, bun.List(activeTaskStatuses()), - ).Scan(ctx, &refs) - if err != nil && err != sql.ErrNoRows { - return fmt.Errorf("binding readable upload for content: %w", err) - } - return nil - }) - return refs, err -} - -func (r *BunStorageUploadRepo) BindReadableUploadForVersion(ctx context.Context, input BindReadableUploadForVersionInput) ([]ObjectVersionRef, error) { - var refs []ObjectVersionRef - err := r.runMaybeTx(ctx, func(db bun.IDB) error { - upload, err := lockStorageUploadForObjectState(ctx, db, input.UploadID, model.ObjectStateReplicating) - if err != nil { - return fmt.Errorf("locking storage upload for version readable bind: %w", err) - } - now := time.Now() - if err := updateUploadReadable(ctx, db, input.UploadID, derefString(upload.PieceCID), now); err != nil { - return err - } - query := `UPDATE object_versions - SET storage_upload_id = ?, state = ?, failed_at_state = NULL, last_error = NULL, updated_at = ? - WHERE version_id = ? AND bucket_id = ? AND size = ? AND checksum = ? AND state = ? - AND NOT EXISTS ( - SELECT 1 FROM storage_uploads AS active_upload - WHERE active_upload.source_version_id = object_versions.version_id - AND active_upload.id <> ? - AND active_upload.status IN ('running', 'ingress_ready', 'readable') - ) - RETURNING object_id, version_id` - err = db.NewRaw(query, - input.UploadID, model.ObjectStateReplicating, now, - input.VersionID, input.BucketID, input.ContentSize, input.Checksum, model.ObjectStateUploading, - input.UploadID, - ).Scan(ctx, &refs) - if err != nil && err != sql.ErrNoRows { - return fmt.Errorf("binding readable upload for version: %w", err) - } - if len(refs) > 0 { - if err := completeUploadTasksForVersion(ctx, db, input.VersionID, now, activeTaskStatuses()); err != nil { - return err - } - } - return nil - }) - return refs, err -} - -func (r *BunStorageUploadRepo) FinalizeUploadIfTargetCopiesMet(ctx context.Context, input FinalizeUploadInput) (bool, []ObjectVersionRef, error) { - var refs []ObjectVersionRef - finalized := false - err := r.runMaybeTx(ctx, func(db bun.IDB) error { - var bucketID int64 - if err := db.NewSelect(). - Model((*model.StorageUpload)(nil)). - Column("bucket_id"). - Where("id = ?", input.UploadID). - Scan(ctx, &bucketID); err != nil { - if err == sql.ErrNoRows { - return fmt.Errorf("loading storage upload %d: %w", input.UploadID, ErrNotFound) - } - return fmt.Errorf("loading storage upload bucket: %w", err) - } - bucket, err := lockBucketByID(ctx, db, bucketID) - if err != nil { - return err - } - if bucket == nil { - return fmt.Errorf("storage upload %d bucket not found: %w", input.UploadID, ErrNotFound) - } - locked, err := lockStorageUploadsByID(ctx, db, []int64{input.UploadID}) - if err != nil { - return fmt.Errorf("locking storage upload for finalization: %w", err) - } - upload := locked[input.UploadID] - if upload == nil || upload.Status == model.StorageUploadStatusRejected || upload.Status == model.StorageUploadStatusSuperseded { - return fmt.Errorf("storage upload %d cannot be finalized: %w", input.UploadID, ErrConflict) - } - readable, err := countReadableReplicaSlots(ctx, db, input.UploadID) - if err != nil { - return err - } - minimum := minimumDurableCopiesForUpload(bucket, upload.RequestedCopies) - if minimum <= 0 || readable < minimum { - return nil - } - now := time.Now() - err = db.NewRaw(`UPDATE object_versions - SET state = ?, updated_at = ? - WHERE storage_upload_id = ? AND state = ? - RETURNING object_id, version_id`, - model.ObjectStateStored, now, input.UploadID, model.ObjectStateReplicating, - ).Scan(ctx, &refs) - if err != nil && err != sql.ErrNoRows { - return fmt.Errorf("marking durable object versions stored: %w", err) - } - for _, ref := range refs { - if input.EnqueueAfterUploadEviction { - evictions := &BunCacheEvictionRepo{db: db} - if _, err := evictions.EnsureAfterUploadTask( - ctx, - ref.ObjectID, - ref.VersionID, - input.EvictionMaxRetries, - ); err != nil { - return fmt.Errorf( - "creating after-upload eviction task for version %s: %w", - ref.VersionID, - err, - ) - } - } - } - if upload.RequestedCopies <= 0 || readable < upload.RequestedCopies { - return nil - } - _, err = db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("status = ?", model.StorageUploadStatusComplete). - Set("accepted_at = COALESCE(accepted_at, ?)", now). - Set("accept_error = NULL"). - Set("updated_at = ?", now). - Where("id = ?", input.UploadID). - Exec(ctx) - if err != nil { - return fmt.Errorf("marking storage upload complete: %w", err) - } - if err := completeUploadTasksForUpload(ctx, db, input.UploadID, now, unclaimedTaskStatuses()); err != nil { - return err - } - finalized = true - return nil - }) - return finalized, refs, err -} - -func minimumDurableCopiesForUpload(bucket *model.Bucket, requestedCopies int) int { - if requestedCopies <= 0 { - return 0 - } - if bucket == nil || bucket.MinimumDurableCopies == nil || *bucket.MinimumDurableCopies >= requestedCopies { - return requestedCopies - } - return *bucket.MinimumDurableCopies -} - -func requireCurrentMinimumDurableCopies(ctx context.Context, db bun.IDB, upload *model.StorageUpload) error { - if upload == nil { - return fmt.Errorf("storage upload is required: %w", ErrInvalidInput) - } - bucket := new(model.Bucket) - if err := db.NewSelect().Model(bucket).Where("id = ?", upload.BucketID).Scan(ctx); err != nil { - if err == sql.ErrNoRows { - return fmt.Errorf("storage upload bucket not found: %w", ErrNotFound) - } - return fmt.Errorf("loading storage upload bucket durability policy: %w", err) - } - minimum := minimumDurableCopiesForUpload(bucket, upload.RequestedCopies) - readable, err := countReadableReplicaSlots(ctx, db, upload.ID) - if err != nil { - return err - } - if minimum <= 0 || readable < minimum { - return fmt.Errorf("storage upload %d has %d of %d required durable copies: %w", upload.ID, readable, minimum, ErrConflict) - } - return nil -} - -func (r *BunStorageUploadRepo) FindActiveUploadBySourceVersion(ctx context.Context, versionID string) (*model.StorageUpload, error) { - return r.findActiveUploadBySourceVersion(ctx, versionID) -} - -func (r *BunStorageUploadRepo) FindLatestUploadBySourceVersion(ctx context.Context, versionID string) (*model.StorageUpload, error) { - upload := new(model.StorageUpload) - err := r.db.NewSelect(). - Model(upload). - Where("source_version_id = ?", versionID). - OrderExpr("id DESC"). - Limit(1). - Scan(ctx) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("selecting latest storage upload: %w", err) - } - return upload, nil -} - -func (r *BunStorageUploadRepo) FindLatestUploadsBySourceVersions(ctx context.Context, versionIDs []string) (map[string]model.StorageUpload, error) { - uploadsByVersionID := make(map[string]model.StorageUpload, len(versionIDs)) - if len(versionIDs) == 0 { - return uploadsByVersionID, nil - } - var uploads []model.StorageUpload - err := r.db.NewRaw(`SELECT storage_upload.* - FROM storage_uploads AS storage_upload - JOIN ( - SELECT source_version_id, MAX(id) AS id - FROM storage_uploads - WHERE source_version_id IN (?) - GROUP BY source_version_id - ) AS latest_upload ON latest_upload.id = storage_upload.id`, - bun.List(versionIDs), - ).Scan(ctx, &uploads) - if err != nil { - return nil, fmt.Errorf("selecting latest storage uploads by source version: %w", err) - } - for _, upload := range uploads { - uploadsByVersionID[upload.SourceVersionID] = upload - } - return uploadsByVersionID, nil -} - -func (r *BunStorageUploadRepo) SetAcceptError(ctx context.Context, uploadID int64, message string) error { - _, err := r.db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("accept_error = ?", message). - Set("updated_at = ?", time.Now()). - Where("id = ?", uploadID). - Exec(ctx) - if err != nil { - return fmt.Errorf("setting storage upload accept error: %w", err) - } - return nil -} - -func (r *BunStorageUploadRepo) runMaybeTx(ctx context.Context, fn func(bun.IDB) error) error { - return runMaybeTx(ctx, r.db, fn) -} - -func (r *BunStorageUploadRepo) findActiveUploadBySourceVersion(ctx context.Context, versionID string) (*model.StorageUpload, error) { - upload := new(model.StorageUpload) - err := r.db.NewSelect(). - Model(upload). - Where("source_version_id = ?", versionID). - Where("status IN (?)", bun.List(activeUploadStatuses())). - OrderExpr("id DESC"). - Limit(1). - Scan(ctx) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("selecting active storage upload: %w", err) - } - return upload, nil -} - -// Generations are numbered per replica slot and never reused, so a retired -// generation stays distinguishable from the one that replaced it. -func nextDataSetGeneration(ctx context.Context, db bun.IDB, bucketID int64, copyIndex int) (int, error) { - var generation int - err := db.NewRaw( - `SELECT COALESCE(MAX(generation), 0) + 1 FROM storage_data_sets WHERE bucket_id = ? AND copy_index = ?`, - bucketID, copyIndex, - ).Scan(ctx, &generation) - if err != nil { - return 0, fmt.Errorf("selecting next storage data set generation: %w", err) - } - return generation, nil -} - -func ensureDataSetBinding(ctx context.Context, db bun.IDB, input EnsureDataSetBindingInput) (*model.StorageDataSet, error) { - if input.BucketID == 0 || input.ProviderID.IsZero() || input.CopyIndex < 0 { - return nil, fmt.Errorf("invalid storage data set binding input: %w", ErrInvalidInput) - } - // Only live generations reserve a provider or a slot; a retired generation - // leaves both free so an operator can reuse a provider they used before. - existingByProvider := new(model.StorageDataSet) - err := db.NewSelect(). - Model(existingByProvider). - Where("bucket_id = ? AND provider_id = ? AND is_current", input.BucketID, input.ProviderID). - Scan(ctx) - if err == nil { - if existingByProvider.CopyIndex != input.CopyIndex { - return nil, fmt.Errorf("provider %s already bound to copy_index %d: %w", input.ProviderID, existingByProvider.CopyIndex, ErrAlreadyExists) - } - return existingByProvider, nil - } - if err != sql.ErrNoRows { - return nil, fmt.Errorf("selecting storage data set by provider: %w", err) - } - existingByIndex := new(model.StorageDataSet) - err = db.NewSelect(). - Model(existingByIndex). - Where("bucket_id = ? AND copy_index = ? AND is_current", input.BucketID, input.CopyIndex). - Scan(ctx) - if err == nil { - return nil, fmt.Errorf("copy_index %d already bound to provider %s: %w", input.CopyIndex, existingByIndex.ProviderID, ErrAlreadyExists) - } - if err != sql.ErrNoRows { - return nil, fmt.Errorf("selecting storage data set by copy index: %w", err) - } - generation, err := nextDataSetGeneration(ctx, db, input.BucketID, input.CopyIndex) - if err != nil { - return nil, err - } - now := time.Now() - binding := &model.StorageDataSet{ - BucketID: input.BucketID, - ProviderID: input.ProviderID, - CopyIndex: input.CopyIndex, - Generation: generation, - IsCurrent: true, - Status: model.StorageDataSetStatusPending, - CreatedByUploadID: nullableInt64(input.CreatedByUploadID), - LastUsedUploadID: nullableInt64(input.CreatedByUploadID), - CreatedAt: now, - UpdatedAt: now, - } - res, err := db.NewInsert(). - Model(binding). - On("CONFLICT DO NOTHING"). - Exec(ctx) - if err != nil { - return nil, fmt.Errorf("inserting storage data set binding: %w", err) - } - rows, _ := res.RowsAffected() - if rows == 0 { - existing := new(model.StorageDataSet) - selectErr := db.NewSelect(). - Model(existing). - Where("bucket_id = ? AND provider_id = ? AND is_current", input.BucketID, input.ProviderID). - Scan(ctx) - if selectErr == nil && existing.CopyIndex == input.CopyIndex { - return existing, nil - } - if selectErr != nil && selectErr != sql.ErrNoRows { - return nil, fmt.Errorf("selecting storage data set after conflict: %w", selectErr) - } - return nil, fmt.Errorf("storage data set binding already exists: %w", ErrAlreadyExists) - } - return binding, nil -} - -func markDataSetReady(ctx context.Context, db bun.IDB, id int64, uploadID int64, dataSetID types.OnChainID, clientDataSetID *types.OnChainID) error { - if dataSetID.IsZero() { - return fmt.Errorf("dataSetID is required: %w", ErrInvalidInput) - } - res, err := db.NewUpdate(). - Model((*model.StorageDataSet)(nil)). - Set("status = ?", model.StorageDataSetStatusReady). - Set("data_set_id = ?", dataSetID). - Set("client_data_set_id = COALESCE(?, client_data_set_id)", clientDataSetID). - Set("last_used_upload_id = ?", nullableInt64(uploadID)). - Set("last_error = NULL"). - Set("updated_at = ?", time.Now()). - Where("id = ?", id). - Where(`NOT EXISTS ( - SELECT 1 FROM storage_data_sets AS other - WHERE other.id <> ? - AND other.provider_id = (SELECT provider_id FROM storage_data_sets WHERE id = ?) - AND other.data_set_id = ? - )`, id, id, dataSetID). - Exec(ctx) - if err != nil { - return fmt.Errorf("marking storage data set ready: %w", err) - } - rows, _ := res.RowsAffected() - if rows > 0 { - return nil - } - count, err := db.NewSelect(). - Model((*model.StorageDataSet)(nil)). - Where("id = ?", id). - Count(ctx) - if err != nil { - return fmt.Errorf("checking storage data set ready result: %w", err) - } - if count == 0 { - return fmt.Errorf("storage data set %d not found: %w", id, ErrNotFound) - } - return fmt.Errorf("provider data set already bound to another bucket: %w", ErrAlreadyExists) -} - -// The result counts logical replica slots, so several data set generations of -// one slot never inflate an upload's durability. -func countReadableReplicaSlots(ctx context.Context, db bun.IDB, uploadID int64) (int, error) { - var row struct { - Count int `bun:"count"` - } - err := db.NewRaw(fmt.Sprintf(`SELECT %s AS count`, - distinctReadableSlotCountSQL("storage_copy", "storage_data_set", "?"), - ), - uploadID, - ).Scan(ctx, &row) - if err != nil { - return 0, fmt.Errorf("counting readable replica slots: %w", err) - } - return row.Count, nil -} - -func requireReadableCommittedCopy(ctx context.Context, db bun.IDB, uploadID int64) error { - count, err := countReadableReplicaSlots(ctx, db, uploadID) - if err != nil { - return err - } - if count == 0 { - return fmt.Errorf("storage upload %d has no readable committed copy: %w", uploadID, ErrNotFound) - } - return nil -} - -// Addressing a copy by slot alone became ambiguous once a slot can own several -// generations, so every write resolves to one concrete row first. Returning -// zero means the slot has no copy yet; an ambiguous slot is a conflict rather -// than a silent multi-row update. -// slotCopyTarget picks the concrete copy a mutation must touch. Zero means the -// slot has no copy, which every caller already handles as "no rows updated". -func slotCopyTarget(ctx context.Context, db bun.IDB, copyID, uploadID int64, copyIndex int) (int64, error) { - if copyID > 0 { - return copyID, nil - } - return resolveSlotCopyID(ctx, db, uploadID, copyIndex) -} - -func resolveSlotCopyID(ctx context.Context, db bun.IDB, uploadID int64, copyIndex int) (int64, error) { - var ids []int64 - err := db.NewSelect(). - Model((*model.StorageUploadCopy)(nil)). - Column("id"). - Where("storage_upload_copy.upload_id = ? AND storage_upload_copy.copy_index = ?", uploadID, copyIndex). - Where(currentGenerationCopySQL("storage_upload_copy")). - Scan(ctx, &ids) - if err != nil { - return 0, fmt.Errorf("resolving storage upload copy for slot: %w", err) - } - switch len(ids) { - case 0: - return 0, nil - case 1: - return ids[0], nil - default: - return 0, fmt.Errorf( - "storage upload %d replica slot %d matches %d copies: %w", - uploadID, copyIndex, len(ids), ErrConflict, - ) - } -} - -func uploadCopyTransferMethod(ctx context.Context, db bun.IDB, uploadID int64, copyIndex int) (model.StorageCopyTransferMethod, error) { - var row struct { - TransferMethod model.StorageCopyTransferMethod `bun:"transfer_method"` - } - err := db.NewSelect(). - Model((*model.StorageUploadCopy)(nil)). - Column("transfer_method"). - Where("storage_upload_copy.upload_id = ? AND storage_upload_copy.copy_index = ?", uploadID, copyIndex). - Where(currentGenerationCopySQL("storage_upload_copy")). - OrderExpr("(storage_upload_copy.storage_data_set_id IS NULL) ASC"). - Limit(1). - Scan(ctx, &row) - if err != nil { - if err == sql.ErrNoRows { - return "", nil - } - return "", fmt.Errorf("loading storage upload copy transfer method: %w", err) - } - return row.TransferMethod, nil -} - -func uploadCopyDataSetCreatedByUpload(ctx context.Context, db bun.IDB, copyID int64) (bool, error) { - var row struct { - IsNewDataSet bool `bun:"is_new_data_set"` - } - err := db.NewRaw(`SELECT CASE - WHEN storage_data_set.created_by_upload_id = storage_copy.upload_id THEN TRUE - ELSE FALSE - END AS is_new_data_set - FROM storage_upload_copies AS storage_copy - LEFT JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id - WHERE storage_copy.id = ?`, copyID).Scan(ctx, &row) - if err != nil { - if err == sql.ErrNoRows { - return false, nil - } - return false, fmt.Errorf("checking storage upload copy data set origin: %w", err) - } - return row.IsNewDataSet, nil -} - -func storageDataSetCreatedByUpload(ctx context.Context, db bun.IDB, storageDataSetID int64, uploadID int64) (bool, error) { - count, err := db.NewSelect(). - Model((*model.StorageDataSet)(nil)). - Where("id = ? AND created_by_upload_id = ?", storageDataSetID, uploadID). - Count(ctx) - if err != nil { - return false, fmt.Errorf("checking storage data set origin: %w", err) - } - return count > 0, nil -} - -func updateUploadIngressReady(ctx context.Context, db bun.IDB, uploadID int64, pieceCID string, now time.Time) error { - _, err := db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("status = ?", model.StorageUploadStatusIngressReady). - Set("piece_cid = COALESCE(?, piece_cid)", nullableString(pieceCID)). - Set("ingress_bytes_transferred = content_size"). - Set("progress_updated_at = ?", now). - Set("updated_at = ?", now). - Where("id = ?", uploadID). - Where("status IN (?)", bun.List([]model.StorageUploadStatus{ - model.StorageUploadStatusRunning, - model.StorageUploadStatusIngressReady, - model.StorageUploadStatusFailed, - })). - Exec(ctx) - if err != nil { - return fmt.Errorf("marking storage upload ingress ready: %w", err) - } - return nil -} - -func updateUploadReadable(ctx context.Context, db bun.IDB, uploadID int64, pieceCID string, now time.Time) error { - _, err := db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("status = ?", model.StorageUploadStatusReadable). - Set("piece_cid = COALESCE(?, piece_cid)", nullableString(pieceCID)). - Set("updated_at = ?", now). - Where("id = ?", uploadID). - Where("status IN (?)", bun.List([]model.StorageUploadStatus{ - model.StorageUploadStatusRunning, - model.StorageUploadStatusIngressReady, - model.StorageUploadStatusReadable, - model.StorageUploadStatusFailed, - })). - Exec(ctx) - if err != nil { - return fmt.Errorf("marking storage upload readable: %w", err) - } - return nil -} - -func completeUploadTasksForVersion(ctx context.Context, db bun.IDB, versionID string, now time.Time, statuses []model.TaskStatus) error { - if versionID == "" { - return nil - } - // Callers must pass an explicit active task status set. An empty set would - // silently leave upload tasks active after the upload becomes readable or - // stored. - if len(statuses) == 0 { - return fmt.Errorf("completing upload tasks for bound version: %w", ErrInvalidInput) - } - _, err := db.NewUpdate(). - Model((*model.Task)(nil)). - Set("status = ?", model.TaskStatusCompleted). - Set("completed_at = ?", now). - Set("last_error = NULL"). - Set("wait_reason = NULL"). - Set("status_message = NULL"). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("started_at = NULL"). - Where("ref_type = ? AND ref_version_id = ? AND type = ?", "object", versionID, model.TaskTypeUpload). - Where("status IN (?)", bun.List(statuses)). - Exec(ctx) - if err != nil { - return fmt.Errorf("completing upload tasks for bound version: %w", err) - } - return nil -} - -func completeUploadTasksForUpload(ctx context.Context, db bun.IDB, uploadID int64, now time.Time, statuses []model.TaskStatus) error { - if uploadID <= 0 || len(statuses) == 0 { - return fmt.Errorf("completing upload tasks for storage upload: %w", ErrInvalidInput) - } - _, err := db.NewUpdate(). - Model((*model.Task)(nil)). - Set("status = ?", model.TaskStatusCompleted). - Set("completed_at = ?", now). - Set("last_error = NULL"). - Set("wait_reason = NULL"). - Set("status_message = NULL"). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("started_at = NULL"). - Where("ref_type = ? AND type = ?", "object", model.TaskTypeUpload). - Where("status IN (?)", bun.List(statuses)). - Where("ref_version_id IN (SELECT version_id FROM object_versions WHERE storage_upload_id = ?)", uploadID). - Exec(ctx) - if err != nil { - return fmt.Errorf("completing upload tasks for storage upload: %w", err) - } - return nil -} - -func activeUploadStatuses() []model.StorageUploadStatus { - return []model.StorageUploadStatus{ - model.StorageUploadStatusRunning, - model.StorageUploadStatusIngressReady, - model.StorageUploadStatusReadable, - } -} - -func nullableString(value string) *string { - if value == "" { - return nil - } - return &value -} - -func nullableInt64(value int64) *int64 { - if value == 0 { - return nil - } - return &value -} - -func derefString(value *string) string { - if value == nil { - return "" - } - return *value -} diff --git a/internal/db/repository/storage_upload_repo_internal_test.go b/internal/db/repository/storage_upload_repo_internal_test.go deleted file mode 100644 index 9986268..0000000 --- a/internal/db/repository/storage_upload_repo_internal_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package repository - -import ( - "context" - "errors" - "testing" - "time" -) - -func TestCompleteUploadTasksForVersionRequiresStatuses(t *testing.T) { - if err := completeUploadTasksForVersion(context.Background(), nil, "", time.Now(), nil); err != nil { - t.Fatalf("empty versionID error = %v, want nil", err) - } - - err := completeUploadTasksForVersion(context.Background(), nil, "version-1", time.Now(), nil) - if !errors.Is(err, ErrInvalidInput) { - t.Fatalf("empty statuses error = %v, want ErrInvalidInput", err) - } -} diff --git a/internal/db/repository/storage_upload_repo_test.go b/internal/db/repository/storage_upload_repo_test.go deleted file mode 100644 index d53cc90..0000000 --- a/internal/db/repository/storage_upload_repo_test.go +++ /dev/null @@ -1,2988 +0,0 @@ -package repository_test - -import ( - "context" - "database/sql" - "errors" - "path/filepath" - "reflect" - "strconv" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/strahe/synaps3/internal/cacheeviction" - "github.com/strahe/synaps3/internal/db/migrations" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/observability" - "github.com/strahe/synaps3/internal/storagecommit" - "github.com/strahe/synaps3/internal/types" - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect/sqlitedialect" -) - -func TestStorageUploadRepo_RecordCompleteResultAndAcceptsUploadingContent(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "upload-provenance-bucket") - - version := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000010001", 10) - objectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version) - if err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("committing: %v", err) - } - task := seedRunningUploadTask(t, repos, objectID, version.VersionID) - - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceTaskID: task.ID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - seedCommittedUploadCopies(t, repos, bucket.ID, upload.ID, "bafk2bzaceprovenance", []storageUploadCopySeed{ - {ProviderID: onChainIDPtr(t, "101"), DataSetID: onChainIDPtr(t, "1001"), PieceID: onChainIDPtr(t, "2001"), TransferMethod: model.StorageCopyTransferMethodIngress, RetrievalURL: strPtr("https://ingress.example/piece"), IsNewDataSet: true}, - {ProviderID: onChainIDPtr(t, "202"), DataSetID: onChainIDPtr(t, "2002"), PieceID: onChainIDPtr(t, "3001"), TransferMethod: model.StorageCopyTransferMethodPeerPull, RetrievalURL: strPtr("https://peer.example/piece"), IsNewDataSet: true}, - }) - refs := bindReadableUploadForContent(t, repos, upload.ID, bucket.ID, version.Size, version.Checksum) - finalizeUploadForTest(t, repos, upload.ID) - if len(refs) != 1 || refs[0].VersionID != version.VersionID { - t.Fatalf("accepted refs = %#v, want source version", refs) - } - - got, err := repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil || got == nil { - t.Fatalf("GetVersionByID: got=%v err=%v", got, err) - } - if got.State != model.ObjectStateStored { - t.Fatalf("state = %s, want stored", got.State) - } - if got.StorageUploadID == nil || *got.StorageUploadID != upload.ID { - t.Fatalf("storage_upload_id = %#v, want %d", got.StorageUploadID, upload.ID) - } - if got.PieceCID == nil || *got.PieceCID != "bafk2bzaceprovenance" { - t.Fatalf("piece_cid = %#v, want derived piece cid", got.PieceCID) - } - if !got.InFilecoin { - t.Fatal("in_filecoin = false, want derived true") - } - - _ = task -} - -func TestStorageUploadRepo_StartObjectUploadAttemptRequiresRequestedCopies(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - tests := []struct { - name string - requestedCopies int - }{ - {name: "zero", requestedCopies: 0}, - {name: "negative", requestedCopies: -1}, - {name: "too-high", requestedCopies: 9}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - bucket := seedBucket(t, db, "upload-invalid-copies-"+tt.name) - - if _, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J000000000000000DEF" + strings.ToUpper(tt.name), - ContentSize: 10, - Checksum: "checksum-invalid-copies-" + tt.name, - RequestedCopies: tt.requestedCopies, - }); err == nil { - t.Fatal("StartObjectUploadAttempt succeeded, want error") - } - }) - } -} - -func TestStorageUploadRepo_OnChainIDsRoundTripLargeValuesAndZeroPieceID(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "onchain-id-round-trip-bucket") - - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J000000000000000000BIG01", - ContentSize: 10, - Checksum: "checksum-onchain-id", - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - providerID := onChainID(t, "18446744073709551616") - dataSetID := onChainID(t, "18446744073709551617") - clientDataSetID := onChainIDPtr(t, "0") - pieceID := onChainIDPtr(t, "0") - - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: providerID, - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - gotPending, err := repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) - if err != nil || gotPending == nil { - t.Fatalf("GetDataSetBindingByCopyIndex pending: binding=%v err=%v", gotPending, err) - } - gotPendingByID, err := repos.Uploads.GetDataSetBindingByID(ctx, binding.ID) - if err != nil || gotPendingByID == nil { - t.Fatalf("GetDataSetBindingByID pending: binding=%v err=%v", gotPendingByID, err) - } - if gotPendingByID.ID != binding.ID || gotPendingByID.BucketID != bucket.ID || gotPendingByID.CopyIndex != 0 { - t.Fatalf("binding by ID = %#v, want id/bucket/copy index", gotPendingByID) - } - if gotPending.DataSetID != nil || gotPending.ClientDataSetID != nil { - t.Fatalf("pending binding ids = data:%v client:%v, want nil SQL NULLs", gotPending.DataSetID, gotPending.ClientDataSetID) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: upload.ID, - DataSetID: dataSetID, - ClientDataSetID: clientDataSetID, - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: providerID}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "piece-onchain-id", - PieceID: pieceID, - RetrievalURL: "https://provider.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - - gotBinding, err := repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) - if err != nil || gotBinding == nil { - t.Fatalf("GetDataSetBindingByCopyIndex ready: binding=%v err=%v", gotBinding, err) - } - if gotBinding.ProviderID.String() != providerID.String() || gotBinding.DataSetID == nil || gotBinding.DataSetID.String() != dataSetID.String() || gotBinding.ClientDataSetID == nil || gotBinding.ClientDataSetID.String() != "0" { - t.Fatalf("ready binding = %#v, want large provider/data set and client 0", gotBinding) - } - gotReadyByID, err := repos.Uploads.GetDataSetBindingByID(ctx, binding.ID) - if err != nil || gotReadyByID == nil { - t.Fatalf("GetDataSetBindingByID ready: binding=%v err=%v", gotReadyByID, err) - } - if gotReadyByID.DataSetID == nil || gotReadyByID.DataSetID.String() != dataSetID.String() { - t.Fatalf("ready binding by ID = %#v, want data set %s", gotReadyByID, dataSetID.String()) - } - copies, err := repos.Uploads.ListCopies(ctx, upload.ID) - if err != nil { - t.Fatalf("ListCopies: %v", err) - } - if len(copies) != 1 || copies[0].ProviderID == nil || copies[0].ProviderID.String() != providerID.String() || copies[0].DataSetID == nil || copies[0].DataSetID.String() != dataSetID.String() || copies[0].PieceID == nil || copies[0].PieceID.String() != "0" { - t.Fatalf("copy = %#v, want large IDs and piece ID 0", copies) - } -} - -func TestStorageUploadRepo_BackfillClientDataSetIDPreservesLifecycleAndRejectsConflicts(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "client-data-set-id-backfill-bucket") - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J000000000000000BACKFILL", - ContentSize: 10, - Checksum: "checksum-client-data-set-id-backfill", - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - dataSetID := onChainID(t, "1001") - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, UploadID: upload.ID, DataSetID: dataSetID, - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.MarkDataSetDraining(ctx, binding.ID, "replacement in progress"); err != nil { - t.Fatalf("MarkDataSetDraining: %v", err) - } - - clientDataSetID := onChainID(t, "9001") - input := repository.BackfillClientDataSetIDInput{ - ID: binding.ID, DataSetID: dataSetID, ClientDataSetID: clientDataSetID, - } - if err := repos.Uploads.BackfillClientDataSetID(ctx, input); err != nil { - t.Fatalf("BackfillClientDataSetID: %v", err) - } - if err := repos.Uploads.BackfillClientDataSetID(ctx, input); err != nil { - t.Fatalf("BackfillClientDataSetID idempotent call: %v", err) - } - got, err := repos.Uploads.GetDataSetBindingByID(ctx, binding.ID) - if err != nil || got == nil { - t.Fatalf("GetDataSetBindingByID: binding=%#v err=%v", got, err) - } - if got.Status != model.StorageDataSetStatusDraining || got.LastError == nil || *got.LastError != "replacement in progress" { - t.Fatalf("binding lifecycle = status:%s error:%v, want unchanged draining state", got.Status, got.LastError) - } - if got.ClientDataSetID == nil || !got.ClientDataSetID.Equal(clientDataSetID) { - t.Fatalf("client data set ID = %v, want %s", got.ClientDataSetID, clientDataSetID.String()) - } - - conflict := input - conflict.ClientDataSetID = onChainID(t, "9002") - if err := repos.Uploads.BackfillClientDataSetID(ctx, conflict); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("conflicting BackfillClientDataSetID error = %v, want ErrConflict", err) - } -} - -func TestStorageUploadRepo_PrimaryStoreProgressTracksAttemptsAndClamps(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "primary-store-progress-bucket") - - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J000000000000000000PRG01", - ContentSize: 10, - Checksum: "checksum-primary-progress", - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - if upload.IngressBytesTransferred != 0 || upload.IngressStoreAttempt != 0 || upload.ProgressUpdatedAt != nil { - t.Fatalf("new upload progress = bytes:%d attempt:%d updated:%v, want zero values", upload.IngressBytesTransferred, upload.IngressStoreAttempt, upload.ProgressUpdatedAt) - } - - attemptOne, err := repos.Uploads.BeginIngressStoreProgress(ctx, upload.ID) - if err != nil { - t.Fatalf("BeginIngressStoreProgress first: %v", err) - } - if attemptOne.IngressStoreAttempt != 1 || attemptOne.IngressBytesTransferred != 0 || attemptOne.ProgressUpdatedAt == nil { - t.Fatalf("first attempt progress = bytes:%d attempt:%d updated:%v, want reset attempt 1", attemptOne.IngressBytesTransferred, attemptOne.IngressStoreAttempt, attemptOne.ProgressUpdatedAt) - } - - if _, err := repos.Uploads.RecordIngressStoreProgress(ctx, repository.RecordIngressStoreProgressInput{ - UploadID: upload.ID, - Attempt: attemptOne.IngressStoreAttempt, - BytesUploaded: 7, - }); err != nil { - t.Fatalf("RecordIngressStoreProgress 7: %v", err) - } - if _, err := repos.Uploads.RecordIngressStoreProgress(ctx, repository.RecordIngressStoreProgressInput{ - UploadID: upload.ID, - Attempt: attemptOne.IngressStoreAttempt, - BytesUploaded: 4, - }); err != nil { - t.Fatalf("RecordIngressStoreProgress stale bytes: %v", err) - } - if _, err := repos.Uploads.RecordIngressStoreProgress(ctx, repository.RecordIngressStoreProgressInput{ - UploadID: upload.ID, - Attempt: attemptOne.IngressStoreAttempt, - BytesUploaded: 99, - }); err != nil { - t.Fatalf("RecordIngressStoreProgress clamp: %v", err) - } - got, err := repos.Uploads.GetByID(ctx, upload.ID) - if err != nil || got == nil { - t.Fatalf("GetByID after progress: got=%v err=%v", got, err) - } - if got.IngressBytesTransferred != 10 { - t.Fatalf("primary bytes after attempt one = %d, want clamped content size", got.IngressBytesTransferred) - } - - attemptTwo, err := repos.Uploads.BeginIngressStoreProgress(ctx, upload.ID) - if err != nil { - t.Fatalf("BeginIngressStoreProgress second: %v", err) - } - if attemptTwo.IngressStoreAttempt != 2 || attemptTwo.IngressBytesTransferred != 0 { - t.Fatalf("second attempt progress = bytes:%d attempt:%d, want reset attempt 2", attemptTwo.IngressBytesTransferred, attemptTwo.IngressStoreAttempt) - } - if _, err := repos.Uploads.RecordIngressStoreProgress(ctx, repository.RecordIngressStoreProgressInput{ - UploadID: upload.ID, - Attempt: attemptOne.IngressStoreAttempt, - BytesUploaded: 8, - }); err != nil { - t.Fatalf("RecordIngressStoreProgress old attempt: %v", err) - } - got, err = repos.Uploads.GetByID(ctx, upload.ID) - if err != nil || got == nil { - t.Fatalf("GetByID after old attempt: got=%v err=%v", got, err) - } - if got.IngressStoreAttempt != 2 || got.IngressBytesTransferred != 0 { - t.Fatalf("old attempt progress changed current attempt = bytes:%d attempt:%d, want reset attempt 2", got.IngressBytesTransferred, got.IngressStoreAttempt) - } -} - -func TestStorageUploadRepo_GetUploadProvenanceIncludesCopiesAndFailures(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "upload-provenance-detail-bucket") - - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J000000000000000PROV001", - ContentSize: 10, - Checksum: "checksum-provenance-detail", - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - seedCommittedUploadCopies(t, repos, bucket.ID, upload.ID, "bafk2bzaceprovenancedetail", []storageUploadCopySeed{ - {ProviderID: onChainIDPtr(t, "101"), DataSetID: onChainIDPtr(t, "1001"), PieceID: onChainIDPtr(t, "2001"), TransferMethod: model.StorageCopyTransferMethodIngress, RetrievalURL: strPtr("https://ingress.example/piece"), IsNewDataSet: true}, - {ProviderID: onChainIDPtr(t, "202"), DataSetID: onChainIDPtr(t, "2002"), PieceID: onChainIDPtr(t, "3001"), TransferMethod: model.StorageCopyTransferMethodPeerPull, RetrievalURL: strPtr("https://peer.example/piece")}, - }) - if err := repos.Uploads.AppendUploadFailure(ctx, repository.AppendUploadFailureInput{ - UploadID: upload.ID, - ProviderID: onChainIDPtr(t, "303"), - TransferMethod: string(model.StorageCopyTransferMethodPeerPull), - Stage: "peer_pull", - ErrorMessage: "provider timed out", - Explicit: true, - }); err != nil { - t.Fatalf("AppendUploadFailure: %v", err) - } - - got, err := repos.Uploads.GetUploadProvenance(ctx, upload.ID) - if err != nil { - t.Fatalf("GetUploadProvenance: %v", err) - } - if got == nil || got.Upload.ID != upload.ID { - t.Fatalf("provenance upload = %#v, want upload %d", got, upload.ID) - } - if len(got.Copies) != 2 { - t.Fatalf("copies len = %d, want 2", len(got.Copies)) - } - if !got.Copies[0].IsNewDataSet { - t.Fatalf("primary is_new_data_set = false, want true") - } - if got.Copies[1].IsNewDataSet { - t.Fatalf("secondary is_new_data_set = true, want false") - } - if got.Copies[1].DataSetID == nil || got.Copies[1].DataSetID.String() != "2002" { - t.Fatalf("secondary data_set_id = %#v, want 2002", got.Copies[1].DataSetID) - } - if len(got.Failures) != 1 { - t.Fatalf("failures len = %d, want 1", len(got.Failures)) - } - if got.Failures[0].ProviderID == nil || got.Failures[0].ProviderID.String() != "303" || got.Failures[0].Stage == nil || *got.Failures[0].Stage != "peer_pull" { - t.Fatalf("failure = %#v, want provider 303 stage peer_pull", got.Failures[0]) - } -} - -func TestStorageUploadRepo_ListBucketStorageHealthSummariesClassifiesRetainedVersionRisk(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "storage-health-risk-bucket") - otherBucket := seedBucket(t, db, "storage-health-other-bucket") - checkedAt := time.Date(2026, 5, 23, 12, 0, 0, 0, time.UTC) - staleBefore := checkedAt.Add(-time.Hour) - - redundantVersion := newObjectVersion(bucket.ID, "redundant.txt", "01J0000000000000000SHREDU", 4) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, redundantVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent redundant: %v", err) - } - redundantUpload := startCopyHealthUpload(t, repos, bucket.ID, redundantVersion.VersionID, redundantVersion.Size, redundantVersion.Checksum, 2) - degraded := commitStorageHealthCopy(t, repos, bucket.ID, redundantUpload.ID, 0, "101", "2101", "3101", "https://provider.example/degraded") - readable := commitStorageHealthCopy(t, repos, bucket.ID, redundantUpload.ID, 1, "202", "2202", "3202", "https://provider.example/readable") - bindStorageHealthVersion(t, repos, bucket.ID, redundantUpload.ID, redundantVersion) - - unavailableVersion := newObjectVersion(bucket.ID, "unavailable.txt", "01J0000000000000000SHUNAV", 5) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, unavailableVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent unavailable: %v", err) - } - unavailableUpload := startCopyHealthUpload(t, repos, bucket.ID, unavailableVersion.VersionID, unavailableVersion.Size, unavailableVersion.Checksum, 1) - unavailable := commitStorageHealthCopy(t, repos, bucket.ID, unavailableUpload.ID, 2, "303", "2303", "3303", "https://provider.example/unavailable") - bindStorageHealthVersion(t, repos, bucket.ID, unavailableUpload.ID, unavailableVersion) - if err := repos.Uploads.MarkDataSetUnavailable(ctx, unavailable.ID, "provider offline"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - - unknownVersion := newObjectVersion(bucket.ID, "unknown.txt", "01J0000000000000000SHUNKN", 6) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, unknownVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent unknown: %v", err) - } - unknownUpload := startCopyHealthUpload(t, repos, bucket.ID, unknownVersion.VersionID, unknownVersion.Size, unknownVersion.Checksum, 1) - commitStorageHealthCopy(t, repos, bucket.ID, unknownUpload.ID, 3, "404", "2404", "3404", "https://provider.example/unknown") - bindStorageHealthVersion(t, repos, bucket.ID, unknownUpload.ID, unknownVersion) - - oldVersion := newObjectVersion(bucket.ID, "old.txt", "01J0000000000000000SHOLD1", 7) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent old: %v", err) - } - oldUpload := startCopyHealthUpload(t, repos, bucket.ID, oldVersion.VersionID, oldVersion.Size, oldVersion.Checksum, 1) - oldUnavailable := commitStorageHealthCopy(t, repos, bucket.ID, oldUpload.ID, 4, "505", "2505", "3505", "https://provider.example/old") - bindStorageHealthVersion(t, repos, bucket.ID, oldUpload.ID, oldVersion) - replacementVersion := newObjectVersion(bucket.ID, "old.txt", "01J0000000000000000SHCURR", 8) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, replacementVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent replacement: %v", err) - } - - deleteVersion := newObjectVersion(bucket.ID, "deleted.txt", "01J0000000000000000SHDEL1", 9) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, deleteVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent delete data: %v", err) - } - deleteUpload := startCopyHealthUpload(t, repos, bucket.ID, deleteVersion.VersionID, deleteVersion.Size, deleteVersion.Checksum, 1) - deleteUnavailable := commitStorageHealthCopy(t, repos, bucket.ID, deleteUpload.ID, 5, "606", "2606", "3606", "https://provider.example/delete") - bindStorageHealthVersion(t, repos, bucket.ID, deleteUpload.ID, deleteVersion) - if _, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "deleted.txt", "01J0000000000000000SHDELM"); err != nil { - t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) - } - - unreferencedUpload := startCopyHealthUpload(t, repos, bucket.ID, "01J0000000000000000SHUNRF", 1, "checksum-unreferenced", 1) - unreferenced := commitStorageHealthCopy(t, repos, bucket.ID, unreferencedUpload.ID, 6, "707", "2707", "3707", "https://provider.example/unreferenced") - - otherVersion := newObjectVersion(otherBucket.ID, "other.txt", "01J0000000000000000SHOTHR", 9) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, otherVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent other: %v", err) - } - otherUpload := startCopyHealthUpload(t, repos, otherBucket.ID, otherVersion.VersionID, otherVersion.Size, otherVersion.Checksum, 1) - otherUnavailable := commitStorageHealthCopy(t, repos, otherBucket.ID, otherUpload.ID, 0, "808", "2808", "3808", "https://provider.example/other") - bindStorageHealthVersion(t, repos, otherBucket.ID, otherUpload.ID, otherVersion) - - if err := repos.Observability.ReplaceDataSetStates(ctx, checkedAt, []observability.DataSetState{ - {LocalDataSetID: degraded.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: 0, ProviderID: onChainID(t, "101"), Status: observability.StatusDegraded, ReasonCodes: []observability.ReasonCode{observability.ReasonChainDataSetUnmanaged}, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: readable.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: 1, ProviderID: onChainID(t, "202"), Status: observability.StatusAvailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: unavailable.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: 0, ProviderID: onChainID(t, "303"), Status: observability.StatusUnavailable, ReasonCodes: []observability.ReasonCode{observability.ReasonChainDataSetMissing}, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: oldUnavailable.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: 0, ProviderID: onChainID(t, "505"), Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: deleteUnavailable.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: 0, ProviderID: onChainID(t, "606"), Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: unreferenced.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: 0, ProviderID: onChainID(t, "707"), Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: otherUnavailable.ID, BucketID: otherBucket.ID, BucketName: otherBucket.Name, CopyIndex: 0, ProviderID: onChainID(t, "808"), Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - }); err != nil { - t.Fatalf("ReplaceDataSetStates: %v", err) - } - - summaries, err := repos.Uploads.ListBucketStorageHealthSummaries(ctx, bucket.ID, staleBefore, 4) - if err != nil { - t.Fatalf("ListBucketStorageHealthSummaries: %v", err) - } - if len(summaries) != 1 { - t.Fatalf("summaries len = %d, want one bucket summary: %#v", len(summaries), summaries) - } - got := summaries[0] - if got.BucketID != bucket.ID || - got.AbnormalDataSets != 6 || - got.AffectedVersionsCapped != 4 || - !got.AffectedVersionsExceedsCap || - got.AffectedVersionsCap != 4 || - !got.LocalStatusNotReady || - !got.ObservationUnavailable || - !got.ObservationDegraded || - !got.ObservationMissing || - got.ObservationStale || - !hasStorageHealthReason(got.ReasonCodes, observability.ReasonChainDataSetUnmanaged) || - !hasStorageHealthReason(got.ReasonCodes, observability.ReasonChainDataSetMissing) || - !hasStorageHealthReason(got.ReasonCodes, observability.ReasonLocalStatusNotReady) || - hasStorageHealthReason(got.ReasonCodes, observability.ReasonChainDataSetInactive) || - got.LastCheckedAt == nil || - !got.LastCheckedAt.Equal(checkedAt) { - t.Fatalf("summary = %#v, want retained version risk from affected abnormal data sets only", got) - } - - allSummaries, err := repos.Uploads.ListBucketStorageHealthSummaries(ctx, 0, staleBefore, 200) - if err != nil { - t.Fatalf("ListBucketStorageHealthSummaries all: %v", err) - } - if len(allSummaries) != 2 { - t.Fatalf("all summaries len = %d, want both buckets with abnormal data sets: %#v", len(allSummaries), allSummaries) - } -} - -func TestStorageUploadRepo_ListBucketStorageHealthSummariesReturnsHealthyBucketFreshness(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "storage-health-healthy-bucket") - checkedAt := time.Date(2026, 5, 23, 12, 0, 0, 0, time.UTC) - staleBefore := checkedAt.Add(-time.Hour) - - version := newObjectVersion(bucket.ID, "healthy.txt", "01J0000000000000000SHGOOD", 4) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - upload := startCopyHealthUpload(t, repos, bucket.ID, version.VersionID, version.Size, version.Checksum, 1) - ready := commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 0, "101", "2101", "3101", "https://provider.example/healthy") - bindStorageHealthVersion(t, repos, bucket.ID, upload.ID, version) - if err := repos.Observability.ReplaceDataSetStates(ctx, checkedAt, []observability.DataSetState{{ - LocalDataSetID: ready.ID, - BucketID: bucket.ID, - BucketName: bucket.Name, - CopyIndex: 0, - ProviderID: onChainID(t, "101"), - Status: observability.StatusAvailable, - LastCheckedAt: checkedAt, - Evidence: map[string]any{}, - }}); err != nil { - t.Fatalf("ReplaceDataSetStates: %v", err) - } - - summaries, err := repos.Uploads.ListBucketStorageHealthSummaries(ctx, bucket.ID, staleBefore, 200) - if err != nil { - t.Fatalf("ListBucketStorageHealthSummaries: %v", err) - } - if len(summaries) != 1 { - t.Fatalf("summaries len = %d, want one healthy bucket summary: %#v", len(summaries), summaries) - } - got := summaries[0] - if got.BucketID != bucket.ID || - got.AbnormalDataSets != 0 || - got.AffectedVersionsCapped != 0 || - got.AffectedVersionsExceedsCap || - got.ObservationStale || - len(got.ReasonCodes) != 0 || - got.LastCheckedAt == nil || - !got.LastCheckedAt.Equal(checkedAt) { - t.Fatalf("summary = %#v, want available bucket observation freshness without data risk", got) - } -} - -func TestStorageUploadRepo_ListBucketStorageHealthSummariesReportsNoAffectedStaleFreshness(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "storage-health-stale-unaffected-bucket") - checkedAt := time.Date(2026, 5, 23, 12, 0, 0, 0, time.UTC) - staleBefore := checkedAt.Add(-time.Hour) - staleCheckedAt := checkedAt.Add(-2 * time.Hour) - - upload := startCopyHealthUpload(t, repos, bucket.ID, "01J0000000000000000SHORPH", 1, "checksum-orphan", 1) - stale := commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 0, "101", "2101", "3101", "https://provider.example/stale-unaffected") - if err := repos.Observability.ReplaceDataSetStates(ctx, checkedAt, []observability.DataSetState{{ - LocalDataSetID: stale.ID, - BucketID: bucket.ID, - BucketName: bucket.Name, - CopyIndex: 0, - ProviderID: onChainID(t, "101"), - Status: observability.StatusAvailable, - LastCheckedAt: staleCheckedAt, - Evidence: map[string]any{}, - }}); err != nil { - t.Fatalf("ReplaceDataSetStates: %v", err) - } - - summaries, err := repos.Uploads.ListBucketStorageHealthSummaries(ctx, bucket.ID, staleBefore, 200) - if err != nil { - t.Fatalf("ListBucketStorageHealthSummaries: %v", err) - } - if len(summaries) != 1 { - t.Fatalf("summaries len = %d, want one bucket summary: %#v", len(summaries), summaries) - } - got := summaries[0] - if got.BucketID != bucket.ID || - got.AbnormalDataSets != 1 || - got.AffectedVersionsCapped != 0 || - got.AffectedVersionsExceedsCap || - !got.ObservationStale || - got.ObservationMissing || - got.ObservationUnavailable || - got.ObservationDegraded || - got.ObservationUnknown || - len(got.ReasonCodes) != 0 || - got.LastCheckedAt == nil || - !got.LastCheckedAt.Equal(staleCheckedAt) { - t.Fatalf("summary = %#v, want stale bucket observation freshness without affected retained versions", got) - } -} - -func TestStorageUploadRepo_ListBucketStorageHealthSummariesTreatsStaleOnlyRiskAsUnknown(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "storage-health-stale-bucket") - checkedAt := time.Date(2026, 5, 23, 12, 0, 0, 0, time.UTC) - staleBefore := checkedAt.Add(-time.Hour) - - version := newObjectVersion(bucket.ID, "stale.txt", "01J0000000000000000SHSTAL", 4) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - upload := startCopyHealthUpload(t, repos, bucket.ID, version.VersionID, version.Size, version.Checksum, 1) - stale := commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 0, "101", "2101", "3101", "https://provider.example/stale") - bindStorageHealthVersion(t, repos, bucket.ID, upload.ID, version) - staleCheckedAt := checkedAt.Add(-2 * time.Hour) - if err := repos.Observability.ReplaceDataSetStates(ctx, checkedAt, []observability.DataSetState{{ - LocalDataSetID: stale.ID, - BucketID: bucket.ID, - BucketName: bucket.Name, - CopyIndex: 0, - ProviderID: onChainID(t, "101"), - Status: observability.StatusAvailable, - LastCheckedAt: staleCheckedAt, - Evidence: map[string]any{}, - }}); err != nil { - t.Fatalf("ReplaceDataSetStates: %v", err) - } - - summaries, err := repos.Uploads.ListBucketStorageHealthSummaries(ctx, bucket.ID, staleBefore, 200) - if err != nil { - t.Fatalf("ListBucketStorageHealthSummaries: %v", err) - } - if len(summaries) != 1 { - t.Fatalf("summaries len = %d, want one bucket summary: %#v", len(summaries), summaries) - } - got := summaries[0] - if got.AffectedVersionsCapped != 1 || - got.AffectedVersionsCap != 200 || - got.AffectedVersionsExceedsCap || - !got.ObservationStale || - got.LastCheckedAt == nil || - !got.LastCheckedAt.Equal(staleCheckedAt) { - t.Fatalf("summary = %#v, want stale-only risk classified as unknown", got) - } -} - -func TestStorageUploadRepo_ListBucketStorageHealthAffectedVersionsReportsRetainedVersionRisk(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "storage-health-affected-bucket") - otherBucket := seedBucket(t, db, "storage-health-affected-other-bucket") - checkedAt := time.Date(2026, 5, 23, 12, 0, 0, 0, time.UTC) - staleBefore := checkedAt.Add(-time.Hour) - - currentVersion := newObjectVersion(bucket.ID, "current.txt", "01J0000000000000000SHD001", 4) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, currentVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent current: %v", err) - } - currentUpload := startCopyHealthUpload(t, repos, bucket.ID, currentVersion.VersionID, currentVersion.Size, currentVersion.Checksum, 3) - currentRiskA := commitStorageHealthCopy(t, repos, bucket.ID, currentUpload.ID, 0, "101", "2101", "3101", "https://provider.example/current-risk-a") - currentRiskB := commitStorageHealthCopy(t, repos, bucket.ID, currentUpload.ID, 1, "102", "2102", "3102", "https://provider.example/current-risk-b") - currentReadable := commitStorageHealthCopy(t, repos, bucket.ID, currentUpload.ID, 2, "103", "2103", "3103", "https://provider.example/current-readable") - bindStorageHealthVersion(t, repos, bucket.ID, currentUpload.ID, currentVersion) - - oldVersion := newObjectVersion(bucket.ID, "archive/old.txt", "01J0000000000000000SHD002", 5) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent old: %v", err) - } - oldUpload := startCopyHealthUpload(t, repos, bucket.ID, oldVersion.VersionID, oldVersion.Size, oldVersion.Checksum, 1) - oldRisk := commitStorageHealthCopy(t, repos, bucket.ID, oldUpload.ID, 3, "104", "2104", "3104", "https://provider.example/old-risk") - bindStorageHealthVersion(t, repos, bucket.ID, oldUpload.ID, oldVersion) - replacementVersion := newObjectVersion(bucket.ID, "archive/old.txt", "01J0000000000000000SHD003", 6) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, replacementVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent replacement: %v", err) - } - - trashVersion := newObjectVersion(bucket.ID, "trash.txt", "01J0000000000000000SHD004", 7) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, trashVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent trash: %v", err) - } - trashUpload := startCopyHealthUpload(t, repos, bucket.ID, trashVersion.VersionID, trashVersion.Size, trashVersion.Checksum, 1) - trashRisk := commitStorageHealthCopy(t, repos, bucket.ID, trashUpload.ID, 4, "105", "2105", "3105", "https://provider.example/trash-risk") - bindStorageHealthVersion(t, repos, bucket.ID, trashUpload.ID, trashVersion) - if _, err := repos.Objects.CreateDeleteMarkerAndSetCurrent(ctx, bucket.ID, "trash.txt", "01J0000000000000000SHD005"); err != nil { - t.Fatalf("CreateDeleteMarkerAndSetCurrent: %v", err) - } - - missingPieceVersion := newObjectVersion(bucket.ID, "missing-piece.txt", "01J0000000000000000SHD008", 9) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, missingPieceVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent missing piece: %v", err) - } - missingPieceUpload := startCopyHealthUpload(t, repos, bucket.ID, missingPieceVersion.VersionID, missingPieceVersion.Size, missingPieceVersion.Checksum, 2) - missingPieceRisk := commitStorageHealthCopy(t, repos, bucket.ID, missingPieceUpload.ID, 6, "107", "2107", "3107", "https://provider.example/missing-piece-risk") - missingPieceReadable := commitStorageHealthCopy(t, repos, bucket.ID, missingPieceUpload.ID, 7, "108", "2108", "3108", "https://provider.example/missing-piece-readable") - bindStorageHealthVersion(t, repos, bucket.ID, missingPieceUpload.ID, missingPieceVersion) - mustExec(t, db, `UPDATE storage_uploads SET piece_cid = NULL WHERE id = ?`, missingPieceUpload.ID) - - unreferencedUpload := startCopyHealthUpload(t, repos, bucket.ID, "01J0000000000000000SHD006", 1, "checksum-unreferenced-risk", 1) - unreferencedRisk := commitStorageHealthCopy(t, repos, bucket.ID, unreferencedUpload.ID, 5, "106", "2106", "3106", "https://provider.example/unreferenced-risk") - - otherVersion := newObjectVersion(otherBucket.ID, "other.txt", "01J0000000000000000SHD007", 8) - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, otherVersion); err != nil { - t.Fatalf("CreateVersionAndSetCurrent other: %v", err) - } - otherUpload := startCopyHealthUpload(t, repos, otherBucket.ID, otherVersion.VersionID, otherVersion.Size, otherVersion.Checksum, 1) - otherRisk := commitStorageHealthCopy(t, repos, otherBucket.ID, otherUpload.ID, 0, "201", "2201", "3201", "https://provider.example/other-risk") - bindStorageHealthVersion(t, repos, otherBucket.ID, otherUpload.ID, otherVersion) - - if err := repos.Observability.ReplaceDataSetStates(ctx, checkedAt, []observability.DataSetState{ - {LocalDataSetID: currentRiskA.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: currentRiskA.CopyIndex, ProviderID: currentRiskA.ProviderID, ChainDataSetID: currentRiskA.DataSetID, Status: observability.StatusDegraded, ReasonCodes: []observability.ReasonCode{observability.ReasonChainDataSetUnmanaged}, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: currentRiskB.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: currentRiskB.CopyIndex, ProviderID: currentRiskB.ProviderID, ChainDataSetID: currentRiskB.DataSetID, Status: observability.StatusUnavailable, ReasonCodes: []observability.ReasonCode{observability.ReasonChainDataSetMissing}, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: currentReadable.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: currentReadable.CopyIndex, ProviderID: currentReadable.ProviderID, ChainDataSetID: currentReadable.DataSetID, Status: observability.StatusAvailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: oldRisk.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: oldRisk.CopyIndex, ProviderID: oldRisk.ProviderID, ChainDataSetID: oldRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: trashRisk.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: trashRisk.CopyIndex, ProviderID: trashRisk.ProviderID, ChainDataSetID: trashRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: missingPieceRisk.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: missingPieceRisk.CopyIndex, ProviderID: missingPieceRisk.ProviderID, ChainDataSetID: missingPieceRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: missingPieceReadable.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: missingPieceReadable.CopyIndex, ProviderID: missingPieceReadable.ProviderID, ChainDataSetID: missingPieceReadable.DataSetID, Status: observability.StatusAvailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: unreferencedRisk.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: unreferencedRisk.CopyIndex, ProviderID: unreferencedRisk.ProviderID, ChainDataSetID: unreferencedRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: otherRisk.ID, BucketID: otherBucket.ID, BucketName: otherBucket.Name, CopyIndex: otherRisk.CopyIndex, ProviderID: otherRisk.ProviderID, ChainDataSetID: otherRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - }); err != nil { - t.Fatalf("ReplaceDataSetStates: %v", err) - } - - page, err := repos.Uploads.ListBucketStorageHealthAffectedVersions(ctx, repository.BucketStorageHealthAffectedVersionsInput{ - BucketID: bucket.ID, - StaleBefore: staleBefore, - Limit: 10, - }) - if err != nil { - t.Fatalf("ListBucketStorageHealthAffectedVersions: %v", err) - } - if page.HasMore || page.NextKeyMarker != "" || page.NextVersionIDMarker != "" || !page.NextCreatedAtMarker.IsZero() { - t.Fatalf("pagination = %#v, want single complete page", page) - } - if got, want := affectedVersionIDs(page.Versions), []string{oldVersion.VersionID, currentVersion.VersionID, missingPieceVersion.VersionID, trashVersion.VersionID}; !reflect.DeepEqual(got, want) { - t.Fatalf("version ids = %#v, want %#v", got, want) - } - current := affectedVersionByID(page.Versions, currentVersion.VersionID) - if current == nil || !current.Version.IsCurrent || current.ReadableAlternativeCount != 1 || len(current.RiskDataSets) != 2 { - t.Fatalf("current affected version = %#v, want current with two risk data sets and one alternative", current) - } - old := affectedVersionByID(page.Versions, oldVersion.VersionID) - if old == nil || old.Version.IsCurrent || old.ReadableAlternativeCount != 0 || len(old.RiskDataSets) != 1 { - t.Fatalf("old affected version = %#v, want retained old version without alternative", old) - } - trash := affectedVersionByID(page.Versions, trashVersion.VersionID) - if trash == nil || trash.Version.IsCurrent || trash.ReadableAlternativeCount != 0 || len(trash.RiskDataSets) != 1 { - t.Fatalf("trash affected version = %#v, want retained trashed data version without alternative", trash) - } - missingPiece := affectedVersionByID(page.Versions, missingPieceVersion.VersionID) - if missingPiece == nil || missingPiece.ReadableAlternativeCount != 0 || len(missingPiece.RiskDataSets) != 1 { - t.Fatalf("missing piece affected version = %#v, want no readable alternative without upload piece cid", missingPiece) - } -} - -func TestStorageUploadRepo_ListBucketStorageHealthAffectedVersionsFiltersAndPaginates(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "storage-health-affected-filter-bucket") - checkedAt := time.Date(2026, 5, 23, 12, 0, 0, 0, time.UTC) - staleBefore := checkedAt.Add(-time.Hour) - - baseCreatedAt := time.Date(2026, 5, 23, 12, 30, 0, 123456000, time.UTC) - firstOldVersion := newObjectVersion(bucket.ID, "docs/a.txt", "01J0000000000000000SHF000", 1) - firstOldVersion.CreatedAt = baseCreatedAt.Add(-time.Minute) - firstOldVersion.UpdatedAt = firstOldVersion.CreatedAt - firstVersion := newObjectVersion(bucket.ID, "docs/a.txt", "01J0000000000000000SHF001", 1) - firstVersion.CreatedAt = baseCreatedAt - firstVersion.UpdatedAt = firstVersion.CreatedAt - secondVersion := newObjectVersion(bucket.ID, "docs/b.txt", "01J0000000000000000SHF002", 1) - secondVersion.CreatedAt = baseCreatedAt.Add(time.Minute) - secondVersion.UpdatedAt = secondVersion.CreatedAt - thirdVersion := newObjectVersion(bucket.ID, "logs/c.txt", "01J0000000000000000SHF003", 1) - thirdVersion.CreatedAt = baseCreatedAt.Add(2 * time.Minute) - thirdVersion.UpdatedAt = thirdVersion.CreatedAt - for _, version := range []*model.ObjectVersion{firstOldVersion, firstVersion, secondVersion, thirdVersion} { - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent %s: %v", version.VersionID, err) - } - } - firstOldUpload := startCopyHealthUpload(t, repos, bucket.ID, firstOldVersion.VersionID, firstOldVersion.Size, firstOldVersion.Checksum, 1) - firstUpload := startCopyHealthUpload(t, repos, bucket.ID, firstVersion.VersionID, firstVersion.Size, firstVersion.Checksum, 1) - secondUpload := startCopyHealthUpload(t, repos, bucket.ID, secondVersion.VersionID, secondVersion.Size, secondVersion.Checksum, 1) - thirdUpload := startCopyHealthUpload(t, repos, bucket.ID, thirdVersion.VersionID, thirdVersion.Size, thirdVersion.Checksum, 1) - firstOldRisk := commitStorageHealthCopy(t, repos, bucket.ID, firstOldUpload.ID, 3, "304", "3304", "4304", "https://provider.example/a-old") - firstRisk := commitStorageHealthCopy(t, repos, bucket.ID, firstUpload.ID, 0, "301", "3301", "4301", "https://provider.example/a") - secondRisk := commitStorageHealthCopy(t, repos, bucket.ID, secondUpload.ID, 1, "302", "3302", "4302", "https://provider.example/b") - thirdRisk := commitStorageHealthCopy(t, repos, bucket.ID, thirdUpload.ID, 2, "303", "3303", "4303", "https://provider.example/c") - bindStorageHealthVersion(t, repos, bucket.ID, firstOldUpload.ID, firstOldVersion) - bindStorageHealthVersion(t, repos, bucket.ID, firstUpload.ID, firstVersion) - bindStorageHealthVersion(t, repos, bucket.ID, secondUpload.ID, secondVersion) - bindStorageHealthVersion(t, repos, bucket.ID, thirdUpload.ID, thirdVersion) - - if err := repos.Observability.ReplaceDataSetStates(ctx, checkedAt, []observability.DataSetState{ - {LocalDataSetID: firstOldRisk.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: firstOldRisk.CopyIndex, ProviderID: firstOldRisk.ProviderID, ChainDataSetID: firstOldRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: firstRisk.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: firstRisk.CopyIndex, ProviderID: firstRisk.ProviderID, ChainDataSetID: firstRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: secondRisk.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: secondRisk.CopyIndex, ProviderID: secondRisk.ProviderID, ChainDataSetID: secondRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: thirdRisk.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: thirdRisk.CopyIndex, ProviderID: thirdRisk.ProviderID, ChainDataSetID: thirdRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - }); err != nil { - t.Fatalf("ReplaceDataSetStates: %v", err) - } - - prefixPage, err := repos.Uploads.ListBucketStorageHealthAffectedVersions(ctx, repository.BucketStorageHealthAffectedVersionsInput{ - BucketID: bucket.ID, - Prefix: "docs/", - StaleBefore: staleBefore, - Limit: 10, - }) - if err != nil { - t.Fatalf("ListBucketStorageHealthAffectedVersions prefix: %v", err) - } - if got, want := affectedVersionIDs(prefixPage.Versions), []string{firstVersion.VersionID, firstOldVersion.VersionID, secondVersion.VersionID}; !reflect.DeepEqual(got, want) { - t.Fatalf("prefix version ids = %#v, want %#v", got, want) - } - - keyPage, err := repos.Uploads.ListBucketStorageHealthAffectedVersions(ctx, repository.BucketStorageHealthAffectedVersionsInput{ - BucketID: bucket.ID, - Key: "docs/b.txt", - StaleBefore: staleBefore, - Limit: 10, - }) - if err != nil { - t.Fatalf("ListBucketStorageHealthAffectedVersions key: %v", err) - } - if got, want := affectedVersionIDs(keyPage.Versions), []string{secondVersion.VersionID}; !reflect.DeepEqual(got, want) { - t.Fatalf("key version ids = %#v, want %#v", got, want) - } - - dataSetPage, err := repos.Uploads.ListBucketStorageHealthAffectedVersions(ctx, repository.BucketStorageHealthAffectedVersionsInput{ - BucketID: bucket.ID, - LocalDataSetID: secondRisk.ID, - StaleBefore: staleBefore, - Limit: 10, - }) - if err != nil { - t.Fatalf("ListBucketStorageHealthAffectedVersions dataset: %v", err) - } - if got, want := affectedVersionIDs(dataSetPage.Versions), []string{secondVersion.VersionID}; !reflect.DeepEqual(got, want) { - t.Fatalf("dataset version ids = %#v, want %#v", got, want) - } - - firstPage, err := repos.Uploads.ListBucketStorageHealthAffectedVersions(ctx, repository.BucketStorageHealthAffectedVersionsInput{ - BucketID: bucket.ID, - StaleBefore: staleBefore, - Limit: 1, - }) - if err != nil { - t.Fatalf("ListBucketStorageHealthAffectedVersions first page: %v", err) - } - if !firstPage.HasMore || firstPage.NextKeyMarker != "docs/a.txt" || firstPage.NextVersionIDMarker != firstVersion.VersionID || !firstPage.NextCreatedAtMarker.Equal(firstVersion.CreatedAt) { - t.Fatalf("first page pagination = %#v, want marker for docs/a.txt", firstPage) - } - secondPage, err := repos.Uploads.ListBucketStorageHealthAffectedVersions(ctx, repository.BucketStorageHealthAffectedVersionsInput{ - BucketID: bucket.ID, - KeyMarker: firstPage.NextKeyMarker, - VersionIDMarker: firstPage.NextVersionIDMarker, - CreatedAtMarker: firstPage.NextCreatedAtMarker, - StaleBefore: staleBefore, - Limit: 2, - }) - if err != nil { - t.Fatalf("ListBucketStorageHealthAffectedVersions second page: %v", err) - } - if got, want := affectedVersionIDs(secondPage.Versions), []string{firstOldVersion.VersionID, secondVersion.VersionID}; !reflect.DeepEqual(got, want) { - t.Fatalf("second page version ids = %#v, want %#v", got, want) - } - - seedPrefixRisk := func(key, versionID string, copyIndex int, providerID, dataSetID, pieceID string) *model.StorageDataSet { - t.Helper() - version := newObjectVersion(bucket.ID, key, versionID, 1) - version.CreatedAt = baseCreatedAt.Add(time.Duration(copyIndex) * time.Minute) - version.UpdatedAt = version.CreatedAt - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent %s: %v", versionID, err) - } - upload := startCopyHealthUpload(t, repos, bucket.ID, version.VersionID, version.Size, version.Checksum, 1) - risk := commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, copyIndex, providerID, dataSetID, pieceID, "https://provider.example/"+versionID) - bindStorageHealthVersion(t, repos, bucket.ID, upload.ID, version) - return risk - } - wildPercentRisk := seedPrefixRisk("wild%/literal.txt", "01J0000000000000000SHF101", 10, "401", "5401", "6401") - wildSiblingRisk := seedPrefixRisk("wildX/literal.txt", "01J0000000000000000SHF102", 11, "402", "5402", "6402") - underScoreRisk := seedPrefixRisk("under_/literal.txt", "01J0000000000000000SHF103", 12, "403", "5403", "6403") - underSiblingRisk := seedPrefixRisk("underX/literal.txt", "01J0000000000000000SHF104", 13, "404", "5404", "6404") - backslashRisk := seedPrefixRisk(`back\slash/literal.txt`, "01J0000000000000000SHF105", 14, "405", "5405", "6405") - unicodeRisk := seedPrefixRisk("¿/literal.txt", "01J0000000000000000SHF106", 15, "406", "5406", "6406") - unicodeSiblingRisk := seedPrefixRisk("?/literal.txt", "01J0000000000000000SHF107", 16, "407", "5407", "6407") - caseRisk := seedPrefixRisk("case/literal.txt", "01J0000000000000000SHF108", 17, "408", "5408", "6408") - caseSiblingRisk := seedPrefixRisk("Case/literal.txt", "01J0000000000000000SHF109", 18, "409", "5409", "6409") - if err := repos.Observability.ReplaceDataSetStates(ctx, checkedAt, []observability.DataSetState{ - {LocalDataSetID: wildPercentRisk.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: wildPercentRisk.CopyIndex, ProviderID: wildPercentRisk.ProviderID, ChainDataSetID: wildPercentRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: wildSiblingRisk.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: wildSiblingRisk.CopyIndex, ProviderID: wildSiblingRisk.ProviderID, ChainDataSetID: wildSiblingRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: underScoreRisk.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: underScoreRisk.CopyIndex, ProviderID: underScoreRisk.ProviderID, ChainDataSetID: underScoreRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: underSiblingRisk.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: underSiblingRisk.CopyIndex, ProviderID: underSiblingRisk.ProviderID, ChainDataSetID: underSiblingRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: backslashRisk.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: backslashRisk.CopyIndex, ProviderID: backslashRisk.ProviderID, ChainDataSetID: backslashRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: unicodeRisk.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: unicodeRisk.CopyIndex, ProviderID: unicodeRisk.ProviderID, ChainDataSetID: unicodeRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: unicodeSiblingRisk.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: unicodeSiblingRisk.CopyIndex, ProviderID: unicodeSiblingRisk.ProviderID, ChainDataSetID: unicodeSiblingRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: caseRisk.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: caseRisk.CopyIndex, ProviderID: caseRisk.ProviderID, ChainDataSetID: caseRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - {LocalDataSetID: caseSiblingRisk.ID, BucketID: bucket.ID, BucketName: bucket.Name, CopyIndex: caseSiblingRisk.CopyIndex, ProviderID: caseSiblingRisk.ProviderID, ChainDataSetID: caseSiblingRisk.DataSetID, Status: observability.StatusUnavailable, LastCheckedAt: checkedAt, Evidence: map[string]any{}}, - }); err != nil { - t.Fatalf("ReplaceDataSetStates special prefixes: %v", err) - } - for _, tc := range []struct { - name string - prefix string - want []string - }{ - {name: "percent", prefix: "wild%/", want: []string{"01J0000000000000000SHF101"}}, - {name: "underscore", prefix: "under_/", want: []string{"01J0000000000000000SHF103"}}, - {name: "backslash", prefix: `back\slash/`, want: []string{"01J0000000000000000SHF105"}}, - {name: "unicode", prefix: "¿/", want: []string{"01J0000000000000000SHF106"}}, - {name: "case-sensitive", prefix: "case/", want: []string{"01J0000000000000000SHF108"}}, - } { - page, err := repos.Uploads.ListBucketStorageHealthAffectedVersions(ctx, repository.BucketStorageHealthAffectedVersionsInput{ - BucketID: bucket.ID, - Prefix: tc.prefix, - StaleBefore: staleBefore, - Limit: 10, - }) - if err != nil { - t.Fatalf("ListBucketStorageHealthAffectedVersions prefix %s: %v", tc.name, err) - } - if got := affectedVersionIDs(page.Versions); !reflect.DeepEqual(got, tc.want) { - t.Fatalf("prefix %s version ids = %#v, want %#v", tc.name, got, tc.want) - } - } -} - -func TestStorageUploadRepo_StagedProvenanceInfersNewDataSetAndAppendsFailures(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "staged-provenance-detail-bucket") - - first, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J000000000000000PROV101", - ContentSize: 10, - Checksum: "checksum-staged-provenance-1", - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt first: %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: first.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, UploadID: first.ID, DataSetID: onChainID(t, "1001")}); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, first.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: onChainID(t, "101"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings first: %v", err) - } - pendingFirstProvenance, err := repos.Uploads.GetUploadProvenance(ctx, first.ID) - if err != nil { - t.Fatalf("GetUploadProvenance pending first: %v", err) - } - if len(pendingFirstProvenance.Copies) != 1 || !pendingFirstProvenance.Copies[0].IsNewDataSet { - t.Fatalf("pending first copies = %#v, want inferred new data set before commit", pendingFirstProvenance.Copies) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: first.ID, - CopyIndex: 0, - PieceCID: "piece-staged-first", - PieceID: onChainIDPtr(t, "2001"), - RetrievalURL: "https://provider.example/first", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted first: %v", err) - } - if err := repos.Uploads.AppendUploadFailure(ctx, repository.AppendUploadFailureInput{ - UploadID: first.ID, - CopyIndex: 0, - Stage: "ingress_commit", - ErrorMessage: "temporary commit failure", - ProviderID: nil, - TransferMethod: "", - Explicit: false, - }); err != nil { - t.Fatalf("AppendUploadFailure: %v", err) - } - - firstProvenance, err := repos.Uploads.GetUploadProvenance(ctx, first.ID) - if err != nil { - t.Fatalf("GetUploadProvenance first: %v", err) - } - if len(firstProvenance.Copies) != 1 || !firstProvenance.Copies[0].IsNewDataSet { - t.Fatalf("first copies = %#v, want inferred new data set", firstProvenance.Copies) - } - if len(firstProvenance.Failures) != 1 || firstProvenance.Failures[0].ProviderID == nil || firstProvenance.Failures[0].ProviderID.String() != "101" { - t.Fatalf("first failures = %#v, want provider inferred from copy", firstProvenance.Failures) - } - - second, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J000000000000000PROV102", - ContentSize: 10, - Checksum: "checksum-staged-provenance-2", - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt second: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, second.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: onChainID(t, "101"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings second: %v", err) - } - pendingSecondProvenance, err := repos.Uploads.GetUploadProvenance(ctx, second.ID) - if err != nil { - t.Fatalf("GetUploadProvenance pending second: %v", err) - } - if len(pendingSecondProvenance.Copies) != 1 || pendingSecondProvenance.Copies[0].IsNewDataSet { - t.Fatalf("pending second copies = %#v, want reused data set before commit", pendingSecondProvenance.Copies) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: second.ID, - CopyIndex: 0, - PieceCID: "piece-staged-second", - PieceID: onChainIDPtr(t, "2002"), - RetrievalURL: "https://provider.example/second", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted second: %v", err) - } - - secondProvenance, err := repos.Uploads.GetUploadProvenance(ctx, second.ID) - if err != nil { - t.Fatalf("GetUploadProvenance second: %v", err) - } - if len(secondProvenance.Copies) != 1 || secondProvenance.Copies[0].IsNewDataSet { - t.Fatalf("second copies = %#v, want reused data set", secondProvenance.Copies) - } -} - -func TestStorageUploadRepo_RequiredOnChainIDValidationUsesInvalidInput(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "onchain-id-validation-bucket") - - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J000000000000000000BAD01", - ContentSize: 10, - Checksum: "checksum-onchain-validation", - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - - if _, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: types.OnChainID{}, - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }); !errors.Is(err, repository.ErrInvalidInput) || errors.Is(err, repository.ErrNotFound) { - t.Fatalf("EnsureDataSetBinding zero provider error = %v, want ErrInvalidInput only", err) - } - - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding valid: %v", err) - } - - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: types.OnChainID{}}, - }); !errors.Is(err, repository.ErrInvalidInput) || errors.Is(err, repository.ErrNotFound) { - t.Fatalf("CreateUploadCopiesForBindings zero provider error = %v, want ErrInvalidInput only", err) - } - - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: upload.ID, - DataSetID: types.OnChainID{}, - }); !errors.Is(err, repository.ErrInvalidInput) || errors.Is(err, repository.ErrNotFound) { - t.Fatalf("MarkDataSetReady zero data set error = %v, want ErrInvalidInput only", err) - } -} - -func TestStorageUploadRepo_MarkUploadCopyCommittedValidatesReadableIdentity(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "committed-copy-validation-bucket") - - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J000000000000000000BAD02", - ContentSize: 10, - Checksum: "checksum-committed-validation", - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - - valid := repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "piece-committed-validation", - PieceID: onChainIDPtr(t, "2001"), - RetrievalURL: "https://provider.example/piece", - } - tests := []struct { - name string - input repository.MarkUploadCopyCommittedInput - }{ - {name: "missing upload", input: repository.MarkUploadCopyCommittedInput{UploadID: 0, CopyIndex: valid.CopyIndex, PieceCID: valid.PieceCID, PieceID: valid.PieceID, RetrievalURL: valid.RetrievalURL}}, - {name: "negative copy index", input: repository.MarkUploadCopyCommittedInput{UploadID: valid.UploadID, CopyIndex: -1, PieceCID: valid.PieceCID, PieceID: valid.PieceID, RetrievalURL: valid.RetrievalURL}}, - {name: "missing piece cid", input: repository.MarkUploadCopyCommittedInput{UploadID: valid.UploadID, CopyIndex: valid.CopyIndex, PieceID: valid.PieceID, RetrievalURL: valid.RetrievalURL}}, - {name: "missing piece id", input: repository.MarkUploadCopyCommittedInput{UploadID: valid.UploadID, CopyIndex: valid.CopyIndex, PieceCID: valid.PieceCID, RetrievalURL: valid.RetrievalURL}}, - {name: "missing retrieval url", input: repository.MarkUploadCopyCommittedInput{UploadID: valid.UploadID, CopyIndex: valid.CopyIndex, PieceCID: valid.PieceCID, PieceID: valid.PieceID}}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := repos.Uploads.MarkUploadCopyCommitted(ctx, tt.input) - if !errors.Is(err, repository.ErrInvalidInput) || errors.Is(err, repository.ErrNotFound) { - t.Fatalf("MarkUploadCopyCommitted error = %v, want ErrInvalidInput only", err) - } - }) - } -} - -func TestStorageUploadRepo_MarkUploadCopyCommittedMissingCopyDoesNotMarkReadable(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "missing-committed-copy-bucket") - - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J000000000000000000BAD03", - ContentSize: 10, - Checksum: "checksum-missing-committed-copy", - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - - err = repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "piece-missing-copy", - PieceID: onChainIDPtr(t, "2001"), - RetrievalURL: "https://provider.example/missing-copy", - }) - if !errors.Is(err, repository.ErrNotFound) || errors.Is(err, repository.ErrInvalidInput) { - t.Fatalf("MarkUploadCopyCommitted missing copy error = %v, want ErrNotFound only", err) - } - - got, err := repos.Uploads.GetByID(ctx, upload.ID) - if err != nil || got == nil { - t.Fatalf("GetByID: upload=%v err=%v", got, err) - } - if got.Status != model.StorageUploadStatusRunning { - t.Fatalf("upload status = %s, want running", got.Status) - } - if got.PieceCID != nil { - t.Fatalf("upload piece cid = %q, want nil", *got.PieceCID) - } -} - -func TestStorageUploadRepo_DataSetBindingIsBucketProviderCopySlot(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "dataset-binding-bucket") - - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J00000000000000000020001", - ContentSize: 10, - Checksum: "checksum-binding", - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - - primary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding primary: %v", err) - } - again, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding again: %v", err) - } - if again.ID != primary.ID { - t.Fatalf("binding id = %d, want reused %d", again.ID, primary.ID) - } - if primary.Status != model.StorageDataSetStatusPending || primary.DataSetID != nil || primary.ClientDataSetID != nil { - t.Fatalf("new binding = status:%s dataSet:%v client:%v, want pending without ids", primary.Status, primary.DataSetID, primary.ClientDataSetID) - } - - if _, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "202"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }); err == nil { - t.Fatal("same bucket copy_index with different provider should be rejected") - } - if _, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 1, - CreatedByUploadID: upload.ID, - }); err == nil { - t.Fatal("same bucket provider with different copy_index should be rejected") - } -} - -func TestStorageUploadRepo_BindReadableUploadForContentMovesFollowersToReplicating(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "primary-commit-bind-bucket") - - leader := newObjectVersion(bucket.ID, "leader.txt", "01J00000000000000000020002", 10) - leader.Checksum = "same-primary-commit" - leaderObjectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, leader) - if err != nil { - t.Fatalf("create leader: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, leader.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("leader uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, leader.VersionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("leader committing: %v", err) - } - if err := repos.Objects.UpdateVersionStateToFailed(ctx, leader.VersionID, model.ObjectStateCommitting, "stale primary failure"); err != nil { - t.Fatalf("leader stale failed: %v", err) - } - follower := newObjectVersion(bucket.ID, "follower.txt", "01J00000000000000000020003", 10) - follower.Checksum = leader.Checksum - follower.State = model.ObjectStateUploading - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, follower); err != nil { - t.Fatalf("create follower: %v", err) - } - independent := newObjectVersion(bucket.ID, "independent.txt", "01J00000000000000000020004", 10) - independent.Checksum = leader.Checksum - independent.State = model.ObjectStateUploading - independentObjectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, independent) - if err != nil { - t.Fatalf("create independent: %v", err) - } - if err := repos.Tasks.Create(ctx, &model.Task{ - Type: model.TaskTypeUpload, - RefType: "object", - RefID: independentObjectID, - RefVersionID: independent.VersionID, - IdempotencyKey: "upload:" + independent.VersionID, - Status: model.TaskStatusQueued, - MaxRetries: 3, - ScheduledAt: time.Now(), - }); err != nil { - t.Fatalf("create independent task: %v", err) - } - - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: leader.VersionID, - ContentSize: leader.Size, - Checksum: leader.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - primary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("primary binding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: primary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "bafk2bzaceprimarybind", - PieceID: onChainIDPtr(t, "301"), - RetrievalURL: "https://primary.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: leader.Size, - Checksum: leader.Checksum, - }); err != nil { - t.Fatalf("BindReadableUploadForContent: %v", err) - } - - for _, versionID := range []string{leader.VersionID, follower.VersionID} { - got, err := repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || got == nil { - t.Fatalf("GetVersionByID(%s): got=%v err=%v", versionID, got, err) - } - if got.State != model.ObjectStateReplicating || got.StorageUploadID == nil || *got.StorageUploadID != upload.ID || !got.InFilecoin { - t.Fatalf("version %s = state:%s upload:%v in_filecoin:%v, want replicating bound to %d", versionID, got.State, got.StorageUploadID, got.InFilecoin, upload.ID) - } - if versionID == leader.VersionID && (got.FailedAtState != nil || got.LastError != nil) { - t.Fatalf("leader failure details = failed_at_state:%#v last_error:%#v, want nil", got.FailedAtState, got.LastError) - } - } - gotIndependent, err := repos.Objects.GetVersionByID(ctx, independent.VersionID) - if err != nil || gotIndependent == nil { - t.Fatalf("GetVersionByID(independent): got=%v err=%v", gotIndependent, err) - } - if gotIndependent.State != model.ObjectStateUploading || gotIndependent.StorageUploadID != nil { - t.Fatalf("independent = state:%s upload:%v, want untouched uploading", gotIndependent.State, gotIndependent.StorageUploadID) - } - if leaderObjectID == 0 { - t.Fatal("leader object id should be set") - } -} - -func TestStorageUploadRepo_BindReadableUploadForVersionCompletesFollowerTask(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "primary-commit-version-bind-bucket") - - leader := newObjectVersion(bucket.ID, "leader.txt", "01J00000000000000000020102", 10) - leader.Checksum = "same-primary-version-bind" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, leader); err != nil { - t.Fatalf("create leader: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, leader.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("leader uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, leader.VersionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("leader committing: %v", err) - } - follower := newObjectVersion(bucket.ID, "follower.txt", "01J00000000000000000020103", 10) - follower.Checksum = leader.Checksum - follower.State = model.ObjectStateUploading - followerObjectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, follower) - if err != nil { - t.Fatalf("create follower: %v", err) - } - task := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "object", - RefID: followerObjectID, - RefVersionID: follower.VersionID, - IdempotencyKey: "upload:" + follower.VersionID, - Status: model.TaskStatusQueued, - MaxRetries: 3, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("create follower task: %v", err) - } - - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: leader.VersionID, - ContentSize: leader.Size, - Checksum: leader.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - primary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("primary binding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: primary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "bafk2bzaceversionbind", - PieceID: onChainIDPtr(t, "301"), - RetrievalURL: "https://primary.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - - refs, err := repos.Uploads.BindReadableUploadForVersion(ctx, repository.BindReadableUploadForVersionInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: follower.Size, - Checksum: follower.Checksum, - VersionID: follower.VersionID, - }) - if err != nil { - t.Fatalf("BindReadableUploadForVersion: %v", err) - } - if len(refs) != 1 || refs[0].VersionID != follower.VersionID { - t.Fatalf("bound refs = %#v, want follower version", refs) - } - gotFollower, err := repos.Objects.GetVersionByID(ctx, follower.VersionID) - if err != nil || gotFollower == nil { - t.Fatalf("GetVersionByID(follower): got=%v err=%v", gotFollower, err) - } - if gotFollower.State != model.ObjectStateReplicating || gotFollower.StorageUploadID == nil || *gotFollower.StorageUploadID != upload.ID || !gotFollower.InFilecoin { - t.Fatalf("follower = state:%s upload:%v in_filecoin:%v, want replicating bound to %d", gotFollower.State, gotFollower.StorageUploadID, gotFollower.InFilecoin, upload.ID) - } - gotTask, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil || gotTask == nil { - t.Fatalf("GetByID(follower task): task=%v err=%v", gotTask, err) - } - if gotTask.Status != model.TaskStatusCompleted { - t.Fatalf("follower task status = %s, want completed", gotTask.Status) - } -} - -func TestStorageUploadRepo_FinalizeUploadIfTargetCopiesMetMovesReplicatingToStored(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "finalize-upload-bucket") - - version := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000020005", 10) - version.Checksum = "finalize-checksum" - objectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version) - if err != nil { - t.Fatalf("create version: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("committing: %v", err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - primary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("primary binding: %v", err) - } - secondary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "202"), CopyIndex: 1, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("secondary binding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: primary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}); err != nil { - t.Fatalf("primary ready: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: secondary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "2002"), ClientDataSetID: onChainIDPtr(t, "9002")}); err != nil { - t.Fatalf("secondary ready: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - {StorageDataSetID: secondary.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{UploadID: upload.ID, CopyIndex: 0, PieceCID: "bafk2bzacefinalize", PieceID: onChainIDPtr(t, "301"), RetrievalURL: "https://primary.example/piece"}); err != nil { - t.Fatalf("MarkUploadCopyCommitted primary: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: version.Size, - Checksum: version.Checksum, - }); err != nil { - t.Fatalf("BindReadableUploadForContent: %v", err) - } - - done, refs, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}) - if err != nil { - t.Fatalf("FinalizeUploadIfTargetCopiesMet partial: %v", err) - } - if done || len(refs) != 0 { - t.Fatalf("partial finalize = done:%v refs:%v, want no-op", done, refs) - } - got, err := repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil || got == nil { - t.Fatalf("GetVersionByID partial: got=%v err=%v", got, err) - } - if got.State != model.ObjectStateReplicating { - t.Fatalf("partial state = %s, want replicating", got.State) - } - - if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: upload.ID, CopyIndex: 1, LastError: "peer pull: dataset unavailable"}); err != nil { - t.Fatalf("MarkUploadCopyFailed peer: %v", err) - } - replacement, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "303"), CopyIndex: 2, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("replacement binding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: replacement.ID, UploadID: upload.ID, DataSetID: onChainID(t, "3003"), ClientDataSetID: onChainIDPtr(t, "9003")}); err != nil { - t.Fatalf("replacement ready: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: replacement.ID, CopyIndex: 2, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "303")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings replacement: %v", err) - } - upload, err = repos.Uploads.GetByID(ctx, upload.ID) - if err != nil { - t.Fatalf("GetByID after replacement: %v", err) - } - if upload.RequestedCopies != 2 { - t.Fatalf("requested copies after replacement append = %d, want 2", upload.RequestedCopies) - } - done, refs, err = repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}) - if err != nil { - t.Fatalf("FinalizeUploadIfTargetCopiesMet after replacement append: %v", err) - } - if done || len(refs) != 0 { - t.Fatalf("replacement append finalize = done:%v refs:%v, want no-op until target copies are met", done, refs) - } - - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{UploadID: upload.ID, CopyIndex: 2, PieceCID: "bafk2bzacefinalize", PieceID: onChainIDPtr(t, "302"), RetrievalURL: "https://replacement.example/piece"}); err != nil { - t.Fatalf("MarkUploadCopyCommitted replacement: %v", err) - } - stage := "peer_commit" - claimedTask := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objectID, - RefVersionID: version.VersionID, - IdempotencyKey: "upload:finalize-preserves-claimed-task", - Status: model.TaskStatusQueued, - ScheduledAt: time.Now().Add(-time.Second), - } - if err := repos.Tasks.Create(ctx, claimedTask); err != nil { - t.Fatalf("Create claimed upload task: %v", err) - } - claimedTask, err = repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil { - t.Fatalf("ClaimReady upload task: %v", err) - } - if claimedTask == nil { - t.Fatal("ClaimReady upload task returned nil") - } - pendingTask := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objectID, - RefVersionID: version.VersionID, - IdempotencyKey: "upload:finalize-clears-pending-task", - Status: model.TaskStatusQueued, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, pendingTask); err != nil { - t.Fatalf("Create pending upload task: %v", err) - } - maxEvictionRetries := 7 - mustExec(t, db, `CREATE TRIGGER reject_after_upload_eviction - BEFORE INSERT ON tasks - WHEN NEW.type = 'evict_cache' - BEGIN - SELECT RAISE(FAIL, 'injected eviction task failure'); - END`) - _, _, err = repos.Uploads.FinalizeUploadIfTargetCopiesMet( - ctx, - repository.NewFinalizeUploadInput( - upload.ID, - true, - maxEvictionRetries, - ), - ) - if err == nil { - t.Fatal("FinalizeUploadIfTargetCopiesMet with task insert failure returned nil error") - } - got, getErr := repos.Objects.GetVersionByID(ctx, version.VersionID) - if getErr != nil || got == nil { - t.Fatalf("GetVersionByID after rolled-back finalize: got=%v err=%v", got, getErr) - } - if got.State != model.ObjectStateReplicating { - t.Fatalf("state after rolled-back finalize = %s, want replicating", got.State) - } - uploadAfterRollback, getErr := repos.Uploads.GetByID(ctx, upload.ID) - if getErr != nil || uploadAfterRollback == nil { - t.Fatalf("GetByID after rolled-back finalize: upload=%v err=%v", uploadAfterRollback, getErr) - } - if uploadAfterRollback.Status != model.StorageUploadStatusReadable { - t.Fatalf("upload status after rolled-back finalize = %s, want readable", uploadAfterRollback.Status) - } - pendingAfterRollback, getErr := repos.Tasks.GetByID(ctx, pendingTask.ID) - if getErr != nil || pendingAfterRollback == nil { - t.Fatalf("GetByID pending task after rollback: task=%v err=%v", pendingAfterRollback, getErr) - } - if pendingAfterRollback.Status != model.TaskStatusQueued { - t.Fatalf("pending task status after rollback = %s, want queued", pendingAfterRollback.Status) - } - mustExec(t, db, `DROP TRIGGER reject_after_upload_eviction`) - - done, refs, err = repos.Uploads.FinalizeUploadIfTargetCopiesMet( - ctx, - repository.NewFinalizeUploadInput( - upload.ID, - true, - maxEvictionRetries, - ), - ) - if err != nil { - t.Fatalf("FinalizeUploadIfTargetCopiesMet complete: %v", err) - } - if !done || len(refs) != 1 || refs[0].VersionID != version.VersionID { - t.Fatalf("complete finalize = done:%v refs:%v, want stored source version", done, refs) - } - got, err = repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil || got == nil { - t.Fatalf("GetVersionByID complete: got=%v err=%v", got, err) - } - if got.State != model.ObjectStateStored { - t.Fatalf("complete state = %s, want stored", got.State) - } - preservedTask, err := repos.Tasks.GetByID(ctx, claimedTask.ID) - if err != nil { - t.Fatalf("GetByID claimed task: %v", err) - } - if preservedTask.Status != model.TaskStatusRunning { - t.Fatalf("claimed upload task status = %s, want running", preservedTask.Status) - } - if err := repos.Tasks.Complete(ctx, claimedTask); err != nil { - t.Fatalf("Complete claimed upload task after finalization: %v", err) - } - finalizedTask, err := repos.Tasks.GetByID(ctx, pendingTask.ID) - if err != nil { - t.Fatalf("GetByID pending task: %v", err) - } - if finalizedTask.Status != model.TaskStatusCompleted { - t.Fatalf("pending upload task status = %s, want completed", finalizedTask.Status) - } - evictionTasks, total, err := repos.Tasks.List( - ctx, - string(model.TaskTypeEvictCache), - cacheeviction.StageAfterUpload, - "", - 10, - 0, - ) - if err != nil { - t.Fatalf("List after-upload eviction tasks: %v", err) - } - if total != 1 || len(evictionTasks) != 1 { - t.Fatalf("after-upload eviction tasks total=%d tasks=%#v, want one", total, evictionTasks) - } - if evictionTasks[0].RefVersionID != version.VersionID || - evictionTasks[0].MaxRetries != maxEvictionRetries { - t.Fatalf( - "after-upload eviction task = version:%s retries:%d, want version:%s retries:%d", - evictionTasks[0].RefVersionID, - evictionTasks[0].MaxRetries, - version.VersionID, - maxEvictionRetries, - ) - } -} - -func TestStorageUploadRepo_MinimumDurabilityStoresBeforeTargetAndKeepsRepairWork(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "minimum-durability-finalize-bucket") - minimum := 2 - if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimum, - }); err != nil { - t.Fatalf("UpdateCopyPolicy: %v", err) - } - - version := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000MIN02", 10) - version.Checksum = "minimum-durability-checksum" - objectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version) - if err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - upload := startCopyHealthUpload(t, repos, bucket.ID, version.VersionID, version.Size, version.Checksum, 3) - commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 0, "101", "1001", "2001", "https://one.example/piece") - commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 1, "202", "2002", "2002", "https://two.example/piece") - bindStorageHealthVersion(t, repos, bucket.ID, upload.ID, version) - - stage := "peer_pull" - repairTask := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objectID, - RefVersionID: version.VersionID, - IdempotencyKey: "upload:minimum-durability-third-copy", - Payload: map[string]interface{}{"upload_id": upload.ID, "copy_index": 2}, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, repairTask); err != nil { - t.Fatalf("Create repair task: %v", err) - } - - done, refs, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}) - if err != nil { - t.Fatalf("FinalizeUploadIfTargetCopiesMet minimum: %v", err) - } - if done || len(refs) != 1 || refs[0].VersionID != version.VersionID { - t.Fatalf("minimum finalize = done:%v refs:%#v, want stored without upload completion", done, refs) - } - gotVersion, err := repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil || gotVersion == nil || gotVersion.State != model.ObjectStateStored { - t.Fatalf("version after minimum = %#v err=%v, want stored", gotVersion, err) - } - gotUpload, err := repos.Uploads.GetByID(ctx, upload.ID) - if err != nil || gotUpload == nil || gotUpload.Status != model.StorageUploadStatusReadable { - t.Fatalf("upload after minimum = %#v err=%v, want readable", gotUpload, err) - } - gotTask, err := repos.Tasks.GetByID(ctx, repairTask.ID) - if err != nil || gotTask == nil || gotTask.Status != model.TaskStatusQueued { - t.Fatalf("repair task after minimum = %#v err=%v, want queued", gotTask, err) - } - - commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 2, "303", "3003", "2003", "https://three.example/piece") - done, refs, err = repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}) - if err != nil { - t.Fatalf("FinalizeUploadIfTargetCopiesMet target: %v", err) - } - if !done || len(refs) != 0 { - t.Fatalf("target finalize = done:%v refs:%#v, want complete without another state transition", done, refs) - } - gotUpload, err = repos.Uploads.GetByID(ctx, upload.ID) - if err != nil || gotUpload == nil || gotUpload.Status != model.StorageUploadStatusComplete { - t.Fatalf("upload after target = %#v err=%v, want complete", gotUpload, err) - } - gotTask, err = repos.Tasks.GetByID(ctx, repairTask.ID) - if err != nil || gotTask == nil || gotTask.Status != model.TaskStatusCompleted { - t.Fatalf("repair task after target = %#v err=%v, want completed", gotTask, err) - } -} - -func TestStorageUploadRepo_MinimumDurabilityEnqueuesAfterUploadBeforeTarget(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "minimum-durability-after-upload-bucket") - minimum := 2 - if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimum, - }); err != nil { - t.Fatalf("UpdateCopyPolicy: %v", err) - } - - version := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000MINAU", 10) - version.Checksum = "minimum-durability-after-upload" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - upload := startCopyHealthUpload(t, repos, bucket.ID, version.VersionID, version.Size, version.Checksum, 3) - commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 0, "101", "1001", "2001", "https://one.example/piece") - commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 1, "202", "2002", "2002", "https://two.example/piece") - bindStorageHealthVersion(t, repos, bucket.ID, upload.ID, version) - - maxEvictionRetries := 7 - done, refs, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet( - ctx, - repository.NewFinalizeUploadInput(upload.ID, true, maxEvictionRetries), - ) - if err != nil { - t.Fatalf("FinalizeUploadIfTargetCopiesMet minimum: %v", err) - } - if done || len(refs) != 1 || refs[0].VersionID != version.VersionID { - t.Fatalf("minimum finalize = done:%v refs:%#v, want stored without upload completion", done, refs) - } - gotUpload, err := repos.Uploads.GetByID(ctx, upload.ID) - if err != nil || gotUpload == nil || gotUpload.Status != model.StorageUploadStatusReadable { - t.Fatalf("upload after minimum = %#v err=%v, want readable", gotUpload, err) - } - evictionTasks, total, err := repos.Tasks.List( - ctx, - string(model.TaskTypeEvictCache), - cacheeviction.StageAfterUpload, - "", - 10, - 0, - ) - if err != nil { - t.Fatalf("List after-upload eviction tasks: %v", err) - } - if total != 1 || len(evictionTasks) != 1 { - t.Fatalf("after-upload eviction tasks after minimum total=%d tasks=%#v, want one", total, evictionTasks) - } - if evictionTasks[0].RefVersionID != version.VersionID || evictionTasks[0].MaxRetries != maxEvictionRetries { - t.Fatalf( - "after-upload eviction task = version:%s retries:%d, want version:%s retries:%d", - evictionTasks[0].RefVersionID, - evictionTasks[0].MaxRetries, - version.VersionID, - maxEvictionRetries, - ) - } - - commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 2, "303", "3003", "2003", "https://three.example/piece") - done, refs, err = repos.Uploads.FinalizeUploadIfTargetCopiesMet( - ctx, - repository.NewFinalizeUploadInput(upload.ID, true, maxEvictionRetries), - ) - if err != nil { - t.Fatalf("FinalizeUploadIfTargetCopiesMet target: %v", err) - } - if !done || len(refs) != 0 { - t.Fatalf("target finalize = done:%v refs:%#v, want complete without another state transition", done, refs) - } - evictionTasks, total, err = repos.Tasks.List( - ctx, - string(model.TaskTypeEvictCache), - cacheeviction.StageAfterUpload, - "", - 10, - 0, - ) - if err != nil { - t.Fatalf("List after-upload eviction tasks after target: %v", err) - } - if total != 1 || len(evictionTasks) != 1 || evictionTasks[0].RefVersionID != version.VersionID { - t.Fatalf("after-upload eviction tasks after target total=%d tasks=%#v, want the same task", total, evictionTasks) - } -} - -func TestStorageUploadRepo_ListIncompleteReadableUploadsIncludesCommittedUnavailableSlot(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "minimum-durability-recovery-bucket") - minimum := 1 - if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimum, - }); err != nil { - t.Fatalf("UpdateCopyPolicy: %v", err) - } - - version := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000MIN03", 10) - version.Checksum = "minimum-durability-recovery-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - upload := startCopyHealthUpload(t, repos, bucket.ID, version.VersionID, version.Size, version.Checksum, 2) - commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 0, "101", "1001", "2001", "https://one.example/piece") - unavailable := commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 1, "202", "2002", "2002", "https://two.example/piece") - bindStorageHealthVersion(t, repos, bucket.ID, upload.ID, version) - if err := repos.Uploads.MarkDataSetUnavailable(ctx, unavailable.ID, "temporary outage"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - if complete, _, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}); err != nil || complete { - t.Fatalf("FinalizeUploadIfTargetCopiesMet = complete:%t err:%v, want durable before target", complete, err) - } - - items, err := repos.Uploads.ListIncompleteReadableUploads(ctx, 0, 100) - if err != nil { - t.Fatalf("ListIncompleteReadableUploads: %v", err) - } - if len(items) != 1 || items[0].Upload.ID != upload.ID || items[0].Version.VersionID != version.VersionID { - t.Fatalf("incomplete readable uploads = %#v, want upload %d version %s", items, upload.ID, version.VersionID) - } -} - -func TestStorageUploadRepo_PrimaryCopyFailureMarksUploadFailed(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "primary-copy-failure-bucket") - - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J00000000000000000010030", - ContentSize: 10, - Checksum: "checksum-primary-failure", - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - - if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: upload.ID, CopyIndex: 0, LastError: "ingress store: provider rejected piece"}); err != nil { - t.Fatalf("MarkUploadCopyFailed: %v", err) - } - got, err := repos.Uploads.GetByID(ctx, upload.ID) - if err != nil || got == nil { - t.Fatalf("GetByID: got=%v err=%v", got, err) - } - if got.Status != model.StorageUploadStatusFailed { - t.Fatalf("upload status = %s, want failed", got.Status) - } - if got.ErrorMessage == nil || *got.ErrorMessage != "ingress store: provider rejected piece" { - t.Fatalf("upload error_message = %#v, want ingress failure reason", got.ErrorMessage) - } - - if err := repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "piece-after-store-retry", - RetrievalURL: "https://provider.example/retry", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady after failure: %v", err) - } - got, err = repos.Uploads.GetByID(ctx, upload.ID) - if err != nil || got == nil { - t.Fatalf("GetByID after store retry: got=%v err=%v", got, err) - } - if got.Status != model.StorageUploadStatusIngressReady { - t.Fatalf("upload status after store retry = %s, want ingress_ready", got.Status) - } - - if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: upload.ID, CopyIndex: 0, LastError: "ingress commit: provider rejected piece"}); err != nil { - t.Fatalf("MarkUploadCopyFailed after store retry: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "piece-after-store-retry", - PieceID: onChainIDPtr(t, "2001"), - RetrievalURL: "https://provider.example/retry", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted after failure: %v", err) - } - got, err = repos.Uploads.GetByID(ctx, upload.ID) - if err != nil || got == nil { - t.Fatalf("GetByID after commit retry: got=%v err=%v", got, err) - } - if got.Status != model.StorageUploadStatusReadable { - t.Fatalf("upload status after commit retry = %s, want readable", got.Status) - } -} - -func TestStorageUploadRepo_ResetRejectedCommitUsesAttemptFence(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "rejected-commit-reset-bucket") - - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J00000000000000000010031", - ContentSize: 10, - Checksum: "checksum-rejected-commit-reset", - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: onChainID(t, "101"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "piece-rejected-commit-reset", - RetrievalURL: "https://provider.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - copyRow, err := repos.Uploads.GetUploadCopy(ctx, upload.ID, 0) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - seedRepositoryCommitAttempt(t, repos, *copyRow, "rejected-attempt", "01", "0xrejected") - - err = repos.Uploads.ResetCommitAttempt(ctx, storagecommit.ResetInput{ - Copy: commitCopyIdentity(*copyRow), AttemptID: "newer-attempt", LastError: "rejected", - }) - if !errors.Is(err, repository.ErrConflict) { - t.Fatalf("stale reset error = %v, want ErrConflict", err) - } - copyRow, err = repos.Uploads.GetUploadCopy(ctx, upload.ID, 0) - if err != nil || copyRow.Status != model.StorageUploadCopyStatusCommitting || copyRow.CommitTransactionID == nil || *copyRow.CommitTransactionID != "0xrejected" { - t.Fatalf("copy after stale reset = %#v err=%v, want original submitted commit", copyRow, err) - } - - if err := repos.Uploads.ResetCommitAttempt(ctx, storagecommit.ResetInput{ - Copy: commitCopyIdentity(*copyRow), AttemptID: "rejected-attempt", LastError: "commit transaction rejected", - }); err != nil { - t.Fatalf("ResetCommitAttempt: %v", err) - } - copyRow, err = repos.Uploads.GetUploadCopy(ctx, upload.ID, 0) - if err != nil || copyRow.Status != model.StorageUploadCopyStatusPieceReady || copyRow.CommitTransactionID != nil || copyRow.CommitExtraDataHex != nil { - t.Fatalf("copy after rejected reset = %#v err=%v, want piece_ready without commit data", copyRow, err) - } - if copyRow.LastError == nil || *copyRow.LastError != "commit transaction rejected" { - t.Fatalf("copy last error = %#v, want rejected reason", copyRow.LastError) - } -} - -func TestStorageUploadRepo_PeerPieceReadyPersistsPieceCIDWithoutIngressTransition(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "peer-piece-ready-bucket") - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J000000000000000PEERPIECE", - ContentSize: 10, - Checksum: "checksum-peer-piece-ready", - RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "202"), - CopyIndex: 1, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, - CopyIndex: 1, - TransferMethod: model.StorageCopyTransferMethodPeerPull, - ProviderID: onChainID(t, "202"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - pieceCID := "bafkreifm6jgq3qxvcvul2woy6t3vht5m6wkh5jgsslnnq3qjm2f2x7x2hu" - if err := repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: upload.ID, - CopyIndex: 1, - PieceCID: pieceCID, - RetrievalURL: "https://provider.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - got, err := repos.Uploads.GetByID(ctx, upload.ID) - if err != nil || got == nil { - t.Fatalf("GetByID: upload=%#v err=%v", got, err) - } - if got.PieceCID == nil || *got.PieceCID != pieceCID { - t.Fatalf("piece CID = %#v, want %q", got.PieceCID, pieceCID) - } - if got.Status != model.StorageUploadStatusRunning { - t.Fatalf("upload status = %s, want running for peer transfer", got.Status) - } - if got.IngressBytesTransferred != 0 { - t.Fatalf("ingress bytes = %d, want unchanged", got.IngressBytesTransferred) - } -} - -func TestStorageUploadRepo_CommittedCopyIgnoresStaleStatusUpdates(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "committed-copy-stale-status-bucket") - - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J00000000000000000010031", - ContentSize: 10, - Checksum: "checksum-committed-stale-status", - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - primary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding primary: %v", err) - } - secondary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "202"), - CopyIndex: 1, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding secondary: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: primary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}); err != nil { - t.Fatalf("MarkDataSetReady primary: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: secondary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "2002"), ClientDataSetID: onChainIDPtr(t, "9002")}); err != nil { - t.Fatalf("MarkDataSetReady secondary: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - {StorageDataSetID: secondary.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "piece-committed-stale-status", - PieceID: onChainIDPtr(t, "3001"), - RetrievalURL: "https://primary.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted primary: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 1, - PieceCID: "piece-committed-stale-status", - PieceID: onChainIDPtr(t, "3002"), - RetrievalURL: "https://secondary.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted secondary: %v", err) - } - - if err := repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: upload.ID, - CopyIndex: 1, - PieceCID: "piece-stale-ready", - RetrievalURL: "https://secondary.example/stale", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady stale: %v", err) - } - if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: upload.ID, CopyIndex: 1, LastError: "secondary pull: stale failure"}); err != nil { - t.Fatalf("MarkUploadCopyFailed stale: %v", err) - } - - copyRow, err := repos.Uploads.GetUploadCopy(ctx, upload.ID, 1) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%v err=%v", copyRow, err) - } - if copyRow.Status != model.StorageUploadCopyStatusCommitted { - t.Fatalf("copy status = %s, want committed", copyRow.Status) - } -} - -func TestStorageUploadRepo_AppendUploadFailureRetriesRacedAttemptIndex(t *testing.T) { - sqldb, err := sql.Open("sqlite", "file:"+filepath.Join(t.TempDir(), "storage-upload-failure-race.db")+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(1)") - if err != nil { - t.Fatalf("opening sqlite db: %v", err) - } - sqldb.SetMaxOpenConns(8) - db := bun.NewDB(sqldb, sqlitedialect.New()) - t.Cleanup(func() { _ = db.Close() }) - - ctx := context.Background() - migrator := migrations.NewMigrator(db) - if err := migrator.Init(ctx); err != nil { - t.Fatalf("init migrator: %v", err) - } - if _, err := migrator.Migrate(ctx); err != nil { - t.Fatalf("running migrations: %v", err) - } - repos := repository.NewRepositories(db) - bucket := seedBucket(t, db, "failure-race-bucket") - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - ContentSize: 1, - Checksum: "failure-race", - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - - hook := &storageUploadFailureRaceHook{uploadID: upload.ID, collisionCount: 4} - db.AddQueryHook(hook) - - if err := repos.Uploads.AppendUploadFailure(ctx, repository.AppendUploadFailureInput{ - UploadID: upload.ID, - CopyIndex: 0, - ProviderID: onChainIDPtr(t, "101"), - TransferMethod: string(model.StorageCopyTransferMethodIngress), - Stage: "ingress_store", - ErrorMessage: "provider store failed", - }); err != nil { - t.Fatalf("AppendUploadFailure: %v", err) - } - if hookErr := hook.err.Load(); hookErr != nil { - t.Fatalf("race hook insert: %v", hookErr) - } - if !hook.triggered.Load() { - t.Fatal("race hook did not run") - } - provenance, err := repos.Uploads.GetUploadProvenance(ctx, upload.ID) - if err != nil { - t.Fatalf("GetUploadProvenance: %v", err) - } - if len(provenance.Failures) != 5 { - t.Fatalf("failures len = %d, want four raced failures and retried append", len(provenance.Failures)) - } - for index, failure := range provenance.Failures { - if failure.AttemptIndex != index { - t.Fatalf("attempt index at row %d = %d, want %d", index, failure.AttemptIndex, index) - } - } - lastFailure := provenance.Failures[len(provenance.Failures)-1] - if lastFailure.ErrorMessage == nil || *lastFailure.ErrorMessage != "provider store failed" { - t.Fatalf("retried failure = %#v, want original append data", lastFailure) - } -} - -func TestStorageUploadRepo_UnavailableDataSetRecoveryUsesIncompleteCopies(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "unavailable-recovery-bucket") - - newUpload := func(versionID string) *model.StorageUpload { - version := newObjectVersion(bucket.ID, versionID+".txt", versionID, 10) - version.Checksum = "shared-recovery-content" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(%s): %v", versionID, err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: versionID, ContentSize: 10, Checksum: "shared-recovery-content", RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt(%s): %v", versionID, err) - } - return upload - } - firstUpload := newUpload("01J000000000000000REPAIR01") - secondUpload := newUpload("01J000000000000000REPAIR02") - orphanUpload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: "01J000000000000000ORPHAN01", ContentSize: 10, Checksum: "shared-recovery-content", RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt(orphan): %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: firstUpload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, UploadID: firstUpload.ID, DataSetID: onChainID(t, "1001")}); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - for _, upload := range []*model.StorageUpload{firstUpload, secondUpload, orphanUpload} { - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "101"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings(%d): %v", upload.ID, err) - } - } - if err := repos.Uploads.MarkDataSetUnavailable(ctx, binding.ID, "temporary outage"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - mustExec(t, db, `UPDATE storage_uploads SET status = ? WHERE id = ?`, model.StorageUploadStatusComplete, firstUpload.ID) - mustExec(t, db, `UPDATE storage_uploads SET status = ? WHERE id = ?`, model.StorageUploadStatusFailed, secondUpload.ID) - - bindings, err := repos.Uploads.ListUnavailableDataSetsWithIncompleteCopies(ctx, 0, 10) - if err != nil || len(bindings) != 1 || bindings[0].ID != binding.ID { - t.Fatalf("ListUnavailableDataSetsWithIncompleteCopies = %#v err=%v", bindings, err) - } - firstCopy, err := repos.Uploads.NextIncompleteCopyForDataSet(ctx, binding.ID) - if err != nil || firstCopy == nil || firstCopy.UploadID != firstUpload.ID { - t.Fatalf("first incomplete copy = %#v err=%v", firstCopy, err) - } - if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: firstUpload.ID, CopyIndex: firstCopy.CopyIndex, LastError: "skip completed repair item"}); err != nil { - t.Fatalf("MarkUploadCopyFailed: %v", err) - } - secondCopy, err := repos.Uploads.NextIncompleteCopyForDataSet(ctx, binding.ID) - if err != nil || secondCopy == nil || secondCopy.UploadID != secondUpload.ID { - t.Fatalf("second incomplete copy = %#v err=%v", secondCopy, err) - } - if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: secondUpload.ID, CopyIndex: secondCopy.CopyIndex, LastError: "skip second repair item"}); err != nil { - t.Fatalf("MarkUploadCopyFailed second: %v", err) - } - next, err := repos.Uploads.NextIncompleteCopyForDataSet(ctx, binding.ID) - if err != nil || next != nil { - t.Fatalf("next incomplete copy after live references = %#v err=%v, want orphan ignored", next, err) - } - bindings, err = repos.Uploads.ListUnavailableDataSetsWithIncompleteCopies(ctx, 0, 10) - if err != nil || len(bindings) != 0 { - t.Fatalf("unavailable data sets after live references = %#v err=%v, want orphan ignored", bindings, err) - } - recovered, err := repos.Uploads.RecoverDataSet(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, UploadID: secondUpload.ID, DataSetID: onChainID(t, "1001")}) - if err != nil || !recovered { - t.Fatalf("RecoverDataSet unavailable: recovered=%t err=%v", recovered, err) - } - if err := repos.Uploads.MarkDataSetDraining(ctx, binding.ID, "service ended"); err != nil { - t.Fatalf("MarkDataSetDraining: %v", err) - } - recovered, err = repos.Uploads.RecoverDataSet(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, UploadID: secondUpload.ID, DataSetID: onChainID(t, "1001")}) - if err != nil || recovered { - t.Fatalf("RecoverDataSet draining: recovered=%t err=%v", recovered, err) - } -} - -func TestStorageUploadRepo_DataSetOutageTransitionsRejectStaleState(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "outage-transition-cas-bucket") - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: "01J000000000000000CASSTATE1", ContentSize: 10, Checksum: "cas-state", RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001")}); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.MarkDataSetFailed(ctx, binding.ID, "stale creation failure"); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("MarkDataSetFailed after ready error = %v, want ErrConflict", err) - } - if err := repos.Uploads.MarkDataSetDraining(ctx, binding.ID, "service ended"); err != nil { - t.Fatalf("MarkDataSetDraining: %v", err) - } - if err := repos.Uploads.MarkDataSetUnavailable(ctx, binding.ID, "stale outage"); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("MarkDataSetUnavailable after draining error = %v, want ErrConflict", err) - } - if err := repos.Uploads.MarkDataSetDraining(ctx, binding.ID+1000, "missing"); !errors.Is(err, repository.ErrNotFound) { - t.Fatalf("MarkDataSetDraining missing error = %v, want ErrNotFound", err) - } -} - -func TestStorageUploadRepo_ReassignIngressUsesOnlyPendingReadyCopy(t *testing.T) { - for _, tc := range []struct { - name string - candidateStatus model.StorageUploadCopyStatus - sourceSubmitted bool - rejectPromotion bool - wantReassigned bool - wantConflict bool - wantIngress int - }{ - {name: "pending", candidateStatus: model.StorageUploadCopyStatusPending, wantReassigned: true, wantIngress: 1}, - {name: "piece ready", candidateStatus: model.StorageUploadCopyStatusPieceReady, wantIngress: 0}, - {name: "submitted source", candidateStatus: model.StorageUploadCopyStatusPending, sourceSubmitted: true, wantConflict: true, wantIngress: 0}, - {name: "promotion conflict", candidateStatus: model.StorageUploadCopyStatusPending, rejectPromotion: true, wantConflict: true, wantIngress: 0}, - } { - t.Run(tc.name, func(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "reassign-ingress-"+strings.ReplaceAll(tc.name, " ", "-")) - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: "01J000000000000000INGRESS1", ContentSize: 10, Checksum: "reassign-ingress", RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - bindings := make([]*model.StorageDataSet, 0, 2) - for copyIndex, ids := range [][2]string{{"101", "1001"}, {"202", "2002"}} { - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, ids[0]), CopyIndex: copyIndex, CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding(%d): %v", copyIndex, err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, UploadID: upload.ID, DataSetID: onChainID(t, ids[1])}); err != nil { - t.Fatalf("MarkDataSetReady(%d): %v", copyIndex, err) - } - bindings = append(bindings, binding) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: bindings[0].ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - {StorageDataSetID: bindings[1].ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkDataSetUnavailable(ctx, bindings[0].ID, "temporary outage"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - if tc.candidateStatus != model.StorageUploadCopyStatusPending { - mustExec(t, db, `UPDATE storage_upload_copies SET status = ? WHERE upload_id = ? AND copy_index = 1`, tc.candidateStatus, upload.ID) - } - if tc.sourceSubmitted { - mustExec(t, db, `UPDATE storage_upload_copies SET status = ? WHERE upload_id = ? AND copy_index = 0`, model.StorageUploadCopyStatusPieceReady, upload.ID) - copyRow, loadErr := repos.Uploads.GetUploadCopy(ctx, upload.ID, 0) - if loadErr != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, loadErr) - } - seedRepositoryCommitAttempt(t, repos, *copyRow, "submitted-source", "abcd", "0xsubmitted") - } - if tc.rejectPromotion { - mustExec(t, db, `CREATE TRIGGER reject_ingress_promotion - BEFORE UPDATE OF transfer_method ON storage_upload_copies - WHEN OLD.copy_index = 1 AND NEW.transfer_method = 'ingress' - BEGIN - SELECT RAISE(IGNORE); - END`) - } - reassigned, err := repos.Uploads.ReassignIngressCopy(ctx, upload.ID, 0) - if tc.wantConflict { - if !errors.Is(err, repository.ErrConflict) { - t.Fatalf("ReassignIngressCopy error = %v, want ErrConflict", err) - } - } else if err != nil { - t.Fatalf("ReassignIngressCopy: %v", err) - } - if (reassigned != nil) != tc.wantReassigned { - t.Fatalf("reassigned = %#v, want %t", reassigned, tc.wantReassigned) - } - if reassigned != nil && (reassigned.CopyIndex != 1 || reassigned.TransferMethod != model.StorageCopyTransferMethodIngress) { - t.Fatalf("reassigned copy = %#v", reassigned) - } - copies, err := repos.Uploads.ListCopies(ctx, upload.ID) - if err != nil { - t.Fatalf("ListCopies: %v", err) - } - ingressCount := 0 - for i := range copies { - copyRow := &copies[i] - if copyRow.TransferMethod == model.StorageCopyTransferMethodIngress { - ingressCount++ - if copyRow.CopyIndex != tc.wantIngress { - t.Fatalf("ingress copy index = %d, want %d", copyRow.CopyIndex, tc.wantIngress) - } - } - } - if ingressCount != 1 { - t.Fatalf("ingress copy count = %d, want 1: %#v", ingressCount, copies) - } - }) - } -} - -func TestStorageUploadRepo_DiscardFailedCandidateIsAtomicWithSharedReferences(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "failed-candidate-shared-refs") - first, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: "01J000000000000000FAILED01", ContentSize: 10, Checksum: "failed-candidate", RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt first: %v", err) - } - second, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: "01J000000000000000FAILED02", ContentSize: 10, Checksum: "failed-candidate", RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt second: %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, "303"), CopyIndex: 0, CreatedByUploadID: first.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - for _, upload := range []*model.StorageUpload{first, second} { - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "303"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings(%d): %v", upload.ID, err) - } - } - if err := repos.Uploads.MarkDataSetFailed(ctx, binding.ID, "creation rejected"); err != nil { - t.Fatalf("MarkDataSetFailed: %v", err) - } - if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: first.ID, CopyIndex: 0, LastError: "creation rejected"}); err != nil { - t.Fatalf("MarkUploadCopyFailed: %v", err) - } - discarded, err := repos.Uploads.DiscardFailedDataSetCandidate(ctx, first.ID, 0, binding.ID) - if err != nil || discarded { - t.Fatalf("DiscardFailedDataSetCandidate: discarded=%t err=%v", discarded, err) - } - copyRow, err := repos.Uploads.GetUploadCopy(ctx, first.ID, 0) - if err != nil || copyRow == nil || copyRow.Status != model.StorageUploadCopyStatusFailed { - t.Fatalf("first copy after guarded discard = %#v err=%v", copyRow, err) - } - retained, err := repos.Uploads.GetDataSetBindingByID(ctx, binding.ID) - if err != nil || retained == nil { - t.Fatalf("binding after guarded discard = %#v err=%v", retained, err) - } -} - -type storageUploadFailureRaceHook struct { - uploadID int64 - collisionCount int64 - inserted atomic.Int64 - triggered atomic.Bool - insertInProgress atomic.Bool - err atomic.Value -} - -func (h *storageUploadFailureRaceHook) BeforeQuery(ctx context.Context, event *bun.QueryEvent) context.Context { - if !strings.Contains(event.Query, "INSERT INTO") || !strings.Contains(event.Query, "storage_upload_failures") { - return ctx - } - if !h.insertInProgress.CompareAndSwap(false, true) { - return ctx - } - defer h.insertInProgress.Store(false) - attemptIndex := int(h.inserted.Load()) - if int64(attemptIndex) >= h.collisionCount { - return ctx - } - h.inserted.Add(1) - h.triggered.Store(true) - providerID, err := types.ParseOnChainID("providerID", "101") - if err != nil { - h.err.Store(err) - return ctx - } - failure := &model.StorageUploadFailure{ - UploadID: h.uploadID, - AttemptIndex: attemptIndex, - ProviderID: &providerID, - TransferMethod: string(model.StorageCopyTransferMethodIngress), - Stage: strPtr("raced_failure"), - ErrorMessage: strPtr("raced insert"), - } - if _, err := event.DB.NewInsert().Model(failure).Exec(ctx); err != nil { - h.err.Store(err) - } - return ctx -} - -func (h *storageUploadFailureRaceHook) AfterQuery(context.Context, *bun.QueryEvent) {} - -func seedRunningUploadTask(t *testing.T, repos *repository.Repositories, objectID int64, versionID string) *model.Task { - t.Helper() - task := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "object", - RefID: objectID, - RefVersionID: versionID, - IdempotencyKey: "upload:" + versionID, - Status: model.TaskStatusQueued, - MaxRetries: 3, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(context.Background(), task); err != nil { - t.Fatalf("Create task: %v", err) - } - claimed, err := repos.Tasks.ClaimReady(context.Background(), model.TaskTypeUpload, time.Minute) - if err != nil { - t.Fatalf("ClaimReady: %v", err) - } - if claimed == nil { - t.Fatal("ClaimReady returned nil") - } - return claimed -} - -func acceptTestStorageUploadForVersion(t *testing.T, repos *repository.Repositories, bucketID int64, version *model.ObjectVersion, pieceCID string) int64 { - t.Helper() - upload, err := repos.Uploads.StartObjectUploadAttempt(context.Background(), repository.StartObjectUploadAttemptInput{ - BucketID: bucketID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - dataSetID := "1001" + strconv.FormatInt(upload.ID, 10) - seedCommittedUploadCopies(t, repos, bucketID, upload.ID, pieceCID, []storageUploadCopySeed{ - {ProviderID: onChainIDPtr(t, "101"), DataSetID: onChainIDPtr(t, dataSetID), PieceID: onChainIDPtr(t, "2001"), TransferMethod: model.StorageCopyTransferMethodIngress, RetrievalURL: strPtr("https://provider.example/" + version.VersionID), IsNewDataSet: true}, - }) - bindReadableUploadForContent(t, repos, upload.ID, bucketID, version.Size, version.Checksum) - finalizeUploadForTest(t, repos, upload.ID) - return upload.ID -} - -func startCopyHealthUpload(t *testing.T, repos *repository.Repositories, bucketID int64, versionID string, size int64, checksum string, requestedCopies int) *model.StorageUpload { - t.Helper() - upload, err := repos.Uploads.StartObjectUploadAttempt(context.Background(), repository.StartObjectUploadAttemptInput{ - BucketID: bucketID, - SourceVersionID: versionID, - ContentSize: size, - Checksum: checksum, - RequestedCopies: requestedCopies, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - return upload -} - -func ensureCopyHealthBinding(t *testing.T, repos *repository.Repositories, bucketID int64, uploadID int64, copyIndex int, providerID string) *model.StorageDataSet { - t.Helper() - binding, err := repos.Uploads.EnsureDataSetBinding(context.Background(), repository.EnsureDataSetBindingInput{ - BucketID: bucketID, - ProviderID: onChainID(t, providerID), - CopyIndex: copyIndex, - CreatedByUploadID: uploadID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - return binding -} - -func commitStorageHealthCopy(t *testing.T, repos *repository.Repositories, bucketID int64, uploadID int64, copyIndex int, providerID, dataSetID, pieceID, retrievalURL string) *model.StorageDataSet { - t.Helper() - binding := ensureCopyHealthBinding(t, repos, bucketID, uploadID, copyIndex, providerID) - if err := repos.Uploads.MarkDataSetReady(context.Background(), repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: uploadID, - DataSetID: onChainID(t, dataSetID), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(context.Background(), uploadID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, - CopyIndex: copyIndex, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: onChainID(t, providerID), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(context.Background(), repository.MarkUploadCopyCommittedInput{ - UploadID: uploadID, - CopyIndex: copyIndex, - PieceCID: "bafk2bzacestorhealth", - PieceID: onChainIDPtr(t, pieceID), - RetrievalURL: retrievalURL, - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - got, err := repos.Uploads.GetDataSetBindingByID(context.Background(), binding.ID) - if err != nil { - t.Fatalf("GetDataSetBindingByID: %v", err) - } - return got -} - -func bindStorageHealthVersion(t *testing.T, repos *repository.Repositories, bucketID int64, uploadID int64, version *model.ObjectVersion) { - t.Helper() - if err := repos.Objects.UpdateVersionState(context.Background(), version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("UpdateVersionState uploading: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForVersion(context.Background(), repository.BindReadableUploadForVersionInput{ - UploadID: uploadID, - BucketID: bucketID, - ContentSize: version.Size, - Checksum: version.Checksum, - VersionID: version.VersionID, - }); err != nil { - t.Fatalf("BindReadableUploadForVersion: %v", err) - } -} - -func hasStorageHealthReason(reasons []observability.ReasonCode, want observability.ReasonCode) bool { - for _, reason := range reasons { - if reason == want { - return true - } - } - return false -} - -func affectedVersionIDs(versions []repository.BucketStorageHealthAffectedVersion) []string { - out := make([]string, 0, len(versions)) - for _, version := range versions { - out = append(out, version.Version.VersionID) - } - return out -} - -func affectedVersionByID(versions []repository.BucketStorageHealthAffectedVersion, versionID string) *repository.BucketStorageHealthAffectedVersion { - for i := range versions { - if versions[i].Version.VersionID == versionID { - return &versions[i] - } - } - return nil -} - -func strPtr(v string) *string { - return &v -} - -// A provider replacement gives one replica slot two data set generations that -// can both hold a committed copy. Counting them as two replicas would release -// cache while only one provider actually holds the data. -func TestStorageUploadRepo_DurabilityCountsDistinctSlotsAcrossGenerations(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "generation-durability-bucket") - - version := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000GEN01", 10) - version.Checksum = "generation-durability-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - upload := startCopyHealthUpload(t, repos, bucket.ID, version.VersionID, version.Size, version.Checksum, 3) - commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 0, "101", "1001", "2001", "https://one.example/piece") - commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 1, "202", "2002", "2002", "https://two.example/piece") - bindStorageHealthVersion(t, repos, bucket.ID, upload.ID, version) - - source, err := repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) - if err != nil || source == nil { - t.Fatalf("GetDataSetBindingByCopyIndex = %#v err=%v", source, err) - } - // Stand in for an activated replacement until the replacement repository - // owns this transition. - mustExec(t, db, `UPDATE storage_data_sets SET is_current = FALSE, status = ? WHERE id = ?`, - model.StorageDataSetStatusDraining, source.ID) - mustExec(t, db, `INSERT INTO storage_data_sets (bucket_id, provider_id, copy_index, generation, is_current, data_set_id, status, created_at, updated_at) - VALUES (?, '909', 0, 2, TRUE, '9009', ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, - bucket.ID, model.StorageDataSetStatusReady) - target, err := repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) - if err != nil || target == nil || target.ID == source.ID || target.Generation != 2 { - t.Fatalf("current binding after activation = %#v err=%v, want the second generation", target, err) - } - mustExec(t, db, `INSERT INTO storage_upload_copies (upload_id, copy_index, provider_id, piece_id, transfer_method, status, retrieval_url, storage_data_set_id, created_at, updated_at) - VALUES (?, 0, '909', '9001', ?, ?, 'https://three.example/piece', ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, - upload.ID, model.StorageCopyTransferMethodPeerPull, model.StorageUploadCopyStatusCommitted, target.ID) - - // All three physical copies stay retrievable, including the retiring one. - copies, err := repos.Uploads.ListReadableCommittedCopies(ctx, upload.ID) - if err != nil { - t.Fatalf("ListReadableCommittedCopies: %v", err) - } - if len(copies) != 3 { - t.Fatalf("readable copies = %d, want 3 physical copies", len(copies)) - } - - // Durability still sees two slots, so the third requested copy is owed. - done, refs, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}) - if err != nil { - t.Fatalf("FinalizeUploadIfTargetCopiesMet: %v", err) - } - if done { - t.Fatalf("two generations of one slot satisfied a 3-copy target, want them counted as one replica (refs=%#v)", refs) - } - gotVersion, err := repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil || gotVersion == nil || gotVersion.State == model.ObjectStateStored { - t.Fatalf("version after activation = %#v err=%v, want it to stay short of the durability threshold", gotVersion, err) - } -} - -// A staged task records the copy it stored to. If the slot's current generation -// changes before the task runs, the write must still land on the generation -// that actually holds the piece rather than on its replacement. -func TestStorageUploadRepo_CopyWritesFollowTheRecordedCopyNotTheCurrentGeneration(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "generation-addressing-bucket") - - version := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000ADR01", 10) - version.Checksum = "generation-addressing-checksum" - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - upload := startCopyHealthUpload(t, repos, bucket.ID, version.VersionID, version.Size, version.Checksum, 1) - source := ensureCopyHealthBinding(t, repos, bucket.ID, upload.ID, 0, "101") - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: source.ID, - UploadID: upload.ID, - DataSetID: onChainID(t, "1001"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: source.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: onChainID(t, "101"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - sourceCopy, err := repos.Uploads.GetUploadCopyForDataSet(ctx, upload.ID, source.ID) - if err != nil || sourceCopy == nil { - t.Fatalf("GetUploadCopyForDataSet source = %#v err=%v", sourceCopy, err) - } - - // Stand in for an activated replacement that also staged its own copy. - mustExec(t, db, `UPDATE storage_data_sets SET is_current = FALSE, status = ? WHERE id = ?`, - model.StorageDataSetStatusDraining, source.ID) - mustExec(t, db, `INSERT INTO storage_data_sets (bucket_id, provider_id, copy_index, generation, is_current, data_set_id, status, created_at, updated_at) - VALUES (?, '909', 0, 2, TRUE, '9009', ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, - bucket.ID, model.StorageDataSetStatusReady) - target, err := repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) - if err != nil || target == nil || target.ID == source.ID { - t.Fatalf("current binding after activation = %#v err=%v, want the new generation", target, err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: target.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodPeerPull, - ProviderID: onChainID(t, "909"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings target: %v", err) - } - targetCopy, err := repos.Uploads.GetUploadCopyForDataSet(ctx, upload.ID, target.ID) - if err != nil || targetCopy == nil { - t.Fatalf("GetUploadCopyForDataSet target = %#v err=%v", targetCopy, err) - } - - if err := repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - StorageUploadCopyID: sourceCopy.ID, - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "bafk2bzacegeneration", - PieceID: onChainIDPtr(t, "2001"), - RetrievalURL: "https://source.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - - gotSource, err := repos.Uploads.GetUploadCopyByID(ctx, sourceCopy.ID) - if err != nil || gotSource == nil || gotSource.Status != model.StorageUploadCopyStatusPieceReady { - t.Fatalf("recorded copy = %#v err=%v, want piece_ready on the generation that stored it", gotSource, err) - } - gotTarget, err := repos.Uploads.GetUploadCopyByID(ctx, targetCopy.ID) - if err != nil || gotTarget == nil || gotTarget.Status != model.StorageUploadCopyStatusPending { - t.Fatalf("replacement copy = %#v err=%v, want it untouched", gotTarget, err) - } - - // A task queued before copy ids existed still resolves through its slot, - // which names the current generation. - if err := repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "bafk2bzacegeneration", - PieceID: onChainIDPtr(t, "9001"), - RetrievalURL: "https://target.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady legacy: %v", err) - } - gotTarget, err = repos.Uploads.GetUploadCopyByID(ctx, targetCopy.ID) - if err != nil || gotTarget == nil || gotTarget.Status != model.StorageUploadCopyStatusPieceReady { - t.Fatalf("legacy addressed copy = %#v err=%v, want the current generation", gotTarget, err) - } -} - -func TestStorageUploadRepo_CountCurrentGenerationCopySlotsIgnoresHistoricalCopies(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := seedBucket(t, db, "current-generation-copy-count") - upload := startCopyHealthUpload(t, repos, bucket.ID, model.NewVersionID(), 10, "generation-count", 2) - source := ensureCopyHealthBinding(t, repos, bucket.ID, upload.ID, 0, "101") - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: source.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), - }); err != nil { - t.Fatalf("MarkDataSetReady source: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: source.ID, CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings source: %v", err) - } - - mustExec(t, db, `UPDATE storage_data_sets SET is_current = FALSE, status = ? WHERE id = ?`, - model.StorageDataSetStatusDraining, source.ID) - mustExec(t, db, `INSERT INTO storage_data_sets - (bucket_id, provider_id, copy_index, generation, is_current, data_set_id, status, created_at, updated_at) - VALUES (?, '202', 0, 2, TRUE, '2002', ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, - bucket.ID, model.StorageDataSetStatusReady) - target, err := repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) - if err != nil || target == nil || target.ID == source.ID { - t.Fatalf("current target = %#v err=%v", target, err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: target.ID, CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings target: %v", err) - } - - count, err := repos.Uploads.CountCurrentGenerationCopySlots(ctx, upload.ID) - if err != nil { - t.Fatalf("CountCurrentGenerationCopySlots: %v", err) - } - if count != 1 { - t.Fatalf("current logical slots = %d, want one occupied slot and one missing slot", count) - } -} diff --git a/internal/db/repository/task_repo.go b/internal/db/repository/task_repo.go index 31039b7..a799ed6 100644 --- a/internal/db/repository/task_repo.go +++ b/internal/db/repository/task_repo.go @@ -3,854 +3,637 @@ package repository import ( "context" "database/sql" + "errors" "fmt" - "slices" "time" "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagereplacement" "github.com/uptrace/bun" "github.com/uptrace/bun/dialect" ) -// BunTaskRepo implements TaskRepository using Bun ORM. +const claimExpiredTaskSQLiteSQL = `UPDATE tasks +SET status = 'running', + resume_mode = 'recover', + claim_generation = claim_generation + 1, + claimed_at = ?, + lease_until = ?, + started_at = COALESCE(started_at, ?), + finished_at = NULL, + updated_at = ? +WHERE id = ( + SELECT id + FROM tasks + WHERE status = 'running' AND lease_until <= ? + ORDER BY lease_until, id + LIMIT 1 +) +AND status = 'running' AND lease_until <= ? +RETURNING *` + +const claimPendingTaskSQLiteSQL = `UPDATE tasks +SET status = 'running', + claim_generation = claim_generation + 1, + claimed_at = ?, + lease_until = ?, + started_at = COALESCE(started_at, ?), + finished_at = NULL, + updated_at = ? +WHERE id = ( + SELECT id + FROM tasks + WHERE status = 'pending' AND available_at <= ? + ORDER BY available_at, id + LIMIT 1 +) +AND status = 'pending' AND available_at <= ? +RETURNING *` + +const claimExpiredTaskPostgresSQL = `UPDATE tasks +SET status = 'running', + resume_mode = 'recover', + claim_generation = claim_generation + 1, + claimed_at = ?, + lease_until = ?, + started_at = COALESCE(started_at, ?), + finished_at = NULL, + updated_at = ? +WHERE id = ( + SELECT id + FROM tasks + WHERE status = 'running' AND lease_until <= ? + ORDER BY lease_until, id + LIMIT 1 + FOR UPDATE SKIP LOCKED +) +AND status = 'running' AND lease_until <= ? +RETURNING *` + +const claimPendingTaskPostgresSQL = `UPDATE tasks +SET status = 'running', + claim_generation = claim_generation + 1, + claimed_at = ?, + lease_until = ?, + started_at = COALESCE(started_at, ?), + finished_at = NULL, + updated_at = ? +WHERE id = ( + SELECT id + FROM tasks + WHERE status = 'pending' AND available_at <= ? + ORDER BY available_at, id + LIMIT 1 + FOR UPDATE SKIP LOCKED +) +AND status = 'pending' AND available_at <= ? +RETURNING *` + +// BunTaskRepo is the persistence boundary used by TaskService and Engine. type BunTaskRepo struct { db bun.IDB } var _ TaskRepository = (*BunTaskRepo)(nil) -const claimReadySQL = `UPDATE tasks - SET status = ?, claimed_at = ?, lease_until = ?, started_at = ?, - last_error = NULL, wait_reason = NULL, status_message = NULL - WHERE id = ( - SELECT id FROM tasks - WHERE type = ? - AND status IN ('queued', 'scheduled', 'waiting') - AND scheduled_at <= ? - ORDER BY scheduled_at ASC, id ASC - LIMIT 1 - ) - AND status IN ('queued', 'scheduled', 'waiting') - RETURNING *` - -func (r *BunTaskRepo) Create(ctx context.Context, task *model.Task) error { - if task != nil && task.Status == "" { - task.Status = model.TaskStatusQueued - } - normalizeTaskStage(task) - _, err := r.db.NewInsert().Model(task).Exec(ctx) - if err != nil { - if isUniqueViolation(err) { - return fmt.Errorf("inserting task %q: %w", task.IdempotencyKey, ErrAlreadyExists) - } - return fmt.Errorf("inserting task: %w", err) - } - return nil -} - -func (r *BunTaskRepo) EnsureRecurring(ctx context.Context, task *model.Task) (bool, error) { - // Automatic recurrence never revives work that gave up. Exhausted and failed - // coordinators wait for an operator, who resumes them through ResumeCoordinator. - return r.ensureRecurringTask(ctx, task, model.TaskStatusCompleted) -} - -// ResumeCoordinator restarts a singleton coordinator on an operator's request. -// It differs from EnsureRecurring in exactly one way: it also revives a task -// that exhausted its retries or failed outright, which is the state the -// dedicated replacement retry exists to recover from. Without it the retry -// would move the replacement back into a working status with nothing queued to -// do the work, and the record would never leave it. -func (r *BunTaskRepo) ResumeCoordinator(ctx context.Context, task *model.Task) (bool, error) { - if task != nil && !storagereplacement.IsCoordinatorTask(task.Type, task.Stage) { - return false, fmt.Errorf("resuming a task that is not a coordinator: %w", ErrInvalidInput) - } - return r.ensureRecurringTask(ctx, task, model.TaskStatusCompleted, model.TaskStatusExhausted, model.TaskStatusFailed) -} - -func (r *BunTaskRepo) ensureRecurringTask(ctx context.Context, task *model.Task, revivable ...model.TaskStatus) (bool, error) { - if task == nil || task.IdempotencyKey == "" || task.Type == "" || task.RefType == "" { - return false, fmt.Errorf("recurring task identity is required: %w", ErrInvalidInput) +func (r *BunTaskRepo) Enqueue(ctx context.Context, task *model.Task) (*model.Task, bool, error) { + if task == nil || task.Type == "" || task.IdempotencyKey == "" || task.InputVersion < 1 || len(task.Input) == 0 || task.InputHash == "" { + return nil, false, fmt.Errorf("task identity and canonical input are required: %w", ErrInvalidInput) } if task.Status == "" { - task.Status = model.TaskStatusQueued - } - requestedMaxRetries := task.MaxRetries - normalizeTaskStage(task) - created := false - err := r.runMaybeTx(ctx, func(db bun.IDB) error { - existing, err := loadAndLockTaskByIdempotencyKey(ctx, db, task.IdempotencyKey) - if err == sql.ErrNoRows { - res, err := db.NewInsert(). - Model(task). - On("CONFLICT (idempotency_key) DO NOTHING"). - Exec(ctx) - if err != nil { - return fmt.Errorf("inserting recurring task: %w", err) - } - rows, _ := res.RowsAffected() - created = rows == 1 - if created && requestedMaxRetries == 0 { - // Bun otherwise substitutes the SQL default for this zero-valued field. - if _, err := db.NewUpdate(). - Model((*model.Task)(nil)). - Set("max_retries = ?", requestedMaxRetries). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - return fmt.Errorf("preserving recurring task zero retries: %w", err) - } - task.MaxRetries = 0 - } - return nil - } + task.Status = model.TaskStatusPending + } + if task.ResumeMode == "" { + task.ResumeMode = model.TaskResumeModeExecute + } + if task.AvailableAt.IsZero() { + task.AvailableAt = time.Now() + } + + inserted := false + if err := runMaybeTx(ctx, r.db, func(db bun.IDB) error { + result, err := db.NewInsert(). + Model(task). + On("CONFLICT (type, idempotency_key) DO NOTHING"). + Exec(ctx) if err != nil { - return fmt.Errorf("loading recurring task: %w", err) + return fmt.Errorf("enqueuing task %s/%s: %w", task.Type, task.IdempotencyKey, err) } - if !slices.Contains(revivable, existing.Status) { + if rows, _ := result.RowsAffected(); rows != 1 { return nil } - now := time.Now() - res, err := db.NewUpdate(). - Model((*model.Task)(nil)). - Set("stage = ?", task.Stage). - Set("ref_type = ?", task.RefType). - Set("ref_id = ?", task.RefID). - Set("ref_version_id = ?", task.RefVersionID). - Set("payload = ?", task.Payload). - Set("status = ?", model.TaskStatusQueued). - Set("retry_count = 0"). - Set("max_retries = ?", task.MaxRetries). - Set("last_error = NULL"). - Set("status_message = NULL"). - Set("wait_reason = NULL"). - Set("scheduled_at = ?", now). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("started_at = NULL"). - Set("completed_at = NULL"). - Where("id = ? AND status = ?", existing.ID, existing.Status). - Exec(ctx) - if err != nil { - return fmt.Errorf("reactivating recurring task: %w", err) + inserted = true + payload := &model.TaskPayload{TaskID: task.ID, Input: task.Input, Checkpoint: task.Checkpoint} + if _, err := db.NewInsert().Model(payload).Exec(ctx); err != nil { + return fmt.Errorf("enqueuing task %s/%s payload: %w", task.Type, task.IdempotencyKey, err) } - rows, _ := res.RowsAffected() - created = rows == 1 return nil - }) - return created, err -} - -func loadAndLockTaskByIdempotencyKey(ctx context.Context, db bun.IDB, idempotencyKey string) (*model.Task, error) { - task := new(model.Task) - err := db.NewRaw(`UPDATE tasks - SET status = status - WHERE idempotency_key = ? - RETURNING *`, idempotencyKey).Scan(ctx, task) + }); err != nil { + return nil, false, err + } + if inserted { + return task, true, nil + } + existing, err := r.GetByIdentity(ctx, task.Type, task.IdempotencyKey) if err != nil { - return nil, err + return nil, false, err } - return task, nil + if existing == nil { + return nil, false, fmt.Errorf("loading task after identity conflict: %w", ErrNotFound) + } + return existing, false, nil } -func normalizeTaskStage(task *model.Task) { - if task == nil || task.Stage != nil || task.Type != model.TaskTypeUpload { - return - } - stage, _ := task.Payload["stage"].(string) - if stage == "" { - stage = "prepare_upload" +// withTaskPayload projects the JSON a task carries from the row that holds it. +func withTaskPayload(q *bun.SelectQuery) *bun.SelectQuery { + return q. + ColumnExpr("task.*"). + ColumnExpr("task_payload.input_json AS input"). + ColumnExpr("task_payload.checkpoint_json AS checkpoint"). + Join("JOIN task_payloads AS task_payload ON task_payload.task_id = task.id") +} + +// loadTaskPayload fills in the JSON for a task read without the join, such as +// one returned by the claim statement. +func loadTaskPayload(ctx context.Context, db bun.IDB, task *model.Task) error { + payload := new(model.TaskPayload) + if err := db.NewSelect().Model(payload).Where("task_id = ?", task.ID).Scan(ctx); err != nil { + return fmt.Errorf("selecting task %d payload: %w", task.ID, err) } - task.Stage = &stage + task.Input = payload.Input + task.Checkpoint = payload.Checkpoint + return nil } func (r *BunTaskRepo) GetByID(ctx context.Context, id int64) (*model.Task, error) { task := new(model.Task) - err := r.db.NewSelect(). - Model(task). - Where("id = ?", id). - Scan(ctx) + err := withTaskPayload(r.db.NewSelect().Model(task)).Where("task.id = ?", id).Scan(ctx) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("selecting task by id: %w", err) + return nil, fmt.Errorf("selecting task %d: %w", id, err) } return task, nil } -func (r *BunTaskRepo) GetByIdempotencyKey(ctx context.Context, idempotencyKey string) (*model.Task, error) { +func (r *BunTaskRepo) GetByIdentity(ctx context.Context, taskType model.TaskType, key string) (*model.Task, error) { task := new(model.Task) - err := r.db.NewSelect(). - Model(task). - Where("idempotency_key = ?", idempotencyKey). + err := withTaskPayload(r.db.NewSelect().Model(task)). + Where("task.type = ? AND task.idempotency_key = ?", taskType, key). Scan(ctx) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("selecting task by idempotency key: %w", err) + return nil, fmt.Errorf("selecting task %s/%s: %w", taskType, key, err) } return task, nil } -func (r *BunTaskRepo) HasActiveByIdempotencyKey(ctx context.Context, idempotencyKey string) (bool, error) { - count, err := r.db.NewSelect(). - Model((*model.Task)(nil)). - Where("idempotency_key = ?", idempotencyKey). - Where("status IN (?)", bun.List(activeTaskStatuses())). - Count(ctx) - if err != nil { - return false, fmt.Errorf("checking active task by idempotency key: %w", err) +func (r *BunTaskRepo) ClaimNext(ctx context.Context, leaseDuration time.Duration) (*model.Task, error) { + if leaseDuration <= 0 { + return nil, fmt.Errorf("lease duration must be positive: %w", ErrInvalidInput) } - return count > 0, nil -} - -func (r *BunTaskRepo) HasEarlierRunningUploadCopyTask(ctx context.Context, claimedTask *model.Task, uploadID int64, copyIndex int) (bool, error) { - taskID, claimedAt, err := runningTaskClaim(claimedTask) - if err != nil { - return false, err + if r.db.Dialect().Name() != dialect.PG { + return r.claimNextSQLite(ctx, r.db, leaseDuration) } - uploadIDExpr, copyIndexExpr := runningUploadCopyTaskPayloadExpressions(r.db.Dialect().Name()) - exists, err := r.db.NewSelect(). - Model((*model.Task)(nil)). - Where("type = ?", model.TaskTypeUpload). - Where("status = ?", model.TaskStatusRunning). - Where(uploadIDExpr+" = ?", uploadID). - Where(copyIndexExpr+" = ?", copyIndex). - Where("(claimed_at < ? OR (claimed_at = ? AND id < ?))", claimedAt, claimedAt, taskID). - Exists(ctx) - if err != nil { - return false, fmt.Errorf("checking earlier running upload copy task: %w", err) + db, ok := r.db.(*bun.DB) + if !ok { + return r.claimNextPostgres(ctx, r.db, leaseDuration) } - return exists, nil + var claimed *model.Task + err := db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + var err error + claimed, err = r.claimNextPostgres(ctx, tx, leaseDuration) + return err + }) + return claimed, err } -// HasEarlierRunningUploadCopyClaim checks upload precedence for a replacement item. -func (r *BunTaskRepo) HasEarlierRunningUploadCopyClaim( - ctx context.Context, - claimedAt time.Time, - uploadID int64, - copyIndex int, -) (bool, error) { - if claimedAt.IsZero() || uploadID <= 0 || copyIndex < 0 { - return false, fmt.Errorf("checking earlier upload copy claim: %w", ErrInvalidInput) - } - uploadIDExpr, copyIndexExpr := runningUploadCopyTaskPayloadExpressions(r.db.Dialect().Name()) - exists, err := r.db.NewSelect(). - Model((*model.Task)(nil)). - Where("type = ?", model.TaskTypeUpload). - Where("status = ?", model.TaskStatusRunning). - Where(uploadIDExpr+" = ?", uploadID). - Where(copyIndexExpr+" = ?", copyIndex). - Where("claimed_at <= ?", claimedAt). - Exists(ctx) - if err != nil { - return false, fmt.Errorf("checking earlier upload copy claim: %w", err) - } - return exists, nil +func (r *BunTaskRepo) claimNextPostgres(ctx context.Context, db bun.IDB, leaseDuration time.Duration) (*model.Task, error) { + return claimNextTask(ctx, db, leaseDuration, claimExpiredTaskPostgresSQL, claimPendingTaskPostgresSQL) } -func runningUploadCopyTaskPayloadExpressions(dialectName dialect.Name) (string, string) { - if dialectName == dialect.PG { - return "CAST(payload ->> 'upload_id' AS BIGINT)", "CAST(payload ->> 'copy_index' AS INTEGER)" - } - return "CAST(json_extract(payload, '$.upload_id') AS INTEGER)", "CAST(json_extract(payload, '$.copy_index') AS INTEGER)" +func (r *BunTaskRepo) claimNextSQLite(ctx context.Context, db bun.IDB, leaseDuration time.Duration) (*model.Task, error) { + return claimNextTask(ctx, db, leaseDuration, claimExpiredTaskSQLiteSQL, claimPendingTaskSQLiteSQL) } -// ClaimReady atomically claims one ready task of the given type. -// Uses a SQLite-safe atomic UPDATE (no SELECT FOR UPDATE). -// Returns nil, nil if no task is ready. -func (r *BunTaskRepo) ClaimReady(ctx context.Context, taskType model.TaskType, leaseDuration time.Duration) (*model.Task, error) { +func claimNextTask( + ctx context.Context, + db bun.IDB, + leaseDuration time.Duration, + recoverySQL string, + pendingSQL string, +) (*model.Task, error) { now := time.Now() leaseUntil := now.Add(leaseDuration) - - task := new(model.Task) - // Atomic claim: UPDATE ... WHERE id = (subquery) RETURNING * - // The scheduled_at filter ensures future retry/wait tasks are not claimed prematurely. - err := r.db.NewRaw( - claimReadySQL, - model.TaskStatusRunning, now, leaseUntil, now, - taskType, - now, - ).Scan(ctx, task) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("claiming ready task: %w", err) + task, err := claimTaskWithSQL(ctx, db, recoverySQL, now, leaseUntil) + if err != nil || task != nil { + return task, err } - return task, nil + return claimTaskWithSQL(ctx, db, pendingSQL, now, leaseUntil) } -func (r *BunTaskRepo) RenewLease(ctx context.Context, claimedTask *model.Task, leaseDuration time.Duration) error { - if leaseDuration < 0 { - leaseDuration = 0 - } - taskID, claimedAt, err := runningTaskClaim(claimedTask) - if err != nil { - return err - } - now := time.Now() - res, err := r.db.NewUpdate(). - Model((*model.Task)(nil)). - Set("lease_until = ?", now.Add(leaseDuration)). - Where("id = ? AND status = ?", taskID, model.TaskStatusRunning). - Where("claimed_at = ?", claimedAt). - Where("lease_until IS NOT NULL AND lease_until > ?", now). - Exec(ctx) - if err != nil { - return fmt.Errorf("renewing task lease: %w", err) - } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("renewing task %d lease: not in active running claim", taskID) - } - return nil -} - -// Complete marks a running task as completed. -func (r *BunTaskRepo) Complete(ctx context.Context, claimedTask *model.Task) error { - return r.complete(ctx, claimedTask, "") -} - -func (r *BunTaskRepo) CompleteWithMessage(ctx context.Context, claimedTask *model.Task, message string) error { - return r.complete(ctx, claimedTask, message) -} - -func (r *BunTaskRepo) complete(ctx context.Context, claimedTask *model.Task, message string) error { - taskID, claimedAt, err := runningTaskClaim(claimedTask) - if err != nil { - return err - } - now := time.Now() - q := r.db.NewUpdate(). - Model((*model.Task)(nil)). - Set("status = ?", model.TaskStatusCompleted). - Set("completed_at = ?", now). - Set("last_error = NULL"). - Set("wait_reason = NULL"). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("started_at = NULL"). - Where("id = ? AND status = ?", taskID, model.TaskStatusRunning). - Where("claimed_at = ?", claimedAt). - Where("lease_until IS NOT NULL AND lease_until > ?", now) - if message == "" { - q = q.Set("status_message = NULL") - } else { - q = q.Set("status_message = ?", message) +func claimTaskWithSQL(ctx context.Context, db bun.IDB, query string, now, leaseUntil time.Time) (*model.Task, error) { + task := new(model.Task) + err := db.NewRaw(query, now, leaseUntil, now, now, now, now).Scan(ctx, task) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil } - res, err := q.Exec(ctx) if err != nil { - return fmt.Errorf("completing task: %w", err) + return nil, fmt.Errorf("claiming next task: %w", err) } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("completing task %d: not in active running claim", taskID) + if err := loadTaskPayload(ctx, db, task); err != nil { + return nil, err } - return nil + return task, nil } -// FailRunning marks a running task as failed without scheduling automatic retry. -func (r *BunTaskRepo) FailRunning(ctx context.Context, claimedTask *model.Task, lastError string) error { - taskID, claimedAt, err := runningTaskClaim(claimedTask) - if err != nil { - return err - } +func (r *BunTaskRepo) RenewLease(ctx context.Context, id, generation int64, leaseDuration time.Duration) (time.Time, error) { now := time.Now() - res, err := r.db.NewUpdate(). + until := now.Add(leaseDuration) + result, err := r.db.NewUpdate(). Model((*model.Task)(nil)). - Set("status = ?", model.TaskStatusFailed). - Set("last_error = ?", lastError). - Set("status_message = NULL"). - Set("wait_reason = NULL"). - Set("completed_at = ?", now). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("started_at = NULL"). - Where("id = ? AND status = ?", taskID, model.TaskStatusRunning). - Where("claimed_at = ?", claimedAt). - Where("lease_until IS NOT NULL AND lease_until > ?", now). + Set("lease_until = ?", until). + Set("updated_at = ?", now). + Where("id = ? AND status = ?", id, model.TaskStatusRunning). + Where("claim_generation = ?", generation). + Where("lease_until > ?", now). Exec(ctx) if err != nil { - return fmt.Errorf("failing task: %w", err) + return time.Time{}, fmt.Errorf("renewing task %d lease: %w", id, err) } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("failing task %d: not in active running claim", taskID) + rows, _ := result.RowsAffected() + if rows != 1 { + return time.Time{}, ErrTaskLeaseLost } - return nil + return until, nil } -func (r *BunTaskRepo) ScheduleRetryRunning(ctx context.Context, claimedTask *model.Task, lastError string, backoff time.Duration) (model.TaskStatus, error) { - if backoff < 0 { - backoff = 0 - } - taskID, claimedAt, err := runningTaskClaim(claimedTask) - if err != nil { - return "", err +func (r *BunTaskRepo) WriteCheckpoint(ctx context.Context, id, generation int64, checkpoint []byte) error { + if len(checkpoint) == 0 { + return fmt.Errorf("checkpoint is required: %w", ErrInvalidInput) } - - var nextStatus model.TaskStatus - err = r.runMaybeTx(ctx, func(db bun.IDB) error { - now := time.Now() - task, err := loadRunningTaskClaim(ctx, db, taskID, claimedAt, now) - if err != nil { - return err - } - - nextRetryCount := task.RetryCount + 1 - nextStatus = retryStatusForTask(task) - completedAt := (*time.Time)(nil) - scheduledAt := now.Add(backoff) - if nextStatus == model.TaskStatusExhausted { - completedAt = &now - scheduledAt = now - } - - q := db.NewUpdate(). + now := time.Now() + return runMaybeTx(ctx, r.db, func(db bun.IDB) error { + result, err := db.NewUpdate(). Model((*model.Task)(nil)). - Set("status = ?", nextStatus). - Set("retry_count = ?", nextRetryCount). - Set("last_error = ?", lastError). - Set("status_message = NULL"). - Set("wait_reason = NULL"). - Set("scheduled_at = ?", scheduledAt). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("started_at = NULL"). - Set("completed_at = ?", completedAt). - Where("id = ? AND status = ?", taskID, model.TaskStatusRunning). - Where("claimed_at = ?", claimedAt). - Where("lease_until IS NOT NULL AND lease_until > ?", now) - res, err := q.Exec(ctx) + Set("resume_mode = ?", model.TaskResumeModeRecover). + Set("updated_at = ?", now). + Where("id = ? AND status = ?", id, model.TaskStatusRunning). + Where("claim_generation = ?", generation). + Where("lease_until > ?", now). + Exec(ctx) if err != nil { - return fmt.Errorf("scheduling task retry: %w", err) + return fmt.Errorf("writing task %d checkpoint: %w", id, err) } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("scheduling retry for task %d: not in same running claim", taskID) + if rows, _ := result.RowsAffected(); rows != 1 { + return ErrTaskLeaseLost + } + if _, err := db.NewUpdate(). + Model((*model.TaskPayload)(nil)). + Set("checkpoint_json = ?", checkpoint). + Where("task_id = ?", id). + Exec(ctx); err != nil { + return fmt.Errorf("writing task %d checkpoint: %w", id, err) } return nil }) - if err != nil { - return "", err - } - return nextStatus, nil } -func loadRunningTaskClaim(ctx context.Context, db bun.IDB, taskID int64, claimedAt time.Time, now time.Time) (*model.Task, error) { - task := new(model.Task) - if err := db.NewSelect(). - Model(task). - Where("id = ? AND status = ?", taskID, model.TaskStatusRunning). - Where("claimed_at = ?", claimedAt). - Where("lease_until IS NOT NULL AND lease_until > ?", now). - Scan(ctx); err != nil { - if err == sql.ErrNoRows { - return nil, fmt.Errorf("loading running task claim %d: not in active running claim", taskID) - } - return nil, fmt.Errorf("loading running task claim: %w", err) - } - return task, nil -} - -func runningTaskClaim(task *model.Task) (int64, time.Time, error) { - if task == nil || task.ID == 0 || task.ClaimedAt == nil { - return 0, time.Time{}, fmt.Errorf("running task claim is required: %w", ErrInvalidInput) +func (r *BunTaskRepo) ValidateClaim(ctx context.Context, id, generation int64) error { + now := time.Now() + var found int64 + err := r.db.NewRaw(`UPDATE tasks + SET updated_at = updated_at + WHERE id = ? AND status = 'running' AND claim_generation = ? AND lease_until > ? + RETURNING id`, id, generation, now).Scan(ctx, &found) + if errors.Is(err, sql.ErrNoRows) { + return ErrTaskLeaseLost } - return task.ID, *task.ClaimedAt, nil -} - -func retryStatusForTask(task *model.Task) model.TaskStatus { - if task.RetryCount+1 >= task.MaxRetries { - return model.TaskStatusExhausted + if err != nil { + return fmt.Errorf("validating task %d claim: %w", id, err) } - return model.TaskStatusScheduled + return nil } -func (r *BunTaskRepo) WaitRunning(ctx context.Context, claimedTask *model.Task, reason model.TaskWaitReason, message string, delay time.Duration) error { - if delay < 0 { - delay = 0 - } - taskID, claimedAt, err := runningTaskClaim(claimedTask) - if err != nil { - return err +func (r *BunTaskRepo) Settle(ctx context.Context, id, generation int64, transition TaskTransition) error { + if !validTaskTransition(transition) { + return fmt.Errorf("invalid task transition: %w", ErrInvalidInput) } now := time.Now() - res, err := r.db.NewUpdate(). + query := r.db.NewUpdate(). Model((*model.Task)(nil)). - Set("status = ?", model.TaskStatusWaiting). - Set("last_error = NULL"). - Set("wait_reason = ?", reason). - Set("status_message = ?", message). - Set("scheduled_at = ?", now.Add(delay)). + Set("status = ?", transition.Status). + Set("wait_reason = ?", transition.WaitReason). + Set("failure_reason = ?", transition.FailureReason). + Set("last_error = ?", transition.LastError). + Set("status_message = ?", transition.StatusMessage). Set("claimed_at = NULL"). Set("lease_until = NULL"). - Set("started_at = NULL"). - Where("id = ? AND status = ?", taskID, model.TaskStatusRunning). - Where("claimed_at = ?", claimedAt). - Where("lease_until IS NOT NULL AND lease_until > ?", now). - Exec(ctx) + Set("updated_at = ?", now). + Where("id = ? AND status = ?", id, model.TaskStatusRunning). + Where("claim_generation = ?", generation). + Where("lease_until > ?", now) + if transition.IncrementRetry { + query = query.Set("retry_count = retry_count + 1") + } + if transition.Status == model.TaskStatusPending { + query = query. + Set("resume_mode = CASE WHEN cancellation_requested_at IS NOT NULL THEN ? ELSE ? END", model.TaskResumeModeRecover, transition.ResumeMode). + Set("available_at = CASE WHEN cancellation_requested_at IS NOT NULL THEN ? ELSE ? END", now, transition.AvailableAt). + Set("finished_at = NULL"). + Set("retention_until = NULL"). + Set("acknowledged_at = NULL") + } else { + query = query. + Set("resume_mode = ?", transition.ResumeMode). + Set("finished_at = ?", now). + Set("retention_until = ?", transition.RetentionUntil) + } + result, err := query.Exec(ctx) if err != nil { - return fmt.Errorf("waiting running task: %w", err) + return fmt.Errorf("settling task %d: %w", id, err) } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("waiting task %d: not in active running claim", taskID) + rows, _ := result.RowsAffected() + if rows != 1 { + return ErrTaskLeaseLost } return nil } -func (r *BunTaskRepo) LockRunningClaim(ctx context.Context, claimedTask *model.Task) error { - taskID, claimedAt, err := runningTaskClaim(claimedTask) - if err != nil { - return err - } - now := time.Now() - res, err := r.db.NewUpdate(). - Model((*model.Task)(nil)). - Set("status = status"). - Where("id = ? AND status = ?", taskID, model.TaskStatusRunning). - Where("claimed_at = ?", claimedAt). - Where("lease_until IS NOT NULL AND lease_until > ?", now). - Exec(ctx) - if err != nil { - return fmt.Errorf("locking running task claim: %w", err) +func validTaskTransition(transition TaskTransition) bool { + switch transition.Status { + case model.TaskStatusPending: + return !transition.AvailableAt.IsZero() && + (transition.ResumeMode == model.TaskResumeModeExecute || transition.ResumeMode == model.TaskResumeModeRecover) && + transition.RetentionUntil == nil + case model.TaskStatusCompleted, model.TaskStatusCancelled: + return transition.RetentionUntil != nil + case model.TaskStatusFailed: + return transition.RetentionUntil == nil + default: + return false } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("locking task %d: not in active running claim", taskID) - } - return nil } -func (r *BunTaskRepo) ContinueRunning(ctx context.Context, claimedTask *model.Task, refVersionID string, payload map[string]interface{}) error { - if payload == nil { - return fmt.Errorf("continuation payload is required: %w", ErrInvalidInput) - } - // An object coordinator must keep naming the version it is working on. A - // bucket-scoped coordinator legitimately has none between items. - if refVersionID == "" && claimedTask != nil && claimedTask.RefType == "object" { - return fmt.Errorf("continuation version is required: %w", ErrInvalidInput) - } - taskID, claimedAt, err := runningTaskClaim(claimedTask) - if err != nil { - return err +func (r *BunTaskRepo) ShortenLease(ctx context.Context, id, generation int64, duration time.Duration) error { + if duration <= 0 { + return fmt.Errorf("lease duration must be positive: %w", ErrInvalidInput) } now := time.Now() - res, err := r.db.NewUpdate(). + result, err := r.db.NewUpdate(). Model((*model.Task)(nil)). - Set("ref_version_id = COALESCE(NULLIF(?, ''), ref_version_id)", refVersionID). - Set("payload = ?", payload). - Set("status = ?", model.TaskStatusQueued). - Set("retry_count = 0"). - Set("last_error = NULL"). - Set("status_message = NULL"). - Set("wait_reason = NULL"). - Set("scheduled_at = ?", now). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("started_at = NULL"). - Set("completed_at = NULL"). - Where("id = ? AND status = ?", taskID, model.TaskStatusRunning). - Where("claimed_at = ?", claimedAt). - Where("lease_until IS NOT NULL AND lease_until > ?", now). + Set("resume_mode = ?", model.TaskResumeModeRecover). + Set("lease_until = ?", now.Add(duration)). + Set("updated_at = ?", now). + Where("id = ? AND status = ?", id, model.TaskStatusRunning). + Where("claim_generation = ?", generation). + Where("lease_until > ?", now). Exec(ctx) if err != nil { - return fmt.Errorf("continuing running task: %w", err) + return fmt.Errorf("shortening task %d lease: %w", id, err) } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("continuing task %d: not in active running claim", taskID) + rows, _ := result.RowsAffected() + if rows != 1 { + return ErrTaskLeaseLost } return nil } -func (r *BunTaskRepo) ReleaseRunning(ctx context.Context, claimedTask *model.Task) error { - taskID, claimedAt, err := runningTaskClaim(claimedTask) - if err != nil { - return err +func (r *BunTaskRepo) WakePending(ctx context.Context, ids []int64) (int, error) { + if len(ids) == 0 { + return 0, nil + } + for _, id := range ids { + if id < 1 { + return 0, fmt.Errorf("task IDs must be positive: %w", ErrInvalidInput) + } } now := time.Now() - res, err := r.db.NewUpdate(). + result, err := r.db.NewUpdate(). Model((*model.Task)(nil)). - Set("status = ?", model.TaskStatusQueued). - Set("scheduled_at = ?", now). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("started_at = NULL"). - Set("last_error = NULL"). - Set("wait_reason = NULL"). - Set("status_message = NULL"). - Where("id = ? AND status = ?", taskID, model.TaskStatusRunning). - Where("claimed_at = ?", claimedAt). - Where("lease_until IS NOT NULL AND lease_until > ?", now). + Set("available_at = ?", now). + Set("updated_at = ?", now). + Where("id IN (?)", bun.List(ids)). + Where("status = ?", model.TaskStatusPending). + Where("available_at > ?", now). Exec(ctx) if err != nil { - return fmt.Errorf("releasing running task: %w", err) - } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("releasing task %d: not in active running claim", taskID) + return 0, fmt.Errorf("waking pending tasks: %w", err) } - return nil + rows, _ := result.RowsAffected() + return int(rows), nil } -func (r *BunTaskRepo) CancelRunning(ctx context.Context, claimedTask *model.Task, message string) error { - taskID, claimedAt, err := runningTaskClaim(claimedTask) - if err != nil { - return err - } +func (r *BunTaskRepo) RequestCancellation(ctx context.Context, id int64, reason string) error { now := time.Now() - q := r.db.NewUpdate(). + result, err := r.db.NewUpdate(). Model((*model.Task)(nil)). - Set("status = ?", model.TaskStatusCancelled). - Set("completed_at = ?", now). - Set("last_error = NULL"). - Set("wait_reason = NULL"). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("started_at = NULL"). - Where("id = ? AND status = ?", taskID, model.TaskStatusRunning). - Where("claimed_at = ?", claimedAt). - Where("lease_until IS NOT NULL AND lease_until > ?", now) - if message == "" { - q = q.Set("status_message = NULL") - } else { - q = q.Set("status_message = ?", message) - } - res, err := q.Exec(ctx) + Set("cancellation_requested_at = COALESCE(cancellation_requested_at, ?)", now). + Set("cancellation_reason = COALESCE(cancellation_reason, ?)", nullableText(reason)). + Set("resume_mode = ?", model.TaskResumeModeRecover). + Set("available_at = CASE WHEN status = ? THEN ? ELSE available_at END", model.TaskStatusPending, now). + Set("updated_at = ?", now). + Where("id = ? AND status IN (?, ?)", id, model.TaskStatusPending, model.TaskStatusRunning). + Exec(ctx) if err != nil { - return fmt.Errorf("cancelling running task: %w", err) + return fmt.Errorf("requesting cancellation for task %d: %w", id, err) } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("cancelling task %d: not in active running claim", taskID) + rows, _ := result.RowsAffected() + if rows != 1 { + return ErrNotFound } return nil } -// ReleaseExpiredLeases resets running tasks whose lease has expired back to queued. -func (r *BunTaskRepo) ReleaseExpiredLeases(ctx context.Context) (int, error) { +func (r *BunTaskRepo) RetryFailed(ctx context.Context, id int64) error { now := time.Now() - res, err := r.db.NewUpdate(). + result, err := r.db.NewUpdate(). Model((*model.Task)(nil)). - Set("status = ?", model.TaskStatusQueued). - Set("scheduled_at = ?", now). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("started_at = NULL"). + Set("status = ?", model.TaskStatusPending). + Set("resume_mode = ?", model.TaskResumeModeRecover). + Set("available_at = ?", now). + Set("retry_count = 0"). + Set("failure_reason = NULL"). Set("last_error = NULL"). - Set("wait_reason = NULL"). - Set("status_message = NULL"). - Where("status = ? AND (lease_until IS NULL OR lease_until < ?)", model.TaskStatusRunning, now). - Exec(ctx) - if err != nil { - return 0, fmt.Errorf("releasing expired leases: %w", err) - } - rows, _ := res.RowsAffected() - return int(rows), nil -} - -// MarkRunningExhausted marks a running task as exhausted. -func (r *BunTaskRepo) MarkRunningExhausted(ctx context.Context, claimedTask *model.Task, lastError string) error { - taskID, claimedAt, err := runningTaskClaim(claimedTask) - if err != nil { - return err - } - now := time.Now() - res, err := r.db.NewUpdate(). - Model((*model.Task)(nil)). - Set("status = ?", model.TaskStatusExhausted). - Set("last_error = ?", lastError). Set("status_message = NULL"). Set("wait_reason = NULL"). - Set("retry_count = retry_count + 1"). - Set("completed_at = ?", now). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("started_at = NULL"). - Where("id = ? AND status = ?", taskID, model.TaskStatusRunning). - Where("claimed_at = ?", claimedAt). - Where("lease_until IS NOT NULL AND lease_until > ?", now). + Set("finished_at = NULL"). + Set("acknowledged_at = NULL"). + Set("retention_until = NULL"). + Set("updated_at = ?", now). + Where("id = ? AND status = ?", id, model.TaskStatusFailed). Exec(ctx) if err != nil { - return fmt.Errorf("marking task exhausted: %w", err) + return fmt.Errorf("retrying task %d: %w", id, err) } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("marking task %d exhausted: not in active running claim", taskID) + rows, _ := result.RowsAffected() + if rows != 1 { + return ErrNotFound } return nil } -// ListExhausted returns exhausted tasks, ordered by most recent first. -func (r *BunTaskRepo) ListExhausted(ctx context.Context, limit int) ([]model.Task, error) { - var tasks []model.Task - q := r.db.NewSelect(). - Model(&tasks). - Where("status = ?", model.TaskStatusExhausted). - OrderExpr("COALESCE(completed_at, started_at, scheduled_at) DESC") - if limit > 0 { - q = q.Limit(limit) - } - err := q.Scan(ctx) - if err != nil { - return nil, fmt.Errorf("listing exhausted tasks: %w", err) +// ReactivateTerminal reuses a terminal idempotency record as a fresh execute +// run. It is intentionally narrower than manual retry: callers must first +// verify that the task's immutable input still describes the desired work. +func (r *BunTaskRepo) ReactivateTerminal(ctx context.Context, id int64) error { + if id < 1 { + return fmt.Errorf("reactivating terminal task: %w", ErrInvalidInput) } - return tasks, nil -} - -func (r *BunTaskRepo) RetryExhausted(ctx context.Context, taskID int64) error { - return r.runMaybeTx(ctx, func(db bun.IDB) error { - task := new(model.Task) - err := db.NewSelect(). - Model(task). - Where("id = ? AND status = ?", taskID, model.TaskStatusExhausted). - Scan(ctx) - if err != nil { - if err == sql.ErrNoRows { - return fmt.Errorf("retrying exhausted task %d: %w", taskID, ErrNotFound) - } - return fmt.Errorf("loading exhausted task: %w", err) - } - - // Replacement work carries state the generic queue knows nothing about, - // so it must resume through the dedicated replacement action instead. - if storagereplacement.IsCoordinatorTask(task.Type, task.Stage) { - return ErrReplacementRetryUnsupported - } - - now := time.Now() - if err := resetFailedObjectForTaskRetry(ctx, db, task, now); err != nil { - return err - } - - res, err := db.NewUpdate(). + now := time.Now() + return runMaybeTx(ctx, r.db, func(db bun.IDB) error { + result, err := db.NewUpdate(). Model((*model.Task)(nil)). - Set("status = ?", model.TaskStatusQueued). + Set("status = ?", model.TaskStatusPending). + Set("resume_mode = ?", model.TaskResumeModeExecute). + Set("available_at = ?", now). + Set("wait_reason = NULL"). Set("retry_count = 0"). - Set("scheduled_at = ?", now). + Set("failure_reason = NULL"). + Set("last_error = NULL"). + Set("status_message = NULL"). + Set("cancellation_requested_at = NULL"). + Set("cancellation_reason = NULL"). Set("claimed_at = NULL"). Set("lease_until = NULL"). Set("started_at = NULL"). - Set("completed_at = NULL"). - Set("last_error = NULL"). - Set("wait_reason = NULL"). - Set("status_message = NULL"). - Where("id = ? AND status = ?", taskID, model.TaskStatusExhausted). + Set("finished_at = NULL"). + Set("acknowledged_at = NULL"). + Set("retention_until = NULL"). + Set("updated_at = ?", now). + Where("id = ? AND type = ? AND status IN (?, ?)", id, model.TaskTypeUploadPlan, model.TaskStatusFailed, model.TaskStatusCancelled). Exec(ctx) if err != nil { - return fmt.Errorf("retrying exhausted task: %w", err) + return fmt.Errorf("reactivating terminal task %d: %w", id, err) } - rows, _ := res.RowsAffected() - if rows == 0 { - return fmt.Errorf("retrying exhausted task %d: %w", taskID, ErrNotFound) + if rows, _ := result.RowsAffected(); rows != 1 { + return fmt.Errorf("reactivating terminal task %d: %w", id, ErrConflict) + } + result, err = db.NewUpdate(). + Model((*model.TaskPayload)(nil)). + Set("checkpoint_json = NULL"). + Where("task_id = ?", id). + Exec(ctx) + if err != nil { + return fmt.Errorf("clearing terminal task %d checkpoint: %w", id, err) + } + if rows, _ := result.RowsAffected(); rows != 1 { + return fmt.Errorf("clearing terminal task %d checkpoint: %w", id, ErrConflict) } return nil }) } -func resetFailedObjectForTaskRetry(ctx context.Context, db bun.IDB, task *model.Task, now time.Time) error { - if task.Type != model.TaskTypeUpload || task.RefType != "object" || task.RefVersionID == "" { - return nil - } - - target := retryObjectState(task) - uploadID := int64(0) - if target == model.ObjectStateReplicating { - uploadID = taskPayloadInt64(task.Payload, "upload_id") - if uploadID > 0 { - if _, err := lockStorageUploadForObjectState(ctx, db, uploadID, target); err != nil { - return fmt.Errorf("locking storage upload for task retry: %w", err) - } - } +func (r *BunTaskRepo) AcknowledgeFailed(ctx context.Context, id int64, retention time.Duration) error { + if retention <= 0 { + return fmt.Errorf("retention must be positive: %w", ErrInvalidInput) } - q := db.NewUpdate(). - Model((*model.ObjectVersion)(nil)). - Set("state = ?", target). - Set("failed_at_state = NULL"). - Set("last_error = NULL"). + now := time.Now() + result, err := r.db.NewUpdate(). + Model((*model.Task)(nil)). + Set("acknowledged_at = COALESCE(acknowledged_at, ?)", now). + Set("retention_until = COALESCE(retention_until, ?)", now.Add(retention)). Set("updated_at = ?", now). - Where("version_id = ? AND state = ?", task.RefVersionID, model.ObjectStateFailed) - if uploadID > 0 { - q = q.Set("storage_upload_id = ?", uploadID) + Where("id = ? AND status = ?", id, model.TaskStatusFailed). + Exec(ctx) + if err != nil { + return fmt.Errorf("acknowledging task %d: %w", id, err) } - if _, err := q.Exec(ctx); err != nil { - return fmt.Errorf("resetting failed object for task retry: %w", err) + rows, _ := result.RowsAffected() + if rows != 1 { + return ErrNotFound } return nil } -func retryObjectState(task *model.Task) model.ObjectState { - stage := "" - if task.Stage != nil { - stage = *task.Stage - } - if stage == "" { - stage, _ = task.Payload["stage"].(string) - } - switch stage { - case "ingress_commit": - return model.ObjectStateCommitting - case "peer_pull", "peer_commit": - return model.ObjectStateReplicating - case "ensure_dataset": - if taskPayloadString(task.Payload, "transfer_method") == string(model.StorageCopyTransferMethodPeerPull) { - return model.ObjectStateReplicating - } +func (r *BunTaskRepo) DeleteRetained(ctx context.Context, now time.Time, limit int) (int, error) { + if now.IsZero() || limit < 1 { + return 0, fmt.Errorf("deleting retained tasks: %w", ErrInvalidInput) + } + if limit > 1000 { + limit = 1000 + } + var ids []int64 + err := r.db.NewSelect(). + Model((*model.Task)(nil)). + Column("id"). + Where("retention_until IS NOT NULL AND retention_until <= ?", now). + Where(`NOT EXISTS (SELECT 1 FROM object_cache WHERE cache_active_task_id = task.id)`). + Where(`NOT EXISTS (SELECT 1 FROM buckets WHERE durability_task_id = task.id)`). + Where(`NOT EXISTS (SELECT 1 FROM storage_contents WHERE cleanup_task_id = task.id)`). + Where(`NOT EXISTS (SELECT 1 FROM storage_copies WHERE active_task_id = task.id)`). + Where(`NOT EXISTS (SELECT 1 FROM storage_data_sets WHERE ensure_task_id = task.id OR retirement_task_id = task.id)`). + Where(`NOT EXISTS (SELECT 1 FROM wallet_operations WHERE task_id = task.id)`). + Where(`NOT EXISTS (SELECT 1 FROM storage_replacements WHERE task_id = task.id)`). + OrderExpr("retention_until, id"). + Limit(limit). + Scan(ctx, &ids) + if err != nil { + return 0, fmt.Errorf("selecting retained tasks: %w", err) + } + if len(ids) == 0 { + return 0, nil + } + result, err := r.db.NewDelete(). + Model((*model.Task)(nil)). + Where("id IN (?)", bun.List(ids)). + Where("retention_until IS NOT NULL AND retention_until <= ?", now). + Exec(ctx) + if err != nil { + return 0, fmt.Errorf("deleting retained tasks: %w", err) } - return model.ObjectStateUploading + rows, _ := result.RowsAffected() + return int(rows), nil } -func taskPayloadInt64(payload map[string]interface{}, key string) int64 { - if payload == nil { - return 0 +func (r *BunTaskRepo) List(ctx context.Context, filter TaskListFilter) (TaskPage, error) { + limit := filter.Limit + if limit <= 0 || limit > 100 { + limit = 50 } - raw, ok := payload[key] - if !ok { - return 0 + var tasks []model.Task + query := withTaskPayload(r.db.NewSelect().Model(&tasks)). + OrderExpr("task.id DESC"). + Limit(limit + 1) + if filter.Type != "" { + query = query.Where("task.type = ?", filter.Type) + } + if filter.Status != "" { + query = query.Where("task.status = ?", filter.Status) + } + if filter.Acknowledged != nil { + if *filter.Acknowledged { + query = query.Where("task.acknowledged_at IS NOT NULL") + } else { + query = query.Where("task.acknowledged_at IS NULL") + } } - switch v := raw.(type) { - case int: - return int64(v) - case int64: - return v - case float64: - return int64(v) + if filter.HideHealthyRecurringSystem { + query = query.Where("(task.type NOT IN (?) OR task.status = ?)", bun.List(model.RecurringSystemTaskTypes()), model.TaskStatusFailed) } - return 0 -} - -func taskPayloadString(payload map[string]interface{}, key string) string { - if payload == nil { - return "" + if filter.BeforeID > 0 { + query = query.Where("task.id < ?", filter.BeforeID) } - raw, ok := payload[key] - if !ok { - return "" + if err := query.Scan(ctx); err != nil { + return TaskPage{}, fmt.Errorf("listing tasks: %w", err) } - value, _ := raw.(string) - return value -} - -func (r *BunTaskRepo) runMaybeTx(ctx context.Context, fn func(bun.IDB) error) error { - if db, ok := r.db.(*bun.DB); ok { - return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { - return fn(tx) - }) + page := TaskPage{Tasks: tasks} + if len(page.Tasks) > limit { + page.Tasks = page.Tasks[:limit] + page.NextBeforeID = page.Tasks[len(page.Tasks)-1].ID } - return fn(r.db) + return page, nil } -// CountByStatus returns task counts grouped by type and status. func (r *BunTaskRepo) CountByStatus(ctx context.Context) ([]TaskStatusCount, error) { var counts []TaskStatusCount err := r.db.NewSelect(). - TableExpr("tasks"). - ColumnExpr("type, status, COUNT(*) AS count"). - GroupExpr("type, status"). + Model((*model.Task)(nil)). + Column("type", "status"). + ColumnExpr("COUNT(*) AS count"). + Group("type", "status"). Scan(ctx, &counts) if err != nil { return nil, fmt.Errorf("counting tasks by status: %w", err) @@ -858,134 +641,80 @@ func (r *BunTaskRepo) CountByStatus(ctx context.Context) ([]TaskStatusCount, err return counts, nil } -func (r *BunTaskRepo) CountOverviewActivePipeline(ctx context.Context) ([]TaskPipelineCount, error) { - var counts []TaskPipelineCount - err := r.db.NewRaw(`SELECT pipeline, status, COUNT(*) AS count - FROM ( - SELECT - CASE - WHEN type = ? AND (stage IS NULL OR stage = '' OR stage IN (?, ?)) THEN 'prepare' - WHEN type = ? AND stage = ? THEN 'upload' - WHEN type = ? AND stage = ? THEN 'commit' - WHEN type = ? AND stage IN (?, ?) THEN 'sync' - WHEN type = ? THEN 'evict' - WHEN type = ? THEN 'cleanup' - ELSE '' - END AS pipeline, - status - FROM tasks - WHERE status IN (?) - ) AS active_tasks - WHERE pipeline <> '' - GROUP BY pipeline, status`, - model.TaskTypeUpload, - "prepare_upload", - "ensure_dataset", - model.TaskTypeUpload, - "ingress_store", - model.TaskTypeUpload, - "ingress_commit", - model.TaskTypeUpload, - "peer_pull", - "peer_commit", - model.TaskTypeEvictCache, - model.TaskTypeStorageCleanup, - bun.List(activeTaskStatuses()), - ).Scan(ctx, &counts) +func (r *BunTaskRepo) CountByPresentationStatus(ctx context.Context) ([]TaskStatusCount, error) { + const presentationStatus = "CASE WHEN status = 'failed' AND acknowledged_at IS NOT NULL THEN 'dismissed' ELSE status END" + var counts []TaskStatusCount + err := r.db.NewSelect(). + Model((*model.Task)(nil)). + Column("type"). + ColumnExpr(presentationStatus+" AS status"). + ColumnExpr("COUNT(*) AS count"). + GroupExpr("type, "+presentationStatus). + Scan(ctx, &counts) if err != nil { - return nil, fmt.Errorf("counting overview active task pipeline: %w", err) + return nil, fmt.Errorf("counting tasks by presentation status: %w", err) } return counts, nil } -func activeTaskStatuses() []model.TaskStatus { - return append(unclaimedTaskStatuses(), model.TaskStatusRunning) +func (r *BunTaskRepo) CountUnacknowledgedFailed(ctx context.Context) (int64, error) { + count, err := r.db.NewSelect(). + Model((*model.Task)(nil)). + Where("status = ?", model.TaskStatusFailed). + Where("acknowledged_at IS NULL"). + Count(ctx) + if err != nil { + return 0, fmt.Errorf("counting unacknowledged failed tasks: %w", err) + } + return int64(count), nil } -func unclaimedTaskStatuses() []model.TaskStatus { - return []model.TaskStatus{ - model.TaskStatusQueued, - model.TaskStatusScheduled, - model.TaskStatusWaiting, +func (r *BunTaskRepo) CountOverviewActivePipeline(ctx context.Context) ([]TaskPipelineCount, error) { + var counts []TaskPipelineCount + err := r.db.NewSelect(). + Model((*model.Task)(nil)). + ColumnExpr("type AS pipeline"). + Column("status"). + ColumnExpr("COUNT(*) AS count"). + Where("status IN (?, ?)", model.TaskStatusPending, model.TaskStatusRunning). + Where("type NOT IN (?)", bun.List(model.RecurringSystemTaskTypes())). + Group("type", "status"). + Scan(ctx, &counts) + if err != nil { + return nil, fmt.Errorf("counting active task pipeline: %w", err) } + return counts, nil } func (r *BunTaskRepo) CountActiveObjectTasksByBucket(ctx context.Context, bucketID int64) (int64, error) { - count, err := r.db.NewSelect(). - TableExpr("tasks AS t"). - Join("JOIN objects AS o ON o.id = t.ref_id"). - Where("t.ref_type = ?", "object"). - Where("o.bucket_id = ?", bucketID). - Where("t.status IN (?)", bun.List(activeTaskStatuses())). - Count(ctx) + var count int64 + err := r.db.NewRaw(`SELECT COUNT(*) + FROM tasks AS t + JOIN object_versions AS ov ON ov.version_id = t.subject_key + WHERE t.subject_type = 'object_version' + AND t.status IN ('pending', 'running') + AND ov.bucket_id = ?`, bucketID).Scan(ctx, &count) if err != nil { return 0, fmt.Errorf("counting active object tasks by bucket: %w", err) } - return int64(count), nil + return count, nil } func (r *BunTaskRepo) CountActiveBucketTasksByBucketID(ctx context.Context, bucketID int64) (int64, error) { count, err := r.db.NewSelect(). - TableExpr("tasks"). - Where("ref_type = ?", "bucket"). - Where("ref_id = ?", bucketID). - Where("status IN (?)", bun.List(activeTaskStatuses())). + Model((*model.Task)(nil)). + Where("subject_type = 'bucket' AND subject_key = ?", fmt.Sprint(bucketID)). + Where("status IN (?, ?)", model.TaskStatusPending, model.TaskStatusRunning). Count(ctx) if err != nil { - return 0, fmt.Errorf("counting active bucket tasks by bucket ID: %w", err) + return 0, fmt.Errorf("counting active bucket tasks: %w", err) } return int64(count), nil } -// List returns tasks with optional type/stage/status filters, paginated by offset/limit. -func (r *BunTaskRepo) List(ctx context.Context, taskType string, stage string, status string, limit, offset int) ([]model.Task, int, error) { - applyFilters := func(q *bun.SelectQuery) *bun.SelectQuery { - if taskType != "" { - q = q.Where("type = ?", taskType) - } - if stage != "" { - q = q.Where("stage = ?", stage) - } - if status != "" { - q = q.Where("status = ?", status) - } - return q - } - - total, err := applyFilters(r.db.NewSelect().Model((*model.Task)(nil))).Count(ctx) - if err != nil { - return nil, 0, fmt.Errorf("counting tasks: %w", err) - } - var tasks []model.Task - err = applyFilters(r.db.NewSelect().Model(&tasks)).OrderExpr("id DESC").Limit(limit).Offset(offset).Scan(ctx) - if err != nil { - return nil, 0, fmt.Errorf("listing tasks: %w", err) - } - return tasks, total, nil -} - -func (r *BunTaskRepo) CompleteByRef(ctx context.Context, refType string, refID int64, taskType model.TaskType) error { - now := time.Now() - res, err := r.db.NewUpdate().Model((*model.Task)(nil)). - Set("status = ?", model.TaskStatusCompleted). - Set("completed_at = ?", now). - Set("last_error = NULL"). - Set("wait_reason = NULL"). - Set("status_message = NULL"). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("started_at = NULL"). - Where("ref_type = ?", refType). - Where("ref_id = ?", refID). - Where("type = ?", taskType). - Where("status IN (?)", bun.List(activeTaskStatuses())). - Exec(ctx) - if err != nil { - return fmt.Errorf("completing tasks by ref: %w", err) - } - n, _ := res.RowsAffected() - if n == 0 { - return fmt.Errorf("no matching %s task for %s/%d: %w", taskType, refType, refID, ErrNotFound) +func nullableText(value string) any { + if value == "" { + return nil } - return nil + return value } diff --git a/internal/db/repository/task_repo_internal_test.go b/internal/db/repository/task_repo_internal_test.go deleted file mode 100644 index 0f2c539..0000000 --- a/internal/db/repository/task_repo_internal_test.go +++ /dev/null @@ -1,194 +0,0 @@ -package repository - -import ( - "context" - "database/sql" - "path/filepath" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - synaps3db "github.com/strahe/synaps3/internal/db" - "github.com/strahe/synaps3/internal/model" - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" - "github.com/uptrace/bun/dialect/sqlitedialect" - _ "modernc.org/sqlite" -) - -type recurringTaskLockHook struct { - queries atomic.Int32 - second chan struct{} - once sync.Once -} - -func (h *recurringTaskLockHook) BeforeQuery(ctx context.Context, event *bun.QueryEvent) context.Context { - if strings.Contains(event.Query, "SET status = status") && h.queries.Add(1) == 2 { - h.once.Do(func() { close(h.second) }) - } - return ctx -} - -func (*recurringTaskLockHook) AfterQuery(context.Context, *bun.QueryEvent) {} - -func TestClaimReadySQLUsesReadyScheduledIndex(t *testing.T) { - sqldb, err := sql.Open("sqlite", "file:"+filepath.Join(t.TempDir(), "claim-ready-plan.db")+"?_pragma=foreign_keys(1)") - if err != nil { - t.Fatalf("open sqlite: %v", err) - } - db := bun.NewDB(sqldb, sqlitedialect.New()) - t.Cleanup(func() { _ = db.Close() }) - - ctx := context.Background() - if err := synaps3db.RunMigrations(ctx, db); err != nil { - t.Fatalf("RunMigrations: %v", err) - } - - now := time.Now() - rows, err := sqldb.QueryContext(ctx, "EXPLAIN QUERY PLAN "+claimReadySQL, - model.TaskStatusRunning, now, now.Add(time.Minute), now, - model.TaskTypeUpload, - now, - ) - if err != nil { - t.Fatalf("EXPLAIN QUERY PLAN ClaimReady: %v", err) - } - defer func() { _ = rows.Close() }() - - var details []string - for rows.Next() { - var id int - var parent int - var notUsed int - var detail string - if err := rows.Scan(&id, &parent, ¬Used, &detail); err != nil { - t.Fatalf("scan plan row: %v", err) - } - details = append(details, detail) - } - if err := rows.Err(); err != nil { - t.Fatalf("iterate plan rows: %v", err) - } - - plan := strings.Join(details, "\n") - if !strings.Contains(plan, "idx_tasks_type_ready_scheduled") { - t.Fatalf("ClaimReady plan =\n%s\nwant idx_tasks_type_ready_scheduled", plan) - } - if strings.Contains(plan, "USE TEMP B-TREE") { - t.Fatalf("ClaimReady plan =\n%s\nwant no temp sort", plan) - } -} - -func TestRunningUploadCopyTaskPayloadExpressionsAreDialectSpecific(t *testing.T) { - tests := []struct { - name string - dialect dialect.Name - wantUploadID string - wantCopyIndex string - }{ - { - name: "sqlite", - dialect: dialect.SQLite, - wantUploadID: "CAST(json_extract(payload, '$.upload_id') AS INTEGER)", - wantCopyIndex: "CAST(json_extract(payload, '$.copy_index') AS INTEGER)", - }, - { - name: "postgres", - dialect: dialect.PG, - wantUploadID: "CAST(payload ->> 'upload_id' AS BIGINT)", - wantCopyIndex: "CAST(payload ->> 'copy_index' AS INTEGER)", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - uploadID, copyIndex := runningUploadCopyTaskPayloadExpressions(tt.dialect) - if uploadID != tt.wantUploadID || copyIndex != tt.wantCopyIndex { - t.Fatalf("payload expressions = %q, %q; want %q, %q", uploadID, copyIndex, tt.wantUploadID, tt.wantCopyIndex) - } - }) - } -} - -func TestEnsureRecurringCannotLoseWakeupWhileCoordinatorCompletes(t *testing.T) { - sqldb, err := sql.Open("sqlite", "file:"+filepath.Join(t.TempDir(), "recurring-lock.db")+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(1)") - if err != nil { - t.Fatalf("open sqlite: %v", err) - } - sqldb.SetMaxOpenConns(4) - db := bun.NewDB(sqldb, sqlitedialect.New()) - t.Cleanup(func() { _ = db.Close() }) - ctx := context.Background() - if err := synaps3db.RunMigrations(ctx, db); err != nil { - t.Fatalf("RunMigrations: %v", err) - } - repos := NewRepositories(db) - stage := "repair_replica" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "bucket", RefID: 1, - IdempotencyKey: "upload:repair-data-set:lost-wakeup", Status: model.TaskStatusQueued, - MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create: %v", err) - } - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil || claimed == nil { - t.Fatalf("ClaimReady: task=%#v err=%v", claimed, err) - } - - hook := &recurringTaskLockHook{second: make(chan struct{})} - db.AddQueryHook(hook) - locked := make(chan struct{}) - release := make(chan struct{}) - completeResult := make(chan error, 1) - go func() { - completeResult <- repos.WithTx(ctx, func(txRepos *Repositories) error { - if err := txRepos.Tasks.LockRunningClaim(ctx, claimed); err != nil { - return err - } - close(locked) - <-release - return txRepos.Tasks.Complete(ctx, claimed) - }) - }() - waitTestSignal(t, locked, "coordinator task lock") - - wakeup := *task - wakeup.ID = 0 - wakeup.Payload = map[string]interface{}{"storage_upload_copy_id": int64(202)} - wakeupResult := make(chan struct { - created bool - err error - }, 1) - go func() { - created, err := repos.Tasks.EnsureRecurring(ctx, &wakeup) - wakeupResult <- struct { - created bool - err error - }{created: created, err: err} - }() - waitTestSignal(t, hook.second, "concurrent recurring wakeup") - close(release) - if err := <-completeResult; err != nil { - t.Fatalf("complete coordinator: %v", err) - } - result := <-wakeupResult - if result.err != nil || !result.created { - t.Fatalf("EnsureRecurring after coordinator completion: created=%t err=%v", result.created, result.err) - } - got, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil || got == nil || got.Status != model.TaskStatusQueued { - t.Fatalf("recurring task after wakeup = %#v err=%v, want queued", got, err) - } -} - -func waitTestSignal(t *testing.T, ch <-chan struct{}, name string) { - t.Helper() - select { - case <-ch: - case <-time.After(5 * time.Second): - t.Fatalf("timed out waiting for %s", name) - } -} diff --git a/internal/db/repository/task_repo_postgres_test.go b/internal/db/repository/task_repo_postgres_test.go new file mode 100644 index 0000000..4ea3986 --- /dev/null +++ b/internal/db/repository/task_repo_postgres_test.go @@ -0,0 +1,173 @@ +package repository_test + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/stdlib" + "github.com/strahe/synaps3/internal/db/migrations" + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect/pgdialect" +) + +func TestPostgresConcurrentTaskClaimsAreUnique(t *testing.T) { + db := newPostgresTaskDB(t) + repos := repository.NewRepositories(db) + const taskCount = 24 + input := json.RawMessage(`{}`) + sum := sha256.Sum256(input) + for index := range taskCount { + if _, created, err := repos.Tasks.Enqueue(t.Context(), &model.Task{ + Type: model.TaskType("postgres_claim_test"), IdempotencyKey: fmt.Sprintf("claim-%d", index), + InputVersion: 1, Input: input, InputHash: hex.EncodeToString(sum[:]), + Status: model.TaskStatusPending, ResumeMode: model.TaskResumeModeExecute, AvailableAt: time.Now(), + }); err != nil || !created { + t.Fatalf("enqueue task %d: created=%t err=%v", index, created, err) + } + } + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + claimed := make(chan *model.Task, taskCount) + errorsFound := make(chan error, taskCount) + var workers sync.WaitGroup + for range taskCount { + workers.Go(func() { + row, err := repos.Tasks.ClaimNext(ctx, time.Minute) + if err != nil { + errorsFound <- err + return + } + claimed <- row + }) + } + workers.Wait() + close(claimed) + close(errorsFound) + for err := range errorsFound { + t.Errorf("claim task: %v", err) + } + seen := make(map[int64]struct{}, taskCount) + for row := range claimed { + if row == nil { + t.Error("concurrent claim returned no task") + continue + } + if _, exists := seen[row.ID]; exists { + t.Errorf("task %d was claimed more than once", row.ID) + } + seen[row.ID] = struct{}{} + } + if len(seen) != taskCount { + t.Fatalf("unique claims = %d, want %d", len(seen), taskCount) + } +} + +func TestPostgresTaskClaimSkipsLockedHeadWithoutLegacyAdvisoryLock(t *testing.T) { + db := newPostgresTaskDB(t) + repos := repository.NewRepositories(db) + availableAt := time.Now().Add(-time.Minute) + for index := range 2 { + if _, created, err := repos.Tasks.Enqueue(t.Context(), &model.Task{ + Type: model.TaskType("postgres_skip_locked_test"), IdempotencyKey: fmt.Sprintf("claim-%d", index), + InputVersion: 1, Input: json.RawMessage(`{}`), InputHash: fmt.Sprintf("hash-%d", index), + Status: model.TaskStatusPending, ResumeMode: model.TaskResumeModeExecute, + AvailableAt: availableAt.Add(time.Duration(index) * time.Second), + }); err != nil || !created { + t.Fatalf("enqueue task %d: created=%t err=%v", index, created, err) + } + } + + tx, err := db.BeginTx(t.Context(), nil) + if err != nil { + t.Fatalf("begin blocker transaction: %v", err) + } + defer func() { _ = tx.Rollback() }() + if _, err := tx.ExecContext(t.Context(), "SELECT pg_advisory_xact_lock(384)"); err != nil { + t.Fatalf("hold legacy advisory lock: %v", err) + } + var lockedID int64 + if err := tx.NewRaw(`SELECT id FROM tasks + WHERE status = 'pending' AND available_at <= ? + ORDER BY available_at, id + LIMIT 1 + FOR UPDATE`, time.Now()).Scan(t.Context(), &lockedID); err != nil { + t.Fatalf("lock queue head: %v", err) + } + + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + type claimResult struct { + task *model.Task + err error + } + result := make(chan claimResult, 1) + go func() { + claimed, claimErr := repos.Tasks.ClaimNext(ctx, time.Minute) + result <- claimResult{task: claimed, err: claimErr} + }() + select { + case claimed := <-result: + if claimed.err != nil || claimed.task == nil { + t.Fatalf("claim unlocked task = %#v err=%v", claimed.task, claimed.err) + } + if claimed.task.ID == lockedID { + t.Fatalf("claimed locked queue head %d", lockedID) + } + case <-ctx.Done(): + t.Fatal("claim blocked behind the queue head or legacy advisory lock") + } +} + +func newPostgresTaskDB(t *testing.T) *bun.DB { + t.Helper() + dsn := os.Getenv("SYNAPS3_POSTGRES_TEST_DSN") + if dsn == "" { + t.Skip("SYNAPS3_POSTGRES_TEST_DSN is not set") + } + config, err := pgx.ParseConfig(dsn) + if err != nil { + t.Fatalf("parse PostgreSQL DSN: %v", err) + } + adminSQL, err := sql.Open("pgx", dsn) + if err != nil { + t.Fatalf("open PostgreSQL admin connection: %v", err) + } + adminDB := bun.NewDB(adminSQL, pgdialect.New()) + schema := fmt.Sprintf("task_claim_%x", sha256.Sum256([]byte(fmt.Sprintf("%s-%d", t.Name(), time.Now().UnixNano()))))[:40] + quotedSchema := `"` + strings.ReplaceAll(schema, `"`, `""`) + `"` + if _, err := adminDB.Exec("CREATE SCHEMA " + quotedSchema); err != nil { + _ = adminDB.Close() + t.Fatalf("create PostgreSQL schema: %v", err) + } + config.RuntimeParams["search_path"] = schema + db := bun.NewDB(stdlib.OpenDB(*config), pgdialect.New()) + t.Cleanup(func() { + _ = db.Close() + _, _ = adminDB.Exec("DROP SCHEMA " + quotedSchema + " CASCADE") + _ = adminDB.Close() + }) + if err := migrations.ValidateTarget(t.Context(), db); err != nil { + t.Fatalf("validate empty PostgreSQL schema: %v", err) + } + migrator := migrations.NewMigrator(db) + if err := migrator.Init(t.Context()); err != nil { + t.Fatalf("initialize PostgreSQL migrations: %v", err) + } + if _, err := migrator.Migrate(t.Context()); err != nil { + t.Fatalf("migrate PostgreSQL schema: %v", err) + } + return db +} diff --git a/internal/db/repository/task_repo_test.go b/internal/db/repository/task_repo_test.go index cee9e74..64e3cd5 100644 --- a/internal/db/repository/task_repo_test.go +++ b/internal/db/repository/task_repo_test.go @@ -2,1671 +2,99 @@ package repository_test import ( "context" - "errors" + "strings" + "sync" "testing" "time" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/testutil" + "github.com/uptrace/bun" ) -func seedTask(t *testing.T, repos *repository.Repositories, taskType model.TaskType) *model.Task { - t.Helper() - task := &model.Task{ - Type: taskType, - RefType: "object", - RefID: 1, - RefVersionID: "01J0000000000000000000TASK", - IdempotencyKey: "idem-" + string(taskType) + "-" + time.Now().Format(time.RFC3339Nano), - Status: model.TaskStatusQueued, - } - if err := repos.Tasks.Create(context.Background(), task); err != nil { - t.Fatalf("seeding task: %v", err) - } - return task -} - -func assertTaskIDs(t *testing.T, tasks []model.Task, want ...int64) { - t.Helper() - if len(tasks) != len(want) { - t.Fatalf("task count = %d, want %d; tasks=%#v", len(tasks), len(want), tasks) - } - for i, task := range tasks { - if task.ID != want[i] { - t.Fatalf("task[%d].ID = %d, want %d; tasks=%#v", i, task.ID, want[i], tasks) - } - } -} - -func TestTaskRepo_ClaimReady(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - // No tasks yet — should return nil. - task, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if err != nil { - t.Fatalf("ClaimReady empty: %v", err) - } - if task != nil { - t.Fatal("expected nil when no tasks") - } - - // Seed a task and claim it. - seeded := seedTask(t, repos, model.TaskTypeUpload) - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if err != nil { - t.Fatalf("ClaimReady: %v", err) - } - if claimed == nil { - t.Fatal("expected claimed task, got nil") - } - if claimed.ID != seeded.ID { - t.Errorf("expected task ID %d, got %d", seeded.ID, claimed.ID) - } - if claimed.Status != model.TaskStatusRunning { - t.Errorf("expected status running, got %s", claimed.Status) - } - if claimed.ClaimedAt == nil || claimed.LeaseUntil == nil { - t.Error("expected claimed_at and lease_until to be set") - } - - // Claiming again should return nil (already running). - again, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if err != nil { - t.Fatalf("ClaimReady again: %v", err) - } - if again != nil { - t.Fatal("expected nil when no ready tasks left") - } -} - -func TestTaskRepo_ClaimReadyHandlesQueuedScheduledAndWaiting(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - now := time.Now() - - queued := seedTask(t, repos, model.TaskTypeUpload) - scheduled := seedTask(t, repos, model.TaskTypeUpload) - waiting := seedTask(t, repos, model.TaskTypeUpload) - futureScheduled := seedTask(t, repos, model.TaskTypeUpload) - - mustExec(t, db, `UPDATE tasks SET scheduled_at = ? WHERE id = ?`, now.Add(-3*time.Minute), queued.ID) - mustExec(t, db, `UPDATE tasks SET status = ?, scheduled_at = ? WHERE id = ?`, model.TaskStatusScheduled, now.Add(-2*time.Minute), scheduled.ID) - mustExec(t, db, `UPDATE tasks SET status = ?, wait_reason = ?, status_message = ?, scheduled_at = ? WHERE id = ?`, model.TaskStatusWaiting, model.TaskWaitReasonDependency, "waiting for all copies", now.Add(-time.Minute), waiting.ID) - mustExec(t, db, `UPDATE tasks SET status = ?, scheduled_at = ? WHERE id = ?`, model.TaskStatusScheduled, now.Add(time.Hour), futureScheduled.ID) - - for _, wantID := range []int64{queued.ID, scheduled.ID, waiting.ID} { - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if err != nil { - t.Fatalf("ClaimReady: %v", err) - } - if claimed == nil { - t.Fatalf("ClaimReady returned nil, want task %d", wantID) - } - if claimed.ID != wantID { - t.Fatalf("ClaimReady ID = %d, want %d", claimed.ID, wantID) - } - if claimed.Status != model.TaskStatusRunning { - t.Fatalf("ClaimReady status = %s, want running", claimed.Status) - } - if claimed.LastError != nil || claimed.WaitReason != nil || claimed.StatusMessage != nil { - t.Fatalf("claimed task diagnostics = last:%v wait:%v message:%v, want cleared", claimed.LastError, claimed.WaitReason, claimed.StatusMessage) - } - } - - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if err != nil { - t.Fatalf("ClaimReady future scheduled: %v", err) - } - if claimed != nil { - t.Fatalf("ClaimReady claimed future scheduled task: %#v", claimed) - } -} - -func TestTaskRepo_ClaimReadyBreaksScheduledTiesByID(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - scheduledAt := time.Now().Add(-time.Minute) - - first := seedTask(t, repos, model.TaskTypeUpload) - second := seedTask(t, repos, model.TaskTypeUpload) - mustExec(t, db, `UPDATE tasks SET scheduled_at = ? WHERE id IN (?, ?)`, scheduledAt, first.ID, second.ID) - - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if err != nil { - t.Fatalf("ClaimReady: %v", err) - } - if claimed == nil { - t.Fatal("ClaimReady returned nil, want task") - } - if claimed.ID != first.ID { - t.Fatalf("ClaimReady ID = %d, want lowest ID %d", claimed.ID, first.ID) - } -} - -func TestTaskRepo_RunningFailureTransitions(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - retryable := seedTask(t, repos, model.TaskTypeUpload) - retryable.MaxRetries = 3 - mustExec(t, db, `UPDATE tasks SET max_retries = ? WHERE id = ?`, retryable.MaxRetries, retryable.ID) - claimed, _ := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - - status, err := repos.Tasks.ScheduleRetryRunning(ctx, claimed, "temporary rpc error", time.Minute) - if err != nil { - t.Fatalf("ScheduleRetryRunning: %v", err) - } - if status != model.TaskStatusScheduled { - t.Fatalf("ScheduleRetryRunning status = %s, want scheduled", status) - } - got, _ := repos.Tasks.GetByID(ctx, claimed.ID) - if got.Status != model.TaskStatusScheduled { - t.Fatalf("retryable status = %s, want scheduled", got.Status) - } - if got.RetryCount != 1 { - t.Fatalf("retry_count = %d, want 1", got.RetryCount) - } - if got.LastError == nil || *got.LastError != "temporary rpc error" { - t.Fatalf("last_error = %v, want temporary rpc error", got.LastError) - } - if got.WaitReason != nil || got.StatusMessage != nil { - t.Fatalf("wait diagnostics = %v/%v, want nil", got.WaitReason, got.StatusMessage) - } - - exhausted := seedTask(t, repos, model.TaskTypeUpload) - mustExec(t, db, `UPDATE tasks SET retry_count = 2, max_retries = 3 WHERE id = ?`, exhausted.ID) - claimed, _ = repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - status, err = repos.Tasks.ScheduleRetryRunning(ctx, claimed, "permanent timeout", time.Minute) - if err != nil { - t.Fatalf("ScheduleRetryRunning exhausted: %v", err) - } - if status != model.TaskStatusExhausted { - t.Fatalf("ScheduleRetryRunning exhausted status = %s, want exhausted", status) - } - got, _ = repos.Tasks.GetByID(ctx, claimed.ID) - if got.Status != model.TaskStatusExhausted { - t.Fatalf("exhausted status = %s, want exhausted", got.Status) - } - if got.CompletedAt == nil { - t.Fatal("completed_at is nil, want set for exhausted task") - } - - zeroRetry := seedTask(t, repos, model.TaskTypeUpload) - mustExec(t, db, `UPDATE tasks SET max_retries = 0 WHERE id = ?`, zeroRetry.ID) - claimed, _ = repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - status, err = repos.Tasks.ScheduleRetryRunning(ctx, claimed, "no retries configured", time.Minute) - if err != nil { - t.Fatalf("ScheduleRetryRunning zero retry: %v", err) - } - if status != model.TaskStatusExhausted { - t.Fatalf("ScheduleRetryRunning zero retry status = %s, want exhausted", status) - } - got, _ = repos.Tasks.GetByID(ctx, claimed.ID) - if got.Status != model.TaskStatusExhausted { - t.Fatalf("zero retry status = %s, want exhausted", got.Status) - } - if got.RetryCount != 1 { - t.Fatalf("zero retry count = %d, want 1", got.RetryCount) - } - - seedTask(t, repos, model.TaskTypeUpload) - claimed, _ = repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err := repos.Tasks.FailRunning(ctx, claimed, "invalid object state"); err != nil { - t.Fatalf("FailRunning: %v", err) - } - got, _ = repos.Tasks.GetByID(ctx, claimed.ID) - if got.Status != model.TaskStatusFailed { - t.Fatalf("failed status = %s, want failed", got.Status) - } - if got.LastError == nil || *got.LastError != "invalid object state" { - t.Fatalf("last_error = %v, want invalid object state", got.LastError) - } -} - -func TestTaskRepo_ScheduleRetryRunningRejectsStaleClaim(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - seedTask(t, repos, model.TaskTypeUpload) - staleClaim, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, -time.Second) - if err != nil { - t.Fatalf("ClaimReady stale: %v", err) - } - if staleClaim == nil || staleClaim.ClaimedAt == nil { - t.Fatal("ClaimReady stale returned no claimed task") - } - if _, err := repos.Tasks.ScheduleRetryRunning(ctx, staleClaim, "expired worker failure", time.Minute); err == nil { - t.Fatal("expected expired claim retry to fail") - } - got, err := repos.Tasks.GetByID(ctx, staleClaim.ID) - if err != nil { - t.Fatalf("GetByID expired: %v", err) - } - if got.Status != model.TaskStatusRunning { - t.Fatalf("status after expired retry = %s, want running", got.Status) - } - if got.RetryCount != 0 { - t.Fatalf("retry_count after expired retry = %d, want 0", got.RetryCount) - } - if _, err := repos.Tasks.ReleaseExpiredLeases(ctx); err != nil { - t.Fatalf("ReleaseExpiredLeases: %v", err) - } - freshClaim, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil { - t.Fatalf("ClaimReady fresh: %v", err) - } - if freshClaim == nil || freshClaim.ClaimedAt == nil { - t.Fatal("ClaimReady fresh returned no claimed task") - } - if staleClaim.ID != freshClaim.ID { - t.Fatalf("claim ids = stale:%d fresh:%d, want same task", staleClaim.ID, freshClaim.ID) - } - - if _, err := repos.Tasks.ScheduleRetryRunning(ctx, staleClaim, "stale worker failure", time.Minute); err == nil { - t.Fatal("expected stale claim retry to fail") - } - got, err = repos.Tasks.GetByID(ctx, freshClaim.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.Status != model.TaskStatusRunning { - t.Fatalf("status after stale retry = %s, want running", got.Status) - } - if got.RetryCount != 0 { - t.Fatalf("retry_count after stale retry = %d, want 0", got.RetryCount) - } - if got.LastError != nil { - t.Fatalf("last_error after stale retry = %v, want nil", got.LastError) - } - if got.ClaimedAt == nil || !got.ClaimedAt.Equal(*freshClaim.ClaimedAt) { - t.Fatalf("claimed_at after stale retry = %v, want fresh claim %v", got.ClaimedAt, freshClaim.ClaimedAt) - } - - status, err := repos.Tasks.ScheduleRetryRunning(ctx, freshClaim, "fresh worker failure", time.Minute) - if err != nil { - t.Fatalf("ScheduleRetryRunning fresh: %v", err) - } - if status != model.TaskStatusScheduled { - t.Fatalf("fresh retry status = %s, want scheduled", status) - } -} - -func TestTaskRepo_WaitRunningStoresNonErrorReason(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - seedTask(t, repos, model.TaskTypeEvictCache) - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeEvictCache, time.Minute) - if err != nil { - t.Fatalf("ClaimReady: %v", err) - } - if claimed == nil { - t.Fatal("ClaimReady returned nil") - } - - if err := repos.Tasks.WaitRunning(ctx, claimed, model.TaskWaitReasonDependency, "waiting for all copies to commit", time.Minute); err != nil { - t.Fatalf("WaitRunning: %v", err) - } - got, _ := repos.Tasks.GetByID(ctx, claimed.ID) - if got.Status != model.TaskStatusWaiting { - t.Fatalf("status = %s, want waiting", got.Status) - } - if got.RetryCount != 0 { - t.Fatalf("retry_count = %d, want 0", got.RetryCount) - } - if got.LastError != nil { - t.Fatalf("last_error = %v, want nil", got.LastError) - } - if got.WaitReason == nil || *got.WaitReason != model.TaskWaitReasonDependency { - t.Fatalf("wait_reason = %v, want dependency", got.WaitReason) - } - if got.StatusMessage == nil || *got.StatusMessage != "waiting for all copies to commit" { - t.Fatalf("status_message = %v, want waiting message", got.StatusMessage) - } -} - -func TestTaskRepo_RenewLease(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - seedTask(t, repos, model.TaskTypeUpload) - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil { - t.Fatalf("ClaimReady: %v", err) - } - if claimed == nil || claimed.LeaseUntil == nil { - t.Fatal("expected claimed task with lease") - } - oldLeaseUntil := *claimed.LeaseUntil - - if err := repos.Tasks.RenewLease(ctx, claimed, 10*time.Minute); err != nil { - t.Fatalf("RenewLease: %v", err) - } - - task, err := repos.Tasks.GetByID(ctx, claimed.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if task.Status != model.TaskStatusRunning { - t.Fatalf("task status = %s, want running", task.Status) - } - if task.LeaseUntil == nil || !task.LeaseUntil.After(oldLeaseUntil) { - t.Fatalf("lease_until = %v, want after %s", task.LeaseUntil, oldLeaseUntil) - } - if task.ClaimedAt == nil || task.StartedAt == nil { - t.Fatalf("claimed_at/start_at = %v/%v, want preserved", task.ClaimedAt, task.StartedAt) - } -} - -func TestTaskRepo_RenewLeaseRejectsStaleClaim(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - seedTask(t, repos, model.TaskTypeUpload) - staleClaim, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, -time.Second) - if err != nil { - t.Fatalf("ClaimReady stale: %v", err) - } - if staleClaim == nil || staleClaim.ClaimedAt == nil { - t.Fatal("ClaimReady stale returned no claimed task") - } - if err := repos.Tasks.RenewLease(ctx, staleClaim, time.Minute); err == nil { - t.Fatal("expected expired lease renewal to fail") - } - if _, err := repos.Tasks.ReleaseExpiredLeases(ctx); err != nil { - t.Fatalf("ReleaseExpiredLeases: %v", err) - } - freshClaim, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil { - t.Fatalf("ClaimReady fresh: %v", err) - } - if freshClaim == nil || freshClaim.ClaimedAt == nil || freshClaim.LeaseUntil == nil { - t.Fatal("ClaimReady fresh returned no active claim") - } - freshLeaseUntil := *freshClaim.LeaseUntil - - if err := repos.Tasks.RenewLease(ctx, staleClaim, 10*time.Minute); err == nil { - t.Fatal("expected stale claim renewal to fail") - } - got, err := repos.Tasks.GetByID(ctx, freshClaim.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.LeaseUntil == nil || !got.LeaseUntil.Equal(freshLeaseUntil) { - t.Fatalf("lease_until after stale renewal = %v, want %s", got.LeaseUntil, freshLeaseUntil) - } -} - -func TestTaskRepo_Complete(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - seedTask(t, repos, model.TaskTypeUpload) - claimed, _ := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if claimed == nil { - t.Fatal("setup: could not claim task") - } - mustExec(t, db, `UPDATE tasks SET status_message = ? WHERE id = ?`, "stale status", claimed.ID) - - if err := repos.Tasks.Complete(ctx, claimed); err != nil { - t.Fatalf("Complete: %v", err) - } - - task, _ := repos.Tasks.GetByID(ctx, claimed.ID) - if task.Status != model.TaskStatusCompleted { - t.Errorf("expected completed, got %s", task.Status) - } - if task.CompletedAt == nil { - t.Error("expected completed_at to be set") - } - if task.ClaimedAt != nil || task.LeaseUntil != nil || task.StartedAt != nil { - t.Fatalf("completed task lease fields = claimed:%v lease:%v started:%v, want cleared", task.ClaimedAt, task.LeaseUntil, task.StartedAt) - } - if task.StatusMessage != nil { - t.Fatalf("completed task status_message = %q, want nil", *task.StatusMessage) - } -} - -func TestTaskRepo_CompleteWithMessage(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - seedTask(t, repos, model.TaskTypeUpload) - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if err != nil { - t.Fatalf("ClaimReady: %v", err) - } - if claimed == nil { - t.Fatal("setup: could not claim task") - } - - message := "completed successfully with special condition" - if err := repos.Tasks.CompleteWithMessage(ctx, claimed, message); err != nil { - t.Fatalf("CompleteWithMessage: %v", err) - } - - task, err := repos.Tasks.GetByID(ctx, claimed.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if task == nil { - t.Fatal("expected task to be found, got nil") - } - if task.Status != model.TaskStatusCompleted { - t.Errorf("expected status %s, got %s", model.TaskStatusCompleted, task.Status) - } - if task.StatusMessage == nil { - t.Errorf("expected status_message %q, got nil", message) - } else if *task.StatusMessage != message { - t.Errorf("expected status_message %q, got %q", message, *task.StatusMessage) - } -} - -func TestTaskRepo_StateTransitionsRejectInvalidStatus(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - for _, tc := range []struct { - name string - fn func(seeded *model.Task) error - }{ - { - name: "Complete not running", - fn: func(seeded *model.Task) error { - return repos.Tasks.Complete(ctx, seeded) - }, - }, - { - name: "MarkRunningExhausted not running", - fn: func(seeded *model.Task) error { - return repos.Tasks.MarkRunningExhausted(ctx, seeded, "should fail") - }, - }, - { - name: "RetryExhausted not exhausted", - fn: func(seeded *model.Task) error { - return repos.Tasks.RetryExhausted(ctx, seeded.ID) - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - seeded := seedTask(t, repos, model.TaskTypeUpload) - if err := tc.fn(seeded); err == nil { - t.Fatal("expected error, got nil") - } - }) - } -} - -func TestTaskRepo_FailRunning(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - seedTask(t, repos, model.TaskTypeUpload) - claimed, _ := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - - if err := repos.Tasks.FailRunning(ctx, claimed, "SP unreachable"); err != nil { - t.Fatalf("FailRunning: %v", err) - } - - task, _ := repos.Tasks.GetByID(ctx, claimed.ID) - if task.Status != model.TaskStatusFailed { - t.Errorf("expected failed, got %s", task.Status) - } - if task.LastError == nil || *task.LastError != "SP unreachable" { - t.Error("expected last_error to be set") - } - if task.RetryCount != 0 { - t.Errorf("expected retry_count 0, got %d", task.RetryCount) - } - if task.CompletedAt == nil { - t.Error("expected completed_at to be set") - } - if task.ClaimedAt != nil || task.LeaseUntil != nil || task.StartedAt != nil { - t.Fatalf("failed task lease fields = claimed:%v lease:%v started:%v, want cleared", task.ClaimedAt, task.LeaseUntil, task.StartedAt) - } - - queued := seedTask(t, repos, model.TaskTypeUpload) - now := time.Now() - leaseUntil := now.Add(5 * time.Minute) - mustExec(t, db, `UPDATE tasks SET claimed_at = ?, lease_until = ?, started_at = ? WHERE id = ?`, now, leaseUntil, now, queued.ID) - queued.ClaimedAt = &now - queued.LeaseUntil = &leaseUntil - queued.StartedAt = &now - err := repos.Tasks.FailRunning(ctx, queued, "should fail") - if err == nil { - t.Fatal("expected error failing queued task") - } -} - -func TestTaskRepo_CancelRunning(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - seedTask(t, repos, model.TaskTypeUpload) - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if err != nil { - t.Fatalf("ClaimReady: %v", err) - } - if claimed == nil { - t.Fatal("setup: could not claim task") - } - mustExec(t, db, `UPDATE tasks SET last_error = ? WHERE id = ?`, "stale error", claimed.ID) - - if err := repos.Tasks.CancelRunning(ctx, claimed, "cancelled by user"); err != nil { - t.Fatalf("CancelRunning: %v", err) - } - - task, err := repos.Tasks.GetByID(ctx, claimed.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if task.Status != model.TaskStatusCancelled { - t.Errorf("expected cancelled, got %s", task.Status) - } - if task.LastError != nil { - t.Errorf("expected last_error to be cleared, got %q", *task.LastError) - } - if task.StatusMessage == nil || *task.StatusMessage != "cancelled by user" { - t.Errorf("expected status_message %q, got %v", "cancelled by user", task.StatusMessage) - } - if task.CompletedAt == nil { - t.Error("expected completed_at to be set") - } - if task.ClaimedAt != nil || task.LeaseUntil != nil || task.StartedAt != nil { - t.Fatalf("cancelled task lease fields = claimed:%v lease:%v started:%v, want cleared", task.ClaimedAt, task.LeaseUntil, task.StartedAt) - } - - queued := seedTask(t, repos, model.TaskTypeUpload) - now := time.Now() - leaseUntil := now.Add(5 * time.Minute) - mustExec(t, db, `UPDATE tasks SET claimed_at = ?, lease_until = ?, started_at = ? WHERE id = ?`, now, leaseUntil, now, queued.ID) - queued.ClaimedAt = &now - queued.LeaseUntil = &leaseUntil - queued.StartedAt = &now - if err := repos.Tasks.CancelRunning(ctx, queued, "should fail"); err == nil { - t.Fatal("expected error cancelling queued task") - } -} - -func TestTaskRepo_CancelRunningWithoutMessage(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - seedTask(t, repos, model.TaskTypeUpload) - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if err != nil { - t.Fatalf("ClaimReady: %v", err) - } - if claimed == nil { - t.Fatal("setup: could not claim task") - } - mustExec(t, db, `UPDATE tasks SET status_message = ? WHERE id = ?`, "stale status", claimed.ID) - - if err := repos.Tasks.CancelRunning(ctx, claimed, ""); err != nil { - t.Fatalf("CancelRunning: %v", err) - } - - task, err := repos.Tasks.GetByID(ctx, claimed.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if task.Status != model.TaskStatusCancelled { - t.Errorf("expected cancelled, got %s", task.Status) - } - if task.StatusMessage != nil { - t.Errorf("expected status_message nil, got %q", *task.StatusMessage) - } -} - -func TestTaskRepo_CancelRunningRejectsStaleClaim(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - seedTask(t, repos, model.TaskTypeUpload) - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if err != nil { - t.Fatalf("ClaimReady: %v", err) - } - if claimed == nil { - t.Fatal("setup: could not claim task") - } - - if err := repos.Tasks.ReleaseRunning(ctx, claimed); err != nil { - t.Fatalf("ReleaseRunning: %v", err) - } - - if err := repos.Tasks.CancelRunning(ctx, claimed, "cancel"); err == nil { - t.Fatal("expected error cancelling stale claim") - } - - task, err := repos.Tasks.GetByID(ctx, claimed.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if task.Status != model.TaskStatusQueued { - t.Fatalf("status after stale cancel = %s, want queued", task.Status) - } -} - -func TestTaskRepo_ReleaseExpiredLeases(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - seedTask(t, repos, model.TaskTypeUpload) - // Claim with a very short lease that will be expired - claimed, _ := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, -1*time.Second) - if claimed == nil { - t.Fatal("setup: could not claim task") - } - mustExec(t, db, `UPDATE tasks SET last_error = ? WHERE id = ?`, "stale lease diagnostic", claimed.ID) - - released, err := repos.Tasks.ReleaseExpiredLeases(ctx) - if err != nil { - t.Fatalf("ReleaseExpiredLeases: %v", err) - } - if released != 1 { - t.Errorf("expected 1 released, got %d", released) - } - - // Task should be queued again. - task, _ := repos.Tasks.GetByID(ctx, claimed.ID) - if task.Status != model.TaskStatusQueued { - t.Errorf("expected queued after release, got %s", task.Status) - } - if task.LastError != nil { - t.Fatalf("last_error = %v, want cleared", task.LastError) - } - - // Can claim it again - reclaimed, _ := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if reclaimed == nil { - t.Fatal("expected to reclaim released task") - } -} - -func TestTaskRepo_ReleaseRunningClearsError(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - seedTask(t, repos, model.TaskTypeUpload) - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if err != nil { - t.Fatalf("ClaimReady: %v", err) - } - if claimed == nil { - t.Fatal("setup: could not claim task") - } - mustExec(t, db, `UPDATE tasks SET last_error = ? WHERE id = ?`, "stale running diagnostic", claimed.ID) - - if err := repos.Tasks.ReleaseRunning(ctx, claimed); err != nil { - t.Fatalf("ReleaseRunning: %v", err) - } - - got, err := repos.Tasks.GetByID(ctx, claimed.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.Status != model.TaskStatusQueued { - t.Fatalf("status = %s, want queued", got.Status) - } - if got.LastError != nil || got.ClaimedAt != nil || got.LeaseUntil != nil || got.StartedAt != nil { - t.Fatalf("released task fields = error:%v claimed:%v lease:%v started:%v, want cleared", got.LastError, got.ClaimedAt, got.LeaseUntil, got.StartedAt) - } -} - -func TestTaskRepo_ReleaseExpiredLeasesPreservesActiveLease(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - seedTask(t, repos, model.TaskTypeUpload) - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if err != nil { - t.Fatalf("ClaimReady: %v", err) - } - if claimed == nil { - t.Fatal("setup: could not claim task") - } - - released, err := repos.Tasks.ReleaseExpiredLeases(ctx) - if err != nil { - t.Fatalf("ReleaseExpiredLeases: %v", err) - } - if released != 0 { - t.Fatalf("released = %d, want 0", released) - } - - task, err := repos.Tasks.GetByID(ctx, claimed.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if task.Status != model.TaskStatusRunning { - t.Fatalf("task status = %s, want running", task.Status) - } +type taskGCSelectionBarrier struct { + selected chan struct{} + release chan struct{} + once sync.Once } -func TestTaskRepo_MarkRunningExhausted(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - seedTask(t, repos, model.TaskTypeUpload) - claimed, _ := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if claimed == nil { - t.Fatal("setup: could not claim task") - } - - if err := repos.Tasks.MarkRunningExhausted(ctx, claimed, "max retries reached"); err != nil { - t.Fatalf("MarkRunningExhausted: %v", err) - } - - task, _ := repos.Tasks.GetByID(ctx, claimed.ID) - if task.Status != model.TaskStatusExhausted { - t.Errorf("expected exhausted, got %s", task.Status) - } - if task.LastError == nil || *task.LastError != "max retries reached" { - t.Error("expected last_error to be set") - } - if task.RetryCount != 1 { - t.Errorf("expected retry_count 1, got %d", task.RetryCount) - } - if task.CompletedAt == nil { - t.Error("expected completed_at to be set") - } +func (b *taskGCSelectionBarrier) BeforeQuery(ctx context.Context, _ *bun.QueryEvent) context.Context { + return ctx } -func TestTaskRepo_ListExhausted(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - // Initially empty - tasks, err := repos.Tasks.ListExhausted(ctx, 100) - if err != nil { - t.Fatalf("ListExhausted empty: %v", err) - } - if len(tasks) != 0 { - t.Fatalf("expected 0 exhausted tasks, got %d", len(tasks)) - } - - // Create a exhausted task - seedTask(t, repos, model.TaskTypeUpload) - claimed, _ := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - _ = repos.Tasks.MarkRunningExhausted(ctx, claimed, "permanent failure") - - tasks, err = repos.Tasks.ListExhausted(ctx, 100) - if err != nil { - t.Fatalf("ListExhausted: %v", err) +func (b *taskGCSelectionBarrier) AfterQuery(ctx context.Context, event *bun.QueryEvent) { + query := strings.ToLower(event.Query) + if event.Err != nil || event.Operation() != "SELECT" || + !strings.Contains(query, "retention_until is not null") || + !strings.Contains(query, "not exists") { + return } - if len(tasks) != 1 { - t.Fatalf("expected 1 exhausted task, got %d", len(tasks)) - } - if tasks[0].ID != claimed.ID { - t.Errorf("expected task ID %d, got %d", claimed.ID, tasks[0].ID) - } - - // Test limit - seedTask(t, repos, model.TaskTypeEvictCache) - claimed2, _ := repos.Tasks.ClaimReady(ctx, model.TaskTypeEvictCache, 5*time.Minute) - _ = repos.Tasks.MarkRunningExhausted(ctx, claimed2, "permanent failure 2") - - tasks, err = repos.Tasks.ListExhausted(ctx, 1) - if err != nil { - t.Fatalf("ListExhausted limit: %v", err) - } - if len(tasks) != 1 { - t.Fatalf("expected 1 task with limit=1, got %d", len(tasks)) - } -} - -func TestTaskRepo_RetryExhausted(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - // Create and move to exhausted - seedTask(t, repos, model.TaskTypeUpload) - claimed, _ := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - _ = repos.Tasks.MarkRunningExhausted(ctx, claimed, "permanent failure") - - // Retry - if err := repos.Tasks.RetryExhausted(ctx, claimed.ID); err != nil { - t.Fatalf("RetryExhausted: %v", err) - } - - task, _ := repos.Tasks.GetByID(ctx, claimed.ID) - if task.Status != model.TaskStatusQueued { - t.Errorf("expected queued after retry, got %s", task.Status) - } - if task.RetryCount != 0 { - t.Errorf("expected retry_count 0, got %d", task.RetryCount) - } - if task.ClaimedAt != nil { - t.Error("expected claimed_at to be nil") - } - if task.LastError != nil { - t.Error("expected last_error to be nil") - } - if task.CompletedAt != nil { - t.Error("expected completed_at to be nil") - } - - // Can be claimed again - reclaimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if err != nil { - t.Fatalf("ClaimReady after retry: %v", err) - } - if reclaimed == nil { - t.Fatal("expected to reclaim retried task") - } -} - -func TestTaskRepo_RetryExhaustedClearsFailedUploadObject(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := seedBucket(t, db, "retry-object-bucket") - version := newObjectVersion(bucket.ID, "file.txt", "01J00000000000000000RETRY", 10) - objectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version) - if err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading state: %v", err) - } - if err := repos.Objects.UpdateVersionStateToFailed(ctx, version.VersionID, model.ObjectStateUploading, "create dataset failed"); err != nil { - t.Fatalf("failed state: %v", err) - } - - stage := "ensure_dataset" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objectID, - RefVersionID: version.VersionID, - IdempotencyKey: "retry-object-failed", - Payload: map[string]interface{}{"copy_index": 0}, - Status: model.TaskStatusExhausted, - RetryCount: 5, - MaxRetries: 5, - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task: %v", err) - } - - if err := repos.Tasks.RetryExhausted(ctx, task.ID); err != nil { - t.Fatalf("RetryExhausted: %v", err) - } - - got, err := repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil { - t.Fatalf("GetVersionByID: %v", err) - } - if got.State != model.ObjectStateUploading { - t.Fatalf("state = %s, want uploading", got.State) - } - if got.FailedAtState != nil || got.LastError != nil { - t.Fatalf("failure details = failed_at_state:%#v last_error:%#v, want nil", got.FailedAtState, got.LastError) - } -} - -func TestTaskRepo_RetryPrimaryCommitExhaustedRestoresCommittingState(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := seedBucket(t, db, "retry-commit-bucket") - version := newObjectVersion(bucket.ID, "file.txt", "01J0000000000000000COMMIT", 10) - objectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version) - if err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading state: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("committing state: %v", err) - } - if err := repos.Objects.UpdateVersionStateToFailed(ctx, version.VersionID, model.ObjectStateCommitting, "commit failed"); err != nil { - t.Fatalf("failed state: %v", err) - } - - stage := "ingress_commit" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objectID, - RefVersionID: version.VersionID, - IdempotencyKey: "retry-primary-commit-failed", - Payload: map[string]interface{}{"upload_id": 12}, - Status: model.TaskStatusExhausted, - RetryCount: 5, - MaxRetries: 5, - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task: %v", err) - } - - if err := repos.Tasks.RetryExhausted(ctx, task.ID); err != nil { - t.Fatalf("RetryExhausted: %v", err) - } - - got, err := repos.Objects.GetVersionByID(ctx, version.VersionID) - if err != nil { - t.Fatalf("GetVersionByID: %v", err) - } - if got.State != model.ObjectStateCommitting { - t.Fatalf("state = %s, want committing", got.State) - } - if got.FailedAtState != nil || got.LastError != nil { - t.Fatalf("failure details = failed_at_state:%#v last_error:%#v, want nil", got.FailedAtState, got.LastError) - } -} - -func TestTaskRepo_List(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - // Empty table. - tasks, total, err := repos.Tasks.List(ctx, "", "", "", 10, 0) - if err != nil { - t.Fatalf("List empty: %v", err) - } - if total != 0 || len(tasks) != 0 { - t.Fatalf("expected 0/0, got %d/%d", len(tasks), total) - } - - // Seed tasks: 2 upload (queued), 1 evict_cache (queued). - firstUpload := seedTask(t, repos, model.TaskTypeUpload) - secondUpload := seedTask(t, repos, model.TaskTypeUpload) - evict := seedTask(t, repos, model.TaskTypeEvictCache) - - // Claim one upload task to make it running. - claimed, _ := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if claimed == nil { - t.Fatal("setup: could not claim task") - } - runningUploadID := claimed.ID - queuedUploadID := firstUpload.ID - if runningUploadID == firstUpload.ID { - queuedUploadID = secondUpload.ID - } - - // List all — should return 3. - tasks, total, err = repos.Tasks.List(ctx, "", "", "", 10, 0) - if err != nil { - t.Fatalf("List all: %v", err) - } - if total != 3 { - t.Errorf("expected total 3, got %d", total) - } - if len(tasks) != 3 { - t.Errorf("expected 3 tasks, got %d", len(tasks)) - } - assertTaskIDs(t, tasks, evict.ID, secondUpload.ID, firstUpload.ID) - - // Filter by type. - tasks, total, err = repos.Tasks.List(ctx, string(model.TaskTypeUpload), "", "", 10, 0) - if err != nil { - t.Fatalf("List by type: %v", err) - } - if total != 2 { - t.Errorf("expected 2 upload, got %d", total) - } - assertTaskIDs(t, tasks, secondUpload.ID, firstUpload.ID) - - // Filter by status. - tasks, total, err = repos.Tasks.List(ctx, "", "", string(model.TaskStatusQueued), 10, 0) - if err != nil { - t.Fatalf("List by status: %v", err) - } - if total != 2 { - t.Errorf("expected 2 queued, got %d", total) - } - assertTaskIDs(t, tasks, evict.ID, queuedUploadID) - - // Filter by type + status. - tasks, total, err = repos.Tasks.List(ctx, string(model.TaskTypeUpload), "", string(model.TaskStatusRunning), 10, 0) - if err != nil { - t.Fatalf("List by type+status: %v", err) - } - if total != 1 { - t.Errorf("expected 1 running upload, got %d", total) - } - if len(tasks) != 1 { - t.Errorf("expected 1 task, got %d", len(tasks)) - } - assertTaskIDs(t, tasks, runningUploadID) - - // Pagination: limit 2, offset 0. - tasks, total, err = repos.Tasks.List(ctx, "", "", "", 2, 0) - if err != nil { - t.Fatalf("List paginated: %v", err) - } - if total != 3 { - t.Errorf("expected total 3, got %d", total) - } - if len(tasks) != 2 { - t.Errorf("expected 2 tasks with limit=2, got %d", len(tasks)) - } - assertTaskIDs(t, tasks, evict.ID, secondUpload.ID) - - // Pagination: limit 2, offset 2 — should return 1. - tasks, total, err = repos.Tasks.List(ctx, "", "", "", 2, 2) - if err != nil { - t.Fatalf("List paginated offset: %v", err) - } - if total != 3 { - t.Errorf("expected total 3, got %d", total) - } - if len(tasks) != 1 { - t.Errorf("expected 1 task at offset 2, got %d", len(tasks)) - } - assertTaskIDs(t, tasks, firstUpload.ID) - - // Pagination: offset beyond the result set returns an empty page with the total unchanged. - tasks, total, err = repos.Tasks.List(ctx, "", "", "", 2, 4) - if err != nil { - t.Fatalf("List paginated beyond total: %v", err) - } - if total != 3 || len(tasks) != 0 { - t.Fatalf("expected 0 tasks with total 3 beyond the last page, got %d/%d", len(tasks), total) - } -} - -func TestTaskRepo_ListFiltersByStage(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - ingressCommit := "ingress_commit" - prepare := "prepare_upload" - for _, task := range []*model.Task{ - { - Type: model.TaskTypeUpload, - Stage: &ingressCommit, - RefType: "object", - RefID: 1, - RefVersionID: "01J000000000000000STAGE01", - IdempotencyKey: "stage-primary-commit", - Status: model.TaskStatusQueued, - ScheduledAt: time.Now(), - }, - { - Type: model.TaskTypeUpload, - Stage: &prepare, - RefType: "object", - RefID: 2, - RefVersionID: "01J000000000000000STAGE02", - IdempotencyKey: "stage-prepare", - Status: model.TaskStatusQueued, - ScheduledAt: time.Now(), - }, - { - Type: model.TaskTypeEvictCache, - RefType: "object", - RefID: 3, - RefVersionID: "01J000000000000000STAGE03", - IdempotencyKey: "stage-evict", - Status: model.TaskStatusQueued, - ScheduledAt: time.Now(), - }, - } { - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create task %q: %v", task.IdempotencyKey, err) + b.once.Do(func() { + close(b.selected) + select { + case <-b.release: + case <-ctx.Done(): } - } - - tasks, total, err := repos.Tasks.List(ctx, string(model.TaskTypeUpload), ingressCommit, string(model.TaskStatusQueued), 10, 0) - if err != nil { - t.Fatalf("List by stage: %v", err) - } - if total != 1 || len(tasks) != 1 || tasks[0].IdempotencyKey != "stage-primary-commit" { - t.Fatalf("stage filtered tasks = total:%d tasks:%#v, want ingress_commit only", total, tasks) - } - - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if err != nil { - t.Fatalf("ClaimReady: %v", err) - } - if claimed == nil { - t.Fatal("ClaimReady returned nil") - } - if claimed.Type != model.TaskTypeUpload { - t.Fatalf("claimed type = %s, want upload", claimed.Type) - } + }) } -func TestTaskRepo_CountByStatus(t *testing.T) { - db := testDB(t) +func TestTaskGCDoesNotDeleteTaskRecoveredAfterSelection(t *testing.T) { + db := testutil.NewTestFileDB(t) + db.SetMaxOpenConns(2) repos := repository.NewRepositories(db) - ctx := context.Background() - - // Empty table — should return empty slice. - counts, err := repos.Tasks.CountByStatus(ctx) - if err != nil { - t.Fatalf("CountByStatus empty: %v", err) - } - if len(counts) != 0 { - t.Fatalf("expected 0 counts, got %d", len(counts)) - } - - // Seed tasks of different types and statuses. - seedTask(t, repos, model.TaskTypeUpload) - seedTask(t, repos, model.TaskTypeUpload) - seedTask(t, repos, model.TaskTypeEvictCache) - - // Claim one upload task to get a running status. - claimed, _ := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 5*time.Minute) - if claimed == nil { - t.Fatal("setup: could not claim task") - } - - counts, err = repos.Tasks.CountByStatus(ctx) - if err != nil { - t.Fatalf("CountByStatus: %v", err) - } - - // Build lookup map for verification. - lookup := make(map[string]int64) - for _, c := range counts { - lookup[c.Type+"/"+c.Status] = c.Count - } - - if lookup[string(model.TaskTypeUpload)+"/"+string(model.TaskStatusQueued)] != 1 { - t.Errorf("expected 1 queued upload, got %d", lookup[string(model.TaskTypeUpload)+"/"+string(model.TaskStatusQueued)]) - } - if lookup[string(model.TaskTypeUpload)+"/"+string(model.TaskStatusRunning)] != 1 { - t.Errorf("expected 1 running upload, got %d", lookup[string(model.TaskTypeUpload)+"/"+string(model.TaskStatusRunning)]) - } - if lookup[string(model.TaskTypeEvictCache)+"/"+string(model.TaskStatusQueued)] != 1 { - t.Errorf("expected 1 queued evict_cache, got %d", lookup[string(model.TaskTypeEvictCache)+"/"+string(model.TaskStatusQueued)]) - } -} - -func TestTaskRepo_OverviewActivePipeline(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - prepare := seedTask(t, repos, model.TaskTypeUpload) - ensure := seedTask(t, repos, model.TaskTypeUpload) - upload := seedTask(t, repos, model.TaskTypeUpload) - commit := seedTask(t, repos, model.TaskTypeUpload) - syncPull := seedTask(t, repos, model.TaskTypeUpload) - syncCommit := seedTask(t, repos, model.TaskTypeUpload) - evict := seedTask(t, repos, model.TaskTypeEvictCache) - cleanup := seedTask(t, repos, model.TaskTypeStorageCleanup) - completed := seedTask(t, repos, model.TaskTypeUpload) - failed := seedTask(t, repos, model.TaskTypeUpload) - exhausted := seedTask(t, repos, model.TaskTypeEvictCache) - - mustExec(t, db, `UPDATE tasks SET stage = NULL, status = ? WHERE id = ?`, model.TaskStatusQueued, prepare.ID) - mustExec(t, db, `UPDATE tasks SET stage = ?, status = ? WHERE id = ?`, "ensure_dataset", model.TaskStatusScheduled, ensure.ID) - mustExec(t, db, `UPDATE tasks SET stage = ?, status = ? WHERE id = ?`, "ingress_store", model.TaskStatusRunning, upload.ID) - mustExec(t, db, `UPDATE tasks SET stage = ?, status = ? WHERE id = ?`, "ingress_commit", model.TaskStatusWaiting, commit.ID) - mustExec(t, db, `UPDATE tasks SET stage = ?, status = ? WHERE id = ?`, "peer_pull", model.TaskStatusQueued, syncPull.ID) - mustExec(t, db, `UPDATE tasks SET stage = ?, status = ? WHERE id = ?`, "peer_commit", model.TaskStatusScheduled, syncCommit.ID) - mustExec(t, db, `UPDATE tasks SET status = ? WHERE id = ?`, model.TaskStatusRunning, evict.ID) - mustExec(t, db, `UPDATE tasks SET status = ? WHERE id = ?`, model.TaskStatusWaiting, cleanup.ID) - mustExec(t, db, `UPDATE tasks SET stage = ?, status = ? WHERE id = ?`, "ingress_store", model.TaskStatusCompleted, completed.ID) - mustExec(t, db, `UPDATE tasks SET stage = ?, status = ? WHERE id = ?`, "ingress_store", model.TaskStatusFailed, failed.ID) - mustExec(t, db, `UPDATE tasks SET status = ? WHERE id = ?`, model.TaskStatusExhausted, exhausted.ID) - - counts, err := repos.Tasks.CountOverviewActivePipeline(ctx) - if err != nil { - t.Fatalf("CountOverviewActivePipeline: %v", err) - } - lookup := make(map[string]int64) - for _, count := range counts { - lookup[count.Pipeline+"/"+count.Status] = count.Count - } - assertPipelineCount := func(pipeline string, status model.TaskStatus, want int64) { - t.Helper() - if got := lookup[pipeline+"/"+string(status)]; got != want { - t.Fatalf("%s/%s = %d, want %d", pipeline, status, got, want) - } - } - assertPipelineCount("prepare", model.TaskStatusQueued, 1) - assertPipelineCount("prepare", model.TaskStatusScheduled, 1) - assertPipelineCount("upload", model.TaskStatusRunning, 1) - assertPipelineCount("commit", model.TaskStatusWaiting, 1) - assertPipelineCount("sync", model.TaskStatusQueued, 1) - assertPipelineCount("sync", model.TaskStatusScheduled, 1) - assertPipelineCount("evict", model.TaskStatusRunning, 1) - assertPipelineCount("cleanup", model.TaskStatusWaiting, 1) - if _, ok := lookup["upload/"+string(model.TaskStatusCompleted)]; ok { - t.Fatal("completed upload should not appear in active pipeline") - } - if _, ok := lookup["upload/"+string(model.TaskStatusFailed)]; ok { - t.Fatal("failed upload should not appear in active pipeline") - } - if _, ok := lookup["evict/"+string(model.TaskStatusExhausted)]; ok { - t.Fatal("exhausted evict task should not appear in active pipeline") - } -} - -func TestTaskRepo_CountActiveObjectTasksByBucket(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucketA := seedBucket(t, db, "task-bucket-a") - bucketB := seedBucket(t, db, "task-bucket-b") - - versionA := newObjectVersion(bucketA.ID, "a.txt", "01J00000000000000000000TA", 1) - objectA, err := repos.Objects.CreateVersionAndSetCurrent(ctx, versionA) - if err != nil { - t.Fatalf("CreateVersionAndSetCurrent objectA: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, versionA.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("objectA uploading: %v", err) - } - acceptTestStorageUploadForVersion(t, repos, bucketA.ID, versionA, "piece-a") - - versionB := newObjectVersion(bucketB.ID, "b.txt", "01J00000000000000000000TB", 1) - objectB, err := repos.Objects.CreateVersionAndSetCurrent(ctx, versionB) - if err != nil { - t.Fatalf("CreateVersionAndSetCurrent objectB: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, versionB.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("objectB uploading: %v", err) - } - acceptTestStorageUploadForVersion(t, repos, bucketB.ID, versionB, "piece-b") - - for _, task := range []*model.Task{ - { - Type: model.TaskTypeEvictCache, - RefType: "object", - RefID: objectA, - RefVersionID: versionA.VersionID, - IdempotencyKey: "count-active-queued", - Status: model.TaskStatusQueued, - }, - { - Type: model.TaskTypeEvictCache, - RefType: "object", - RefID: objectA, - RefVersionID: versionA.VersionID, - IdempotencyKey: "count-active-running", - Status: model.TaskStatusRunning, - }, - { - Type: model.TaskTypeUpload, - RefType: "object", - RefID: objectA, - RefVersionID: versionA.VersionID, - IdempotencyKey: "count-active-completed", - Status: model.TaskStatusCompleted, - }, - { - Type: model.TaskTypeEvictCache, - RefType: "bucket", - RefID: bucketA.ID, - RefVersionID: "", - IdempotencyKey: "count-active-bucket-task", - Status: model.TaskStatusQueued, - }, - { - Type: model.TaskTypeEvictCache, - RefType: "object", - RefID: objectB, - RefVersionID: versionB.VersionID, - IdempotencyKey: "count-active-other-bucket", - Status: model.TaskStatusQueued, - }, - } { - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Tasks.Create(%s): %v", task.IdempotencyKey, err) - } - } - - count, err := repos.Tasks.CountActiveObjectTasksByBucket(ctx, bucketA.ID) - if err != nil { - t.Fatalf("CountActiveObjectTasksByBucket: %v", err) - } - if count != 2 { - t.Fatalf("active object task count = %d, want 2", count) - } -} - -func TestTaskRepo_CountActiveBucketTasksByBucketID(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucketA := seedBucket(t, db, "bucket-task-a") - bucketB := seedBucket(t, db, "bucket-task-b") - - for _, task := range []*model.Task{ - { - Type: model.TaskTypeUpload, - RefType: "bucket", - RefID: bucketA.ID, - RefVersionID: "", - IdempotencyKey: "create-ps-a", - Status: model.TaskStatusQueued, - }, - { - Type: model.TaskTypeEvictCache, - RefType: "bucket", - RefID: bucketA.ID, - RefVersionID: "", - IdempotencyKey: "delete-ps-a", - Status: model.TaskStatusRunning, - }, - { - Type: model.TaskTypeUpload, - RefType: "bucket", - RefID: bucketA.ID, - RefVersionID: "", - IdempotencyKey: "create-ps-a-completed", - Status: model.TaskStatusCompleted, - }, - { - Type: model.TaskTypeUpload, - RefType: "bucket", - RefID: bucketB.ID, - RefVersionID: "", - IdempotencyKey: "create-ps-b", - Status: model.TaskStatusQueued, - }, - } { - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Tasks.Create(%s): %v", task.IdempotencyKey, err) - } - } - - count, err := repos.Tasks.CountActiveBucketTasksByBucketID(ctx, bucketA.ID) - if err != nil { - t.Fatalf("CountActiveBucketTasksByBucketID: %v", err) - } - if count != 2 { - t.Fatalf("active bucket task count = %d, want 2 (queued upload + running evict_cache)", count) - } - - countB, err := repos.Tasks.CountActiveBucketTasksByBucketID(ctx, bucketB.ID) - if err != nil { - t.Fatalf("CountActiveBucketTasksByBucketID bucketB: %v", err) - } - if countB != 1 { - t.Fatalf("active bucket task count for B = %d, want 1", countB) - } -} - -func TestTaskRepo_CompleteByRef(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := seedBucket(t, db, "complete-ref-bucket") - - // Seed a queued bucket-scoped task. - queuedTask := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "bucket", - RefID: bucket.ID, - RefVersionID: "", - IdempotencyKey: "cbr-queued-" + time.Now().Format(time.RFC3339Nano), - Status: model.TaskStatusQueued, - } - if err := repos.Tasks.Create(ctx, queuedTask); err != nil { - t.Fatalf("seed queued task: %v", err) - } - - // Happy path: complete the queued task by ref. - if err := repos.Tasks.CompleteByRef(ctx, "bucket", bucket.ID, model.TaskTypeUpload); err != nil { - t.Fatalf("CompleteByRef (happy): %v", err) - } - - // Verify the task is now completed. - got, err := repos.Tasks.GetByID(ctx, queuedTask.ID) - if err != nil { - t.Fatalf("GetByID after complete: %v", err) - } - if got.Status != model.TaskStatusCompleted { - t.Fatalf("status = %s, want completed", got.Status) - } - if got.CompletedAt.IsZero() { - t.Fatal("CompletedAt should be set after CompleteByRef") - } - - // Idempotency: calling again on already-completed task returns ErrNotFound - // because no active rows match. - err = repos.Tasks.CompleteByRef(ctx, "bucket", bucket.ID, model.TaskTypeUpload) - if err == nil { - t.Fatal("CompleteByRef on completed task should return error") - } - if !errors.Is(err, repository.ErrNotFound) { - t.Fatalf("CompleteByRef on completed = %v, want ErrNotFound", err) - } - - // Zero-match: non-existent ref returns ErrNotFound. - err = repos.Tasks.CompleteByRef(ctx, "bucket", 999999, model.TaskTypeUpload) - if err == nil { - t.Fatal("CompleteByRef on non-existent ref should return error") - } - if !errors.Is(err, repository.ErrNotFound) { - t.Fatalf("CompleteByRef non-existent = %v, want ErrNotFound", err) - } -} - -func TestTaskRepo_CompleteByRefClearsWaitingFields(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := seedBucket(t, db, "complete-ref-waiting-bucket") - waitReason := model.TaskWaitReasonDependency - statusMessage := "waiting for dependency" - lastError := "previous error" - task := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "bucket", - RefID: bucket.ID, - RefVersionID: "", - IdempotencyKey: "cbr-waiting-" + time.Now().Format(time.RFC3339Nano), - Status: model.TaskStatusWaiting, - WaitReason: &waitReason, - StatusMessage: &statusMessage, - LastError: &lastError, - ScheduledAt: time.Now().Add(time.Minute), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("seed waiting task: %v", err) - } - - if err := repos.Tasks.CompleteByRef(ctx, "bucket", bucket.ID, model.TaskTypeUpload); err != nil { - t.Fatalf("CompleteByRef: %v", err) - } - - got, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.Status != model.TaskStatusCompleted { - t.Fatalf("status = %s, want completed", got.Status) - } - if got.WaitReason != nil || got.StatusMessage != nil || got.LastError != nil { - t.Fatalf("completed task diagnostics = wait:%v message:%v error:%v, want cleared", got.WaitReason, got.StatusMessage, got.LastError) - } -} - -func TestTaskRepo_RecurringTaskReusesOneRowAcrossRepairItems(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - stage := "repair_replica" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "bucket", - RefID: 1, - RefVersionID: "01J000000000000000REPAIR01", - IdempotencyKey: "upload:repair-data-set:41", - Payload: map[string]interface{}{"storage_data_set_id": int64(41), "storage_upload_copy_id": int64(101)}, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - created, err := repos.Tasks.EnsureRecurring(ctx, task) - if err != nil || !created || task.ID == 0 { - t.Fatalf("EnsureRecurring create: created=%t task=%#v err=%v", created, task, err) - } - duplicate := *task - duplicate.ID = 0 - duplicate.Payload = map[string]interface{}{"storage_data_set_id": int64(41), "storage_upload_copy_id": int64(102)} - created, err = repos.Tasks.EnsureRecurring(ctx, &duplicate) - if err != nil || created { - t.Fatalf("EnsureRecurring active duplicate: created=%t err=%v", created, err) - } - - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil || claimed == nil || claimed.ID != task.ID { - t.Fatalf("ClaimReady: task=%#v err=%v", claimed, err) - } - nextPayload := map[string]interface{}{"storage_data_set_id": int64(41), "storage_upload_copy_id": int64(102)} - nextVersionID := "01J000000000000000REPAIR02" - if err := repos.Tasks.ContinueRunning(ctx, claimed, nextVersionID, nextPayload); err != nil { - t.Fatalf("ContinueRunning: %v", err) - } - continued, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil || continued.Status != model.TaskStatusQueued || continued.RetryCount != 0 || continued.RefVersionID != nextVersionID || testTaskPayloadInt64(continued.Payload, "storage_upload_copy_id") != 102 { - t.Fatalf("continued task = %#v err=%v", continued, err) - } - - claimed, err = repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil || claimed == nil { - t.Fatalf("ClaimReady continued: task=%#v err=%v", claimed, err) - } - if err := repos.Tasks.Complete(ctx, claimed); err != nil { - t.Fatalf("Complete: %v", err) - } - reactivated := *task - reactivated.ID = 0 - reactivated.Payload = map[string]interface{}{"storage_data_set_id": int64(41), "storage_upload_copy_id": int64(103)} - created, err = repos.Tasks.EnsureRecurring(ctx, &reactivated) + taskRow, created, err := repos.Tasks.Enqueue(t.Context(), &model.Task{ + Type: "gc_race", IdempotencyKey: "recover-after-selection", InputVersion: 1, + Input: []byte(`{}`), InputHash: "test", Status: model.TaskStatusPending, + ResumeMode: model.TaskResumeModeExecute, AvailableAt: time.Now(), + }) if err != nil || !created { - t.Fatalf("EnsureRecurring completed task: created=%t err=%v", created, err) + t.Fatalf("enqueue task = %#v created=%v err=%v", taskRow, created, err) } - got, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil || got.Status != model.TaskStatusQueued || testTaskPayloadInt64(got.Payload, "storage_upload_copy_id") != 103 { - t.Fatalf("reactivated task = %#v err=%v", got, err) - } -} - -func TestTaskRepo_HasEarlierRunningUploadCopyTaskMatchesConcreteCopy(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - stage := "ingress_store" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: 1, - RefVersionID: "01J000000000000000RUNNING1", - IdempotencyKey: "upload:running-copy", - Payload: map[string]interface{}{"upload_id": int64(41), "copy_index": 2}, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create: %v", err) - } - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) + claimed, err := repos.Tasks.ClaimNext(t.Context(), time.Minute) if err != nil || claimed == nil { - t.Fatalf("ClaimReady: task=%#v err=%v", claimed, err) - } - - repairStage := "repair_replica" - repairTask := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &repairStage, - RefType: "bucket", - RefID: 1, - RefVersionID: "01J000000000000000RUNNING1", - IdempotencyKey: "upload:repair-data-set:1", - Payload: map[string]interface{}{"storage_data_set_id": int64(1), "storage_upload_copy_id": int64(1)}, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, repairTask); err != nil { - t.Fatalf("Create repair: %v", err) - } - claimedRepair, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil || claimedRepair == nil || claimedRepair.ID != repairTask.ID { - t.Fatalf("ClaimReady repair: task=%#v err=%v", claimedRepair, err) - } - - earlier, err := repos.Tasks.HasEarlierRunningUploadCopyTask(ctx, claimedRepair, 41, 2) - if err != nil || !earlier { - t.Fatalf("HasEarlierRunningUploadCopyTask matching copy = %t, %v; want true", earlier, err) - } - earlier, err = repos.Tasks.HasEarlierRunningUploadCopyTask(ctx, claimedRepair, 41, 1) - if err != nil || earlier { - t.Fatalf("HasEarlierRunningUploadCopyTask different copy = %t, %v; want false", earlier, err) - } -} - -func testTaskPayloadInt64(payload map[string]interface{}, key string) int64 { - switch value := payload[key].(type) { - case int64: - return value - case float64: - return int64(value) - default: - return 0 - } -} - -func TestTaskRepo_RecurringTaskDoesNotReviveExhaustedWork(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - stage := "repair_replica" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "bucket", RefID: 1, - IdempotencyKey: "upload:repair-data-set:42", Status: model.TaskStatusExhausted, - MaxRetries: 5, RetryCount: 5, ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create exhausted task: %v", err) - } - retry := *task - retry.ID = 0 - retry.Status = model.TaskStatusQueued - created, err := repos.Tasks.EnsureRecurring(ctx, &retry) - if err != nil || created { - t.Fatalf("EnsureRecurring exhausted task: created=%t err=%v", created, err) - } - got, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil || got.Status != model.TaskStatusExhausted || got.RetryCount != 5 { - t.Fatalf("exhausted task = %#v err=%v", got, err) + t.Fatalf("claim task = %#v err=%v", claimed, err) + } + if err := repos.Tasks.Settle(t.Context(), claimed.ID, claimed.ClaimGeneration, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, + }); err != nil { + t.Fatalf("fail task: %v", err) + } + if err := repos.Tasks.AcknowledgeFailed(t.Context(), taskRow.ID, time.Hour); err != nil { + t.Fatalf("acknowledge task: %v", err) + } + if _, err := db.NewRaw(`UPDATE tasks SET retention_until = ? WHERE id = ?`, time.Now().Add(-time.Minute), taskRow.ID).Exec(t.Context()); err != nil { + t.Fatalf("expire task retention: %v", err) + } + + barrier := &taskGCSelectionBarrier{selected: make(chan struct{}), release: make(chan struct{})} + db.AddQueryHook(barrier) + type gcResult struct { + deleted int + err error + } + result := make(chan gcResult, 1) + go func() { + deleted, deleteErr := repos.Tasks.DeleteRetained(context.Background(), time.Now(), 10) + result <- gcResult{deleted: deleted, err: deleteErr} + }() + select { + case <-barrier.selected: + case <-time.After(time.Second): + close(barrier.release) + t.Fatal("task GC did not reach the selection barrier") + } + if err := repos.Tasks.RetryFailed(t.Context(), taskRow.ID); err != nil { + close(barrier.release) + t.Fatalf("recover task during GC: %v", err) + } + close(barrier.release) + out := <-result + if out.err != nil || out.deleted != 0 { + t.Fatalf("task GC result = deleted:%d err:%v", out.deleted, out.err) + } + stored, err := repos.Tasks.GetByID(t.Context(), taskRow.ID) + if err != nil || stored == nil || stored.Status != model.TaskStatusPending || stored.ResumeMode != model.TaskResumeModeRecover { + t.Fatalf("recovered task = %#v err=%v", stored, err) } } diff --git a/internal/db/repository/wallet_operation_repo.go b/internal/db/repository/wallet_operation_repo.go index e923baf..2a759db 100644 --- a/internal/db/repository/wallet_operation_repo.go +++ b/internal/db/repository/wallet_operation_repo.go @@ -72,139 +72,113 @@ func (r *BunWalletOperationRepo) GetByID(ctx context.Context, id int64) (*model. return op, nil } -func (r *BunWalletOperationRepo) ClaimPending(ctx context.Context, leaseDuration time.Duration) (*model.WalletOperation, error) { - now := time.Now() - leaseUntil := now.Add(leaseDuration) - op := new(model.WalletOperation) - err := r.db.NewRaw( - `UPDATE wallet_operations SET status = ?, started_at = ?, lease_until = ?, updated_at = ? - WHERE id = ( - SELECT id FROM wallet_operations - WHERE status = ? - AND NOT EXISTS ( - SELECT 1 FROM wallet_operations in_flight - WHERE in_flight.status IN (?, ?) - ) - ORDER BY created_at ASC, id ASC - LIMIT 1 - ) - AND status = ? - RETURNING *`, - model.WalletOperationStatusRunning, now, leaseUntil, now, - model.WalletOperationStatusPending, - model.WalletOperationStatusRunning, - model.WalletOperationStatusSubmitted, - model.WalletOperationStatusPending, - ).Scan(ctx, op) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - return nil, fmt.Errorf("claiming pending wallet operation: %w", err) +func (r *BunWalletOperationRepo) BindTask(ctx context.Context, id, taskID int64) error { + if id < 1 || taskID < 1 { + return ErrInvalidInput } - return op, nil + res, err := r.db.NewUpdate(). + Model((*model.WalletOperation)(nil)). + Set("task_id = ?", taskID). + Set("updated_at = ?", time.Now()). + Where("id = ? AND status = ?", id, model.WalletOperationStatusPending). + Where("task_id IS NULL OR task_id = ?", taskID). + Exec(ctx) + return requireRows(res, err, "binding wallet operation task") } -func (r *BunWalletOperationRepo) MarkSubmitted(ctx context.Context, id int64, txHash string) error { +func (r *BunWalletOperationRepo) MarkBroadcastAttempted(ctx context.Context, id, taskID int64) error { + now := time.Now() + res, err := r.db.NewUpdate(). + Model((*model.WalletOperation)(nil)). + Set("broadcast_attempted_at = COALESCE(broadcast_attempted_at, ?)", now). + Set("started_at = COALESCE(started_at, ?)", now). + Set("updated_at = ?", now). + Where("id = ? AND task_id = ? AND status = ?", id, taskID, model.WalletOperationStatusPending). + Exec(ctx) + return requireRows(res, err, "marking wallet operation broadcast attempted") +} + +func (r *BunWalletOperationRepo) MarkSubmitted(ctx context.Context, id, taskID int64, txHash string) error { + if txHash == "" { + return ErrInvalidInput + } now := time.Now() res, err := r.db.NewUpdate(). Model((*model.WalletOperation)(nil)). Set("status = ?", model.WalletOperationStatusSubmitted). Set("tx_hash = ?", txHash). - Set("submitted_at = ?", now). - Set("lease_until = NULL"). + Set("submitted_at = COALESCE(submitted_at, ?)", now). Set("updated_at = ?", now). - Where("id = ?", id). - Where("status IN (?, ?)", model.WalletOperationStatusRunning, model.WalletOperationStatusSubmitted). + Where("id = ? AND task_id = ?", id, taskID). + Where("status IN (?, ?)", model.WalletOperationStatusPending, model.WalletOperationStatusSubmitted). + Where("tx_hash IS NULL OR tx_hash = ?", txHash). Exec(ctx) return requireRows(res, err, "marking wallet operation submitted") } -func (r *BunWalletOperationRepo) MarkConfirmed(ctx context.Context, id int64) error { +func (r *BunWalletOperationRepo) MarkConfirmed(ctx context.Context, id, taskID int64, txHash string) error { now := time.Now() res, err := r.db.NewUpdate(). Model((*model.WalletOperation)(nil)). Set("status = ?", model.WalletOperationStatusConfirmed). + Set("tx_hash = ?", txHash). Set("last_error = NULL"). - Set("lease_until = NULL"). + Set("task_id = NULL"). Set("completed_at = ?", now). Set("updated_at = ?", now). - Where("id = ?", id). - Where("status = ?", model.WalletOperationStatusSubmitted). + Where("id = ? AND task_id = ?", id, taskID). + Where("status IN (?, ?)", model.WalletOperationStatusPending, model.WalletOperationStatusSubmitted). + Where("tx_hash IS NULL OR tx_hash = ?", txHash). Exec(ctx) return requireRows(res, err, "marking wallet operation confirmed") } -func (r *BunWalletOperationRepo) MarkConfirmedWithoutTransaction(ctx context.Context, id int64) error { +func (r *BunWalletOperationRepo) MarkConfirmedWithoutTransaction(ctx context.Context, id, taskID int64) error { now := time.Now() res, err := r.db.NewUpdate(). Model((*model.WalletOperation)(nil)). Set("status = ?", model.WalletOperationStatusConfirmed). Set("tx_hash = NULL"). Set("last_error = NULL"). - Set("lease_until = NULL"). + Set("task_id = NULL"). Set("submitted_at = NULL"). Set("completed_at = ?", now). Set("updated_at = ?", now). - Where("id = ?", id). - Where("status = ?", model.WalletOperationStatusRunning). + Where("id = ? AND task_id = ?", id, taskID). + Where("status = ?", model.WalletOperationStatusPending). Exec(ctx) return requireRows(res, err, "marking wallet operation confirmed without transaction") } -func (r *BunWalletOperationRepo) MarkFailed(ctx context.Context, id int64, lastError string) error { +func (r *BunWalletOperationRepo) MarkFailed(ctx context.Context, id, taskID int64, lastError string) error { now := time.Now() res, err := r.db.NewUpdate(). Model((*model.WalletOperation)(nil)). Set("status = ?", model.WalletOperationStatusFailed). Set("last_error = ?", lastError). - Set("lease_until = NULL"). + Set("task_id = NULL"). Set("completed_at = ?", now). Set("updated_at = ?", now). - Where("id = ?", id). - Where("status IN (?, ?, ?)", model.WalletOperationStatusRunning, model.WalletOperationStatusSubmitted, model.WalletOperationStatusPending). + Where("id = ? AND task_id = ?", id, taskID). + Where("status IN (?, ?)", model.WalletOperationStatusSubmitted, model.WalletOperationStatusPending). Exec(ctx) return requireRows(res, err, "marking wallet operation failed") } -func (r *BunWalletOperationRepo) MarkExpiredRunningUnknown(ctx context.Context) ([]model.WalletOperation, error) { +func (r *BunWalletOperationRepo) MarkUnknown(ctx context.Context, id, taskID int64, lastError string) error { now := time.Now() - var ops []model.WalletOperation - err := r.db.NewRaw( - `UPDATE wallet_operations - SET status = ?, last_error = ?, lease_until = NULL, completed_at = ?, updated_at = ? - WHERE status = ? - AND (tx_hash IS NULL OR tx_hash = '') - AND lease_until IS NOT NULL - AND lease_until < ? - RETURNING *`, - model.WalletOperationStatusUnknown, - "operation state is unknown after restart before transaction hash was recorded", - now, - now, - model.WalletOperationStatusRunning, - now, - ).Scan(ctx, &ops) - if err != nil { - return nil, fmt.Errorf("marking expired wallet operations unknown: %w", err) - } - return ops, nil -} - -func (r *BunWalletOperationRepo) ListSubmitted(ctx context.Context, limit int) ([]model.WalletOperation, error) { - limit = normalizeWalletOperationLimit(limit) - var ops []model.WalletOperation - err := r.db.NewSelect(). - Model(&ops). - Where("status = ?", model.WalletOperationStatusSubmitted). - Where("tx_hash IS NOT NULL AND tx_hash <> ''"). - OrderExpr("submitted_at ASC, id ASC"). - Limit(limit). - Scan(ctx) - if err != nil { - return nil, fmt.Errorf("listing submitted wallet operations: %w", err) - } - return ops, nil + res, err := r.db.NewUpdate(). + Model((*model.WalletOperation)(nil)). + Set("status = ?", model.WalletOperationStatusUnknown). + Set("last_error = ?", lastError). + Set("task_id = NULL"). + Set("completed_at = ?", now). + Set("updated_at = ?", now). + Where("id = ? AND task_id = ?", id, taskID). + Where("status = ?", model.WalletOperationStatusPending). + Where("broadcast_attempted_at IS NOT NULL"). + Exec(ctx) + return requireRows(res, err, "marking wallet operation unknown") } func (r *BunWalletOperationRepo) ListRecent(ctx context.Context, limit int) ([]model.WalletOperation, error) { diff --git a/internal/db/repository/wallet_operation_repo_test.go b/internal/db/repository/wallet_operation_repo_test.go index e700166..a4fabbc 100644 --- a/internal/db/repository/wallet_operation_repo_test.go +++ b/internal/db/repository/wallet_operation_repo_test.go @@ -1,812 +1,144 @@ package repository_test import ( - "context" "errors" - "strings" - "sync" "testing" "time" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" - "github.com/uptrace/bun" ) -type walletCreateBarrierHook struct { - participants int - release chan struct{} - mu sync.Mutex - selected int - failedInserts int -} - -func newWalletCreateBarrierHook(participants int) *walletCreateBarrierHook { - return &walletCreateBarrierHook{ - participants: participants, - release: make(chan struct{}), - } -} - -func (h *walletCreateBarrierHook) BeforeQuery(ctx context.Context, _ *bun.QueryEvent) context.Context { - return ctx -} - -func (h *walletCreateBarrierHook) AfterQuery(ctx context.Context, event *bun.QueryEvent) { - if !strings.Contains(event.Query, "wallet_operations") { - return +func TestWalletOperationRepoCreateOrGet(t *testing.T) { + repos := repository.NewRepositories(testDB(t)) + ctx := t.Context() + input := repository.CreateWalletOperationInput{ + Type: model.WalletOperationTypeFund, ClientRequestID: "request-1", Amount: "100", } - if event.Operation() == "INSERT" { - if event.Err != nil { - h.mu.Lock() - h.failedInserts++ - h.mu.Unlock() - } - return - } - if event.Operation() != "SELECT" { - return + first, created, err := repos.WalletOperations.CreateOrGet(ctx, input) + if err != nil || !created { + t.Fatalf("first CreateOrGet operation=%#v created=%t err=%v", first, created, err) } - - h.mu.Lock() - if h.selected >= h.participants { - h.mu.Unlock() - return + second, created, err := repos.WalletOperations.CreateOrGet(ctx, input) + if err != nil || created || second == nil || second.ID != first.ID { + t.Fatalf("second CreateOrGet operation=%#v created=%t err=%v", second, created, err) } - h.selected++ - if h.selected == h.participants { - close(h.release) + input.Amount = "101" + if _, _, err := repos.WalletOperations.CreateOrGet(ctx, input); !errors.Is(err, repository.ErrWalletOperationConflict) { + t.Fatalf("conflicting CreateOrGet error=%v", err) } - release := h.release - h.mu.Unlock() - - select { - case <-release: - case <-ctx.Done(): - } -} - -func (h *walletCreateBarrierHook) failedInsertCount() int { - h.mu.Lock() - defer h.mu.Unlock() - return h.failedInserts } -func TestWalletOperationRepo_CreateOrGetIsIdempotentByTypeAndClientRequestID(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - first, created, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "request-1", - Amount: "1000000000000000000", - }) - if err != nil { - t.Fatalf("CreateOrGet first: %v", err) - } - if !created { - t.Fatal("first CreateOrGet created = false, want true") - } - - second, created, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "request-1", - Amount: "1000000000000000000", - }) - if err != nil { - t.Fatalf("CreateOrGet second: %v", err) - } - if created { - t.Fatal("second CreateOrGet created = true, want false") +func TestWalletOperationRepoValidatesAmounts(t *testing.T) { + repos := repository.NewRepositories(testDB(t)) + ctx := t.Context() + if op, created, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ + Type: model.WalletOperationTypeApprove, ClientRequestID: "approve", Amount: "0", + }); err != nil || !created || op.Amount != "0" { + t.Fatalf("approve operation=%#v created=%t err=%v", op, created, err) } - if second.ID != first.ID { - t.Fatalf("second ID = %d, want %d", second.ID, first.ID) - } -} - -func TestWalletOperationRepo_CreateOrGetValidatesAmount(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - approve, created, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeApprove, - ClientRequestID: "approve-1", - Amount: "0", - }) - if err != nil { - t.Fatalf("CreateOrGet approve: %v", err) - } - if !created || approve.Amount != "0" { - t.Fatalf("approve operation = %#v created=%v, want amount 0 and created", approve, created) - } - - for _, tc := range []struct { - name string - opType model.WalletOperationType - amount string - }{ - {name: "fund empty", opType: model.WalletOperationTypeFund, amount: ""}, - {name: "fund zero", opType: model.WalletOperationTypeFund, amount: "0"}, - {name: "fund leading zero", opType: model.WalletOperationTypeFund, amount: "00"}, - {name: "fund negative", opType: model.WalletOperationTypeFund, amount: "-1"}, - {name: "fund float", opType: model.WalletOperationTypeFund, amount: "1.5"}, - {name: "fund string", opType: model.WalletOperationTypeFund, amount: "abc"}, - {name: "approve positive", opType: model.WalletOperationTypeApprove, amount: "1"}, - {name: "withdraw zero", opType: model.WalletOperationTypeWithdraw, amount: "0"}, - {name: "unknown type", opType: model.WalletOperationType("unknown"), amount: "1"}, + for _, input := range []repository.CreateWalletOperationInput{ + {Type: model.WalletOperationTypeFund, ClientRequestID: "zero", Amount: "0"}, + {Type: model.WalletOperationTypeWithdraw, ClientRequestID: "negative", Amount: "-1"}, + {Type: model.WalletOperationTypeApprove, ClientRequestID: "approve-positive", Amount: "1"}, + {Type: model.WalletOperationType("other"), ClientRequestID: "unknown", Amount: "1"}, } { - t.Run(tc.name, func(t *testing.T) { - _, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: tc.opType, - ClientRequestID: tc.name, - Amount: tc.amount, - }) - if !errors.Is(err, repository.ErrWalletOperationInvalidAmount) { - t.Fatalf("CreateOrGet type=%s amount=%q error = %v, want ErrWalletOperationInvalidAmount", tc.opType, tc.amount, err) - } - }) - } -} - -func TestWalletOperationRepo_CreateOrGetRejectsSameRequestWithDifferentAmount(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - if _, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeWithdraw, - ClientRequestID: "request-1", - Amount: "100", - }); err != nil { - t.Fatalf("CreateOrGet first: %v", err) - } - - if _, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeWithdraw, - ClientRequestID: "request-1", - Amount: "101", - }); err == nil { - t.Fatal("CreateOrGet with different amount error = nil, want conflict") - } -} - -func TestWalletOperationRepo_CreateOrGetConcurrentSameAmount(t *testing.T) { - db := concurrentTestDB(t) - repos := repository.NewRepositories(db) - - const workers = 8 - barrier := newWalletCreateBarrierHook(workers) - db.AddQueryHook(barrier) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - start := make(chan struct{}) - var wg sync.WaitGroup - ops := make([]*model.WalletOperation, workers) - created := make([]bool, workers) - errs := make([]error, workers) - - wg.Add(workers) - for i := range workers { - go func(i int) { - defer wg.Done() - <-start - op, wasCreated, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "concurrent-request", - Amount: "100", - }) - ops[i] = op - created[i] = wasCreated - errs[i] = err - }(i) - } - close(start) - wg.Wait() - - var createdCount int - var id int64 - for i := range workers { - if errs[i] != nil { - t.Fatalf("worker %d: %v", i, errs[i]) - } - if ops[i] == nil { - t.Fatalf("worker %d returned nil operation", i) - } - if created[i] { - createdCount++ - } - if id == 0 { - id = ops[i].ID - } else if ops[i].ID != id { - t.Fatalf("worker %d ID = %d, want %d", i, ops[i].ID, id) - } - } - if createdCount != 1 { - t.Fatalf("created count = %d, want 1", createdCount) - } - if got := barrier.failedInsertCount(); got != workers-1 { - t.Fatalf("failed insert count = %d, want %d unique-conflict retries", got, workers-1) - } - - got, err := repos.WalletOperations.GetByID(ctx, id) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got == nil || got.ID != id || got.Amount != "100" { - t.Fatalf("stored operation = %#v, want id %d amount 100", got, id) - } -} - -func TestWalletOperationRepo_CreateOrGetConcurrentAmountConflict(t *testing.T) { - db := concurrentTestDB(t) - repos := repository.NewRepositories(db) - - const workers = 8 - barrier := newWalletCreateBarrierHook(workers) - db.AddQueryHook(barrier) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - start := make(chan struct{}) - var wg sync.WaitGroup - ops := make([]*model.WalletOperation, workers) - errs := make([]error, workers) - - wg.Add(workers) - for i := range workers { - go func(i int) { - defer wg.Done() - <-start - amount := "100" - if i%2 == 1 { - amount = "200" - } - op, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "concurrent-conflict", - Amount: amount, - }) - ops[i] = op - errs[i] = err - }(i) - } - close(start) - wg.Wait() - - var successID int64 - var successAmount string - var successCount int - var conflictCount int - for i := range workers { - if errors.Is(errs[i], repository.ErrWalletOperationConflict) { - conflictCount++ - continue - } - if errs[i] != nil { - t.Fatalf("worker %d: unexpected error %v", i, errs[i]) - } - if ops[i] == nil { - t.Fatalf("worker %d returned nil operation", i) + if _, _, err := repos.WalletOperations.CreateOrGet(ctx, input); !errors.Is(err, repository.ErrWalletOperationInvalidAmount) { + t.Errorf("CreateOrGet(%#v) error=%v", input, err) } - successCount++ - if successID == 0 { - successID = ops[i].ID - successAmount = ops[i].Amount - } else if ops[i].ID != successID { - t.Fatalf("worker %d ID = %d, want %d", i, ops[i].ID, successID) - } else if ops[i].Amount != successAmount { - t.Fatalf("worker %d amount = %q, want %q", i, ops[i].Amount, successAmount) - } - } - if successCount != workers/2 { - t.Fatalf("success count = %d, want %d", successCount, workers/2) - } - if conflictCount != workers/2 { - t.Fatalf("conflict count = %d, want %d", conflictCount, workers/2) - } - if got := barrier.failedInsertCount(); got != workers-1 { - t.Fatalf("failed insert count = %d, want %d unique-conflict retries", got, workers-1) - } - - got, err := repos.WalletOperations.GetByID(ctx, successID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got == nil || got.Amount != successAmount { - t.Fatalf("stored operation = %#v, want amount %q", got, successAmount) } } -func TestWalletOperationRepo_ClaimPendingAndMarkSubmitted(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - created, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "request-1", - Amount: "100", - }) - if err != nil { - t.Fatalf("CreateOrGet: %v", err) - } - - claimed, err := repos.WalletOperations.ClaimPending(ctx, time.Minute) - if err != nil { - t.Fatalf("ClaimPending: %v", err) - } - if claimed == nil || claimed.ID != created.ID { - t.Fatalf("claimed = %#v, want operation %d", claimed, created.ID) +func TestWalletOperationRepoFencesBroadcastAndConfirmationByTask(t *testing.T) { + repos := repository.NewRepositories(testDB(t)) + ctx := t.Context() + op := createWalletOperation(t, repos, model.WalletOperationTypeFund, "fenced", "100") + taskID := createWalletTask(t, repos, "fenced") + if err := repos.WalletOperations.BindTask(ctx, op.ID, taskID); err != nil { + t.Fatalf("BindTask: %v", err) } - if claimed.Status != model.WalletOperationStatusRunning { - t.Fatalf("claimed status = %q, want running", claimed.Status) + if err := repos.WalletOperations.MarkBroadcastAttempted(ctx, op.ID, taskID); err != nil { + t.Fatalf("MarkBroadcastAttempted: %v", err) } - - if err := repos.WalletOperations.MarkSubmitted(ctx, claimed.ID, "0xabc"); err != nil { + if err := repos.WalletOperations.MarkSubmitted(ctx, op.ID, taskID, "0xabc"); err != nil { t.Fatalf("MarkSubmitted: %v", err) } - got, err := repos.WalletOperations.GetByID(ctx, claimed.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.Status != model.WalletOperationStatusSubmitted { - t.Fatalf("status = %q, want submitted", got.Status) - } - if got.TxHash == nil || *got.TxHash != "0xabc" { - t.Fatalf("tx_hash = %v, want 0xabc", got.TxHash) - } -} - -func TestWalletOperationRepo_MarkConfirmed(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - op, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "fund-confirm-1", - Amount: "100", - }) - if err != nil { - t.Fatalf("CreateOrGet: %v", err) + if err := repos.WalletOperations.MarkConfirmed(ctx, op.ID, taskID+1, "0xabc"); !errors.Is(err, repository.ErrNotFound) { + t.Fatalf("stale MarkConfirmed error=%v", err) } - claimed, err := repos.WalletOperations.ClaimPending(ctx, time.Minute) - if err != nil { - t.Fatalf("ClaimPending: %v", err) - } - if claimed == nil || claimed.ID != op.ID { - t.Fatalf("claimed = %#v, want operation %d", claimed, op.ID) - } - if err := repos.WalletOperations.MarkSubmitted(ctx, claimed.ID, "0xabc"); err != nil { - t.Fatalf("MarkSubmitted: %v", err) - } - if err := repos.WalletOperations.MarkConfirmed(ctx, claimed.ID); err != nil { + if err := repos.WalletOperations.MarkConfirmed(ctx, op.ID, taskID, "0xabc"); err != nil { t.Fatalf("MarkConfirmed: %v", err) } - got, err := repos.WalletOperations.GetByID(ctx, op.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.Status != model.WalletOperationStatusConfirmed { - t.Fatalf("status = %q, want confirmed", got.Status) + if err != nil || got == nil { + t.Fatalf("GetByID operation=%#v err=%v", got, err) } - if got.TxHash == nil || *got.TxHash != "0xabc" { - t.Fatalf("tx_hash = %v, want 0xabc", got.TxHash) - } - if got.SubmittedAt == nil { - t.Fatal("submitted_at = nil, want timestamp") - } - if got.CompletedAt == nil { - t.Fatal("completed_at = nil, want timestamp") + if got.Status != model.WalletOperationStatusConfirmed || got.TxHash == nil || *got.TxHash != "0xabc" || got.TaskID != nil || got.BroadcastAttemptedAt == nil || got.CompletedAt == nil { + t.Fatalf("confirmed operation=%#v", got) } } -func TestWalletOperationRepo_MarkConfirmedWithoutTransaction(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - op, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeApprove, - ClientRequestID: "approve-1", - Amount: "0", - }) - if err != nil { - t.Fatalf("CreateOrGet: %v", err) +func TestWalletOperationRepoUnknownRequiresAttemptEvidence(t *testing.T) { + repos := repository.NewRepositories(testDB(t)) + ctx := t.Context() + op := createWalletOperation(t, repos, model.WalletOperationTypeWithdraw, "unknown", "100") + taskID := createWalletTask(t, repos, "unknown") + if err := repos.WalletOperations.BindTask(ctx, op.ID, taskID); err != nil { + t.Fatalf("BindTask: %v", err) } - claimed, err := repos.WalletOperations.ClaimPending(ctx, time.Minute) - if err != nil { - t.Fatalf("ClaimPending: %v", err) + if err := repos.WalletOperations.MarkUnknown(ctx, op.ID, taskID, "outcome unavailable"); !errors.Is(err, repository.ErrNotFound) { + t.Fatalf("MarkUnknown without attempt error=%v", err) } - if claimed == nil || claimed.ID != op.ID { - t.Fatalf("claimed = %#v, want operation %d", claimed, op.ID) + if err := repos.WalletOperations.MarkBroadcastAttempted(ctx, op.ID, taskID); err != nil { + t.Fatalf("MarkBroadcastAttempted: %v", err) } - if err := repos.WalletOperations.MarkConfirmedWithoutTransaction(ctx, claimed.ID); err != nil { - t.Fatalf("MarkConfirmedWithoutTransaction: %v", err) + if err := repos.WalletOperations.MarkUnknown(ctx, op.ID, taskID, "outcome unavailable"); err != nil { + t.Fatalf("MarkUnknown: %v", err) } - got, err := repos.WalletOperations.GetByID(ctx, op.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.Status != model.WalletOperationStatusConfirmed { - t.Fatalf("status = %q, want confirmed", got.Status) - } - if got.TxHash != nil { - t.Fatalf("tx_hash = %v, want nil", *got.TxHash) - } - if got.SubmittedAt != nil { - t.Fatalf("submitted_at = %v, want nil", got.SubmittedAt) - } - if got.CompletedAt == nil { - t.Fatal("completed_at = nil, want timestamp") + if err != nil || got == nil || got.Status != model.WalletOperationStatusUnknown || got.TaskID != nil || got.CompletedAt == nil { + t.Fatalf("unknown operation=%#v err=%v", got, err) } } -func TestWalletOperationRepo_MarkFailed(t *testing.T) { - claimPending := func(t *testing.T, repos *repository.Repositories, ctx context.Context, op *model.WalletOperation) *model.WalletOperation { - t.Helper() - claimed, err := repos.WalletOperations.ClaimPending(ctx, time.Minute) - if err != nil { - t.Fatalf("ClaimPending: %v", err) - } - if claimed == nil || claimed.ID != op.ID { - t.Fatalf("claimed = %#v, want operation %d", claimed, op.ID) - } - return claimed - } - - for _, tc := range []struct { - name string - prepare func(t *testing.T, repos *repository.Repositories, ctx context.Context, op *model.WalletOperation) - }{ - {name: "pending"}, - { - name: "running", - prepare: func(t *testing.T, repos *repository.Repositories, ctx context.Context, op *model.WalletOperation) { - claimPending(t, repos, ctx, op) - }, - }, - { - name: "submitted", - prepare: func(t *testing.T, repos *repository.Repositories, ctx context.Context, op *model.WalletOperation) { - claimed := claimPending(t, repos, ctx, op) - if err := repos.WalletOperations.MarkSubmitted(ctx, claimed.ID, "0xbad"); err != nil { - t.Fatalf("MarkSubmitted: %v", err) - } - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - op, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "request-failed-" + tc.name, - Amount: "100", - }) - if err != nil { - t.Fatalf("CreateOrGet: %v", err) - } - if tc.prepare != nil { - tc.prepare(t, repos, ctx, op) - } - - lastError := "failed from " + tc.name - if err := repos.WalletOperations.MarkFailed(ctx, op.ID, lastError); err != nil { - t.Fatalf("MarkFailed: %v", err) - } - - got, err := repos.WalletOperations.GetByID(ctx, op.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got == nil { - t.Fatal("GetByID returned nil operation") - } - if got.Status != model.WalletOperationStatusFailed { - t.Fatalf("status = %q, want failed", got.Status) - } - if got.LastError == nil || *got.LastError != lastError { - t.Fatalf("last_error = %v, want %q", got.LastError, lastError) - } - if got.LeaseUntil != nil { - t.Fatalf("lease_until = %v, want nil", got.LeaseUntil) - } - if got.CompletedAt == nil { - t.Fatal("completed_at = nil, want timestamp") - } - }) +func TestWalletOperationRepoConfirmedWithoutTransaction(t *testing.T) { + repos := repository.NewRepositories(testDB(t)) + ctx := t.Context() + op := createWalletOperation(t, repos, model.WalletOperationTypeApprove, "approve-no-tx", "0") + taskID := createWalletTask(t, repos, "approve-no-tx") + if err := repos.WalletOperations.BindTask(ctx, op.ID, taskID); err != nil { + t.Fatalf("BindTask: %v", err) } - - t.Run("confirmed rejected", func(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - op, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeApprove, - ClientRequestID: "approve-not-failed", - Amount: "0", - }) - if err != nil { - t.Fatalf("CreateOrGet: %v", err) - } - claimed := claimPending(t, repos, ctx, op) - if err := repos.WalletOperations.MarkConfirmedWithoutTransaction(ctx, claimed.ID); err != nil { - t.Fatalf("MarkConfirmedWithoutTransaction: %v", err) - } - confirmed, err := repos.WalletOperations.GetByID(ctx, op.ID) - if err != nil { - t.Fatalf("GetByID confirmed: %v", err) - } - if confirmed == nil || confirmed.CompletedAt == nil { - t.Fatalf("confirmed operation = %#v, want completed timestamp", confirmed) - } - - if err := repos.WalletOperations.MarkFailed(ctx, op.ID, "failed after confirm"); !errors.Is(err, repository.ErrNotFound) { - t.Fatalf("MarkFailed error = %v, want ErrNotFound", err) - } - got, err := repos.WalletOperations.GetByID(ctx, op.ID) - if err != nil { - t.Fatalf("GetByID after rejected MarkFailed: %v", err) - } - if got == nil { - t.Fatal("GetByID returned nil operation") - } - if got.Status != model.WalletOperationStatusConfirmed { - t.Fatalf("status = %q, want confirmed", got.Status) - } - if got.LastError != nil { - t.Fatalf("last_error = %v, want nil", got.LastError) - } - if got.CompletedAt == nil || !got.CompletedAt.Equal(*confirmed.CompletedAt) { - t.Fatalf("completed_at = %v, want unchanged %v", got.CompletedAt, confirmed.CompletedAt) - } - }) -} - -func TestWalletOperationRepo_ListSubmitted(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - submitted, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "request-submitted", - Amount: "100", - }) - if err != nil { - t.Fatalf("CreateOrGet submitted: %v", err) - } - pending, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "request-pending", - Amount: "200", - }) - if err != nil { - t.Fatalf("CreateOrGet pending: %v", err) - } - emptyTx, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "request-empty-tx", - Amount: "300", - }) - if err != nil { - t.Fatalf("CreateOrGet empty tx: %v", err) - } - - claimed, err := repos.WalletOperations.ClaimPending(ctx, time.Minute) - if err != nil { - t.Fatalf("ClaimPending: %v", err) - } - if claimed == nil || claimed.ID != submitted.ID { - t.Fatalf("claimed = %#v, want operation %d", claimed, submitted.ID) - } - if err := repos.WalletOperations.MarkSubmitted(ctx, claimed.ID, "0x123"); err != nil { - t.Fatalf("MarkSubmitted: %v", err) - } - - now := time.Now() - if _, err := db.NewUpdate(). - Model(emptyTx). - Set("status = ?", model.WalletOperationStatusSubmitted). - Set("tx_hash = ?", ""). - Set("submitted_at = ?", now). - Set("updated_at = ?", now). - WherePK(). - Exec(ctx); err != nil { - t.Fatalf("seed submitted empty tx: %v", err) - } - - ops, err := repos.WalletOperations.ListSubmitted(ctx, 10) - if err != nil { - t.Fatalf("ListSubmitted: %v", err) - } - if len(ops) != 1 { - t.Fatalf("submitted count = %d, want 1: %#v", len(ops), ops) - } - if ops[0].ID != submitted.ID { - t.Fatalf("submitted ID = %d, want %d", ops[0].ID, submitted.ID) - } - if ops[0].TxHash == nil || *ops[0].TxHash != "0x123" { - t.Fatalf("tx_hash = %v, want 0x123", ops[0].TxHash) + if err := repos.WalletOperations.MarkConfirmedWithoutTransaction(ctx, op.ID, taskID); err != nil { + t.Fatalf("MarkConfirmedWithoutTransaction: %v", err) } - for _, op := range ops { - if op.ID == pending.ID { - t.Fatalf("ListSubmitted returned pending operation %d", pending.ID) - } - if op.ID == emptyTx.ID { - t.Fatalf("ListSubmitted returned submitted operation %d with empty tx hash", emptyTx.ID) - } + got, err := repos.WalletOperations.GetByID(ctx, op.ID) + if err != nil || got == nil || got.Status != model.WalletOperationStatusConfirmed || got.TxHash != nil || got.SubmittedAt != nil || got.TaskID != nil { + t.Fatalf("confirmed operation=%#v err=%v", got, err) } } -func TestWalletOperationRepo_ClaimPendingWaitsForInFlightOperation(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - first, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "request-1", - Amount: "100", +func createWalletOperation(t *testing.T, repos *repository.Repositories, operationType model.WalletOperationType, requestID, amount string) *model.WalletOperation { + t.Helper() + op, _, err := repos.WalletOperations.CreateOrGet(t.Context(), repository.CreateWalletOperationInput{ + Type: operationType, ClientRequestID: requestID, Amount: amount, }) if err != nil { - t.Fatalf("CreateOrGet first: %v", err) - } - second, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "request-2", - Amount: "200", - }) - if err != nil { - t.Fatalf("CreateOrGet second: %v", err) - } - - claimed, err := repos.WalletOperations.ClaimPending(ctx, time.Minute) - if err != nil { - t.Fatalf("ClaimPending first: %v", err) - } - if claimed == nil || claimed.ID != first.ID { - t.Fatalf("claimed = %#v, want first operation %d", claimed, first.ID) - } - blocked, err := repos.WalletOperations.ClaimPending(ctx, time.Minute) - if err != nil { - t.Fatalf("ClaimPending while running: %v", err) - } - if blocked != nil { - t.Fatalf("ClaimPending while running returned operation %d, want nil", blocked.ID) - } - - if err := repos.WalletOperations.MarkSubmitted(ctx, first.ID, "0xabc"); err != nil { - t.Fatalf("MarkSubmitted: %v", err) - } - blocked, err = repos.WalletOperations.ClaimPending(ctx, time.Minute) - if err != nil { - t.Fatalf("ClaimPending while submitted: %v", err) - } - if blocked != nil { - t.Fatalf("ClaimPending while submitted returned operation %d, want nil", blocked.ID) - } - - if err := repos.WalletOperations.MarkConfirmed(ctx, first.ID); err != nil { - t.Fatalf("MarkConfirmed: %v", err) - } - claimed, err = repos.WalletOperations.ClaimPending(ctx, time.Minute) - if err != nil { - t.Fatalf("ClaimPending after confirmed: %v", err) - } - if claimed == nil || claimed.ID != second.ID { - t.Fatalf("claimed after confirmed = %#v, want second operation %d", claimed, second.ID) + t.Fatalf("CreateOrGet: %v", err) } + return op } -func TestWalletOperationRepo_MarkExpiredRunningUnknownLeavesPendingAndSubmittedAlone(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - running, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "running", - Amount: "100", - }) - if err != nil { - t.Fatalf("CreateOrGet running: %v", err) - } - submitted, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "submitted", - Amount: "100", +func createWalletTask(t *testing.T, repos *repository.Repositories, key string) int64 { + t.Helper() + row, created, err := repos.Tasks.Enqueue(t.Context(), &model.Task{ + Type: model.TaskTypeWalletOperation, IdempotencyKey: "wallet:" + key, + InputVersion: 1, Input: []byte(`{"operation_id":1}`), InputHash: key, + Status: model.TaskStatusPending, ResumeMode: model.TaskResumeModeExecute, + AvailableAt: time.Now(), }) - if err != nil { - t.Fatalf("CreateOrGet submitted: %v", err) - } - pending, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "pending", - Amount: "100", - }) - if err != nil { - t.Fatalf("CreateOrGet pending: %v", err) - } - - now := time.Now() - expiredLease := now.Add(-time.Second) - if _, err := db.NewUpdate(). - Model(running). - Set("status = ?", model.WalletOperationStatusRunning). - Set("started_at = ?", now.Add(-2*time.Second)). - Set("lease_until = ?", expiredLease). - Set("updated_at = ?", now). - WherePK(). - Exec(ctx); err != nil { - t.Fatalf("seed running operation: %v", err) - } - if _, err := db.NewUpdate(). - Model(submitted). - Set("status = ?", model.WalletOperationStatusSubmitted). - Set("tx_hash = ?", "0xdef"). - Set("submitted_at = ?", now). - Set("updated_at = ?", now). - WherePK(). - Exec(ctx); err != nil { - t.Fatalf("seed submitted operation: %v", err) - } - - expired, err := repos.WalletOperations.MarkExpiredRunningUnknown(ctx) - if err != nil { - t.Fatalf("MarkExpiredRunningUnknown: %v", err) - } - if len(expired) != 1 { - t.Fatalf("expired count = %d, want 1", len(expired)) - } - if expired[0].ID != running.ID { - t.Fatalf("expired ID = %d, want %d", expired[0].ID, running.ID) - } - - gotRunning, _ := repos.WalletOperations.GetByID(ctx, running.ID) - gotSubmitted, _ := repos.WalletOperations.GetByID(ctx, submitted.ID) - gotPending, _ := repos.WalletOperations.GetByID(ctx, pending.ID) - if gotRunning.Status != model.WalletOperationStatusUnknown { - t.Fatalf("running status = %q, want unknown", gotRunning.Status) - } - if gotSubmitted.Status != model.WalletOperationStatusSubmitted { - t.Fatalf("submitted status = %q, want submitted", gotSubmitted.Status) - } - if gotPending.Status != model.WalletOperationStatusPending { - t.Fatalf("pending status = %q, want pending", gotPending.Status) - } -} - -func TestWalletOperationRepo_ListRecentClampsLimitToMax(t *testing.T) { - db := testDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - for i := 0; i < 101; i++ { - _, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "request-" + time.Unix(int64(i), 0).UTC().Format("20060102150405"), - Amount: "100", - }) - if err != nil { - t.Fatalf("CreateOrGet %d: %v", i, err) - } - } - - ops, err := repos.WalletOperations.ListRecent(ctx, 101) - if err != nil { - t.Fatalf("ListRecent: %v", err) - } - if len(ops) != 100 { - t.Fatalf("operation count = %d, want 100", len(ops)) + if err != nil || !created { + t.Fatalf("enqueue wallet task row=%#v created=%t err=%v", row, created, err) } + return row.ID } diff --git a/internal/model/bucket.go b/internal/model/bucket.go index 994b0e9..9939b8a 100644 --- a/internal/model/bucket.go +++ b/internal/model/bucket.go @@ -1,6 +1,7 @@ package model import ( + "context" "time" "github.com/uptrace/bun" @@ -10,31 +11,108 @@ import ( type BucketStatus string const ( - BucketStatusActive BucketStatus = "active" + BucketStatusProvisioning BucketStatus = "provisioning" + BucketStatusReady BucketStatus = "ready" + + // BucketStatusActive is kept as a source-compatible alias for callers that + // create already-provisioned test fixtures. + BucketStatusActive = BucketStatusReady ) -// IsVisible returns true — all buckets are active and visible. -func (s BucketStatus) IsVisible() bool { return true } +// IsVisible reports whether S3 clients may discover the bucket namespace. +func (s BucketStatus) IsVisible() bool { + return s == BucketStatusProvisioning || s == BucketStatusReady +} -// IsAdminVisible returns true — all buckets are visible to admin. -func (s BucketStatus) IsAdminVisible() bool { return true } +// IsAdminVisible reports whether the bucket belongs in the admin inventory. +func (s BucketStatus) IsAdminVisible() bool { return s.IsVisible() } -// IsWritable returns true — all active buckets accept writes. -func (s BucketStatus) IsWritable() bool { return true } +// IsWritable reports whether provider storage is ready for object writes. +func (s BucketStatus) IsWritable() bool { return s == BucketStatusReady } // Bucket stores S3 bucket metadata. type Bucket struct { bun.BaseModel `bun:"table:buckets"` - ID int64 `bun:",pk,autoincrement"` - Name string `bun:",unique,notnull"` - ACL []byte `bun:",nullzero"` - OwnerAccessKey *string `bun:",nullzero"` - DefaultCopies *int `bun:",nullzero"` - MinimumDurableCopies *int `bun:",nullzero"` - Status BucketStatus `bun:",notnull,default:'active'"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - - Owner *S3Account `bun:"rel:belongs-to,join:owner_access_key=access_key,on_update:cascade,on_delete:restrict"` + ID int64 `bun:",pk,autoincrement,identity"` + Name string `bun:"type:text,unique,notnull"` + ACL []byte `bun:",nullzero"` + OwnerAccessKey *string `bun:"type:text,nullzero"` + // The durability policy is materialised at creation from configuration, so + // every bucket answers "how many replicas" without consulting config. + DefaultCopies int `bun:"type:integer,notnull"` + MinimumDurableCopies int `bun:"type:integer,notnull"` + DurabilityGeneration int64 `bun:",notnull,default:0"` + DurabilityTaskID *int64 `bun:",nullzero"` + Status BucketStatus `bun:"type:text,notnull,default:'provisioning'"` + CreatedAt time.Time `bun:",nullzero,notnull"` + UpdatedAt time.Time `bun:",nullzero,notnull"` + + Owner *S3Account `bun:"rel:belongs-to,join:owner_access_key=access_key,on_update:restrict,on_delete:restrict"` +} + +// BucketReplicaSlotStatus tracks whether a slot still accepts new writes. A +// closed slot is decommissioned rather than deleted: retired data set +// generations still reference it, and this schema never deletes that history. +// +// Nothing writes Decommissioned yet. Closing a slot only makes sense together +// with retiring the paid storage service that sits on it, and that retirement +// path does not exist, so lowering a bucket's replica target is refused instead. +type BucketReplicaSlotStatus string + +const ( + BucketReplicaSlotStatusActive BucketReplicaSlotStatus = "active" + BucketReplicaSlotStatusDecommissioned BucketReplicaSlotStatus = "decommissioned" +) + +// BucketReplicaSlot is one logical replica position in a bucket. Data sets, +// copies, replacements, cleanups and observability rows all reference it, so a +// copy index cannot name a slot the bucket never opened. +type BucketReplicaSlot struct { + bun.BaseModel `bun:"table:bucket_replica_slots"` + + ID int64 `bun:",pk,autoincrement,identity"` + BucketID int64 `bun:",notnull"` + CopyIndex int `bun:"type:integer,notnull"` + Status BucketReplicaSlotStatus `bun:"type:text,notnull,default:'active'"` + CreatedAt time.Time `bun:",nullzero,notnull"` + UpdatedAt time.Time `bun:",nullzero,notnull"` +} + +var _ bun.BeforeAppendModelHook = (*Bucket)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (b *Bucket) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if b.CreatedAt.IsZero() { + b.CreatedAt = now + } + if b.UpdatedAt.IsZero() { + b.UpdatedAt = now + } + return nil +} + +var _ bun.BeforeAppendModelHook = (*BucketReplicaSlot)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (b *BucketReplicaSlot) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if b.CreatedAt.IsZero() { + b.CreatedAt = now + } + if b.UpdatedAt.IsZero() { + b.UpdatedAt = now + } + return nil } diff --git a/internal/model/deletion.go b/internal/model/deletion.go index e8967b3..d120cda 100644 --- a/internal/model/deletion.go +++ b/internal/model/deletion.go @@ -1,21 +1,13 @@ package model import ( + "context" "time" "github.com/strahe/synaps3/internal/types" "github.com/uptrace/bun" ) -type CacheCleanupStatus string - -const ( - CacheCleanupStatusPending CacheCleanupStatus = "pending" - CacheCleanupStatusDeleted CacheCleanupStatus = "deleted" - CacheCleanupStatusSkipped CacheCleanupStatus = "skipped" - CacheCleanupStatusFailed CacheCleanupStatus = "failed" -) - type StorageCleanupCopyStatus string const ( @@ -26,49 +18,62 @@ const ( StorageCleanupCopyStatusUnsupported StorageCleanupCopyStatus = "unsupported" ) -// ObjectDeletion records an accepted permanent deletion of one object version. +// ObjectDeletion is an append-only tombstone for one permanently deleted +// object version. Cache cleanup is not tracked here: residency is keyed by +// content, so the trigger is the content's reference count reaching zero +// rather than the removal of any single version. type ObjectDeletion struct { bun.BaseModel `bun:"table:object_deletions"` - ID int64 `bun:",pk,autoincrement"` - BucketID int64 `bun:",notnull"` - ObjectID int64 `bun:",notnull"` - Key string `bun:",notnull"` - VersionID string `bun:",unique,notnull"` - CacheKey string `bun:",notnull"` - StorageUploadID *int64 `bun:",nullzero"` - Size int64 `bun:",notnull"` - Checksum string `bun:",notnull"` - CacheCleanupStatus CacheCleanupStatus `bun:",notnull,default:'pending'"` - CacheError *string `bun:",nullzero"` - CacheCleanedAt *time.Time `bun:",nullzero"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - DeletedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` + ID int64 `bun:",pk,autoincrement,identity"` + BucketID int64 `bun:",notnull"` + ObjectID int64 `bun:",notnull"` + Key string `bun:"type:text,notnull"` + VersionID string `bun:"type:text,unique,notnull"` + ContentID *int64 `bun:",nullzero"` + Size int64 `bun:",notnull"` + DeletedAt time.Time `bun:",nullzero,notnull"` } // StorageCleanupCopy tracks PDP cleanup for one committed storage copy. type StorageCleanupCopy struct { bun.BaseModel `bun:"table:storage_cleanup_copies"` - ID int64 `bun:",pk,autoincrement"` - TaskID int64 `bun:",notnull"` - UploadID int64 `bun:",notnull"` - CopyIndex int `bun:",notnull"` - ProviderID *types.OnChainID `bun:"type:text"` - StorageDataSetID *int64 `bun:",nullzero"` + ID int64 `bun:",pk,autoincrement,identity"` + ContentID int64 `bun:",notnull"` + BucketID int64 `bun:",notnull"` + CopyIndex int `bun:"type:integer,notnull"` + ProviderID types.OnChainID `bun:"type:text,notnull"` + StorageDataSetID int64 `bun:",notnull"` DataSetID *types.OnChainID `bun:"type:text"` ClientDataSetID *types.OnChainID `bun:"type:text"` - PieceID *types.OnChainID `bun:"type:text"` - PieceCID string `bun:",notnull"` - RetrievalURL *string `bun:",nullzero"` - Status StorageCleanupCopyStatus `bun:",notnull,default:'pending'"` - DeleteTxHash *string `bun:",nullzero"` - LastError *string `bun:",nullzero"` + PieceID types.OnChainID `bun:"type:text,notnull"` + PieceCID string `bun:"type:text,notnull"` + RetrievalURL *string `bun:"type:text,nullzero"` + Status StorageCleanupCopyStatus `bun:"type:text,notnull,default:'pending'"` + DeleteTxHash *string `bun:"type:text,nullzero"` + LastError *string `bun:"type:text,nullzero"` ScheduledAt *time.Time `bun:",nullzero"` RemovedAt *time.Time `bun:",nullzero"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` + CreatedAt time.Time `bun:",nullzero,notnull"` + UpdatedAt time.Time `bun:",nullzero,notnull"` +} + +var _ bun.BeforeAppendModelHook = (*StorageCleanupCopy)(nil) - Task *Task `bun:"rel:belongs-to,join:task_id=id"` +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (s *StorageCleanupCopy) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if s.CreatedAt.IsZero() { + s.CreatedAt = now + } + if s.UpdatedAt.IsZero() { + s.UpdatedAt = now + } + return nil } diff --git a/internal/model/multipart.go b/internal/model/multipart.go index 1097ff6..6b4218e 100644 --- a/internal/model/multipart.go +++ b/internal/model/multipart.go @@ -1,6 +1,7 @@ package model import ( + "context" "time" "github.com/uptrace/bun" @@ -20,15 +21,14 @@ const ( type MultipartUpload struct { bun.BaseModel `bun:"table:multipart_uploads"` - ID int64 `bun:",pk,autoincrement"` + UploadID string `bun:"type:text,pk"` BucketID int64 `bun:",notnull"` - Key string `bun:",notnull"` - UploadID string `bun:",notnull,unique"` - ContentType string `bun:",notnull,default:'application/octet-stream'"` - Metadata map[string]string `bun:"type:jsonb"` - Status MultipartStatus `bun:",notnull,default:'initiated'"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` + Key string `bun:"type:text,notnull"` + ContentType string `bun:"type:text,notnull,default:'application/octet-stream'"` + Metadata map[string]string `bun:"type:jsonb,notnull,default:'{}'"` + Status MultipartStatus `bun:"type:text,notnull,default:'initiated'"` + CreatedAt time.Time `bun:",nullzero,notnull"` + UpdatedAt time.Time `bun:",nullzero,notnull"` Bucket *Bucket `bun:"rel:belongs-to,join:bucket_id=id"` } @@ -37,13 +37,48 @@ type MultipartUpload struct { type MultipartPart struct { bun.BaseModel `bun:"table:multipart_parts"` - ID int64 `bun:",pk,autoincrement"` - UploadID string `bun:",notnull"` - PartNumber int `bun:",notnull"` + ID int64 `bun:",pk,autoincrement,identity"` + UploadID string `bun:"type:text,notnull"` + PartNumber int `bun:"type:integer,notnull"` Size int64 `bun:",notnull"` - ETag string `bun:",notnull"` - Checksum *string `bun:",nullzero"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` + ETag string `bun:"type:text,notnull"` + Checksum *string `bun:"type:text,nullzero"` + CreatedAt time.Time `bun:",nullzero,notnull"` Upload *MultipartUpload `bun:"rel:belongs-to,join:upload_id=upload_id"` } + +var _ bun.BeforeAppendModelHook = (*MultipartUpload)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (m *MultipartUpload) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if m.CreatedAt.IsZero() { + m.CreatedAt = now + } + if m.UpdatedAt.IsZero() { + m.UpdatedAt = now + } + return nil +} + +var _ bun.BeforeAppendModelHook = (*MultipartPart)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (m *MultipartPart) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if m.CreatedAt.IsZero() { + m.CreatedAt = now + } + return nil +} diff --git a/internal/model/object.go b/internal/model/object.go index 2bb4a32..5c22645 100644 --- a/internal/model/object.go +++ b/internal/model/object.go @@ -1,34 +1,48 @@ package model import ( + "context" + "path" + "strconv" "time" "github.com/uptrace/bun" ) -// ObjectState represents the lifecycle state of an object. -// State tracks the storage pipeline lifecycle only. +// ObjectState is a derived view of how far a version's content has travelled +// through the storage pipeline. It is computed from the content's copy rows on +// read and is deliberately not a stored column: persisting it would duplicate +// facts the copies already own, and the two would drift. type ObjectState string const ( - ObjectStateCached ObjectState = "cached" - ObjectStateUploading ObjectState = "uploading" - ObjectStateCommitting ObjectState = "committing" - ObjectStateReplicating ObjectState = "replicating" - ObjectStateStored ObjectState = "stored" - ObjectStateFailed ObjectState = "failed" - ObjectStateCacheEvicted ObjectState = "cache_evicted" + ObjectStateCached ObjectState = "cached" + ObjectStateUploading ObjectState = "uploading" + ObjectStateCommitting ObjectState = "committing" + ObjectStateReplicating ObjectState = "replicating" + ObjectStateStored ObjectState = "stored" + ObjectStateFailed ObjectState = "failed" ) +// ContentCacheKey returns the local cache path for one content payload. Cache +// residency is content-addressed, so versions that share bytes share a file +// and the key is derived rather than stored. +func ContentCacheKey(contentID int64) string { + return path.Join(".contents", strconv.FormatInt(contentID, 10)) +} + // Object stores the stable identity for an S3 object key. type Object struct { bun.BaseModel `bun:"table:objects"` - ID int64 `bun:",pk,autoincrement"` - BucketID int64 `bun:",notnull"` - Key string `bun:",notnull"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` + ID int64 `bun:",pk,autoincrement,identity"` + BucketID int64 `bun:",notnull"` + Key string `bun:"type:text,notnull"` + // CurrentVersionID names the version reads serve. It is the single authority + // for "current"; a version carries no flag of its own. + CurrentVersionID *string `bun:"type:text,nullzero"` + CreatedAt time.Time `bun:",nullzero,notnull"` + UpdatedAt time.Time `bun:",nullzero,notnull"` Bucket *Bucket `bun:"rel:belongs-to,join:bucket_id=id"` } @@ -37,31 +51,115 @@ type Object struct { type ObjectVersion struct { bun.BaseModel `bun:"table:object_versions"` - VersionID string `bun:",pk"` + VersionID string `bun:"type:text,pk"` ObjectID int64 `bun:",notnull"` BucketID int64 `bun:",notnull"` - Key string `bun:",notnull"` + Key string `bun:"type:text,notnull"` + ContentID *int64 `bun:",nullzero"` Size int64 `bun:",notnull"` - ETag string `bun:",notnull"` - Checksum string `bun:",notnull"` - ContentType string `bun:",notnull,default:'application/octet-stream'"` - Metadata map[string]string `bun:"type:jsonb"` - CacheKey string `bun:",notnull"` - MultipartUploadID *string `bun:",nullzero"` - StorageUploadID *int64 `bun:",nullzero"` - PieceCID *string `bun:",scanonly"` - RetrievalURL *string `bun:",scanonly"` - InCache bool `bun:",notnull,default:true"` - InFilecoin bool `bun:",scanonly"` - IsCurrent bool `bun:",notnull,default:false"` + ETag string `bun:"type:text,notnull"` + ContentType string `bun:"type:text,notnull,default:'application/octet-stream'"` + Metadata map[string]string `bun:"type:jsonb,notnull,default:'{}'"` + MultipartUploadID *string `bun:"type:text,nullzero"` IsDeleteMarker bool `bun:",notnull,default:false"` - State ObjectState `bun:",notnull,default:'cached'"` - FailedAtState *ObjectState `bun:",nullzero"` - LastError *string `bun:",nullzero"` - CacheAccessedAt *time.Time `bun:",nullzero"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` + CreatedAt time.Time `bun:",nullzero,notnull"` + UpdatedAt time.Time `bun:",nullzero,notnull"` + + // Everything below is projected by repository reads rather than stored. + // Durability and pipeline position are functions of the copy rows, cache + // residency belongs to object_cache, and "current" is the object's pointer. + IsCurrent bool `bun:",scanonly"` + Checksum string `bun:",scanonly"` + PieceCID *string `bun:"type:text,scanonly"` + RetrievalURL *string `bun:"type:text,scanonly"` + InCache bool `bun:",scanonly"` + CacheAccessedAt *time.Time `bun:",scanonly"` + InFilecoin bool `bun:",scanonly"` + State ObjectState `bun:",scanonly"` Object *Object `bun:"rel:belongs-to,join:object_id=id"` Bucket *Bucket `bun:"rel:belongs-to,join:bucket_id=id"` } + +// CacheKey returns the local cache path backing this version. Residency is +// content-addressed, so a delete marker has no key and versions sharing bytes +// share one. +func (v *ObjectVersion) CacheKey() string { + if v == nil || v.ContentID == nil { + return "" + } + return ContentCacheKey(*v.ContentID) +} + +// ObjectCache records local cache residency for one content payload. Two +// versions of identical bytes share a single entry, so eviction is driven by +// the content's reference count rather than by any one version. +type ObjectCache struct { + bun.BaseModel `bun:"table:object_cache"` + + ContentID int64 `bun:",pk"` + InCache bool `bun:",notnull"` + CacheAccessedAt *time.Time `bun:",nullzero"` + CachePresenceGeneration int64 `bun:",notnull,default:0"` + CacheOperationGeneration int64 `bun:",notnull,default:0"` + CacheActiveTaskID *int64 `bun:",nullzero"` + CreatedAt time.Time `bun:",nullzero,notnull"` + UpdatedAt time.Time `bun:",nullzero,notnull"` +} + +var _ bun.BeforeAppendModelHook = (*Object)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (o *Object) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if o.CreatedAt.IsZero() { + o.CreatedAt = now + } + if o.UpdatedAt.IsZero() { + o.UpdatedAt = now + } + return nil +} + +var _ bun.BeforeAppendModelHook = (*ObjectVersion)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (o *ObjectVersion) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if o.CreatedAt.IsZero() { + o.CreatedAt = now + } + if o.UpdatedAt.IsZero() { + o.UpdatedAt = now + } + return nil +} + +var _ bun.BeforeAppendModelHook = (*ObjectCache)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (o *ObjectCache) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if o.CreatedAt.IsZero() { + o.CreatedAt = now + } + if o.UpdatedAt.IsZero() { + o.UpdatedAt = now + } + return nil +} diff --git a/internal/model/s3_account.go b/internal/model/s3_account.go index 9c8c821..a730645 100644 --- a/internal/model/s3_account.go +++ b/internal/model/s3_account.go @@ -1,6 +1,7 @@ package model import ( + "context" "time" "github.com/uptrace/bun" @@ -11,10 +12,29 @@ import ( type S3Account struct { bun.BaseModel `bun:"table:s3_accounts"` - AccessKey string `bun:",pk"` - SecretKey string `bun:",notnull"` - Role auth.Role `bun:",notnull"` + AccessKey string `bun:"type:text,pk"` + SecretKey string `bun:"type:text,notnull"` + Role auth.Role `bun:"type:text,notnull"` IsRoot bool `bun:",notnull,default:false"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` + CreatedAt time.Time `bun:",nullzero,notnull"` + UpdatedAt time.Time `bun:",nullzero,notnull"` +} + +var _ bun.BeforeAppendModelHook = (*S3Account)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (s *S3Account) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if s.CreatedAt.IsZero() { + s.CreatedAt = now + } + if s.UpdatedAt.IsZero() { + s.UpdatedAt = now + } + return nil } diff --git a/internal/model/storage.go b/internal/model/storage.go index 52a9385..94b6b37 100644 --- a/internal/model/storage.go +++ b/internal/model/storage.go @@ -1,7 +1,7 @@ package model import ( - "encoding/json" + "context" "math/bits" "time" @@ -9,18 +9,6 @@ import ( "github.com/uptrace/bun" ) -type StorageUploadStatus string - -const ( - StorageUploadStatusRunning StorageUploadStatus = "running" - StorageUploadStatusIngressReady StorageUploadStatus = "ingress_ready" - StorageUploadStatusReadable StorageUploadStatus = "readable" - StorageUploadStatusComplete StorageUploadStatus = "complete" - StorageUploadStatusFailed StorageUploadStatus = "failed" - StorageUploadStatusRejected StorageUploadStatus = "rejected" - StorageUploadStatusSuperseded StorageUploadStatus = "superseded" -) - // UploadProgressPercent returns a byte-based integer percent when the total is known. func UploadProgressPercent(uploaded, total int64) *int { if total <= 0 { @@ -41,13 +29,12 @@ func UploadProgressPercent(uploaded, total int64) *int { type StorageDataSetStatus string const ( - StorageDataSetStatusPending StorageDataSetStatus = "pending" - StorageDataSetStatusCreating StorageDataSetStatus = "creating" - StorageDataSetStatusReady StorageDataSetStatus = "ready" - StorageDataSetStatusFailed StorageDataSetStatus = "failed" - StorageDataSetStatusUnavailable StorageDataSetStatus = "unavailable" - StorageDataSetStatusDraining StorageDataSetStatus = "draining" - StorageDataSetStatusRetired StorageDataSetStatus = "retired" + StorageDataSetStatusPending StorageDataSetStatus = "pending" + StorageDataSetStatusCreating StorageDataSetStatus = "creating" + StorageDataSetStatusReady StorageDataSetStatus = "ready" + StorageDataSetStatusFailed StorageDataSetStatus = "failed" + StorageDataSetStatusDraining StorageDataSetStatus = "draining" + StorageDataSetStatusRetired StorageDataSetStatus = "retired" ) type StorageCopyTransferMethod string @@ -57,41 +44,39 @@ const ( StorageCopyTransferMethodPeerPull StorageCopyTransferMethod = "peer_pull" ) -type StorageUploadCopyStatus string +type StorageCopyStatus string const ( - StorageUploadCopyStatusPending StorageUploadCopyStatus = "pending" - StorageUploadCopyStatusPieceReady StorageUploadCopyStatus = "piece_ready" - StorageUploadCopyStatusCommitting StorageUploadCopyStatus = "committing" - StorageUploadCopyStatusCommitted StorageUploadCopyStatus = "committed" - StorageUploadCopyStatusFailed StorageUploadCopyStatus = "failed" + StorageCopyStatusPending StorageCopyStatus = "pending" + StorageCopyStatusPieceReady StorageCopyStatus = "piece_ready" + StorageCopyStatusCommitting StorageCopyStatus = "committing" + StorageCopyStatusCommitted StorageCopyStatus = "committed" + StorageCopyStatusFailed StorageCopyStatus = "failed" ) -// StorageUpload records one SDK upload attempt and its persisted outcome. -type StorageUpload struct { - bun.BaseModel `bun:"table:storage_uploads"` - - ID int64 `bun:",pk,autoincrement"` - BucketID int64 `bun:",notnull"` - SourceTaskID *int64 `bun:",nullzero"` - SourceVersionID string `bun:",nullzero"` - ContentSize int64 `bun:",notnull"` - Checksum string `bun:",notnull"` - Status StorageUploadStatus `bun:",notnull,default:'running'"` - PieceCID *string `bun:",nullzero"` - RequestedCopies int `bun:",notnull"` - IngressBytesTransferred int64 `bun:",notnull,default:0"` - IngressStoreAttempt int `bun:",notnull,default:0"` - ProgressUpdatedAt *time.Time `bun:",nullzero"` - RawResultJSON json.RawMessage `bun:"type:jsonb,nullzero"` - ErrorMessage *string `bun:",nullzero"` - AcceptError *string `bun:",nullzero"` - AcceptedAt *time.Time `bun:",nullzero"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` +// StorageContent is the identity of one bucket-scoped byte payload. Object +// versions point at it, copies place it with providers, and dedup is a lookup +// on (bucket_id, checksum, content_size) rather than a scan. +type StorageContent struct { + bun.BaseModel `bun:"table:storage_contents"` + + ID int64 `bun:",pk,autoincrement,identity"` + BucketID int64 `bun:",notnull"` + Checksum string `bun:"type:text,notnull"` + ContentSize int64 `bun:",notnull"` + PieceCID *string `bun:"type:text,nullzero"` + // RequestedCopies is the durability target frozen when the content is first + // created. Later bucket-policy changes do not rewrite this target, and a + // deduplicated write inherits it. + RequestedCopies int `bun:"type:integer,notnull"` + ErrorMessage *string `bun:"type:text,nullzero"` + AcceptedAt *time.Time `bun:",nullzero"` + CleanupGeneration int64 `bun:",notnull,default:0"` + CleanupTaskID *int64 `bun:",nullzero"` + CreatedAt time.Time `bun:",nullzero,notnull"` + UpdatedAt time.Time `bun:",nullzero,notnull"` Bucket *Bucket `bun:"rel:belongs-to,join:bucket_id=id"` - Task *Task `bun:"rel:belongs-to,join:source_task_id=id"` } // StorageDataSet records the bucket ownership of a provider-scoped data set. @@ -102,73 +87,137 @@ type StorageUpload struct { type StorageDataSet struct { bun.BaseModel `bun:"table:storage_data_sets"` - ID int64 `bun:",pk,autoincrement"` - BucketID int64 `bun:",notnull"` - ProviderID types.OnChainID `bun:"type:text,notnull"` - CopyIndex int `bun:",notnull"` - Generation int `bun:",notnull"` - IsCurrent bool `bun:",notnull"` - DataSetID *types.OnChainID `bun:"type:text"` - ClientDataSetID *types.OnChainID `bun:"type:text"` - Status StorageDataSetStatus `bun:",notnull,default:'pending'"` - CreateTransactionID *string `bun:",nullzero"` - CreateStatusURL *string `bun:",nullzero"` - CreatedByUploadID *int64 `bun:",nullzero"` - LastUsedUploadID *int64 `bun:",nullzero"` - LastError *string `bun:",nullzero"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - - Bucket *Bucket `bun:"rel:belongs-to,join:bucket_id=id"` - CreatedByUpload *StorageUpload `bun:"rel:belongs-to,join:created_by_upload_id=id"` - LastUsedUpload *StorageUpload `bun:"rel:belongs-to,join:last_used_upload_id=id"` + ID int64 `bun:",pk,autoincrement,identity"` + BucketID int64 `bun:",notnull"` + ProviderID types.OnChainID `bun:"type:text,notnull"` + CopyIndex int `bun:"type:integer,notnull"` + Generation int64 `bun:",notnull,default:1"` + IsCurrent bool `bun:",notnull"` + DataSetID *types.OnChainID `bun:"type:text"` + ClientDataSetID *types.OnChainID `bun:"type:text"` + Status StorageDataSetStatus `bun:"type:text,notnull,default:'pending'"` + CreateTransactionID *string `bun:"type:text,nullzero"` + CreateStatusURL *string `bun:"type:text,nullzero"` + CreatedByContentID *int64 `bun:",nullzero"` + LastUsedContentID *int64 `bun:",nullzero"` + LastError *string `bun:"type:text,nullzero"` + EnsureTaskID *int64 `bun:",nullzero"` + RetirementGeneration int64 `bun:",notnull,default:0"` + RetirementTaskID *int64 `bun:",nullzero"` + CreatedAt time.Time `bun:",nullzero,notnull"` + UpdatedAt time.Time `bun:",nullzero,notnull"` + + Bucket *Bucket `bun:"rel:belongs-to,join:bucket_id=id"` + CreatedByContent *StorageContent `bun:"rel:belongs-to,join:created_by_content_id=id"` + LastUsedContent *StorageContent `bun:"rel:belongs-to,join:last_used_content_id=id"` } -// StorageUploadCopy stores one successful copy returned by the SDK. -type StorageUploadCopy struct { - bun.BaseModel `bun:"table:storage_upload_copies"` - - ID int64 `bun:",pk,autoincrement"` - UploadID int64 `bun:",notnull"` - CopyIndex int `bun:",notnull"` - ProviderID *types.OnChainID `bun:"type:text"` - DataSetID *types.OnChainID `bun:"type:text,scanonly"` - PieceID *types.OnChainID `bun:"type:text"` - TransferMethod StorageCopyTransferMethod `bun:",notnull"` - Status StorageUploadCopyStatus `bun:",notnull,default:'pending'"` - RetrievalURL *string `bun:",nullzero"` - IsNewDataSet bool `bun:",notnull,default:false"` - StorageDataSetID *int64 `bun:",nullzero"` - CommitExtraDataHex *string `bun:",nullzero"` - CommitTransactionID *string `bun:",nullzero"` - CommitReadyAt *time.Time `bun:",nullzero"` - CommitAttemptID *string `bun:",nullzero"` - CommitAttemptedAt *time.Time `bun:",nullzero"` - CommitSubmissionJSON *string `bun:",nullzero"` - CommitConfirmedTransactionID *string `bun:",nullzero"` - CommitAttentionCode *string `bun:",nullzero"` - CommitAttentionAt *time.Time `bun:",nullzero"` - LastError *string `bun:",nullzero"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - - Upload *StorageUpload `bun:"rel:belongs-to,join:upload_id=id"` +// StorageCopy places one content payload on one data set generation. Ingress +// progress lives here rather than on the content because it belongs to the +// concrete transfer that produced it. +type StorageCopy struct { + bun.BaseModel `bun:"table:storage_copies"` + + ID int64 `bun:",pk,autoincrement,identity"` + ContentID int64 `bun:",notnull"` + BucketID int64 `bun:",notnull"` + // ContentSize repeats the content size so the ingress bound stays a local + // check; a composite foreign key keeps the repetition from drifting. + ContentSize int64 `bun:",notnull"` + StorageDataSetID int64 `bun:",notnull"` + CopyIndex int `bun:"type:integer,notnull"` + ProviderID types.OnChainID `bun:"type:text,notnull"` + PieceID *types.OnChainID `bun:"type:text"` + TransferMethod StorageCopyTransferMethod `bun:"type:text,notnull"` + Status StorageCopyStatus `bun:"type:text,notnull,default:'pending'"` + RetrievalURL *string `bun:"type:text,nullzero"` + // IsNewDataSet is derived by repository reads from the data set's creator. + IsNewDataSet bool `bun:",scanonly"` + CommitExtraDataHex *string `bun:"type:text,nullzero"` + CommitReadyAt *time.Time `bun:",nullzero"` + // ConfirmedAttemptID and ConfirmedAttemptStatus project the ledger row that + // proves this copy is committed. A composite foreign key requires the named + // attempt to actually be confirmed, so the projection cannot drift. + ConfirmedAttemptID *string `bun:"type:text,nullzero"` + ConfirmedAttemptStatus *string `bun:"type:text,nullzero"` + IngressBytesTransferred int64 `bun:",notnull,default:0"` + IngressStoreAttempt int `bun:"type:integer,notnull,default:0"` + ProgressUpdatedAt *time.Time `bun:",nullzero"` + WorkGeneration int64 `bun:",notnull,default:0"` + ActiveTaskID *int64 `bun:",nullzero"` + LastError *string `bun:"type:text,nullzero"` + CreatedAt time.Time `bun:",nullzero,notnull"` + UpdatedAt time.Time `bun:",nullzero,notnull"` + + DataSetID *types.OnChainID `bun:"type:text,scanonly"` + + // Commit evidence is projected from storage_commit_attempts by repository + // reads. These fields are not columns on the copy table. + CommitAttemptID *string `bun:",scanonly"` + CommitAttemptedAt *time.Time `bun:",scanonly"` + CommitTransactionID *string `bun:",scanonly"` + CommitSubmissionJSON *string `bun:",scanonly"` + CommitConfirmedTransactionID *string `bun:",scanonly"` + CommitAttentionCode *string `bun:",scanonly"` + CommitAttentionAt *time.Time `bun:",scanonly"` + + Content *StorageContent `bun:"rel:belongs-to,join:content_id=id"` StorageSet *StorageDataSet `bun:"rel:belongs-to,join:storage_data_set_id=id"` } -// StorageUploadFailure stores one failed provider attempt returned by the SDK. -type StorageUploadFailure struct { - bun.BaseModel `bun:"table:storage_upload_failures"` - - ID int64 `bun:",pk,autoincrement"` - UploadID int64 `bun:",notnull"` - AttemptIndex int `bun:",notnull"` - ProviderID *types.OnChainID `bun:"type:text"` - TransferMethod string `bun:",notnull"` - Stage *string `bun:",nullzero"` - ErrorMessage *string `bun:",nullzero"` - Explicit bool `bun:",notnull,default:false"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - - Upload *StorageUpload `bun:"rel:belongs-to,join:upload_id=id"` +var _ bun.BeforeAppendModelHook = (*StorageContent)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (s *StorageContent) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if s.CreatedAt.IsZero() { + s.CreatedAt = now + } + if s.UpdatedAt.IsZero() { + s.UpdatedAt = now + } + return nil +} + +var _ bun.BeforeAppendModelHook = (*StorageDataSet)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (s *StorageDataSet) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if s.CreatedAt.IsZero() { + s.CreatedAt = now + } + if s.UpdatedAt.IsZero() { + s.UpdatedAt = now + } + return nil +} + +var _ bun.BeforeAppendModelHook = (*StorageCopy)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (s *StorageCopy) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if s.CreatedAt.IsZero() { + s.CreatedAt = now + } + if s.UpdatedAt.IsZero() { + s.UpdatedAt = now + } + return nil } diff --git a/internal/model/storage_test.go b/internal/model/storage_test.go index 221034b..9eb3715 100644 --- a/internal/model/storage_test.go +++ b/internal/model/storage_test.go @@ -22,7 +22,7 @@ func TestStorageOnChainIDColumnsUseTextInPostgresDDL(t *testing.T) { tests := []struct { name string - model interface{} + model any columns []string }{ { @@ -31,15 +31,10 @@ func TestStorageOnChainIDColumnsUseTextInPostgresDDL(t *testing.T) { columns: []string{"provider_id", "data_set_id", "client_data_set_id"}, }, { - name: "storage_upload_copies", - model: (*StorageUploadCopy)(nil), + name: "storage_copies", + model: (*StorageCopy)(nil), columns: []string{"provider_id", "piece_id"}, }, - { - name: "storage_upload_failures", - model: (*StorageUploadFailure)(nil), - columns: []string{"provider_id"}, - }, } for _, tt := range tests { diff --git a/internal/model/task.go b/internal/model/task.go index ba29fb0..5f7923e 100644 --- a/internal/model/task.go +++ b/internal/model/task.go @@ -1,64 +1,138 @@ package model import ( + "context" + "encoding/json" + "slices" "time" "github.com/uptrace/bun" ) -// TaskType represents the kind of async operation. +// TaskType is the only persistent classification of asynchronous work. type TaskType string const ( - TaskTypeUpload TaskType = "upload" - TaskTypeEvictCache TaskType = "evict_cache" - TaskTypeStorageCleanup TaskType = "storage_cleanup" + TaskTypeBucketProvision TaskType = "bucket_provision" + TaskTypeUploadPlan TaskType = "upload_plan" + TaskTypeStorageDataSetEnsure TaskType = "storage_dataset_ensure" + TaskTypeStorageTransferPlan TaskType = "storage_transfer_plan" + TaskTypeStorageStore TaskType = "storage_store" + TaskTypeStoragePull TaskType = "storage_pull" + TaskTypeStorageCommitCoordinate TaskType = "storage_commit_coordinate" + TaskTypeStorageCommit TaskType = "storage_commit" + TaskTypeProviderReplacementCoordinate TaskType = "provider_replacement_coordinate" + TaskTypeCacheCapacityReconcile TaskType = "cache_capacity_reconcile" + TaskTypeCacheEvict TaskType = "cache_evict" + TaskTypeCacheReconcileDurability TaskType = "cache_reconcile_durability" + TaskTypeStorageCleanup TaskType = "storage_cleanup" + TaskTypeStorageDataSetRetire TaskType = "storage_dataset_retire" + TaskTypeWalletOperation TaskType = "wallet_operation" + TaskTypeObservabilityRefresh TaskType = "observability_refresh" + TaskTypeGC TaskType = "task_gc" ) -// TaskStatus represents the processing state of a task. +// RecurringSystemTaskTypes returns the perpetual maintenance task types. +func RecurringSystemTaskTypes() []TaskType { + return []TaskType{TaskTypeCacheCapacityReconcile, TaskTypeObservabilityRefresh, TaskTypeGC} +} + +// IsRecurringSystem reports whether the task is a perpetual maintenance loop +// rather than a finite user or domain operation. +func (t TaskType) IsRecurringSystem() bool { + return slices.Contains(RecurringSystemTaskTypes(), t) +} + +// TaskStatus is the complete task lifecycle. type TaskStatus string const ( - TaskStatusQueued TaskStatus = "queued" - TaskStatusScheduled TaskStatus = "scheduled" + TaskStatusPending TaskStatus = "pending" TaskStatusRunning TaskStatus = "running" - TaskStatusWaiting TaskStatus = "waiting" TaskStatusCompleted TaskStatus = "completed" TaskStatusFailed TaskStatus = "failed" - TaskStatusExhausted TaskStatus = "exhausted" TaskStatusCancelled TaskStatus = "cancelled" ) -// TaskWaitReason identifies why a task is waiting without treating the wait as -// an error. -type TaskWaitReason string +// TaskResumeMode controls whether a handler may initiate an external effect. +type TaskResumeMode string const ( - TaskWaitReasonDependency TaskWaitReason = "dependency" - TaskWaitReasonExternalConfirmation TaskWaitReason = "external_confirmation" + TaskResumeModeExecute TaskResumeMode = "execute" + TaskResumeModeRecover TaskResumeMode = "recover" ) -// Task represents an async job in the DB-backed queue with lease semantics. +// Task is a reclaimable execution record. Domain tables retain durable safety +// evidence independently of this row. type Task struct { bun.BaseModel `bun:"table:tasks"` - ID int64 `bun:",pk,autoincrement"` - Type TaskType `bun:",notnull"` - Stage *string `bun:",nullzero"` - RefType string `bun:",notnull"` // "object" or "bucket" - RefID int64 `bun:",notnull"` - RefVersionID string `bun:",notnull"` - IdempotencyKey string `bun:",unique,notnull"` - Payload map[string]interface{} `bun:"type:jsonb"` - Status TaskStatus `bun:",notnull,default:'queued'"` - RetryCount int `bun:",notnull,default:0"` - MaxRetries int `bun:",notnull,default:5"` - LastError *string `bun:",nullzero"` - StatusMessage *string `bun:",nullzero"` - WaitReason *TaskWaitReason `bun:",nullzero"` - ScheduledAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - ClaimedAt *time.Time `bun:",nullzero"` - LeaseUntil *time.Time `bun:",nullzero"` - StartedAt *time.Time `bun:",nullzero"` - CompletedAt *time.Time `bun:",nullzero"` + ID int64 `bun:",pk,autoincrement,identity"` + Type TaskType `bun:"type:text,notnull"` + IdempotencyKey string `bun:"type:text,notnull"` + InputVersion int `bun:"type:integer,notnull"` + InputHash string `bun:"type:text,notnull"` + SubjectType *string `bun:"type:text,nullzero"` + SubjectKey *string `bun:"type:text,nullzero"` + + // Input and Checkpoint live in task_payloads and are projected on read, so + // renewing a lease never rewrites the JSON a task carries. + Input json.RawMessage `bun:",scanonly"` + Checkpoint json.RawMessage `bun:",scanonly"` + + Status TaskStatus `bun:"type:text,notnull,default:'pending'"` + ResumeMode TaskResumeMode `bun:"type:text,notnull,default:'execute'"` + AvailableAt time.Time `bun:",nullzero,notnull"` + WaitReason *string `bun:"type:text,nullzero"` + RetryCount int `bun:"type:integer,notnull,default:0"` + RetryLimit *int `bun:"type:integer,nullzero"` + FailureReason *string `bun:"type:text,nullzero"` + LastError *string `bun:"type:text,nullzero"` + StatusMessage *string `bun:"type:text,nullzero"` + + CancellationRequestedAt *time.Time `bun:",nullzero"` + CancellationReason *string `bun:"type:text,nullzero"` + ClaimGeneration int64 `bun:",notnull,default:0"` + ClaimedAt *time.Time `bun:",nullzero"` + LeaseUntil *time.Time `bun:",nullzero"` + StartedAt *time.Time `bun:",nullzero"` + FinishedAt *time.Time `bun:",nullzero"` + AcknowledgedAt *time.Time `bun:",nullzero"` + RetentionUntil *time.Time `bun:",nullzero"` + CreatedAt time.Time `bun:",nullzero,notnull"` + UpdatedAt time.Time `bun:",nullzero,notnull"` +} + +// TaskPayload carries a task's input and checkpoint JSON. It is a separate row +// because tasks.lease_until is indexed and renewed on every heartbeat, which on +// PostgreSQL rewrites the whole row. +type TaskPayload struct { + bun.BaseModel `bun:"table:task_payloads"` + + TaskID int64 `bun:",pk"` + Input json.RawMessage `bun:"input_json,type:jsonb,notnull"` + Checkpoint json.RawMessage `bun:"checkpoint_json,type:jsonb,nullzero"` +} + +func (t *Task) CancellationRequested() bool { + return t != nil && t.CancellationRequestedAt != nil +} + +var _ bun.BeforeAppendModelHook = (*Task)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (t *Task) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if t.CreatedAt.IsZero() { + t.CreatedAt = now + } + if t.UpdatedAt.IsZero() { + t.UpdatedAt = now + } + return nil } diff --git a/internal/model/wallet_operation.go b/internal/model/wallet_operation.go index df3aa77..8e0c9b3 100644 --- a/internal/model/wallet_operation.go +++ b/internal/model/wallet_operation.go @@ -1,6 +1,7 @@ package model import ( + "context" "time" "github.com/uptrace/bun" @@ -18,7 +19,6 @@ type WalletOperationStatus string const ( WalletOperationStatusPending WalletOperationStatus = "pending" - WalletOperationStatusRunning WalletOperationStatus = "running" WalletOperationStatusSubmitted WalletOperationStatus = "submitted" WalletOperationStatusConfirmed WalletOperationStatus = "confirmed" WalletOperationStatusFailed WalletOperationStatus = "failed" @@ -28,17 +28,37 @@ const ( type WalletOperation struct { bun.BaseModel `bun:"table:wallet_operations"` - ID int64 `bun:",pk,autoincrement"` - Type WalletOperationType `bun:",notnull"` - ClientRequestID string `bun:",notnull"` - Amount string `bun:",notnull"` - Status WalletOperationStatus `bun:",notnull,default:'pending'"` - TxHash *string `bun:",nullzero"` - LastError *string `bun:",nullzero"` - LeaseUntil *time.Time `bun:",nullzero"` - StartedAt *time.Time `bun:",nullzero"` - SubmittedAt *time.Time `bun:",nullzero"` - CompletedAt *time.Time `bun:",nullzero"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` + ID int64 `bun:",pk,autoincrement,identity"` + Type WalletOperationType `bun:"type:text,notnull"` + ClientRequestID string `bun:"type:text,notnull"` + Amount string `bun:"type:text,notnull"` + Status WalletOperationStatus `bun:"type:text,notnull,default:'pending'"` + TxHash *string `bun:"type:text,nullzero"` + LastError *string `bun:"type:text,nullzero"` + BroadcastAttemptedAt *time.Time `bun:",nullzero"` + TaskID *int64 `bun:",nullzero"` + StartedAt *time.Time `bun:",nullzero"` + SubmittedAt *time.Time `bun:",nullzero"` + CompletedAt *time.Time `bun:",nullzero"` + CreatedAt time.Time `bun:",nullzero,notnull"` + UpdatedAt time.Time `bun:",nullzero,notnull"` +} + +var _ bun.BeforeAppendModelHook = (*WalletOperation)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (w *WalletOperation) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if w.CreatedAt.IsZero() { + w.CreatedAt = now + } + if w.UpdatedAt.IsZero() { + w.UpdatedAt = now + } + return nil } diff --git a/internal/objectdeletion/cache_cleanup.go b/internal/objectdeletion/cache_cleanup.go index ced2907..290d941 100644 --- a/internal/objectdeletion/cache_cleanup.go +++ b/internal/objectdeletion/cache_cleanup.go @@ -2,54 +2,60 @@ package objectdeletion import ( "context" - "log/slog" "github.com/strahe/synaps3/internal/cache" "github.com/strahe/synaps3/internal/cacheaccess" "github.com/strahe/synaps3/internal/model" ) -type cacheCleanupRecorder interface { - UpdateObjectDeletionCacheCleanup(ctx context.Context, versionID string, status model.CacheCleanupStatus, cacheError string) error +type cacheReleaseRepository interface { + ReleaseContentCacheIfUnreferenced(ctx context.Context, contentID int64, release func() error) (bool, error) } -func RecordCacheCleanup( +type CacheReleaseOutcome string + +const ( + CacheReleaseRetained CacheReleaseOutcome = "retained" + CacheReleaseReleased CacheReleaseOutcome = "released" +) + +// ReleaseContentCache removes the cached bytes of one content payload after the +// last object version referencing it has been permanently deleted. +// +// Cache residency is content-addressed, so several versions can share a single +// file. Deleting one of them must leave the file alone; only the disappearance +// of the final reference releases it. The reference decision is rechecked while +// the content row and deletion gate are both held. +// +// The deletion gate is held on the content key for the same reason: two +// versions of identical bytes contend for one file, not one file each. +func ReleaseContentCache( ctx context.Context, c cache.Cache, gate *cacheaccess.Gate, tracker *cacheaccess.Tracker, - recorder cacheCleanupRecorder, - logger *slog.Logger, + repository cacheReleaseRepository, bucketName string, - versionID string, - cacheKey string, -) model.CacheCleanupStatus { + contentID int64, +) (CacheReleaseOutcome, error) { if gate == nil { - panic("cache cleanup requires a cache access gate") + panic("cache release requires a cache access gate") } if tracker == nil { - panic("cache cleanup requires a cache access tracker") + panic("cache release requires a cache access tracker") } - status := model.CacheCleanupStatusSkipped - cacheErr := "" - var deleteErr error - gate.GuardDeletion(versionID, func() { - if cacheKey != "" { - deleteErr = c.Delete(ctx, bucketName, cacheKey) + cacheKey := model.ContentCacheKey(contentID) + outcome := CacheReleaseRetained + var releaseErr error + gate.GuardDeletion(cacheKey, func() { + var released bool + released, releaseErr = repository.ReleaseContentCacheIfUnreferenced(ctx, contentID, func() error { + return c.Delete(ctx, bucketName, cacheKey) + }) + if releaseErr == nil && released { + tracker.Forget(contentID) + outcome = CacheReleaseReleased } - tracker.Forget(versionID) }) - if cacheKey != "" { - if deleteErr != nil { - status = model.CacheCleanupStatusFailed - cacheErr = deleteErr.Error() - logger.Warn("permanent delete cache cleanup failed", "bucket", bucketName, "versionID", versionID, "cacheKey", cacheKey, "error", deleteErr) - } else { - status = model.CacheCleanupStatusDeleted - } - } - if err := recorder.UpdateObjectDeletionCacheCleanup(ctx, versionID, status, cacheErr); err != nil { - logger.Warn("recording permanent delete cache cleanup failed", "bucket", bucketName, "versionID", versionID, "status", status, "error", err) - } - return status + return outcome, releaseErr } diff --git a/internal/objectdeletion/cache_cleanup_test.go b/internal/objectdeletion/cache_cleanup_test.go index 04d1b5d..1afe3e7 100644 --- a/internal/objectdeletion/cache_cleanup_test.go +++ b/internal/objectdeletion/cache_cleanup_test.go @@ -1,11 +1,9 @@ package objectdeletion_test import ( - "bytes" "context" "errors" "io" - "log/slog" "sync" "testing" "time" @@ -17,133 +15,112 @@ import ( "github.com/strahe/synaps3/internal/testutil" ) -type cleanupRecorder struct { - mu sync.Mutex - status model.CacheCleanupStatus +type noopAccessStore struct{} + +func (*noopAccessStore) RecordContentCacheAccess(context.Context, int64, time.Time) error { + return nil } -func (r *cleanupRecorder) UpdateObjectDeletionCacheCleanup( - _ context.Context, - _ string, - status model.CacheCleanupStatus, - _ string, -) error { - r.mu.Lock() - defer r.mu.Unlock() - r.status = status +func (*noopAccessStore) RecordContentCacheCommit(context.Context, int64, time.Time) error { return nil } -type cleanupAccessStore struct{} +func newReleaseTracker() *cacheaccess.Tracker { + return cacheaccess.NewTracker(0, &noopAccessStore{}) +} -func (*cleanupAccessStore) RecordVersionCacheAccess(context.Context, string, time.Time) error { - return nil +type presenceRecorder struct { + mu sync.Mutex + cleared []int64 + clearErr error + referenced bool } -func (*cleanupAccessStore) RecordVersionCacheCommit(context.Context, string, time.Time) error { - return nil +func (r *presenceRecorder) ClearContentCachePresence(_ context.Context, contentID int64) error { + r.mu.Lock() + defer r.mu.Unlock() + r.cleared = append(r.cleared, contentID) + return r.clearErr } -func TestRecordCacheCleanupWaitsForOpenResponseBody(t *testing.T) { - deleted := make(chan struct{}) - mockCache := &testutil.MockCache{ - DeleteFunc: func(context.Context, string, string) error { - close(deleted) - return nil - }, +func (r *presenceRecorder) ReleaseContentCacheIfUnreferenced( + ctx context.Context, + contentID int64, + release func() error, +) (bool, error) { + if r.referenced { + return false, nil } - gate := cacheaccess.NewGate() - tracker := cacheaccess.NewTracker( - cacheaccess.DefaultPersistenceInterval, - new(cleanupAccessStore), - ) - if err := tracker.RecordAccess(context.Background(), "version-1", nil); err != nil { - t.Fatalf("RecordAccess: %v", err) + if err := release(); err != nil { + return false, err } - opened, err := gate.Open( - "version-1", - func() (io.ReadCloser, *cache.ObjectInfo, error) { - return io.NopCloser(bytes.NewReader([]byte("cached"))), &cache.ObjectInfo{Size: 6}, nil - }, - ) - if err != nil { - t.Fatalf("Open: %v", err) + if err := r.ClearContentCachePresence(ctx, contentID); err != nil { + return false, err } + return true, nil +} - recorder := new(cleanupRecorder) - cleanupDone := make(chan model.CacheCleanupStatus, 1) - go func() { - cleanupDone <- objectdeletion.RecordCacheCleanup( - context.Background(), - mockCache, - gate, - tracker, - recorder, - slog.Default(), - "bucket", - "version-1", - ".versions/version-1", - ) - }() - - select { - case <-deleted: - t.Fatal("permanent deletion removed a cache file while its response body was open") - case <-time.After(20 * time.Millisecond): - } - if tracker.Latest("version-1").IsZero() { - t.Fatal("permanent deletion retired access tracking while the response body was open") - } - if err := opened.Body.Close(); err != nil { - t.Fatalf("Close: %v", err) +func newReleaseCache(deleteErr error) (*testutil.MockCache, *[]string) { + deleted := new([]string) + return &testutil.MockCache{ + DeleteFunc: func(_ context.Context, _, key string) error { + *deleted = append(*deleted, key) + return deleteErr + }, + GetFunc: func(context.Context, string, string) (io.ReadCloser, *cache.ObjectInfo, error) { + return nil, nil, errors.New("unused") + }, + }, deleted +} + +func TestReleaseContentCacheDeletesTheContentKeyAndClearsPresence(t *testing.T) { + mockCache, deleted := newReleaseCache(nil) + recorder := &presenceRecorder{} + gate := cacheaccess.NewGate() + tracker := newReleaseTracker() + + outcome, err := objectdeletion.ReleaseContentCache( + context.Background(), mockCache, gate, tracker, recorder, "bucket", 41, + ) + if err != nil || outcome != objectdeletion.CacheReleaseReleased { + t.Fatalf("ReleaseContentCache = %q, %v, want released", outcome, err) } - select { - case status := <-cleanupDone: - if status != model.CacheCleanupStatusDeleted { - t.Fatalf("cleanup status = %s, want deleted", status) - } - case <-time.After(time.Second): - t.Fatal("permanent deletion did not resume after the response body closed") + if want := model.ContentCacheKey(41); len(*deleted) != 1 || (*deleted)[0] != want { + t.Fatalf("deleted keys = %v, want [%s]", *deleted, want) } - if got := tracker.Latest("version-1"); !got.IsZero() { - t.Fatalf("tracking entry after permanent deletion = %s, want retired", got) + if len(recorder.cleared) != 1 || recorder.cleared[0] != 41 { + t.Fatalf("cleared presence = %v, want [41]", recorder.cleared) } } -func TestRecordCacheCleanupRetiresTrackingWhenCacheDeleteFails(t *testing.T) { - deleteErr := errors.New("cache delete failed") - mockCache := &testutil.MockCache{ - DeleteFunc: func(context.Context, string, string) error { - return deleteErr - }, - } - gate := cacheaccess.NewGate() - tracker := cacheaccess.NewTracker( - cacheaccess.DefaultPersistenceInterval, - new(cleanupAccessStore), +func TestReleaseContentCacheReportsFailureWithoutClearingPresence(t *testing.T) { + mockCache, _ := newReleaseCache(errors.New("disk is busy")) + recorder := &presenceRecorder{} + + outcome, err := objectdeletion.ReleaseContentCache( + context.Background(), mockCache, cacheaccess.NewGate(), newReleaseTracker(), recorder, "bucket", 41, ) - if err := tracker.RecordAccess(context.Background(), "version-failed", nil); err != nil { - t.Fatalf("RecordAccess: %v", err) + if err == nil || outcome != objectdeletion.CacheReleaseRetained { + t.Fatalf("ReleaseContentCache = %q, %v, want retained with an error", outcome, err) } - if tracker.Latest("version-failed").IsZero() { - t.Fatal("tracker did not retain the initial cache access") + // Presence must survive a failed delete, or the next reader would be told + // bytes are gone while the file is still there. + if len(recorder.cleared) != 0 { + t.Fatalf("cleared presence = %v, want none", recorder.cleared) } +} + +func TestReleaseContentCacheRetainsReferencedContent(t *testing.T) { + mockCache, deleted := newReleaseCache(nil) + recorder := &presenceRecorder{referenced: true} - status := objectdeletion.RecordCacheCleanup( - context.Background(), - mockCache, - gate, - tracker, - new(cleanupRecorder), - slog.Default(), - "bucket", - "version-failed", - ".versions/version-failed", + outcome, err := objectdeletion.ReleaseContentCache( + context.Background(), mockCache, cacheaccess.NewGate(), newReleaseTracker(), recorder, "bucket", 41, ) - if status != model.CacheCleanupStatusFailed { - t.Fatalf("cleanup status = %s, want failed", status) + if err != nil || outcome != objectdeletion.CacheReleaseRetained { + t.Fatalf("ReleaseContentCache = %q, %v, want retained", outcome, err) } - if got := tracker.Latest("version-failed"); !got.IsZero() { - t.Fatalf("tracking entry after permanent deletion = %s, want retired", got) + if len(*deleted) != 0 || len(recorder.cleared) != 0 { + t.Fatalf("retained content deleted=%v cleared=%v", *deleted, recorder.cleared) } } diff --git a/internal/objectreader/reader.go b/internal/objectreader/reader.go index 2878470..bf899a9 100644 --- a/internal/objectreader/reader.go +++ b/internal/objectreader/reader.go @@ -143,9 +143,7 @@ func (r *Reader) openVersion(ctx context.Context, bucketName, key, versionID str if !os.IsNotExist(cacheErr) { return nil, fmt.Errorf("%w: %w", ErrCacheRead, cacheErr) } - if version.InCache { - r.markCachePresence(ctx, version.VersionID, false) - } + r.reconcileCacheMiss(ctx, bucketName, version) rc, err := r.downloadVersionFromProvider(ctx, key, version) if errors.Is(err, ErrCacheMiss) { @@ -160,7 +158,7 @@ func (r *Reader) openVersion(ctx context.Context, bucketName, key, versionID str body = r.streamAndRehydrate( ctx, bucketName, - version.CacheKey, + version.CacheKey(), version.VersionID, rc, ) @@ -200,9 +198,7 @@ func (r *Reader) open(ctx context.Context, bucketName, key string, visible Bucke return nil, fmt.Errorf("%w: %w", ErrCacheRead, cacheErr) } cacheMiss = true - if version.InCache { - r.markCachePresence(ctx, version.VersionID, false) - } + r.reconcileCacheMiss(ctx, bucketName, version) rc, err := r.downloadVersionFromProvider(ctx, key, version) if errors.Is(err, ErrCacheMiss) { @@ -229,7 +225,7 @@ func (r *Reader) open(ctx context.Context, bucketName, key string, visible Bucke body = r.streamAndRehydrate( ctx, bucketName, - version.CacheKey, + version.CacheKey(), version.VersionID, rc, ) @@ -252,10 +248,10 @@ func resultFromVersion(version *model.ObjectVersion, body io.ReadCloser, source } func (r *Reader) downloadVersionFromProvider(ctx context.Context, key string, version *model.ObjectVersion) (io.ReadCloser, error) { - if version.StorageUploadID == nil || r.storage == nil { + if version.ContentID == nil || r.storage == nil { return nil, ErrCacheMiss } - copies, err := r.repos.Uploads.ListReadableCommittedCopies(ctx, *version.StorageUploadID) + copies, err := r.repos.Contents.ListReadableCommittedCopies(ctx, *version.ContentID) if err != nil { return nil, err } @@ -309,7 +305,7 @@ func (r *Reader) streamAndRehydrate( skipRehydration bool ) err := r.cacheGate.Commit( - versionID, + cacheKey, func() error { persistedVersion, err := r.repos.Objects.GetVersionByID(ctx, versionID) if err != nil { @@ -317,7 +313,7 @@ func (r *Reader) streamAndRehydrate( } if persistedVersion == nil || persistedVersion.IsDeleteMarker || - persistedVersion.CacheKey != cacheKey { + persistedVersion.CacheKey() != cacheKey { skipRehydration = true return nil } @@ -325,11 +321,13 @@ func (r *Reader) streamAndRehydrate( if err != nil { return err } - persistErr = r.accessTracker.RecordCommit( - ctx, - versionID, - persistedVersion.CacheAccessedAt, - ) + if persistedVersion.ContentID != nil { + persistErr = r.accessTracker.RecordCommit( + ctx, + *persistedVersion.ContentID, + persistedVersion.CacheAccessedAt, + ) + } return nil }, ) @@ -359,13 +357,28 @@ func (r *Reader) streamAndRehydrate( return body } -func (r *Reader) markCachePresence(ctx context.Context, versionID string, inCache bool) { - if r == nil || r.repos == nil || r.repos.Objects == nil || versionID == "" { +func (r *Reader) reconcileCacheMiss(ctx context.Context, bucketName string, version *model.ObjectVersion) { + if r == nil || r.repos == nil || r.repos.Objects == nil || version == nil || !version.InCache || version.ContentID == nil { return } - if err := r.repos.Objects.SetVersionCachePresence(ctx, versionID, inCache); err != nil { - r.logger.Warn("cache location update failed", "versionID", versionID, "inCache", inCache, "error", err) - } + cacheKey := version.CacheKey() + r.cacheGate.GuardDeletion(cacheKey, func() { + body, _, err := r.cache.Get(ctx, bucketName, cacheKey) + switch { + case err == nil && body != nil: + if closeErr := body.Close(); closeErr != nil { + r.logger.Warn("cache miss recheck close failed", "cacheKey", cacheKey, "error", closeErr) + } + case err == nil: + r.logger.Warn("cache miss recheck returned an empty read handle", "cacheKey", cacheKey) + case os.IsNotExist(err): + if persistErr := r.repos.Objects.SetVersionCachePresence(ctx, version.VersionID, false); persistErr != nil { + r.logger.Warn("cache location update failed", "versionID", version.VersionID, "inCache", false, "error", persistErr) + } + default: + r.logger.Warn("cache miss recheck failed", "cacheKey", cacheKey, "error", err) + } + }) } func (r *Reader) openCached( @@ -374,19 +387,23 @@ func (r *Reader) openCached( version *model.ObjectVersion, ) (io.ReadCloser, error) { opened, err := r.cacheGate.Open( - version.VersionID, + version.CacheKey(), func() (io.ReadCloser, *cache.ObjectInfo, error) { - return r.cache.Get(ctx, bucketName, version.CacheKey) + return r.cache.Get(ctx, bucketName, version.CacheKey()) }, ) if err != nil { return nil, err } + // Recency belongs to the bytes, so it is recorded against the content that + // backs this version rather than against the version itself. var persistErr error - if version.InCache { - persistErr = r.accessTracker.RecordAccess(ctx, version.VersionID, version.CacheAccessedAt) - } else { - persistErr = r.accessTracker.RecordCommit(ctx, version.VersionID, version.CacheAccessedAt) + if version.ContentID != nil { + if version.InCache { + persistErr = r.accessTracker.RecordAccess(ctx, *version.ContentID, version.CacheAccessedAt) + } else { + persistErr = r.accessTracker.RecordCommit(ctx, *version.ContentID, version.CacheAccessedAt) + } } if persistErr != nil { r.logger.Warn( diff --git a/internal/objectreader/reader_deletion_test.go b/internal/objectreader/reader_deletion_test.go index d7b4739..06d4a79 100644 --- a/internal/objectreader/reader_deletion_test.go +++ b/internal/objectreader/reader_deletion_test.go @@ -20,6 +20,10 @@ import ( "github.com/strahe/synapse-go/storage" ) +// TestOpenVersionDoesNotRehydrateAfterPermanentDeletion pins the race between a +// provider download and a permanent delete. Cache residency is content +// addressed, so the deletion gate is held on the content key and a reader that +// started before the delete must not write the file back afterwards. func TestOpenVersionDoesNotRehydrateAfterPermanentDeletion(t *testing.T) { downloadStarted := make(chan struct{}) releaseDownload := make(chan struct{}) @@ -49,7 +53,7 @@ func TestOpenVersionDoesNotRehydrateAfterPermanentDeletion(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "reader-permanent-delete-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "reader-permanent-delete-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -61,14 +65,13 @@ func TestOpenVersionDoesNotRehydrateAfterPermanentDeletion(t *testing.T) { ETag: "object-etag", Checksum: "object-checksum", ContentType: "text/plain", - CacheKey: ".versions/01J0000000000000000000OR12", - State: model.ObjectStateUploading, } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, withReaderContent(t, repos, version)); err != nil { t.Fatalf("Objects.CreateVersionAndSetCurrent: %v", err) } acceptReaderVersionUpload( t, + db, repos, version.VersionID, buildTestCID(t), @@ -118,19 +121,20 @@ func TestOpenVersionDoesNotRehydrateAfterPermanentDeletion(t *testing.T) { if err != nil { t.Fatalf("DeleteObjectVersionPermanently: %v", err) } - status := objectdeletion.RecordCacheCleanup( + if deletion.ContentID == nil { + t.Fatalf("deletion = %#v, want content identity", deletion) + } + outcome, releaseErr := objectdeletion.ReleaseContentCache( ctx, mc, gate, tracker, repos.Objects, - slog.Default(), bucket.Name, - version.VersionID, - deletion.CacheKey, + *deletion.ContentID, ) - if status != model.CacheCleanupStatusDeleted { - t.Fatalf("cache cleanup status = %s, want deleted", status) + if releaseErr != nil || outcome != objectdeletion.CacheReleaseReleased { + t.Fatalf("ReleaseContentCache = %q, %v, want released", outcome, releaseErr) } releaseDownloadOnce.Do(func() { close(releaseDownload) @@ -159,7 +163,7 @@ func TestOpenVersionDoesNotRehydrateAfterPermanentDeletion(t *testing.T) { if putCalls != 0 { t.Fatalf("cache Put calls after permanent deletion = %d, want 0", putCalls) } - if got := tracker.Latest(version.VersionID); !got.IsZero() { + if got := tracker.Latest(*deletion.ContentID); !got.IsZero() { t.Fatalf("tracking entry after permanent deletion = %s, want none", got) } } diff --git a/internal/objectreader/reader_test.go b/internal/objectreader/reader_test.go index 6296740..aca6532 100644 --- a/internal/objectreader/reader_test.go +++ b/internal/objectreader/reader_test.go @@ -7,6 +7,7 @@ import ( "io" "log/slog" "os" + "sync/atomic" "testing" "time" @@ -19,6 +20,7 @@ import ( "github.com/strahe/synaps3/internal/synapse" "github.com/strahe/synaps3/internal/testutil" "github.com/strahe/synapse-go/storage" + "github.com/uptrace/bun" ) func newTestReader( @@ -57,7 +59,7 @@ func TestOpenUsesProviderFallbackAndRehydratesCache(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "reader-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "reader-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -70,17 +72,15 @@ func TestOpenUsesProviderFallbackAndRehydratesCache(t *testing.T) { ETag: "object-etag", Checksum: "object-checksum", ContentType: "text/plain", - CacheKey: ".versions/01J0000000000000000000OR01", - State: model.ObjectStateUploading, } - _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version) + _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, withReaderContent(t, repos, version)) if err != nil { t.Fatalf("Objects.CreateVersionAndSetCurrent: %v", err) } - acceptReaderVersionUpload(t, repos, version.VersionID, pieceCID, "https://provider.example/piece") + acceptReaderVersionUpload(t, db, repos, version.VersionID, pieceCID, "https://provider.example/piece") previousAccess := time.Now().Add(-24 * time.Hour) - if err := repos.Objects.RecordVersionCacheAccess(ctx, version.VersionID, previousAccess); err != nil { - t.Fatalf("RecordVersionCacheAccess before rehydrate: %v", err) + if err := repos.Objects.RecordContentCacheAccess(ctx, *version.ContentID, previousAccess); err != nil { + t.Fatalf("RecordContentCacheAccess before rehydrate: %v", err) } storageClient := &testutil.MockStorageClient{ @@ -144,7 +144,7 @@ func TestOpenVersionForCopyUsesProviderWithoutRehydratingSourceCache(t *testing. db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "copy-reader-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "copy-reader-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -157,13 +157,11 @@ func TestOpenVersionForCopyUsesProviderWithoutRehydratingSourceCache(t *testing. ETag: "object-etag", Checksum: "object-checksum", ContentType: "text/plain", - CacheKey: ".versions/01J0000000000000000000OR09", - State: model.ObjectStateUploading, } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, withReaderContent(t, repos, version)); err != nil { t.Fatalf("Objects.CreateVersionAndSetCurrent: %v", err) } - acceptReaderVersionUpload(t, repos, version.VersionID, pieceCID, "https://provider.example/copy") + acceptReaderVersionUpload(t, db, repos, version.VersionID, pieceCID, "https://provider.example/copy") if err := repos.Objects.SetVersionCachePresence(ctx, version.VersionID, false); err != nil { t.Fatalf("SetVersionCachePresence: %v", err) } @@ -218,7 +216,7 @@ func TestOpenReplicatingVersionUsesPrimaryCopyOnly(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "replicating-reader-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "replicating-reader-bucket", Status: model.BucketStatusActive, DefaultCopies: 3, MinimumDurableCopies: 3} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -231,22 +229,18 @@ func TestOpenReplicatingVersionUsesPrimaryCopyOnly(t *testing.T) { ETag: "object-etag", Checksum: "object-checksum", ContentType: "text/plain", - CacheKey: ".versions/01J0000000000000000000OR05", - State: model.ObjectStateCached, } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, withReaderContent(t, repos, version)); err != nil { t.Fatalf("Objects.CreateVersionAndSetCurrent: %v", err) } - uploadID := bindReaderPrimaryCommittedUpload(t, repos, version.VersionID, pieceCID, "https://primary.example/piece") - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: uploadID, + contentID := bindReaderPrimaryCommittedUpload(t, db, repos, version.VersionID, pieceCID, "https://primary.example/piece") + testutil.CommitStorageCopy(t, db, repos, repository.MarkUploadCopyCommittedInput{ + ContentID: contentID, CopyIndex: 1, PieceCID: pieceCID, PieceID: onChainIDPtr(t, "2"), RetrievalURL: "https://secondary.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted secondary: %v", err) - } + }) storageClient := &testutil.MockStorageClient{ DownloadFunc: func(_ context.Context, _ cid.Cid, opts *storage.DownloadOptions) (io.ReadCloser, error) { @@ -284,7 +278,7 @@ func TestOpenCacheHitCoalescesAccessPersistenceWithoutLifecyclePresenceWrite(t * db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "cache-hit-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "cache-hit-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -296,10 +290,8 @@ func TestOpenCacheHitCoalescesAccessPersistenceWithoutLifecyclePresenceWrite(t * ETag: "object-etag", Checksum: "object-checksum", ContentType: "text/plain", - CacheKey: ".versions/01J0000000000000000000OR08", - State: model.ObjectStateCached, } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, withReaderContent(t, repos, version)); err != nil { t.Fatalf("Objects.CreateVersionAndSetCurrent: %v", err) } objects := &countingObjectRepo{ObjectRepository: repos.Objects} @@ -345,21 +337,21 @@ func (r *countingObjectRepo) SetVersionCachePresence(ctx context.Context, versio return r.ObjectRepository.SetVersionCachePresence(ctx, versionID, inCache) } -func (r *countingObjectRepo) RecordVersionCacheAccess(ctx context.Context, versionID string, accessedAt time.Time) error { +func (r *countingObjectRepo) RecordContentCacheAccess(ctx context.Context, contentID int64, accessedAt time.Time) error { r.cacheAccessWrites++ if r.cacheAccessErr != nil { return r.cacheAccessErr } - return r.ObjectRepository.RecordVersionCacheAccess(ctx, versionID, accessedAt) + return r.ObjectRepository.RecordContentCacheAccess(ctx, contentID, accessedAt) } -func (r *countingObjectRepo) RecordVersionCacheCommit(ctx context.Context, versionID string, accessedAt time.Time) error { +func (r *countingObjectRepo) RecordContentCacheCommit(ctx context.Context, contentID int64, accessedAt time.Time) error { r.cacheAccessWrites++ r.cacheCommitWrites++ if r.cacheAccessErr != nil { return r.cacheAccessErr } - return r.ObjectRepository.RecordVersionCacheCommit(ctx, versionID, accessedAt) + return r.ObjectRepository.RecordContentCacheCommit(ctx, contentID, accessedAt) } func TestOpenCacheHitReconcilesStaleAbsentPresence(t *testing.T) { @@ -371,7 +363,7 @@ func TestOpenCacheHitReconcilesStaleAbsentPresence(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "cache-presence-reconcile-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "cache-presence-reconcile-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -383,10 +375,8 @@ func TestOpenCacheHitReconcilesStaleAbsentPresence(t *testing.T) { ETag: "object-etag", Checksum: "object-checksum", ContentType: "text/plain", - CacheKey: ".versions/01J0000000000000000000OR11", - State: model.ObjectStateCached, } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, withReaderContent(t, repos, version)); err != nil { t.Fatalf("Objects.CreateVersionAndSetCurrent: %v", err) } if err := repos.Objects.SetVersionCachePresence(ctx, version.VersionID, false); err != nil { @@ -422,7 +412,7 @@ func TestOpenCacheHitIgnoresCacheAccessPersistenceFailure(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "cache-access-failure-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "cache-access-failure-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -434,10 +424,8 @@ func TestOpenCacheHitIgnoresCacheAccessPersistenceFailure(t *testing.T) { ETag: "object-etag", Checksum: "object-checksum", ContentType: "text/plain", - CacheKey: ".versions/01J0000000000000000000OR10", - State: model.ObjectStateCached, } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, withReaderContent(t, repos, version)); err != nil { t.Fatalf("Objects.CreateVersionAndSetCurrent: %v", err) } objects := &countingObjectRepo{ @@ -473,7 +461,7 @@ func TestOpenCacheMissMarksCacheLocationAbsent(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "cache-miss-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "cache-miss-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -485,10 +473,8 @@ func TestOpenCacheMissMarksCacheLocationAbsent(t *testing.T) { ETag: "object-etag", Checksum: "object-checksum", ContentType: "text/plain", - CacheKey: ".versions/01J0000000000000000000OR06", - State: model.ObjectStateCached, } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, withReaderContent(t, repos, version)); err != nil { t.Fatalf("Objects.CreateVersionAndSetCurrent: %v", err) } @@ -511,6 +497,80 @@ func TestOpenCacheMissMarksCacheLocationAbsent(t *testing.T) { } } +func TestOpenCacheMissDoesNotOverwriteConcurrentCommitPresence(t *testing.T) { + firstGetStarted := make(chan struct{}) + allowFirstGetToReturn := make(chan struct{}) + var present atomic.Bool + var getCalls atomic.Int64 + mc := &testutil.MockCache{ + GetFunc: func(_ context.Context, _, _ string) (io.ReadCloser, *cache.ObjectInfo, error) { + if getCalls.Add(1) == 1 { + close(firstGetStarted) + <-allowFirstGetToReturn + return nil, nil, os.ErrNotExist + } + if present.Load() { + return io.NopCloser(bytes.NewReader([]byte("cached"))), &cache.ObjectInfo{Size: 6}, nil + } + return nil, nil, os.ErrNotExist + }, + } + db := testutil.NewTestDB(t) + repos := repository.NewRepositories(db) + ctx := t.Context() + bucket := &model.Bucket{Name: "cache-miss-commit-race-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) + } + version := &model.ObjectVersion{ + VersionID: "01J0000000000000000000OR12", + BucketID: bucket.ID, + Key: "concurrent-cache.txt", + Size: 6, + ETag: "object-etag", + Checksum: "object-checksum", + ContentType: "text/plain", + } + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, withReaderContent(t, repos, version)); err != nil { + t.Fatalf("Objects.CreateVersionAndSetCurrent: %v", err) + } + + gate := cacheaccess.NewGate() + tracker := cacheaccess.NewTracker(cacheaccess.DefaultPersistenceInterval, repos.Objects) + reader := New(repos, mc, nil, gate, tracker, slog.Default()) + openDone := make(chan error, 1) + go func() { + result, err := reader.Open(ctx, bucket.Name, version.Key, S3Visibility) + if result != nil && result.Body != nil { + _ = result.Body.Close() + } + openDone <- err + }() + + <-firstGetStarted + commitErr := gate.Commit(version.CacheKey(), func() error { + present.Store(true) + return repos.Objects.RecordContentCacheCommit(ctx, *version.ContentID, time.Now()) + }) + close(allowFirstGetToReturn) + if commitErr != nil { + t.Fatalf("commit cache presence: %v", commitErr) + } + if err := <-openDone; !errors.Is(err, ErrNoSuchKey) { + t.Fatalf("Open error = %v, want ErrNoSuchKey after provider fallback", err) + } + if getCalls.Load() != 2 { + t.Fatalf("cache get calls = %d, want initial miss plus guarded recheck", getCalls.Load()) + } + stored, err := repos.Objects.GetVersionByID(ctx, version.VersionID) + if err != nil || stored == nil { + t.Fatalf("version after cache commit race: version=%v err=%v", stored, err) + } + if !stored.InCache { + t.Fatal("concurrent cache commit was overwritten by stale miss") + } +} + func TestOpenRehydrateFailureDoesNotMarkCacheLocationPresent(t *testing.T) { mc := &testutil.MockCache{ GetFunc: func(_ context.Context, _, _ string) (io.ReadCloser, *cache.ObjectInfo, error) { @@ -524,7 +584,7 @@ func TestOpenRehydrateFailureDoesNotMarkCacheLocationPresent(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "rehydrate-fail-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "rehydrate-fail-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -537,13 +597,11 @@ func TestOpenRehydrateFailureDoesNotMarkCacheLocationPresent(t *testing.T) { ETag: "object-etag", Checksum: "object-checksum", ContentType: "text/plain", - CacheKey: ".versions/01J0000000000000000000OR07", - State: model.ObjectStateUploading, } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, withReaderContent(t, repos, version)); err != nil { t.Fatalf("Objects.CreateVersionAndSetCurrent: %v", err) } - acceptReaderVersionUpload(t, repos, version.VersionID, pieceCID, "https://provider.example/piece") + acceptReaderVersionUpload(t, db, repos, version.VersionID, pieceCID, "https://provider.example/piece") if err := repos.Objects.SetVersionCachePresence(ctx, version.VersionID, false); err != nil { t.Fatalf("SetVersionCachePresence: %v", err) } @@ -592,7 +650,7 @@ func TestOpenTreatsCurrentVersionChangeAfterProviderDownloadAsMissing(t *testing db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "deleted-reader-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "deleted-reader-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -605,14 +663,12 @@ func TestOpenTreatsCurrentVersionChangeAfterProviderDownloadAsMissing(t *testing ETag: "object-etag", Checksum: "object-checksum", ContentType: "text/plain", - CacheKey: ".versions/01J0000000000000000000OR02", - State: model.ObjectStateUploading, } - _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version) + _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, withReaderContent(t, repos, version)) if err != nil { t.Fatalf("Objects.CreateVersionAndSetCurrent: %v", err) } - acceptReaderVersionUpload(t, repos, version.VersionID, pieceCID, "https://provider.example/deleted") + acceptReaderVersionUpload(t, db, repos, version.VersionID, pieceCID, "https://provider.example/deleted") storageClient := &testutil.MockStorageClient{ DownloadFunc: func(_ context.Context, _ cid.Cid, _ *storage.DownloadOptions) (io.ReadCloser, error) { @@ -624,10 +680,8 @@ func TestOpenTreatsCurrentVersionChangeAfterProviderDownloadAsMissing(t *testing ETag: "new-etag", Checksum: "new-checksum", ContentType: "text/plain", - CacheKey: ".versions/01J0000000000000000000OR03", - State: model.ObjectStateCached, } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, replacement); err != nil { + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, withReaderContent(t, repos, replacement)); err != nil { t.Fatalf("Objects.CreateVersionAndSetCurrent replacement: %v", err) } return io.NopCloser(bytes.NewReader([]byte("remote"))), nil @@ -661,7 +715,7 @@ func TestOpenVersionDoesNotRestartWhenCurrentVersionChanges(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) ctx := context.Background() - bucket := &model.Bucket{Name: "version-reader-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "version-reader-bucket", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("Buckets.Create: %v", err) } @@ -674,13 +728,11 @@ func TestOpenVersionDoesNotRestartWhenCurrentVersionChanges(t *testing.T) { ETag: "old-etag", Checksum: "old-checksum", ContentType: "text/plain", - CacheKey: ".versions/01J0000000000000000000OR04", - State: model.ObjectStateUploading, } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, oldVersion); err != nil { + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, withReaderContent(t, repos, oldVersion)); err != nil { t.Fatalf("Objects.CreateVersionAndSetCurrent old: %v", err) } - acceptReaderVersionUpload(t, repos, oldVersion.VersionID, pieceCID, "https://provider.example/old") + acceptReaderVersionUpload(t, db, repos, oldVersion.VersionID, pieceCID, "https://provider.example/old") storageClient := &testutil.MockStorageClient{ DownloadFunc: func(_ context.Context, _ cid.Cid, _ *storage.DownloadOptions) (io.ReadCloser, error) { @@ -692,10 +744,8 @@ func TestOpenVersionDoesNotRestartWhenCurrentVersionChanges(t *testing.T) { ETag: "new-etag", Checksum: "new-checksum", ContentType: "text/plain", - CacheKey: ".versions/01J0000000000000000000OR05", - State: model.ObjectStateCached, } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, replacement); err != nil { + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, withReaderContent(t, repos, replacement)); err != nil { t.Fatalf("Objects.CreateVersionAndSetCurrent replacement: %v", err) } return io.NopCloser(bytes.NewReader([]byte("old"))), nil @@ -723,7 +773,51 @@ func TestOpenVersionDoesNotRestartWhenCurrentVersionChanges(t *testing.T) { } } -func acceptReaderVersionUpload(t *testing.T, repos *repository.Repositories, versionID string, pieceCID string, retrievalURL string) { +// withReaderContent gives a seeded version the content identity it now requires, +// freezing the bucket's durability policy the way production ingest does. +func withReaderContent(t *testing.T, repos *repository.Repositories, version *model.ObjectVersion) *model.ObjectVersion { + t.Helper() + if version.IsDeleteMarker || version.ContentID != nil { + return version + } + checksum := version.Checksum + if checksum == "" { + checksum = "checksum-" + version.VersionID + } + requestedCopies := 1 + if bucket, err := repos.Buckets.GetByID(t.Context(), version.BucketID); err != nil { + t.Fatalf("load bucket %d: %v", version.BucketID, err) + } else if bucket != nil { + requestedCopies = bucket.DefaultCopies + } + content, err := repos.Contents.EnsureContent(t.Context(), repository.EnsureContentInput{ + BucketID: version.BucketID, + ContentSize: version.Size, + Checksum: testutil.StorageChecksum(checksum), + RequestedCopies: requestedCopies, + }) + if err != nil { + t.Fatalf("EnsureContent for %s: %v", version.VersionID, err) + } + version.ContentID = &content.ID + return version +} + +// readerContentForVersion returns the content a seeded version points at, so +// upload seeding attaches copies to it instead of minting a second identity. +func readerContentForVersion(t *testing.T, repos *repository.Repositories, version *model.ObjectVersion) *model.StorageContent { + t.Helper() + if version.ContentID == nil { + t.Fatalf("version %s has no content", version.VersionID) + } + content, err := repos.Contents.GetByID(context.Background(), *version.ContentID) + if err != nil || content == nil { + t.Fatalf("get content %d: content=%v err=%v", *version.ContentID, content, err) + } + return content +} + +func acceptReaderVersionUpload(t *testing.T, db *bun.DB, repos *repository.Repositories, versionID string, pieceCID string, retrievalURL string) { t.Helper() ctx := context.Background() version, err := repos.Objects.GetVersionByID(ctx, versionID) @@ -733,29 +827,20 @@ func acceptReaderVersionUpload(t *testing.T, repos *repository.Repositories, ver providerID := onChainID(t, "101") dataSetID := onChainID(t, "1001") pieceID := onChainIDPtr(t, "1") - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: version.BucketID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("start upload attempt: %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: version.BucketID, - ProviderID: providerID, - CopyIndex: 0, - CreatedByUploadID: upload.ID, + upload := readerContentForVersion(t, repos, version) + binding, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: version.BucketID, + ProviderID: providerID, + CopyIndex: 0, + CreatedByContentID: upload.ID, }) if err != nil { t.Fatalf("ensure dataset binding: %v", err) } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, UploadID: upload.ID, DataSetID: dataSetID}); err != nil { + if err := repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, ContentID: upload.ID, DataSetID: dataSetID}); err != nil { t.Fatalf("mark dataset ready: %v", err) } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ + if err := repos.Contents.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, @@ -763,87 +848,64 @@ func acceptReaderVersionUpload(t *testing.T, repos *repository.Repositories, ver }}); err != nil { t.Fatalf("create upload copy: %v", err) } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, + testutil.CommitStorageCopy(t, db, repos, repository.MarkUploadCopyCommittedInput{ + ContentID: upload.ID, CopyIndex: 0, PieceCID: pieceCID, PieceID: pieceID, RetrievalURL: retrievalURL, - }); err != nil { - t.Fatalf("mark copy committed: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: version.BucketID, - ContentSize: version.Size, - Checksum: version.Checksum, + }) + if _, err := repos.Contents.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ + ContentID: upload.ID, + BucketID: version.BucketID, }); err != nil { t.Fatalf("bind readable upload: %v", err) } - if finalized, _, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}); err != nil { + if finalized, _, err := repos.Contents.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{ContentID: upload.ID}); err != nil { t.Fatalf("finalize upload: %v", err) } else if !finalized { t.Fatal("finalize upload = false, want true") } } -func bindReaderPrimaryCommittedUpload(t *testing.T, repos *repository.Repositories, versionID string, pieceCID string, retrievalURL string) int64 { +func bindReaderPrimaryCommittedUpload(t *testing.T, db *bun.DB, repos *repository.Repositories, versionID string, pieceCID string, retrievalURL string) int64 { t.Helper() ctx := context.Background() version, err := repos.Objects.GetVersionByID(ctx, versionID) if err != nil || version == nil { t.Fatalf("get version for primary bind: version=%v err=%v", version, err) } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("mark committing: %v", err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: version.BucketID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("start upload attempt: %v", err) - } - primary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: version.BucketID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID}) + upload := readerContentForVersion(t, repos, version) + primary, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: version.BucketID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByContentID: upload.ID}) if err != nil { t.Fatalf("primary binding: %v", err) } - secondary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: version.BucketID, ProviderID: onChainID(t, "202"), CopyIndex: 1, CreatedByUploadID: upload.ID}) + secondary, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: version.BucketID, ProviderID: onChainID(t, "202"), CopyIndex: 1, CreatedByContentID: upload.ID}) if err != nil { t.Fatalf("secondary binding: %v", err) } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: primary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}); err != nil { + if err := repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: primary.ID, ContentID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}); err != nil { t.Fatalf("primary ready: %v", err) } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: secondary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "2002"), ClientDataSetID: onChainIDPtr(t, "9002")}); err != nil { + if err := repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: secondary.ID, ContentID: upload.ID, DataSetID: onChainID(t, "2002"), ClientDataSetID: onChainIDPtr(t, "9002")}); err != nil { t.Fatalf("secondary ready: %v", err) } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ + if err := repos.Contents.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, {StorageDataSetID: secondary.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202")}, }); err != nil { t.Fatalf("create copy rows: %v", err) } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, + testutil.CommitStorageCopy(t, db, repos, repository.MarkUploadCopyCommittedInput{ + ContentID: upload.ID, CopyIndex: 0, PieceCID: pieceCID, PieceID: onChainIDPtr(t, "1"), RetrievalURL: retrievalURL, - }); err != nil { - t.Fatalf("primary committed: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: version.BucketID, - ContentSize: version.Size, - Checksum: version.Checksum, + }) + if _, err := repos.Contents.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ + ContentID: upload.ID, + BucketID: version.BucketID, }); err != nil { t.Fatalf("bind primary committed: %v", err) } diff --git a/internal/observability/checker.go b/internal/observability/checker.go index c038439..e25c076 100644 --- a/internal/observability/checker.go +++ b/internal/observability/checker.go @@ -4,6 +4,7 @@ import ( "context" "errors" "log/slog" + "slices" "sort" "sync" "time" @@ -497,12 +498,7 @@ func lessDecimalString(a, b string) bool { } func reasonCodeContains(codes []ReasonCode, want ReasonCode) bool { - for _, code := range codes { - if code == want { - return true - } - } - return false + return slices.Contains(codes, want) } func worseStatus(current, next Status) Status { @@ -531,7 +527,7 @@ func localStatusSeverity(status model.StorageDataSetStatus) Status { return StatusAvailable case model.StorageDataSetStatusPending, model.StorageDataSetStatusCreating: return StatusDegraded - case model.StorageDataSetStatusFailed, model.StorageDataSetStatusUnavailable, model.StorageDataSetStatusRetired: + case model.StorageDataSetStatusFailed, model.StorageDataSetStatusRetired: return StatusUnavailable default: return StatusDegraded diff --git a/internal/observability/checker_test.go b/internal/observability/checker_test.go index 1cacfdb..8a637ac 100644 --- a/internal/observability/checker_test.go +++ b/internal/observability/checker_test.go @@ -384,7 +384,7 @@ func TestCheckDataSetsMergesWalletScanFailureWithLocalStatus(t *testing.T) { BucketName: "alpha", ProviderID: onChainID(t, "103"), DataSetID: onChainIDPtr(t, "1003"), - Status: model.StorageDataSetStatusUnavailable, + Status: model.StorageDataSetStatusFailed, }, }) if err != nil { diff --git a/internal/observability/interpretation.go b/internal/observability/interpretation.go index d4e2130..36c8beb 100644 --- a/internal/observability/interpretation.go +++ b/internal/observability/interpretation.go @@ -122,13 +122,13 @@ func dataSetHasActivePieces(state DataSetState) *bool { func CopyHealthFromFacts(facts CopyFacts, dataSetObservation *DataSetObservation, interval time.Duration, now time.Time) Signal { switch facts.Status { - case model.StorageUploadCopyStatusPending, model.StorageUploadCopyStatusPieceReady: + case model.StorageCopyStatusPending, model.StorageCopyStatusPieceReady: return BuildSignal(StatusDegraded, []ReasonCode{ReasonCopyPending}, facts.LastError, nil, interval, now) - case model.StorageUploadCopyStatusCommitting: + case model.StorageCopyStatusCommitting: return BuildSignal(StatusDegraded, []ReasonCode{ReasonCopyCommitting}, facts.LastError, nil, interval, now) - case model.StorageUploadCopyStatusFailed: + case model.StorageCopyStatusFailed: return BuildSignal(StatusUnavailable, []ReasonCode{ReasonCopyFailed}, facts.LastError, nil, interval, now) - case model.StorageUploadCopyStatusCommitted: + case model.StorageCopyStatusCommitted: return committedCopyHealthFromFacts(facts, dataSetObservation, interval, now) default: return BuildSignal(StatusUnknown, []ReasonCode{ReasonCopyObservationMissing}, facts.LastError, nil, interval, now) diff --git a/internal/observability/interpretation_test.go b/internal/observability/interpretation_test.go index 6b3578e..a46424d 100644 --- a/internal/observability/interpretation_test.go +++ b/internal/observability/interpretation_test.go @@ -97,7 +97,7 @@ func TestCopyHealthFromFactsClassifiesLocalCopyState(t *testing.T) { { name: "healthy committed copy", facts: CopyFacts{ - Status: model.StorageUploadCopyStatusCommitted, + Status: model.StorageCopyStatusCommitted, ProviderID: onChainIDPtr(t, "101"), LocalDataSetID: &localDataSetID, ChainDataSetID: onChainIDPtr(t, "1001"), @@ -111,7 +111,7 @@ func TestCopyHealthFromFactsClassifiesLocalCopyState(t *testing.T) { { name: "pending copy", facts: CopyFacts{ - Status: model.StorageUploadCopyStatusPending, + Status: model.StorageCopyStatusPending, }, wantStatus: StatusDegraded, wantReasons: []ReasonCode{ReasonCopyPending}, @@ -119,7 +119,7 @@ func TestCopyHealthFromFactsClassifiesLocalCopyState(t *testing.T) { { name: "committing copy", facts: CopyFacts{ - Status: model.StorageUploadCopyStatusCommitting, + Status: model.StorageCopyStatusCommitting, }, wantStatus: StatusDegraded, wantReasons: []ReasonCode{ReasonCopyCommitting}, @@ -127,7 +127,7 @@ func TestCopyHealthFromFactsClassifiesLocalCopyState(t *testing.T) { { name: "failed copy", facts: CopyFacts{ - Status: model.StorageUploadCopyStatusFailed, + Status: model.StorageCopyStatusFailed, LastError: stringPtr("provider rejected piece"), }, wantStatus: StatusUnavailable, @@ -136,7 +136,7 @@ func TestCopyHealthFromFactsClassifiesLocalCopyState(t *testing.T) { { name: "committed copy missing evidence", facts: CopyFacts{ - Status: model.StorageUploadCopyStatusCommitted, + Status: model.StorageCopyStatusCommitted, LocalDataSetID: &localDataSetID, ChainDataSetID: onChainIDPtr(t, "1001"), }, @@ -147,7 +147,7 @@ func TestCopyHealthFromFactsClassifiesLocalCopyState(t *testing.T) { { name: "committed copy missing observation", facts: CopyFacts{ - Status: model.StorageUploadCopyStatusCommitted, + Status: model.StorageCopyStatusCommitted, ProviderID: onChainIDPtr(t, "101"), LocalDataSetID: &localDataSetID, ChainDataSetID: onChainIDPtr(t, "1001"), @@ -160,7 +160,7 @@ func TestCopyHealthFromFactsClassifiesLocalCopyState(t *testing.T) { { name: "unavailable data set observation", facts: CopyFacts{ - Status: model.StorageUploadCopyStatusCommitted, + Status: model.StorageCopyStatusCommitted, ProviderID: onChainIDPtr(t, "101"), LocalDataSetID: &localDataSetID, ChainDataSetID: onChainIDPtr(t, "1001"), @@ -192,7 +192,7 @@ func TestCopyHealthFromFactsTreatsStaleDataSetAsUnknown(t *testing.T) { observation := copyHealthDataSetObservation(t, localDataSetID, StatusAvailable, nil, checkedAt) observation.Signal = BuildSignal(StatusAvailable, nil, nil, &checkedAt, time.Hour, now) got := CopyHealthFromFacts(CopyFacts{ - Status: model.StorageUploadCopyStatusCommitted, + Status: model.StorageCopyStatusCommitted, ProviderID: onChainIDPtr(t, "101"), LocalDataSetID: &localDataSetID, ChainDataSetID: onChainIDPtr(t, "1001"), diff --git a/internal/observability/model.go b/internal/observability/model.go index 92f3641..78bbb2e 100644 --- a/internal/observability/model.go +++ b/internal/observability/model.go @@ -1,6 +1,7 @@ package observability import ( + "context" "errors" "time" @@ -21,37 +22,26 @@ const ( type ReasonCode string const ( - ReasonRegistryLookupFailed ReasonCode = "registry_lookup_failed" - ReasonProviderInactive ReasonCode = "provider_inactive" - ReasonProviderMissingPDP ReasonCode = "provider_missing_pdp" - ReasonProviderHTTPUnreachable ReasonCode = "provider_http_unreachable" - ReasonChainLookupFailed ReasonCode = "chain_lookup_failed" - ReasonChainDataSetMissing ReasonCode = "chain_data_set_missing" - ReasonChainDataSetInactive ReasonCode = "chain_data_set_inactive" - ReasonChainDataSetUnmanaged ReasonCode = "chain_data_set_unmanaged" - ReasonLocalStatusNotReady ReasonCode = "local_status_not_ready" - ReasonProviderMismatch ReasonCode = "provider_mismatch" - ReasonMetadataMismatch ReasonCode = "metadata_mismatch" - ReasonCopyUnderReplicated ReasonCode = "copy_under_replicated" - ReasonCopyPending ReasonCode = "copy_pending" - ReasonCopyCommitting ReasonCode = "copy_committing" - ReasonCopyFailed ReasonCode = "copy_failed" - ReasonCopyMissingProvider ReasonCode = "copy_missing_provider" - ReasonCopyMissingDataSet ReasonCode = "copy_missing_data_set" - ReasonCopyMissingPiece ReasonCode = "copy_missing_piece" - ReasonCopyMissingRetrievalURL ReasonCode = "copy_missing_retrieval_url" - ReasonCopyObservationMissing ReasonCode = "copy_observation_missing" - ReasonTaskNotApplicable ReasonCode = "task_not_applicable" - ReasonTaskMissingEvidence ReasonCode = "task_missing_evidence" - ReasonTaskInsufficientFunds ReasonCode = "task_insufficient_funds" - ReasonTaskMissingApproval ReasonCode = "task_missing_approval" - ReasonTaskRPCUnavailable ReasonCode = "task_rpc_unavailable" - ReasonTaskChainPending ReasonCode = "task_chain_pending" - ReasonTaskChainConfirmed ReasonCode = "task_chain_confirmed" - ReasonTaskTransactionRejected ReasonCode = "task_transaction_rejected" - ReasonTaskPieceStatusMismatch ReasonCode = "task_piece_status_mismatch" - ReasonTaskDiagnosticUnavailable ReasonCode = "task_diagnostic_unavailable" - ReasonTaskUnknownStatus ReasonCode = "task_unknown_status" + ReasonRegistryLookupFailed ReasonCode = "registry_lookup_failed" + ReasonProviderInactive ReasonCode = "provider_inactive" + ReasonProviderMissingPDP ReasonCode = "provider_missing_pdp" + ReasonProviderHTTPUnreachable ReasonCode = "provider_http_unreachable" + ReasonChainLookupFailed ReasonCode = "chain_lookup_failed" + ReasonChainDataSetMissing ReasonCode = "chain_data_set_missing" + ReasonChainDataSetInactive ReasonCode = "chain_data_set_inactive" + ReasonChainDataSetUnmanaged ReasonCode = "chain_data_set_unmanaged" + ReasonLocalStatusNotReady ReasonCode = "local_status_not_ready" + ReasonProviderMismatch ReasonCode = "provider_mismatch" + ReasonMetadataMismatch ReasonCode = "metadata_mismatch" + ReasonCopyUnderReplicated ReasonCode = "copy_under_replicated" + ReasonCopyPending ReasonCode = "copy_pending" + ReasonCopyCommitting ReasonCode = "copy_committing" + ReasonCopyFailed ReasonCode = "copy_failed" + ReasonCopyMissingProvider ReasonCode = "copy_missing_provider" + ReasonCopyMissingDataSet ReasonCode = "copy_missing_data_set" + ReasonCopyMissingPiece ReasonCode = "copy_missing_piece" + ReasonCopyMissingRetrievalURL ReasonCode = "copy_missing_retrieval_url" + ReasonCopyObservationMissing ReasonCode = "copy_observation_missing" ) var ErrProviderNotFound = errors.New("provider not found") @@ -112,38 +102,37 @@ type ProviderState struct { bun.BaseModel `bun:"table:observability_provider_states"` ProviderID types.OnChainID `bun:"provider_id,pk,type:text" json:"provider_id"` - Status Status `bun:"status,notnull" json:"status"` + Status Status `bun:"status,type:text,notnull" json:"status"` ReasonCodes []ReasonCode `bun:"reason_codes,type:jsonb,notnull" json:"reason_codes"` Active *bool `bun:"active" json:"active,omitempty"` HasPDP *bool `bun:"has_pdp" json:"has_pdp,omitempty"` - ServiceURL *string `bun:"service_url" json:"service_url,omitempty"` - HealthStatus *string `bun:"health_status" json:"health_status,omitempty"` + ServiceURL *string `bun:"service_url,type:text" json:"service_url,omitempty"` + HealthStatus *string `bun:"health_status,type:text" json:"health_status,omitempty"` LastCheckedAt time.Time `bun:"last_checked_at,nullzero,notnull" json:"last_checked_at"` - LastError *string `bun:"last_error,nullzero" json:"last_error,omitempty"` + LastError *string `bun:"last_error,type:text,nullzero" json:"last_error,omitempty"` Evidence map[string]any `bun:"evidence_json,type:jsonb,notnull" json:"evidence"` - CreatedAt time.Time `bun:"created_at,nullzero,notnull,default:current_timestamp" json:"created_at"` - UpdatedAt time.Time `bun:"updated_at,nullzero,notnull,default:current_timestamp" json:"updated_at"` } type DataSetState struct { - bun.BaseModel `bun:"table:observability_data_set_states"` - - LocalDataSetID int64 `bun:"local_data_set_id,pk" json:"local_data_set_id"` - BucketID int64 `bun:"bucket_id,notnull" json:"bucket_id"` - BucketName string `bun:"bucket_name,notnull" json:"bucket_name"` - CopyIndex int `bun:"copy_index,notnull" json:"copy_index"` - ProviderID types.OnChainID `bun:"provider_id,type:text,notnull" json:"provider_id"` - ChainDataSetID *types.OnChainID `bun:"chain_data_set_id,type:text" json:"chain_data_set_id,omitempty"` - ClientDataSetID *types.OnChainID `bun:"client_data_set_id,type:text" json:"client_data_set_id,omitempty"` - LocalStatus model.StorageDataSetStatus `bun:"local_status,notnull" json:"local_status"` - Status Status `bun:"status,notnull" json:"status"` + bun.BaseModel `bun:"table:observability_data_set_states,alias:observability_data_set_state"` + + LocalDataSetID int64 `bun:"local_data_set_id,pk" json:"local_data_set_id"` + BucketID int64 `bun:"bucket_id,notnull" json:"bucket_id"` + CopyIndex int `bun:"copy_index,type:integer,notnull" json:"copy_index"` + ProviderID types.OnChainID `bun:"provider_id,type:text,notnull" json:"provider_id"` + ChainDataSetID *types.OnChainID `bun:"chain_data_set_id,type:text" json:"chain_data_set_id,omitempty"` + ClientDataSetID *types.OnChainID `bun:"client_data_set_id,type:text" json:"client_data_set_id,omitempty"` + // BucketName and LocalStatus are projected from the rows that own them. + // Observations are rebuilt wholesale on every refresh, so a copy here would + // only add a second, staler answer to the same question. + BucketName string `bun:",scanonly" json:"bucket_name"` + LocalStatus model.StorageDataSetStatus `bun:",scanonly" json:"local_status"` + Status Status `bun:"status,type:text,notnull" json:"status"` ReasonCodes []ReasonCode `bun:"reason_codes,type:jsonb,notnull" json:"reason_codes"` ActivePieceCount *int64 `bun:"active_piece_count,nullzero" json:"active_piece_count,omitempty"` LastCheckedAt time.Time `bun:"last_checked_at,nullzero,notnull" json:"last_checked_at"` - LastError *string `bun:"last_error,nullzero" json:"last_error,omitempty"` + LastError *string `bun:"last_error,type:text,nullzero" json:"last_error,omitempty"` Evidence map[string]any `bun:"evidence_json,type:jsonb,notnull" json:"evidence"` - CreatedAt time.Time `bun:"created_at,nullzero,notnull,default:current_timestamp" json:"created_at"` - UpdatedAt time.Time `bun:"updated_at,nullzero,notnull,default:current_timestamp" json:"updated_at"` } type CollectionState struct { @@ -151,8 +140,8 @@ type CollectionState struct { CollectionType CollectionType `bun:"collection_type,pk,type:text" json:"collection_type"` LastCheckedAt time.Time `bun:"last_checked_at,nullzero,notnull" json:"last_checked_at"` - CreatedAt time.Time `bun:"created_at,nullzero,notnull,default:current_timestamp" json:"created_at"` - UpdatedAt time.Time `bun:"updated_at,nullzero,notnull,default:current_timestamp" json:"updated_at"` + CreatedAt time.Time `bun:"created_at,nullzero,notnull" json:"created_at"` + UpdatedAt time.Time `bun:"updated_at,nullzero,notnull" json:"updated_at"` } type Summary struct { @@ -202,13 +191,13 @@ type DataSetFacts struct { } type CopyFacts struct { - Status model.StorageUploadCopyStatus `json:"status"` - ProviderID *types.OnChainID `json:"provider_id,omitempty"` - LocalDataSetID *int64 `json:"local_data_set_id,omitempty"` - ChainDataSetID *types.OnChainID `json:"chain_data_set_id,omitempty"` - PieceID *types.OnChainID `json:"piece_id,omitempty"` - RetrievalURL *string `json:"retrieval_url,omitempty"` - LastError *string `json:"last_error,omitempty"` + Status model.StorageCopyStatus `json:"status"` + ProviderID *types.OnChainID `json:"provider_id,omitempty"` + LocalDataSetID *int64 `json:"local_data_set_id,omitempty"` + ChainDataSetID *types.OnChainID `json:"chain_data_set_id,omitempty"` + PieceID *types.OnChainID `json:"piece_id,omitempty"` + RetrievalURL *string `json:"retrieval_url,omitempty"` + LastError *string `json:"last_error,omitempty"` } type Freshness struct { @@ -266,3 +255,22 @@ type DataSetStatePage struct { Limit int `json:"limit"` Offset int `json:"offset"` } + +var _ bun.BeforeAppendModelHook = (*CollectionState)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (c *CollectionState) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if c.CreatedAt.IsZero() { + c.CreatedAt = now + } + if c.UpdatedAt.IsZero() { + c.UpdatedAt = now + } + return nil +} diff --git a/internal/observability/runner.go b/internal/observability/runner.go deleted file mode 100644 index 5a36e72..0000000 --- a/internal/observability/runner.go +++ /dev/null @@ -1,49 +0,0 @@ -package observability - -import ( - "context" - "log/slog" - "time" -) - -type Runner struct { - service *Service - interval time.Duration - logger *slog.Logger -} - -func NewRunner(service *Service, logger *slog.Logger) *Runner { - interval := 5 * time.Minute - if service != nil { - interval = service.RefreshInterval() - } - return &Runner{ - service: service, - interval: interval, - logger: logger, - } -} - -func (r *Runner) Run(ctx context.Context) { - if r == nil || r.service == nil { - return - } - r.refresh(ctx) - - ticker := time.NewTicker(r.interval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - r.refresh(ctx) - } - } -} - -func (r *Runner) refresh(ctx context.Context) { - if err := r.service.RefreshAll(ctx); err != nil && r.logger != nil && ctx.Err() == nil { - r.logger.Warn("observability refresh failed", "error", err) - } -} diff --git a/internal/observability/task_diagnostics.go b/internal/observability/task_diagnostics.go deleted file mode 100644 index 89e1b88..0000000 --- a/internal/observability/task_diagnostics.go +++ /dev/null @@ -1,462 +0,0 @@ -package observability - -import ( - "encoding/json" - "strings" - "time" - - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/types" -) - -type TaskDiagnosticCurrentState string - -const ( - TaskDiagnosticStateNotApplicable TaskDiagnosticCurrentState = "not_applicable" - TaskDiagnosticStatePreparing TaskDiagnosticCurrentState = "preparing" - TaskDiagnosticStateTransferring TaskDiagnosticCurrentState = "transferring" - TaskDiagnosticStateWaitingForChain TaskDiagnosticCurrentState = "waiting_for_chain" - TaskDiagnosticStateConfirmed TaskDiagnosticCurrentState = "confirmed" - TaskDiagnosticStateRejected TaskDiagnosticCurrentState = "rejected" - TaskDiagnosticStateMismatch TaskDiagnosticCurrentState = "mismatch" - TaskDiagnosticStateUnavailable TaskDiagnosticCurrentState = "unavailable" - TaskDiagnosticStateUnknown TaskDiagnosticCurrentState = "unknown" -) - -type TaskDiagnosticNextAction string - -const ( - TaskDiagnosticActionNone TaskDiagnosticNextAction = "none" - TaskDiagnosticActionWait TaskDiagnosticNextAction = "wait" - TaskDiagnosticActionRetryTask TaskDiagnosticNextAction = "retry_task" - TaskDiagnosticActionCheckWalletFunds TaskDiagnosticNextAction = "check_wallet_funds" - TaskDiagnosticActionCheckWalletApproval TaskDiagnosticNextAction = "check_wallet_approval" - TaskDiagnosticActionInspectProvider TaskDiagnosticNextAction = "inspect_provider" - TaskDiagnosticActionInspectTask TaskDiagnosticNextAction = "inspect_task" -) - -type TaskDiagnosticOperation string - -const ( - TaskDiagnosticOperationNone TaskDiagnosticOperation = "none" - TaskDiagnosticOperationPrepareUpload TaskDiagnosticOperation = "prepare_upload" - TaskDiagnosticOperationTransferPiece TaskDiagnosticOperation = "transfer_piece" - TaskDiagnosticOperationCreateDataSet TaskDiagnosticOperation = "create_data_set" - TaskDiagnosticOperationAddPieces TaskDiagnosticOperation = "add_pieces" -) - -type TaskDiagnosticLiveState string - -const ( - TaskDiagnosticLiveSkipped TaskDiagnosticLiveState = "skipped" - TaskDiagnosticLivePending TaskDiagnosticLiveState = "pending" - TaskDiagnosticLiveConfirmed TaskDiagnosticLiveState = "confirmed" - TaskDiagnosticLiveRejected TaskDiagnosticLiveState = "rejected" - TaskDiagnosticLiveMismatch TaskDiagnosticLiveState = "mismatch" - TaskDiagnosticLiveUnavailable TaskDiagnosticLiveState = "unavailable" - TaskDiagnosticLiveUnknown TaskDiagnosticLiveState = "unknown" -) - -type TaskDiagnostic struct { - CheckedAt time.Time `json:"checked_at"` - CurrentState TaskDiagnosticCurrentState `json:"current_state"` - Signal Signal `json:"signal"` - ReasonCodes []ReasonCode `json:"reason_codes"` - NextAction TaskDiagnosticNextAction `json:"next_action"` - Evidence TaskDiagnosticEvidence `json:"evidence"` -} - -type TaskDiagnosticEvidence struct { - Task TaskDiagnosticTaskFacts `json:"task"` - Upload *TaskDiagnosticUploadFacts `json:"upload,omitempty"` - Copy *TaskDiagnosticCopyFacts `json:"copy,omitempty"` - DataSet *TaskDiagnosticDataSetFacts `json:"data_set,omitempty"` - Provider *TaskDiagnosticProviderFacts `json:"provider,omitempty"` - Transaction *TaskDiagnosticTransactionFacts `json:"transaction,omitempty"` - LiveCheck *TaskDiagnosticLiveCheck `json:"live_check,omitempty"` - Operation TaskDiagnosticOperation `json:"operation"` -} - -type TaskDiagnosticFacts struct { - Task TaskDiagnosticTaskFacts - Upload *TaskDiagnosticUploadFacts - Copy *TaskDiagnosticCopyFacts - DataSet *TaskDiagnosticDataSetFacts - Provider *TaskDiagnosticProviderFacts - Transaction *TaskDiagnosticTransactionFacts -} - -type TaskDiagnosticTaskFacts struct { - ID int64 `json:"id,omitempty"` - Type model.TaskType `json:"type"` - Stage string `json:"stage,omitempty"` - Status model.TaskStatus `json:"status"` - RetryCount int `json:"retry_count"` - MaxRetries int `json:"max_retries"` - LastError *string `json:"last_error,omitempty"` - StatusMessage *string `json:"status_message,omitempty"` - WaitReason *model.TaskWaitReason `json:"wait_reason,omitempty"` - ScheduledAt *time.Time `json:"scheduled_at,omitempty"` -} - -type TaskDiagnosticUploadFacts struct { - ID int64 `json:"id,omitempty"` - Status model.StorageUploadStatus `json:"status,omitempty"` - RequestedCopies int `json:"requested_copies,omitempty"` - ErrorMessage *string `json:"error_message,omitempty"` - AcceptError *string `json:"accept_error,omitempty"` -} - -type TaskDiagnosticCopyFacts struct { - UploadID int64 `json:"upload_id,omitempty"` - CopyIndex int `json:"copy_index,omitempty"` - Status model.StorageUploadCopyStatus `json:"status,omitempty"` - ProviderID *types.OnChainID `json:"provider_id,omitempty"` - StorageDataSetID *int64 `json:"storage_data_set_id,omitempty"` - ChainDataSetID *types.OnChainID `json:"chain_data_set_id,omitempty"` - PieceID *types.OnChainID `json:"piece_id,omitempty"` - TransferMethod model.StorageCopyTransferMethod `json:"transfer_method,omitempty"` - CommitTransactionID *string `json:"commit_transaction_id,omitempty"` - LastError *string `json:"last_error,omitempty"` -} - -type TaskDiagnosticDataSetFacts struct { - ID int64 `json:"id,omitempty"` - Status model.StorageDataSetStatus `json:"status,omitempty"` - ProviderID types.OnChainID `json:"provider_id,omitempty"` - CopyIndex int `json:"copy_index,omitempty"` - ChainDataSetID *types.OnChainID `json:"chain_data_set_id,omitempty"` - ClientDataSetID *types.OnChainID `json:"client_data_set_id,omitempty"` - CreateTransactionID *string `json:"create_transaction_id,omitempty"` - CreateStatusURL *string `json:"create_status_url,omitempty"` - LastError *string `json:"last_error,omitempty"` -} - -type TaskDiagnosticProviderFacts struct { - ProviderID types.OnChainID `json:"provider_id,omitempty"` - Status Status `json:"status,omitempty"` - ReasonCodes []ReasonCode `json:"reason_codes,omitempty"` - ServiceURL *string `json:"service_url,omitempty"` - HealthStatus *string `json:"health_status,omitempty"` - LastError *string `json:"last_error,omitempty"` -} - -type TaskDiagnosticTransactionFacts struct { - Kind TaskDiagnosticOperation `json:"kind"` - StatusURL string `json:"status_url,omitempty"` - ServiceURL string `json:"service_url,omitempty"` - DataSetID string `json:"data_set_id,omitempty"` - TransactionID string `json:"transaction_id,omitempty"` - PieceCount int `json:"piece_count,omitempty"` -} - -type TaskDiagnosticLiveCheck struct { - State TaskDiagnosticLiveState `json:"state"` - StatusURL string `json:"status_url,omitempty"` - TxStatus string `json:"tx_status,omitempty"` - DataSetID string `json:"data_set_id,omitempty"` - DataSetCreated *bool `json:"data_set_created,omitempty"` - PiecesAdded *bool `json:"pieces_added,omitempty"` - PieceCount *int `json:"piece_count,omitempty"` - ConfirmedPieceIDs []string `json:"confirmed_piece_ids,omitempty"` - Error string `json:"error,omitempty"` - - confirmedPieceIDsSet bool -} - -// SetConfirmedPieceIDs records provider evidence even when the provider returns an empty list. -func (live *TaskDiagnosticLiveCheck) SetConfirmedPieceIDs(ids []string) { - live.ConfirmedPieceIDs = ids - if live.ConfirmedPieceIDs == nil { - live.ConfirmedPieceIDs = []string{} - } - live.confirmedPieceIDsSet = true -} - -func (live TaskDiagnosticLiveCheck) MarshalJSON() ([]byte, error) { - type taskDiagnosticLiveCheckJSON struct { - State TaskDiagnosticLiveState `json:"state"` - StatusURL string `json:"status_url,omitempty"` - TxStatus string `json:"tx_status,omitempty"` - DataSetID string `json:"data_set_id,omitempty"` - DataSetCreated *bool `json:"data_set_created,omitempty"` - PiecesAdded *bool `json:"pieces_added,omitempty"` - PieceCount *int `json:"piece_count,omitempty"` - ConfirmedPieceIDs any `json:"confirmed_piece_ids,omitempty"` - Error string `json:"error,omitempty"` - } - ids := any(nil) - if live.confirmedPieceIDsSet { - confirmed := live.ConfirmedPieceIDs - if confirmed == nil { - confirmed = []string{} - } - ids = confirmed - } - return json.Marshal(taskDiagnosticLiveCheckJSON{ - State: live.State, - StatusURL: live.StatusURL, - TxStatus: live.TxStatus, - DataSetID: live.DataSetID, - DataSetCreated: live.DataSetCreated, - PiecesAdded: live.PiecesAdded, - PieceCount: live.PieceCount, - ConfirmedPieceIDs: ids, - Error: live.Error, - }) -} - -func TaskDiagnosticFromFacts(facts TaskDiagnosticFacts, live *TaskDiagnosticLiveCheck, now time.Time) TaskDiagnostic { - if now.IsZero() { - now = time.Now().UTC() - } - operation := taskDiagnosticOperation(facts) - state, status, reasons, next := taskDiagnosticAssessment(facts, live, operation) - signal := BuildSignal(status, reasons, taskDiagnosticLastError(facts, live), &now, time.Hour, now) - return TaskDiagnostic{ - CheckedAt: now, - CurrentState: state, - Signal: signal, - ReasonCodes: signal.ReasonCodes, - NextAction: next, - Evidence: TaskDiagnosticEvidence{ - Task: facts.Task, - Upload: facts.Upload, - Copy: facts.Copy, - DataSet: facts.DataSet, - Provider: facts.Provider, - Transaction: facts.Transaction, - LiveCheck: live, - Operation: operation, - }, - } -} - -func taskDiagnosticAssessment(facts TaskDiagnosticFacts, live *TaskDiagnosticLiveCheck, operation TaskDiagnosticOperation) (TaskDiagnosticCurrentState, Status, []ReasonCode, TaskDiagnosticNextAction) { - if facts.Task.Type != model.TaskTypeUpload { - return TaskDiagnosticStateNotApplicable, StatusAvailable, []ReasonCode{ReasonTaskNotApplicable}, TaskDiagnosticActionNone - } - if live != nil && live.State != "" && live.State != TaskDiagnosticLiveSkipped { - return taskDiagnosticFromLive(*live) - } - if state, status, reason, next, ok := taskDiagnosticFromError(taskDiagnosticTextEvidence(facts)); ok { - return state, status, []ReasonCode{reason}, next - } - if facts.Task.WaitReason != nil && *facts.Task.WaitReason == model.TaskWaitReasonExternalConfirmation { - return TaskDiagnosticStateWaitingForChain, StatusDegraded, []ReasonCode{ReasonTaskChainPending}, TaskDiagnosticActionWait - } - if facts.DataSet != nil { - switch facts.DataSet.Status { - case model.StorageDataSetStatusCreating: - return TaskDiagnosticStateWaitingForChain, StatusDegraded, []ReasonCode{ReasonTaskChainPending}, TaskDiagnosticActionWait - case model.StorageDataSetStatusFailed, model.StorageDataSetStatusUnavailable: - return TaskDiagnosticStateUnavailable, StatusUnavailable, []ReasonCode{ReasonTaskDiagnosticUnavailable}, TaskDiagnosticActionInspectProvider - case model.StorageDataSetStatusReady: - if operation == TaskDiagnosticOperationCreateDataSet { - return TaskDiagnosticStateConfirmed, StatusAvailable, []ReasonCode{ReasonTaskChainConfirmed}, TaskDiagnosticActionNone - } - } - } - if facts.Copy != nil { - switch facts.Copy.Status { - case model.StorageUploadCopyStatusCommitting: - return TaskDiagnosticStateWaitingForChain, StatusDegraded, []ReasonCode{ReasonTaskChainPending}, TaskDiagnosticActionWait - case model.StorageUploadCopyStatusCommitted: - return TaskDiagnosticStateConfirmed, StatusAvailable, []ReasonCode{ReasonTaskChainConfirmed}, TaskDiagnosticActionNone - case model.StorageUploadCopyStatusFailed: - return TaskDiagnosticStateUnavailable, StatusUnavailable, []ReasonCode{ReasonTaskDiagnosticUnavailable}, TaskDiagnosticActionInspectProvider - } - } - switch operation { - case TaskDiagnosticOperationPrepareUpload: - return TaskDiagnosticStatePreparing, StatusDegraded, []ReasonCode{ReasonCopyPending}, TaskDiagnosticActionWait - case TaskDiagnosticOperationTransferPiece: - return TaskDiagnosticStateTransferring, StatusDegraded, []ReasonCode{ReasonCopyPending}, TaskDiagnosticActionWait - case TaskDiagnosticOperationCreateDataSet, TaskDiagnosticOperationAddPieces: - return TaskDiagnosticStateUnknown, StatusUnknown, []ReasonCode{ReasonTaskMissingEvidence}, TaskDiagnosticActionInspectTask - default: - return TaskDiagnosticStateUnknown, StatusUnknown, []ReasonCode{ReasonTaskUnknownStatus}, TaskDiagnosticActionInspectTask - } -} - -func taskDiagnosticFromLive(live TaskDiagnosticLiveCheck) (TaskDiagnosticCurrentState, Status, []ReasonCode, TaskDiagnosticNextAction) { - switch live.State { - case TaskDiagnosticLivePending: - return TaskDiagnosticStateWaitingForChain, StatusDegraded, []ReasonCode{ReasonTaskChainPending}, TaskDiagnosticActionWait - case TaskDiagnosticLiveConfirmed: - return TaskDiagnosticStateConfirmed, StatusAvailable, []ReasonCode{ReasonTaskChainConfirmed}, TaskDiagnosticActionNone - case TaskDiagnosticLiveRejected: - return TaskDiagnosticStateRejected, StatusUnavailable, []ReasonCode{ReasonTaskTransactionRejected}, TaskDiagnosticActionRetryTask - case TaskDiagnosticLiveMismatch: - return TaskDiagnosticStateMismatch, StatusUnavailable, []ReasonCode{ReasonTaskPieceStatusMismatch}, TaskDiagnosticActionInspectProvider - case TaskDiagnosticLiveUnavailable: - if state, status, reason, next, ok := taskDiagnosticFromError(live.Error); ok { - return state, status, []ReasonCode{reason}, next - } - return TaskDiagnosticStateUnknown, StatusUnknown, []ReasonCode{ReasonTaskDiagnosticUnavailable}, TaskDiagnosticActionInspectProvider - default: - return TaskDiagnosticStateUnknown, StatusUnknown, []ReasonCode{ReasonTaskUnknownStatus}, TaskDiagnosticActionInspectTask - } -} - -func taskDiagnosticFromError(text string) (TaskDiagnosticCurrentState, Status, ReasonCode, TaskDiagnosticNextAction, bool) { - lower := strings.ToLower(text) - for _, matcher := range taskDiagnosticErrorMatchers { - if matcher.matches(lower) { - return matcher.state, matcher.status, matcher.reason, matcher.next, true - } - } - return "", "", "", "", false -} - -type taskDiagnosticErrorMatcher struct { - needles []string - state TaskDiagnosticCurrentState - status Status - reason ReasonCode - next TaskDiagnosticNextAction -} - -var taskDiagnosticErrorMatchers = []taskDiagnosticErrorMatcher{ - { - needles: []string{"available funds = 0", "insufficient funds", "not enough funds"}, - state: TaskDiagnosticStateUnavailable, - status: StatusUnavailable, - reason: ReasonTaskInsufficientFunds, - next: TaskDiagnosticActionCheckWalletFunds, - }, - { - needles: []string{"approval", "allowance"}, - state: TaskDiagnosticStateUnavailable, - status: StatusUnavailable, - reason: ReasonTaskMissingApproval, - next: TaskDiagnosticActionCheckWalletApproval, - }, - { - needles: []string{"transaction rejected", "tx rejected", "pdp: transaction rejected"}, - state: TaskDiagnosticStateRejected, - status: StatusUnavailable, - reason: ReasonTaskTransactionRejected, - next: TaskDiagnosticActionRetryTask, - }, - { - needles: []string{"mismatch", "confirmed without", "piecesadded=false"}, - state: TaskDiagnosticStateMismatch, - status: StatusUnavailable, - reason: ReasonTaskPieceStatusMismatch, - next: TaskDiagnosticActionInspectProvider, - }, - { - needles: []string{"timeout", "deadline exceeded", "rpc"}, - state: TaskDiagnosticStateUnknown, - status: StatusUnknown, - reason: ReasonTaskRPCUnavailable, - next: TaskDiagnosticActionInspectProvider, - }, - { - needles: []string{"missing service url", "missing status url"}, - state: TaskDiagnosticStateUnavailable, - status: StatusUnavailable, - reason: ReasonTaskDiagnosticUnavailable, - next: TaskDiagnosticActionInspectProvider, - }, - { - needles: []string{"missing data set id", "missing transaction id"}, - state: TaskDiagnosticStateUnavailable, - status: StatusUnavailable, - reason: ReasonTaskDiagnosticUnavailable, - next: TaskDiagnosticActionInspectTask, - }, -} - -func (m taskDiagnosticErrorMatcher) matches(text string) bool { - for _, needle := range m.needles { - if strings.Contains(text, needle) { - return true - } - } - return false -} - -func taskDiagnosticOperation(facts TaskDiagnosticFacts) TaskDiagnosticOperation { - return TaskDiagnosticOperationForTask(facts.Task.Type, facts.Task.Stage) -} - -// TaskDiagnosticOperationForTask returns the diagnostic operation represented by a task type and stage. -func TaskDiagnosticOperationForTask(taskType model.TaskType, stage string) TaskDiagnosticOperation { - if taskType != model.TaskTypeUpload { - return TaskDiagnosticOperationNone - } - switch stage { - case "", "prepare_upload": - return TaskDiagnosticOperationPrepareUpload - case "ensure_dataset": - return TaskDiagnosticOperationCreateDataSet - case "ingress_store", "peer_pull": - return TaskDiagnosticOperationTransferPiece - case "ingress_commit", "peer_commit": - return TaskDiagnosticOperationAddPieces - default: - return TaskDiagnosticOperationNone - } -} - -func taskDiagnosticTextEvidence(facts TaskDiagnosticFacts) string { - var parts []string - if facts.Task.LastError != nil { - parts = append(parts, *facts.Task.LastError) - } - if facts.Task.StatusMessage != nil { - parts = append(parts, *facts.Task.StatusMessage) - } - if facts.Upload != nil { - if facts.Upload.ErrorMessage != nil { - parts = append(parts, *facts.Upload.ErrorMessage) - } - if facts.Upload.AcceptError != nil { - parts = append(parts, *facts.Upload.AcceptError) - } - } - if facts.Copy != nil && facts.Copy.LastError != nil { - parts = append(parts, *facts.Copy.LastError) - } - if facts.DataSet != nil && facts.DataSet.LastError != nil { - parts = append(parts, *facts.DataSet.LastError) - } - if facts.Provider != nil && facts.Provider.LastError != nil { - parts = append(parts, *facts.Provider.LastError) - } - return strings.Join(parts, "\n") -} - -func taskDiagnosticLastError(facts TaskDiagnosticFacts, live *TaskDiagnosticLiveCheck) *string { - if live != nil && live.Error != "" { - return &live.Error - } - for _, value := range []*string{ - facts.Task.LastError, - facts.Task.StatusMessage, - } { - if value != nil && *value != "" { - return value - } - } - if facts.Upload != nil { - if facts.Upload.ErrorMessage != nil && *facts.Upload.ErrorMessage != "" { - return facts.Upload.ErrorMessage - } - if facts.Upload.AcceptError != nil && *facts.Upload.AcceptError != "" { - return facts.Upload.AcceptError - } - } - if facts.Copy != nil && facts.Copy.LastError != nil && *facts.Copy.LastError != "" { - return facts.Copy.LastError - } - if facts.DataSet != nil && facts.DataSet.LastError != nil && *facts.DataSet.LastError != "" { - return facts.DataSet.LastError - } - if facts.Provider != nil && facts.Provider.LastError != nil && *facts.Provider.LastError != "" { - return facts.Provider.LastError - } - return nil -} diff --git a/internal/observability/task_diagnostics_test.go b/internal/observability/task_diagnostics_test.go deleted file mode 100644 index b5cfdd9..0000000 --- a/internal/observability/task_diagnostics_test.go +++ /dev/null @@ -1,213 +0,0 @@ -package observability - -import ( - "reflect" - "testing" - "time" - - "github.com/strahe/synaps3/internal/model" -) - -func TestTaskDiagnosticFromFactsClassifiesUploadEvidence(t *testing.T) { - now := time.Date(2026, 5, 22, 10, 0, 0, 0, time.UTC) - - tests := []struct { - name string - facts TaskDiagnosticFacts - wantState TaskDiagnosticCurrentState - wantStatus Status - wantReasons []ReasonCode - wantNext TaskDiagnosticNextAction - wantOperation TaskDiagnosticOperation - }{ - { - name: "insufficient funds", - facts: taskDiagnosticFactsWithLastError("USDFC available funds = 0"), - wantState: TaskDiagnosticStateUnavailable, - wantStatus: StatusUnavailable, - wantReasons: []ReasonCode{ReasonTaskInsufficientFunds}, - wantNext: TaskDiagnosticActionCheckWalletFunds, - wantOperation: TaskDiagnosticOperationAddPieces, - }, - { - name: "approval missing", - facts: taskDiagnosticFactsWithLastError("wallet allowance approval missing"), - wantState: TaskDiagnosticStateUnavailable, - wantStatus: StatusUnavailable, - wantReasons: []ReasonCode{ReasonTaskMissingApproval}, - wantNext: TaskDiagnosticActionCheckWalletApproval, - wantOperation: TaskDiagnosticOperationAddPieces, - }, - { - name: "rpc timeout", - facts: taskDiagnosticFactsWithLastError("rpc deadline exceeded while reading transaction receipt"), - wantState: TaskDiagnosticStateUnknown, - wantStatus: StatusUnknown, - wantReasons: []ReasonCode{ReasonTaskRPCUnavailable}, - wantNext: TaskDiagnosticActionInspectProvider, - wantOperation: TaskDiagnosticOperationAddPieces, - }, - { - name: "external confirmation wait", - facts: TaskDiagnosticFacts{ - Task: TaskDiagnosticTaskFacts{ - Type: model.TaskTypeUpload, - Stage: "ensure_dataset", - Status: model.TaskStatusWaiting, - WaitReason: taskWaitReasonPtr(model.TaskWaitReasonExternalConfirmation), - }, - DataSet: &TaskDiagnosticDataSetFacts{Status: model.StorageDataSetStatusCreating}, - }, - wantState: TaskDiagnosticStateWaitingForChain, - wantStatus: StatusDegraded, - wantReasons: []ReasonCode{ReasonTaskChainPending}, - wantNext: TaskDiagnosticActionWait, - wantOperation: TaskDiagnosticOperationCreateDataSet, - }, - { - name: "non upload", - facts: TaskDiagnosticFacts{Task: TaskDiagnosticTaskFacts{Type: model.TaskTypeEvictCache, Status: model.TaskStatusQueued}}, - wantState: TaskDiagnosticStateNotApplicable, - wantStatus: StatusAvailable, - wantReasons: []ReasonCode{ReasonTaskNotApplicable}, - wantNext: TaskDiagnosticActionNone, - wantOperation: TaskDiagnosticOperationNone, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := TaskDiagnosticFromFacts(tt.facts, nil, now) - if got.CurrentState != tt.wantState || got.Signal.Status != tt.wantStatus || got.NextAction != tt.wantNext || got.Evidence.Operation != tt.wantOperation { - t.Fatalf("diagnostic = state:%s status:%s next:%s op:%s, want state:%s status:%s next:%s op:%s", - got.CurrentState, got.Signal.Status, got.NextAction, got.Evidence.Operation, - tt.wantState, tt.wantStatus, tt.wantNext, tt.wantOperation) - } - if !reflect.DeepEqual(got.Signal.ReasonCodes, tt.wantReasons) { - t.Fatalf("reasons = %#v, want %#v", got.Signal.ReasonCodes, tt.wantReasons) - } - }) - } -} - -func TestTaskDiagnosticFromFactsClassifiesLiveStatus(t *testing.T) { - now := time.Date(2026, 5, 22, 10, 0, 0, 0, time.UTC) - baseFacts := taskDiagnosticFactsWithLastError("") - - tests := []struct { - name string - live TaskDiagnosticLiveCheck - wantState TaskDiagnosticCurrentState - wantStatus Status - wantReasons []ReasonCode - wantNext TaskDiagnosticNextAction - }{ - { - name: "pending", - live: TaskDiagnosticLiveCheck{State: TaskDiagnosticLivePending, TxStatus: "pending"}, - wantState: TaskDiagnosticStateWaitingForChain, - wantStatus: StatusDegraded, - wantReasons: []ReasonCode{ReasonTaskChainPending}, - wantNext: TaskDiagnosticActionWait, - }, - { - name: "confirmed", - live: TaskDiagnosticLiveCheck{State: TaskDiagnosticLiveConfirmed, TxStatus: "confirmed", PiecesAdded: boolPtr(true), PieceCount: intPtr(1)}, - wantState: TaskDiagnosticStateConfirmed, - wantStatus: StatusAvailable, - wantReasons: []ReasonCode{ReasonTaskChainConfirmed}, - wantNext: TaskDiagnosticActionNone, - }, - { - name: "rejected", - live: TaskDiagnosticLiveCheck{State: TaskDiagnosticLiveRejected, TxStatus: "rejected"}, - wantState: TaskDiagnosticStateRejected, - wantStatus: StatusUnavailable, - wantReasons: []ReasonCode{ReasonTaskTransactionRejected}, - wantNext: TaskDiagnosticActionRetryTask, - }, - { - name: "mismatch", - live: TaskDiagnosticLiveCheck{State: TaskDiagnosticLiveMismatch, TxStatus: "confirmed", PiecesAdded: boolPtr(false)}, - wantState: TaskDiagnosticStateMismatch, - wantStatus: StatusUnavailable, - wantReasons: []ReasonCode{ReasonTaskPieceStatusMismatch}, - wantNext: TaskDiagnosticActionInspectProvider, - }, - { - name: "timeout unavailable", - live: TaskDiagnosticLiveCheck{State: TaskDiagnosticLiveUnavailable, Error: "context deadline exceeded"}, - wantState: TaskDiagnosticStateUnknown, - wantStatus: StatusUnknown, - wantReasons: []ReasonCode{ReasonTaskRPCUnavailable}, - wantNext: TaskDiagnosticActionInspectProvider, - }, - { - name: "missing service URL unavailable", - live: TaskDiagnosticLiveCheck{State: TaskDiagnosticLiveUnavailable, Error: "missing service URL"}, - wantState: TaskDiagnosticStateUnavailable, - wantStatus: StatusUnavailable, - wantReasons: []ReasonCode{ReasonTaskDiagnosticUnavailable}, - wantNext: TaskDiagnosticActionInspectProvider, - }, - { - name: "unavailable with parsed wallet error", - live: TaskDiagnosticLiveCheck{State: TaskDiagnosticLiveUnavailable, Error: "insufficient funds for add pieces"}, - wantState: TaskDiagnosticStateUnavailable, - wantStatus: StatusUnavailable, - wantReasons: []ReasonCode{ReasonTaskInsufficientFunds}, - wantNext: TaskDiagnosticActionCheckWalletFunds, - }, - { - name: "unknown response", - live: TaskDiagnosticLiveCheck{State: TaskDiagnosticLiveUnknown, TxStatus: "queued"}, - wantState: TaskDiagnosticStateUnknown, - wantStatus: StatusUnknown, - wantReasons: []ReasonCode{ReasonTaskUnknownStatus}, - wantNext: TaskDiagnosticActionInspectTask, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := TaskDiagnosticFromFacts(baseFacts, &tt.live, now) - if got.CurrentState != tt.wantState || got.Signal.Status != tt.wantStatus || got.NextAction != tt.wantNext { - t.Fatalf("diagnostic = state:%s status:%s next:%s, want state:%s status:%s next:%s", - got.CurrentState, got.Signal.Status, got.NextAction, tt.wantState, tt.wantStatus, tt.wantNext) - } - if !reflect.DeepEqual(got.Signal.ReasonCodes, tt.wantReasons) { - t.Fatalf("reasons = %#v, want %#v", got.Signal.ReasonCodes, tt.wantReasons) - } - }) - } -} - -func taskDiagnosticFactsWithLastError(lastError string) TaskDiagnosticFacts { - var lastErrorPtr *string - if lastError != "" { - lastErrorPtr = &lastError - } - return TaskDiagnosticFacts{ - Task: TaskDiagnosticTaskFacts{ - Type: model.TaskTypeUpload, - Stage: "ingress_commit", - Status: model.TaskStatusFailed, - LastError: lastErrorPtr, - }, - Copy: &TaskDiagnosticCopyFacts{ - Status: model.StorageUploadCopyStatusCommitting, - }, - Transaction: &TaskDiagnosticTransactionFacts{ - Kind: TaskDiagnosticOperationAddPieces, - TransactionID: "0xcommit", - }, - } -} - -func taskWaitReasonPtr(value model.TaskWaitReason) *model.TaskWaitReason { - return &value -} - -func intPtr(value int) *int { - return &value -} diff --git a/internal/s3access/logger_test.go b/internal/s3access/logger_test.go index 735c8ea..c21f11a 100644 --- a/internal/s3access/logger_test.go +++ b/internal/s3access/logger_test.go @@ -40,7 +40,7 @@ func TestLoggerOmitsQueryAndObjectDetailFields(t *testing.T) { Action: "GetObject", }) - for _, field := range []string{"query", "bucket", "key", "remote_ip", "bytes_sent", "object_size", "uploadID", "partNumber", "versionID", "bucket_owner"} { + for _, field := range []string{"query", "bucket", "key", "remote_ip", "bytes_sent", "object_size", "contentID", "partNumber", "versionID", "bucket_owner"} { if _, ok := record[field]; ok { t.Fatalf("record should omit %q: %#v", field, record) } diff --git a/internal/s3iam/service_test.go b/internal/s3iam/service_test.go index 0bc58d9..d1138b1 100644 --- a/internal/s3iam/service_test.go +++ b/internal/s3iam/service_test.go @@ -102,6 +102,7 @@ func TestServiceDeleteUserAccountRejectsOwnedBuckets(t *testing.T) { Name: "owned-bucket", OwnerAccessKey: &owner, Status: model.BucketStatusActive, + DefaultCopies: 8, MinimumDurableCopies: 8, }); err != nil { t.Fatalf("Create bucket: %v", err) } diff --git a/internal/state/machine.go b/internal/state/machine.go deleted file mode 100644 index 7a99142..0000000 --- a/internal/state/machine.go +++ /dev/null @@ -1,73 +0,0 @@ -package state - -import ( - "fmt" - "sync" -) - -// Transition defines a valid state change. -type Transition struct { - From string - To string -} - -// Machine is a generic, extensible finite state machine. -// States and transitions can be registered dynamically, supporting -// future additions without modifying existing logic. -type Machine struct { - mu sync.RWMutex - transitions map[string]map[string]bool // from → set of valid to-states -} - -// New creates a new empty state machine. -func New() *Machine { - return &Machine{ - transitions: make(map[string]map[string]bool), - } -} - -// Register adds one or more valid transitions. -func (m *Machine) Register(transitions ...Transition) { - m.mu.Lock() - defer m.mu.Unlock() - - for _, t := range transitions { - if m.transitions[t.From] == nil { - m.transitions[t.From] = make(map[string]bool) - } - m.transitions[t.From][t.To] = true - } -} - -// CanTransition reports whether moving from → to is allowed. -func (m *Machine) CanTransition(from, to string) bool { - m.mu.RLock() - defer m.mu.RUnlock() - - targets, ok := m.transitions[from] - if !ok { - return false - } - return targets[to] -} - -// Validate checks the transition and returns an error if it is not allowed. -func (m *Machine) Validate(from, to string) error { - if !m.CanTransition(from, to) { - return fmt.Errorf("invalid state transition: %s → %s", from, to) - } - return nil -} - -// NextStates returns all states reachable from the given state. -func (m *Machine) NextStates(from string) []string { - m.mu.RLock() - defer m.mu.RUnlock() - - targets := m.transitions[from] - out := make([]string, 0, len(targets)) - for t := range targets { - out = append(out, t) - } - return out -} diff --git a/internal/state/machine_test.go b/internal/state/machine_test.go deleted file mode 100644 index d55931d..0000000 --- a/internal/state/machine_test.go +++ /dev/null @@ -1,162 +0,0 @@ -package state - -import ( - "sort" - "testing" - - "github.com/strahe/synaps3/internal/model" -) - -func TestObjectStateMachine_HappyPath(t *testing.T) { - m := NewObjectStateMachine() - - transitions := []struct{ from, to model.ObjectState }{ - {model.ObjectStateCached, model.ObjectStateUploading}, - {model.ObjectStateUploading, model.ObjectStateCommitting}, - {model.ObjectStateUploading, model.ObjectStateReplicating}, - {model.ObjectStateCommitting, model.ObjectStateReplicating}, - {model.ObjectStateReplicating, model.ObjectStateStored}, - {model.ObjectStateStored, model.ObjectStateCacheEvicted}, - } - for _, tt := range transitions { - if err := m.Validate(string(tt.from), string(tt.to)); err != nil { - t.Errorf("happy path %s→%s: unexpected error: %v", tt.from, tt.to, err) - } - } -} - -func TestObjectStateMachine_FailureTransitions(t *testing.T) { - m := NewObjectStateMachine() - - // States that can transition to failed. - canFail := []model.ObjectState{ - model.ObjectStateUploading, - model.ObjectStateCommitting, - } - for _, from := range canFail { - if err := m.Validate(string(from), string(model.ObjectStateFailed)); err != nil { - t.Errorf("%s→failed: unexpected error: %v", from, err) - } - } - - // States that cannot transition to failed. - cannotFail := []model.ObjectState{ - model.ObjectStateCached, - model.ObjectStateReplicating, - model.ObjectStateStored, - model.ObjectStateFailed, - model.ObjectStateCacheEvicted, - } - for _, from := range cannotFail { - if m.CanTransition(string(from), string(model.ObjectStateFailed)) { - t.Errorf("%s→failed: should not be allowed", from) - } - } -} - -func TestObjectStateMachine_RetryFromFailed(t *testing.T) { - m := NewObjectStateMachine() - - // Allowed retries from failed. - allowed := []model.ObjectState{ - model.ObjectStateUploading, - } - for _, to := range allowed { - if err := m.Validate(string(model.ObjectStateFailed), string(to)); err != nil { - t.Errorf("failed→%s: unexpected error: %v", to, err) - } - } - - // Disallowed retries. - disallowed := []model.ObjectState{ - model.ObjectStateCached, - model.ObjectStateStored, - model.ObjectStateCacheEvicted, - } - for _, to := range disallowed { - if m.CanTransition(string(model.ObjectStateFailed), string(to)) { - t.Errorf("failed→%s: should not be allowed", to) - } - } -} - -func TestObjectStateMachine_ResumeReassignedIngress(t *testing.T) { - m := NewObjectStateMachine() - - if err := m.Validate(string(model.ObjectStateCommitting), string(model.ObjectStateUploading)); err != nil { - t.Fatalf("committing→uploading: unexpected error: %v", err) - } -} - -func TestObjectStateMachine_InvalidTransitions(t *testing.T) { - m := NewObjectStateMachine() - - invalid := []struct{ from, to model.ObjectState }{ - {model.ObjectStateCached, model.ObjectStateStored}, // must go through uploading - {model.ObjectStateCommitting, model.ObjectStateStored}, // secondary commits still pending - {model.ObjectStateCached, model.ObjectStateCacheEvicted}, // can't evict from cached - {model.ObjectStateCacheEvicted, model.ObjectStateCached}, // can't revive - {model.ObjectStateStored, model.ObjectStateUploading}, // backwards - {model.ObjectStateCached, model.ObjectStateFailed}, // can't fail from cached - {model.ObjectStateStored, model.ObjectStateFailed}, // stored failures stay in task state - {model.ObjectStateUploading, model.ObjectStateCacheEvicted}, // must go through stored - } - for _, tt := range invalid { - if m.CanTransition(string(tt.from), string(tt.to)) { - t.Errorf("invalid transition %s→%s: should not be allowed", tt.from, tt.to) - } - } -} - -func TestObjectStateMachine_SameStateSelfTransition(t *testing.T) { - m := NewObjectStateMachine() - - allStates := []model.ObjectState{ - model.ObjectStateCached, - model.ObjectStateUploading, - model.ObjectStateCommitting, - model.ObjectStateReplicating, - model.ObjectStateStored, - model.ObjectStateFailed, - model.ObjectStateCacheEvicted, - } - for _, s := range allStates { - if m.CanTransition(string(s), string(s)) { - t.Errorf("self-transition %s→%s: should not be allowed", s, s) - } - } -} - -func TestObjectStateMachine_NextStates(t *testing.T) { - m := NewObjectStateMachine() - - tests := []struct { - from model.ObjectState - expected []string - }{ - {model.ObjectStateCached, []string{"uploading"}}, - {model.ObjectStateUploading, []string{"committing", "failed", "replicating"}}, - {model.ObjectStateCommitting, []string{"failed", "replicating", "uploading"}}, - {model.ObjectStateReplicating, []string{"stored"}}, - {model.ObjectStateStored, []string{"cache_evicted"}}, - {model.ObjectStateFailed, []string{"uploading"}}, - {model.ObjectStateCacheEvicted, nil}, - } - for _, tt := range tests { - got := m.NextStates(string(tt.from)) - sort.Strings(got) - expected := tt.expected - sort.Strings(expected) - - if len(got) != len(expected) { - t.Errorf("NextStates(%s) = %v, want %v", tt.from, got, expected) - continue - } - for i := range got { - if got[i] != expected[i] { - t.Errorf("NextStates(%s) = %v, want %v", tt.from, got, expected) - break - } - } - } -} diff --git a/internal/state/object_states.go b/internal/state/object_states.go deleted file mode 100644 index dc67e6b..0000000 --- a/internal/state/object_states.go +++ /dev/null @@ -1,29 +0,0 @@ -package state - -import "github.com/strahe/synaps3/internal/model" - -// NewObjectStateMachine returns a state machine pre-configured with -// the standard object lifecycle transitions. -func NewObjectStateMachine() *Machine { - m := New() - m.Register( - // Happy path - Transition{From: string(model.ObjectStateCached), To: string(model.ObjectStateUploading)}, - Transition{From: string(model.ObjectStateUploading), To: string(model.ObjectStateCommitting)}, - // An assigned peer can commit before the original ingress copy resumes. - Transition{From: string(model.ObjectStateUploading), To: string(model.ObjectStateReplicating)}, - Transition{From: string(model.ObjectStateCommitting), To: string(model.ObjectStateReplicating)}, - Transition{From: string(model.ObjectStateReplicating), To: string(model.ObjectStateStored)}, - Transition{From: string(model.ObjectStateStored), To: string(model.ObjectStateCacheEvicted)}, - - // Failure - Transition{From: string(model.ObjectStateUploading), To: string(model.ObjectStateFailed)}, - Transition{From: string(model.ObjectStateCommitting), To: string(model.ObjectStateFailed)}, - - // Retry - Transition{From: string(model.ObjectStateFailed), To: string(model.ObjectStateUploading)}, - // Recovery can reassign an ingress attempt that reached committing without a piece. - Transition{From: string(model.ObjectStateCommitting), To: string(model.ObjectStateUploading)}, - ) - return m -} diff --git a/internal/state/transition.go b/internal/state/transition.go deleted file mode 100644 index e4fd626..0000000 --- a/internal/state/transition.go +++ /dev/null @@ -1,43 +0,0 @@ -package state - -import ( - "context" - "fmt" - - "github.com/strahe/synaps3/internal/model" -) - -// StateUpdater abstracts the DB compare-and-set for state transitions. -type StateUpdater interface { - UpdateVersionState(ctx context.Context, versionID string, from, to model.ObjectState) error - UpdateVersionStateToFailed(ctx context.Context, versionID string, from model.ObjectState, lastError string) error -} - -// TransitionState validates a state change via the FSM, then atomically -// updates the database. This is the single entry point for all object -// state changes, providing dual protection: -// 1. FSM validation (catches programming errors early) -// 2. DB compare-and-set (prevents races at the data level) -// -// When transitioning to ObjectStateFailed, use TransitionToFailed instead -// to record the phase that failed and the error message. -func TransitionState(ctx context.Context, m *Machine, u StateUpdater, versionID string, from, to model.ObjectState) error { - if to == model.ObjectStateFailed { - return fmt.Errorf("use TransitionToFailed for failure transitions to record FailedAtState") - } - if err := m.Validate(string(from), string(to)); err != nil { - return fmt.Errorf("state transition rejected: %w", err) - } - return u.UpdateVersionState(ctx, versionID, from, to) -} - -// TransitionToFailed validates the transition to ObjectStateFailed via the -// FSM, then atomically updates the database, recording the source state -// (FailedAtState) and the error message. Workers use FailedAtState to -// determine the correct retry target. -func TransitionToFailed(ctx context.Context, m *Machine, u StateUpdater, versionID string, from model.ObjectState, lastError string) error { - if err := m.Validate(string(from), string(model.ObjectStateFailed)); err != nil { - return fmt.Errorf("state transition rejected: %w", err) - } - return u.UpdateVersionStateToFailed(ctx, versionID, from, lastError) -} diff --git a/internal/state/transition_test.go b/internal/state/transition_test.go deleted file mode 100644 index f31f35b..0000000 --- a/internal/state/transition_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package state - -import ( - "context" - "errors" - "testing" - - "github.com/strahe/synaps3/internal/model" -) - -// mockStateUpdater implements StateUpdater for testing. -type mockStateUpdater struct { - updateStateCalled bool - updateFailedCalled bool - lastVersionID string - lastFrom, lastTo model.ObjectState - lastError string - returnErr error -} - -func (m *mockStateUpdater) UpdateVersionState(_ context.Context, versionID string, from, to model.ObjectState) error { - m.updateStateCalled = true - m.lastVersionID = versionID - m.lastFrom = from - m.lastTo = to - return m.returnErr -} - -func (m *mockStateUpdater) UpdateVersionStateToFailed(_ context.Context, versionID string, from model.ObjectState, lastError string) error { - m.updateFailedCalled = true - m.lastVersionID = versionID - m.lastFrom = from - m.lastError = lastError - return m.returnErr -} - -func TestTransitionState_ValidTransition(t *testing.T) { - m := NewObjectStateMachine() - u := &mockStateUpdater{} - ctx := context.Background() - - err := TransitionState(ctx, m, u, "version-1", model.ObjectStateCached, model.ObjectStateUploading) - if err != nil { - t.Fatalf("TransitionState: %v", err) - } - if !u.updateStateCalled { - t.Error("UpdateState was not called") - } - if u.lastVersionID != "version-1" || u.lastFrom != model.ObjectStateCached || u.lastTo != model.ObjectStateUploading { - t.Errorf("wrong args: version=%s from=%s to=%s", u.lastVersionID, u.lastFrom, u.lastTo) - } -} - -func TestTransitionState_InvalidTransition(t *testing.T) { - m := NewObjectStateMachine() - u := &mockStateUpdater{} - ctx := context.Background() - - err := TransitionState(ctx, m, u, "version-1", model.ObjectStateCached, model.ObjectStateStored) - if err == nil { - t.Fatal("TransitionState should have failed for invalid transition") - } - if u.updateStateCalled { - t.Error("UpdateState should not be called for invalid transition") - } -} - -func TestTransitionState_DBError(t *testing.T) { - m := NewObjectStateMachine() - dbErr := errors.New("db failure") - u := &mockStateUpdater{returnErr: dbErr} - ctx := context.Background() - - err := TransitionState(ctx, m, u, "version-1", model.ObjectStateCached, model.ObjectStateUploading) - if !errors.Is(err, dbErr) { - t.Errorf("expected db error, got: %v", err) - } -} - -func TestTransitionState_RejectsFailedTarget(t *testing.T) { - m := NewObjectStateMachine() - u := &mockStateUpdater{} - ctx := context.Background() - - err := TransitionState(ctx, m, u, "version-1", model.ObjectStateUploading, model.ObjectStateFailed) - if err == nil { - t.Fatal("TransitionState should reject →failed; use TransitionToFailed instead") - } - if u.updateStateCalled { - t.Error("UpdateState should not be called for →failed transition") - } -} - -func TestTransitionToFailed_Valid(t *testing.T) { - m := NewObjectStateMachine() - u := &mockStateUpdater{} - ctx := context.Background() - - err := TransitionToFailed(ctx, m, u, "version-1", model.ObjectStateUploading, "upload timeout") - if err != nil { - t.Fatalf("TransitionToFailed: %v", err) - } - if !u.updateFailedCalled { - t.Error("UpdateStateToFailed was not called") - } - if u.lastFrom != model.ObjectStateUploading { - t.Errorf("lastFrom = %s, want uploading", u.lastFrom) - } - if u.lastError != "upload timeout" { - t.Errorf("lastError = %q, want %q", u.lastError, "upload timeout") - } -} - -func TestTransitionToFailed_FromStoredRejected(t *testing.T) { - m := NewObjectStateMachine() - u := &mockStateUpdater{} - ctx := context.Background() - - err := TransitionToFailed(ctx, m, u, "version-1", model.ObjectStateStored, "eviction retries exhausted") - if err == nil { - t.Fatal("TransitionToFailed should reject stored→failed") - } - if u.updateFailedCalled { - t.Error("UpdateStateToFailed should not be called for stored→failed") - } -} - -func TestTransitionToFailed_InvalidSource(t *testing.T) { - m := NewObjectStateMachine() - u := &mockStateUpdater{} - ctx := context.Background() - - err := TransitionToFailed(ctx, m, u, "version-1", model.ObjectStateCached, "should not work") - if err == nil { - t.Fatal("TransitionToFailed should reject cached→failed") - } - if u.updateFailedCalled { - t.Error("UpdateStateToFailed should not be called for invalid transition") - } -} diff --git a/internal/storagecleanup/task.go b/internal/storagecleanup/task.go new file mode 100644 index 0000000..4c23dfa --- /dev/null +++ b/internal/storagecleanup/task.go @@ -0,0 +1,25 @@ +// Package storagecleanup owns durable cleanup task inputs. +package storagecleanup + +import ( + "errors" + "fmt" +) + +const TaskKeyPrefix = "storage-cleanup:" + +type Input struct { + ContentID int64 `json:"content_id"` + Generation int64 `json:"generation"` +} + +func TaskKey(contentID, generation int64) string { + return fmt.Sprintf("%s%d:%d", TaskKeyPrefix, contentID, generation) +} + +func ValidateInput(input Input) error { + if input.ContentID < 1 || input.Generation < 1 { + return errors.New("content_id and generation must be positive") + } + return nil +} diff --git a/internal/storagecommit/advancer.go b/internal/storagecommit/advancer.go index 16845e6..9a8e4c7 100644 --- a/internal/storagecommit/advancer.go +++ b/internal/storagecommit/advancer.go @@ -37,7 +37,7 @@ type Advancer struct { } type AdvanceInput struct { - Copy model.StorageUploadCopy + Copy model.StorageCopy Binding model.StorageDataSet Target synapse.DataSetTarget Pieces []storage.PieceInput @@ -49,11 +49,11 @@ type AdvanceInput struct { // not be reconstructed. It never submits or confirms provider work. func (a *Advancer) AdvanceUnavailable( ctx context.Context, - copyRow model.StorageUploadCopy, + copyRow model.StorageCopy, binding model.StorageDataSet, ) (AdvanceResult, error) { - if a == nil || a.Store == nil || copyRow.ID <= 0 || copyRow.UploadID <= 0 || copyRow.CopyIndex < 0 || - copyRow.StorageDataSetID == nil || *copyRow.StorageDataSetID != binding.ID || + if a == nil || a.Store == nil || copyRow.ID <= 0 || copyRow.ContentID <= 0 || copyRow.CopyIndex < 0 || + copyRow.StorageDataSetID != binding.ID || copyRow.CommitAttemptID == nil || *copyRow.CommitAttemptID == "" || copyRow.CommitAttemptedAt == nil { return AdvanceResult{}, errors.New("invalid unavailable storage commit input") } @@ -70,10 +70,10 @@ func (a *Advancer) AdvanceUnavailable( return AdvanceResult{State: AdvancePending, AttemptID: attemptID}, nil } return a.attention(ctx, CopyIdentity{ - StorageUploadCopyID: copyRow.ID, - UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, - StorageDataSetID: binding.ID, + StorageCopyID: copyRow.ID, + ContentID: copyRow.ContentID, + CopyIndex: copyRow.CopyIndex, + StorageDataSetID: binding.ID, }, attemptID, AttentionDataSetUnavailable, true) } @@ -81,19 +81,19 @@ func (a *Advancer) AdvanceUnavailable( // is terminal without requiring a provider context. func (a *Advancer) ReleaseTerminalReservation( ctx context.Context, - copyRow model.StorageUploadCopy, + copyRow model.StorageCopy, binding model.StorageDataSet, ) (AdvanceResult, error) { - if a == nil || a.Store == nil || copyRow.ID <= 0 || copyRow.UploadID <= 0 || copyRow.CopyIndex < 0 || - copyRow.StorageDataSetID == nil || *copyRow.StorageDataSetID != binding.ID || + if a == nil || a.Store == nil || copyRow.ID <= 0 || copyRow.ContentID <= 0 || copyRow.CopyIndex < 0 || + copyRow.StorageDataSetID != binding.ID || copyRow.CommitAttemptID == nil || *copyRow.CommitAttemptID == "" || copyRow.CommitAttemptedAt != nil { return AdvanceResult{}, errors.New("invalid terminal storage commit reservation") } return a.release(ctx, CopyIdentity{ - StorageUploadCopyID: copyRow.ID, - UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, - StorageDataSetID: binding.ID, + StorageCopyID: copyRow.ID, + ContentID: copyRow.ContentID, + CopyIndex: copyRow.CopyIndex, + StorageDataSetID: binding.ID, }, copyRow, ReleaseOwnerTerminal, true, true, false) } @@ -148,7 +148,7 @@ func (a *Advancer) Advance(ctx context.Context, input AdvanceInput) (AdvanceResu func (a *Advancer) submitReserved( ctx context.Context, input AdvanceInput, - copyRow model.StorageUploadCopy, + copyRow model.StorageCopy, ) (AdvanceResult, error) { if copyRow.CommitAttemptID == nil || *copyRow.CommitAttemptID == "" { return AdvanceResult{}, errors.New("reserved storage commit has no attempt token") @@ -268,7 +268,7 @@ type commitEvidenceError struct { func (a *Advancer) observe( ctx context.Context, input AdvanceInput, - copyRow model.StorageUploadCopy, + copyRow model.StorageCopy, ) (AdvanceResult, error) { identity := copyIdentity(input, false) attemptID := *copyRow.CommitAttemptID @@ -329,7 +329,7 @@ func (a *Advancer) observe( func (a *Advancer) observeTransaction( ctx context.Context, input AdvanceInput, - copyRow model.StorageUploadCopy, + copyRow model.StorageCopy, ) (AdvanceResult, error) { identity := copyIdentity(input, false) attemptID := *copyRow.CommitAttemptID @@ -376,7 +376,7 @@ func (a *Advancer) observeTransaction( AttemptID: attemptID, Confirmation: &storage.CommitResult{ TransactionID: transactionID, - ConfirmedTransactionID: result.ConfirmedTransactionID, + ConfirmedTransactionID: confirmedTransactionID(transactionID, result.ConfirmedTransactionID), DataSet: ref, PieceIDs: pieceIDs, }, @@ -396,7 +396,7 @@ func (a *Advancer) observeTransaction( func (a *Advancer) classifySDKStatus( ctx context.Context, identity CopyIdentity, - copyRow model.StorageUploadCopy, + copyRow model.StorageCopy, attemptID string, status *storage.CommitStatus, ) (AdvanceResult, error) { @@ -415,7 +415,7 @@ func (a *Advancer) classifySDKStatus( AttemptID: attemptID, Confirmation: &storage.CommitResult{ TransactionID: status.TransactionID, - ConfirmedTransactionID: status.ConfirmedTransactionID, + ConfirmedTransactionID: confirmedTransactionID(status.TransactionID, status.ConfirmedTransactionID), DataSet: *status.DataSet, PieceIDs: status.PieceIDs, }, @@ -430,10 +430,17 @@ func (a *Advancer) classifySDKStatus( } } +func confirmedTransactionID(transactionID, confirmedTransactionID string) string { + if confirmedTransactionID != "" { + return confirmedTransactionID + } + return transactionID +} + func (a *Advancer) pendingOrAttention( ctx context.Context, identity CopyIdentity, - copyRow model.StorageUploadCopy, + copyRow model.StorageCopy, attemptID string, ) (AdvanceResult, error) { return a.pendingOrAttentionWithCode( @@ -444,7 +451,7 @@ func (a *Advancer) pendingOrAttention( func (a *Advancer) pendingOrAttentionWithCode( ctx context.Context, identity CopyIdentity, - copyRow model.StorageUploadCopy, + copyRow model.StorageCopy, attemptID string, code AttentionCode, ) (AdvanceResult, error) { @@ -457,7 +464,7 @@ func (a *Advancer) pendingOrAttentionWithCode( func (a *Advancer) attentionForCopy( ctx context.Context, identity CopyIdentity, - copyRow model.StorageUploadCopy, + copyRow model.StorageCopy, attemptID string, code AttentionCode, keepObserving bool, @@ -469,7 +476,7 @@ func (a *Advancer) attentionForCopy( } func existingAttentionResult( - copyRow model.StorageUploadCopy, + copyRow model.StorageCopy, attemptID string, fallbackCode AttentionCode, keepObserving bool, @@ -537,7 +544,7 @@ func (a *Advancer) reset(ctx context.Context, identity CopyIdentity, attemptID s func (a *Advancer) release( ctx context.Context, identity CopyIdentity, - copyRow model.StorageUploadCopy, + copyRow model.StorageCopy, reason ReleaseReason, clearReadyAt bool, clearExtraData bool, @@ -550,6 +557,7 @@ func (a *Advancer) release( err := a.Store.ReleaseCommitAttempt(evidenceCtx, ReleaseInput{ Copy: identity, AttemptID: *copyRow.CommitAttemptID, + Reason: reason, KnownNotSubmitted: knownNotSubmitted, ClearReadyAt: clearReadyAt, ClearExtraData: clearExtraData, @@ -571,7 +579,7 @@ func (a *Advancer) release( func (a *Advancer) releaseUnavailable( ctx context.Context, identity CopyIdentity, - copyRow model.StorageUploadCopy, + copyRow model.StorageCopy, cause error, knownNotSubmitted bool, ) (AdvanceResult, error) { @@ -595,7 +603,7 @@ func dataSetRefusesWrites(err error) bool { func (a *Advancer) commitExtraData( ctx context.Context, target synapse.DataSetTarget, - copyRow model.StorageUploadCopy, + copyRow model.StorageCopy, pieces []storage.PieceInput, ) (string, error) { if copyRow.CommitExtraDataHex != nil && *copyRow.CommitExtraDataHex != "" { @@ -614,8 +622,8 @@ func (a *Advancer) commitExtraData( func (a *Advancer) validateInput(input AdvanceInput) error { if a == nil || a.Store == nil || input.Target == nil || input.Binding.ID <= 0 || input.Binding.DataSetID == nil || input.Binding.DataSetID.IsZero() || - input.Copy.ID <= 0 || input.Copy.UploadID <= 0 || input.Copy.CopyIndex < 0 || - input.Copy.StorageDataSetID == nil || *input.Copy.StorageDataSetID != input.Binding.ID || + input.Copy.ID <= 0 || input.Copy.ContentID <= 0 || input.Copy.CopyIndex < 0 || + input.Copy.StorageDataSetID != input.Binding.ID || len(input.Pieces) != 1 || !input.Pieces[0].PieceCID.Defined() { return errors.New("invalid storage commit advance input") } @@ -624,8 +632,8 @@ func (a *Advancer) validateInput(input AdvanceInput) error { func copyIdentity(input AdvanceInput, requireEligibleCopy bool) CopyIdentity { return CopyIdentity{ - StorageUploadCopyID: input.Copy.ID, - UploadID: input.Copy.UploadID, + StorageCopyID: input.Copy.ID, + ContentID: input.Copy.ContentID, CopyIndex: input.Copy.CopyIndex, StorageDataSetID: input.Binding.ID, RequireEligibleCopy: requireEligibleCopy, diff --git a/internal/storagecommit/advancer_test.go b/internal/storagecommit/advancer_test.go index a7332d5..3f408d9 100644 --- a/internal/storagecommit/advancer_test.go +++ b/internal/storagecommit/advancer_test.go @@ -58,11 +58,11 @@ func TestAdvancerPersistsFourSubmissionsBeforeConfirmationAndAdmitsFIFO(t *testi } return &storage.CommitStatus{ State: storage.CommitStateConfirmed, TransactionID: submission.TransactionID, - ConfirmedTransactionID: "0xconfirmed", DataSet: &dataSetRef, + DataSet: &dataSetRef, PieceIDs: []sdktypes.BigInt{sdktypes.NewBigInt(5001)}, }, nil } - advancer := storagecommit.Advancer{Store: repos.Uploads} + advancer := storagecommit.Advancer{Store: repos.Contents} for i := range 4 { result, err := advancer.Advance(t.Context(), storagecommit.AdvanceInput{ @@ -108,15 +108,27 @@ func TestAdvancerPersistsFourSubmissionsBeforeConfirmationAndAdmitsFIFO(t *testi if err != nil || settled.State != storagecommit.AdvanceConfirmed || settled.Confirmation == nil { t.Fatalf("settle first = %#v err=%v", settled, err) } + if settled.Confirmation.ConfirmedTransactionID != settled.Confirmation.TransactionID { + t.Fatalf("confirmed transaction = %q, want fallback %q", settled.Confirmation.ConfirmedTransactionID, settled.Confirmation.TransactionID) + } pieceID := idtypes.OnChainIDFromSDK(settled.Confirmation.PieceIDs[0]) - if err := repos.Uploads.MarkUploadCopyCommitted(t.Context(), repository.MarkUploadCopyCommittedInput{ - StorageUploadCopyID: copies[0].ID, UploadID: copies[0].UploadID, CopyIndex: 0, + settlement := repository.MarkUploadCopyCommittedInput{ + StorageCopyID: copies[0].ID, ContentID: copies[0].ContentID, CopyIndex: 0, PieceCID: pieceCID.String(), PieceID: &pieceID, RetrievalURL: target.PieceURL(pieceCID), CommitExtraDataHex: *first.CommitExtraDataHex, CommitTransactionID: settled.Confirmation.TransactionID, CommitAttemptID: settled.AttemptID, CommitConfirmedTransactionID: settled.Confirmation.ConfirmedTransactionID, - }); err != nil { + } + if err := repos.Contents.MarkUploadCopyCommitted(t.Context(), settlement); err != nil { t.Fatalf("settle first copy: %v", err) } + if err := repos.Contents.MarkUploadCopyCommitted(t.Context(), settlement); err != nil { + t.Fatalf("replay first copy settlement: %v", err) + } + conflictingSettlement := settlement + conflictingSettlement.CommitConfirmedTransactionID = "0xconflicting-confirmation" + if err := repos.Contents.MarkUploadCopyCommitted(t.Context(), conflictingSettlement); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("conflicting first copy settlement = %v, want ErrConflict", err) + } fifthPersisted = loadAdvancerCopy(t, repos, copies[4].ID) fifth, err = advancer.Advance(t.Context(), storagecommit.AdvanceInput{ @@ -136,13 +148,13 @@ func TestAdvancerOwnerTerminalReleasesUnattemptedReservationWithoutSDK(t *testin repos := repository.NewRepositories(db) binding, copies, _ := seedAdvancerCopies(t, db, 1) identity := advancerCopyIdentity(copies[0]) - if _, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ + if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ Copy: identity, AttemptID: "owner-terminal-attempt", }); err != nil { t.Fatalf("reserve: %v", err) } copyRow := loadAdvancerCopy(t, repos, copies[0].ID) - result, err := (&storagecommit.Advancer{Store: repos.Uploads}).ReleaseTerminalReservation( + result, err := (&storagecommit.Advancer{Store: repos.Contents}).ReleaseTerminalReservation( t.Context(), *copyRow, *binding, ) if err != nil || result.State != storagecommit.AdvanceReleased || result.ReleaseReason != storagecommit.ReleaseOwnerTerminal { @@ -159,14 +171,14 @@ func TestAdvancerOwnerTerminalClearsFIFOReadinessWithoutReservation(t *testing.T repos := repository.NewRepositories(db) binding, copies, pieceCID := seedAdvancerCopies(t, db, 5) for i := range 4 { - if _, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ + if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ Copy: advancerCopyIdentity(copies[i]), AttemptID: fmt.Sprintf("capacity-%d", i), }); err != nil { t.Fatalf("reserve capacity %d: %v", i, err) } } target := testutil.NewMockDataSetTarget(binding.ProviderID.SDK(), binding.DataSetID.SDK(), nil) - advancer := storagecommit.Advancer{Store: repos.Uploads} + advancer := storagecommit.Advancer{Store: repos.Contents} waiting, err := advancer.Advance(t.Context(), storagecommit.AdvanceInput{ Copy: copies[4], Binding: *binding, Target: target, Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, @@ -206,7 +218,7 @@ func TestAdvancerDataSetUnavailableSeparatesReservationFromAttempt(t *testing.T) return nil, nil } - result, err := (&storagecommit.Advancer{Store: repos.Uploads}).Advance(t.Context(), storagecommit.AdvanceInput{ + result, err := (&storagecommit.Advancer{Store: repos.Contents}).Advance(t.Context(), storagecommit.AdvanceInput{ Copy: copies[0], Binding: *binding, Target: target, Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, }) @@ -232,7 +244,7 @@ func TestAdvancerDataSetUnavailableSeparatesReservationFromAttempt(t *testing.T) return nil, storage.ErrDataSetUnavailable } - result, err := (&storagecommit.Advancer{Store: repos.Uploads}).Advance(t.Context(), storagecommit.AdvanceInput{ + result, err := (&storagecommit.Advancer{Store: repos.Contents}).Advance(t.Context(), storagecommit.AdvanceInput{ Copy: copies[0], Binding: *binding, Target: target, Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, }) @@ -260,7 +272,7 @@ func TestAdvancerAmbiguousSubmitErrorRetainsAttemptFence(t *testing.T) { return nil, storage.ErrInvalidArgument } - result, err := (&storagecommit.Advancer{Store: repos.Uploads}).Advance(t.Context(), storagecommit.AdvanceInput{ + result, err := (&storagecommit.Advancer{Store: repos.Contents}).Advance(t.Context(), storagecommit.AdvanceInput{ Copy: copies[0], Binding: *binding, Target: target, Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, }) @@ -268,7 +280,7 @@ func TestAdvancerAmbiguousSubmitErrorRetainsAttemptFence(t *testing.T) { t.Fatalf("advance = %#v err=%v, want a fenced pending result carrying the submit error", result, err) } persisted := loadAdvancerCopy(t, repos, copies[0].ID) - if persisted.Status != model.StorageUploadCopyStatusCommitting || + if persisted.Status != model.StorageCopyStatusCommitting || persisted.CommitAttemptID == nil || persisted.CommitAttemptedAt == nil { t.Fatalf("ambiguous submit error lost attempt fence: %#v", persisted) } @@ -287,7 +299,7 @@ func TestAdvancerProviderUnavailableSubmitKeepsFenceAndSignalsDependency(t *test return nil, providerErr } - result, err := (&storagecommit.Advancer{Store: repos.Uploads}).Advance(t.Context(), storagecommit.AdvanceInput{ + result, err := (&storagecommit.Advancer{Store: repos.Contents}).Advance(t.Context(), storagecommit.AdvanceInput{ Copy: copies[0], Binding: *binding, Target: target, Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, }) @@ -309,7 +321,7 @@ func TestAdvancerReleasesReservationWhenMarkAttemptedFails(t *testing.T) { return []byte{0xab}, nil } injected := errors.New("injected mark-attempted failure") - store := &failingMarkAttemptedStore{Store: repos.Uploads, err: injected} + store := &failingMarkAttemptedStore{Store: repos.Contents, err: injected} result, err := (&storagecommit.Advancer{Store: store}).Advance(t.Context(), storagecommit.AdvanceInput{ Copy: copies[0], Binding: *binding, Target: target, @@ -337,7 +349,7 @@ func TestAdvancerSubmissionCallbackPreventsErrorBasedReset(t *testing.T) { return nil, storage.ErrInvalidArgument } - result, err := (&storagecommit.Advancer{Store: repos.Uploads}).Advance(t.Context(), storagecommit.AdvanceInput{ + result, err := (&storagecommit.Advancer{Store: repos.Contents}).Advance(t.Context(), storagecommit.AdvanceInput{ Copy: copies[0], Binding: *binding, Target: target, Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, }) @@ -374,7 +386,7 @@ func TestAdvancerSurfacesDurableSubmissionEvidenceFailure(t *testing.T) { }, nil } injected := errors.New("injected evidence write failure") - store := &failingCommitEvidenceStore{Store: repos.Uploads, err: injected} + store := &failingCommitEvidenceStore{Store: repos.Contents, err: injected} result, err := (&storagecommit.Advancer{Store: store}).Advance(t.Context(), storagecommit.AdvanceInput{ Copy: copies[0], Binding: *binding, Target: target, @@ -425,7 +437,7 @@ func TestAdvancerUnavailableConfirmationWaitsThenRecoversFromAttention(t *testin PieceIDs: []sdktypes.BigInt{sdktypes.NewBigInt(5001)}, }, nil } - advancer := storagecommit.Advancer{Store: repos.Uploads, Now: func() time.Time { return startedAt }} + advancer := storagecommit.Advancer{Store: repos.Contents, Now: func() time.Time { return startedAt }} result, err := advancer.Advance(t.Context(), storagecommit.AdvanceInput{ Copy: copies[0], Binding: *binding, Target: target, Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, @@ -465,8 +477,8 @@ func TestAdvancerUnavailableConfirmationWaitsThenRecoversFromAttention(t *testin t.Fatalf("recovered observation = %#v err=%v", result, err) } pieceID := idtypes.OnChainIDFromSDK(result.Confirmation.PieceIDs[0]) - if err := repos.Uploads.MarkUploadCopyCommitted(t.Context(), repository.MarkUploadCopyCommittedInput{ - StorageUploadCopyID: copies[0].ID, UploadID: copies[0].UploadID, CopyIndex: copies[0].CopyIndex, + if err := repos.Contents.MarkUploadCopyCommitted(t.Context(), repository.MarkUploadCopyCommittedInput{ + StorageCopyID: copies[0].ID, ContentID: copies[0].ContentID, CopyIndex: copies[0].CopyIndex, PieceCID: pieceCID.String(), PieceID: &pieceID, RetrievalURL: target.PieceURL(pieceCID), CommitExtraDataHex: *copyRow.CommitExtraDataHex, CommitTransactionID: result.Confirmation.TransactionID, CommitAttemptID: result.AttemptID, CommitConfirmedTransactionID: result.Confirmation.ConfirmedTransactionID, @@ -485,19 +497,19 @@ func TestAdvancerUnavailableContextMakesAttemptVisibleAfterThreshold(t *testing. binding, copies, _ := seedAdvancerCopies(t, db, 1) identity := advancerCopyIdentity(copies[0]) startedAt := time.Date(2026, time.August, 30, 12, 0, 0, 0, time.UTC) - if _, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ + if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ Copy: identity, AttemptID: "unavailable-context", Now: startedAt, }); err != nil { t.Fatalf("reserve: %v", err) } - if _, err := repos.Uploads.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ + if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ Copy: identity, AttemptID: "unavailable-context", ExtraDataHex: "abcd", Now: startedAt, }); err != nil { t.Fatalf("mark attempted: %v", err) } copyRow := loadAdvancerCopy(t, repos, copies[0].ID) advancer := storagecommit.Advancer{ - Store: repos.Uploads, + Store: repos.Contents, Now: func() time.Time { return startedAt.Add(14 * time.Minute) }, } @@ -518,24 +530,50 @@ func TestAdvancerUnavailableContextMakesAttemptVisibleAfterThreshold(t *testing. } } +func TestCommitAttemptIdempotencyRejectsChangedExtraData(t *testing.T) { + db := testutil.NewTestDB(t) + repos := repository.NewRepositories(db) + _, copies, _ := seedAdvancerCopies(t, db, 1) + identity := advancerCopyIdentity(copies[0]) + if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ + Copy: identity, AttemptID: "immutable-extra-data", + }); err != nil { + t.Fatalf("reserve: %v", err) + } + if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ + Copy: identity, AttemptID: "immutable-extra-data", ExtraDataHex: "abcd", + }); err != nil { + t.Fatalf("mark attempted: %v", err) + } + if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ + Copy: identity, AttemptID: "immutable-extra-data", ExtraDataHex: "beef", + }); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("changed extra data error = %v, want ErrConflict", err) + } + persisted := loadAdvancerCopy(t, repos, copies[0].ID) + if persisted.CommitExtraDataHex == nil || *persisted.CommitExtraDataHex != "abcd" { + t.Fatalf("persisted extra data = %v, want immutable abcd", persisted.CommitExtraDataHex) + } +} + func TestAdvancerUnavailableContextPreservesCancellationAndUnknownAttention(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) binding, copies, _ := seedAdvancerCopies(t, db, 1) identity := advancerCopyIdentity(copies[0]) startedAt := time.Date(2026, time.August, 30, 12, 0, 0, 0, time.UTC) - if _, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ + if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ Copy: identity, AttemptID: "unavailable-canceled", Now: startedAt, }); err != nil { t.Fatalf("reserve: %v", err) } - if _, err := repos.Uploads.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ + if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ Copy: identity, AttemptID: "unavailable-canceled", ExtraDataHex: "abcd", Now: startedAt, }); err != nil { t.Fatalf("mark attempted: %v", err) } copyRow := loadAdvancerCopy(t, repos, copies[0].ID) - store := &attentionCountingStore{Store: repos.Uploads} + store := &attentionCountingStore{Store: repos.Contents} advancer := storagecommit.Advancer{ Store: store, Now: func() time.Time { return startedAt.Add(16 * time.Minute) }, @@ -549,10 +587,10 @@ func TestAdvancerUnavailableContextPreservesCancellationAndUnknownAttention(t *t attentionAt := startedAt.Add(15 * time.Minute) if _, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("commit_attention_code = ?", "future_attention_code"). - Set("commit_attention_at = ?", attentionAt). - Where("id = ?", copies[0].ID). + Model((*storagecommit.Attempt)(nil)). + Set("attention_code = ?", "future_attention_code"). + Set("attention_at = ?", attentionAt). + Where("attempt_id = ?", "unavailable-canceled"). Exec(t.Context()); err != nil { t.Fatalf("set future attention: %v", err) } @@ -609,7 +647,7 @@ func TestAdvancerFullSubmissionInvalidStatusKeepsStableAttentionAndRecovers(t *t return nil, nil } } - advancer := storagecommit.Advancer{Store: repos.Uploads} + advancer := storagecommit.Advancer{Store: repos.Contents} result, err := advancer.Advance(t.Context(), storagecommit.AdvanceInput{ Copy: copies[0], Binding: *binding, Target: target, Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, @@ -618,7 +656,7 @@ func TestAdvancerFullSubmissionInvalidStatusKeepsStableAttentionAndRecovers(t *t t.Fatalf("submit = %#v err=%v", result, err) } - store := &attentionCountingStore{Store: repos.Uploads} + store := &attentionCountingStore{Store: repos.Contents} advancer.Store = store copyRow := loadAdvancerCopy(t, repos, copies[0].ID) result, err = advancer.Advance(t.Context(), storagecommit.AdvanceInput{ @@ -642,9 +680,9 @@ func TestAdvancerFullSubmissionInvalidStatusKeepsStableAttentionAndRecovers(t *t } if _, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("commit_attention_code = ?", "future_attention_code"). - Where("id = ?", copies[0].ID). + Model((*storagecommit.Attempt)(nil)). + Set("attention_code = ?", "future_attention_code"). + Where("content_id = ? AND storage_data_set_id = ? AND resolved_at IS NULL", copies[0].ContentID, copies[0].StorageDataSetID). Exec(t.Context()); err != nil { t.Fatalf("set future attention: %v", err) } @@ -679,7 +717,7 @@ func TestAdvancerFullSubmissionInvalidStatusKeepsStableAttentionAndRecovers(t *t t.Fatalf("rejected recovery = %#v err=%v statusCalls=%d attentionWrites=%d", result, err, statusCalls, store.calls) } persisted := loadAdvancerCopy(t, repos, copies[0].ID) - if persisted.Status != model.StorageUploadCopyStatusPieceReady || persisted.CommitAttemptID != nil || + if persisted.Status != model.StorageCopyStatusPieceReady || persisted.CommitAttemptID != nil || persisted.CommitAttentionCode != nil || persisted.CommitAttentionAt != nil { t.Fatalf("copy after rejected recovery = %#v, want reset piece-ready copy", persisted) } @@ -690,12 +728,12 @@ func TestAdvancerAttemptOnlyUsesPieceStatusAsDiagnosticEvidence(t *testing.T) { repos := repository.NewRepositories(db) binding, copies, pieceCID := seedAdvancerCopies(t, db, 1) identity := advancerCopyIdentity(copies[0]) - if _, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ + if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ Copy: identity, AttemptID: "attempt-only", }); err != nil { t.Fatalf("reserve: %v", err) } - if _, err := repos.Uploads.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ + if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ Copy: identity, AttemptID: "attempt-only", ExtraDataHex: "abcd", }); err != nil { t.Fatalf("mark attempted: %v", err) @@ -708,7 +746,7 @@ func TestAdvancerAttemptOnlyUsesPieceStatusAsDiagnosticEvidence(t *testing.T) { return &storage.PieceStatus{Exists: true}, nil } - result, err := (&storagecommit.Advancer{Store: repos.Uploads}).Advance(t.Context(), storagecommit.AdvanceInput{ + result, err := (&storagecommit.Advancer{Store: repos.Contents}).Advance(t.Context(), storagecommit.AdvanceInput{ Copy: *copyRow, Binding: *binding, Target: target, Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, }) @@ -717,12 +755,12 @@ func TestAdvancerAttemptOnlyUsesPieceStatusAsDiagnosticEvidence(t *testing.T) { t.Fatalf("advance = %#v err=%v", result, err) } persisted := loadAdvancerCopy(t, repos, copies[0].ID) - if persisted.PieceID != nil || persisted.Status != model.StorageUploadCopyStatusCommitting || + if persisted.PieceID != nil || persisted.Status != model.StorageCopyStatusCommitting || persisted.CommitAttentionAt == nil || persisted.CommitAttentionCode == nil || *persisted.CommitAttentionCode != string(storagecommit.AttentionUnattributedPiece) { t.Fatalf("attempt-only evidence was incorrectly adopted: %#v", persisted) } - result, err = (&storagecommit.Advancer{Store: repos.Uploads}).Advance(t.Context(), storagecommit.AdvanceInput{ + result, err = (&storagecommit.Advancer{Store: repos.Contents}).Advance(t.Context(), storagecommit.AdvanceInput{ Copy: *persisted, Binding: *binding, Target: target, Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, }) @@ -731,31 +769,98 @@ func TestAdvancerAttemptOnlyUsesPieceStatusAsDiagnosticEvidence(t *testing.T) { } } +func TestReleaseCommitAttentionResumesFencedFailedTask(t *testing.T) { + db := testutil.NewTestDB(t) + repos := repository.NewRepositories(db) + _, copies, _ := seedAdvancerCopies(t, db, 1) + copyRow := copies[0] + taskRow, created, err := repos.Tasks.Enqueue(t.Context(), &model.Task{ + Type: model.TaskTypeStorageCommit, IdempotencyKey: "release-attention", InputVersion: 1, + Input: []byte(`{}`), InputHash: "release-attention", Status: model.TaskStatusPending, + ResumeMode: model.TaskResumeModeExecute, AvailableAt: time.Now(), + }) + if err != nil || !created { + t.Fatalf("enqueue commit task = %#v created=%v err=%v", taskRow, created, err) + } + generation, err := repos.Contents.NextCopyWorkGeneration(t.Context(), copyRow.ID) + if err != nil { + t.Fatalf("next copy generation: %v", err) + } + if err := repos.Contents.BindCopyTask(t.Context(), copyRow.ID, generation, taskRow.ID); err != nil { + t.Fatalf("bind commit task: %v", err) + } + identity := advancerCopyIdentity(copyRow) + const attemptID = "release-attention-attempt" + if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ + Copy: identity, AttemptID: attemptID, + }); err != nil { + t.Fatalf("reserve commit attempt: %v", err) + } + if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ + Copy: identity, AttemptID: attemptID, ExtraDataHex: "abcd", + }); err != nil { + t.Fatalf("mark commit attempted: %v", err) + } + if err := repos.Contents.MarkCommitAttention(t.Context(), storagecommit.AttentionInput{ + Copy: identity, AttemptID: attemptID, Code: storagecommit.AttentionAttemptOnlyAmbiguous, + }); err != nil { + t.Fatalf("mark commit attention: %v", err) + } + claimed, err := repos.Tasks.ClaimNext(t.Context(), time.Minute) + if err != nil || claimed == nil || claimed.ID != taskRow.ID { + t.Fatalf("claim commit task = %#v err=%v", claimed, err) + } + reason := string(storagecommit.AttentionAttemptOnlyAmbiguous) + message := "storage registration requires attention" + if err := repos.Tasks.Settle(t.Context(), claimed.ID, claimed.ClaimGeneration, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, + FailureReason: &reason, LastError: &message, + }); err != nil { + t.Fatalf("fail commit task: %v", err) + } + + if err := repos.Contents.ReleaseCommitAttention(t.Context(), storagecommit.ManualReleaseInput{ + CopyID: copyRow.ID, ExpectedAttemptID: attemptID, AcknowledgePossibleDuplicate: true, + }); err != nil { + t.Fatalf("release commit attention: %v", err) + } + resumed, err := repos.Tasks.GetByID(t.Context(), taskRow.ID) + if err != nil || resumed == nil || resumed.Status != model.TaskStatusPending || + resumed.ResumeMode != model.TaskResumeModeRecover || resumed.RetryCount != 0 { + t.Fatalf("resumed task = %#v err=%v", resumed, err) + } + persisted := loadAdvancerCopy(t, repos, copyRow.ID) + if persisted.Status != model.StorageCopyStatusPieceReady || persisted.CommitAttemptID != nil || + persisted.ActiveTaskID == nil || *persisted.ActiveTaskID != taskRow.ID { + t.Fatalf("released copy = %#v, want piece-ready copy fenced to resumed task", persisted) + } +} + func TestAdvancerTxOnlyEvidenceConfirmsWithoutPieceStatus(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) binding, copies, pieceCID := seedAdvancerCopies(t, db, 1) identity := advancerCopyIdentity(copies[0]) - if _, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ + if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ Copy: identity, AttemptID: "tx-only", }); err != nil { t.Fatalf("reserve: %v", err) } - if _, err := repos.Uploads.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ + if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ Copy: identity, AttemptID: "tx-only", ExtraDataHex: "abcd", }); err != nil { t.Fatalf("mark attempted: %v", err) } - if err := repos.Uploads.RecordCommitTransaction(t.Context(), storagecommit.EvidenceInput{ + if err := repos.Contents.RecordCommitTransaction(t.Context(), storagecommit.EvidenceInput{ Copy: identity, AttemptID: "tx-only", TransactionID: "0xtxonly", }); err != nil { t.Fatalf("record transaction: %v", err) } if _, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("commit_attention_code = ?", "future_attention_code"). - Set("commit_attention_at = ?", time.Now()). - Where("id = ?", copies[0].ID). + Model((*storagecommit.Attempt)(nil)). + Set("attention_code = ?", "future_attention_code"). + Set("attention_at = ?", time.Now()). + Where("attempt_id = ?", "tx-only"). Exec(t.Context()); err != nil { t.Fatalf("set future tx-only attention: %v", err) } @@ -771,16 +876,15 @@ func TestAdvancerTxOnlyEvidenceConfirmsWithoutPieceStatus(t *testing.T) { checkerCalls++ return synapse.PDPStatusResult{ State: synapse.PDPStatusConfirmed, ConfirmedPieceIDs: []string{"5001"}, - ConfirmedTransactionID: "0xconfirmedtx", }, nil }) - result, err := (&storagecommit.Advancer{Store: repos.Uploads, StatusChecker: checker}).Advance( + result, err := (&storagecommit.Advancer{Store: repos.Contents, StatusChecker: checker}).Advance( t.Context(), storagecommit.AdvanceInput{ Copy: *copyRow, Binding: *binding, Target: target, Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, }) if err != nil || result.State != storagecommit.AdvanceConfirmed || result.Confirmation == nil || - result.Confirmation.ConfirmedTransactionID != "0xconfirmedtx" || checkerCalls != 1 { + result.Confirmation.ConfirmedTransactionID != "0xtxonly" || checkerCalls != 1 { t.Fatalf("advance = %#v err=%v checkerCalls=%d", result, err, checkerCalls) } } @@ -790,17 +894,17 @@ func TestAdvancerTxOnlyMismatchNeedsAttention(t *testing.T) { repos := repository.NewRepositories(db) binding, copies, pieceCID := seedAdvancerCopies(t, db, 1) identity := advancerCopyIdentity(copies[0]) - if _, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ + if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ Copy: identity, AttemptID: "tx-only-mismatch", }); err != nil { t.Fatalf("reserve: %v", err) } - if _, err := repos.Uploads.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ + if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ Copy: identity, AttemptID: "tx-only-mismatch", ExtraDataHex: "abcd", }); err != nil { t.Fatalf("mark attempted: %v", err) } - if err := repos.Uploads.RecordCommitTransaction(t.Context(), storagecommit.EvidenceInput{ + if err := repos.Contents.RecordCommitTransaction(t.Context(), storagecommit.EvidenceInput{ Copy: identity, AttemptID: "tx-only-mismatch", TransactionID: "0xmismatch", }); err != nil { t.Fatalf("record transaction: %v", err) @@ -811,7 +915,7 @@ func TestAdvancerTxOnlyMismatchNeedsAttention(t *testing.T) { return synapse.PDPStatusResult{State: synapse.PDPStatusMismatch}, nil }) - result, err := (&storagecommit.Advancer{Store: repos.Uploads, StatusChecker: checker}).Advance( + result, err := (&storagecommit.Advancer{Store: repos.Contents, StatusChecker: checker}).Advance( t.Context(), storagecommit.AdvanceInput{ Copy: *copyRow, Binding: *binding, Target: target, Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, @@ -832,17 +936,17 @@ func TestAdvancerTxOnlyRequestTimeoutRemainsPending(t *testing.T) { repos := repository.NewRepositories(db) binding, copies, pieceCID := seedAdvancerCopies(t, db, 1) identity := advancerCopyIdentity(copies[0]) - if _, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ + if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ Copy: identity, AttemptID: "tx-only-timeout", }); err != nil { t.Fatalf("reserve: %v", err) } - if _, err := repos.Uploads.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ + if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ Copy: identity, AttemptID: "tx-only-timeout", ExtraDataHex: "abcd", }); err != nil { t.Fatalf("mark attempted: %v", err) } - if err := repos.Uploads.RecordCommitTransaction(t.Context(), storagecommit.EvidenceInput{ + if err := repos.Contents.RecordCommitTransaction(t.Context(), storagecommit.EvidenceInput{ Copy: identity, AttemptID: "tx-only-timeout", TransactionID: "0xtimeout", }); err != nil { t.Fatalf("record transaction: %v", err) @@ -863,7 +967,7 @@ func TestAdvancerTxOnlyRequestTimeoutRemainsPending(t *testing.T) { parentCtx := t.Context() startedAt := time.Now() result, err := (&storagecommit.Advancer{ - Store: repos.Uploads, StatusChecker: checker, RequestTimeout: requestTimeout, + Store: repos.Contents, StatusChecker: checker, RequestTimeout: requestTimeout, }).Advance(parentCtx, storagecommit.AdvanceInput{ Copy: *copyRow, Binding: *binding, Target: target, Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, @@ -889,12 +993,12 @@ func TestAdvancerCanceledObservationDoesNotWriteAttention(t *testing.T) { repos := repository.NewRepositories(db) binding, copies, pieceCID := seedAdvancerCopies(t, db, 1) identity := advancerCopyIdentity(copies[0]) - if _, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ + if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ Copy: identity, AttemptID: "canceled-observation", }); err != nil { t.Fatalf("reserve: %v", err) } - if _, err := repos.Uploads.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ + if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ Copy: identity, AttemptID: "canceled-observation", ExtraDataHex: "abcd", }); err != nil { t.Fatalf("mark attempted: %v", err) @@ -907,7 +1011,7 @@ func TestAdvancerCanceledObservationDoesNotWriteAttention(t *testing.T) { return nil, context.Canceled } - result, err := (&storagecommit.Advancer{Store: repos.Uploads}).Advance(ctx, storagecommit.AdvanceInput{ + result, err := (&storagecommit.Advancer{Store: repos.Contents}).Advance(ctx, storagecommit.AdvanceInput{ Copy: *copyRow, Binding: *binding, Target: target, Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, }) @@ -964,13 +1068,14 @@ func (f commitStatusCheckerFunc) GetAddPiecesStatus( return f(ctx, input) } -func seedAdvancerCopies(t *testing.T, db *bun.DB, count int) (*model.StorageDataSet, []model.StorageUploadCopy, cid.Cid) { +func seedAdvancerCopies(t *testing.T, db *bun.DB, count int) (*model.StorageDataSet, []model.StorageCopy, cid.Cid) { t.Helper() pieceCID := advancerTestCID(t) - bucket := &model.Bucket{Name: "storage-commit-advancer", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "storage-commit-advancer", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if _, err := db.NewInsert().Model(bucket).Exec(t.Context()); err != nil { t.Fatalf("insert bucket: %v", err) } + testutil.OpenBucketReplicaSlots(t, db, bucket.ID, bucket.DefaultCopies) providerID := idtypes.OnChainIDFromSDK(sdktypes.NewBigInt(101)) dataSetID := idtypes.OnChainIDFromSDK(sdktypes.NewBigInt(1001)) clientDataSetID := idtypes.OnChainIDFromSDK(sdktypes.NewBigInt(9001)) @@ -981,20 +1086,21 @@ func seedAdvancerCopies(t *testing.T, db *bun.DB, count int) (*model.StorageData if _, err := db.NewInsert().Model(binding).Exec(t.Context()); err != nil { t.Fatalf("insert data set: %v", err) } - copies := make([]model.StorageUploadCopy, 0, count) + copies := make([]model.StorageCopy, 0, count) for i := range count { piece := pieceCID.String() - upload := &model.StorageUpload{ - BucketID: bucket.ID, ContentSize: 1, Checksum: fmt.Sprintf("checksum-%d", i), - Status: model.StorageUploadStatusRunning, PieceCID: &piece, RequestedCopies: 1, + upload := &model.StorageContent{ + BucketID: bucket.ID, + ContentSize: 1, Checksum: testutil.StorageChecksum(fmt.Sprintf("checksum-%d", i)), PieceCID: &piece, RequestedCopies: 1, } if _, err := db.NewInsert().Model(upload).Exec(t.Context()); err != nil { t.Fatalf("insert upload %d: %v", i, err) } - copyRow := model.StorageUploadCopy{ - UploadID: upload.ID, CopyIndex: 0, ProviderID: &providerID, + copyRow := model.StorageCopy{ + ContentID: upload.ID, BucketID: bucket.ID, ContentSize: upload.ContentSize, + CopyIndex: 0, ProviderID: providerID, TransferMethod: model.StorageCopyTransferMethodPeerPull, - Status: model.StorageUploadCopyStatusPieceReady, StorageDataSetID: &binding.ID, + Status: model.StorageCopyStatusPieceReady, StorageDataSetID: binding.ID, } if _, err := db.NewInsert().Model(©Row).Exec(t.Context()); err != nil { t.Fatalf("insert copy %d: %v", i, err) @@ -1004,19 +1110,19 @@ func seedAdvancerCopies(t *testing.T, db *bun.DB, count int) (*model.StorageData return binding, copies, pieceCID } -func loadAdvancerCopy(t *testing.T, repos *repository.Repositories, copyID int64) *model.StorageUploadCopy { +func loadAdvancerCopy(t *testing.T, repos *repository.Repositories, copyID int64) *model.StorageCopy { t.Helper() - copyRow, err := repos.Uploads.GetUploadCopyByID(t.Context(), copyID) + copyRow, err := repos.Contents.GetUploadCopyByID(t.Context(), copyID) if err != nil { t.Fatalf("load copy %d: %v", copyID, err) } return copyRow } -func advancerCopyIdentity(copyRow model.StorageUploadCopy) storagecommit.CopyIdentity { +func advancerCopyIdentity(copyRow model.StorageCopy) storagecommit.CopyIdentity { return storagecommit.CopyIdentity{ - StorageUploadCopyID: copyRow.ID, UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, StorageDataSetID: *copyRow.StorageDataSetID, + StorageCopyID: copyRow.ID, ContentID: copyRow.ContentID, + CopyIndex: copyRow.CopyIndex, StorageDataSetID: copyRow.StorageDataSetID, } } @@ -1048,7 +1154,7 @@ func TestAdvancerWriteBlockedDataSetReleasesAttemptWithCause(t *testing.T) { return nil, writeBlocked() } - result, err := (&storagecommit.Advancer{Store: repos.Uploads}).Advance(t.Context(), storagecommit.AdvanceInput{ + result, err := (&storagecommit.Advancer{Store: repos.Contents}).Advance(t.Context(), storagecommit.AdvanceInput{ Copy: copies[0], Binding: *binding, Target: target, Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, }) @@ -1060,7 +1166,7 @@ func TestAdvancerWriteBlockedDataSetReleasesAttemptWithCause(t *testing.T) { t.Fatalf("release cause = %v, want the payment terminated error", result.Cause) } persisted := loadAdvancerCopy(t, repos, copies[0].ID) - if persisted.Status != model.StorageUploadCopyStatusPieceReady || persisted.CommitAttemptID != nil || + if persisted.Status != model.StorageCopyStatusPieceReady || persisted.CommitAttemptID != nil || persisted.CommitAttemptedAt != nil || persisted.CommitReadyAt != nil || persisted.CommitExtraDataHex != nil { t.Fatalf("write-blocked release retained commit state: %#v", persisted) @@ -1080,7 +1186,7 @@ func TestAdvancerWriteBlockedDataSetReleasesAttemptWithCause(t *testing.T) { return nil, writeBlocked() } - result, _ := (&storagecommit.Advancer{Store: repos.Uploads}).Advance(t.Context(), storagecommit.AdvanceInput{ + result, _ := (&storagecommit.Advancer{Store: repos.Contents}).Advance(t.Context(), storagecommit.AdvanceInput{ Copy: copies[0], Binding: *binding, Target: target, Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, }) @@ -1088,7 +1194,7 @@ func TestAdvancerWriteBlockedDataSetReleasesAttemptWithCause(t *testing.T) { t.Fatalf("advance = %#v, want the recorded transaction to refuse the release", result) } persisted := loadAdvancerCopy(t, repos, copies[0].ID) - if persisted.Status != model.StorageUploadCopyStatusCommitting || persisted.CommitAttemptID == nil || + if persisted.Status != model.StorageCopyStatusCommitting || persisted.CommitAttemptID == nil || persisted.CommitAttemptedAt == nil || persisted.CommitTransactionID == nil || *persisted.CommitTransactionID != "0xwriteblocked" { t.Fatalf("write-blocked release dropped submitted evidence: %#v", persisted) @@ -1097,20 +1203,20 @@ func TestAdvancerWriteBlockedDataSetReleasesAttemptWithCause(t *testing.T) { } func TestAdvancerUnreadableSubmissionFallsBackToTransactionEvidence(t *testing.T) { - seed := func(t *testing.T, db *bun.DB, repos *repository.Repositories, copyRow model.StorageUploadCopy, keepTransaction bool) { + seed := func(t *testing.T, repos *repository.Repositories, copyRow model.StorageCopy) { t.Helper() identity := advancerCopyIdentity(copyRow) - if _, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ + if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ Copy: identity, AttemptID: "unreadable", }); err != nil { t.Fatalf("reserve: %v", err) } - if _, err := repos.Uploads.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ + if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ Copy: identity, AttemptID: "unreadable", ExtraDataHex: "abcd", }); err != nil { t.Fatalf("mark attempted: %v", err) } - if err := repos.Uploads.RecordCommitSubmission(t.Context(), storagecommit.EvidenceInput{ + if err := repos.Contents.RecordCommitSubmission(t.Context(), storagecommit.EvidenceInput{ Copy: identity, AttemptID: "unreadable", TransactionID: "0xunreadable", // A future envelope version stands in for any submission this build can // no longer decode, such as an SDK that renamed a field. @@ -1118,23 +1224,13 @@ func TestAdvancerUnreadableSubmissionFallsBackToTransactionEvidence(t *testing.T }); err != nil { t.Fatalf("record submission: %v", err) } - if keepTransaction { - return - } - if _, err := db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("commit_transaction_id = NULL"). - Where("id = ?", copyRow.ID). - Exec(t.Context()); err != nil { - t.Fatalf("clear transaction evidence: %v", err) - } } t.Run("confirms through the recorded transaction", func(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) binding, copies, pieceCID := seedAdvancerCopies(t, db, 1) - seed(t, db, repos, copies[0], true) + seed(t, repos, copies[0]) target := testutil.NewMockDataSetTarget(binding.ProviderID.SDK(), binding.DataSetID.SDK(), nil) target.ClientDataSetIDValue = sdktypes.NewBigInt(9001) target.GetCommitStatusFunc = func(context.Context, storage.CommitSubmission) (*storage.CommitStatus, error) { @@ -1148,7 +1244,7 @@ func TestAdvancerUnreadableSubmissionFallsBackToTransactionEvidence(t *testing.T }, nil }) - result, err := (&storagecommit.Advancer{Store: repos.Uploads, StatusChecker: checker}).Advance( + result, err := (&storagecommit.Advancer{Store: repos.Contents, StatusChecker: checker}).Advance( t.Context(), storagecommit.AdvanceInput{ Copy: *loadAdvancerCopy(t, repos, copies[0].ID), Binding: *binding, Target: target, Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, @@ -1159,21 +1255,17 @@ func TestAdvancerUnreadableSubmissionFallsBackToTransactionEvidence(t *testing.T } }) - t.Run("needs attention without transaction evidence", func(t *testing.T) { + t.Run("rejects submission without transaction evidence", func(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) - binding, copies, pieceCID := seedAdvancerCopies(t, db, 1) - seed(t, db, repos, copies[0], false) - target := testutil.NewMockDataSetTarget(binding.ProviderID.SDK(), binding.DataSetID.SDK(), nil) - - result, err := (&storagecommit.Advancer{Store: repos.Uploads}).Advance( - t.Context(), storagecommit.AdvanceInput{ - Copy: *loadAdvancerCopy(t, repos, copies[0].ID), Binding: *binding, Target: target, - Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, - }) - if err != nil || result.State != storagecommit.AdvanceNeedsAttention || - result.AttentionCode != storagecommit.AttentionInvalidSubmission { - t.Fatalf("advance = %#v err=%v, want invalid submission attention", result, err) + _, copies, _ := seedAdvancerCopies(t, db, 1) + seed(t, repos, copies[0]) + if _, err := db.NewUpdate(). + Model((*storagecommit.Attempt)(nil)). + Set("transaction_id = NULL"). + Where("attempt_id = ?", "unreadable"). + Exec(t.Context()); err == nil { + t.Fatal("commit ledger accepted submission evidence without a transaction") } }) } diff --git a/internal/storagecommit/attempt.go b/internal/storagecommit/attempt.go new file mode 100644 index 0000000..ceaf999 --- /dev/null +++ b/internal/storagecommit/attempt.go @@ -0,0 +1,62 @@ +package storagecommit + +import ( + "context" + "time" + + "github.com/uptrace/bun" +) + +// AttemptStatus is the durable lifecycle of one possible external commit +// effect. Terminal attempts remain as evidence after the copy is re-queued. +type AttemptStatus string + +const ( + AttemptStatusReserved AttemptStatus = "reserved" + AttemptStatusAttempted AttemptStatus = "attempted" + AttemptStatusConfirmed AttemptStatus = "confirmed" + AttemptStatusReleased AttemptStatus = "released" + AttemptStatusRejected AttemptStatus = "rejected" +) + +// Attempt is the append-preserving ledger for storage commit side effects. +// ContentID and StorageDataSetID are the stable business key of the bound copy. +type Attempt struct { + bun.BaseModel `bun:"table:storage_commit_attempts"` + + AttemptID string `bun:"attempt_id,type:text,pk"` + ContentID int64 `bun:"content_id,notnull"` + StorageDataSetID int64 `bun:"storage_data_set_id,notnull"` + Status AttemptStatus `bun:"status,type:text,notnull,default:'reserved'"` + ExtraDataHex *string `bun:"extra_data_hex,type:text,nullzero"` + TransactionID *string `bun:"transaction_id,type:text,nullzero"` + SubmissionJSON *string `bun:"submission_json,type:text,nullzero"` + ConfirmedTransactionID *string `bun:"confirmed_transaction_id,type:text,nullzero"` + AttentionCode *string `bun:"attention_code,type:text,nullzero"` + AttentionAt *time.Time `bun:"attention_at,nullzero"` + ReleaseReason *string `bun:"release_reason,type:text,nullzero"` + LastError *string `bun:"last_error,type:text,nullzero"` + AttemptedAt *time.Time `bun:"attempted_at,nullzero"` + ResolvedAt *time.Time `bun:"resolved_at,nullzero"` + CreatedAt time.Time `bun:"created_at,nullzero,notnull"` + UpdatedAt time.Time `bun:"updated_at,nullzero,notnull"` +} + +var _ bun.BeforeAppendModelHook = (*Attempt)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (a *Attempt) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if a.CreatedAt.IsZero() { + a.CreatedAt = now + } + if a.UpdatedAt.IsZero() { + a.UpdatedAt = now + } + return nil +} diff --git a/internal/storagecommit/attention.go b/internal/storagecommit/attention.go index 7710d3e..956549b 100644 --- a/internal/storagecommit/attention.go +++ b/internal/storagecommit/attention.go @@ -4,7 +4,7 @@ import "time" type AttentionRecord struct { CopyID int64 - UploadID int64 + ContentID int64 CopyIndex int DataSetRowID int64 ProviderID string diff --git a/internal/storagecommit/result.go b/internal/storagecommit/result.go index d8dcc1c..d1fe8f0 100644 --- a/internal/storagecommit/result.go +++ b/internal/storagecommit/result.go @@ -14,12 +14,24 @@ const ( AdvanceNeedsAttention AdvanceState = "needs_attention" ) +// Valid reports whether the application can write this release reason. The +// database keeps the column open so newer binaries can add reasons safely. +func (r ReleaseReason) Valid() bool { + switch r { + case ReleaseBeforeSubmitCanceled, ReleaseDataSetUnavailable, ReleaseOwnerTerminal, ReleaseManualDuplicateAck: + return true + default: + return false + } +} + type ReleaseReason string const ( ReleaseBeforeSubmitCanceled ReleaseReason = "before_submit_canceled" ReleaseDataSetUnavailable ReleaseReason = "data_set_unavailable" ReleaseOwnerTerminal ReleaseReason = "owner_terminal" + ReleaseManualDuplicateAck ReleaseReason = "manual_duplicate_acknowledgement" ) type AdvanceResult struct { diff --git a/internal/storagecommit/store.go b/internal/storagecommit/store.go index 22a2e5a..34a8ffb 100644 --- a/internal/storagecommit/store.go +++ b/internal/storagecommit/store.go @@ -10,8 +10,8 @@ import ( const MaxActiveAttemptsPerDataSet = 4 type CopyIdentity struct { - StorageUploadCopyID int64 - UploadID int64 + StorageCopyID int64 + ContentID int64 CopyIndex int StorageDataSetID int64 RequireEligibleCopy bool @@ -32,7 +32,7 @@ type ReserveInput struct { type ReserveResult struct { State ReservationState - Copy model.StorageUploadCopy + Copy model.StorageCopy // AttentionHeld counts the data set's active attempts already flagged for // operator attention. It is set only when capacity turned the reservation // away, so a waiting result with zero here is queued behind work that is @@ -49,7 +49,7 @@ type AttemptInput struct { type AttemptResult struct { Entered bool - Copy model.StorageUploadCopy + Copy model.StorageCopy } type EvidenceInput struct { @@ -77,6 +77,7 @@ type ResetInput struct { type ReleaseInput struct { Copy CopyIdentity AttemptID string + Reason ReleaseReason KnownNotSubmitted bool ClearReadyAt bool ClearExtraData bool diff --git a/internal/storagepipeline/task.go b/internal/storagepipeline/task.go new file mode 100644 index 0000000..44ecdd7 --- /dev/null +++ b/internal/storagepipeline/task.go @@ -0,0 +1,98 @@ +// Package storagepipeline owns the workflow-neutral storage task inputs. +package storagepipeline + +import ( + "errors" + "fmt" + "strconv" +) + +const ( + UploadPlanKeyPrefix = "upload-plan:" + DataSetEnsureKeyPrefix = "storage-dataset-ensure:" + TransferPlanKeyPrefix = "storage-transfer-plan:" + StoreKeyPrefix = "storage-store:" + PullKeyPrefix = "storage-pull:" + CommitCoordinateKeyPrefix = "storage-commit-coordinate:" + CommitKeyPrefix = "storage-commit:" +) + +// UploadPlanInput names the content to ingest. Ingest is a property of the +// bytes, so two versions of identical content share one plan. +type UploadPlanInput struct { + ContentID int64 `json:"content_id"` +} + +type DataSetInput struct { + DataSetID int64 `json:"data_set_id"` +} + +type DataSetGenerationInput struct { + DataSetID int64 `json:"data_set_id"` + Generation int64 `json:"generation"` +} + +type CopyGenerationInput struct { + CopyID int64 `json:"copy_id"` + Generation int64 `json:"generation"` +} + +func UploadPlanKey(contentID int64) string { + return UploadPlanKeyPrefix + strconv.FormatInt(contentID, 10) +} + +func DataSetEnsureKey(dataSetID int64) string { + return DataSetEnsureKeyPrefix + strconv.FormatInt(dataSetID, 10) +} + +func TransferPlanKey(copyID, generation int64) string { + return copyGenerationKey(TransferPlanKeyPrefix, copyID, generation) +} + +func StoreKey(copyID, generation int64) string { + return copyGenerationKey(StoreKeyPrefix, copyID, generation) +} + +func PullKey(copyID, generation int64) string { + return copyGenerationKey(PullKeyPrefix, copyID, generation) +} + +func CommitCoordinateKey(copyID, generation int64) string { + return copyGenerationKey(CommitCoordinateKeyPrefix, copyID, generation) +} + +func CommitKey(copyID, generation int64) string { + return copyGenerationKey(CommitKeyPrefix, copyID, generation) +} + +func ValidateUploadPlanInput(input UploadPlanInput) error { + if input.ContentID < 1 { + return errors.New("content_id must be positive") + } + return nil +} + +func ValidateDataSetInput(input DataSetInput) error { + if input.DataSetID < 1 { + return errors.New("data_set_id must be positive") + } + return nil +} + +func ValidateDataSetGenerationInput(input DataSetGenerationInput) error { + if input.DataSetID < 1 || input.Generation < 1 { + return errors.New("data_set_id and generation must be positive") + } + return nil +} + +func ValidateCopyGenerationInput(input CopyGenerationInput) error { + if input.CopyID < 1 || input.Generation < 1 { + return errors.New("copy_id and generation must be positive") + } + return nil +} + +func copyGenerationKey(prefix string, id, generation int64) string { + return fmt.Sprintf("%s%d:%d", prefix, id, generation) +} diff --git a/internal/storagepull/record.go b/internal/storagepull/record.go new file mode 100644 index 0000000..64778c2 --- /dev/null +++ b/internal/storagepull/record.go @@ -0,0 +1,70 @@ +// Package storagepull owns the ledger of provider-side copy attempts. A pull +// asks a target provider to fetch one piece from a source provider. The target +// request is idempotent through its commit extra data, while AttemptID identifies +// the internal ledger row across recovery. +package storagepull + +import ( + "context" + "time" + + "github.com/strahe/synaps3/internal/types" + "github.com/uptrace/bun" +) + +// AttemptStatus is deliberately two-valued. A commit needs five states because +// its effect is an on-chain transaction that can hang and cost gas; a pull's +// effect is a transfer request the target can be asked about directly, and +// success is recorded on the copy reaching piece_ready. Termination is +// resolved_at, so a resolved attempt is a success and a resolved abandoned one +// is a request nobody will observe again. +type AttemptStatus string + +const ( + AttemptStatusAttempted AttemptStatus = "attempted" + AttemptStatusAbandoned AttemptStatus = "abandoned" +) + +// Attempt is one request sent to a target provider. Every source field is NOT +// NULL: an attempt that exists is fully identified, which is what the old +// all-or-nothing check constraint tried to express across nullable columns. +type Attempt struct { + bun.BaseModel `bun:"table:storage_pull_attempts,alias:storage_pull_attempt"` + + AttemptID string `bun:"type:text,pk"` + ContentID int64 `bun:",notnull"` + StorageDataSetID int64 `bun:",notnull"` + Status AttemptStatus `bun:"type:text,notnull"` + // The source the piece was pulled from, recorded before the request is sent + // so recovery never invents a different one. + SourceProviderID types.OnChainID `bun:"type:text,notnull"` + SourceDataSetID types.OnChainID `bun:"type:text,notnull"` + SourcePieceID types.OnChainID `bun:"type:text,notnull"` + // SourcePieceCID names the piece itself, distinct from the content's own CID. + SourcePieceCID string `bun:"type:text,notnull"` + SourceRetrievalURL string `bun:"type:text,notnull"` + LastError *string `bun:"type:text,nullzero"` + AttemptedAt time.Time `bun:",nullzero,notnull"` + ResolvedAt *time.Time `bun:",nullzero"` + CreatedAt time.Time `bun:",nullzero,notnull"` + UpdatedAt time.Time `bun:",nullzero,notnull"` +} + +var _ bun.BeforeAppendModelHook = (*Attempt)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (a *Attempt) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if a.CreatedAt.IsZero() { + a.CreatedAt = now + } + if a.UpdatedAt.IsZero() { + a.UpdatedAt = now + } + return nil +} diff --git a/internal/storagereplacement/codes.go b/internal/storagereplacement/codes.go index de39535..8604cac 100644 --- a/internal/storagereplacement/codes.go +++ b/internal/storagereplacement/codes.go @@ -15,10 +15,6 @@ const ( CodeTargetUnavailable = "replacement_target_unavailable" CodeIdempotencyConflict = "replacement_idempotency_conflict" CodeSourceNotCurrent = "replacement_source_not_current" - - // CodeTaskRetryUnsupported is returned by the generic exhausted-task retry - // endpoint when the task belongs to a replacement. - CodeTaskRetryUnsupported = "replacement_task_retry_unsupported" ) // Code maps a replacement error to its stable API code. It returns an empty diff --git a/internal/storagereplacement/record.go b/internal/storagereplacement/record.go index a2c8085..7a2f6c4 100644 --- a/internal/storagereplacement/record.go +++ b/internal/storagereplacement/record.go @@ -1,6 +1,7 @@ package storagereplacement import ( + "context" "time" "github.com/strahe/synaps3/internal/types" @@ -14,48 +15,70 @@ import ( type Replacement struct { bun.BaseModel `bun:"table:storage_replacements,alias:storage_replacement"` - ID int64 `bun:",pk,autoincrement"` + ID int64 `bun:",pk,autoincrement,identity"` BucketID int64 `bun:",notnull"` // CopyIndex is the logical replica slot both generations belong to. - CopyIndex int `bun:",notnull"` + CopyIndex int `bun:"type:integer,notnull"` SourceDataSetID int64 `bun:",notnull"` TargetDataSetID int64 `bun:",notnull"` - SelectionMode SelectionMode `bun:",notnull"` + SelectionMode SelectionMode `bun:"type:text,notnull"` RequestedProviderID *types.OnChainID `bun:"type:text"` - ClientRequestID string `bun:",notnull"` - Status Status `bun:",notnull"` - WaitReason *WaitReason `bun:",nullzero"` - FailureReason *FailureReason `bun:",nullzero"` - LastError *string `bun:",nullzero"` + ClientRequestID string `bun:"type:text,notnull"` + Status Status `bun:"type:text,notnull"` + WaitReason *WaitReason `bun:"type:text,nullzero"` + FailureReason *FailureReason `bun:"type:text,nullzero"` + LastError *string `bun:"type:text,nullzero"` // ItemsTotal and ItemsCopied are maintained inside the transactions that // seed and complete items, so progress never needs a history-sized count. - ItemsTotal int `bun:",notnull,default:0"` - ItemsCopied int `bun:",notnull,default:0"` - // SeedCursorUploadID advances through storage uploads in bounded batches so + ItemsTotal int `bun:"type:integer,notnull,default:0"` + ItemsCopied int `bun:"type:integer,notnull,default:0"` + // SeedCursorContentID advances through storage uploads in bounded batches so // no single transaction scales with retained bucket history. - SeedCursorUploadID int64 `bun:",notnull,default:0"` - SeedingComplete bool `bun:",notnull,default:false"` - // StateVersion fences item-worker pause requests from a later coordinator - // recovery. LastDispatchedAt is the durable fairness cursor used by the - // global item queue. - StateVersion int64 `bun:",notnull,default:1"` - LastDispatchedAt *time.Time `bun:",nullzero"` - // TerminationEpoch is recorded before the old service is treated as - // terminated, so a crash between termination and observation re-reads it - // instead of terminating twice. - TerminationTxHash *string `bun:",nullzero"` - TerminationEpoch *int64 `bun:",nullzero"` - TerminationObservedAt *time.Time `bun:",nullzero"` - // AbandonedTerminationEpoch records termination of a superseded target. - // It is separate from the source termination fields above because the two - // services belong to opposite generations. - AbandonedTerminationTxHash *string `bun:",nullzero"` - AbandonedTerminationEpoch *int64 `bun:",nullzero"` - AbandonedTerminationObservedAt *time.Time `bun:",nullzero"` - SupersededByID *int64 `bun:",nullzero"` - ConfirmedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` + SeedCursorContentID int64 `bun:",notnull,default:0"` + SeedingComplete bool `bun:",notnull,default:false"` + TaskGeneration int64 `bun:",notnull,default:1"` + TaskID *int64 `bun:",nullzero"` + SupersededByID *int64 `bun:",nullzero"` + CreatedAt time.Time `bun:",nullzero,notnull"` + UpdatedAt time.Time `bun:",nullzero,notnull"` + + // The termination fields below are projected from storage_data_set_terminations + // on read. A termination is a ledger row, so a third kind of termination adds + // a row rather than another repeated column group here. + TerminationTxHash *string `bun:",scanonly"` + TerminationEpoch *int64 `bun:",scanonly"` + AbandonedTerminationTxHash *string `bun:",scanonly"` + AbandonedTerminationEpoch *int64 `bun:",scanonly"` +} + +// TerminationRole names which of a replacement's two data sets a termination +// ended. The source is terminated once migration is safe; the target is +// terminated only after the replacement is superseded and its target abandoned. +type TerminationRole string + +const ( + TerminationRoleSource TerminationRole = "source" + TerminationRoleAbandonedTarget TerminationRole = "abandoned_target" +) + +// Termination records the end of term paid for on one data set. It is written +// before the remote service is treated as terminated, so a crash in between +// re-reads it instead of paying for a second termination. +type Termination struct { + bun.BaseModel `bun:"table:storage_data_set_terminations,alias:storage_data_set_termination"` + + ID int64 `bun:",pk,autoincrement,identity"` + ReplacementID int64 `bun:",notnull"` + Role TerminationRole `bun:"type:text,notnull"` + // Exactly one of the data set columns is set, chosen by Role. Each carries a + // composite foreign key back to the matching column on the replacement, so a + // row cannot name a data set the replacement never held in that role. + SourceDataSetID *int64 `bun:",nullzero"` + AbandonedTargetDataSetID *int64 `bun:",nullzero"` + TxHash *string `bun:"type:text,nullzero"` + Epoch int64 `bun:",notnull"` + CreatedAt time.Time `bun:",nullzero,notnull"` + UpdatedAt time.Time `bun:",nullzero,notnull"` } // Item is one unit of migration work. Items are keyed by storage upload, not by @@ -63,30 +86,18 @@ type Replacement struct { type Item struct { bun.BaseModel `bun:"table:storage_replacement_items,alias:storage_replacement_item"` - ID int64 `bun:",pk,autoincrement"` + ID int64 `bun:",pk,autoincrement,identity"` ReplacementID int64 `bun:",notnull"` - UploadID int64 `bun:",notnull"` - // TargetCopyID is the concrete copy row on the target generation. Tasks - // address it directly so they can never write the wrong generation. - TargetCopyID *int64 `bun:",nullzero"` - Status ItemStatus `bun:",notnull"` - Attempts int `bun:",notnull,default:0"` - ScheduledAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - RetryCount int `bun:",notnull,default:0"` - // MaxRetries is nullable only for rows created before the durable item - // queue migration. Startup recovery initializes it exactly once. - MaxRetries *int `bun:",nullzero"` - ClaimedAt *time.Time `bun:",nullzero"` - LeaseUntil *time.Time `bun:",nullzero"` - LastError *string `bun:",nullzero"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` -} - -// ClaimToken fences lifecycle updates made by one item worker lease. -type ClaimToken struct { - ItemID int64 - ClaimedAt time.Time + ContentID int64 `bun:",notnull"` + // TargetDataSetID identifies the target generation. Together with ContentID + // it resolves exactly one bound copy without retaining a surrogate copy ID. + // It is NOT NULL because a nullable column would make the composite foreign + // key to that copy skip validation whenever it was unset. + TargetDataSetID int64 `bun:",notnull"` + Status ItemStatus `bun:"type:text,notnull,default:'pending'"` + LastError *string `bun:"type:text,nullzero"` + CreatedAt time.Time `bun:",nullzero,notnull"` + UpdatedAt time.Time `bun:",nullzero,notnull"` } // ProgressSnapshot is the UI-neutral aggregate for one replacement. Total is @@ -113,13 +124,65 @@ type ProgressSnapshot struct { // intentionally reports presence rather than item counts; full aggregates are // reserved for operator-facing progress reads. type ExecutionSnapshot struct { - ReplacementID int64 `bun:"replacement_id"` - SeedingComplete bool `bun:"seeding_complete"` - ItemsTotal int `bun:"items_total"` - ItemsCopied int `bun:"items_copied"` - HasPending bool `bun:"has_pending"` - HasActive bool `bun:"has_active"` - HasRetrying bool `bun:"has_retrying"` - HasWaitingSource bool `bun:"has_waiting_source"` - HasFailed bool `bun:"has_failed"` + ReplacementID int64 `bun:"replacement_id"` + SeedingComplete bool `bun:"seeding_complete"` + HasPending bool `bun:"has_pending"` + HasFailed bool `bun:"has_failed"` +} + +var _ bun.BeforeAppendModelHook = (*Replacement)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (r *Replacement) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if r.CreatedAt.IsZero() { + r.CreatedAt = now + } + if r.UpdatedAt.IsZero() { + r.UpdatedAt = now + } + return nil +} + +var _ bun.BeforeAppendModelHook = (*Termination)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (t *Termination) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if t.CreatedAt.IsZero() { + t.CreatedAt = now + } + if t.UpdatedAt.IsZero() { + t.UpdatedAt = now + } + return nil +} + +var _ bun.BeforeAppendModelHook = (*Item)(nil) + +// BeforeAppendModel stamps the audit columns on insert. The database has no +// timestamp default, so every row is written with one encoding instead of two +// that sort against each other inside the same second. +func (i *Item) BeforeAppendModel(_ context.Context, query bun.Query) error { + if _, ok := query.(*bun.InsertQuery); !ok { + return nil + } + now := time.Now().UTC() + if i.CreatedAt.IsZero() { + i.CreatedAt = now + } + if i.UpdatedAt.IsZero() { + i.UpdatedAt = now + } + return nil } diff --git a/internal/storagereplacement/status.go b/internal/storagereplacement/status.go index 8d01aa9..f548049 100644 --- a/internal/storagereplacement/status.go +++ b/internal/storagereplacement/status.go @@ -22,8 +22,8 @@ const ( // StatusCleanupAttention means retirement cannot proceed without an // operator decision. Automatic task retry is suppressed. StatusCleanupAttention Status = "cleanup_attention" - // StatusFailed means the work exhausted its retries and needs the operator - // to retry it from the Data Sets surface. + // StatusFailed means automatic recovery stopped and the operator must retry + // it from the Data Sets surface. StatusFailed Status = "failed" // StatusCompleted means the source service was terminated and observed as // terminated. @@ -92,27 +92,18 @@ type ItemStatus string const ( // ItemStatusPending is seeded work not yet attempted. ItemStatusPending ItemStatus = "pending" - // ItemStatusRunning is an item leased by a replacement worker. - ItemStatusRunning ItemStatus = "running" - // ItemStatusRetrying is temporarily failed work waiting for its persisted - // retry schedule. It remains retirement-blocking. - ItemStatusRetrying ItemStatus = "retrying" - // ItemStatusWaitingSource means no readable copy and no cached content is - // available yet. The coordinator moves on and revisits it later. - ItemStatusWaitingSource ItemStatus = "waiting_source" // ItemStatusCopied means the target holds a committed readable copy. ItemStatusCopied ItemStatus = "copied" // ItemStatusCancelled means the content no longer needs migrating. ItemStatusCancelled ItemStatus = "cancelled" - // ItemStatusFailed exhausted its item-level retry budget. Other items may - // continue, but the source cannot retire until an operator retries it. - ItemStatusFailed ItemStatus = "failed" + // ItemStatusAttention requires an operator decision before retirement. + ItemStatusAttention ItemStatus = "attention" ) // Executable reports whether an item is still active or recoverable. func (s ItemStatus) Executable() bool { switch s { - case ItemStatusPending, ItemStatusRunning, ItemStatusRetrying, ItemStatusWaitingSource: + case ItemStatusPending: return true default: return false @@ -122,8 +113,7 @@ func (s ItemStatus) Executable() bool { // Blocking reports whether the item prevents the source from being retired. func (s ItemStatus) Blocking() bool { switch s { - case ItemStatusPending, ItemStatusRunning, ItemStatusRetrying, - ItemStatusWaitingSource, ItemStatusFailed: + case ItemStatusPending, ItemStatusAttention: return true default: return false @@ -133,9 +123,7 @@ func (s ItemStatus) Blocking() bool { // Valid reports whether the value is a known item status. func (s ItemStatus) Valid() bool { switch s { - case ItemStatusPending, ItemStatusRunning, ItemStatusRetrying, - ItemStatusWaitingSource, ItemStatusCopied, ItemStatusCancelled, - ItemStatusFailed: + case ItemStatusPending, ItemStatusCopied, ItemStatusCancelled, ItemStatusAttention: return true default: return false diff --git a/internal/storagereplacement/task.go b/internal/storagereplacement/task.go index 0bcc267..5e27251 100644 --- a/internal/storagereplacement/task.go +++ b/internal/storagereplacement/task.go @@ -4,193 +4,72 @@ import ( "encoding/json" "errors" "fmt" - "time" "github.com/strahe/synaps3/internal/model" ) const ( - // StageMigrate advances the replacement control plane through the uploader; - // item transfers run on the dedicated replacement worker. - StageMigrate = "replace_provider" - // StageRetire runs the retirement safety gate through the storage cleanup - // worker. - StageRetire = "retire_data_set" - // StageRetireAbandonedTarget ends the service of a target that a later - // confirmation replaced before it ever took over the slot. - StageRetireAbandonedTarget = "retire_abandoned_target" - - replacementIDPayloadKey = "replacement_id" - migrateTaskKeyPrefix = "upload:storage-replacement:" - retireTaskKeyPrefix = "storage_cleanup:storage-replacement:" + CoordinateTaskKeyPrefix = "provider-replacement:" + RetireTaskKeyPrefix = "storage-dataset-retire:" ) -// MigrateTaskKey identifies the single migration coordinator for one -// replacement. Item concurrency is owned by the durable replacement queue, -// not by this control-plane task. -func MigrateTaskKey(replacementID int64) string { - return fmt.Sprintf("%s%d:migrate", migrateTaskKeyPrefix, replacementID) -} - -// RetireTaskKey identifies the single retirement coordinator for one -// replacement. It retires the generation being replaced. -func RetireTaskKey(replacementID int64) string { - return fmt.Sprintf("%s%d:cleanup", retireTaskKeyPrefix, replacementID) -} - -// AbandonedTargetTaskKey identifies the cleanup of a target a later -// confirmation abandoned. It is a separate coordinator because it retires the -// opposite generation and answers a different safety question. -func AbandonedTargetTaskKey(replacementID int64) string { - return fmt.Sprintf("%s%d:abandoned-target", retireTaskKeyPrefix, replacementID) +type CoordinateInput struct { + ReplacementID int64 `json:"replacement_id"` + Generation int64 `json:"generation"` } -// NewMigrateTask builds the singleton migration coordinator for one -// replacement. It prepares the target, seeds durable items, observes their -// bounded execution state, and hands completed migration to retirement. -func NewMigrateTask(replacementID, bucketID int64, versionID string, maxRetries int, scheduledAt time.Time) *model.Task { - stage := StageMigrate - return &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "bucket", - RefID: bucketID, - RefVersionID: versionID, - IdempotencyKey: MigrateTaskKey(replacementID), - Payload: NewMigratePayload(replacementID), - Status: model.TaskStatusQueued, - MaxRetries: maxRetries, - ScheduledAt: scheduledAt, - } -} - -// NewRetireTask builds the singleton retirement coordinator for one -// replacement. It runs on the storage cleanup worker because terminating a -// service is a destructive lifecycle action, not upload work. -func NewRetireTask(replacementID, bucketID int64, maxRetries int, scheduledAt time.Time) *model.Task { - stage := StageRetire - return &model.Task{ - Type: model.TaskTypeStorageCleanup, - Stage: &stage, - RefType: "bucket", - RefID: bucketID, - IdempotencyKey: RetireTaskKey(replacementID), - Payload: NewRetirePayload(replacementID), - Status: model.TaskStatusQueued, - MaxRetries: maxRetries, - ScheduledAt: scheduledAt, - } +type RetireInput struct { + ReplacementID int64 `json:"replacement_id"` + DataSetID int64 `json:"data_set_id"` + Generation int64 `json:"generation"` } -// MigratePayload is the persisted state of the migration coordinator. The -// durable item queue owns all transfer state. -type MigratePayload struct { - ReplacementID int64 +func CoordinateTaskKey(replacementID, generation int64) string { + return fmt.Sprintf("%s%d:%d", CoordinateTaskKeyPrefix, replacementID, generation) } -// NewMigratePayload builds the coordinator payload. -func NewMigratePayload(replacementID int64) map[string]any { - return map[string]any{replacementIDPayloadKey: replacementID} +func RetireTaskKey(dataSetID, generation int64) string { + return fmt.Sprintf("%s%d:%d", RetireTaskKeyPrefix, dataSetID, generation) } -// ParseMigratePayload decodes a migration coordinator payload. These tasks only -// ever exist after the upgrade that introduced them, so the replacement ID is -// required rather than inferred. -func ParseMigratePayload(task *model.Task) (MigratePayload, error) { +func ParseCoordinateInput(task *model.Task) (CoordinateInput, error) { if task == nil { - return MigratePayload{}, errors.New("nil replacement migration task") + return CoordinateInput{}, errors.New("nil provider replacement task") } - replacementID, err := payloadInt64(task.Payload, replacementIDPayloadKey) - if err != nil { - return MigratePayload{}, err + var input CoordinateInput + if err := json.Unmarshal(task.Input, &input); err != nil { + return CoordinateInput{}, fmt.Errorf("decoding provider replacement input: %w", err) } - if replacementID <= 0 { - return MigratePayload{}, fmt.Errorf("replacement migration task %s must be positive", replacementIDPayloadKey) + if input.ReplacementID < 1 || input.Generation < 1 { + return CoordinateInput{}, errors.New("provider replacement input is incomplete") } - return MigratePayload{ReplacementID: replacementID}, nil + return input, nil } -// NewRetirePayload builds the retirement coordinator payload. -func NewRetirePayload(replacementID int64) map[string]any { - return map[string]any{replacementIDPayloadKey: replacementID} -} - -// NewAbandonedTargetTask builds the coordinator that ends the paid service of a -// target no confirmation uses any more. Without it the abandoned service keeps -// costing money after a later confirmation takes over. -func NewAbandonedTargetTask(replacementID, bucketID int64, maxRetries int, scheduledAt time.Time) *model.Task { - stage := StageRetireAbandonedTarget - return &model.Task{ - Type: model.TaskTypeStorageCleanup, - Stage: &stage, - RefType: "bucket", - RefID: bucketID, - IdempotencyKey: AbandonedTargetTaskKey(replacementID), - Payload: NewRetirePayload(replacementID), - Status: model.TaskStatusQueued, - MaxRetries: maxRetries, - ScheduledAt: scheduledAt, - } -} - -// ParseRetirePayload decodes a retirement coordinator payload. -func ParseRetirePayload(task *model.Task) (int64, error) { +func ParseRetireInput(task *model.Task) (RetireInput, error) { if task == nil { - return 0, errors.New("nil replacement retirement task") + return RetireInput{}, errors.New("nil data-set retirement task") } - replacementID, err := payloadInt64(task.Payload, replacementIDPayloadKey) - if err != nil { - return 0, err + var input RetireInput + if err := json.Unmarshal(task.Input, &input); err != nil { + return RetireInput{}, fmt.Errorf("decoding data-set retirement input: %w", err) } - if replacementID <= 0 { - return 0, fmt.Errorf("replacement retirement task %s must be positive", replacementIDPayloadKey) + if input.ReplacementID < 1 || input.DataSetID < 1 || input.Generation < 1 { + return RetireInput{}, errors.New("data-set retirement input is incomplete") } - return replacementID, nil + return input, nil } -// IsCoordinatorTask reports whether a task belongs to a replacement. Generic -// exhausted-task retry uses it to refuse work that must resume through the -// dedicated replacement action. -func IsCoordinatorTask(taskType model.TaskType, stage *string) bool { - if stage == nil { - return false - } - switch { - case taskType == model.TaskTypeUpload && *stage == StageMigrate: - return true - case taskType == model.TaskTypeStorageCleanup && *stage == StageRetire: - return true - case taskType == model.TaskTypeStorageCleanup && *stage == StageRetireAbandonedTarget: - return true - default: - return false +func ValidateCoordinateInput(input CoordinateInput) error { + if input.ReplacementID < 1 || input.Generation < 1 { + return errors.New("replacement_id and generation are required") } + return nil } -// Payload values survive a JSON round trip through the task table, so an -// integer can come back as float64 or json.Number depending on the driver. -func payloadInt64(payload map[string]any, key string) (int64, error) { - if payload == nil { - return 0, fmt.Errorf("replacement task payload is missing %s", key) - } - raw, ok := payload[key] - if !ok { - return 0, fmt.Errorf("replacement task payload is missing %s", key) - } - switch value := raw.(type) { - case int64: - return value, nil - case int: - return int64(value), nil - case float64: - return int64(value), nil - case json.Number: - parsed, err := value.Int64() - if err != nil { - return 0, fmt.Errorf("replacement task payload %s is not an integer: %w", key, err) - } - return parsed, nil - default: - return 0, fmt.Errorf("replacement task payload %s has type %T, want an integer", key, raw) +func ValidateRetireInput(input RetireInput) error { + if input.ReplacementID < 1 || input.DataSetID < 1 || input.Generation < 1 { + return errors.New("replacement_id, data_set_id, and generation are required") } + return nil } diff --git a/internal/storagereplacement/task_test.go b/internal/storagereplacement/task_test.go index 667d922..403a9fd 100644 --- a/internal/storagereplacement/task_test.go +++ b/internal/storagereplacement/task_test.go @@ -3,125 +3,46 @@ package storagereplacement import ( "encoding/json" "errors" - "strings" "testing" "github.com/strahe/synaps3/internal/model" ) -func TestMigratePayloadRoundTrip(t *testing.T) { - task := &model.Task{Payload: NewMigratePayload(7)} - got, err := ParseMigratePayload(task) +func TestCoordinateInputRoundTrip(t *testing.T) { + raw, err := json.Marshal(CoordinateInput{ReplacementID: 7, Generation: 2}) if err != nil { - t.Fatalf("ParseMigratePayload: %v", err) + t.Fatalf("Marshal: %v", err) } - want := MigratePayload{ReplacementID: 7} - if got != want { - t.Fatalf("payload = %+v, want %+v", got, want) - } -} - -// Payloads survive a JSON round trip through the task table, so integers come -// back as float64 or json.Number depending on the driver. -func TestMigratePayloadSurvivesJSONRoundTrip(t *testing.T) { - encoded, err := json.Marshal(NewMigratePayload(7)) + got, err := ParseCoordinateInput(&model.Task{Input: raw}) if err != nil { - t.Fatalf("marshal payload: %v", err) + t.Fatalf("ParseCoordinateInput: %v", err) } - for _, useNumber := range []bool{false, true} { - decoded := map[string]any{} - decoder := json.NewDecoder(strings.NewReader(string(encoded))) - if useNumber { - decoder.UseNumber() - } - if err := decoder.Decode(&decoded); err != nil { - t.Fatalf("decode payload: %v", err) - } - got, err := ParseMigratePayload(&model.Task{Payload: decoded}) - if err != nil { - t.Fatalf("ParseMigratePayload(useNumber=%v): %v", useNumber, err) - } - if got.ReplacementID != 7 { - t.Fatalf("payload = %+v, want replacement 7", got) - } + if got != (CoordinateInput{ReplacementID: 7, Generation: 2}) { + t.Fatalf("input = %#v", got) } } -func TestMigratePayloadRejectsMissingReplacement(t *testing.T) { - cases := map[string]*model.Task{ - "nil task": nil, - "nil payload": {}, - "empty payload": {Payload: map[string]any{}}, - "zero id": {Payload: map[string]any{"replacement_id": 0}}, - "wrong type": {Payload: map[string]any{"replacement_id": "seven"}}, - "only item id": {Payload: map[string]any{"item_id": 42}}, - "negative value": {Payload: map[string]any{"replacement_id": -1}}, +func TestReplacementInputsRejectIncompleteIdentity(t *testing.T) { + if _, err := ParseCoordinateInput(nil); err == nil { + t.Fatal("ParseCoordinateInput accepted nil task") } - for name, task := range cases { - if _, err := ParseMigratePayload(task); err == nil { - t.Fatalf("ParseMigratePayload(%s) succeeded, want an error", name) - } + if _, err := ParseCoordinateInput(&model.Task{Input: json.RawMessage(`{"replacement_id":7}`)}); err == nil { + t.Fatal("ParseCoordinateInput accepted missing generation") } -} - -func TestRetirePayloadRoundTrip(t *testing.T) { - replacementID, err := ParseRetirePayload(&model.Task{Payload: NewRetirePayload(11)}) - if err != nil { - t.Fatalf("ParseRetirePayload: %v", err) - } - if replacementID != 11 { - t.Fatalf("replacementID = %d, want 11", replacementID) - } - if _, err := ParseRetirePayload(&model.Task{}); err == nil { - t.Fatal("ParseRetirePayload accepted an empty payload, want an error") + if _, err := ParseRetireInput(&model.Task{Input: json.RawMessage(`{"replacement_id":7,"data_set_id":3}`)}); err == nil { + t.Fatal("ParseRetireInput accepted missing generation") } } -func TestCoordinatorTaskKeysAreDistinctSingletons(t *testing.T) { - if MigrateTaskKey(1) == MigrateTaskKey(2) { - t.Fatal("migration keys collide across replacements") - } - if MigrateTaskKey(1) == RetireTaskKey(1) { - t.Fatal("migration and retirement keys collide") - } - if AbandonedTargetTaskKey(1) == RetireTaskKey(1) { - t.Fatal("abandoned-target and source retirement keys collide") - } - if AbandonedTargetTaskKey(1) == MigrateTaskKey(1) { - t.Fatal("abandoned-target and migration keys collide") +func TestReplacementTaskKeysAreGenerationScoped(t *testing.T) { + if CoordinateTaskKey(1, 1) == CoordinateTaskKey(1, 2) { + t.Fatal("replacement generations share a coordinator key") } - for _, key := range []string{MigrateTaskKey(1), RetireTaskKey(1), AbandonedTargetTaskKey(1)} { - if !strings.Contains(key, "storage-replacement:1:") { - t.Fatalf("key %q does not identify replacement 1", key) - } + if RetireTaskKey(4, 1) == RetireTaskKey(4, 2) { + t.Fatal("retirement generations share a key") } -} - -// Generic exhausted-task retry must refuse replacement work and send the -// operator back to the Data Sets surface. -func TestIsCoordinatorTask(t *testing.T) { - migrate := StageMigrate - retire := StageRetire - abandoned := StageRetireAbandonedTarget - ingress := "ingress_store" - cases := []struct { - name string - taskType model.TaskType - stage *string - want bool - }{ - {"migration coordinator", model.TaskTypeUpload, &migrate, true}, - {"retirement coordinator", model.TaskTypeStorageCleanup, &retire, true}, - {"abandoned-target coordinator", model.TaskTypeStorageCleanup, &abandoned, true}, - {"ordinary upload", model.TaskTypeUpload, &ingress, false}, - {"no stage", model.TaskTypeUpload, nil, false}, - {"stage on the wrong task type", model.TaskTypeStorageCleanup, &migrate, false}, - {"retire stage on the wrong task type", model.TaskTypeUpload, &retire, false}, - } - for _, tc := range cases { - if got := IsCoordinatorTask(tc.taskType, tc.stage); got != tc.want { - t.Fatalf("IsCoordinatorTask(%s) = %v, want %v", tc.name, got, tc.want) - } + if CoordinateTaskKey(1, 1) == RetireTaskKey(1, 1) { + t.Fatal("coordinator and retirement keys collide") } } @@ -141,7 +62,6 @@ func TestCodeMapsReplacementErrors(t *testing.T) { if got := Code(err); got != want { t.Fatalf("Code(%v) = %q, want %q", err, got, want) } - // Wrapping is how repositories add context, so it must not lose the code. if got := Code(errors.Join(errors.New("context"), err)); got != want { t.Fatalf("Code(wrapped %v) = %q, want %q", err, got, want) } diff --git a/internal/storagereplacement/transition.go b/internal/storagereplacement/transition.go index 40a5d0f..b82c934 100644 --- a/internal/storagereplacement/transition.go +++ b/internal/storagereplacement/transition.go @@ -96,18 +96,3 @@ func PhaseFor(status Status) Phase { return PhaseNone } } - -// OnTaskExhausted maps a status to the state a terminally exhausted coordinator -// task leaves behind. Retirement goes to operator attention rather than failed -// because its remaining work is a cleanup decision, not a retryable copy. -// The second return value reports whether any change is warranted. -func OnTaskExhausted(status Status) (Status, bool) { - switch status { - case StatusPreparingTarget, StatusMigrating, StatusWaiting: - return StatusFailed, true - case StatusRetiring: - return StatusCleanupAttention, true - default: - return status, false - } -} diff --git a/internal/storagereplacement/transition_test.go b/internal/storagereplacement/transition_test.go index 270be5d..47e7de2 100644 --- a/internal/storagereplacement/transition_test.go +++ b/internal/storagereplacement/transition_test.go @@ -77,38 +77,6 @@ func TestFailedResumesAtAnyWorkingPhase(t *testing.T) { } } -func TestOnTaskExhausted(t *testing.T) { - cases := []struct { - status Status - want Status - changed bool - }{ - {StatusPreparingTarget, StatusFailed, true}, - {StatusMigrating, StatusFailed, true}, - {StatusWaiting, StatusFailed, true}, - // Retirement needs an operator decision, not another copy attempt. - {StatusRetiring, StatusCleanupAttention, true}, - {StatusCleanupAttention, StatusCleanupAttention, false}, - {StatusFailed, StatusFailed, false}, - {StatusCompleted, StatusCompleted, false}, - {StatusSuperseded, StatusSuperseded, false}, - } - for _, tc := range cases { - got, changed := OnTaskExhausted(tc.status) - if got != tc.want || changed != tc.changed { - t.Fatalf("OnTaskExhausted(%s) = (%s, %v), want (%s, %v)", tc.status, got, changed, tc.want, tc.changed) - } - } - for _, tc := range cases { - if !tc.changed { - continue - } - if !Allowed(tc.status, tc.want) { - t.Fatalf("exhaustion moves %s to %s, but the state machine forbids it", tc.status, tc.want) - } - } -} - func TestStatusClassification(t *testing.T) { cases := []struct { status Status @@ -164,13 +132,10 @@ func TestPhaseForMatchesCoordinatorWork(t *testing.T) { func TestItemStatusBlocksRetirementWhileExecutable(t *testing.T) { cases := map[ItemStatus]bool{ - ItemStatusPending: true, - ItemStatusRunning: true, - ItemStatusRetrying: true, - ItemStatusWaitingSource: true, - ItemStatusFailed: true, - ItemStatusCopied: false, - ItemStatusCancelled: false, + ItemStatusPending: true, + ItemStatusAttention: true, + ItemStatusCopied: false, + ItemStatusCancelled: false, } for status, want := range cases { if got := status.Blocking(); got != want { diff --git a/internal/synapse/interfaces.go b/internal/synapse/interfaces.go index 79adedc..f6bedb8 100644 --- a/internal/synapse/interfaces.go +++ b/internal/synapse/interfaces.go @@ -59,6 +59,22 @@ type StorageClient interface { OpenCleanupContext(ctx context.Context, dataSetID sdktypes.BigInt, opts storage.NewDataSetContextOptions) (CleanupContext, error) } +// ParkedPieceState describes the provider-local state of a content-addressed +// piece before it has been committed on chain. +type ParkedPieceState string + +const ( + ParkedPieceMissing ParkedPieceState = "missing" + ParkedPieceProcessing ParkedPieceState = "processing" + ParkedPieceReady ParkedPieceState = "parked" +) + +// ParkedPieceChecker observes the provider's parked-piece endpoint without +// treating an on-chain lookup as evidence that an upload did not happen. +type ParkedPieceChecker interface { + FindParkedPiece(context.Context, string, cid.Cid) (ParkedPieceState, error) +} + // ServiceTerminator is the destructive service-lifecycle boundary. It is used // only after replacement cleanup authorization and the retirement safety gate // have both passed. diff --git a/internal/synapse/pdp_status.go b/internal/synapse/pdp_status.go index db1cec9..3bea9c4 100644 --- a/internal/synapse/pdp_status.go +++ b/internal/synapse/pdp_status.go @@ -12,6 +12,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ipfs/go-cid" "github.com/strahe/synapse-go/pdp" ) @@ -196,18 +197,67 @@ func (c *PDPStatusChecker) GetAddPiecesStatus(ctx context.Context, input AddPiec return result, nil } +// FindParkedPiece checks the provider-local piece endpoint once. Its client +// uses the same redirect, DNS, and private-network protections as status +// polling. +func (c *PDPStatusChecker) FindParkedPiece(ctx context.Context, serviceURL string, pieceCID cid.Cid) (ParkedPieceState, error) { + client, err := c.clientForServiceURL(serviceURL) + if err != nil { + return "", err + } + found, err := client.FindPiece(ctx, pieceCID) + switch { + case errors.Is(err, pdp.ErrPieceNotFound): + return ParkedPieceMissing, nil + case errors.Is(err, pdp.ErrPieceProcessing): + return ParkedPieceProcessing, nil + case err != nil: + return "", err + case found == nil: + return "", errors.New("empty parked-piece response") + } + foundCID, err := cid.Parse(found.PieceCID) + if err != nil { + return "", fmt.Errorf("parse parked piece identity: %w", err) + } + if !foundCID.Equals(pieceCID) { + return "", fmt.Errorf("parked piece identity mismatch: got %s want %s", foundCID, pieceCID) + } + return ParkedPieceReady, nil +} + +func (c *PDPStatusChecker) clientForServiceURL(serviceURL string) (*pdp.Client, error) { + parsed, err := parsePDPHTTPURL(serviceURL, "service URL") + if err != nil { + return nil, err + } + return c.newPDPClient(parsed.String()) +} + func (c *PDPStatusChecker) clientForStatusURL(statusURL string) (*pdp.Client, error) { - parsed, err := url.Parse(statusURL) + parsed, err := parsePDPHTTPURL(statusURL, "status URL") if err != nil { - return nil, fmt.Errorf("parse status URL: %w", err) + return nil, err + } + base := &url.URL{Scheme: parsed.Scheme, Host: parsed.Host} + return c.newPDPClient(base.String()) +} + +func parsePDPHTTPURL(rawURL, label string) (*url.URL, error) { + parsed, err := url.Parse(rawURL) + if err != nil { + return nil, fmt.Errorf("parse %s: %w", label, err) } if parsed.Scheme != "http" && parsed.Scheme != "https" { - return nil, fmt.Errorf("unsupported status URL scheme %q", parsed.Scheme) + return nil, fmt.Errorf("unsupported %s scheme %q", label, parsed.Scheme) } if parsed.Host == "" { - return nil, fmt.Errorf("missing status URL host") + return nil, fmt.Errorf("missing %s host", label) } - base := &url.URL{Scheme: parsed.Scheme, Host: parsed.Host} + return parsed, nil +} + +func (c *PDPStatusChecker) newPDPClient(baseURL string) (*pdp.Client, error) { httpClient := c.httpClient if httpClient == nil { timeout := c.timeout @@ -216,7 +266,7 @@ func (c *PDPStatusChecker) clientForStatusURL(statusURL string) (*pdp.Client, er } httpClient = newPDPStatusHTTPClient(timeout, c.allowPrivateNetworks) } - return pdp.New(base.String(), + return pdp.New(baseURL, pdp.WithHTTPClient(httpClient), pdp.WithMaxRetries(0), ) diff --git a/internal/synapse/pdp_status_test.go b/internal/synapse/pdp_status_test.go index 8ff2c9e..29532e9 100644 --- a/internal/synapse/pdp_status_test.go +++ b/internal/synapse/pdp_status_test.go @@ -7,6 +7,8 @@ import ( "strings" "testing" "time" + + "github.com/strahe/synapse-go/piece" ) const ( @@ -15,6 +17,61 @@ const ( testConfirmedTxHash = "0xabcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" ) +func TestPDPStatusCheckerFindsProviderParkedPiece(t *testing.T) { + info, err := piece.Calculate(strings.NewReader(strings.Repeat("p", 128))) + if err != nil { + t.Fatalf("calculate piece identity: %v", err) + } + tests := []struct { + name string + statusCode int + body string + want ParkedPieceState + }{ + {name: "parked", statusCode: http.StatusOK, body: fmt.Sprintf(`{"pieceCid":%q}`, info.CIDv2), want: ParkedPieceReady}, + {name: "processing", statusCode: http.StatusAccepted, want: ParkedPieceProcessing}, + {name: "missing", statusCode: http.StatusNotFound, want: ParkedPieceMissing}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/provider/api/pdp/piece" || r.URL.Query().Get("pieceCid") != info.CIDv2.String() { + t.Fatalf("parked-piece request = %s?%s", r.URL.Path, r.URL.RawQuery) + } + w.WriteHeader(tt.statusCode) + _, _ = fmt.Fprint(w, tt.body) + })) + defer server.Close() + + checker := NewPDPStatusChecker(PDPStatusCheckerOptions{Timeout: time.Second, AllowPrivateNetworks: true}) + got, err := checker.FindParkedPiece(t.Context(), server.URL+"/provider/api", info.CIDv2) + if err != nil || got != tt.want { + t.Fatalf("FindParkedPiece = %q, %v; want %q", got, err, tt.want) + } + }) + } +} + +func TestPDPStatusCheckerRejectsMismatchedParkedPiece(t *testing.T) { + want, err := piece.Calculate(strings.NewReader(strings.Repeat("w", 128))) + if err != nil { + t.Fatalf("calculate wanted piece identity: %v", err) + } + other, err := piece.Calculate(strings.NewReader(strings.Repeat("o", 128))) + if err != nil { + t.Fatalf("calculate other piece identity: %v", err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprintf(w, `{"pieceCid":%q}`, other.CIDv2) + })) + defer server.Close() + + checker := NewPDPStatusChecker(PDPStatusCheckerOptions{Timeout: time.Second, AllowPrivateNetworks: true}) + if _, err := checker.FindParkedPiece(t.Context(), server.URL, want.CIDv2); err == nil || !strings.Contains(err.Error(), "identity mismatch") { + t.Fatalf("mismatched parked-piece error = %v", err) + } +} + func TestPDPStatusCheckerChecksDataSetCreationStatusOnce(t *testing.T) { var requests int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/synapse/storage_errors.go b/internal/synapse/storage_errors.go index 055eb8f..dc897da 100644 --- a/internal/synapse/storage_errors.go +++ b/internal/synapse/storage_errors.go @@ -58,6 +58,18 @@ type DataSetServiceEndedError struct { Cause error } +type PullErrorDisposition uint8 + +const ( + PullErrorUnknown PullErrorDisposition = iota + PullErrorRetryable + PullErrorTerminal +) + +// ErrProviderTransactionRejected is the adapter-level identity for a provider +// transaction that reached a terminal rejected state. +var ErrProviderTransactionRejected = pdp.ErrTxRejected + func (e *DataSetServiceEndedError) Error() string { if e == nil || e.Cause == nil { return "storage data set service ended" @@ -82,6 +94,31 @@ func IsProviderUnavailable(err error) bool { return errors.As(err, &unavailable) } +// ClassifyPullError keeps SDK-specific pull failures at the adapter boundary. +// Provider and caller interruptions remain retryable; deterministic request or +// provider rejection errors are terminal. Unknown errors use the task's bounded +// retry budget so a newly introduced SDK error cannot create a permanent loop. +func ClassifyPullError(err error) PullErrorDisposition { + if err == nil { + return PullErrorUnknown + } + if IsProviderUnavailable(err) || errors.Is(err, context.Canceled) || providerOperationUnavailable(err) { + return PullErrorRetryable + } + if errors.Is(err, pdp.ErrPullFailed) || errors.Is(err, storage.ErrInvalidArgument) || IsDataSetServiceEnded(err) { + return PullErrorTerminal + } + if httpErr, ok := errors.AsType[*pdp.HTTPError](err); ok { + if providerHTTPStatusUnavailable(httpErr.StatusCode) { + return PullErrorRetryable + } + if httpErr.StatusCode >= http.StatusBadRequest && httpErr.StatusCode < http.StatusInternalServerError { + return PullErrorTerminal + } + } + return PullErrorUnknown +} + func providerOperationUnavailable(err error) bool { if err == nil || errors.Is(err, context.Canceled) { return false diff --git a/internal/synapse/storage_errors_test.go b/internal/synapse/storage_errors_test.go index 31b6d1c..e2fe65c 100644 --- a/internal/synapse/storage_errors_test.go +++ b/internal/synapse/storage_errors_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net" + "net/http" "testing" "github.com/strahe/synapse-go/pdp" @@ -12,6 +13,38 @@ import ( "github.com/strahe/synapse-go/storage" ) +func TestClassifyPullError(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + want PullErrorDisposition + }{ + {name: "provider failed", err: fmt.Errorf("pull: %w", pdp.ErrPullFailed), want: PullErrorTerminal}, + {name: "invalid request", err: fmt.Errorf("pull: %w", storage.ErrInvalidArgument), want: PullErrorTerminal}, + {name: "bad request", err: &pdp.HTTPError{StatusCode: http.StatusBadRequest}, want: PullErrorTerminal}, + {name: "not found", err: &pdp.HTTPError{StatusCode: http.StatusNotFound}, want: PullErrorTerminal}, + {name: "request timeout", err: &pdp.HTTPError{StatusCode: http.StatusRequestTimeout}, want: PullErrorRetryable}, + {name: "too early", err: &pdp.HTTPError{StatusCode: http.StatusTooEarly}, want: PullErrorRetryable}, + {name: "rate limited", err: &pdp.HTTPError{StatusCode: http.StatusTooManyRequests}, want: PullErrorRetryable}, + {name: "server error", err: &pdp.HTTPError{StatusCode: http.StatusBadGateway}, want: PullErrorRetryable}, + {name: "network error", err: &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection reset")}, want: PullErrorRetryable}, + {name: "caller cancelled", err: context.Canceled, want: PullErrorRetryable}, + {name: "caller deadline", err: context.DeadlineExceeded, want: PullErrorRetryable}, + {name: "normalized provider unavailable", err: &ProviderUnavailableError{Cause: errors.New("offline")}, want: PullErrorRetryable}, + {name: "unknown", err: errors.New("new sdk error"), want: PullErrorUnknown}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := ClassifyPullError(tt.err); got != tt.want { + t.Fatalf("ClassifyPullError(%v) = %v, want %v", tt.err, got, tt.want) + } + }) + } +} + func TestNormalizeSelectUploadTargetsError(t *testing.T) { t.Parallel() diff --git a/internal/systemtask/task.go b/internal/systemtask/task.go new file mode 100644 index 0000000..1303210 --- /dev/null +++ b/internal/systemtask/task.go @@ -0,0 +1,8 @@ +// Package systemtask owns inputs for recurring built-in tasks. +package systemtask + +const CacheCapacityKey = "system:cache-capacity" + +type Input struct{} + +func ValidateInput(Input) error { return nil } diff --git a/internal/systemtest/filecoin.go b/internal/systemtest/filecoin.go index 4554c55..4ad478e 100644 --- a/internal/systemtest/filecoin.go +++ b/internal/systemtest/filecoin.go @@ -18,10 +18,10 @@ import ( "github.com/ethereum/go-ethereum/common" ethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/ipfs/go-cid" - "github.com/multiformats/go-multihash" "github.com/strahe/synaps3/internal/observability" "github.com/strahe/synaps3/internal/synapse" appTypes "github.com/strahe/synaps3/internal/types" + "github.com/strahe/synapse-go/piece" "github.com/strahe/synapse-go/storage" sdktypes "github.com/strahe/synapse-go/types" ) @@ -395,11 +395,11 @@ func (c *memoryDataSetTarget) Store(ctx context.Context, reader io.Reader, opts if err != nil { return nil, err } - digest, err := multihash.Sum(content, multihash.SHA2_256, -1) + pieceInfo, err := piece.CalculateFromBytes(content) if err != nil { - return nil, fmt.Errorf("memory filecoin: hashing piece: %w", err) + return nil, fmt.Errorf("memory filecoin: calculating piece identity: %w", err) } - pieceCID := cid.NewCidV1(cid.Raw, digest) + pieceCID := pieceInfo.CIDv2 if opts != nil && opts.PieceCID.Defined() && !opts.PieceCID.Equals(pieceCID) { return nil, fmt.Errorf("memory filecoin: supplied CID does not match content") } diff --git a/internal/systemtest/harness.go b/internal/systemtest/harness.go index f97f1c3..6ea49ea 100644 --- a/internal/systemtest/harness.go +++ b/internal/systemtest/harness.go @@ -111,9 +111,15 @@ func newHarness(ctx context.Context, logger *slog.Logger, s3Address string) (_ * cfg.Filecoin.Observability.Interval = 40 * time.Millisecond cfg.Filecoin.Observability.Timeout = time.Second cfg.Filecoin.Observability.Concurrency = 3 - cfg.Worker.Upload = config.WorkerPoolConfig{Concurrency: 1, PollInterval: 15 * time.Millisecond, MaxRetries: 3} - cfg.Worker.Evictor = config.WorkerPoolConfig{Concurrency: 1, PollInterval: 15 * time.Millisecond, MaxRetries: 3} - cfg.Worker.StorageCleanup = config.WorkerPoolConfig{Concurrency: 1, PollInterval: 25 * time.Millisecond, MaxRetries: 3} + cfg.Worker.Tasks = config.TaskWorkerConfig{ + Concurrency: 4, + PollInterval: 15 * time.Millisecond, + LeaseDuration: time.Second, + MaxRetries: 3, + Retention: time.Hour, + ProviderMutationConcurrency: 4, + DestructiveMutationConcurrency: 2, + } passwordHash, err := bcrypt.GenerateFromPassword([]byte(AdminPassword), bcrypt.MinCost) if err != nil { return nil, fmt.Errorf("hashing systemtest admin password: %w", err) diff --git a/internal/task/contract.go b/internal/task/contract.go new file mode 100644 index 0000000..f8f028b --- /dev/null +++ b/internal/task/contract.go @@ -0,0 +1,293 @@ +// Package task owns the workflow-neutral task contract and execution engine. +package task + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "time" + + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" +) + +var ( + ErrUnknownType = errors.New("unknown task type") + ErrInputConflict = errors.New("task input conflict") + ErrRetryUnsupported = errors.New("task retry is not supported") + ErrInvalidResult = errors.New("invalid task result") + ErrRegistryFrozen = errors.New("task registry is frozen") + ErrEffectForbidden = errors.New("external effects are forbidden during recovery") + ErrCodecPanic = errors.New("task input codec panicked") + ErrInvalidCanonical = errors.New("task input codec returned invalid canonical JSON") +) + +// Codec validates input and returns its canonical JSON representation. +type Codec interface { + Canonicalize(json.RawMessage) (json.RawMessage, error) +} + +// CodecFunc adapts a validation function to Codec. +type CodecFunc func(json.RawMessage) (json.RawMessage, error) + +func (f CodecFunc) Canonicalize(input json.RawMessage) (json.RawMessage, error) { + return f(input) +} + +func canonicalizeInput(codec Codec, input json.RawMessage) (canonical json.RawMessage, err error) { + if codec == nil { + return nil, ErrInvalidCanonical + } + defer func() { + if recovered := recover(); recovered != nil { + canonical = nil + err = fmt.Errorf("%w: %v", ErrCodecPanic, recovered) + } + }() + canonical, err = codec.Canonicalize(input) + if err != nil { + return nil, err + } + if !json.Valid(canonical) { + return nil, ErrInvalidCanonical + } + return bytes.Clone(canonical), nil +} + +// StrictJSONCodec rejects unknown fields and trailing values before applying +// validate. Re-marshalling produces stable object-key ordering. +func StrictJSONCodec[T any](validate func(*T) error) Codec { + return CodecFunc(func(input json.RawMessage) (json.RawMessage, error) { + var value T + decoder := json.NewDecoder(bytes.NewReader(input)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&value); err != nil { + return nil, fmt.Errorf("decoding task input: %w", err) + } + if err := decoder.Decode(new(any)); !errors.Is(err, io.EOF) { + return nil, errors.New("decoding task input: trailing JSON value") + } + if validate != nil { + if err := validate(&value); err != nil { + return nil, err + } + } + canonical, err := json.Marshal(value) + if err != nil { + return nil, fmt.Errorf("encoding canonical task input: %w", err) + } + return canonical, nil + }) +} + +// Definition is the complete persistent contract for a task type. +type Definition struct { + Type model.TaskType + InputVersion int + Codec Codec + RetryLimit *int + AllowRetry bool + // CanManualRetry optionally narrows AllowRetry for one failed task based on + // its durable failure evidence. Nil preserves the type-wide policy. + CanManualRetry func(*model.Task) bool +} + +func (d Definition) manualRetryAllowed(task *model.Task) bool { + return d.AllowRetry && (d.CanManualRetry == nil || d.CanManualRetry(task)) +} + +func (d Definition) validate() error { + if d.Type == "" || d.InputVersion < 1 || d.Codec == nil { + return fmt.Errorf("incomplete definition for %q", d.Type) + } + if d.RetryLimit != nil && *d.RetryLimit < 0 { + return fmt.Errorf("negative retry limit for %q", d.Type) + } + return nil +} + +// Handler owns exactly one task type. +type Handler interface { + Definition() Definition + Execute(context.Context, Execution) Result + Recover(context.Context, Execution) Result +} + +type ResultKind uint8 + +const ( + resultInvalid ResultKind = iota + resultComplete + resultSuspend + resultRetry + resultFail + resultCancel +) + +// Settlement performs database-only domain changes in the task transition +// transaction. It must be safe to invoke again after a rollback. +type Settlement func(context.Context, *repository.Repositories) error + +// Result is a closed set constructed through the helpers below. +type Result struct { + kind ResultKind + delay time.Duration + retryBackoff bool + resumeMode model.TaskResumeMode + waitReason string + failureReason string + err error + message string + settlement Settlement +} + +func Complete(message string, settlement Settlement) Result { + return Result{kind: resultComplete, message: message, settlement: settlement} +} + +func Suspend(mode model.TaskResumeMode, delay time.Duration, reason, message string, settlement Settlement) Result { + return Result{ + kind: resultSuspend, resumeMode: mode, delay: delay, + waitReason: reason, message: message, settlement: settlement, + } +} + +func Retry(err error, failureReason string, delay time.Duration, settlement Settlement) Result { + return Result{ + kind: resultRetry, resumeMode: model.TaskResumeModeRecover, delay: delay, + failureReason: failureReason, err: err, settlement: settlement, + } +} + +// RetryBackoff retries with the engine's bounded exponential backoff. +func RetryBackoff(err error, failureReason string, settlement Settlement) Result { + return Result{ + kind: resultRetry, resumeMode: model.TaskResumeModeRecover, retryBackoff: true, + failureReason: failureReason, err: err, settlement: settlement, + } +} + +func Fail(err error, failureReason string, settlement Settlement) Result { + return Result{kind: resultFail, err: err, failureReason: failureReason, settlement: settlement} +} + +func Cancel(message string, settlement Settlement) Result { + return Result{kind: resultCancel, message: message, settlement: settlement} +} + +type Resource string + +const ( + ResourceProviderMutation Resource = "provider_mutation" + ResourceDestructiveMutation Resource = "destructive_mutation" + ResourceWallet Resource = "wallet" +) + +type ( + checkpointWriter func(context.Context, any, Settlement) error + resourceRunner func(context.Context, Resource, func(context.Context) error) error +) + +// Execution is an immutable view of one fenced claim. +type Execution struct { + task model.Task + checkpoint checkpointWriter + resource resourceRunner +} + +func (e Execution) ID() int64 { return e.task.ID } +func (e Execution) Type() model.TaskType { return e.task.Type } +func (e Execution) InputVersion() int { return e.task.InputVersion } +func (e Execution) ClaimGeneration() int64 { return e.task.ClaimGeneration } +func (e Execution) Mode() model.TaskResumeMode { return e.task.ResumeMode } +func (e Execution) RetryCount() int { return e.task.RetryCount } +func (e Execution) RetryLimit() (int, bool) { + if e.task.RetryLimit == nil { + return 0, false + } + return *e.task.RetryLimit, true +} + +func (e Execution) RetryWillFail() bool { + limit, limited := e.RetryLimit() + return limited && e.RetryCount() >= limit +} +func (e Execution) CancellationRequested() bool { return e.task.CancellationRequested() } +func (e Execution) CancellationReason() string { return dereference(e.task.CancellationReason) } +func (e Execution) Input() json.RawMessage { return bytes.Clone(e.task.Input) } +func (e Execution) Checkpoint() json.RawMessage { return bytes.Clone(e.task.Checkpoint) } + +func (e Execution) WriteCheckpoint(ctx context.Context, value any) error { + return e.WriteCheckpointWith(ctx, value, nil) +} + +// WriteCheckpointWith atomically persists recovery identity and monotonic +// domain evidence before an external effect can be attempted. +func (e Execution) WriteCheckpointWith(ctx context.Context, value any, settlement Settlement) error { + if e.checkpoint == nil { + return errors.New("checkpoint writer is unavailable") + } + return e.checkpoint(ctx, value, settlement) +} + +func (e Execution) WithResource(ctx context.Context, resource Resource, fn func(context.Context) error) error { + if e.Mode() != model.TaskResumeModeExecute { + return ErrEffectForbidden + } + if e.resource == nil { + return errors.New("resource gate is unavailable") + } + return e.resource(ctx, resource, fn) +} + +// WithCheckpointedEffect admits an external effect, persists its recovery +// evidence, and only then invokes effect. A false attempted result guarantees +// that effect was not called; recovery after a process crash must still rely on +// the durable checkpoint rather than this return value. +func (e Execution) WithCheckpointedEffect( + ctx context.Context, + resource Resource, + checkpoint any, + settlement Settlement, + effect func(context.Context) error, +) (attempted bool, err error) { + if effect == nil { + return false, errors.New("external effect is required") + } + err = e.WithResource(ctx, resource, func(ctx context.Context) error { + if err := e.WriteCheckpointWith(ctx, checkpoint, settlement); err != nil { + return err + } + attempted = true + return effect(ctx) + }) + return attempted, err +} + +func DecodeInput[T any](execution Execution) (T, error) { + var value T + err := json.Unmarshal(execution.Input(), &value) + return value, err +} + +func DecodeCheckpoint[T any](execution Execution) (T, bool, error) { + var value T + raw := execution.Checkpoint() + if len(raw) == 0 { + return value, false, nil + } + if err := json.Unmarshal(raw, &value); err != nil { + return value, true, err + } + return value, true, nil +} + +func dereference(value *string) string { + if value == nil { + return "" + } + return *value +} diff --git a/internal/task/engine.go b/internal/task/engine.go new file mode 100644 index 0000000..91086de --- /dev/null +++ b/internal/task/engine.go @@ -0,0 +1,677 @@ +package task + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log/slog" + "math/rand/v2" + "runtime/debug" + "sync" + "sync/atomic" + "time" + + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" +) + +type EngineConfig struct { + Concurrency int + PollInterval time.Duration + LeaseDuration time.Duration + Retention time.Duration + ProviderMutationConcurrency int + DestructiveMutationConcurrency int + OnTaskSettled func(*model.Task, repository.TaskTransition) +} + +const ( + retryBaseDelay = 10 * time.Second + retryMaximumDelay = 5 * time.Minute + retryJitterFraction = 0.20 +) + +// Engine is the only task claimant and lease owner. +type Engine struct { + config EngineConfig + repos *repository.Repositories + registry *Registry + logger *slog.Logger + gates map[Resource]chan struct{} + recoveryMu sync.Mutex + recovery map[recoveryRequest]struct{} + recoveryWake chan struct{} + settlementRetryDelays []time.Duration + renewalRetryDelays []time.Duration + retryDelay func(int) time.Duration + lastTick atomic.Int64 +} + +type recoveryRequest struct { + id int64 + generation int64 +} + +func NewEngine(config EngineConfig, repos *repository.Repositories, registry *Registry, logger *slog.Logger) (*Engine, error) { + if config.Concurrency < 1 || config.PollInterval <= 0 || config.LeaseDuration <= config.PollInterval || + config.Retention <= 0 || config.ProviderMutationConcurrency < 1 || config.DestructiveMutationConcurrency < 1 { + return nil, errors.New("invalid task engine configuration") + } + if repos == nil || repos.Tasks == nil || registry == nil { + return nil, errors.New("task engine requires repositories and registry") + } + if logger == nil { + logger = slog.Default() + } + registry.freeze() + return &Engine{ + config: config, repos: repos, registry: registry, logger: logger, + gates: map[Resource]chan struct{}{ + ResourceProviderMutation: make(chan struct{}, config.ProviderMutationConcurrency), + ResourceDestructiveMutation: make(chan struct{}, config.DestructiveMutationConcurrency), + ResourceWallet: make(chan struct{}, 1), + }, + recovery: make(map[recoveryRequest]struct{}), + recoveryWake: make(chan struct{}, 1), + settlementRetryDelays: []time.Duration{0, time.Second, 2 * time.Second, 4 * time.Second}, + renewalRetryDelays: []time.Duration{0, time.Second, 2 * time.Second, 4 * time.Second}, + retryDelay: defaultRetryDelay, + }, nil +} + +func (e *Engine) Name() string { return "tasks" } + +func (e *Engine) Healthy() bool { + last := time.Unix(0, e.lastTick.Load()) + return !last.IsZero() && time.Since(last) <= maxDuration(3*e.config.PollInterval, time.Minute) +} + +func (e *Engine) Run(ctx context.Context) error { + var workers sync.WaitGroup + for range e.config.Concurrency { + workers.Go(func() { + e.runSlot(ctx) + }) + } + workers.Go(func() { + e.runRecoveryQueue(ctx) + }) + workers.Wait() + return nil +} + +func (e *Engine) runSlot(ctx context.Context) { + for ctx.Err() == nil { + e.lastTick.Store(time.Now().UnixNano()) + claimed, err := e.repos.Tasks.ClaimNext(ctx, e.config.LeaseDuration) + if err != nil { + if ctx.Err() == nil { + e.logger.Error("claiming task", "error", err) + } + if !sleepContext(ctx, e.config.PollInterval) { + return + } + continue + } + if claimed == nil { + if !sleepContext(ctx, e.config.PollInterval) { + return + } + continue + } + e.executeClaimSafely(ctx, claimed) + } +} + +func (e *Engine) executeClaimSafely(parent context.Context, claimed *model.Task) { + defer func() { + if recovered := recover(); recovered != nil { + e.logger.Error("task execution panicked outside the handler boundary", + "task_id", claimed.ID, + "task_type", claimed.Type, + "claim_generation", claimed.ClaimGeneration, + "error", recovered, + "stack", string(debug.Stack()), + ) + e.abandonClaim(claimed) + } + }() + e.executeClaim(parent, claimed) +} + +func (e *Engine) executeClaim(parent context.Context, claimed *model.Task) { + logger := e.logger.With("task_id", claimed.ID, "task_type", claimed.Type, "claim_generation", claimed.ClaimGeneration) + handler, ok := e.registry.Handler(claimed.Type) + if !ok { + logger.Error("claimed task has no registered handler") + if e.failClaim(parent, claimed, "handler_unavailable", fmt.Errorf("no handler is registered for task type %q", claimed.Type)) != nil { + e.abandonClaim(claimed) + } + return + } + definition, ok := e.registry.Definition(claimed.Type) + if !ok { + logger.Error("claimed task definition is unavailable") + if e.failClaim(parent, claimed, "handler_unavailable", fmt.Errorf("no definition is registered for task type %q", claimed.Type)) != nil { + e.abandonClaim(claimed) + } + return + } + if claimed.InputVersion != definition.InputVersion { + logger.Error("claimed task input version is unsupported", "input_version", claimed.InputVersion, "current_version", definition.InputVersion) + if e.failClaim(parent, claimed, "input_version_unsupported", fmt.Errorf("task input version %d is unsupported", claimed.InputVersion)) != nil { + e.abandonClaim(claimed) + } + return + } + canonical, err := canonicalizeInput(definition.Codec, claimed.Input) + if err != nil { + logger.Error("claimed task input is invalid", "error", err) + reason := "invalid_input" + if errors.Is(err, ErrCodecPanic) { + reason = "input_codec_panic" + } + if e.failClaim(parent, claimed, reason, err) != nil { + e.abandonClaim(claimed) + } + return + } + inputSum := sha256.Sum256(canonical) + if !bytes.Equal(inputSum[:], decodeHash(claimed.InputHash)) { + err := errors.New("stored task input hash does not match its canonical input") + logger.Error("claimed task input hash is invalid") + if e.failClaim(parent, claimed, "invalid_input_hash", err) != nil { + e.abandonClaim(claimed) + } + return + } + // PostgreSQL JSONB may normalize whitespace and key order. Handler input is + // always the codec's canonical representation after its hash is verified. + claimed.Input = canonical + + handlerCtx, cancelHandler := context.WithCancel(parent) + defer cancelHandler() + var leaseSafe atomic.Bool + leaseSafe.Store(true) + stopRenewal := make(chan struct{}) + renewalStopped := make(chan struct{}) + go e.renewLease(handlerCtx, claimed, &leaseSafe, cancelHandler, stopRenewal, renewalStopped) + renewalDone := false + stopLeaseRenewal := func() { + if renewalDone { + return + } + close(stopRenewal) + <-renewalStopped + renewalDone = true + } + defer stopLeaseRenewal() + + execution := Execution{ + task: *claimed, + checkpoint: func(ctx context.Context, value any, settlement Settlement) error { + if !leaseSafe.Load() { + return repository.ErrTaskLeaseLost + } + checkpoint, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("encoding checkpoint: %w", err) + } + return e.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { + if err := txRepos.Tasks.ValidateClaim(ctx, claimed.ID, claimed.ClaimGeneration); err != nil { + return err + } + if settlement != nil { + if err := invokeSettlement(ctx, settlement, txRepos); err != nil { + return err + } + } + return txRepos.Tasks.WriteCheckpoint(ctx, claimed.ID, claimed.ClaimGeneration, checkpoint) + }) + }, + resource: func(ctx context.Context, resource Resource, fn func(context.Context) error) error { + if !leaseSafe.Load() { + return repository.ErrTaskLeaseLost + } + return e.withResource(ctx, resource, func(ctx context.Context) error { + if !leaseSafe.Load() { + return repository.ErrTaskLeaseLost + } + if err := e.repos.Tasks.ValidateClaim(ctx, claimed.ID, claimed.ClaimGeneration); err != nil { + return err + } + if !leaseSafe.Load() { + return repository.ErrTaskLeaseLost + } + return fn(ctx) + }) + }, + } + + result, panicked := invokeHandler(handlerCtx, handler, execution) + if panicked != nil { + logger.Error("task handler panicked", "error", panicked, "stack", string(debug.Stack())) + stopLeaseRenewal() + if e.failClaim(parent, claimed, "handler_panic", fmt.Errorf("task handler panicked: %v", panicked)) != nil { + e.abandonClaim(claimed) + } + return + } + if parent.Err() != nil || !leaseSafe.Load() { + logger.Warn("task result discarded because the lease is uncertain") + stopLeaseRenewal() + e.abandonClaim(claimed) + return + } + if err := validateResult(result, claimed); err != nil { + logger.Error("task handler returned an invalid result", "error", err) + stopLeaseRenewal() + if e.failClaim(parent, claimed, "invalid_result", err) != nil { + e.abandonClaim(claimed) + } + return + } + stopLeaseRenewal() + if err := e.commitResult(parent, claimed, result); err != nil { + logger.Error("settling task result", "error", err) + e.abandonClaim(claimed) + } +} + +func (e *Engine) failClaim(ctx context.Context, claimed *model.Task, reason string, cause error) error { + if err := e.commitResult(ctx, claimed, Fail(cause, reason, nil)); err != nil { + e.logger.Error("recording task engine failure", "task_id", claimed.ID, "claim_generation", claimed.ClaimGeneration, "error", err) + return err + } + return nil +} + +func invokeHandler(ctx context.Context, handler Handler, execution Execution) (result Result, panicValue any) { + defer func() { + panicValue = recover() + }() + if execution.Mode() == model.TaskResumeModeRecover { + return handler.Recover(ctx, execution), nil + } + return handler.Execute(ctx, execution), nil +} + +func validateResult(result Result, _ *model.Task) error { + switch result.kind { + case resultComplete: + return nil + case resultFail: + if result.err == nil || result.failureReason == "" { + return ErrInvalidResult + } + return nil + case resultCancel: + return nil + case resultSuspend: + if result.delay < 0 || (result.resumeMode != model.TaskResumeModeExecute && result.resumeMode != model.TaskResumeModeRecover) { + return ErrInvalidResult + } + return nil + case resultRetry: + if result.delay < 0 || result.err == nil || result.failureReason == "" { + return ErrInvalidResult + } + return nil + default: + return ErrInvalidResult + } +} + +func (e *Engine) commitResult(ctx context.Context, claimed *model.Task, result Result) error { + transition := e.transitionFor(claimed, result) + var lastErr error + for _, delay := range e.settlementRetryDelays { + if delay > 0 { + if err := e.waitSettlementRetry(ctx, claimed, delay); err != nil { + return err + } + } + leaseUntil, err := e.repos.Tasks.RenewLease(ctx, claimed.ID, claimed.ClaimGeneration, e.config.LeaseDuration) + if err != nil { + return err + } + settlementDeadline := leaseUntil.Add(-e.config.LeaseDuration / 3) + if !time.Now().Before(settlementDeadline) { + return repository.ErrTaskLeaseLost + } + settlementCtx, cancel := context.WithDeadline(ctx, settlementDeadline) + lastErr = e.repos.WithTx(settlementCtx, func(txRepos *repository.Repositories) error { + if err := txRepos.Tasks.ValidateClaim(settlementCtx, claimed.ID, claimed.ClaimGeneration); err != nil { + return err + } + if result.settlement != nil { + if err := invokeSettlement(settlementCtx, result.settlement, txRepos); err != nil { + return err + } + } + return txRepos.Tasks.Settle(settlementCtx, claimed.ID, claimed.ClaimGeneration, transition) + }) + cancel() + if lastErr == nil { + e.notifyTaskSettled(claimed, transition) + return nil + } + if errors.Is(lastErr, repository.ErrTaskLeaseLost) { + return lastErr + } + } + return lastErr +} + +func (e *Engine) notifyTaskSettled(claimed *model.Task, transition repository.TaskTransition) { + if e.config.OnTaskSettled == nil { + return + } + defer func() { + if recovered := recover(); recovered != nil { + e.logger.Error("task settlement callback panicked", "task_id", claimed.ID, "panic", recovered) + } + }() + e.config.OnTaskSettled(claimed, transition) +} + +func (e *Engine) waitSettlementRetry(ctx context.Context, claimed *model.Task, delay time.Duration) error { + if _, err := e.repos.Tasks.RenewLease(ctx, claimed.ID, claimed.ClaimGeneration, e.config.LeaseDuration); err != nil { + return err + } + deadline := time.Now().Add(delay) + interval := minDuration(30*time.Second, e.config.LeaseDuration/3) + for { + remaining := time.Until(deadline) + if remaining <= 0 { + return nil + } + wait := minDuration(interval, remaining) + if !sleepContext(ctx, wait) { + return ctx.Err() + } + if wait == remaining { + return nil + } + if _, err := e.repos.Tasks.RenewLease(ctx, claimed.ID, claimed.ClaimGeneration, e.config.LeaseDuration); err != nil { + return err + } + } +} + +func invokeSettlement(ctx context.Context, settlement Settlement, repos *repository.Repositories) (err error) { + defer func() { + if recovered := recover(); recovered != nil { + err = fmt.Errorf("task settlement panicked: %v", recovered) + } + }() + return settlement(ctx, repos) +} + +func (e *Engine) transitionFor(claimed *model.Task, result Result) repository.TaskTransition { + now := time.Now() + transition := repository.TaskTransition{ + ResumeMode: model.TaskResumeModeRecover, + WaitReason: textPointer(result.waitReason), + FailureReason: textPointer(result.failureReason), + LastError: errorPointer(result.err), + StatusMessage: textPointer(result.message), + } + switch result.kind { + case resultComplete: + retention := now.Add(e.config.Retention) + transition.Status = model.TaskStatusCompleted + transition.RetentionUntil = &retention + case resultSuspend: + transition.Status = model.TaskStatusPending + transition.ResumeMode = result.resumeMode + transition.AvailableAt = now.Add(result.delay) + case resultRetry: + if claimed.RetryLimit != nil && claimed.RetryCount >= *claimed.RetryLimit { + transition.Status = model.TaskStatusFailed + } else { + delay := result.delay + if result.retryBackoff { + delay = e.retryDelay(claimed.RetryCount) + } + transition.IncrementRetry = true + transition.Status = model.TaskStatusPending + transition.AvailableAt = now.Add(delay) + } + case resultFail: + transition.Status = model.TaskStatusFailed + case resultCancel: + retention := now.Add(e.config.Retention) + transition.Status = model.TaskStatusCancelled + transition.RetentionUntil = &retention + } + return transition +} + +func defaultRetryDelay(retryCount int) time.Duration { + return retryDelayWithJitter(retryCount, rand.Float64()) +} + +func retryDelayWithJitter(retryCount int, jitterUnit float64) time.Duration { + if retryCount < 0 { + retryCount = 0 + } + delay := retryBaseDelay + for range retryCount { + if delay >= retryMaximumDelay/2 { + delay = retryMaximumDelay + break + } + delay *= 2 + } + jitterUnit = min(max(jitterUnit, 0), 1) + jitter := 1 + retryJitterFraction*(2*jitterUnit-1) + return min(time.Duration(float64(delay)*jitter), retryMaximumDelay) +} + +func (e *Engine) renewLease( + ctx context.Context, + claimed *model.Task, + leaseSafe *atomic.Bool, + cancel context.CancelFunc, + stop <-chan struct{}, + stopped chan<- struct{}, +) { + defer close(stopped) + interval := minDuration(30*time.Second, e.config.LeaseDuration/3) + confirmedUntil := time.Now().Add(e.config.LeaseDuration) + if claimed.LeaseUntil != nil && claimed.LeaseUntil.After(time.Now()) { + confirmedUntil = *claimed.LeaseUntil + } + safetyMargin := e.config.LeaseDuration / 3 + timer := time.NewTimer(interval) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + leaseSafe.Store(false) + return + case <-stop: + return + case <-timer.C: + } + deadline := confirmedUntil.Add(-safetyMargin) + var ( + err error + renewed time.Time + ) + for attempt, delay := range e.renewalRetryDelays { + if attempt > 0 { + retry, stopped := waitRenewalRetry(ctx, stop, deadline, delay) + if stopped { + return + } + if !retry { + leaseSafe.Store(false) + cancel() + return + } + } + if !time.Now().Before(deadline) { + err = context.DeadlineExceeded + break + } + renewCtx, cancelRenew := context.WithDeadline(ctx, deadline) + renewed, err = e.repos.Tasks.RenewLease(renewCtx, claimed.ID, claimed.ClaimGeneration, e.config.LeaseDuration) + cancelRenew() + if err == nil { + confirmedUntil = renewed + e.lastTick.Store(time.Now().UnixNano()) + break + } + if errors.Is(err, repository.ErrTaskLeaseLost) { + break + } + } + if err != nil { + leaseSafe.Store(false) + cancel() + return + } + timer.Reset(interval) + } +} + +func waitRenewalRetry(ctx context.Context, stop <-chan struct{}, deadline time.Time, delay time.Duration) (retry, stopped bool) { + remaining := time.Until(deadline) + if remaining <= 0 || delay >= remaining { + return false, false + } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return false, false + case <-stop: + return false, true + case <-timer.C: + return true, false + } +} + +func (e *Engine) abandonClaim(claimed *model.Task) { + ctx, cancel := context.WithTimeout(context.Background(), minDuration(e.config.PollInterval, 5*time.Second)) + defer cancel() + err := e.repos.Tasks.ShortenLease(ctx, claimed.ID, claimed.ClaimGeneration, e.config.PollInterval) + if err == nil || errors.Is(err, repository.ErrTaskLeaseLost) { + return + } + e.enqueueRecovery(recoveryRequest{id: claimed.ID, generation: claimed.ClaimGeneration}) +} + +func (e *Engine) runRecoveryQueue(ctx context.Context) { + for { + if ctx.Err() != nil { + return + } + request, ok := e.nextRecovery() + if !ok { + select { + case <-ctx.Done(): + return + case <-e.recoveryWake: + continue + } + } + recoveryCtx, cancel := context.WithTimeout(context.Background(), minDuration(e.config.PollInterval, 5*time.Second)) + err := e.repos.Tasks.ShortenLease(recoveryCtx, request.id, request.generation, e.config.PollInterval) + cancel() + if err != nil && !errors.Is(err, repository.ErrTaskLeaseLost) { + e.logger.Error("shortening uncertain task lease", "task_id", request.id, "claim_generation", request.generation, "error", err) + if sleepContext(ctx, e.config.PollInterval) { + e.enqueueRecovery(request) + } + } + } +} + +func (e *Engine) enqueueRecovery(request recoveryRequest) { + e.recoveryMu.Lock() + e.recovery[request] = struct{}{} + e.recoveryMu.Unlock() + select { + case e.recoveryWake <- struct{}{}: + default: + } +} + +func (e *Engine) nextRecovery() (recoveryRequest, bool) { + e.recoveryMu.Lock() + defer e.recoveryMu.Unlock() + for request := range e.recovery { + delete(e.recovery, request) + return request, true + } + return recoveryRequest{}, false +} + +func (e *Engine) withResource(ctx context.Context, resource Resource, fn func(context.Context) error) error { + gate, ok := e.gates[resource] + if !ok || fn == nil { + return fmt.Errorf("unknown task resource %q", resource) + } + select { + case gate <- struct{}{}: + defer func() { <-gate }() + return fn(ctx) + case <-ctx.Done(): + return ctx.Err() + } +} + +func sleepContext(ctx context.Context, delay time.Duration) bool { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +func minDuration(left, right time.Duration) time.Duration { + if left < right { + return left + } + return right +} + +func maxDuration(left, right time.Duration) time.Duration { + if left > right { + return left + } + return right +} + +func textPointer(value string) *string { + if value == "" { + return nil + } + return &value +} + +func errorPointer(err error) *string { + if err == nil { + return nil + } + message := err.Error() + return &message +} + +func decodeHash(value string) []byte { + decoded, err := hex.DecodeString(value) + if err != nil { + return nil + } + return decoded +} diff --git a/internal/task/registry.go b/internal/task/registry.go new file mode 100644 index 0000000..764d5c0 --- /dev/null +++ b/internal/task/registry.go @@ -0,0 +1,87 @@ +package task + +import ( + "fmt" + "slices" + "sync" + + "github.com/strahe/synaps3/internal/model" +) + +// Registry is immutable after Engine starts. +type Registry struct { + mu sync.RWMutex + handlers map[model.TaskType]Handler + definitions map[model.TaskType]Definition + frozen bool +} + +func NewRegistry() *Registry { + return &Registry{ + handlers: make(map[model.TaskType]Handler), + definitions: make(map[model.TaskType]Definition), + } +} + +func (r *Registry) Register(handler Handler) error { + if r == nil || handler == nil { + return fmt.Errorf("registering nil handler: %w", ErrUnknownType) + } + definition := handler.Definition() + if err := definition.validate(); err != nil { + return err + } + r.mu.Lock() + defer r.mu.Unlock() + if r.frozen { + return ErrRegistryFrozen + } + if _, exists := r.handlers[definition.Type]; exists { + return fmt.Errorf("handler %q already registered", definition.Type) + } + definition.RetryLimit = cloneInt(definition.RetryLimit) + r.handlers[definition.Type] = handler + r.definitions[definition.Type] = definition + return nil +} + +func (r *Registry) freeze() { + r.mu.Lock() + defer r.mu.Unlock() + r.frozen = true +} + +func (r *Registry) Handler(taskType model.TaskType) (Handler, bool) { + if r == nil { + return nil, false + } + r.mu.RLock() + defer r.mu.RUnlock() + handler, ok := r.handlers[taskType] + return handler, ok +} + +func (r *Registry) Definition(taskType model.TaskType) (Definition, bool) { + if r == nil { + return Definition{}, false + } + r.mu.RLock() + defer r.mu.RUnlock() + definition, ok := r.definitions[taskType] + if !ok { + return Definition{}, false + } + definition.RetryLimit = cloneInt(definition.RetryLimit) + return definition, true +} + +func (r *Registry) Types() []model.TaskType { + r.mu.RLock() + defer r.mu.RUnlock() + types := make([]model.TaskType, 0, len(r.handlers)) + for taskType := range r.handlers { + types = append(types, taskType) + } + slices.Sort(types) + return types +} diff --git a/internal/task/service.go b/internal/task/service.go new file mode 100644 index 0000000..c18eddf --- /dev/null +++ b/internal/task/service.go @@ -0,0 +1,226 @@ +package task + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" +) + +type Service struct { + registry *Registry + repos *repository.Repositories + retention time.Duration +} + +type EnqueueRequest struct { + Type model.TaskType + IdempotencyKey string + Input any + SubjectType string + SubjectKey string + AvailableAt time.Time +} + +func NewService(registry *Registry, repos *repository.Repositories, retention time.Duration) (*Service, error) { + if registry == nil || repos == nil || repos.Tasks == nil || retention <= 0 { + return nil, errors.New("task service requires registry, repository, and positive retention") + } + return &Service{registry: registry, repos: repos, retention: retention}, nil +} + +func (s *Service) Enqueue(ctx context.Context, request EnqueueRequest) (*model.Task, bool, error) { + task, err := s.prepare(request) + if err != nil { + return nil, false, err + } + return s.enqueuePrepared(ctx, s.repos.Tasks, task) +} + +// EnqueueInTransaction uses a transaction-scoped repository set supplied by +// the caller. It exists for domain mutations that must bind the task in the +// same transaction; callers still cannot bypass registry validation. +func (s *Service) EnqueueInTransaction( + ctx context.Context, + txRepos *repository.Repositories, + request EnqueueRequest, +) (*model.Task, bool, error) { + if txRepos == nil || txRepos.Tasks == nil { + return nil, false, errors.New("transaction task repository is required") + } + prepared, err := s.prepare(request) + if err != nil { + return nil, false, err + } + return s.enqueuePrepared(ctx, txRepos.Tasks, prepared) +} + +// EnqueueOrReactivateTerminalInTransaction is the cached-content upload-plan +// escape hatch. It preserves one idempotency identity while allowing a new +// live reference to restart work that ended before creating any copies. +func (s *Service) EnqueueOrReactivateTerminalInTransaction( + ctx context.Context, + txRepos *repository.Repositories, + request EnqueueRequest, +) (*model.Task, bool, error) { + if txRepos == nil || txRepos.Tasks == nil { + return nil, false, errors.New("transaction task repository is required") + } + if request.Type != model.TaskTypeUploadPlan { + return nil, false, fmt.Errorf("terminal reactivation is limited to upload plans: %w", repository.ErrInvalidInput) + } + prepared, err := s.prepare(request) + if err != nil { + return nil, false, err + } + stored, created, err := s.enqueuePrepared(ctx, txRepos.Tasks, prepared) + if err != nil || created { + return stored, created, err + } + switch stored.Status { + case model.TaskStatusPending, model.TaskStatusRunning: + return stored, false, nil + case model.TaskStatusFailed, model.TaskStatusCancelled: + if err := txRepos.Tasks.ReactivateTerminal(ctx, stored.ID); err != nil { + return nil, false, err + } + stored, err = txRepos.Tasks.GetByID(ctx, stored.ID) + return stored, false, err + case model.TaskStatusCompleted: + return nil, false, fmt.Errorf("completed task conflicts with cached content: %w", repository.ErrConflict) + default: + return nil, false, fmt.Errorf("task has invalid status %q: %w", stored.Status, repository.ErrConflict) + } +} + +// EnqueueTx creates a task and binds its domain owner in one transaction. +// bind must perform database work only and tolerate a transaction retry. +func (s *Service) EnqueueTx( + ctx context.Context, + request EnqueueRequest, + bind func(context.Context, *repository.Repositories, *model.Task, bool) error, +) (*model.Task, bool, error) { + prepared, err := s.prepare(request) + if err != nil { + return nil, false, err + } + var stored *model.Task + var created bool + err = s.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { + var enqueueErr error + stored, created, enqueueErr = s.enqueuePrepared(ctx, txRepos.Tasks, prepared) + if enqueueErr != nil { + return enqueueErr + } + if bind != nil { + return bind(ctx, txRepos, stored, created) + } + return nil + }) + return stored, created, err +} + +func (s *Service) prepare(request EnqueueRequest) (*model.Task, error) { + definition, ok := s.registry.Definition(request.Type) + if !ok { + return nil, fmt.Errorf("%w: %s", ErrUnknownType, request.Type) + } + if request.IdempotencyKey == "" || (request.SubjectType == "") != (request.SubjectKey == "") { + return nil, fmt.Errorf("task identity is incomplete: %w", repository.ErrInvalidInput) + } + raw, err := json.Marshal(request.Input) + if err != nil { + return nil, fmt.Errorf("encoding task input: %w", err) + } + canonical, err := canonicalizeInput(definition.Codec, raw) + if err != nil { + return nil, fmt.Errorf("validating %s input: %w", request.Type, err) + } + sum := sha256.Sum256(canonical) + availableAt := request.AvailableAt + if availableAt.IsZero() { + availableAt = time.Now() + } + task := &model.Task{ + Type: request.Type, IdempotencyKey: request.IdempotencyKey, + InputVersion: definition.InputVersion, Input: canonical, + InputHash: hex.EncodeToString(sum[:]), + Status: model.TaskStatusPending, ResumeMode: model.TaskResumeModeExecute, + AvailableAt: availableAt, RetryLimit: cloneInt(definition.RetryLimit), + } + if request.SubjectType != "" { + task.SubjectType = &request.SubjectType + task.SubjectKey = &request.SubjectKey + } + return task, nil +} + +func (s *Service) enqueuePrepared(ctx context.Context, tasks repository.TaskRepository, task *model.Task) (*model.Task, bool, error) { + stored, created, err := tasks.Enqueue(ctx, task) + if err != nil { + return nil, false, err + } + if stored.InputVersion != task.InputVersion || stored.InputHash != task.InputHash { + return nil, false, fmt.Errorf("%w for %s/%s", ErrInputConflict, task.Type, task.IdempotencyKey) + } + return stored, created, nil +} + +func (s *Service) Get(ctx context.Context, id int64) (*model.Task, error) { + return s.repos.Tasks.GetByID(ctx, id) +} + +func (s *Service) Retry(ctx context.Context, id int64) error { + task, err := s.repos.Tasks.GetByID(ctx, id) + if err != nil { + return err + } + if task == nil || task.Status != model.TaskStatusFailed { + return repository.ErrNotFound + } + definition, ok := s.registry.Definition(task.Type) + if !ok { + return fmt.Errorf("%w: %s", ErrUnknownType, task.Type) + } + if !definition.manualRetryAllowed(task) { + return ErrRetryUnsupported + } + return s.repos.Tasks.RetryFailed(ctx, id) +} + +func (s *Service) Acknowledge(ctx context.Context, id int64) error { + return s.repos.Tasks.AcknowledgeFailed(ctx, id, s.retention) +} + +func (s *Service) WakeInTransaction(ctx context.Context, txRepos *repository.Repositories, ids []int64) (int, error) { + if txRepos == nil || txRepos.Tasks == nil { + return 0, errors.New("transaction task repository is required") + } + return txRepos.Tasks.WakePending(ctx, ids) +} + +func (s *Service) Retryable(task *model.Task) bool { + if task == nil || task.Status != model.TaskStatusFailed { + return false + } + definition, ok := s.registry.Definition(task.Type) + return ok && definition.manualRetryAllowed(task) +} + +func (s *Service) Acknowledgeable(task *model.Task) bool { + return task != nil && task.Status == model.TaskStatusFailed && task.AcknowledgedAt == nil +} + +func cloneInt(value *int) *int { + if value == nil { + return nil + } + cloned := *value + return &cloned +} diff --git a/internal/task/task_test.go b/internal/task/task_test.go new file mode 100644 index 0000000..ac00b43 --- /dev/null +++ b/internal/task/task_test.go @@ -0,0 +1,1260 @@ +package task + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/testutil" + "github.com/uptrace/bun" +) + +const testTaskType model.TaskType = "test_operation" + +type testInput struct { + Value string `json:"value"` +} + +type scriptedHandler struct { + definition Definition + execute func(context.Context, Execution) Result + recover func(context.Context, Execution) Result +} + +type panicError struct{} + +func (panicError) Error() string { panic("error formatting failure") } + +func (h scriptedHandler) Definition() Definition { return h.definition } + +func (h scriptedHandler) Execute(ctx context.Context, execution Execution) Result { + if h.execute == nil { + return Complete("executed", nil) + } + return h.execute(ctx, execution) +} + +func (h scriptedHandler) Recover(ctx context.Context, execution Execution) Result { + if h.recover == nil { + return Complete("recovered", nil) + } + return h.recover(ctx, execution) +} + +func testDefinition(retryLimit *int, allowRetry bool) Definition { + return Definition{ + Type: testTaskType, + InputVersion: 1, + Codec: StrictJSONCodec(func(input *testInput) error { + if input.Value == "" { + return errors.New("value is required") + } + return nil + }), + RetryLimit: retryLimit, + AllowRetry: allowRetry, + } +} + +type taskHarness struct { + db *bun.DB + repos *repository.Repositories + registry *Registry + service *Service + engine *Engine +} + +func newTaskHarness(t *testing.T, handler Handler, config *EngineConfig) taskHarness { + t.Helper() + db := testutil.NewTestFileDB(t) + repos := repository.NewRepositories(db) + registry := NewRegistry() + if err := registry.Register(handler); err != nil { + t.Fatalf("register handler: %v", err) + } + service, err := NewService(registry, repos, time.Hour) + if err != nil { + t.Fatalf("new service: %v", err) + } + engineConfig := EngineConfig{ + Concurrency: 4, + PollInterval: 10 * time.Millisecond, + LeaseDuration: 5 * time.Second, + Retention: time.Hour, + ProviderMutationConcurrency: 4, + DestructiveMutationConcurrency: 2, + } + if config != nil { + engineConfig = *config + } + engine, err := NewEngine(engineConfig, repos, registry, slog.Default()) + if err != nil { + t.Fatalf("new engine: %v", err) + } + return taskHarness{db: db, repos: repos, registry: registry, service: service, engine: engine} +} + +func enqueueTestTask(t *testing.T, harness taskHarness, key, value string) *model.Task { + t.Helper() + taskRow, created, err := harness.service.Enqueue(t.Context(), EnqueueRequest{ + Type: testTaskType, + IdempotencyKey: key, + Input: testInput{Value: value}, + SubjectType: "test_subject", + SubjectKey: key, + }) + if err != nil { + t.Fatalf("enqueue task: %v", err) + } + if !created { + t.Fatalf("task %s was not created", key) + } + return taskRow +} + +func claimTestTask(t *testing.T, harness taskHarness) *model.Task { + t.Helper() + claimed, err := harness.repos.Tasks.ClaimNext(t.Context(), harness.engine.config.LeaseDuration) + if err != nil { + t.Fatalf("claim task: %v", err) + } + if claimed == nil { + t.Fatal("claim returned no task") + } + return claimed +} + +func TestServiceCanonicalEnqueueAndConflict(t *testing.T) { + limit := 5 + harness := newTaskHarness(t, scriptedHandler{definition: testDefinition(&limit, true)}, nil) + + first, created, err := harness.service.Enqueue(t.Context(), EnqueueRequest{ + Type: testTaskType, IdempotencyKey: "same", Input: map[string]any{"value": "alpha"}, + }) + if err != nil || !created { + t.Fatalf("first enqueue = task:%#v created:%v err:%v", first, created, err) + } + if string(first.Input) != `{"value":"alpha"}` { + t.Fatalf("canonical input = %s", first.Input) + } + + same, created, err := harness.service.Enqueue(t.Context(), EnqueueRequest{ + Type: testTaskType, IdempotencyKey: "same", Input: testInput{Value: "alpha"}, + }) + if err != nil || created || same.ID != first.ID { + t.Fatalf("idempotent enqueue = task:%#v created:%v err:%v", same, created, err) + } + + _, _, err = harness.service.Enqueue(t.Context(), EnqueueRequest{ + Type: testTaskType, IdempotencyKey: "same", Input: testInput{Value: "beta"}, + }) + if !errors.Is(err, ErrInputConflict) { + t.Fatalf("conflicting enqueue error = %v, want %v", err, ErrInputConflict) + } + + _, _, err = harness.service.Enqueue(t.Context(), EnqueueRequest{ + Type: testTaskType, IdempotencyKey: "unknown-field", Input: map[string]any{"value": "alpha", "extra": true}, + }) + if err == nil { + t.Fatal("enqueue with unknown input field succeeded") + } +} + +func TestServiceWakeMakesFuturePendingTaskReady(t *testing.T) { + limit := 5 + harness := newTaskHarness(t, scriptedHandler{definition: testDefinition(&limit, true)}, nil) + row, created, err := harness.service.Enqueue(t.Context(), EnqueueRequest{ + Type: testTaskType, IdempotencyKey: "wake", Input: testInput{Value: "wake"}, + AvailableAt: time.Now().Add(time.Hour), + }) + if err != nil || !created { + t.Fatalf("enqueue future task = %#v created=%v err=%v", row, created, err) + } + if claimed, err := harness.repos.Tasks.ClaimNext(t.Context(), time.Minute); err != nil || claimed != nil { + t.Fatalf("claim before wake = %#v err=%v", claimed, err) + } + if err := harness.repos.WithTx(t.Context(), func(txRepos *repository.Repositories) error { + woken, wakeErr := harness.service.WakeInTransaction(t.Context(), txRepos, []int64{row.ID}) + if wakeErr != nil { + return wakeErr + } + if woken != 1 { + return fmt.Errorf("woken tasks = %d, want 1", woken) + } + return nil + }); err != nil { + t.Fatalf("wake future task: %v", err) + } + claimed := claimTestTask(t, harness) + if claimed.ID != row.ID { + t.Fatalf("claimed task = %d, want %d", claimed.ID, row.ID) + } +} + +func TestEngineConstructionFreezesRegistry(t *testing.T) { + limit := 5 + harness := newTaskHarness(t, scriptedHandler{definition: testDefinition(&limit, true)}, nil) + err := harness.registry.Register(scriptedHandler{definition: Definition{ + Type: "late_operation", InputVersion: 1, + Codec: StrictJSONCodec(func(input *testInput) error { return nil }), + }}) + if !errors.Is(err, ErrRegistryFrozen) { + t.Fatalf("late registry mutation error = %v, want frozen registry", err) + } +} + +func TestRegistryFreezesDefinitionAtRegistration(t *testing.T) { + limit := 5 + handler := &scriptedHandler{definition: testDefinition(&limit, true)} + registry := NewRegistry() + if err := registry.Register(handler); err != nil { + t.Fatalf("register handler: %v", err) + } + handler.definition.InputVersion = 2 + handler.definition.AllowRetry = false + limit = 9 + + definition, ok := registry.Definition(testTaskType) + if !ok || definition.InputVersion != 1 || !definition.AllowRetry || definition.RetryLimit == nil || *definition.RetryLimit != 5 { + t.Fatalf("registered definition changed = %#v", definition) + } + *definition.RetryLimit = 11 + again, ok := registry.Definition(testTaskType) + if !ok || again.RetryLimit == nil || *again.RetryLimit != 5 { + t.Fatalf("returned definition mutated registry = %#v", again) + } +} + +func TestEngineSettlesAllFiveStates(t *testing.T) { + limit := 0 + tests := []struct { + name string + result Result + wantStatus model.TaskStatus + wantMode model.TaskResumeMode + wantRetry int + wantRetention bool + }{ + {name: "completed", result: Complete("done", nil), wantStatus: model.TaskStatusCompleted, wantMode: model.TaskResumeModeRecover, wantRetention: true}, + {name: "pending", result: Suspend(model.TaskResumeModeExecute, time.Hour, "scheduled", "later", nil), wantStatus: model.TaskStatusPending, wantMode: model.TaskResumeModeExecute}, + {name: "failed", result: Fail(errors.New("permanent"), "permanent_failure", nil), wantStatus: model.TaskStatusFailed, wantMode: model.TaskResumeModeRecover}, + {name: "cancelled", result: Cancel("superseded", nil), wantStatus: model.TaskStatusCancelled, wantMode: model.TaskResumeModeRecover, wantRetention: true}, + {name: "retry limit reached", result: Retry(errors.New("temporary"), "temporary_failure", 0, nil), wantStatus: model.TaskStatusFailed, wantMode: model.TaskResumeModeRecover}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&limit, true), + execute: func(context.Context, Execution) Result { return tt.result }, + }, nil) + row := enqueueTestTask(t, harness, tt.name, tt.name) + claimed := claimTestTask(t, harness) + harness.engine.executeClaim(t.Context(), claimed) + + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil { + t.Fatalf("get settled task: %v", err) + } + if stored.Status != tt.wantStatus || stored.ResumeMode != tt.wantMode || stored.RetryCount != tt.wantRetry { + t.Fatalf("settled task = status:%s mode:%s retries:%d", stored.Status, stored.ResumeMode, stored.RetryCount) + } + if (stored.RetentionUntil != nil) != tt.wantRetention { + t.Fatalf("retention = %v, want present %v", stored.RetentionUntil, tt.wantRetention) + } + if stored.Status != model.TaskStatusPending && stored.FinishedAt == nil { + t.Fatal("terminal task has no finished time") + } + if stored.Status == model.TaskStatusCompleted || stored.Status == model.TaskStatusCancelled { + again, claimErr := harness.repos.Tasks.ClaimNext(t.Context(), time.Minute) + if claimErr != nil || again != nil { + t.Fatalf("terminal task revived = %#v, err=%v", again, claimErr) + } + } + }) + } +} + +func TestEngineNotifiesAfterSettlementCommits(t *testing.T) { + limit := 1 + var notified atomic.Int64 + config := EngineConfig{ + Concurrency: 1, PollInterval: 10 * time.Millisecond, LeaseDuration: 5 * time.Second, Retention: time.Hour, + ProviderMutationConcurrency: 1, DestructiveMutationConcurrency: 1, + OnTaskSettled: func(taskRow *model.Task, transition repository.TaskTransition) { + if taskRow.IdempotencyKey != "notify" || transition.Status != model.TaskStatusCompleted { + t.Errorf("settlement notification = task:%#v transition:%#v", taskRow, transition) + } + notified.Add(1) + }, + } + harness := newTaskHarness(t, scriptedHandler{definition: testDefinition(&limit, true)}, &config) + enqueueTestTask(t, harness, "notify", "notify") + claimed := claimTestTask(t, harness) + harness.engine.executeClaim(t.Context(), claimed) + if notified.Load() != 1 { + t.Fatalf("settlement notifications = %d, want 1", notified.Load()) + } +} + +func TestRetryBackoffUsesPersistedRetryCount(t *testing.T) { + limit := 5 + harness := newTaskHarness(t, scriptedHandler{definition: testDefinition(&limit, true)}, nil) + harness.engine.retryDelay = func(retryCount int) time.Duration { + if retryCount != 3 { + t.Fatalf("retry count = %d, want 3", retryCount) + } + return 37 * time.Second + } + now := time.Now() + transition := harness.engine.transitionFor(&model.Task{RetryCount: 3, RetryLimit: &limit}, RetryBackoff(errors.New("temporary"), "temporary_failure", nil)) + if transition.Status != model.TaskStatusPending || !transition.IncrementRetry { + t.Fatalf("transition = %#v, want pending retry", transition) + } + if transition.AvailableAt.Before(now.Add(37*time.Second)) || transition.AvailableAt.After(time.Now().Add(37*time.Second)) { + t.Fatalf("available at = %s, want 37 second delay", transition.AvailableAt) + } +} + +func TestRetryDelayIsExponentialJitteredAndCapped(t *testing.T) { + tests := []struct { + name string + retryCount int + jitter float64 + want time.Duration + }{ + {name: "first low jitter", retryCount: 0, jitter: 0, want: 8 * time.Second}, + {name: "first midpoint", retryCount: 0, jitter: 0.5, want: 10 * time.Second}, + {name: "third midpoint", retryCount: 2, jitter: 0.5, want: 40 * time.Second}, + {name: "negative count", retryCount: -1, jitter: 0.5, want: 10 * time.Second}, + {name: "hard cap", retryCount: 20, jitter: 1, want: 5 * time.Minute}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := retryDelayWithJitter(tt.retryCount, tt.jitter); got != tt.want { + t.Fatalf("delay = %s, want %s", got, tt.want) + } + }) + } +} + +func TestServiceRetryForcesRecoverAndAcknowledgeStartsRetention(t *testing.T) { + limit := 5 + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&limit, true), + execute: func(context.Context, Execution) Result { + return Fail(errors.New("failed"), "test_failure", nil) + }, + }, nil) + row := enqueueTestTask(t, harness, "retry", "retry") + harness.engine.executeClaim(t.Context(), claimTestTask(t, harness)) + + if err := harness.service.Retry(t.Context(), row.ID); err != nil { + t.Fatalf("retry failed task: %v", err) + } + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusPending || stored.ResumeMode != model.TaskResumeModeRecover || stored.RetryCount != 0 { + t.Fatalf("retried task = %#v, err=%v", stored, err) + } + + claimed := claimTestTask(t, harness) + if claimed.ResumeMode != model.TaskResumeModeRecover { + t.Fatalf("manual retry claim mode = %s", claimed.ResumeMode) + } + if err := harness.repos.Tasks.Settle(t.Context(), claimed.ID, claimed.ClaimGeneration, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, + }); err != nil { + t.Fatalf("fail retried task: %v", err) + } + if err := harness.service.Acknowledge(t.Context(), row.ID); err != nil { + t.Fatalf("acknowledge failed task: %v", err) + } + stored, err = harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.AcknowledgedAt == nil || stored.RetentionUntil == nil { + t.Fatalf("acknowledged task = %#v, err=%v", stored, err) + } +} + +func TestServiceManualRetryPredicateUsesFailureEvidence(t *testing.T) { + limit := 5 + definition := testDefinition(&limit, true) + definition.CanManualRetry = func(task *model.Task) bool { + return task != nil && task.FailureReason != nil && *task.FailureReason != "unsafe_outcome" + } + harness := newTaskHarness(t, scriptedHandler{ + definition: definition, + execute: func(context.Context, Execution) Result { + return Fail(errors.New("outcome is unknown"), "unsafe_outcome", nil) + }, + }, nil) + row := enqueueTestTask(t, harness, "unsafe-retry", "unsafe-retry") + harness.engine.executeClaim(t.Context(), claimTestTask(t, harness)) + + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored == nil || stored.Status != model.TaskStatusFailed { + t.Fatalf("failed task = %#v, err=%v", stored, err) + } + if harness.service.Retryable(stored) { + t.Fatal("unsafe failed task is retryable") + } + if err := harness.service.Retry(t.Context(), row.ID); !errors.Is(err, ErrRetryUnsupported) { + t.Fatalf("Retry = %v, want ErrRetryUnsupported", err) + } +} + +func TestCancellationWakesPendingTaskAndForcesRecovery(t *testing.T) { + limit := 5 + var executed atomic.Bool + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&limit, true), + execute: func(context.Context, Execution) Result { + executed.Store(true) + return Complete("must not execute", nil) + }, + recover: func(_ context.Context, execution Execution) Result { + if !execution.CancellationRequested() || execution.CancellationReason() != "owner stopped" { + return Fail(errors.New("missing cancellation request"), "test_failure", nil) + } + return Cancel("cancelled", nil) + }, + }, nil) + row, created, err := harness.service.Enqueue(t.Context(), EnqueueRequest{ + Type: testTaskType, IdempotencyKey: "cancel-pending", Input: testInput{Value: "cancel-pending"}, + AvailableAt: time.Now().Add(24 * time.Hour), + }) + if err != nil || !created { + t.Fatalf("enqueue delayed task = %#v created=%v err=%v", row, created, err) + } + if err := harness.repos.Tasks.RequestCancellation(t.Context(), row.ID, "owner stopped"); err != nil { + t.Fatalf("request cancellation: %v", err) + } + claimed := claimTestTask(t, harness) + if claimed.ResumeMode != model.TaskResumeModeRecover || claimed.CancellationRequestedAt == nil || + claimed.AvailableAt.Before(*claimed.CancellationRequestedAt) { + t.Fatalf("cancelled claim = %#v", claimed) + } + harness.engine.executeClaim(t.Context(), claimed) + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusCancelled || executed.Load() { + t.Fatalf("cancelled task = %#v executed=%v err=%v", stored, executed.Load(), err) + } +} + +func TestCancellationOverridesPendingSettlementScheduleAndMode(t *testing.T) { + limit := 5 + harness := newTaskHarness(t, scriptedHandler{definition: testDefinition(&limit, true)}, nil) + row := enqueueTestTask(t, harness, "cancel-running", "cancel-running") + claimed := claimTestTask(t, harness) + if err := harness.repos.Tasks.RequestCancellation(t.Context(), row.ID, "owner stopped"); err != nil { + t.Fatalf("request cancellation: %v", err) + } + if err := harness.repos.Tasks.Settle(t.Context(), claimed.ID, claimed.ClaimGeneration, repository.TaskTransition{ + Status: model.TaskStatusPending, ResumeMode: model.TaskResumeModeExecute, + AvailableAt: time.Now().Add(24 * time.Hour), + }); err != nil { + t.Fatalf("suspend cancellation-requested task: %v", err) + } + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusPending || stored.ResumeMode != model.TaskResumeModeRecover || + stored.CancellationRequestedAt == nil || stored.AvailableAt.Before(*stored.CancellationRequestedAt) || + stored.AvailableAt.After(time.Now().Add(time.Second)) { + t.Fatalf("suspended cancellation-requested task = %#v, err=%v", stored, err) + } +} + +func TestExpiredClaimIsRecoveredAndStaleGenerationIsFenced(t *testing.T) { + limit := 5 + var executes, recovers atomic.Int64 + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&limit, true), + execute: func(context.Context, Execution) Result { + executes.Add(1) + return Complete("executed", nil) + }, + recover: func(context.Context, Execution) Result { + recovers.Add(1) + return Complete("recovered", nil) + }, + }, nil) + enqueueTestTask(t, harness, "takeover", "takeover") + stale := claimTestTask(t, harness) + if _, err := harness.db.NewRaw(`UPDATE tasks SET lease_until = ? WHERE id = ?`, time.Now().Add(-time.Second), stale.ID).Exec(t.Context()); err != nil { + t.Fatalf("expire first claim: %v", err) + } + fresh := claimTestTask(t, harness) + if fresh.ClaimGeneration != stale.ClaimGeneration+1 || fresh.ResumeMode != model.TaskResumeModeRecover { + t.Fatalf("takeover claim = generation:%d mode:%s", fresh.ClaimGeneration, fresh.ResumeMode) + } + if err := harness.repos.Tasks.WriteCheckpoint(t.Context(), stale.ID, stale.ClaimGeneration, []byte(`{"stale":true}`)); !errors.Is(err, repository.ErrTaskLeaseLost) { + t.Fatalf("stale checkpoint error = %v", err) + } + if err := harness.repos.Tasks.Settle(t.Context(), stale.ID, stale.ClaimGeneration, repository.TaskTransition{ + Status: model.TaskStatusCompleted, ResumeMode: model.TaskResumeModeRecover, RetentionUntil: new(time.Now().Add(time.Hour)), + }); !errors.Is(err, repository.ErrTaskLeaseLost) { + t.Fatalf("stale settlement error = %v", err) + } + + harness.engine.executeClaim(t.Context(), fresh) + if executes.Load() != 0 || recovers.Load() != 1 { + t.Fatalf("handler calls = execute:%d recover:%d", executes.Load(), recovers.Load()) + } +} + +func TestEngineRejectsCorruptInputBeforeHandler(t *testing.T) { + limit := 5 + var called atomic.Bool + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&limit, true), + execute: func(context.Context, Execution) Result { + called.Store(true) + return Complete("unexpected", nil) + }, + }, nil) + row := enqueueTestTask(t, harness, "corrupt", "corrupt") + if _, err := harness.db.NewRaw(`UPDATE tasks SET input_hash = ? WHERE id = ?`, "00", row.ID).Exec(t.Context()); err != nil { + t.Fatalf("corrupt task hash: %v", err) + } + harness.engine.executeClaim(t.Context(), claimTestTask(t, harness)) + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusFailed || stored.FailureReason == nil || *stored.FailureReason != "invalid_input_hash" { + t.Fatalf("corrupt task result = %#v, err=%v", stored, err) + } + if called.Load() { + t.Fatal("handler ran with corrupt input") + } +} + +func TestEngineAcceptsDatabaseJSONNormalizationAndUsesCanonicalInput(t *testing.T) { + limit := 5 + var gotInput string + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&limit, true), + execute: func(_ context.Context, execution Execution) Result { + gotInput = string(execution.Input()) + return Complete("normalized", nil) + }, + }, nil) + row := enqueueTestTask(t, harness, "normalized", "normalized") + if _, err := harness.db.NewRaw(`UPDATE task_payloads SET input_json = ? WHERE task_id = ?`, `{ "value" : "normalized" }`, row.ID).Exec(t.Context()); err != nil { + t.Fatalf("normalize stored task JSON: %v", err) + } + harness.engine.executeClaim(t.Context(), claimTestTask(t, harness)) + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusCompleted { + t.Fatalf("normalized task result = %#v, err=%v", stored, err) + } + if gotInput != `{"value":"normalized"}` { + t.Fatalf("handler input = %q, want canonical JSON", gotInput) + } +} + +func TestCodecPanicFailsClosedDuringEnqueueAndExecution(t *testing.T) { + limit := 5 + var panicCodec atomic.Bool + var handlerCalled atomic.Bool + strict := StrictJSONCodec(func(input *testInput) error { + if input.Value == "" { + return errors.New("value is required") + } + return nil + }) + definition := testDefinition(&limit, true) + definition.Codec = CodecFunc(func(input json.RawMessage) (json.RawMessage, error) { + if panicCodec.Load() { + panic("codec failure") + } + return strict.Canonicalize(input) + }) + harness := newTaskHarness(t, scriptedHandler{ + definition: definition, + execute: func(context.Context, Execution) Result { + handlerCalled.Store(true) + return Complete("unexpected", nil) + }, + }, nil) + row := enqueueTestTask(t, harness, "codec-panic", "codec-panic") + panicCodec.Store(true) + if _, _, err := harness.service.Enqueue(t.Context(), EnqueueRequest{ + Type: testTaskType, IdempotencyKey: "codec-panic-enqueue", Input: testInput{Value: "value"}, + }); !errors.Is(err, ErrCodecPanic) { + t.Fatalf("enqueue codec panic error = %v, want ErrCodecPanic", err) + } + + harness.engine.executeClaim(t.Context(), claimTestTask(t, harness)) + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusFailed || stored.FailureReason == nil || *stored.FailureReason != "input_codec_panic" { + t.Fatalf("codec panic task result = %#v, err=%v", stored, err) + } + if handlerCalled.Load() { + t.Fatal("handler ran after its input codec panicked") + } +} + +func TestCodecInvalidCanonicalJSONFailsClosed(t *testing.T) { + limit := 5 + definition := testDefinition(&limit, true) + definition.Codec = CodecFunc(func(json.RawMessage) (json.RawMessage, error) { + return json.RawMessage(`not-json`), nil + }) + harness := newTaskHarness(t, scriptedHandler{definition: definition}, nil) + if _, _, err := harness.service.Enqueue(t.Context(), EnqueueRequest{ + Type: testTaskType, IdempotencyKey: "invalid-canonical", Input: testInput{Value: "value"}, + }); !errors.Is(err, ErrInvalidCanonical) { + t.Fatalf("enqueue invalid canonical error = %v, want ErrInvalidCanonical", err) + } +} + +func TestUnexpectedClaimPanicForcesRecoveryWithoutKillingWorker(t *testing.T) { + limit := 5 + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&limit, true), + execute: func(context.Context, Execution) Result { + return Retry(panicError{}, "formatting_failed", 0, nil) + }, + }, nil) + row := enqueueTestTask(t, harness, "claim-panic", "claim-panic") + harness.engine.executeClaimSafely(t.Context(), claimTestTask(t, harness)) + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusRunning || stored.ResumeMode != model.TaskResumeModeRecover || stored.LeaseUntil == nil { + t.Fatalf("task after unexpected panic = %#v, err=%v", stored, err) + } + if stored.LeaseUntil.After(time.Now().Add(2 * harness.engine.config.PollInterval)) { + t.Fatalf("unexpected panic left a long lease: %s", stored.LeaseUntil) + } +} + +func TestEnginePanicAndInvalidResultFailClosed(t *testing.T) { + limit := 5 + tests := []struct { + name string + execute func(context.Context, Execution) Result + wantReason string + }{ + {name: "panic", execute: func(context.Context, Execution) Result { panic("boom") }, wantReason: "handler_panic"}, + {name: "invalid", execute: func(context.Context, Execution) Result { return Result{} }, wantReason: "invalid_result"}, + {name: "failure without evidence", execute: func(context.Context, Execution) Result { + return Fail(nil, "", nil) + }, wantReason: "invalid_result"}, + {name: "retry without reason", execute: func(context.Context, Execution) Result { + return Retry(errors.New("temporary"), "", 0, nil) + }, wantReason: "invalid_result"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + harness := newTaskHarness(t, scriptedHandler{definition: testDefinition(&limit, true), execute: tt.execute}, nil) + row := enqueueTestTask(t, harness, tt.name, tt.name) + harness.engine.executeClaim(t.Context(), claimTestTask(t, harness)) + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusFailed || stored.FailureReason == nil || *stored.FailureReason != tt.wantReason { + t.Fatalf("failed-closed task = %#v, err=%v", stored, err) + } + }) + } +} + +func TestEngineShutdownDiscardsHandlerResultAndForcesRecovery(t *testing.T) { + limit := 5 + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&limit, true), + execute: func(ctx context.Context, _ Execution) Result { + <-ctx.Done() + return Complete("must be discarded", nil) + }, + }, nil) + row := enqueueTestTask(t, harness, "shutdown", "shutdown") + claimed := claimTestTask(t, harness) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + harness.engine.executeClaim(ctx, claimed) + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusRunning || stored.ResumeMode != model.TaskResumeModeRecover { + t.Fatalf("shutdown task = %#v, err=%v", stored, err) + } +} + +func TestRecoverCannotStartExternalEffect(t *testing.T) { + limit := 5 + var called atomic.Bool + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&limit, true), + recover: func(ctx context.Context, execution Execution) Result { + err := execution.WithResource(ctx, ResourceProviderMutation, func(context.Context) error { + called.Store(true) + return nil + }) + if !errors.Is(err, ErrEffectForbidden) { + return Fail(fmt.Errorf("resource error = %v", err), "unexpected_resource_error", nil) + } + return Complete("recovery remained read-only", nil) + }, + }, nil) + row := enqueueTestTask(t, harness, "recover-effect", "recover-effect") + if _, err := harness.db.NewRaw(`UPDATE tasks SET resume_mode = ? WHERE id = ?`, model.TaskResumeModeRecover, row.ID).Exec(t.Context()); err != nil { + t.Fatalf("set recovery mode: %v", err) + } + claimed := claimTestTask(t, harness) + harness.engine.executeClaim(t.Context(), claimed) + if called.Load() { + t.Fatal("recovery handler started an external effect") + } + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusCompleted { + t.Fatalf("recovery task = %#v, err=%v", stored, err) + } +} + +func TestWithCheckpointedEffectSeparatesAdmissionFromAttempt(t *testing.T) { + injected := errors.New("injected failure") + tests := []struct { + name string + mode model.TaskResumeMode + resource resourceRunner + checkpoint checkpointWriter + wantAttempted bool + wantEffectCalls int + wantError error + }{ + { + name: "resource admission fails", mode: model.TaskResumeModeExecute, + resource: func(context.Context, Resource, func(context.Context) error) error { return context.Canceled }, + checkpoint: func(context.Context, any, Settlement) error { + t.Fatal("checkpoint ran before resource admission") + return nil + }, + wantError: context.Canceled, + }, + { + name: "checkpoint fails", mode: model.TaskResumeModeExecute, + resource: func(ctx context.Context, _ Resource, fn func(context.Context) error) error { return fn(ctx) }, + checkpoint: func(context.Context, any, Settlement) error { + return injected + }, + wantError: injected, + }, + { + name: "effect fails", mode: model.TaskResumeModeExecute, + resource: func(ctx context.Context, _ Resource, fn func(context.Context) error) error { return fn(ctx) }, + checkpoint: func(ctx context.Context, _ any, settlement Settlement) error { + return settlement(ctx, nil) + }, + wantAttempted: true, wantEffectCalls: 1, wantError: injected, + }, + { + name: "recovery is forbidden", mode: model.TaskResumeModeRecover, + resource: func(context.Context, Resource, func(context.Context) error) error { + t.Fatal("recovery reached resource runner") + return nil + }, + checkpoint: func(context.Context, any, Settlement) error { + t.Fatal("recovery wrote checkpoint") + return nil + }, + wantError: ErrEffectForbidden, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + effectCalls := 0 + execution := Execution{ + task: model.Task{ResumeMode: tt.mode}, checkpoint: tt.checkpoint, resource: tt.resource, + } + attempted, err := execution.WithCheckpointedEffect(t.Context(), ResourceProviderMutation, map[string]bool{"attempted": true}, func(context.Context, *repository.Repositories) error { + return nil + }, func(context.Context) error { + effectCalls++ + return injected + }) + if attempted != tt.wantAttempted || effectCalls != tt.wantEffectCalls || !errors.Is(err, tt.wantError) { + t.Fatalf("result = attempted:%v calls:%d err:%v, want attempted:%v calls:%d err:%v", attempted, effectCalls, err, tt.wantAttempted, tt.wantEffectCalls, tt.wantError) + } + }) + } +} + +func TestCheckpointedEffectRollsBackEvidenceBeforeEffect(t *testing.T) { + limit := 5 + var effectCalled atomic.Bool + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&limit, true), + execute: func(ctx context.Context, execution Execution) Result { + attempted, err := execution.WithCheckpointedEffect(ctx, ResourceProviderMutation, map[string]bool{"attempted": true}, func(ctx context.Context, repos *repository.Repositories) error { + if err := repos.Tasks.RequestCancellation(ctx, execution.ID(), "must roll back"); err != nil { + return err + } + return errors.New("reject checkpoint evidence") + }, func(context.Context) error { + effectCalled.Store(true) + return nil + }) + if attempted || err == nil { + return Fail(fmt.Errorf("checkpoint result = attempted:%v err:%v", attempted, err), "unexpected_checkpoint_result", nil) + } + return Fail(err, "effect_not_started", nil) + }, + }, nil) + row := enqueueTestTask(t, harness, "checkpoint-rollback", "checkpoint-rollback") + harness.engine.executeClaim(t.Context(), claimTestTask(t, harness)) + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusFailed || stored.FailureReason == nil || *stored.FailureReason != "effect_not_started" || stored.CancellationRequestedAt != nil || len(stored.Checkpoint) != 0 { + t.Fatalf("rolled-back task = %#v, err=%v", stored, err) + } + if effectCalled.Load() { + t.Fatal("effect ran after checkpoint settlement failed") + } +} + +func TestExternalEffectRevalidatesClaimAfterResourceAdmission(t *testing.T) { + limit := 5 + var called atomic.Bool + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&limit, true), + execute: func(ctx context.Context, execution Execution) Result { + err := execution.WithResource(ctx, ResourceProviderMutation, func(context.Context) error { + called.Store(true) + return nil + }) + if err != nil { + return Fail(err, "resource_claim_lost", nil) + } + return Complete("effect completed", nil) + }, + }, nil) + row := enqueueTestTask(t, harness, "stale-effect", "stale-effect") + stale := claimTestTask(t, harness) + if _, err := harness.db.NewRaw(`UPDATE tasks SET lease_until = ? WHERE id = ?`, time.Now().Add(-time.Second), row.ID).Exec(t.Context()); err != nil { + t.Fatalf("expire stale claim: %v", err) + } + fresh := claimTestTask(t, harness) + + harness.engine.executeClaim(t.Context(), stale) + if called.Load() { + t.Fatal("stale claim started an external effect") + } + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusRunning || stored.ClaimGeneration != fresh.ClaimGeneration { + t.Fatalf("task after stale effect attempt = %#v, err=%v", stored, err) + } +} + +type renewFailureRepository struct { + repository.TaskRepository +} + +func (r renewFailureRepository) RenewLease(context.Context, int64, int64, time.Duration) (time.Time, error) { + return time.Time{}, errors.New("injected renewal failure") +} + +type countingRenewRepository struct { + repository.TaskRepository + renewals atomic.Int64 +} + +func (r *countingRenewRepository) RenewLease(ctx context.Context, id, generation int64, duration time.Duration) (time.Time, error) { + r.renewals.Add(1) + return r.TaskRepository.RenewLease(ctx, id, generation, duration) +} + +type controlledShortenRepository struct { + repository.TaskRepository + fail atomic.Bool +} + +func (r *controlledShortenRepository) ShortenLease(ctx context.Context, id, generation int64, duration time.Duration) error { + if r.fail.Load() { + return errors.New("injected lease shortening failure") + } + return r.TaskRepository.ShortenLease(ctx, id, generation, duration) +} + +type delayedShortenRepository struct { + repository.TaskRepository + delay time.Duration + shortened atomic.Bool + renewalsAfterShort atomic.Int64 +} + +func (r *delayedShortenRepository) RenewLease(ctx context.Context, id, generation int64, duration time.Duration) (time.Time, error) { + if r.shortened.Load() { + r.renewalsAfterShort.Add(1) + } + return r.TaskRepository.RenewLease(ctx, id, generation, duration) +} + +func (r *delayedShortenRepository) ShortenLease(ctx context.Context, id, generation int64, duration time.Duration) error { + err := r.TaskRepository.ShortenLease(ctx, id, generation, duration) + if err == nil { + r.shortened.Store(true) + time.Sleep(r.delay) + } + return err +} + +func TestEngineRenewalFailureCancelsBeforeSafetyBoundary(t *testing.T) { + limit := 5 + config := EngineConfig{ + Concurrency: 1, PollInterval: 10 * time.Millisecond, LeaseDuration: 3 * time.Second, Retention: time.Hour, + ProviderMutationConcurrency: 1, DestructiveMutationConcurrency: 1, + } + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&limit, true), + execute: func(ctx context.Context, _ Execution) Result { + <-ctx.Done() + return Complete("must be discarded", nil) + }, + }, &config) + row := enqueueTestTask(t, harness, "renewal", "renewal") + claimed := claimTestTask(t, harness) + harness.repos.Tasks = renewFailureRepository{TaskRepository: harness.repos.Tasks} + harness.engine.renewalRetryDelays = []time.Duration{0, time.Second} + started := time.Now() + harness.engine.executeClaim(t.Context(), claimed) + if elapsed := time.Since(started); elapsed >= config.LeaseDuration-config.LeaseDuration/3+30*time.Millisecond { + t.Fatalf("renewal uncertainty cancelled after %s", elapsed) + } + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusRunning || stored.ResumeMode != model.TaskResumeModeRecover { + t.Fatalf("renewal failure task = %#v, err=%v", stored, err) + } +} + +func TestEngineSuccessfulRenewalKeepsHealthCurrent(t *testing.T) { + limit := 5 + config := EngineConfig{ + Concurrency: 1, PollInterval: 10 * time.Millisecond, LeaseDuration: 3 * time.Second, Retention: time.Hour, + ProviderMutationConcurrency: 1, DestructiveMutationConcurrency: 1, + } + harness := newTaskHarness(t, scriptedHandler{definition: testDefinition(&limit, true)}, &config) + enqueueTestTask(t, harness, "renewal-health", "renewal-health") + claimed := claimTestTask(t, harness) + harness.engine.lastTick.Store(time.Now().Add(-2 * time.Minute).UnixNano()) + + var leaseSafe atomic.Bool + leaseSafe.Store(true) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + stop := make(chan struct{}) + stopped := make(chan struct{}) + go harness.engine.renewLease(ctx, claimed, &leaseSafe, cancel, stop, stopped) + + deadline := time.Now().Add(5 * time.Second) + for !harness.engine.Healthy() && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if !harness.engine.Healthy() { + t.Fatal("successful lease renewal did not refresh engine health") + } + close(stop) + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("lease renewal did not stop") + } + if !leaseSafe.Load() { + t.Fatal("successful lease renewal marked the claim unsafe") + } +} + +func TestEngineSettlementFailureShortensLeaseAndRecovers(t *testing.T) { + limit := 5 + var recovers atomic.Int64 + config := EngineConfig{ + Concurrency: 1, PollInterval: 10 * time.Millisecond, LeaseDuration: 3 * time.Second, Retention: time.Hour, + ProviderMutationConcurrency: 1, DestructiveMutationConcurrency: 1, + } + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&limit, true), + execute: func(context.Context, Execution) Result { return Complete("first", nil) }, + recover: func(context.Context, Execution) Result { + recovers.Add(1) + return Complete("recovered", nil) + }, + }, &config) + row := enqueueTestTask(t, harness, "settlement", "settlement") + claimed := claimTestTask(t, harness) + ordered := &delayedShortenRepository{ + TaskRepository: harness.repos.Tasks, + delay: 2 * config.LeaseDuration / 3, + } + harness.repos.Tasks = ordered + if _, err := harness.db.ExecContext(t.Context(), `CREATE TRIGGER fail_task_settlement + BEFORE UPDATE OF status ON tasks + WHEN OLD.status = 'running' AND NEW.status <> 'running' + BEGIN SELECT RAISE(FAIL, 'injected settlement failure'); END`); err != nil { + t.Fatalf("create settlement fault: %v", err) + } + harness.engine.settlementRetryDelays = []time.Duration{0, 0, 0, 0} + harness.engine.executeClaim(t.Context(), claimed) + if got := ordered.renewalsAfterShort.Load(); got != 0 { + t.Fatalf("lease renewed %d times after it was shortened", got) + } + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusRunning || stored.ResumeMode != model.TaskResumeModeRecover || stored.LeaseUntil == nil { + t.Fatalf("uncertain settlement task = %#v, err=%v", stored, err) + } + if stored.LeaseUntil.After(time.Now().Add(2 * harness.engine.config.PollInterval)) { + t.Fatalf("settlement failure lease was not shortened: %s", stored.LeaseUntil) + } + if _, err := harness.db.ExecContext(t.Context(), `DROP TRIGGER fail_task_settlement`); err != nil { + t.Fatalf("drop settlement fault: %v", err) + } + if _, err := harness.db.NewRaw(`UPDATE tasks SET lease_until = ? WHERE id = ?`, time.Now().Add(-time.Millisecond), row.ID).Exec(t.Context()); err != nil { + t.Fatalf("expire shortened lease: %v", err) + } + fresh := claimTestTask(t, harness) + if fresh.ResumeMode != model.TaskResumeModeRecover { + t.Fatalf("settlement recovery mode = %s", fresh.ResumeMode) + } + harness.engine.executeClaim(t.Context(), fresh) + stored, err = harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusCompleted || recovers.Load() != 1 { + t.Fatalf("recovered settlement task = %#v recover calls=%d err=%v", stored, recovers.Load(), err) + } +} + +func TestEngineSettlementPanicRollsBackAndShortensLease(t *testing.T) { + limit := 5 + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&limit, true), + execute: func(context.Context, Execution) Result { + return Complete("must not settle", func(context.Context, *repository.Repositories) error { + panic("settlement boom") + }) + }, + }, nil) + row := enqueueTestTask(t, harness, "settlement-panic", "settlement-panic") + claimed := claimTestTask(t, harness) + harness.engine.settlementRetryDelays = []time.Duration{0, 0, 0, 0} + + harness.engine.executeClaim(t.Context(), claimed) + + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusRunning || stored.ResumeMode != model.TaskResumeModeRecover || stored.LeaseUntil == nil { + t.Fatalf("task after settlement panic = %#v, err=%v", stored, err) + } + if stored.LeaseUntil.After(time.Now().Add(2 * harness.engine.config.PollInterval)) { + t.Fatalf("settlement panic lease was not shortened: %s", stored.LeaseUntil) + } +} + +func TestEngineRecoveryQueueDoesNotDropLeaseShorteningWork(t *testing.T) { + limit := 5 + config := EngineConfig{ + Concurrency: 1, PollInterval: 10 * time.Millisecond, LeaseDuration: time.Minute, Retention: time.Hour, + ProviderMutationConcurrency: 1, DestructiveMutationConcurrency: 1, + } + harness := newTaskHarness(t, scriptedHandler{definition: testDefinition(&limit, true)}, &config) + controlled := &controlledShortenRepository{TaskRepository: harness.repos.Tasks} + controlled.fail.Store(true) + harness.repos.Tasks = controlled + + claimed := make([]*model.Task, 0, 3) + for i := range 3 { + enqueueTestTask(t, harness, fmt.Sprintf("recovery-queue-%d", i), fmt.Sprintf("value-%d", i)) + claimed = append(claimed, claimTestTask(t, harness)) + } + for _, row := range claimed { + harness.engine.abandonClaim(row) + } + controlled.fail.Store(false) + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan struct{}) + go func() { + harness.engine.runRecoveryQueue(ctx) + close(done) + }() + + deadline := time.Now().Add(time.Second) + for _, row := range claimed { + for { + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil { + t.Fatalf("load recovered task %d: %v", row.ID, err) + } + if stored.LeaseUntil != nil && stored.ResumeMode == model.TaskResumeModeRecover && + stored.LeaseUntil.Before(time.Now().Add(2*config.PollInterval)) { + break + } + if time.Now().After(deadline) { + t.Fatalf("task %d lease was not shortened: %#v", row.ID, stored) + } + time.Sleep(time.Millisecond) + } + } + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("recovery queue did not stop") + } +} + +func TestEngineRenewsLeaseWhileSettlementIsRetried(t *testing.T) { + limit := 5 + var settlements atomic.Int64 + config := EngineConfig{ + Concurrency: 1, PollInterval: 10 * time.Millisecond, LeaseDuration: 3 * time.Second, Retention: time.Hour, + ProviderMutationConcurrency: 1, DestructiveMutationConcurrency: 1, + } + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&limit, true), + execute: func(context.Context, Execution) Result { + return Complete("settled", func(context.Context, *repository.Repositories) error { + if settlements.Add(1) < 4 { + return errors.New("injected transient settlement failure") + } + return nil + }) + }, + }, &config) + row := enqueueTestTask(t, harness, "settlement-renewal", "settlement-renewal") + claimed := claimTestTask(t, harness) + counting := &countingRenewRepository{TaskRepository: harness.repos.Tasks} + harness.repos.Tasks = counting + harness.engine.settlementRetryDelays = []time.Duration{0, 1200 * time.Millisecond, 1200 * time.Millisecond, 1200 * time.Millisecond} + + harness.engine.executeClaim(t.Context(), claimed) + + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusCompleted { + t.Fatalf("task after settlement retries = %#v, err=%v", stored, err) + } + if got := settlements.Load(); got != 4 { + t.Fatalf("settlement calls = %d, want 4", got) + } + if got := counting.renewals.Load(); got != 10 { + t.Fatalf("lease renewals = %d, want retries renewed before and during waits", got) + } +} + +func TestEngineConfirmsLeaseBeforeSettlement(t *testing.T) { + limit := 5 + var settlements atomic.Int64 + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&limit, true), + execute: func(context.Context, Execution) Result { + return Complete("must not settle", func(context.Context, *repository.Repositories) error { + settlements.Add(1) + return nil + }) + }, + }, nil) + row := enqueueTestTask(t, harness, "settlement-lease", "settlement-lease") + claimed := claimTestTask(t, harness) + harness.repos.Tasks = renewFailureRepository{TaskRepository: harness.repos.Tasks} + + harness.engine.executeClaim(t.Context(), claimed) + + if got := settlements.Load(); got != 0 { + t.Fatalf("settlement ran %d times without a confirmed lease", got) + } + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusRunning || stored.ResumeMode != model.TaskResumeModeRecover || stored.LeaseUntil == nil { + t.Fatalf("task after unconfirmed settlement lease = %#v, err=%v", stored, err) + } + if stored.LeaseUntil.After(time.Now().Add(2 * harness.engine.config.PollInterval)) { + t.Fatalf("unconfirmed settlement lease was not shortened: %s", stored.LeaseUntil) + } +} + +func TestClaimNextConcurrentClaimsAreUnique(t *testing.T) { + limit := 5 + harness := newTaskHarness(t, scriptedHandler{definition: testDefinition(&limit, true)}, nil) + harness.db.SetMaxOpenConns(8) + const total = 24 + for i := range total { + enqueueTestTask(t, harness, fmt.Sprintf("claim-%02d", i), fmt.Sprintf("value-%02d", i)) + } + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + claimedIDs := make(chan int64, total) + var workers sync.WaitGroup + for range 8 { + workers.Go(func() { + for ctx.Err() == nil { + claimed, err := harness.repos.Tasks.ClaimNext(ctx, time.Minute) + if err != nil { + continue + } + if claimed == nil { + return + } + claimedIDs <- claimed.ID + } + }) + } + workers.Wait() + close(claimedIDs) + seen := make(map[int64]struct{}, total) + for id := range claimedIDs { + if _, duplicate := seen[id]; duplicate { + t.Fatalf("task %d was claimed twice", id) + } + seen[id] = struct{}{} + } + if len(seen) != total { + t.Fatalf("unique claims = %d, want %d", len(seen), total) + } +} + +func TestProviderResourceGateReachesAndEnforcesConfiguredLimit(t *testing.T) { + limit := 5 + harness := newTaskHarness(t, scriptedHandler{definition: testDefinition(&limit, true)}, nil) + const calls = 12 + var running, maximum atomic.Int64 + entered := make(chan struct{}, calls) + release := make(chan struct{}) + errorsCh := make(chan error, calls) + var workers sync.WaitGroup + for range calls { + workers.Go(func() { + err := harness.engine.withResource(t.Context(), ResourceProviderMutation, func(context.Context) error { + current := running.Add(1) + for { + observed := maximum.Load() + if current <= observed || maximum.CompareAndSwap(observed, current) { + break + } + } + entered <- struct{}{} + <-release + running.Add(-1) + return nil + }) + errorsCh <- err + }) + } + for range harness.engine.config.ProviderMutationConcurrency { + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("provider mutation gate did not reach configured concurrency") + } + } + if maximum.Load() != int64(harness.engine.config.ProviderMutationConcurrency) { + t.Fatalf("provider mutation concurrency = %d", maximum.Load()) + } + select { + case <-entered: + t.Fatal("provider mutation gate exceeded configured concurrency") + case <-time.After(20 * time.Millisecond): + } + close(release) + workers.Wait() + close(errorsCh) + for err := range errorsCh { + if err != nil { + t.Fatalf("provider resource call: %v", err) + } + } + if maximum.Load() != 4 { + t.Fatalf("provider mutation maximum = %d, want 4", maximum.Load()) + } +} diff --git a/internal/testutil/db.go b/internal/testutil/db.go index fafec70..436c477 100644 --- a/internal/testutil/db.go +++ b/internal/testutil/db.go @@ -4,11 +4,14 @@ package testutil import ( "context" + "crypto/sha256" "database/sql" + "encoding/hex" "fmt" "path/filepath" "sync/atomic" "testing" + "time" "github.com/strahe/synaps3/internal/db/migrations" "github.com/strahe/synaps3/internal/db/repository" @@ -21,6 +24,18 @@ import ( var testDBCounter atomic.Uint64 +// StorageChecksum turns a readable fixture identity into the canonical +// lowercase SHA-256 representation required by storage_contents. +func StorageChecksum(identity string) string { + if len(identity) == sha256.Size*2 { + if decoded, err := hex.DecodeString(identity); err == nil && hex.EncodeToString(decoded) == identity { + return identity + } + } + digest := sha256.Sum256([]byte(identity)) + return hex.EncodeToString(digest[:]) +} + // NewTestDB creates a fresh in-memory SQLite DB with all migrations applied. // The database is closed automatically when the test completes. func NewTestDB(t *testing.T) *bun.DB { @@ -74,21 +89,44 @@ func NewTestRepos(t *testing.T) *repository.Repositories { // SeedBucket inserts an active bucket and returns it. func SeedBucket(t *testing.T, db *bun.DB, name string) *model.Bucket { t.Helper() - bucket := &model.Bucket{Name: name, Status: model.BucketStatusActive} - _, err := db.NewInsert().Model(bucket).Exec(context.Background()) - if err != nil { + return SeedBucketWithCopies(t, db, name, model.BucketStatusActive, 1) +} + +// SeedBucketWithCopies inserts a bucket whose durability policy is the given +// replica target, along with the replica slots that target opens. +func SeedBucketWithCopies(t *testing.T, db *bun.DB, name string, status model.BucketStatus, copies int) *model.Bucket { + t.Helper() + bucket := &model.Bucket{Name: name, Status: status, DefaultCopies: copies, MinimumDurableCopies: copies} + if _, err := db.NewInsert().Model(bucket).Exec(context.Background()); err != nil { t.Fatalf("seeding bucket: %v", err) } + OpenBucketReplicaSlots(t, db, bucket.ID, copies) return bucket } // SeedBucketWithStatus inserts a bucket with the specified status. func SeedBucketWithStatus(t *testing.T, db *bun.DB, name string, status model.BucketStatus) *model.Bucket { t.Helper() - bucket := &model.Bucket{Name: name, Status: status} - _, err := db.NewInsert().Model(bucket).Exec(context.Background()) - if err != nil { - t.Fatalf("seeding bucket: %v", err) + return SeedBucketWithCopies(t, db, name, status, 1) +} + +// OpenBucketReplicaSlots inserts the replica slots a bucket seeded outside +// repository.Buckets.Create would otherwise be missing. Data sets, copies, +// replacements, cleanups and observability rows all reference these, so a test +// that seeds a bucket directly must open its slots too. +func OpenBucketReplicaSlots(tb testing.TB, db bun.IDB, bucketID int64, copies int) { + tb.Helper() + slots := make([]model.BucketReplicaSlot, 0, copies) + now := time.Now().UTC() + for copyIndex := range copies { + slots = append(slots, model.BucketReplicaSlot{ + BucketID: bucketID, CopyIndex: copyIndex, + Status: model.BucketReplicaSlotStatusActive, CreatedAt: now, UpdatedAt: now, + }) + } + if _, err := db.NewInsert().Model(&slots). + On("CONFLICT (bucket_id, copy_index) DO NOTHING"). + Exec(context.Background()); err != nil { + tb.Fatalf("opening bucket replica slots: %v", err) } - return bucket } diff --git a/internal/testutil/db_test.go b/internal/testutil/db_test.go index c39a781..cb0d2c8 100644 --- a/internal/testutil/db_test.go +++ b/internal/testutil/db_test.go @@ -28,7 +28,7 @@ func assertTestDBsAreIsolated(t *testing.T, newDB func(*testing.T) *bun.DB) { first := newDB(t) second := newDB(t) - bucket := &model.Bucket{Name: "isolated-db-bucket", Status: model.BucketStatusActive} + bucket := &model.Bucket{Name: "isolated-db-bucket", Status: model.BucketStatusActive, DefaultCopies: 8, MinimumDurableCopies: 8} if _, err := first.NewInsert().Model(bucket).Exec(ctx); err != nil { t.Fatalf("insert bucket into first db: %v", err) } diff --git a/internal/testutil/mocks.go b/internal/testutil/mocks.go index ca18c7d..d740f56 100644 --- a/internal/testutil/mocks.go +++ b/internal/testutil/mocks.go @@ -185,7 +185,11 @@ type MockStorageTarget struct { ClientDataSetIDValue sdktypes.BigInt ServiceURLValue string WithCDNValue bool + CreateDataSetFunc func(context.Context, *storage.CreateDataSetOptions) (*storage.CreateDataSetResult, error) + WaitDataSetFunc func(context.Context, storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) + StoreFunc func(context.Context, io.Reader, *storage.StoreOptions) (*storage.StoreResult, error) PresignForCommitFunc func(context.Context, []storage.PieceInput) ([]byte, error) + PullFunc func(context.Context, storage.PullRequest) (*storage.PullResult, error) SubmitCommitFunc func(context.Context, storage.CommitRequest) (*storage.CommitSubmission, error) GetCommitStatusFunc func(context.Context, storage.CommitSubmission) (*storage.CommitStatus, error) PieceStatusFunc func(context.Context, cid.Cid) (*storage.PieceStatus, error) @@ -240,15 +244,24 @@ func (m *MockStorageTarget) ServiceURL() string { return "https://provider.example" } -func (m *MockStorageTarget) CreateDataSet(context.Context, *storage.CreateDataSetOptions) (*storage.CreateDataSetResult, error) { +func (m *MockStorageTarget) CreateDataSet(ctx context.Context, opts *storage.CreateDataSetOptions) (*storage.CreateDataSetResult, error) { + if m.CreateDataSetFunc != nil { + return m.CreateDataSetFunc(ctx, opts) + } return nil, errors.New("MockStorageTarget.CreateDataSet not configured") } -func (m *MockStorageTarget) WaitForDataSetCreated(context.Context, storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) { +func (m *MockStorageTarget) WaitForDataSetCreated(ctx context.Context, submission storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) { + if m.WaitDataSetFunc != nil { + return m.WaitDataSetFunc(ctx, submission) + } return nil, errors.New("MockStorageTarget.WaitForDataSetCreated not configured") } -func (m *MockStorageTarget) Store(context.Context, io.Reader, *storage.StoreOptions) (*storage.StoreResult, error) { +func (m *MockStorageTarget) Store(ctx context.Context, reader io.Reader, opts *storage.StoreOptions) (*storage.StoreResult, error) { + if m.StoreFunc != nil { + return m.StoreFunc(ctx, reader, opts) + } return nil, errors.New("MockStorageTarget.Store not configured") } @@ -259,7 +272,10 @@ func (m *MockStorageTarget) PresignForCommit(ctx context.Context, pieces []stora return nil, errors.New("MockStorageTarget.PresignForCommit not configured") } -func (m *MockStorageTarget) Pull(context.Context, storage.PullRequest) (*storage.PullResult, error) { +func (m *MockStorageTarget) Pull(ctx context.Context, request storage.PullRequest) (*storage.PullResult, error) { + if m.PullFunc != nil { + return m.PullFunc(ctx, request) + } return nil, errors.New("MockStorageTarget.Pull not configured") } @@ -316,7 +332,7 @@ type MockCache struct { CreateBucketDirFunc func(ctx context.Context, bucket string) error DeleteBucketDirFunc func(ctx context.Context, bucket string) error PutPartFunc func(ctx context.Context, uploadID string, partNumber int, r io.Reader) (*cache.ObjectInfo, error) - AssemblePartsFunc func(ctx context.Context, bucket, key, uploadID string, partNumbers []int) (*cache.ObjectInfo, []string, error) + AssemblePartsFunc func(ctx context.Context, bucket, key, uploadID string, partNumbers []int) (*cache.StagedObject, []string, error) DeleteUploadFunc func(ctx context.Context, uploadID string) error } @@ -383,11 +399,11 @@ func (m *MockCache) PutPart(ctx context.Context, uploadID string, partNumber int return nil, errors.New("MockCache.PutPart not configured") } -func (m *MockCache) AssembleParts(ctx context.Context, bucket, key, uploadID string, partNumbers []int) (*cache.ObjectInfo, []string, error) { +func (m *MockCache) AssemblePartsStaged(ctx context.Context, bucket, key, uploadID string, partNumbers []int) (*cache.StagedObject, []string, error) { if m.AssemblePartsFunc != nil { return m.AssemblePartsFunc(ctx, bucket, key, uploadID, partNumbers) } - return nil, nil, errors.New("MockCache.AssembleParts not configured") + return nil, nil, errors.New("MockCache.AssemblePartsStaged not configured") } func (m *MockCache) DeleteUpload(ctx context.Context, uploadID string) error { diff --git a/internal/testutil/storagecommit.go b/internal/testutil/storagecommit.go new file mode 100644 index 0000000..10f6684 --- /dev/null +++ b/internal/testutil/storagecommit.go @@ -0,0 +1,76 @@ +package testutil + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagecommit" + "github.com/uptrace/bun" +) + +// CommitStorageCopy drives one copy to committed the way production does: a +// commit attempt is recorded in the ledger first, and the copy then projects +// that confirmed row. A committed copy without confirmed evidence is refused by +// the schema, so tests cannot shortcut this. +func CommitStorageCopy( + t *testing.T, + db bun.IDB, + repos *repository.Repositories, + input repository.MarkUploadCopyCommittedInput, +) { + t.Helper() + ctx := context.Background() + if input.StorageCopyID == 0 { + copyRow := new(model.StorageCopy) + if err := db.NewSelect(). + Model(copyRow). + Where("content_id = ? AND copy_index = ?", input.ContentID, input.CopyIndex). + Scan(ctx); err != nil { + t.Fatalf("loading copy for content %d slot %d: %v", input.ContentID, input.CopyIndex, err) + } + input.StorageCopyID = copyRow.ID + } + copyRow := new(model.StorageCopy) + if err := db.NewSelect().Model(copyRow).Where("id = ?", input.StorageCopyID).Scan(ctx); err != nil { + t.Fatalf("loading copy %d: %v", input.StorageCopyID, err) + } + if input.CommitExtraDataHex == "" { + input.CommitExtraDataHex = "abcd" + } + if input.CommitTransactionID == "" { + input.CommitTransactionID = fmt.Sprintf("tx-%d", input.StorageCopyID) + } + if input.CommitConfirmedTransactionID == "" { + input.CommitConfirmedTransactionID = input.CommitTransactionID + } + if input.CommitAttemptID == "" { + input.CommitAttemptID = fmt.Sprintf("attempt-%d-%d", input.ContentID, input.StorageCopyID) + } + now := time.Now() + attempt := &storagecommit.Attempt{ + AttemptID: input.CommitAttemptID, ContentID: copyRow.ContentID, + StorageDataSetID: copyRow.StorageDataSetID, Status: storagecommit.AttemptStatusAttempted, + ExtraDataHex: &input.CommitExtraDataHex, TransactionID: &input.CommitTransactionID, + AttemptedAt: &now, CreatedAt: now, UpdatedAt: now, + } + if _, err := db.NewInsert().Model(attempt).Exec(ctx); err != nil { + t.Fatalf("seeding commit attempt for copy %d: %v", input.StorageCopyID, err) + } + // Confirmation lands on a copy the coordinator already moved to committing. + if _, err := db.NewUpdate(). + Model((*model.StorageCopy)(nil)). + Set("status = ?", model.StorageCopyStatusCommitting). + Set("commit_extra_data_hex = ?", input.CommitExtraDataHex). + Set("updated_at = ?", now). + Where("id = ?", input.StorageCopyID). + Exec(ctx); err != nil { + t.Fatalf("moving copy %d to committing: %v", input.StorageCopyID, err) + } + if err := repos.Contents.MarkUploadCopyCommitted(ctx, input); err != nil { + t.Fatalf("MarkUploadCopyCommitted: %v", err) + } +} diff --git a/internal/walletoperation/task.go b/internal/walletoperation/task.go new file mode 100644 index 0000000..982180b --- /dev/null +++ b/internal/walletoperation/task.go @@ -0,0 +1,29 @@ +// Package walletoperation owns wallet task inputs and recovery checkpoints. +package walletoperation + +import ( + "errors" + "fmt" +) + +const TaskKeyPrefix = "wallet-operation:" + +type Input struct { + OperationID int64 `json:"operation_id"` +} + +type Checkpoint struct { + BroadcastAttempted bool `json:"broadcast_attempted"` + TransactionHash string `json:"transaction_hash,omitempty"` +} + +func TaskKey(operationID int64) string { + return fmt.Sprintf("%s%d", TaskKeyPrefix, operationID) +} + +func ValidateInput(input Input) error { + if input.OperationID < 1 { + return errors.New("operation_id must be positive") + } + return nil +} diff --git a/internal/worker/backoff.go b/internal/worker/backoff.go deleted file mode 100644 index 659146b..0000000 --- a/internal/worker/backoff.go +++ /dev/null @@ -1,33 +0,0 @@ -package worker - -import ( - "math" - "math/rand/v2" - "time" -) - -const ( - baseDelay = 10 * time.Second - maxBackoff = 5 * time.Minute - jitterFraction = 0.20 -) - -// retryDelay computes an exponential backoff delay with jitter. -// Formula: min(base * 2^retryCount + jitter, maxBackoff) -// where jitter is ±20% of the computed delay. -func retryDelay(retryCount int) time.Duration { - delay := float64(baseDelay) * math.Pow(2, float64(retryCount)) - if delay > float64(maxBackoff) { - delay = float64(maxBackoff) - } - - // Apply jitter: ±jitterFraction of the computed delay. - jitter := delay * jitterFraction * (2*rand.Float64() - 1) - delay += jitter - - if delay < float64(baseDelay)*(1-jitterFraction) { - delay = float64(baseDelay) * (1 - jitterFraction) - } - - return time.Duration(delay) -} diff --git a/internal/worker/backoff_test.go b/internal/worker/backoff_test.go deleted file mode 100644 index bb3570c..0000000 --- a/internal/worker/backoff_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package worker - -import ( - "testing" - "time" -) - -func TestRetryDelay_Exponential(t *testing.T) { - // Expected approximate centres: 10s, 20s, 40s, 80s, 160s, 300s (capped). - expected := []time.Duration{ - 10 * time.Second, - 20 * time.Second, - 40 * time.Second, - 80 * time.Second, - 160 * time.Second, - 300 * time.Second, - } - - for i, want := range expected { - got := retryDelay(i) - lo := time.Duration(float64(want) * (1 - jitterFraction)) - hi := time.Duration(float64(want) * (1 + jitterFraction)) - if got < lo || got > hi { - t.Errorf("retryDelay(%d) = %v; want in [%v, %v]", i, got, lo, hi) - } - } -} - -func TestRetryDelay_MaxCap(t *testing.T) { - upper := time.Duration(float64(maxBackoff) * (1 + jitterFraction)) - for _, rc := range []int{10, 20, 50, 100} { - got := retryDelay(rc) - if got > upper { - t.Errorf("retryDelay(%d) = %v; exceeds max %v (with jitter %v)", rc, got, maxBackoff, upper) - } - } -} - -func TestRetryDelay_Jitter(t *testing.T) { - seen := make(map[time.Duration]bool) - for range 100 { - seen[retryDelay(3)] = true - } - if len(seen) < 2 { - t.Errorf("expected jitter spread across runs, got %d unique values", len(seen)) - } -} - -func TestRetryDelay_ZeroRetry(t *testing.T) { - got := retryDelay(0) - lo := time.Duration(float64(baseDelay) * (1 - jitterFraction)) - hi := time.Duration(float64(baseDelay) * (1 + jitterFraction)) - if got < lo || got > hi { - t.Errorf("retryDelay(0) = %v; want in [%v, %v]", got, lo, hi) - } -} - -func TestRetryDelay_Floor(t *testing.T) { - floor := time.Duration(float64(baseDelay) * (1 - jitterFraction)) - for rc := range 20 { - got := retryDelay(rc) - if got < floor { - t.Errorf("retryDelay(%d) = %v; below floor %v", rc, got, floor) - } - } -} diff --git a/internal/worker/bucket_task_handlers.go b/internal/worker/bucket_task_handlers.go new file mode 100644 index 0000000..9569434 --- /dev/null +++ b/internal/worker/bucket_task_handlers.go @@ -0,0 +1,169 @@ +package worker + +import ( + "context" + "errors" + + "github.com/strahe/synaps3/internal/bucketlifecycle" + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/objectlimits" + "github.com/strahe/synaps3/internal/synapse" + taskengine "github.com/strahe/synaps3/internal/task" + idtypes "github.com/strahe/synaps3/internal/types" +) + +func (h *TaskHandlers) bucketProvisionHandler() taskengine.Handler { + definition := taskengine.Definition{ + Type: model.TaskTypeBucketProvision, InputVersion: 1, + Codec: taskengine.StrictJSONCodec(func(input *bucketlifecycle.ProvisionInput) error { + return bucketlifecycle.ValidateProvisionInput(*input) + }), + RetryLimit: h.retryLimit(), AllowRetry: true, + } + run := func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + input, err := taskengine.DecodeInput[bucketlifecycle.ProvisionInput](execution) + if err != nil { + return decodeFailure(string(definition.Type), err) + } + if h.deps.Storage == nil || h.taskService == nil { + return taskengine.Fail(errors.New("storage task dependencies are unavailable"), "dependency_unavailable", nil) + } + bucket, err := h.deps.Repositories.Buckets.GetByID(ctx, input.BucketID) + if err != nil { + return retryTask(err, "bucket_load_failed") + } + if bucket == nil { + return taskengine.Cancel("Bucket no longer exists", nil) + } + // A ready bucket is still provisioned here after its replica target + // grows: readiness is decided by the data sets below, not by the status + // the bucket happened to have when this task was enqueued. + + required := h.effectiveBucketCopies(bucket) + bindings, err := h.deps.Repositories.Contents.ListDataSetBindings(ctx, bucket.ID) + if err != nil { + return retryTask(err, "dataset_bindings_load_failed") + } + // A generation that ended gives up its slot, so only live generations + // reach the body of this loop and provisioning always has a next move. + ready := 0 + covered := 0 + allPendingWorkBound := true + for i := range bindings { + binding := &bindings[i] + if !binding.IsCurrent || binding.CopyIndex >= required { + continue + } + covered++ + if binding.Status == model.StorageDataSetStatusReady { + ready++ + } else if (binding.Status == model.StorageDataSetStatusPending || binding.Status == model.StorageDataSetStatusCreating) && binding.EnsureTaskID == nil { + allPendingWorkBound = false + } + } + if ready >= required { + return taskengine.Complete("Bucket storage is ready", func(ctx context.Context, repos *repository.Repositories) error { + _, err := repos.Buckets.PromoteReadyIfProvisioned(ctx, bucket.ID, required) + return err + }) + } + if covered == required && allPendingWorkBound { + return taskengine.Suspend(model.TaskResumeModeExecute, storagePollInterval, "storage_service", "Preparing bucket storage", nil) + } + + selected, err := h.selectBucketBindings(ctx, bucket, required) + if err != nil { + if synapse.IsProviderUnavailable(err) || synapse.IsNoProviderCandidates(err) { + return taskengine.Suspend(model.TaskResumeModeExecute, storageDependencyWait, "providers", "Waiting for storage providers", nil) + } + return retryTask(err, "storage_selection_failed") + } + if len(selected) < required { + return taskengine.Suspend(model.TaskResumeModeExecute, storageDependencyWait, "providers", "Waiting for storage providers", nil) + } + targets := make([]synapse.StorageTarget, 0, len(selected)) + for i := range selected { + targets = append(targets, selected[i].target) + } + costs, err := h.deps.Storage.PrepareUpload(ctx, uint64(objectlimits.MinFOCUploadSize), targets) + if err != nil { + if synapse.IsProviderUnavailable(err) || synapse.IsNoProviderCandidates(err) { + return taskengine.Suspend(model.TaskResumeModeExecute, storageDependencyWait, "funding", "Waiting for storage funding", nil) + } + return retryTask(err, "storage_funding_failed") + } + if costs == nil || !costs.Ready { + return taskengine.Suspend(model.TaskResumeModeExecute, storageDependencyWait, "funding", uploadFundingWaitMessage(costs), nil) + } + + plan := make([]uploadBindingPlan, 0, len(selected)) + for i := range selected { + entry := selected[i] + frozen := uploadBindingPlan{ + copyIndex: entry.copyIndex, + provider: idtypes.OnChainIDFromSDK(entry.target.ProviderID()), + } + if ref, ok := entry.target.DataSetRef(); ok { + frozen.dataSet = &ref + } + plan = append(plan, frozen) + } + + settlement := func(ctx context.Context, repos *repository.Repositories) error { + created := make([]model.StorageDataSet, 0, len(plan)) + for i := range plan { + entry := plan[i] + binding, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: entry.provider, CopyIndex: entry.copyIndex, + }) + if err != nil { + return err + } + if entry.dataSet != nil { + dataSetID, clientDataSetID, err := dataSetRefIDs(binding, *entry.dataSet) + if err != nil { + return err + } + if err := repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: binding.ID, DataSetID: dataSetID, ClientDataSetID: &clientDataSetID, + }); err != nil { + return err + } + binding.Status = model.StorageDataSetStatusReady + } + created = append(created, *binding) + } + for i := range created { + binding := &created[i] + if binding.Status == model.StorageDataSetStatusReady || binding.EnsureTaskID != nil { + continue + } + if err := h.enqueueDataSetEnsure(ctx, repos, binding); err != nil && !errors.Is(err, repository.ErrConflict) { + return err + } + } + _, err := repos.Buckets.PromoteReadyIfProvisioned(ctx, bucket.ID, required) + return err + } + allResolved := true + for i := range plan { + if plan[i].dataSet == nil { + allResolved = false + break + } + } + if allResolved { + return taskengine.Complete("Bucket storage is ready", settlement) + } + return taskengine.Suspend(model.TaskResumeModeExecute, storagePollInterval, "storage_service", "Preparing bucket storage", settlement) + } + return taskHandler{definition: definition, execute: run, recover: run} +} + +func (h *TaskHandlers) effectiveBucketCopies(bucket *model.Bucket) int { + if bucket == nil { + return model.ClampStorageCopies(h.deps.DefaultCopies) + } + return model.ClampStorageCopies(bucket.DefaultCopies) +} diff --git a/internal/worker/core_task_handlers.go b/internal/worker/core_task_handlers.go new file mode 100644 index 0000000..dd4449d --- /dev/null +++ b/internal/worker/core_task_handlers.go @@ -0,0 +1,852 @@ +package worker + +import ( + "context" + "errors" + "fmt" + "math/big" + "os" + "strconv" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + ethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/ipfs/go-cid" + "github.com/strahe/synaps3/internal/cache" + "github.com/strahe/synaps3/internal/cacheeviction" + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagecleanup" + "github.com/strahe/synaps3/internal/synapse" + "github.com/strahe/synaps3/internal/systemtask" + taskengine "github.com/strahe/synaps3/internal/task" + "github.com/strahe/synaps3/internal/walletoperation" + "github.com/strahe/synapse-go/payments" + "github.com/strahe/synapse-go/storage" +) + +const ( + dependencyWait = time.Minute + externalPollInterval = 5 * time.Second + cleanupAttentionAfter = 30 * time.Minute + taskGCInterval = time.Hour + cleanupGCPageSize = 500 +) + +type cleanupCheckpoint struct { + CopyID int64 `json:"copy_id"` + AttemptedAt time.Time `json:"attempted_at"` +} + +type cacheCapacityCheckpoint struct { + CycleActive bool `json:"cycle_active"` +} + +type cacheEvictionCheckpoint struct { + AttemptedAt time.Time `json:"attempted_at"` +} + +func (h *TaskHandlers) cacheCapacityHandler() taskengine.Handler { + definition := taskengine.Definition{ + Type: model.TaskTypeCacheCapacityReconcile, InputVersion: 1, + Codec: taskengine.StrictJSONCodec(func(input *systemtask.Input) error { return systemtask.ValidateInput(*input) }), + RetryLimit: nil, AllowRetry: true, + } + run := func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + if h.deps.EvictionPolicy != cache.EvictionPolicyLRU { + return taskengine.Suspend(model.TaskResumeModeExecute, taskGCInterval, "scheduled", "Automatic cache cleanup is disabled", nil) + } + if h.deps.Cache == nil || h.deps.CacheTracker == nil || h.taskService == nil || h.deps.MaxCacheBytes <= 0 { + return taskengine.Fail(errors.New("cache capacity dependencies are unavailable"), "dependency_unavailable", nil) + } + checkpoint, _, err := taskengine.DecodeCheckpoint[cacheCapacityCheckpoint](execution) + if err != nil { + return taskengine.Fail(err, "invalid_checkpoint", nil) + } + if !h.deps.CacheTracker.SafeForLRU() { + return taskengine.Suspend(model.TaskResumeModeRecover, dependencyWait, "cache_access", "Waiting for reliable cache access records", nil) + } + usedBytes := h.deps.Cache.UsedBytes() + highBytes := cacheWatermarkBytes(h.deps.MaxCacheBytes, h.deps.LRUHighPercent) + lowBytes := cacheWatermarkBytes(h.deps.MaxCacheBytes, h.deps.LRULowPercent) + cycleActive := checkpoint.CycleActive + switch { + case usedBytes <= lowBytes: + cycleActive = false + case usedBytes >= highBytes: + cycleActive = true + } + if cycleActive != checkpoint.CycleActive { + checkpoint.CycleActive = cycleActive + if err := execution.WriteCheckpoint(ctx, checkpoint); err != nil { + return retryTask(err, "cache_capacity_checkpoint_failed") + } + } + if !cycleActive { + return taskengine.Suspend(model.TaskResumeModeExecute, externalPollInterval, "scheduled", "Local cache usage is within its target", nil) + } + activeBytes, err := h.deps.Repositories.CacheEvictions.ActiveEvictionBytes(ctx) + if err != nil { + return retryTask(err, "cache_capacity_scan_failed") + } + bytesToPlan := usedBytes - lowBytes - activeBytes + if bytesToPlan <= 0 { + return taskengine.Suspend(model.TaskResumeModeExecute, externalPollInterval, "cache_cleanup", "Local cache cleanup is in progress", nil) + } + plannedBytes, plannedTasks, err := h.planLRUEvictions(ctx, bytesToPlan) + if err != nil { + return retryTask(err, "cache_capacity_plan_failed") + } + message := "Waiting for remotely safe cached data" + if plannedTasks > 0 { + message = fmt.Sprintf("Scheduled cleanup for %d cached items (%d bytes)", plannedTasks, plannedBytes) + } + return taskengine.Suspend(model.TaskResumeModeExecute, externalPollInterval, "cache_cleanup", message, nil) + } + return taskHandler{definition: definition, execute: run, recover: run} +} + +func (h *TaskHandlers) planLRUEvictions(ctx context.Context, bytesToPlan int64) (int64, int, error) { + const candidateBatchSize = 100 + var plannedBytes int64 + var plannedTasks int + for bytesToPlan > 0 { + candidates, err := h.deps.Repositories.CacheEvictions.ListLRUCandidates(ctx, candidateBatchSize) + if err != nil { + return plannedBytes, plannedTasks, err + } + if len(candidates) == 0 { + break + } + createdThisBatch := 0 + for i := range candidates { + candidate := candidates[i] + scheduled := false + err := h.deps.Repositories.WithTx(ctx, func(repos *repository.Repositories) error { + reservation, err := repos.CacheEvictions.PrepareEviction(ctx, candidate.ContentID) + if err != nil { + return err + } + if reservation.ActiveTaskID != nil { + return nil + } + generation := reservation.Generation + accessedAt := cacheeviction.NormalizeAccessTime(candidate.AccessedAt) + taskRow, _, err := h.taskService.EnqueueInTransaction(ctx, repos, taskengine.EnqueueRequest{ + Type: model.TaskTypeCacheEvict, IdempotencyKey: cacheeviction.EvictTaskKey(candidate.ContentID, generation), + Input: cacheeviction.EvictInput{ContentID: candidate.ContentID, Generation: generation, AccessedAt: &accessedAt}, + SubjectType: "storage_content", SubjectKey: strconv.FormatInt(candidate.ContentID, 10), + }) + if err != nil { + return err + } + if err := repos.CacheEvictions.BindEvictionTask(ctx, candidate.ContentID, generation, taskRow.ID); err != nil { + return err + } + scheduled = true + return nil + }) + if errors.Is(err, repository.ErrConflict) || errors.Is(err, repository.ErrNotFound) { + continue + } + if err != nil { + return plannedBytes, plannedTasks, fmt.Errorf("planning cache cleanup for content %d: %w", candidate.ContentID, err) + } + if !scheduled { + continue + } + plannedBytes += candidate.Size + bytesToPlan -= candidate.Size + plannedTasks++ + createdThisBatch++ + if bytesToPlan <= 0 { + break + } + } + if len(candidates) < candidateBatchSize || createdThisBatch == 0 { + break + } + } + return plannedBytes, plannedTasks, nil +} + +func (h *TaskHandlers) cacheEvictHandler() taskengine.Handler { + definition := taskengine.Definition{ + Type: model.TaskTypeCacheEvict, InputVersion: 1, + Codec: taskengine.StrictJSONCodec(cacheeviction.ValidateEvictInput), + RetryLimit: h.retryLimit(), AllowRetry: true, + } + return taskHandler{ + definition: definition, + execute: func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + return h.runCacheEviction(ctx, execution, true) + }, + recover: func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + return h.runCacheEviction(ctx, execution, false) + }, + } +} + +func (h *TaskHandlers) runCacheEviction( + ctx context.Context, + execution taskengine.Execution, + allowDelete bool, +) taskengine.Result { + input, err := taskengine.DecodeInput[cacheeviction.EvictInput](execution) + if err != nil { + return decodeFailure(string(model.TaskTypeCacheEvict), err) + } + if h.deps.Cache == nil || h.deps.CacheGate == nil || h.deps.CacheTracker == nil { + return taskengine.Fail( + errors.New("cache deletion dependencies are unavailable"), + "dependency_unavailable", + h.releaseCacheEvictionSettlement(input, execution.ID()), + ) + } + if _, _, checkpointErr := taskengine.DecodeCheckpoint[cacheEvictionCheckpoint](execution); checkpointErr != nil { + return taskengine.Fail(checkpointErr, "invalid_checkpoint", h.releaseCacheEvictionSettlement(input, execution.ID())) + } + if allowDelete { + checkpoint := cacheEvictionCheckpoint{AttemptedAt: time.Now().UTC()} + if err := execution.WriteCheckpoint(ctx, checkpoint); err != nil { + return h.retryCacheEviction(execution, input, err, "cache_checkpoint_failed") + } + } + var result taskengine.Result + h.deps.CacheGate.GuardDeletion(model.ContentCacheKey(input.ContentID), func() { + if input.AccessedAt != nil { + if h.deps.EvictionPolicy != cache.EvictionPolicyLRU || !h.deps.CacheTracker.SafeForLRU() { + result = h.cancelCacheEviction(input, execution.ID(), "Cache removal is no longer needed") + return + } + if h.deps.Cache.UsedBytes() <= cacheWatermarkBytes(h.deps.MaxCacheBytes, h.deps.LRULowPercent) { + result = h.cancelCacheEviction(input, execution.ID(), "Local cache usage reached its target") + return + } + } + result = h.deleteAuthorizedCacheEntry(ctx, execution, input, allowDelete) + }) + return result +} + +func (h *TaskHandlers) deleteAuthorizedCacheEntry( + ctx context.Context, + execution taskengine.Execution, + input cacheeviction.EvictInput, + allowDelete bool, +) taskengine.Result { + deletionSucceeded := false + if input.AccessedAt != nil && allowDelete { + if !h.deps.CacheTracker.SafeForLRU() { + return h.cancelCacheEviction(input, execution.ID(), "Cache removal is no longer needed") + } + entry, err := h.deps.Repositories.CacheEvictions.GetCacheEntry(ctx, input.ContentID) + if err != nil { + return h.retryCacheEviction(execution, input, err, "cache_entry_load_failed") + } + content, err := h.deps.Repositories.Contents.GetByID(ctx, input.ContentID) + if err != nil { + return h.retryCacheEviction(execution, input, err, "cache_content_load_failed") + } + if entry == nil || content == nil || entry.CacheAccessedAt == nil || + !cacheeviction.NormalizeAccessTime(*entry.CacheAccessedAt).Equal(*input.AccessedAt) || + cacheeviction.NormalizeAccessTime(h.deps.CacheTracker.Latest(input.ContentID)).After(*input.AccessedAt) { + if entry != nil && entry.CacheAccessedAt != nil && + cacheeviction.NormalizeAccessTime(h.deps.CacheTracker.Latest(input.ContentID)).After(cacheeviction.NormalizeAccessTime(*entry.CacheAccessedAt)) { + if flushErr := h.deps.CacheTracker.FlushWhileGuarded(ctx, input.ContentID); flushErr != nil { + return h.retryCacheEviction(execution, input, flushErr, "cache_access_flush_failed") + } + } + return h.cancelCacheEviction(input, execution.ID(), "Cached data was used after cleanup was scheduled") + } + if !h.reserveLRUDeletion(content.ContentSize) { + return h.cancelCacheEviction(input, execution.ID(), "Local cache usage reached its target") + } + defer func() { h.finishLRUDeletion(content.ContentSize, deletionSucceeded) }() + } + if !allowDelete { + entry, err := h.deps.Repositories.CacheEvictions.GetCacheEntry(ctx, input.ContentID) + if err != nil { + return h.retryCacheEviction(execution, input, err, "cache_entry_load_failed") + } + if entry == nil { + return h.cancelCacheEviction(input, execution.ID(), "Cache removal is no longer needed") + } + content, err := h.deps.Repositories.Contents.GetByID(ctx, input.ContentID) + if err != nil { + return h.retryCacheEviction(execution, input, err, "cache_content_load_failed") + } + if content == nil { + return h.cancelCacheEviction(input, execution.ID(), "Cache removal is no longer needed") + } + bucket, err := h.deps.Repositories.Buckets.GetByID(ctx, content.BucketID) + if err != nil { + return h.retryCacheEviction(execution, input, err, "cache_bucket_load_failed") + } + if bucket == nil { + return h.cancelCacheEviction(input, execution.ID(), "Cache removal is no longer needed") + } + body, _, err := h.deps.Cache.Get(ctx, bucket.Name, model.ContentCacheKey(input.ContentID)) + switch { + case err == nil: + if body == nil { + return h.retryCacheEviction(execution, input, errors.New("cache returned an empty read handle"), "cache_observation_failed") + } + if closeErr := body.Close(); closeErr != nil { + return h.retryCacheEviction(execution, input, closeErr, "cache_observation_failed") + } + return taskengine.Suspend(model.TaskResumeModeExecute, 0, "safe_to_execute", "Local cache removal is ready", nil) + case os.IsNotExist(err): + finalizeErr := h.deps.Repositories.WithTx(ctx, func(repos *repository.Repositories) error { + if err := repos.Tasks.ValidateClaim(ctx, execution.ID(), execution.ClaimGeneration()); err != nil { + return err + } + return repos.CacheEvictions.RecordDeletion(ctx, input.ContentID, input.Generation, execution.ID()) + }) + if errors.Is(finalizeErr, repository.ErrConflict) || errors.Is(finalizeErr, repository.ErrNotFound) { + return h.cancelCacheEviction(input, execution.ID(), "Cache removal was superseded") + } + if finalizeErr != nil { + return h.retryCacheEviction(execution, input, finalizeErr, "cache_record_failed") + } + h.deps.CacheTracker.Forget(input.ContentID) + return taskengine.Complete("Local cache removed", nil) + default: + return h.retryCacheEviction(execution, input, err, "cache_observation_failed") + } + } + deleteErr := h.deps.Repositories.WithTx(ctx, func(repos *repository.Repositories) error { + if err := repos.Tasks.ValidateClaim(ctx, execution.ID(), execution.ClaimGeneration()); err != nil { + return err + } + authorized, err := repos.CacheEvictions.AuthorizeDeletion( + ctx, input.ContentID, input.Generation, execution.ID(), input.AccessedAt, + ) + if err != nil { + return err + } + if err := h.deps.Cache.Delete(ctx, authorized.BucketName, model.ContentCacheKey(authorized.Content.ID)); err != nil { + return fmt.Errorf("deleting cache file: %w", err) + } + return repos.CacheEvictions.RecordDeletion(ctx, input.ContentID, input.Generation, execution.ID()) + }) + if deleteErr != nil { + recorded, checkErr := h.deps.Repositories.CacheEvictions.DeletionRecorded(ctx, input.ContentID, input.Generation) + if checkErr == nil && recorded { + h.deps.CacheTracker.Forget(input.ContentID) + if input.AccessedAt != nil { + deletionSucceeded = true + } + return taskengine.Complete("Local cache removed", nil) + } + switch { + case errors.Is(deleteErr, cacheeviction.ErrDurabilityThreshold) && input.AccessedAt == nil: + return taskengine.Suspend(model.TaskResumeModeExecute, dependencyWait, "durability", "Waiting for durable storage", nil) + case errors.Is(deleteErr, cacheeviction.ErrDurabilityThreshold), errors.Is(deleteErr, cacheeviction.ErrNoLongerEligible), + errors.Is(deleteErr, cacheeviction.ErrAccessChanged), errors.Is(deleteErr, repository.ErrNotFound), errors.Is(deleteErr, repository.ErrConflict): + return h.cancelCacheEviction(input, execution.ID(), "Cache removal is no longer needed") + default: + return h.retryCacheEviction(execution, input, errors.Join(deleteErr, checkErr), "cache_delete_failed") + } + } + h.deps.CacheTracker.Forget(input.ContentID) + if input.AccessedAt != nil { + deletionSucceeded = true + } + return taskengine.Complete("Local cache removed", nil) +} + +func (h *TaskHandlers) cancelCacheEviction( + input cacheeviction.EvictInput, + taskID int64, + message string, +) taskengine.Result { + return taskengine.Cancel(message, h.releaseCacheEvictionSettlement(input, taskID)) +} + +func (h *TaskHandlers) releaseCacheEvictionSettlement(input cacheeviction.EvictInput, taskID int64) taskengine.Settlement { + return func(ctx context.Context, repos *repository.Repositories) error { + err := repos.CacheEvictions.ReleaseEviction(ctx, input.ContentID, input.Generation, taskID) + if errors.Is(err, repository.ErrConflict) || errors.Is(err, repository.ErrNotFound) { + return nil + } + return err + } +} + +func (h *TaskHandlers) retryCacheEviction( + execution taskengine.Execution, + input cacheeviction.EvictInput, + err error, + reason string, +) taskengine.Result { + if execution.RetryWillFail() { + return taskengine.Fail(err, reason, h.releaseCacheEvictionSettlement(input, execution.ID())) + } + return taskengine.RetryBackoff(err, reason, nil) +} + +func (h *TaskHandlers) reserveLRUDeletion(size int64) bool { + h.lruCapacityMu.Lock() + defer h.lruCapacityMu.Unlock() + if h.lruInFlightDeletes == 0 { + h.lruProjectedBytes = h.deps.Cache.UsedBytes() + } + if h.lruProjectedBytes <= cacheWatermarkBytes(h.deps.MaxCacheBytes, h.deps.LRULowPercent) { + return false + } + h.lruProjectedBytes -= size + h.lruInFlightDeletes++ + return true +} + +func (h *TaskHandlers) finishLRUDeletion(size int64, deleted bool) { + h.lruCapacityMu.Lock() + defer h.lruCapacityMu.Unlock() + if !deleted { + h.lruProjectedBytes += size + } + h.lruInFlightDeletes-- + if h.lruInFlightDeletes <= 0 { + h.lruInFlightDeletes = 0 + h.lruProjectedBytes = 0 + } +} + +func cacheWatermarkBytes(maxBytes int64, percent int) int64 { + if maxBytes <= 0 || percent <= 0 { + return 0 + } + if percent >= 100 { + return maxBytes + } + return (maxBytes/100)*int64(percent) + (maxBytes%100)*int64(percent)/100 +} + +func (h *TaskHandlers) cacheDurabilityHandler() taskengine.Handler { + definition := taskengine.Definition{ + Type: model.TaskTypeCacheReconcileDurability, InputVersion: 1, + Codec: taskengine.StrictJSONCodec(cacheeviction.ValidateDurabilityInput), + RetryLimit: h.retryLimit(), AllowRetry: true, + } + run := func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + input, err := taskengine.DecodeInput[cacheeviction.DurabilityInput](execution) + if err != nil { + return decodeFailure(string(definition.Type), err) + } + candidate, err := h.deps.Repositories.CacheEvictions.NextBucketDurabilityCandidate(ctx, input.BucketID, input.Generation, execution.ID()) + if err != nil { + if errors.Is(err, repository.ErrConflict) { + return taskengine.Cancel("A newer storage policy update replaced this operation", nil) + } + return retryTask(err, "durability_scan_failed") + } + if candidate == nil { + return taskengine.Complete("Bucket storage policy applied", func(ctx context.Context, repos *repository.Repositories) error { + return repos.CacheEvictions.CompleteBucketDurability(ctx, input.BucketID, input.Generation, execution.ID()) + }) + } + return taskengine.Suspend(model.TaskResumeModeExecute, 0, "more_work", "Applying bucket storage policy", func(ctx context.Context, repos *repository.Repositories) error { + if h.deps.EvictionPolicy != cache.EvictionPolicyAfterUpload { + return nil + } + if h.taskService == nil { + return errors.New("task service is unavailable") + } + reservation, err := repos.CacheEvictions.PrepareEviction(ctx, candidate.ID) + if err != nil { + return err + } + if reservation.ActiveTaskID != nil { + return nil + } + generation := reservation.Generation + taskRow, _, err := h.taskService.EnqueueInTransaction(ctx, repos, taskengine.EnqueueRequest{ + Type: model.TaskTypeCacheEvict, IdempotencyKey: cacheeviction.EvictTaskKey(candidate.ID, generation), + Input: cacheeviction.EvictInput{ContentID: candidate.ID, Generation: generation}, + SubjectType: "storage_content", SubjectKey: strconv.FormatInt(candidate.ID, 10), + }) + if err != nil { + return err + } + return repos.CacheEvictions.BindEvictionTask(ctx, candidate.ID, generation, taskRow.ID) + }) + } + return taskHandler{definition: definition, execute: run, recover: run} +} + +func (h *TaskHandlers) storageCleanupHandler() taskengine.Handler { + definition := taskengine.Definition{ + Type: model.TaskTypeStorageCleanup, InputVersion: 1, + Codec: taskengine.StrictJSONCodec(func(input *storagecleanup.Input) error { return storagecleanup.ValidateInput(*input) }), + RetryLimit: h.retryLimit(), AllowRetry: true, + } + return taskHandler{ + definition: definition, + execute: func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + return h.runStorageCleanup(ctx, execution, true) + }, + recover: func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + return h.runStorageCleanup(ctx, execution, false) + }, + } +} + +func (h *TaskHandlers) runStorageCleanup(ctx context.Context, execution taskengine.Execution, allowDelete bool) taskengine.Result { + input, err := taskengine.DecodeInput[storagecleanup.Input](execution) + if err != nil { + return decodeFailure(string(model.TaskTypeStorageCleanup), err) + } + if h.deps.Storage == nil { + return taskengine.Fail(errors.New("storage cleanup client is unavailable"), "dependency_unavailable", nil) + } + copies, err := h.deps.Repositories.StorageCleanup.AuthorizeTask(ctx, input.ContentID, input.Generation, execution.ID()) + if err != nil { + if errors.Is(err, repository.ErrConflict) || errors.Is(err, repository.ErrNotFound) { + return taskengine.Cancel("Remote cleanup was superseded", nil) + } + return retryTask(err, "cleanup_authorization_failed") + } + hasReferences, err := h.deps.Repositories.StorageCleanup.UploadHasObjectReferences(ctx, input.ContentID) + if err == nil && !hasReferences { + hasReferences, err = h.deps.Repositories.StorageCleanup.CleanupHasObjectReferences(ctx, input.ContentID) + } + if err != nil { + return retryTask(err, "cleanup_reference_check_failed") + } + if hasReferences { + return taskengine.Suspend(model.TaskResumeModeRecover, dependencyWait, "references", "Waiting for stored data references", nil) + } + checkpoint, hasCheckpoint, err := taskengine.DecodeCheckpoint[cleanupCheckpoint](execution) + if err != nil { + return taskengine.Fail(err, "invalid_checkpoint", nil) + } + hasUnsupported := false + for i := range copies { + copyRow := copies[i] + switch copyRow.Status { + case model.StorageCleanupCopyStatusRemoved: + continue + case model.StorageCleanupCopyStatusUnsupported: + hasUnsupported = true + continue + } + if copyRow.DataSetID == nil || copyRow.DataSetID.IsZero() || copyRow.ProviderID.IsZero() || copyRow.PieceID.IsZero() || copyRow.PieceCID == "" { + message := "Storage provider details are incomplete" + if err := h.deps.Repositories.StorageCleanup.MarkCopyUnsupported(ctx, copyRow.ID, message); err != nil { + return retryTask(err, "cleanup_evidence_failed") + } + hasUnsupported = true + continue + } + pieceCID, err := cid.Parse(copyRow.PieceCID) + if err != nil { + message := "Stored data identifier is invalid" + if markErr := h.deps.Repositories.StorageCleanup.MarkCopyUnsupported(ctx, copyRow.ID, message); markErr != nil { + return retryTask(markErr, "cleanup_evidence_failed") + } + hasUnsupported = true + continue + } + providerID := copyRow.ProviderID.SDK() + cleanupContext, err := h.deps.Storage.OpenCleanupContext(ctx, copyRow.DataSetID.SDK(), storage.NewDataSetContextOptions{ProviderID: &providerID}) + if err != nil { + return retryTask(err, "cleanup_context_failed") + } + status, err := cleanupContext.PieceStatus(ctx, pieceCID) + if err != nil { + return taskengine.Suspend(model.TaskResumeModeRecover, externalPollInterval, "provider_confirmation", "Checking remote cleanup", nil) + } + if status == nil || !status.Exists { + if err := h.deps.Repositories.StorageCleanup.MarkCopyRemoved(ctx, copyRow.ID); err != nil { + return retryTask(err, "cleanup_evidence_failed") + } + continue + } + if copyRow.Status == model.StorageCleanupCopyStatusDeleteScheduled || (hasCheckpoint && checkpoint.CopyID == copyRow.ID) { + attemptedAt := checkpoint.AttemptedAt + if attemptedAt.IsZero() && copyRow.ScheduledAt != nil { + attemptedAt = *copyRow.ScheduledAt + } + if !attemptedAt.IsZero() && time.Since(attemptedAt) >= cleanupAttentionAfter { + err := errors.New("remote replica deletion could not be confirmed") + return taskengine.Fail(err, "cleanup_outcome_unknown", func(ctx context.Context, repos *repository.Repositories) error { + return repos.StorageCleanup.MarkCopyFailed(ctx, copyRow.ID, err.Error()) + }) + } + return taskengine.Suspend(model.TaskResumeModeRecover, externalPollInterval, "provider_confirmation", "Waiting for remote cleanup", nil) + } + if !allowDelete { + return taskengine.Suspend(model.TaskResumeModeExecute, 0, "safe_to_execute", "Remote cleanup is ready", nil) + } + checkpoint = cleanupCheckpoint{CopyID: copyRow.ID, AttemptedAt: time.Now().UTC()} + var txHash string + attempted, err := execution.WithCheckpointedEffect(ctx, taskengine.ResourceDestructiveMutation, checkpoint, nil, func(ctx context.Context) error { + result, deleteErr := cleanupContext.DeletePieceByID(ctx, copyRow.PieceID.SDK()) + if result != nil { + txHash = result.Hash.String() + } + return deleteErr + }) + if err != nil { + if !attempted { + return retryTask(err, "cleanup_not_started") + } + return taskengine.Suspend(model.TaskResumeModeRecover, externalPollInterval, "provider_confirmation", "Checking remote cleanup", nil) + } + if err := h.deps.Repositories.StorageCleanup.MarkCopyDeleteScheduled(ctx, copyRow.ID, txHash); err != nil { + return taskengine.Suspend(model.TaskResumeModeRecover, externalPollInterval, "provider_confirmation", "Recording remote cleanup", nil) + } + return taskengine.Suspend(model.TaskResumeModeRecover, externalPollInterval, "provider_confirmation", "Waiting for remote cleanup", nil) + } + if hasUnsupported { + return taskengine.Fail(errors.New("one or more remote replica deletions are unsupported"), "cleanup_unsupported", nil) + } + return taskengine.Complete("Remote replicas removed", func(ctx context.Context, repos *repository.Repositories) error { + return repos.StorageCleanup.CompleteTask(ctx, input.ContentID, input.Generation, execution.ID()) + }) +} + +func (h *TaskHandlers) walletHandler() taskengine.Handler { + definition := taskengine.Definition{ + Type: model.TaskTypeWalletOperation, InputVersion: 1, + Codec: taskengine.StrictJSONCodec(func(input *walletoperation.Input) error { return walletoperation.ValidateInput(*input) }), + RetryLimit: h.retryLimit(), AllowRetry: true, + CanManualRetry: func(task *model.Task) bool { + return task != nil && task.FailureReason != nil && *task.FailureReason == "wallet_broadcast_not_started" + }, + } + return taskHandler{ + definition: definition, + execute: func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + return h.executeWalletOperation(ctx, execution) + }, + recover: func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + return h.recoverWalletOperation(ctx, execution) + }, + } +} + +func (h *TaskHandlers) executeWalletOperation(ctx context.Context, execution taskengine.Execution) taskengine.Result { + input, err := taskengine.DecodeInput[walletoperation.Input](execution) + if err != nil { + return decodeFailure(string(model.TaskTypeWalletOperation), err) + } + op, err := h.deps.Repositories.WalletOperations.GetByID(ctx, input.OperationID) + if err != nil { + return h.retryWalletOperation(execution, input.OperationID, err, "wallet_load_failed") + } + if result, done := h.walletTerminalResult(op, execution.ID()); done { + return result + } + if op.Status == model.WalletOperationStatusSubmitted || op.BroadcastAttemptedAt != nil { + return h.recoverWalletOperation(ctx, execution) + } + if h.deps.Wallet == nil { + return taskengine.Fail(errors.New("wallet operator is unavailable"), "dependency_unavailable", nil) + } + amount, ok := new(big.Int).SetString(op.Amount, 10) + if !ok || !validTaskWalletAmount(op.Type, amount) { + return taskengine.Fail(errors.New("invalid wallet operation amount"), "invalid_input", func(ctx context.Context, repos *repository.Repositories) error { + return repos.WalletOperations.MarkFailed(ctx, op.ID, execution.ID(), "invalid wallet operation amount") + }) + } + checkpoint := walletoperation.Checkpoint{BroadcastAttempted: true} + var txHash string + var alreadyComplete bool + attempted, err := execution.WithCheckpointedEffect(ctx, taskengine.ResourceWallet, checkpoint, func(ctx context.Context, repos *repository.Repositories) error { + return repos.WalletOperations.MarkBroadcastAttempted(ctx, op.ID, execution.ID()) + }, func(ctx context.Context) error { + requestCtx, cancel := context.WithTimeout(ctx, h.deps.WalletBroadcastTimeout) + defer cancel() + var broadcastErr error + txHash, alreadyComplete, broadcastErr = broadcastWalletOperation(requestCtx, h.deps.Wallet, op.Type, amount) + return broadcastErr + }) + if err != nil { + if !attempted { + return retryTask(err, "wallet_broadcast_not_started") + } + message := fmt.Sprintf("wallet broadcast outcome is unknown: %v", err) + return taskengine.Fail(err, "wallet_broadcast_unknown", func(ctx context.Context, repos *repository.Repositories) error { + return repos.WalletOperations.MarkUnknown(ctx, op.ID, execution.ID(), message) + }) + } + if alreadyComplete { + return taskengine.Complete("Wallet authorization already satisfied", func(ctx context.Context, repos *repository.Repositories) error { + return repos.WalletOperations.MarkConfirmedWithoutTransaction(ctx, op.ID, execution.ID()) + }) + } + if txHash == "" { + err := errors.New("wallet broadcast returned no transaction hash") + return taskengine.Fail(err, "wallet_broadcast_unknown", func(ctx context.Context, repos *repository.Repositories) error { + return repos.WalletOperations.MarkUnknown(ctx, op.ID, execution.ID(), err.Error()) + }) + } + checkpoint.TransactionHash = txHash + if err := execution.WriteCheckpointWith(ctx, checkpoint, func(ctx context.Context, repos *repository.Repositories) error { + return repos.WalletOperations.MarkSubmitted(ctx, op.ID, execution.ID(), txHash) + }); err != nil { + return taskengine.Suspend(model.TaskResumeModeRecover, externalPollInterval, "transaction_confirmation", "Recording wallet transaction", func(ctx context.Context, repos *repository.Repositories) error { + return repos.WalletOperations.MarkSubmitted(ctx, op.ID, execution.ID(), txHash) + }) + } + return taskengine.Suspend(model.TaskResumeModeRecover, externalPollInterval, "transaction_confirmation", "Waiting for wallet transaction", nil) +} + +func (h *TaskHandlers) recoverWalletOperation(ctx context.Context, execution taskengine.Execution) taskengine.Result { + input, err := taskengine.DecodeInput[walletoperation.Input](execution) + if err != nil { + return decodeFailure(string(model.TaskTypeWalletOperation), err) + } + op, err := h.deps.Repositories.WalletOperations.GetByID(ctx, input.OperationID) + if err != nil { + return h.retryWalletOperation(execution, input.OperationID, err, "wallet_load_failed") + } + if result, done := h.walletTerminalResult(op, execution.ID()); done { + return result + } + checkpoint, hasCheckpoint, err := taskengine.DecodeCheckpoint[walletoperation.Checkpoint](execution) + if err != nil { + return taskengine.Fail(err, "invalid_checkpoint", nil) + } + txHash := "" + if op.TxHash != nil { + txHash = *op.TxHash + } + if txHash == "" && hasCheckpoint { + txHash = checkpoint.TransactionHash + } + if txHash == "" { + if !hasCheckpoint && op.BroadcastAttemptedAt == nil { + return taskengine.Suspend(model.TaskResumeModeExecute, 0, "safe_to_execute", "Wallet operation is ready", nil) + } + err := errors.New("wallet transaction identity could not be recovered") + return taskengine.Fail(err, "wallet_broadcast_unknown", func(ctx context.Context, repos *repository.Repositories) error { + return repos.WalletOperations.MarkUnknown(ctx, op.ID, execution.ID(), err.Error()) + }) + } + if h.deps.Receipts == nil { + return taskengine.Suspend(model.TaskResumeModeRecover, externalPollInterval, "transaction_confirmation", "Waiting for wallet transaction", func(ctx context.Context, repos *repository.Repositories) error { + return repos.WalletOperations.MarkSubmitted(ctx, op.ID, execution.ID(), txHash) + }) + } + requestCtx, cancel := context.WithTimeout(ctx, h.deps.WalletReceiptTimeout) + receipt, err := h.deps.Receipts.TransactionReceipt(requestCtx, common.HexToHash(txHash)) + cancel() + if errors.Is(err, ethereum.NotFound) || receipt == nil { + return taskengine.Suspend(model.TaskResumeModeRecover, externalPollInterval, "transaction_confirmation", "Waiting for wallet transaction", func(ctx context.Context, repos *repository.Repositories) error { + return repos.WalletOperations.MarkSubmitted(ctx, op.ID, execution.ID(), txHash) + }) + } + if err != nil { + return taskengine.Suspend(model.TaskResumeModeRecover, externalPollInterval, "transaction_confirmation", "Checking wallet transaction", nil) + } + if receipt.Status == ethtypes.ReceiptStatusSuccessful { + return taskengine.Complete("Wallet transaction confirmed", func(ctx context.Context, repos *repository.Repositories) error { + return repos.WalletOperations.MarkConfirmed(ctx, op.ID, execution.ID(), txHash) + }) + } + message := fmt.Sprintf("wallet transaction reverted: status=%d transaction=%s", receipt.Status, txHash) + return taskengine.Fail(errors.New(message), "wallet_transaction_reverted", func(ctx context.Context, repos *repository.Repositories) error { + return repos.WalletOperations.MarkFailed(ctx, op.ID, execution.ID(), message) + }) +} + +func (h *TaskHandlers) retryWalletOperation(execution taskengine.Execution, operationID int64, err error, reason string) taskengine.Result { + if !execution.RetryWillFail() { + return retryTask(err, reason) + } + return taskengine.Fail(err, reason, func(ctx context.Context, repos *repository.Repositories) error { + return repos.WalletOperations.MarkFailed(ctx, operationID, execution.ID(), err.Error()) + }) +} + +func (h *TaskHandlers) walletTerminalResult(op *model.WalletOperation, taskID int64) (taskengine.Result, bool) { + if op == nil { + return taskengine.Fail(repository.ErrNotFound, "wallet_operation_missing", nil), true + } + if op.TaskID == nil || *op.TaskID != taskID { + switch op.Status { + case model.WalletOperationStatusConfirmed: + return taskengine.Complete("Wallet operation completed", nil), true + case model.WalletOperationStatusFailed, model.WalletOperationStatusUnknown: + return taskengine.Fail(errors.New("wallet operation requires attention"), "wallet_operation_terminal", nil), true + default: + return taskengine.Fail(repository.ErrConflict, "wallet_task_superseded", nil), true + } + } + return taskengine.Result{}, false +} + +func validTaskWalletAmount(operationType model.WalletOperationType, amount *big.Int) bool { + if amount == nil { + return false + } + switch operationType { + case model.WalletOperationTypeApprove: + return amount.Sign() == 0 + case model.WalletOperationTypeFund, model.WalletOperationTypeWithdraw: + return amount.Sign() > 0 + default: + return false + } +} + +func broadcastWalletOperation(ctx context.Context, operator synapse.WalletOperator, operationType model.WalletOperationType, amount *big.Int) (string, bool, error) { + switch operationType { + case model.WalletOperationTypeFund: + hash, err := operator.FundUSDFC(ctx, amount) + return hash, false, err + case model.WalletOperationTypeWithdraw: + hash, err := operator.WithdrawUSDFC(ctx, amount) + return hash, false, err + case model.WalletOperationTypeApprove: + hash, err := operator.ApproveFWSS(ctx) + if errors.Is(err, payments.ErrNothingToFund) { + return "", true, nil + } + return hash, false, err + default: + return "", false, fmt.Errorf("unsupported wallet operation type %q", operationType) + } +} + +func (h *TaskHandlers) observabilityHandler() taskengine.Handler { + definition := taskengine.Definition{ + Type: model.TaskTypeObservabilityRefresh, InputVersion: 1, + Codec: taskengine.StrictJSONCodec(func(input *systemtask.Input) error { return systemtask.ValidateInput(*input) }), + RetryLimit: nil, AllowRetry: true, + } + run := func(ctx context.Context, _ taskengine.Execution) taskengine.Result { + if h.deps.Observability == nil { + return taskengine.Fail(errors.New("observability service is unavailable"), "dependency_unavailable", nil) + } + if err := h.deps.Observability.RefreshAll(ctx); err != nil { + return retryTask(err, "observability_refresh_failed") + } + return taskengine.Suspend(model.TaskResumeModeExecute, h.deps.Observability.RefreshInterval(), "scheduled", "Storage health refreshed", nil) + } + return taskHandler{definition: definition, execute: run, recover: run} +} + +func (h *TaskHandlers) gcHandler() taskengine.Handler { + definition := taskengine.Definition{ + Type: model.TaskTypeGC, InputVersion: 1, + Codec: taskengine.StrictJSONCodec(func(input *systemtask.Input) error { return systemtask.ValidateInput(*input) }), + RetryLimit: nil, AllowRetry: true, + } + run := func(ctx context.Context, _ taskengine.Execution) taskengine.Result { + deleted, err := h.deps.Repositories.Tasks.DeleteRetained(ctx, time.Now(), cleanupGCPageSize) + if err != nil { + return retryTask(err, "task_gc_failed") + } + delay := taskGCInterval + if deleted == cleanupGCPageSize { + delay = 0 + } + return taskengine.Suspend(model.TaskResumeModeExecute, delay, "scheduled", fmt.Sprintf("Removed %d expired task records", deleted), nil) + } + return taskHandler{definition: definition, execute: run, recover: run} +} diff --git a/internal/worker/evictor.go b/internal/worker/evictor.go deleted file mode 100644 index f007b05..0000000 --- a/internal/worker/evictor.go +++ /dev/null @@ -1,321 +0,0 @@ -package worker - -import ( - "context" - "log/slog" - "sync" - "time" - - "github.com/strahe/synaps3/internal/admin" - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/cacheaccess" - "github.com/strahe/synaps3/internal/cacheeviction" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/state" -) - -const ( - replicatingEvictDeferDelay = 30 * time.Second - lruCandidateBatchSize = 100 - lruTerminalRetryDelay = time.Hour -) - -// Evictor claims cache eviction tasks and removes remotely durable objects -// according to the configured policy. -type Evictor struct { - repos *repository.Repositories - cache cache.Cache - cacheGate *cacheaccess.Gate - cacheAccessTracker *cacheaccess.Tracker - stateMachine *state.Machine - policy cache.EvictionPolicy - maxCacheBytes int64 - highWatermarkPercent int - lowWatermarkPercent int - maxRetries int - concurrency int - pollInterval time.Duration - leaseTTL time.Duration - logger *slog.Logger - lruCapacityMu sync.Mutex - lruCycleActive bool - lruProjectedBytes int64 - lruInFlightDeletes int - lruPauseLogOnce sync.Once - *livenessTracker -} - -// EvictorOption configures cache eviction behavior. -type EvictorOption func(*Evictor) - -// WithCacheEvictionPolicy configures the eviction strategy and LRU capacity -// thresholds. The thresholds are ignored outside LRU mode. -func WithCacheEvictionPolicy( - policy cache.EvictionPolicy, - maxCacheBytes int64, - highWatermarkPercent int, - lowWatermarkPercent int, - maxRetries int, -) EvictorOption { - return func(e *Evictor) { - e.policy = policy - e.maxCacheBytes = maxCacheBytes - e.highWatermarkPercent = highWatermarkPercent - e.lowWatermarkPercent = lowWatermarkPercent - e.maxRetries = maxRetries - } -} - -// NewEvictor creates a new cache evictor worker. -func NewEvictor( - repos *repository.Repositories, - c cache.Cache, - cacheGate *cacheaccess.Gate, - cacheAccessTracker *cacheaccess.Tracker, - sm *state.Machine, - concurrency int, - pollInterval time.Duration, - logger *slog.Logger, - opts ...EvictorOption, -) *Evictor { - if cacheGate == nil { - panic("evictor requires a cache access gate") - } - if cacheAccessTracker == nil { - panic("evictor requires a cache access tracker") - } - e := &Evictor{ - repos: repos, - cache: c, - cacheGate: cacheGate, - cacheAccessTracker: cacheAccessTracker, - stateMachine: sm, - policy: cache.EvictionPolicyNone, - highWatermarkPercent: 90, - lowWatermarkPercent: 80, - maxRetries: defaultEvictMaxRetries, - concurrency: concurrency, - pollInterval: pollInterval, - leaseTTL: 5 * time.Minute, - logger: logger, - livenessTracker: newLivenessTracker(pollInterval), - } - for _, opt := range opts { - opt(e) - } - return e -} - -func (e *Evictor) Name() string { return "evictor" } - -func (e *Evictor) Run(ctx context.Context) error { - var wg sync.WaitGroup - if e.policy == cache.EvictionPolicyLRU { - wg.Add(1) - go func() { - defer wg.Done() - e.runLRUPlanner(ctx) - }() - } - for range e.concurrency { - wg.Add(1) - go func() { - defer wg.Done() - e.runSlot(ctx) - }() - } - - wg.Wait() - return ctx.Err() -} - -func (e *Evictor) runSlot(ctx context.Context) { - if !sleepUntilNextWorkerPoll(ctx, e.pollInterval) { - return - } - - for { - if ctx.Err() != nil { - return - } - - e.recordTick() - task, err := e.repos.Tasks.ClaimReady(ctx, model.TaskTypeEvictCache, e.leaseTTL) - if err != nil { - if ctx.Err() != nil { - return - } - e.logger.Error("claiming evict_cache task", "error", err) - if !sleepUntilNextWorkerPoll(ctx, e.pollInterval) { - return - } - continue - } - if task == nil { - if !sleepUntilNextWorkerPoll(ctx, e.pollInterval) { - return - } - continue - } - - e.recordWorkStarted() - func() { - defer e.recordWorkFinished() - stopLeaseRenewal := startTaskLeaseRenewal(e.logger, e.repos, task, e.leaseTTL) - defer stopLeaseRenewal() - e.processTask(ctx, task) - }() - releaseTaskOnWorkerShutdown(ctx, e.logger, e.repos, task) - } -} - -// Healthy returns true if the worker has ticked recently. -func (e *Evictor) Healthy() bool { return e.healthy() } - -func (e *Evictor) lruAccessTrackingSafe() bool { - if e.cacheAccessTracker.SafeForLRU() { - return true - } - admin.CacheLRUEvictionPaused.Set(1) - e.lruPauseLogOnce.Do(func() { - e.logger.Error( - "LRU cache eviction paused until restart", - "reason", - cacheaccess.ErrLRUAccessUncertain, - "action", - "resolve cache access persistence errors and restart SynapS3", - ) - }) - return false -} - -func (e *Evictor) processTask(ctx context.Context, task *model.Task) { - start := time.Now() - defer func() { - admin.WorkerTaskDuration.WithLabelValues("evictor").Observe(time.Since(start).Seconds()) - }() - - var decision *evictionDecision - switch taskStage(task) { - case cacheeviction.StageLRU: - decision = e.processLRUEviction(ctx, task) - case cacheeviction.StageAfterUpload: - decision = e.processAfterUploadEviction(ctx, task) - case cacheeviction.StageReconcileBucketDurability: - e.processBucketDurabilityReconciliation(ctx, task) - return - default: - decision = cancelEviction("Cache eviction task uses an unsupported stage") - } - e.applyEvictionDecision(ctx, task, decision) -} - -func (e *Evictor) reserveLRUDeletion(size int64) bool { - e.lruCapacityMu.Lock() - defer e.lruCapacityMu.Unlock() - - // Use one projected-usage ledger for the whole concurrent deletion batch. - // Reading live usage mid-batch can count a completed unlink and its - // still-finishing reservation twice. - if e.lruInFlightDeletes == 0 { - e.lruProjectedBytes = e.cache.UsedBytes() - } - lowBytes := watermarkBytes(e.maxCacheBytes, e.lowWatermarkPercent) - if e.lruProjectedBytes <= lowBytes { - return false - } - if size > 0 { - e.lruProjectedBytes -= size - e.lruInFlightDeletes++ - } - return true -} - -func (e *Evictor) finishLRUDeletion(size int64, deleted bool) { - if size <= 0 { - return - } - e.lruCapacityMu.Lock() - defer e.lruCapacityMu.Unlock() - - if !deleted { - e.lruProjectedBytes += size - } - e.lruInFlightDeletes-- - if e.lruInFlightDeletes <= 0 { - e.lruInFlightDeletes = 0 - e.lruProjectedBytes = 0 - } -} - -func (e *Evictor) shouldPlanLRUCycle(usedBytes, highBytes, lowBytes int64) bool { - e.lruCapacityMu.Lock() - defer e.lruCapacityMu.Unlock() - - // Preserve the hysteresis cycle after crossing the high watermark even when - // individual planned tasks are cancelled or retried. - switch { - case usedBytes <= lowBytes: - e.lruCycleActive = false - case usedBytes >= highBytes: - e.lruCycleActive = true - } - return e.lruCycleActive -} - -func (e *Evictor) deferReplicatingEviction(ctx context.Context, task *model.Task) { - logger := e.taskLogger(task) - if err := e.repos.Tasks.WaitRunning( - ctx, - task, - model.TaskWaitReasonDependency, - "Waiting for enough durable replicas to release the cache", - replicatingEvictDeferDelay, - ); err != nil { - logger.Error("failed to defer replicating cache eviction", "error", err) - _ = e.repos.Tasks.FailRunning(ctx, task, err.Error()) - admin.WorkerTasksProcessed.WithLabelValues("evictor", "failure").Inc() - return - } - admin.WorkerTasksProcessed.WithLabelValues("evictor", "success").Inc() - logger.Info("cache eviction deferred until durability policy is satisfied") -} - -func (e *Evictor) completeTask(ctx context.Context, task *model.Task, logger *slog.Logger, message string) { - if !completeWorkerTask(ctx, e.repos, task, "evictor", logger) { - return - } - logger.Info(message) -} - -func (e *Evictor) cancelTask(ctx context.Context, task *model.Task, message string) { - logger := e.taskLogger(task) - if err := e.repos.Tasks.CancelRunning(ctx, task, message); err != nil { - logger.Error("failed to cancel cache eviction task", "error", err) - admin.WorkerTasksProcessed.WithLabelValues("evictor", "failure").Inc() - return - } - admin.WorkerTasksProcessed.WithLabelValues("evictor", "success").Inc() - logger.Info("cache eviction task cancelled", "reason", message) -} - -func (e *Evictor) failTask(ctx context.Context, task *model.Task, lastError string, logMessage string) { - logger := e.taskLogger(task) - logger.Warn(logMessage) - if err := e.repos.Tasks.FailRunning(ctx, task, lastError); err != nil { - logger.Error("failed to record cache eviction failure", "error", err) - } - admin.WorkerTasksProcessed.WithLabelValues("evictor", "failure").Inc() -} - -func (e *Evictor) retryTask(ctx context.Context, task *model.Task, err error, logMessage string) { - logger := e.taskLogger(task) - logger.Warn(logMessage, "error", err) - scheduleTaskRetry(ctx, e.repos, task, "evictor", logger, err) - admin.WorkerTasksProcessed.WithLabelValues("evictor", "failure").Inc() -} - -func (e *Evictor) taskLogger(task *model.Task) *slog.Logger { - return e.logger.With("taskID", task.ID, "objectID", task.RefID, "versionID", task.RefVersionID) -} diff --git a/internal/worker/evictor_after_upload.go b/internal/worker/evictor_after_upload.go deleted file mode 100644 index 4afca1b..0000000 --- a/internal/worker/evictor_after_upload.go +++ /dev/null @@ -1,49 +0,0 @@ -package worker - -import ( - "context" - "errors" - - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/cacheeviction" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" -) - -func (e *Evictor) processAfterUploadEviction( - ctx context.Context, - task *model.Task, -) *evictionDecision { - authorized, err := cacheeviction.DeleteAuthorized(task) - if err != nil { - return cancelEviction("Cache deletion authorization is invalid") - } - if !authorized && e.policy != cache.EvictionPolicyAfterUpload { - return cancelEviction("Cache eviction policy no longer removes objects after upload") - } - - var decision *evictionDecision - e.cacheGate.GuardDeletion(task.RefVersionID, func() { - decision = e.finalizeAfterUploadEviction(ctx, task) - }) - return decision -} - -func (e *Evictor) finalizeAfterUploadEviction( - ctx context.Context, - task *model.Task, -) *evictionDecision { - deletion, err := e.repos.CacheEvictions.AuthorizeDeletion(ctx, task, nil) - switch { - case err == nil: - return e.deleteCacheEntry(ctx, task, deletion) - case errors.Is(err, cacheeviction.ErrDurabilityThreshold): - return waitForEvictionDependency() - case errors.Is(err, repository.ErrNotFound): - return failEviction("object not found", "object version not found for after-upload cache eviction") - case errors.Is(err, cacheeviction.ErrNoLongerEligible), errors.Is(err, cacheeviction.ErrAccessChanged): - return failEviction("not stored", "object version not in stored state") - default: - return retryEviction(err, "authorizing after-upload cache eviction") - } -} diff --git a/internal/worker/evictor_after_upload_failure_test.go b/internal/worker/evictor_after_upload_failure_test.go deleted file mode 100644 index bf3f917..0000000 --- a/internal/worker/evictor_after_upload_failure_test.go +++ /dev/null @@ -1,241 +0,0 @@ -package worker_test - -import ( - "context" - "errors" - "io" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/cacheeviction" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/testutil" -) - -func TestEvictor_Preconditions(t *testing.T) { - tests := []struct { - name string - setup func(ctx context.Context, t *testing.T, env *testWorkerEnv) *model.Task - wantLastError string - }{ - { - name: "MissingVersion", - setup: func(ctx context.Context, t *testing.T, env *testWorkerEnv) *model.Task { - _, objID, _ := seedStoredObject(t, env) - stage := cacheeviction.StageAfterUpload - task := &model.Task{ - Type: model.TaskTypeEvictCache, - Stage: &stage, - RefType: "object", - RefID: objID, - RefVersionID: "01J000000000000000MISSING1", - IdempotencyKey: "evict_cache:missing", - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("creating task: %v", err) - } - return task - }, - wantLastError: "object not found", - }, - { - name: "WrongState", - setup: func(ctx context.Context, t *testing.T, env *testWorkerEnv) *model.Task { - _, objID, versionID := seedObjectInDB(t, env, model.BucketStatusActive) - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("transition: %v", err) - } - return seedTask(t, env, model.TaskTypeEvictCache, objID, versionID, 5, 0) - }, - wantLastError: "not stored", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - mc := &testutil.MockCache{} - env := newTestWorkerEnvWithMockCache(t, mc) - ctx := context.Background() - - task := tt.setup(ctx, t, env) - - evictor := newAfterUploadEvictor(env, 1, 50*time.Millisecond) - runWorkerUntilTask(t, env, evictor, task.ID, 5*time.Second) - - got, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil { - t.Fatalf("getting task: %v", err) - } - if got == nil { - t.Fatalf("task %d not found", task.ID) - } - if got.Status != model.TaskStatusFailed { - t.Errorf("expected task failed, got %s", got.Status) - } - if got.LastError == nil || !strings.Contains(*got.LastError, tt.wantLastError) { - t.Errorf("expected last error to contain %q, got %v", tt.wantLastError, got.LastError) - } - }) - } -} - -func TestEvictor_AfterUploadWaitsWhenMinimumIsNoLongerMet(t *testing.T) { - var deleteCalls atomic.Int64 - mc := &testutil.MockCache{DeleteFunc: func(context.Context, string, string) error { - deleteCalls.Add(1) - return nil - }} - env := newTestWorkerEnvWithMockCache(t, mc) - ctx := context.Background() - _, objID, versionID := seedStoredObject(t, env) - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil || version.StorageUploadID == nil { - t.Fatalf("stored version upload: version=%v err=%v", version, err) - } - if _, err := env.db.NewDelete().Model((*model.StorageUploadCopy)(nil)).Where("upload_id = ?", *version.StorageUploadID).Exec(ctx); err != nil { - t.Fatalf("remove readable copies: %v", err) - } - task := seedTask(t, env, model.TaskTypeEvictCache, objID, versionID, 5, 0) - evictor := newAfterUploadEvictor(env, 1, 10*time.Millisecond) - - runCtx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = evictor.Run(runCtx) - close(done) - }() - waitForTaskStatus(t, env, task.ID, model.TaskStatusWaiting, 3*time.Second) - cancel() - waitForSignal(t, done, time.Second, "after-upload durability wait shutdown") - - got, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || got == nil || got.RetryCount != 0 || got.WaitReason == nil || *got.WaitReason != model.TaskWaitReasonDependency { - t.Fatalf("waiting task = %#v err=%v", got, err) - } - if deleteCalls.Load() != 0 { - t.Fatalf("cache delete calls = %d, want 0", deleteCalls.Load()) - } -} - -func TestEvictor_CacheDeleteFailureLeavesObjectUnchangedAndKeepsTaskRecoverable(t *testing.T) { - for _, tc := range []struct { - name string - retryCount int - wantStatus model.TaskStatus - wantRetries int - }{ - {name: "requeue", retryCount: 0, wantStatus: model.TaskStatusScheduled, wantRetries: 1}, - {name: "exhausted", retryCount: 4, wantStatus: model.TaskStatusExhausted, wantRetries: 5}, - } { - t.Run(tc.name, func(t *testing.T) { - mc := &testutil.MockCache{ - DeleteFunc: func(_ context.Context, _, _ string) error { - return errors.New("permission denied") - }, - GetFunc: func(_ context.Context, _, _ string) (io.ReadCloser, *cache.ObjectInfo, error) { - return nil, nil, errors.New("not needed") - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - _, objID, versionID := seedStoredObject(t, env) - - task := seedTask(t, env, model.TaskTypeEvictCache, objID, versionID, 5, tc.retryCount) - - evictor := newAfterUploadEvictor(env, 1, 50*time.Millisecond) - if tc.wantStatus == model.TaskStatusScheduled { - runWorkerUntilTaskRetryCount(t, env, evictor, task.ID, tc.wantRetries, 5*time.Second) - } else { - runWorkerUntilTask(t, env, evictor, task.ID, 5*time.Second) - } - - got, err := env.repos.Tasks.GetByID(context.Background(), task.ID) - if err != nil || got == nil { - t.Fatalf("get task after cache delete failure: task=%v err=%v", got, err) - } - if got.Status != tc.wantStatus { - t.Errorf("expected task %s after cache delete failure, got %s", tc.wantStatus, got.Status) - } - if got.RetryCount != tc.wantRetries { - t.Errorf("expected retry_count=%d, got %d", tc.wantRetries, got.RetryCount) - } - - obj, err := env.repos.Objects.GetCurrentVersionByObjectID(context.Background(), objID) - if err != nil || obj == nil { - t.Fatalf("get object after cache delete failure: object=%v err=%v", obj, err) - } - if obj.State != model.ObjectStateStored { - t.Errorf("expected object state stored after cache delete failure, got %s", obj.State) - } - if !obj.InCache { - t.Error("expected object cache location to remain true after cache delete failure") - } - }) - } -} - -func TestEvictor_DeletionAuthorizationSurvivesStateWriteFailure(t *testing.T) { - var deleteCalls atomic.Int64 - mc := &testutil.MockCache{DeleteFunc: func(context.Context, string, string) error { - deleteCalls.Add(1) - return nil - }} - env := newTestWorkerEnvWithMockCache(t, mc) - ctx := context.Background() - _, objID, versionID := seedStoredObject(t, env) - task := seedTask(t, env, model.TaskTypeEvictCache, objID, versionID, 5, 0) - records := &failFirstDeletionRecordRepo{CacheEvictionRepository: env.repos.CacheEvictions} - env.repos.CacheEvictions = records - evictor := newAfterUploadEvictor(env, 1, 10*time.Millisecond) - - runWorkerUntilTaskRetryCount(t, env, evictor, task.ID, 1, 5*time.Second) - interrupted, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || interrupted == nil { - t.Fatalf("GetByID after interrupted record: task=%#v err=%v", interrupted, err) - } - authorized, err := cacheeviction.DeleteAuthorized(interrupted) - if err != nil || !authorized || interrupted.RefVersionID != versionID { - t.Fatalf("interrupted authorization = task:%#v authorized:%t err:%v", interrupted, authorized, err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil || version.State != model.ObjectStateStored || !version.InCache { - t.Fatalf("version before state convergence = %#v err=%v", version, err) - } - if _, err := env.db.NewUpdate(). - Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - t.Fatalf("make authorized retry ready: %v", err) - } - - completed := runWorkerUntilTask(t, env, evictor, task.ID, 5*time.Second) - if completed.Status != model.TaskStatusCompleted || completed.RetryCount != 1 { - t.Fatalf("completed authorized retry = %#v", completed) - } - version, err = env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil || version.State != model.ObjectStateCacheEvicted || version.InCache { - t.Fatalf("version after state convergence = %#v err=%v", version, err) - } - if deleteCalls.Load() != 2 || records.calls.Load() != 2 { - t.Fatalf("recovery calls = delete:%d record:%d, want 2/2", deleteCalls.Load(), records.calls.Load()) - } -} - -type failFirstDeletionRecordRepo struct { - repository.CacheEvictionRepository - calls atomic.Int64 -} - -func (r *failFirstDeletionRecordRepo) RecordAuthorizedDeletion(ctx context.Context, task *model.Task) error { - if r.calls.Add(1) == 1 { - return errors.New("injected state write failure") - } - return r.CacheEvictionRepository.RecordAuthorizedDeletion(ctx, task) -} diff --git a/internal/worker/evictor_after_upload_test.go b/internal/worker/evictor_after_upload_test.go deleted file mode 100644 index ce2dbbd..0000000 --- a/internal/worker/evictor_after_upload_test.go +++ /dev/null @@ -1,201 +0,0 @@ -package worker_test - -import ( - "context" - "io" - "log/slog" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/cacheeviction" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/testutil" - "github.com/strahe/synaps3/internal/worker" -) - -// seedStoredObject creates a bucket+object version in stored state with an accepted upload. -func seedStoredObject(t *testing.T, env *testWorkerEnv) (*model.Bucket, int64, string) { - t.Helper() - ctx := context.Background() - - versionID := model.NewVersionID() - bucket := &model.Bucket{Name: "b-" + strings.ToLower(versionID), Status: model.BucketStatusActive} - if err := env.repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("creating bucket: %v", err) - } - - version := &model.ObjectVersion{ - VersionID: versionID, - BucketID: bucket.ID, - Key: "hello-" + versionID + ".txt", - Size: 11, - ETag: "etag-" + versionID, - Checksum: "sha256-" + versionID, - ContentType: "text/plain", - CacheKey: ".versions/" + versionID, - } - objID, err := env.repos.Objects.CreateVersionAndSetCurrent(ctx, version) - if err != nil { - t.Fatalf("creating object version: %v", err) - } - - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("transition to uploading: %v", err) - } - pieceCID := testCID(t).String() - acceptWorkerVersionUpload(t, env, versionID, pieceCID, "https://provider.example/pieces/1") - return bucket, objID, versionID -} - -func newAfterUploadEvictor( - env *testWorkerEnv, - concurrency int, - pollInterval time.Duration, -) *worker.Evictor { - return worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - concurrency, - pollInterval, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyAfterUpload, 0, 90, 80, 3), - ) -} - -func futureLRUAccessTime() time.Time { - return time.Now().UTC().Add(time.Hour).Truncate(time.Microsecond) -} - -func TestEvictor_HappyPath(t *testing.T) { - mc := &testutil.MockCache{ - DeleteFunc: func(_ context.Context, _, _ string) error { - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - _, objID, versionID := seedStoredObject(t, env) - - task := seedTask(t, env, model.TaskTypeEvictCache, objID, versionID, 5, 0) - - evictor := newAfterUploadEvictor(env, 1, 50*time.Millisecond) - runWorkerUntilTask(t, env, evictor, task.ID, 5*time.Second) - - ctx := context.Background() - - got, _ := env.repos.Tasks.GetByID(ctx, task.ID) - if got.Status != model.TaskStatusCompleted { - t.Errorf("expected task completed, got %s", got.Status) - } - - obj, _ := env.repos.Objects.GetCurrentVersionByObjectID(ctx, objID) - if obj.State != model.ObjectStateCacheEvicted { - t.Errorf("expected object state cache_evicted, got %s", obj.State) - } - if obj.InCache { - t.Error("expected object cache location to be false after successful eviction") - } -} - -type observableDeletionAuthorizationRepo struct { - repository.CacheEvictionRepository - - calls atomic.Int64 - started chan struct{} -} - -func (r *observableDeletionAuthorizationRepo) AuthorizeDeletion( - ctx context.Context, - task *model.Task, - expectedAccess *time.Time, -) (*cacheeviction.AuthorizedDeletion, error) { - if r.calls.Add(1) == 1 { - close(r.started) - } - return r.CacheEvictionRepository.AuthorizeDeletion(ctx, task, expectedAccess) -} - -func TestEvictor_ChecksRemoteSafetyAfterWaitingForOpenBody(t *testing.T) { - var deleteCalls atomic.Int64 - mc := &testutil.MockCache{ - ExistsFunc: func(context.Context, string, string) bool { return true }, - DeleteFunc: func(context.Context, string, string) error { - deleteCalls.Add(1) - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - _, objectID, versionID := seedStoredObject(t, env) - version, err := env.repos.Objects.GetVersionByID(context.Background(), versionID) - if err != nil || version == nil || version.StorageUploadID == nil { - t.Fatalf("GetVersionByID: version=%#v err=%v", version, err) - } - copies, err := env.repos.Uploads.ListCopies(context.Background(), *version.StorageUploadID) - if err != nil || len(copies) == 0 || copies[0].StorageDataSetID == nil { - t.Fatalf("ListCopies: copies=%#v err=%v", copies, err) - } - dataSetID := *copies[0].StorageDataSetID - - checks := &observableDeletionAuthorizationRepo{ - CacheEvictionRepository: env.repos.CacheEvictions, - started: make(chan struct{}), - } - env.repos.CacheEvictions = checks - opened, err := env.cacheGate.Open( - versionID, - func() (io.ReadCloser, *cache.ObjectInfo, error) { - return io.NopCloser(strings.NewReader("cached")), &cache.ObjectInfo{Size: version.Size}, nil - }, - ) - if err != nil { - t.Fatalf("cache gate Open: %v", err) - } - task := seedTask(t, env, model.TaskTypeEvictCache, objectID, versionID, 5, 0) - evictor := newAfterUploadEvictor(env, 1, 10*time.Millisecond) - - runCtx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = evictor.Run(runCtx) - close(done) - }() - defer func() { - cancel() - _ = opened.Body.Close() - waitForSignal(t, done, time.Second, "remote-safety eviction shutdown") - }() - - waitForTaskStatus(t, env, task.ID, model.TaskStatusRunning, 3*time.Second) - select { - case <-checks.started: - t.Fatal("remote safety was checked before the open cache body released the deletion gate") - case <-time.After(50 * time.Millisecond): - } - if err := env.repos.Uploads.MarkDataSetUnavailable( - context.Background(), - dataSetID, - "provider unavailable", - ); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - if err := opened.Body.Close(); err != nil { - t.Fatalf("close cache body: %v", err) - } - waitForSignal(t, checks.started, time.Second, "final remote-safety check") - waitForTaskStatus(t, env, task.ID, model.TaskStatusWaiting, 3*time.Second) - - if deleteCalls.Load() != 0 { - t.Fatalf("cache Delete calls after remote safety changed = %d, want 0", deleteCalls.Load()) - } - gotVersion, err := env.repos.Objects.GetVersionByID(context.Background(), versionID) - if err != nil || gotVersion == nil || !gotVersion.InCache || - gotVersion.State != model.ObjectStateStored { - t.Fatalf("version after remote safety changed = %#v err=%v", gotVersion, err) - } -} diff --git a/internal/worker/evictor_concurrency_test.go b/internal/worker/evictor_concurrency_test.go deleted file mode 100644 index e730726..0000000 --- a/internal/worker/evictor_concurrency_test.go +++ /dev/null @@ -1,281 +0,0 @@ -package worker_test - -import ( - "context" - "strings" - "sync" - "testing" - "time" - - "github.com/strahe/synaps3/internal/cacheeviction" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/testutil" -) - -func seedLRUEvictionTask( - t *testing.T, - env *testWorkerEnv, - objectID int64, - versionID string, - accessedAt time.Time, -) *model.Task { - t.Helper() - task := cacheeviction.NewLRUTask(cacheeviction.Candidate{ - ObjectID: objectID, - VersionID: versionID, - AccessedAt: accessedAt, - }, 3, time.Now()) - if err := env.repos.Tasks.Create(context.Background(), task); err != nil { - t.Fatalf("Create LRU eviction task: %v", err) - } - return task -} - -func TestEvictor_ClaimsLaterPendingTaskWhileAnotherEvictionRuns(t *testing.T) { - firstDeleteEntered := make(chan struct{}) - releaseFirstDelete := make(chan struct{}) - var enterOnce sync.Once - var releaseOnce sync.Once - - mc := &testutil.MockCache{} - env := newTestWorkerEnvWithMockCache(t, mc) - _, firstObjID, firstVersionID := seedStoredObject(t, env) - firstTask := seedTask(t, env, model.TaskTypeEvictCache, firstObjID, firstVersionID, 5, 0) - firstCacheKey := ".versions/" + firstVersionID - _, secondObjID, secondVersionID := seedStoredObject(t, env) - - mc.DeleteFunc = func(ctx context.Context, _, key string) error { - if key == firstCacheKey { - enterOnce.Do(func() { close(firstDeleteEntered) }) - select { - case <-releaseFirstDelete: - case <-ctx.Done(): - return ctx.Err() - } - } - return nil - } - - evictor := newAfterUploadEvictor(env, 2, 20*time.Millisecond) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = evictor.Run(ctx) - close(done) - }() - defer func() { - releaseOnce.Do(func() { close(releaseFirstDelete) }) - cancel() - waitForSignal(t, done, time.Second, "evictor shutdown") - }() - - waitForSignal(t, firstDeleteEntered, time.Second, "first eviction delete to start") - - secondTask := seedTask(t, env, model.TaskTypeEvictCache, secondObjID, secondVersionID, 5, 0) - - waitForTaskStatus(t, env, secondTask.ID, model.TaskStatusCompleted, 500*time.Millisecond) - - got, err := env.repos.Tasks.GetByID(context.Background(), firstTask.ID) - if err != nil { - t.Fatalf("get first task: %v", err) - } - if got.Status != model.TaskStatusRunning { - t.Fatalf("first task status = %s, want running while second task completed", got.Status) - } -} - -func TestEvictor_HealthyWhileEvictionTaskIsActive(t *testing.T) { - deleteEntered := make(chan struct{}) - releaseDelete := make(chan struct{}) - var enterOnce sync.Once - var releaseOnce sync.Once - pollInterval := 20 * time.Millisecond - - mc := &testutil.MockCache{ - DeleteFunc: func(ctx context.Context, _, _ string) error { - enterOnce.Do(func() { close(deleteEntered) }) - select { - case <-releaseDelete: - case <-ctx.Done(): - return ctx.Err() - } - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - _, objID, versionID := seedStoredObject(t, env) - _ = seedTask(t, env, model.TaskTypeEvictCache, objID, versionID, 5, 0) - - evictor := newAfterUploadEvictor(env, 1, pollInterval) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = evictor.Run(ctx) - close(done) - }() - defer func() { - releaseOnce.Do(func() { close(releaseDelete) }) - cancel() - waitForSignal(t, done, time.Second, "evictor shutdown") - }() - - waitForSignal(t, deleteEntered, time.Second, "eviction delete to start") - time.Sleep(4 * pollInterval) - - if !evictor.Healthy() { - t.Fatal("evictor should remain healthy while eviction task is active") - } -} - -func TestEvictor_ReplicatingVersionDefersEvictionAndKeepsCache(t *testing.T) { - deleteCalled := false - mc := &testutil.MockCache{ - DeleteFunc: func(_ context.Context, _, _ string) error { - deleteCalled = true - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - bucket, objID, versionID := seedObjectInDB(t, env, model.BucketStatusActive) - ctx := context.Background() - - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("committing: %v", err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - primary, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding primary: %v", err) - } - secondary, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "202"), - CopyIndex: 1, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding secondary: %v", err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: primary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}); err != nil { - t.Fatalf("MarkDataSetReady primary: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - {StorageDataSetID: secondary.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: testCID(t).String(), - PieceID: onChainIDPtr(t, "3001"), - RetrievalURL: "https://primary.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted primary: %v", err) - } - if _, err := env.repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: version.Size, - Checksum: version.Checksum, - }); err != nil { - t.Fatalf("BindReadableUploadForContent: %v", err) - } - - task := seedTask(t, env, model.TaskTypeEvictCache, objID, versionID, 5, 0) - originalScheduledAt := task.ScheduledAt - evictor := newAfterUploadEvictor(env, 1, 50*time.Millisecond) - - runCtx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = evictor.Run(runCtx) - close(done) - }() - defer func() { - cancel() - waitForSignal(t, done, time.Second, "evictor shutdown") - }() - - deadline := time.After(3 * time.Second) - ticker := time.NewTicker(20 * time.Millisecond) - defer ticker.Stop() - var gotTask *model.Task - for gotTask == nil { - select { - case <-deadline: - t.Fatal("timed out waiting for replicating evict task to be deferred") - case <-ticker.C: - current, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || current == nil { - continue - } - if current.Status != model.TaskStatusQueued && current.Status != model.TaskStatusWaiting { - if current.Status != model.TaskStatusRunning { - gotTask = current - } - continue - } - if current.StatusMessage != nil && strings.Contains(*current.StatusMessage, "enough durable replicas") { - gotTask = current - } - } - } - if gotTask.Status != model.TaskStatusWaiting { - t.Fatalf("task status = %s, want waiting deferred task", gotTask.Status) - } - if gotTask.RetryCount != 0 { - t.Fatalf("task retry_count = %d, want 0", gotTask.RetryCount) - } - if gotTask.LastError != nil { - t.Fatalf("task last_error = %v, want nil", gotTask.LastError) - } - if gotTask.WaitReason == nil || *gotTask.WaitReason != model.TaskWaitReasonDependency { - t.Fatalf("task wait_reason = %v, want dependency", gotTask.WaitReason) - } - if gotTask.StatusMessage == nil || !strings.Contains(*gotTask.StatusMessage, "enough durable replicas") { - t.Fatalf("task status_message = %v, want waiting-for-copies reason", gotTask.StatusMessage) - } - if !gotTask.ScheduledAt.After(originalScheduledAt) { - t.Fatalf("task scheduled_at = %s, want after %s", gotTask.ScheduledAt, originalScheduledAt) - } - if gotTask.ScheduledAt.Before(originalScheduledAt.Add(20 * time.Second)) { - t.Fatalf("task scheduled_at = %s, want a longer defer while replication is still running", gotTask.ScheduledAt) - } - gotVersion, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || gotVersion == nil { - t.Fatalf("GetVersionByID after evict: version=%v err=%v", gotVersion, err) - } - if gotVersion.State != model.ObjectStateReplicating { - t.Fatalf("version state = %s, want replicating", gotVersion.State) - } - if !gotVersion.InCache { - t.Fatal("version in_cache = false, want cache retained while replicating") - } - if deleteCalled { - t.Fatalf("cache delete was called for replicating version %s in bucket %s", version.CacheKey, bucket.Name) - } -} diff --git a/internal/worker/evictor_finalize.go b/internal/worker/evictor_finalize.go deleted file mode 100644 index 9352448..0000000 --- a/internal/worker/evictor_finalize.go +++ /dev/null @@ -1,133 +0,0 @@ -package worker - -import ( - "context" - "fmt" - - "github.com/strahe/synaps3/internal/cacheeviction" - "github.com/strahe/synaps3/internal/model" -) - -type evictionAction uint8 - -const ( - evictionComplete evictionAction = iota + 1 - evictionCancel - evictionRetry - evictionFail - evictionWait -) - -type evictionDecision struct { - action evictionAction - reason string - logMessage string - cause error -} - -func (e *Evictor) deleteCacheEntry( - ctx context.Context, - task *model.Task, - deletion *cacheeviction.AuthorizedDeletion, -) *evictionDecision { - if deletion == nil { - return failEviction("cache deletion was not authorized", "cache eviction has no authorized target") - } - if deletion.Version.InCache { - if err := e.cache.Delete(ctx, deletion.BucketName, deletion.Version.CacheKey); err != nil { - return retryEviction(err, "deleting cache entry") - } - } - return e.recordCacheEntryDeleted(ctx, task, &deletion.Version) -} - -func (e *Evictor) deleteCoordinatorCacheEntry( - ctx context.Context, - task *model.Task, - deletion *cacheeviction.AuthorizedDeletion, -) error { - if deletion == nil { - return cacheeviction.ErrNoLongerEligible - } - if deletion.Version.InCache { - if err := e.cache.Delete(ctx, deletion.BucketName, deletion.Version.CacheKey); err != nil { - return err - } - } - e.cacheAccessTracker.Forget(deletion.Version.VersionID) - if err := e.repos.CacheEvictions.RecordAuthorizedDeletion(ctx, task); err != nil { - return fmt.Errorf("recording cache eviction state: %w", err) - } - return nil -} - -func (e *Evictor) recordCacheEntryDeleted( - ctx context.Context, - task *model.Task, - version *model.ObjectVersion, -) *evictionDecision { - e.cacheAccessTracker.Forget(version.VersionID) - if err := e.repos.CacheEvictions.RecordAuthorizedDeletion(ctx, task); err != nil { - return retryEviction(err, "recording cache eviction state") - } - return completeEviction() -} - -func (e *Evictor) applyEvictionDecision( - ctx context.Context, - task *model.Task, - decision *evictionDecision, -) { - if decision == nil { - return - } - switch decision.action { - case evictionComplete: - e.completeTask(ctx, task, e.taskLogger(task), "cache eviction completed") - case evictionCancel: - e.cancelTask(ctx, task, decision.reason) - case evictionRetry: - e.retryTask(ctx, task, decision.cause, decision.logMessage) - case evictionFail: - e.failTask(ctx, task, decision.reason, decision.logMessage) - case evictionWait: - e.deferReplicatingEviction(ctx, task) - default: - e.failTask(ctx, task, "invalid cache eviction decision", "cache eviction produced an invalid decision") - } -} - -func taskStage(task *model.Task) string { - if task == nil || task.Stage == nil { - return "" - } - return *task.Stage -} - -func completeEviction() *evictionDecision { - return &evictionDecision{action: evictionComplete} -} - -func cancelEviction(reason string) *evictionDecision { - return &evictionDecision{action: evictionCancel, reason: reason} -} - -func retryEviction(err error, logMessage string) *evictionDecision { - return &evictionDecision{ - action: evictionRetry, - cause: err, - logMessage: logMessage, - } -} - -func failEviction(reason, logMessage string) *evictionDecision { - return &evictionDecision{ - action: evictionFail, - reason: reason, - logMessage: logMessage, - } -} - -func waitForEvictionDependency() *evictionDecision { - return &evictionDecision{action: evictionWait} -} diff --git a/internal/worker/evictor_lru.go b/internal/worker/evictor_lru.go deleted file mode 100644 index ebec026..0000000 --- a/internal/worker/evictor_lru.go +++ /dev/null @@ -1,260 +0,0 @@ -package worker - -import ( - "context" - "errors" - "fmt" - "time" - - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/cacheeviction" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" -) - -func (e *Evictor) processLRUEviction( - ctx context.Context, - task *model.Task, -) *evictionDecision { - authorized, err := cacheeviction.DeleteAuthorized(task) - if err != nil { - return cancelEviction("LRU cache deletion authorization is invalid") - } - if authorized { - var decision *evictionDecision - e.cacheGate.GuardDeletion(task.RefVersionID, func() { - deletion, authorizeErr := e.repos.CacheEvictions.AuthorizeDeletion(ctx, task, nil) - switch { - case authorizeErr == nil: - decision = e.deleteCacheEntry(ctx, task, deletion) - case errors.Is(authorizeErr, repository.ErrNotFound), errors.Is(authorizeErr, cacheeviction.ErrNoLongerEligible): - decision = cancelEviction("Authorized cache entry no longer exists") - default: - decision = retryEviction(authorizeErr, "resuming authorized LRU cache eviction") - } - }) - return decision - } - if e.policy != cache.EvictionPolicyLRU { - return cancelEviction("Cache eviction policy no longer uses LRU") - } - if !e.lruAccessTrackingSafe() { - return cancelEviction("LRU eviction is paused because recent cache access could not be retained") - } - if e.cache.UsedBytes() <= watermarkBytes(e.maxCacheBytes, e.lowWatermarkPercent) { - return cancelEviction("LRU cache usage already reached the low watermark") - } - payload, err := cacheeviction.ParseLRUTaskPayload(task) - if err != nil { - return cancelEviction("LRU eviction plan is no longer valid and can be replanned") - } - - var decision *evictionDecision - e.cacheGate.GuardDeletion(task.RefVersionID, func() { - decision = e.finalizeLRUEviction(ctx, task, payload.AccessedAt) - }) - return decision -} - -func (e *Evictor) finalizeLRUEviction( - ctx context.Context, - task *model.Task, - accessSnapshot time.Time, -) *evictionDecision { - if !e.lruAccessTrackingSafe() { - return cancelEviction("LRU eviction is paused because recent cache access could not be retained") - } - - version, err := e.repos.Objects.GetVersionByID(ctx, task.RefVersionID) - if err != nil { - return retryEviction(err, "loading object version for LRU cache eviction") - } - if version == nil { - return cancelEviction("Object version no longer exists") - } - if !version.InCache { - return cancelEviction("Object is no longer present in the local cache") - } - if version.State != model.ObjectStateStored && - version.State != model.ObjectStateCacheEvicted { - return cancelEviction("Object is no longer eligible for LRU eviction") - } - - durableAccess := effectiveLRUAccessTime(version) - inMemoryAccess := e.cacheAccessTracker.Latest(task.RefVersionID) - if !cacheeviction.NormalizeAccessTime(durableAccess).Equal(accessSnapshot) || - cacheeviction.NormalizeAccessTime(inMemoryAccess).After(accessSnapshot) { - if newerInMemoryAccess(version.CacheAccessedAt, inMemoryAccess).IsZero() { - return cancelEviction("Object was accessed after this LRU eviction was planned") - } - if err := e.cacheAccessTracker.FlushWhileGuarded(ctx, task.RefVersionID); err != nil { - e.taskLogger(task).Warn( - "persisting recent cache access before cancelling LRU eviction", - "error", - err, - ) - } - return cancelEviction("Object was accessed after this LRU eviction was planned") - } - - if !e.reserveLRUDeletion(version.Size) { - return cancelEviction("LRU cache usage already reached the low watermark") - } - deletion, err := e.repos.CacheEvictions.AuthorizeDeletion(ctx, task, &accessSnapshot) - if err != nil { - e.finishLRUDeletion(version.Size, false) - switch { - case errors.Is(err, cacheeviction.ErrDurabilityThreshold), - errors.Is(err, cacheeviction.ErrNoLongerEligible), - errors.Is(err, cacheeviction.ErrAccessChanged): - return cancelEviction("Object is no longer eligible for LRU cache eviction") - default: - return retryEviction(err, "authorizing LRU cache eviction") - } - } - if deletion.Version.InCache { - err = e.cache.Delete(ctx, deletion.BucketName, deletion.Version.CacheKey) - } - e.finishLRUDeletion(version.Size, err == nil) - if err != nil { - return retryEviction(err, "deleting cache entry") - } - return e.recordCacheEntryDeleted(ctx, task, &deletion.Version) -} - -func effectiveLRUAccessTime(version *model.ObjectVersion) time.Time { - if version == nil { - return time.Time{} - } - if version.CacheAccessedAt != nil { - return cacheeviction.NormalizeAccessTime(*version.CacheAccessedAt) - } - return cacheeviction.NormalizeAccessTime(version.CreatedAt) -} - -func newerInMemoryAccess(durable *time.Time, inMemory time.Time) time.Time { - inMemory = cacheeviction.NormalizeAccessTime(inMemory) - if inMemory.IsZero() { - return time.Time{} - } - if durable != nil && - !inMemory.After(cacheeviction.NormalizeAccessTime(*durable)) { - return time.Time{} - } - return inMemory -} - -func (e *Evictor) runLRUPlanner(ctx context.Context) { - for { - if ctx.Err() != nil { - return - } - e.recordTick() - if err := e.planLRUEvictions(ctx); err != nil && ctx.Err() == nil { - e.logger.Error("planning LRU cache evictions", "error", err) - } - if !sleepUntilNextWorkerPoll(ctx, e.pollInterval) { - return - } - } -} - -func (e *Evictor) planLRUEvictions(ctx context.Context) error { - if e.policy != cache.EvictionPolicyLRU || e.maxCacheBytes <= 0 { - return nil - } - if !e.lruAccessTrackingSafe() { - return nil - } - usedBytes := e.cache.UsedBytes() - highBytes := watermarkBytes(e.maxCacheBytes, e.highWatermarkPercent) - lowBytes := watermarkBytes(e.maxCacheBytes, e.lowWatermarkPercent) - if !e.shouldPlanLRUCycle(usedBytes, highBytes, lowBytes) { - return nil - } - activeBytes, err := e.repos.CacheEvictions.ActiveLRUBytes(ctx) - if err != nil { - return err - } - bytesToPlan := usedBytes - lowBytes - activeBytes - if bytesToPlan <= 0 { - return nil - } - - var plannedBytes int64 - plannedTasks := 0 - for bytesToPlan > 0 { - createdThisBatch := 0 - terminalSince := time.Now().Add(-lruTerminalRetryDelay) - candidates, err := e.repos.CacheEvictions.ListLRUCandidates( - ctx, - terminalSince, - lruCandidateBatchSize, - ) - if err != nil { - return err - } - if len(candidates) == 0 { - break - } - for _, candidate := range candidates { - activated, err := e.repos.CacheEvictions.PlanLRU( - ctx, - candidate, - e.maxRetries, - terminalSince, - ) - if err != nil { - return fmt.Errorf("creating LRU eviction task for version %s: %w", candidate.VersionID, err) - } - if !activated { - continue - } - plannedTasks++ - createdThisBatch++ - plannedBytes += candidate.Size - bytesToPlan -= candidate.Size - if bytesToPlan <= 0 { - break - } - } - if len(candidates) < lruCandidateBatchSize || createdThisBatch == 0 { - break - } - } - - if plannedTasks == 0 { - e.logger.Warn( - "LRU cache cleanup is active but no remotely safe entries can be evicted", - "usedBytes", - usedBytes, - "highBytes", - highBytes, - "lowBytes", - lowBytes, - ) - return nil - } - e.logger.Info( - "planned LRU cache evictions", - "tasks", - plannedTasks, - "plannedBytes", - plannedBytes, - "usedBytes", - usedBytes, - "targetBytes", - lowBytes, - ) - return nil -} - -func watermarkBytes(maxBytes int64, percent int) int64 { - if maxBytes <= 0 || percent <= 0 { - return 0 - } - if percent >= 100 { - return maxBytes - } - return (maxBytes/100)*int64(percent) + (maxBytes%100)*int64(percent)/100 -} diff --git a/internal/worker/evictor_lru_capacity_test.go b/internal/worker/evictor_lru_capacity_test.go deleted file mode 100644 index 77b83fb..0000000 --- a/internal/worker/evictor_lru_capacity_test.go +++ /dev/null @@ -1,146 +0,0 @@ -package worker_test - -import ( - "context" - "io" - "log/slog" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/testutil" - "github.com/strahe/synaps3/internal/worker" -) - -type blockingDeletionRecordRepo struct { - repository.CacheEvictionRepository - versionID string - entered chan struct{} - release <-chan struct{} - once sync.Once -} - -func (r *blockingDeletionRecordRepo) RecordAuthorizedDeletion( - ctx context.Context, - task *model.Task, -) error { - if task.RefVersionID == r.versionID { - r.once.Do(func() { close(r.entered) }) - select { - case <-r.release: - case <-ctx.Done(): - return ctx.Err() - } - } - return r.CacheEvictionRepository.RecordAuthorizedDeletion(ctx, task) -} - -func TestEvictor_LRUCapacityReservationEndsAtPhysicalDelete(t *testing.T) { - var used atomic.Int64 - used.Store(33) - var secondDeleteOnce sync.Once - secondDeleteEntered := make(chan struct{}) - - var firstCacheKey, secondCacheKey string - mc := &testutil.MockCache{ - UsedBytesFunc: used.Load, - ExistsFunc: func(_ context.Context, _, _ string) bool { - return true - }, - DeleteFunc: func(_ context.Context, _, key string) error { - switch key { - case firstCacheKey: - used.Add(-11) - case secondCacheKey: - used.Add(-11) - secondDeleteOnce.Do(func() { close(secondDeleteEntered) }) - } - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - _, firstObjectID, firstVersionID := seedStoredObject(t, env) - _, secondObjectID, secondVersionID := seedStoredObject(t, env) - firstCacheKey = ".versions/" + firstVersionID - secondCacheKey = ".versions/" + secondVersionID - - accessedAt := futureLRUAccessTime() - for _, versionID := range []string{firstVersionID, secondVersionID} { - if err := env.repos.Objects.RecordVersionCacheAccess( - context.Background(), - versionID, - accessedAt, - ); err != nil { - t.Fatalf("RecordVersionCacheAccess(%s): %v", versionID, err) - } - } - firstTask := seedLRUEvictionTask(t, env, firstObjectID, firstVersionID, accessedAt) - secondTask := seedLRUEvictionTask(t, env, secondObjectID, secondVersionID, accessedAt) - - stateUpdateEntered := make(chan struct{}) - releaseStateUpdate := make(chan struct{}) - var releaseStateOnce sync.Once - env.repos.CacheEvictions = &blockingDeletionRecordRepo{ - CacheEvictionRepository: env.repos.CacheEvictions, - versionID: firstVersionID, - entered: stateUpdateEntered, - release: releaseStateUpdate, - } - - heldSecond, err := env.cacheGate.Open( - secondVersionID, - func() (io.ReadCloser, *cache.ObjectInfo, error) { - return io.NopCloser(strings.NewReader("cached")), &cache.ObjectInfo{Size: 11}, nil - }, - ) - if err != nil { - t.Fatalf("hold second cache entry open: %v", err) - } - - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 2, - 10*time.Millisecond, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 30, 90, 60, 3), - ) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = evictor.Run(ctx) - close(done) - }() - defer func() { - _ = heldSecond.Body.Close() - releaseStateOnce.Do(func() { close(releaseStateUpdate) }) - cancel() - waitForSignal(t, done, time.Second, "LRU capacity reservation test shutdown") - }() - - waitForSignal(t, stateUpdateEntered, time.Second, "first cache file deletion") - if err := heldSecond.Body.Close(); err != nil { - t.Fatalf("release second cache entry: %v", err) - } - waitForSignal( - t, - secondDeleteEntered, - time.Second, - "second cache file deletion while first database transition is pending", - ) - - releaseStateOnce.Do(func() { close(releaseStateUpdate) }) - waitForTaskStatus(t, env, firstTask.ID, model.TaskStatusCompleted, time.Second) - waitForTaskStatus(t, env, secondTask.ID, model.TaskStatusCompleted, time.Second) - if got := used.Load(); got != 11 { - t.Fatalf("cache used bytes after concurrent LRU deletions = %d, want 11", got) - } -} diff --git a/internal/worker/evictor_lru_finalize_test.go b/internal/worker/evictor_lru_finalize_test.go deleted file mode 100644 index 9d63924..0000000 --- a/internal/worker/evictor_lru_finalize_test.go +++ /dev/null @@ -1,739 +0,0 @@ -package worker_test - -import ( - "context" - "errors" - "io" - "log/slog" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/cacheaccess" - "github.com/strahe/synaps3/internal/cacheeviction" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/objectreader" - "github.com/strahe/synaps3/internal/testutil" - "github.com/strahe/synaps3/internal/worker" -) - -func TestEvictor_LRUAccessAfterPlanningCancelsStaleTask(t *testing.T) { - var deleteCalls atomic.Int64 - mc := &testutil.MockCache{ - UsedBytesFunc: func() int64 { return 11 }, - ExistsFunc: func(_ context.Context, _, _ string) bool { - return true - }, - DeleteFunc: func(_ context.Context, _, _ string) error { - deleteCalls.Add(1) - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - _, objectID, versionID := seedStoredObject(t, env) - plannedAt := futureLRUAccessTime() - if err := env.repos.Objects.RecordVersionCacheAccess(context.Background(), versionID, plannedAt); err != nil { - t.Fatalf("RecordVersionCacheAccess(planned): %v", err) - } - task := seedLRUEvictionTask(t, env, objectID, versionID, plannedAt) - if err := env.repos.Objects.RecordVersionCacheAccess(context.Background(), versionID, plannedAt.Add(time.Minute)); err != nil { - t.Fatalf("RecordVersionCacheAccess(after plan): %v", err) - } - - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 1, - 10*time.Millisecond, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 20, 90, 50, 3), - ) - got := runWorkerUntilTask(t, env, evictor, task.ID, 3*time.Second) - if got.Status != model.TaskStatusCancelled { - t.Fatalf("stale LRU task status = %s, want cancelled", got.Status) - } - if deleteCalls.Load() != 0 { - t.Fatalf("cache Delete calls = %d, want 0 for stale LRU task", deleteCalls.Load()) - } - version, err := env.repos.Objects.GetVersionByID(context.Background(), versionID) - if err != nil || version == nil || !version.InCache || version.State != model.ObjectStateStored { - t.Fatalf("version after stale LRU task = %#v err=%v", version, err) - } -} - -func TestEvictor_LRUTimestampPrecisionMatchesDatabaseValue(t *testing.T) { - var deleteCalls atomic.Int64 - mc := &testutil.MockCache{ - UsedBytesFunc: func() int64 { return 11 }, - ExistsFunc: func(_ context.Context, _, _ string) bool { - return true - }, - DeleteFunc: func(_ context.Context, _, _ string) error { - deleteCalls.Add(1) - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - _, objectID, versionID := seedStoredObject(t, env) - plannedAt := futureLRUAccessTime().Add(789 * time.Nanosecond) - if err := env.repos.Objects.RecordVersionCacheAccess(context.Background(), versionID, plannedAt); err != nil { - t.Fatalf("RecordVersionCacheAccess(planned): %v", err) - } - task := seedLRUEvictionTask(t, env, objectID, versionID, plannedAt) - - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 1, - 10*time.Millisecond, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 20, 90, 50, 3), - ) - got := runWorkerUntilTask(t, env, evictor, task.ID, 3*time.Second) - if got.Status != model.TaskStatusCompleted { - t.Fatalf("precision-normalized LRU task status = %s, want completed", got.Status) - } - if deleteCalls.Load() != 1 { - t.Fatalf("cache Delete calls = %d, want 1", deleteCalls.Load()) - } -} - -func TestEvictor_LRUWaitsForOpenBodyAndPreservesRecentAccess(t *testing.T) { - var used atomic.Int64 - used.Store(11) - var deleteCalls atomic.Int64 - mc := &testutil.MockCache{ - UsedBytesFunc: used.Load, - GetFunc: func(_ context.Context, _, _ string) (io.ReadCloser, *cache.ObjectInfo, error) { - return io.NopCloser(strings.NewReader("cached")), &cache.ObjectInfo{Size: 6}, nil - }, - ExistsFunc: func(_ context.Context, _, _ string) bool { - return true - }, - DeleteFunc: func(_ context.Context, _, _ string) error { - deleteCalls.Add(1) - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - bucket, objectID, versionID := seedStoredObject(t, env) - plannedAt := futureLRUAccessTime() - if err := env.repos.Objects.RecordVersionCacheAccess(context.Background(), versionID, plannedAt); err != nil { - t.Fatalf("RecordVersionCacheAccess(planned): %v", err) - } - task := seedLRUEvictionTask(t, env, objectID, versionID, plannedAt) - reader := objectreader.New( - env.repos, - env.cache, - nil, - env.cacheGate, - env.accessTracker, - slog.Default(), - ) - - opened, err := reader.Open( - context.Background(), - bucket.Name, - "hello-"+versionID+".txt", - objectreader.S3Visibility, - ) - if err != nil { - t.Fatalf("cache Open: %v", err) - } - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 1, - 10*time.Millisecond, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 20, 90, 50, 3), - ) - runCtx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = evictor.Run(runCtx) - close(done) - }() - defer func() { - cancel() - _ = opened.Body.Close() - waitForSignal(t, done, time.Second, "LRU race test shutdown") - }() - - waitForTaskStatus(t, env, task.ID, model.TaskStatusRunning, 3*time.Second) - if deleteCalls.Load() != 0 { - t.Fatalf("cache Delete calls while response body is open = %d, want 0", deleteCalls.Load()) - } - used.Store(0) - if err := opened.Body.Close(); err != nil { - t.Fatalf("close cache body: %v", err) - } - - waitForTaskStatus(t, env, task.ID, model.TaskStatusCancelled, 3*time.Second) - if deleteCalls.Load() != 0 { - t.Fatalf("cache Delete calls = %d, want 0 after protected access", deleteCalls.Load()) - } -} - -func TestEvictor_LRUChecksRemoteSafetyAfterWaitingForOpenBody(t *testing.T) { - var deleteCalls atomic.Int64 - mc := &testutil.MockCache{ - UsedBytesFunc: func() int64 { return 11 }, - DeleteFunc: func(context.Context, string, string) error { - deleteCalls.Add(1) - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - _, objectID, versionID := seedStoredObject(t, env) - version, err := env.repos.Objects.GetVersionByID(context.Background(), versionID) - if err != nil || version == nil || version.StorageUploadID == nil { - t.Fatalf("GetVersionByID: version=%#v err=%v", version, err) - } - copies, err := env.repos.Uploads.ListCopies(context.Background(), *version.StorageUploadID) - if err != nil || len(copies) == 0 || copies[0].StorageDataSetID == nil { - t.Fatalf("ListCopies: copies=%#v err=%v", copies, err) - } - dataSetID := *copies[0].StorageDataSetID - - plannedAt := futureLRUAccessTime() - if err := env.repos.Objects.RecordVersionCacheAccess( - context.Background(), - versionID, - plannedAt, - ); err != nil { - t.Fatalf("RecordVersionCacheAccess: %v", err) - } - task := seedLRUEvictionTask(t, env, objectID, versionID, plannedAt) - checks := &observableDeletionAuthorizationRepo{ - CacheEvictionRepository: env.repos.CacheEvictions, - started: make(chan struct{}), - } - env.repos.CacheEvictions = checks - - opened, err := env.cacheGate.Open( - versionID, - func() (io.ReadCloser, *cache.ObjectInfo, error) { - return io.NopCloser(strings.NewReader("cached")), &cache.ObjectInfo{Size: version.Size}, nil - }, - ) - if err != nil { - t.Fatalf("cache gate Open: %v", err) - } - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 1, - 10*time.Millisecond, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 20, 90, 50, 3), - ) - runCtx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = evictor.Run(runCtx) - close(done) - }() - defer func() { - cancel() - _ = opened.Body.Close() - waitForSignal(t, done, time.Second, "LRU remote-safety test shutdown") - }() - - waitForTaskStatus(t, env, task.ID, model.TaskStatusRunning, 3*time.Second) - select { - case <-checks.started: - t.Fatal("remote safety was checked before the open cache body released the deletion gate") - case <-time.After(50 * time.Millisecond): - } - if err := env.repos.Uploads.MarkDataSetUnavailable( - context.Background(), - dataSetID, - "provider unavailable", - ); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - if err := opened.Body.Close(); err != nil { - t.Fatalf("close cache body: %v", err) - } - waitForSignal(t, checks.started, time.Second, "LRU final remote-safety check") - waitForTaskStatus(t, env, task.ID, model.TaskStatusCancelled, 3*time.Second) - if deleteCalls.Load() != 0 { - t.Fatalf("cache Delete calls after remote safety changed = %d, want 0", deleteCalls.Load()) - } -} - -type cacheAccessRecordingObjectRepo struct { - repository.ObjectRepository - - accessWrites atomic.Int64 -} - -func (r *cacheAccessRecordingObjectRepo) RecordVersionCacheAccess( - ctx context.Context, - versionID string, - accessedAt time.Time, -) error { - r.accessWrites.Add(1) - return r.ObjectRepository.RecordVersionCacheAccess(ctx, versionID, accessedAt) -} - -func TestEvictor_LRUFlushesNewerMemoryAccessBeforeDatabaseStaleCancellation(t *testing.T) { - var deleteCalls atomic.Int64 - mc := &testutil.MockCache{ - UsedBytesFunc: func() int64 { return 11 }, - GetFunc: func(_ context.Context, _, _ string) (io.ReadCloser, *cache.ObjectInfo, error) { - return io.NopCloser(strings.NewReader("cached")), &cache.ObjectInfo{Size: 6}, nil - }, - ExistsFunc: func(_ context.Context, _, _ string) bool { - return true - }, - DeleteFunc: func(_ context.Context, _, _ string) error { - deleteCalls.Add(1) - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - bucket, objectID, versionID := seedStoredObject(t, env) - recordingRepo := &cacheAccessRecordingObjectRepo{ - ObjectRepository: env.repos.Objects, - } - env.repos.Objects = recordingRepo - cacheGate := cacheaccess.NewGate() - accessTracker := cacheaccess.NewTracker(cacheaccess.DefaultPersistenceInterval, recordingRepo) - reader := objectreader.New( - env.repos, - env.cache, - nil, - cacheGate, - accessTracker, - slog.Default(), - ) - - first, err := reader.Open( - context.Background(), - bucket.Name, - "hello-"+versionID+".txt", - objectreader.S3Visibility, - ) - if err != nil { - t.Fatalf("initial cache Open: %v", err) - } - _ = first.Body.Close() - plannedVersion, err := env.repos.Objects.GetVersionByID(context.Background(), versionID) - if err != nil || plannedVersion == nil || plannedVersion.CacheAccessedAt == nil { - t.Fatalf("version after initial cache access: version=%#v err=%v", plannedVersion, err) - } - task := seedLRUEvictionTask( - t, - env, - objectID, - versionID, - *plannedVersion.CacheAccessedAt, - ) - - durableAfterPlan := plannedVersion.CacheAccessedAt.Add(time.Minute) - if err := env.repos.Objects.RecordVersionCacheAccess( - context.Background(), - versionID, - durableAfterPlan, - ); err != nil { - t.Fatalf("RecordVersionCacheAccess(after plan): %v", err) - } - second, err := reader.Open( - context.Background(), - bucket.Name, - "hello-"+versionID+".txt", - objectreader.S3Visibility, - ) - if err != nil { - t.Fatalf("second cache Open: %v", err) - } - _ = second.Body.Close() - - beforeEviction, err := env.repos.Objects.GetVersionByID(context.Background(), versionID) - if err != nil || beforeEviction == nil || beforeEviction.CacheAccessedAt == nil { - t.Fatalf("version before eviction: version=%#v err=%v", beforeEviction, err) - } - if !beforeEviction.CacheAccessedAt.Equal(durableAfterPlan) { - t.Fatalf( - "coalesced cache access persisted early at %v, want durable time %v", - beforeEviction.CacheAccessedAt, - durableAfterPlan, - ) - } - - recordingRepo.accessWrites.Store(0) - evictor := worker.NewEvictor( - env.repos, - env.cache, - cacheGate, - accessTracker, - env.sm, - 1, - 10*time.Millisecond, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 20, 90, 50, 3), - ) - got := runWorkerUntilTask(t, env, evictor, task.ID, 3*time.Second) - if got.Status != model.TaskStatusCancelled { - t.Fatalf("stale LRU task status = %s, want cancelled", got.Status) - } - if deleteCalls.Load() != 0 { - t.Fatalf("cache Delete calls = %d, want 0 after newer cache access", deleteCalls.Load()) - } - if recordingRepo.accessWrites.Load() != 1 { - t.Fatalf( - "access writes before stale cancellation = %d, want one deferred flush", - recordingRepo.accessWrites.Load(), - ) - } - afterEviction, err := env.repos.Objects.GetVersionByID(context.Background(), versionID) - if err != nil || afterEviction == nil || afterEviction.CacheAccessedAt == nil { - t.Fatalf("version after stale cancellation: version=%#v err=%v", afterEviction, err) - } - if !afterEviction.CacheAccessedAt.After(durableAfterPlan) { - t.Fatalf( - "flushed cache access = %v, want after durable time %v", - afterEviction.CacheAccessedAt, - durableAfterPlan, - ) - } -} - -func TestEvictor_LRUCanEvictRehydratedCacheEvictedVersion(t *testing.T) { - var used atomic.Int64 - used.Store(11) - mc := &testutil.MockCache{ - UsedBytesFunc: used.Load, - ExistsFunc: func(_ context.Context, _, _ string) bool { - return true - }, - DeleteFunc: func(_ context.Context, _, _ string) error { - used.Store(0) - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - _, objectID, versionID := seedStoredObject(t, env) - previousAccess := futureLRUAccessTime() - if err := env.repos.Objects.RecordVersionCacheAccess(context.Background(), versionID, previousAccess); err != nil { - t.Fatalf("RecordVersionCacheAccess(previous): %v", err) - } - if err := env.repos.Objects.UpdateVersionState( - context.Background(), - versionID, - model.ObjectStateStored, - model.ObjectStateCacheEvicted, - ); err != nil { - t.Fatalf("UpdateVersionState(cache_evicted): %v", err) - } - completedAt := time.Now() - previousTask := cacheeviction.NewLRUTask(cacheeviction.Candidate{ - ObjectID: objectID, - VersionID: versionID, - AccessedAt: previousAccess, - }, 3, completedAt) - previousTask.Status = model.TaskStatusCompleted - previousTask.CompletedAt = &completedAt - if err := env.repos.Tasks.Create(context.Background(), previousTask); err != nil { - t.Fatalf("Create previous completed LRU task: %v", err) - } - accessedAt := previousAccess.Add(time.Hour) - if err := env.repos.Objects.RecordVersionCacheCommit(context.Background(), versionID, accessedAt); err != nil { - t.Fatalf("RecordVersionCacheCommit(rehydrated): %v", err) - } - - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 1, - 10*time.Millisecond, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 10, 90, 50, 3), - ) - runWorkerUntilCondition(t, evictor, 10*time.Millisecond, 3*time.Second, func() (bool, error) { - version, err := env.repos.Objects.GetVersionByID(context.Background(), versionID) - return version != nil && !version.InCache, err - }) - tasks, total, err := env.repos.Tasks.List( - context.Background(), - string(model.TaskTypeEvictCache), - cacheeviction.StageLRU, - "", - 10, - 0, - ) - if err != nil || total != 1 || len(tasks) != 1 || tasks[0].ID != previousTask.ID { - t.Fatalf("stable rehydrated LRU task total=%d tasks=%#v err=%v, want reused task", total, tasks, err) - } - version, err := env.repos.Objects.GetVersionByID(context.Background(), versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - if version.State != model.ObjectStateCacheEvicted || version.InCache { - t.Fatalf("version state/cache = %s/%v, want cache_evicted/false", version.State, version.InCache) - } -} - -func TestEvictor_LRUReconcilesAlreadyMissingCacheEntry(t *testing.T) { - var deleteCalls atomic.Int64 - mc := &testutil.MockCache{ - UsedBytesFunc: func() int64 { return 11 }, - ExistsFunc: func(_ context.Context, _, _ string) bool { - return false - }, - DeleteFunc: func(_ context.Context, _, _ string) error { - deleteCalls.Add(1) - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - _, objectID, versionID := seedStoredObject(t, env) - accessedAt := futureLRUAccessTime() - if err := env.repos.Objects.RecordVersionCacheAccess(context.Background(), versionID, accessedAt); err != nil { - t.Fatalf("RecordVersionCacheAccess: %v", err) - } - task := seedLRUEvictionTask(t, env, objectID, versionID, accessedAt) - - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 1, - 10*time.Millisecond, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 20, 90, 50, 3), - ) - got := runWorkerUntilTask(t, env, evictor, task.ID, 3*time.Second) - if got.Status != model.TaskStatusCompleted { - t.Fatalf("missing-entry LRU task status = %s, want completed", got.Status) - } - if deleteCalls.Load() != 1 { - t.Fatalf("idempotent cache Delete calls = %d, want 1", deleteCalls.Load()) - } - version, err := env.repos.Objects.GetVersionByID(context.Background(), versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - if version.State != model.ObjectStateCacheEvicted || version.InCache { - t.Fatalf("version state/cache = %s/%v, want cache_evicted/false", version.State, version.InCache) - } -} - -func TestEvictor_LRUTransientVersionLookupFailureSchedulesRetry(t *testing.T) { - mc := &testutil.MockCache{ - UsedBytesFunc: func() int64 { return 11 }, - ExistsFunc: func(_ context.Context, _, _ string) bool { - return true - }, - DeleteFunc: func(_ context.Context, _, _ string) error { - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - _, objectID, versionID := seedStoredObject(t, env) - accessedAt := futureLRUAccessTime() - if err := env.repos.Objects.RecordVersionCacheAccess(context.Background(), versionID, accessedAt); err != nil { - t.Fatalf("RecordVersionCacheAccess: %v", err) - } - task := seedLRUEvictionTask(t, env, objectID, versionID, accessedAt) - env.repos.Objects = &failOnceVersionLookupRepo{ObjectRepository: env.repos.Objects} - - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 1, - 10*time.Millisecond, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 20, 90, 50, 3), - ) - runWorkerUntilTaskRetryCount(t, env, evictor, task.ID, 1, 3*time.Second) - got, err := env.repos.Tasks.GetByID(context.Background(), task.ID) - if err != nil || got == nil { - t.Fatalf("GetByID after transient lookup: task=%v err=%v", got, err) - } - if got.Status != model.TaskStatusScheduled || got.RetryCount != 1 { - t.Fatalf("transient lookup LRU task status/retries = %s/%d, want scheduled/1", got.Status, got.RetryCount) - } -} - -type failOnceVersionLookupRepo struct { - repository.ObjectRepository - failed atomic.Bool -} - -func (r *failOnceVersionLookupRepo) GetVersionByID(ctx context.Context, versionID string) (*model.ObjectVersion, error) { - if r.failed.CompareAndSwap(false, true) { - return nil, errors.New("transient version lookup failure") - } - return r.ObjectRepository.GetVersionByID(ctx, versionID) -} - -func TestEvictor_LRUConcurrentWorkersStopAtLowWatermark(t *testing.T) { - var used atomic.Int64 - used.Store(33) - var deleteCalls atomic.Int64 - var activeDeletes atomic.Int64 - var concurrentOnce sync.Once - concurrentDeletes := make(chan struct{}) - releaseDeletes := make(chan struct{}) - mc := &testutil.MockCache{ - UsedBytesFunc: used.Load, - ExistsFunc: func(_ context.Context, _, _ string) bool { - return true - }, - DeleteFunc: func(ctx context.Context, _, _ string) error { - deleteCalls.Add(1) - if activeDeletes.Add(1) == 2 { - concurrentOnce.Do(func() { close(concurrentDeletes) }) - } - defer activeDeletes.Add(-1) - select { - case <-releaseDeletes: - case <-ctx.Done(): - return ctx.Err() - } - used.Add(-11) - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - accessedAt := futureLRUAccessTime() - var taskIDs []int64 - for range 3 { - _, objectID, versionID := seedStoredObject(t, env) - if err := env.repos.Objects.RecordVersionCacheAccess(context.Background(), versionID, accessedAt); err != nil { - t.Fatalf("RecordVersionCacheAccess(%s): %v", versionID, err) - } - taskIDs = append(taskIDs, seedLRUEvictionTask(t, env, objectID, versionID, accessedAt).ID) - } - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 3, - 10*time.Millisecond, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 30, 90, 60, 3), - ) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = evictor.Run(ctx) - close(done) - }() - select { - case <-concurrentDeletes: - close(releaseDeletes) - case <-time.After(time.Second): - close(releaseDeletes) - cancel() - waitForSignal(t, done, time.Second, "concurrent LRU evictor shutdown") - t.Fatal("LRU cache deletes did not run concurrently") - } - deadline := time.After(3 * time.Second) - ticker := time.NewTicker(10 * time.Millisecond) - defer ticker.Stop() - for { - terminal := 0 - for _, taskID := range taskIDs { - task, err := env.repos.Tasks.GetByID(context.Background(), taskID) - if err == nil && task != nil && !taskStatusActive(task.Status) { - terminal++ - } - } - if terminal == len(taskIDs) { - break - } - select { - case <-deadline: - cancel() - <-done - t.Fatalf("timed out waiting for concurrent LRU tasks; terminal=%d used=%d", terminal, used.Load()) - case <-ticker.C: - } - } - cancel() - waitForSignal(t, done, time.Second, "concurrent LRU evictor shutdown") - - if deleteCalls.Load() != 2 || used.Load() != 11 { - t.Fatalf("concurrent LRU deletes/used = %d/%d, want 2/11", deleteCalls.Load(), used.Load()) - } - var completed, cancelled int - for _, taskID := range taskIDs { - task, err := env.repos.Tasks.GetByID(context.Background(), taskID) - if err != nil || task == nil { - t.Fatalf("GetByID(%d): task=%v err=%v", taskID, task, err) - } - switch task.Status { - case model.TaskStatusCompleted: - completed++ - case model.TaskStatusCancelled: - cancelled++ - default: - t.Fatalf("task %d status = %s, want completed/cancelled", taskID, task.Status) - } - } - if completed != 2 || cancelled != 1 { - t.Fatalf("concurrent LRU terminal counts completed/cancelled = %d/%d, want 2/1", completed, cancelled) - } -} - -func TestEvictor_CancelsTaskThatDoesNotMatchCurrentPolicy(t *testing.T) { - var deleteCalls atomic.Int64 - mc := &testutil.MockCache{ - DeleteFunc: func(_ context.Context, _, _ string) error { - deleteCalls.Add(1) - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - _, objectID, versionID := seedStoredObject(t, env) - task := seedTask(t, env, model.TaskTypeEvictCache, objectID, versionID, 3, 0) - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 1, - 10*time.Millisecond, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyNone, 20, 90, 80, 3), - ) - - got := runWorkerUntilTask(t, env, evictor, task.ID, 3*time.Second) - if got.Status != model.TaskStatusCancelled { - t.Fatalf("incompatible eviction task status = %s, want cancelled", got.Status) - } - if deleteCalls.Load() != 0 { - t.Fatalf("cache Delete calls = %d, want 0 for incompatible policy", deleteCalls.Load()) - } -} diff --git a/internal/worker/evictor_lru_observability_test.go b/internal/worker/evictor_lru_observability_test.go deleted file mode 100644 index ed7c03b..0000000 --- a/internal/worker/evictor_lru_observability_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package worker_test - -import ( - "bytes" - "context" - "errors" - "fmt" - "hash/fnv" - "log/slog" - "strings" - "testing" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/strahe/synaps3/internal/admin" - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/cacheaccess" - "github.com/strahe/synaps3/internal/testutil" - "github.com/strahe/synaps3/internal/worker" -) - -type unavailableCacheAccessStore struct{} - -func (*unavailableCacheAccessStore) RecordVersionCacheAccess( - context.Context, - string, - time.Time, -) error { - return errors.New("database unavailable") -} - -func (*unavailableCacheAccessStore) RecordVersionCacheCommit( - context.Context, - string, - time.Time, -) error { - return errors.New("database unavailable") -} - -func unsafeCacheAccessTracker(t *testing.T) *cacheaccess.Tracker { - t.Helper() - tracker := cacheaccess.NewTracker(time.Minute, new(unavailableCacheAccessStore)) - for index := 0; index < 100_000 && tracker.SafeForLRU(); index++ { - versionID := fmt.Sprintf("unsafe-version-%d", index) - hash := fnv.New32a() - _, _ = hash.Write([]byte(versionID)) - if hash.Sum32()%256 != 0 { - continue - } - _ = tracker.RecordAccess(context.Background(), versionID, nil) - } - if tracker.SafeForLRU() { - t.Fatal("cache access tracker remained safe after bounded dirty entries overflowed") - } - return tracker -} - -func gatheredGaugeValue(name string) float64 { - families, err := prometheus.DefaultGatherer.Gather() - if err != nil { - return -1 - } - for _, family := range families { - if family.GetName() != name || len(family.GetMetric()) == 0 { - continue - } - return family.GetMetric()[0].GetGauge().GetValue() - } - return -1 -} - -func TestEvictor_LRUAccessTrackingPauseIsObservable(t *testing.T) { - admin.CacheLRUEvictionPaused.Set(0) - t.Cleanup(func() { - admin.CacheLRUEvictionPaused.Set(0) - }) - var logs bytes.Buffer - logger := slog.New(slog.NewTextHandler(&logs, nil)) - mc := &testutil.MockCache{ - UsedBytesFunc: func() int64 { - return 100 - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - unsafeCacheAccessTracker(t), - env.sm, - 0, - 10*time.Millisecond, - logger, - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 100, 90, 80, 3), - ) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = evictor.Run(ctx) - close(done) - }() - - deadline := time.After(time.Second) - ticker := time.NewTicker(5 * time.Millisecond) - defer ticker.Stop() - for gatheredGaugeValue("synaps3_cache_lru_eviction_paused") != 1 { - select { - case <-deadline: - cancel() - waitForSignal(t, done, time.Second, "LRU pause observability test shutdown") - t.Fatal("LRU pause metric did not become 1") - case <-ticker.C: - } - } - time.Sleep(40 * time.Millisecond) - cancel() - waitForSignal(t, done, time.Second, "LRU pause observability test shutdown") - - const pauseMessage = "LRU cache eviction paused until restart" - if count := strings.Count(logs.String(), pauseMessage); count != 1 { - t.Fatalf("LRU pause log count = %d, want 1; logs=%s", count, logs.String()) - } -} diff --git a/internal/worker/evictor_lru_planner_test.go b/internal/worker/evictor_lru_planner_test.go deleted file mode 100644 index e042611..0000000 --- a/internal/worker/evictor_lru_planner_test.go +++ /dev/null @@ -1,627 +0,0 @@ -package worker_test - -import ( - "context" - "errors" - "log/slog" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/cacheeviction" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/testutil" - "github.com/strahe/synaps3/internal/worker" -) - -func TestEvictor_LRUBelowHighWatermarkDoesNotPlanEviction(t *testing.T) { - var used atomic.Int64 - used.Store(26) - var deleteCalls atomic.Int64 - mc := &testutil.MockCache{ - UsedBytesFunc: used.Load, - ExistsFunc: func(_ context.Context, _, _ string) bool { - return true - }, - DeleteFunc: func(_ context.Context, _, _ string) error { - deleteCalls.Add(1) - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - seedStoredObject(t, env) - pollInterval := 15 * time.Millisecond - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 1, - pollInterval, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 30, 90, 60, 3), - ) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = evictor.Run(ctx) - close(done) - }() - time.Sleep(4 * pollInterval) - cancel() - waitForSignal(t, done, time.Second, "LRU evictor shutdown") - - tasks, total, err := env.repos.Tasks.List( - context.Background(), - string(model.TaskTypeEvictCache), - cacheeviction.StageLRU, - "", - 10, - 0, - ) - if err != nil { - t.Fatalf("List LRU tasks: %v", err) - } - if total != 0 || len(tasks) != 0 { - t.Fatalf("LRU tasks below high watermark total=%d tasks=%#v, want none", total, tasks) - } - if deleteCalls.Load() != 0 { - t.Fatalf("cache Delete calls below high watermark = %d, want 0", deleteCalls.Load()) - } -} - -func TestEvictor_LRUEvictsLeastRecentlyUsedUntilLowWatermark(t *testing.T) { - var used atomic.Int64 - used.Store(33) - var deletedMu sync.Mutex - var deleted []string - mc := &testutil.MockCache{ - UsedBytesFunc: used.Load, - ExistsFunc: func(_ context.Context, _, _ string) bool { - return true - }, - DeleteFunc: func(_ context.Context, _, key string) error { - deletedMu.Lock() - deleted = append(deleted, key) - deletedMu.Unlock() - used.Add(-11) - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - type seeded struct { - objectID int64 - versionID string - } - var versions []seeded - for range 3 { - _, objectID, versionID := seedStoredObject(t, env) - versions = append(versions, seeded{objectID: objectID, versionID: versionID}) - } - base := futureLRUAccessTime() - for index, version := range versions { - if err := env.repos.Objects.RecordVersionCacheAccess( - context.Background(), - version.versionID, - base.Add(time.Duration(index)*time.Hour), - ); err != nil { - t.Fatalf("RecordVersionCacheAccess(%s): %v", version.versionID, err) - } - } - - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 1, - 10*time.Millisecond, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 30, 90, 60, 3), - ) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = evictor.Run(ctx) - close(done) - }() - deadline := time.After(3 * time.Second) - ticker := time.NewTicker(10 * time.Millisecond) - defer ticker.Stop() - for { - _, completed, err := env.repos.Tasks.List( - context.Background(), - string(model.TaskTypeEvictCache), - cacheeviction.StageLRU, - string(model.TaskStatusCompleted), - 10, - 0, - ) - if err == nil && completed == 2 { - break - } - select { - case <-deadline: - cancel() - <-done - t.Fatalf("timed out waiting for LRU eviction; used=%d completed=%d err=%v", used.Load(), completed, err) - case <-ticker.C: - } - } - cancel() - waitForSignal(t, done, time.Second, "LRU evictor shutdown") - - deletedMu.Lock() - gotDeleted := append([]string(nil), deleted...) - deletedMu.Unlock() - wantDeleted := []string{ - ".versions/" + versions[0].versionID, - ".versions/" + versions[1].versionID, - } - if len(gotDeleted) != len(wantDeleted) { - t.Fatalf("deleted cache keys = %#v, want %#v", gotDeleted, wantDeleted) - } - for index := range wantDeleted { - if gotDeleted[index] != wantDeleted[index] { - t.Fatalf("deleted cache keys = %#v, want LRU order %#v", gotDeleted, wantDeleted) - } - } - for index, version := range versions { - got, err := env.repos.Objects.GetVersionByID(context.Background(), version.versionID) - if err != nil || got == nil { - t.Fatalf("GetVersionByID(%s): version=%v err=%v", version.versionID, got, err) - } - if index < 2 { - if got.State != model.ObjectStateCacheEvicted || got.InCache { - t.Fatalf("evicted version %d state/cache = %s/%v", index, got.State, got.InCache) - } - } else if got.State != model.ObjectStateStored || !got.InCache { - t.Fatalf("retained version state/cache = %s/%v, want stored/true", got.State, got.InCache) - } - } - completed, total, err := env.repos.Tasks.List( - context.Background(), - string(model.TaskTypeEvictCache), - cacheeviction.StageLRU, - string(model.TaskStatusCompleted), - 10, - 0, - ) - if err != nil { - t.Fatalf("List completed LRU tasks: %v", err) - } - if total != 2 || len(completed) != 2 { - t.Fatalf("completed LRU tasks total=%d tasks=%#v, want 2", total, completed) - } - for _, task := range completed { - if _, ok := task.Payload["cache_accessed_at"].(string); !ok { - t.Fatalf("LRU task payload = %#v, want RFC3339Nano string snapshot", task.Payload) - } - } -} - -func TestEvictor_LRUContinuesActiveCycleBelowHighWatermark(t *testing.T) { - var used atomic.Int64 - used.Store(95) - var deleteCalls atomic.Int64 - var touchOnce sync.Once - touchResult := make(chan error, 1) - - var env *testWorkerEnv - var versionToTouch string - mc := &testutil.MockCache{ - UsedBytesFunc: used.Load, - ExistsFunc: func(_ context.Context, _, _ string) bool { - return true - }, - DeleteFunc: func(_ context.Context, _, _ string) error { - deleteCalls.Add(1) - used.Add(-11) - touchOnce.Do(func() { - touchResult <- env.repos.Objects.RecordVersionCacheAccess( - context.Background(), - versionToTouch, - futureLRUAccessTime().Add(24*time.Hour), - ) - }) - return nil - }, - } - env = newTestWorkerEnvWithMockCache(t, mc) - - var versionIDs []string - base := futureLRUAccessTime() - for index := range 3 { - _, _, versionID := seedStoredObject(t, env) - versionIDs = append(versionIDs, versionID) - if err := env.repos.Objects.RecordVersionCacheAccess( - context.Background(), - versionID, - base.Add(time.Duration(index)*time.Hour), - ); err != nil { - t.Fatalf("RecordVersionCacheAccess(%s): %v", versionID, err) - } - } - versionToTouch = versionIDs[1] - - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 1, - 10*time.Millisecond, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 100, 90, 80, 3), - ) - runWorkerUntilCondition(t, evictor, 10*time.Millisecond, 3*time.Second, func() (bool, error) { - if used.Load() > 80 || deleteCalls.Load() != 2 { - return false, nil - } - tasks, total, err := env.repos.Tasks.List( - context.Background(), - string(model.TaskTypeEvictCache), - cacheeviction.StageLRU, - "", - 10, - 0, - ) - if err != nil { - return false, err - } - var completed, cancelled int - for _, task := range tasks { - switch task.Status { - case model.TaskStatusCompleted: - completed++ - case model.TaskStatusCancelled: - cancelled++ - } - } - return total == 3 && completed == 2 && cancelled == 1, nil - }) - - select { - case err := <-touchResult: - if err != nil { - t.Fatalf("touching planned LRU candidate: %v", err) - } - default: - t.Fatal("planned LRU candidate was not touched during the first deletion") - } - - tasks, total, err := env.repos.Tasks.List( - context.Background(), - string(model.TaskTypeEvictCache), - cacheeviction.StageLRU, - "", - 10, - 0, - ) - if err != nil { - t.Fatalf("List LRU tasks: %v", err) - } - var completed, cancelled int - for _, task := range tasks { - switch task.Status { - case model.TaskStatusCompleted: - completed++ - case model.TaskStatusCancelled: - cancelled++ - } - } - if total != 3 || completed != 2 || cancelled != 1 { - t.Fatalf( - "LRU tasks total/completed/cancelled = %d/%d/%d, want 3/2/1", - total, - completed, - cancelled, - ) - } -} - -func TestEvictor_LRUSkipsUnexpectedNullAccessTime(t *testing.T) { - var used atomic.Int64 - used.Store(11) - var deleteCalls atomic.Int64 - mc := &testutil.MockCache{ - UsedBytesFunc: used.Load, - ExistsFunc: func(_ context.Context, _, _ string) bool { - return true - }, - DeleteFunc: func(_ context.Context, _, _ string) error { - deleteCalls.Add(1) - used.Store(0) - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - _, _, versionID := seedStoredObject(t, env) - if _, err := env.db.NewUpdate(). - Model((*model.ObjectVersion)(nil)). - Set("cache_accessed_at = NULL"). - Where("version_id = ?", versionID). - Exec(context.Background()); err != nil { - t.Fatalf("clear cache access time: %v", err) - } - candidates, err := env.repos.CacheEvictions.ListLRUCandidates( - context.Background(), - time.Now().Add(-time.Hour), - 10, - ) - if err != nil || len(candidates) != 0 { - t.Fatalf("NULL-access LRU candidates = %#v err=%v, want none", candidates, err) - } - - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 1, - 10*time.Millisecond, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 10, 90, 50, 3), - ) - runCtx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = evictor.Run(runCtx) - close(done) - }() - time.Sleep(50 * time.Millisecond) - cancel() - waitForSignal(t, done, time.Second, "NULL-access LRU test shutdown") - - tasks, total, err := env.repos.Tasks.List( - context.Background(), - string(model.TaskTypeEvictCache), - cacheeviction.StageLRU, - "", - 10, - 0, - ) - if err != nil { - t.Fatalf("List NULL-access LRU tasks: %v", err) - } - if total != 0 || len(tasks) != 0 { - t.Fatalf("NULL-access LRU tasks total=%d tasks=%#v, want none", total, tasks) - } - if deleteCalls.Load() != 0 { - t.Fatalf("NULL-access LRU delete calls = %d, want 0", deleteCalls.Load()) - } - version, err := env.repos.Objects.GetVersionByID(context.Background(), versionID) - if err != nil || version == nil || version.State != model.ObjectStateStored || !version.InCache { - t.Fatalf("NULL-access LRU version = %#v err=%v, want stored in cache", version, err) - } -} - -func TestEvictor_LRUReactivatesCancelledStableTask(t *testing.T) { - var used atomic.Int64 - used.Store(11) - mc := &testutil.MockCache{ - UsedBytesFunc: used.Load, - ExistsFunc: func(_ context.Context, _, _ string) bool { - return true - }, - DeleteFunc: func(_ context.Context, _, _ string) error { - used.Store(0) - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - _, objectID, versionID := seedStoredObject(t, env) - accessedAt := futureLRUAccessTime() - if err := env.repos.Objects.RecordVersionCacheAccess(context.Background(), versionID, accessedAt); err != nil { - t.Fatalf("RecordVersionCacheAccess: %v", err) - } - completedAt := time.Now() - task := cacheeviction.NewLRUTask(cacheeviction.Candidate{ - ObjectID: objectID, - VersionID: versionID, - AccessedAt: accessedAt, - }, 3, completedAt) - task.Status = model.TaskStatusCancelled - task.CompletedAt = &completedAt - if err := env.repos.Tasks.Create(context.Background(), task); err != nil { - t.Fatalf("Create cancelled LRU task: %v", err) - } - - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 1, - 10*time.Millisecond, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 10, 90, 50, 3), - ) - runWorkerUntilCondition(t, evictor, 10*time.Millisecond, 3*time.Second, func() (bool, error) { - got, err := env.repos.Tasks.GetByID(context.Background(), task.ID) - return got != nil && got.Status == model.TaskStatusCompleted, err - }) - - tasks, total, err := env.repos.Tasks.List( - context.Background(), - string(model.TaskTypeEvictCache), - cacheeviction.StageLRU, - "", - 10, - 0, - ) - if err != nil { - t.Fatalf("List reactivated LRU tasks: %v", err) - } - if total != 1 || len(tasks) != 1 || tasks[0].ID != task.ID || tasks[0].Status != model.TaskStatusCompleted { - t.Fatalf("reactivated LRU tasks total=%d tasks=%#v, want original completed task", total, tasks) - } -} - -func TestEvictor_LRUExhaustedTaskWaitsForCooldownBeforeReplanning(t *testing.T) { - var used atomic.Int64 - used.Store(11) - var deleteCalls atomic.Int64 - var deleteSucceeds atomic.Bool - mc := &testutil.MockCache{ - UsedBytesFunc: used.Load, - ExistsFunc: func(_ context.Context, _, _ string) bool { - return true - }, - DeleteFunc: func(_ context.Context, _, _ string) error { - deleteCalls.Add(1) - if deleteSucceeds.Load() { - used.Store(0) - return nil - } - return errors.New("permission denied") - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - _, _, versionID := seedStoredObject(t, env) - accessedAt := futureLRUAccessTime() - if err := env.repos.Objects.RecordVersionCacheAccess(context.Background(), versionID, accessedAt); err != nil { - t.Fatalf("RecordVersionCacheAccess: %v", err) - } - - pollInterval := 10 * time.Millisecond - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 1, - pollInterval, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 10, 90, 50, 1), - ) - var exhaustedAt time.Time - runWorkerUntilCondition(t, evictor, pollInterval, 3*time.Second, func() (bool, error) { - _, exhausted, err := env.repos.Tasks.List( - context.Background(), - string(model.TaskTypeEvictCache), - cacheeviction.StageLRU, - string(model.TaskStatusExhausted), - 10, - 0, - ) - if err != nil || exhausted != 1 { - return false, err - } - if exhaustedAt.IsZero() { - exhaustedAt = time.Now() - } - return time.Since(exhaustedAt) >= 8*pollInterval, nil - }) - - tasks, total, err := env.repos.Tasks.List( - context.Background(), - string(model.TaskTypeEvictCache), - cacheeviction.StageLRU, - "", - 10, - 0, - ) - if err != nil { - t.Fatalf("List LRU tasks after exhaustion: %v", err) - } - if total != 1 || len(tasks) != 1 || tasks[0].Status != model.TaskStatusExhausted { - t.Fatalf("LRU tasks after exhaustion total=%d tasks=%#v, want one exhausted task", total, tasks) - } - if deleteCalls.Load() != 1 { - t.Fatalf("LRU delete calls after exhaustion = %d, want 1", deleteCalls.Load()) - } - - if _, err := env.db.NewUpdate(). - Model((*model.Task)(nil)). - Set("completed_at = ?", time.Now().Add(-2*time.Hour)). - Where("id = ?", tasks[0].ID). - Exec(context.Background()); err != nil { - t.Fatalf("age exhausted LRU task: %v", err) - } - deleteSucceeds.Store(true) - recoveryEvictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 1, - pollInterval, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 10, 90, 50, 1), - ) - runWorkerUntilCondition(t, recoveryEvictor, pollInterval, 3*time.Second, func() (bool, error) { - _, completed, err := env.repos.Tasks.List( - context.Background(), - string(model.TaskTypeEvictCache), - cacheeviction.StageLRU, - string(model.TaskStatusCompleted), - 10, - 0, - ) - return completed == 1, err - }) - tasks, total, err = env.repos.Tasks.List( - context.Background(), - string(model.TaskTypeEvictCache), - cacheeviction.StageLRU, - "", - 10, - 0, - ) - if err != nil { - t.Fatalf("List LRU tasks after cooldown recovery: %v", err) - } - if total != 1 || len(tasks) != 1 || tasks[0].Status != model.TaskStatusCompleted { - t.Fatalf("LRU tasks after cooldown recovery total=%d tasks=%#v, want original completed task", total, tasks) - } - if deleteCalls.Load() != 2 { - t.Fatalf("LRU delete calls after cooldown recovery = %d, want 2 total attempts", deleteCalls.Load()) - } -} - -func runWorkerUntilCondition( - t *testing.T, - w worker.Worker, - pollInterval time.Duration, - timeout time.Duration, - condition func() (bool, error), -) { - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = w.Run(ctx) - close(done) - }() - defer waitForSignal(t, done, time.Second, "worker condition shutdown") - defer cancel() - - deadline := time.NewTimer(timeout) - defer deadline.Stop() - ticker := time.NewTicker(pollInterval) - defer ticker.Stop() - for { - ok, err := condition() - if err != nil { - t.Fatalf("checking worker condition: %v", err) - } - if ok { - return - } - select { - case <-deadline.C: - t.Fatal("timed out waiting for worker condition") - case <-ticker.C: - } - } -} diff --git a/internal/worker/evictor_payload_test.go b/internal/worker/evictor_payload_test.go deleted file mode 100644 index a7913a9..0000000 --- a/internal/worker/evictor_payload_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package worker_test - -import ( - "context" - "log/slog" - "sync/atomic" - "testing" - "time" - - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/cacheeviction" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/testutil" - "github.com/strahe/synaps3/internal/worker" -) - -func TestEvictor_LRUMalformedAccessSnapshotCancelsWithoutDeleting(t *testing.T) { - var deleteCalls atomic.Int64 - mc := &testutil.MockCache{ - UsedBytesFunc: func() int64 { return 11 }, - DeleteFunc: func(context.Context, string, string) error { - deleteCalls.Add(1) - return nil - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - _, objectID, versionID := seedStoredObject(t, env) - stage := cacheeviction.StageLRU - task := &model.Task{ - Type: model.TaskTypeEvictCache, - Stage: &stage, - RefType: "object", - RefID: objectID, - RefVersionID: versionID, - IdempotencyKey: "evict_cache:lru:" + versionID, - Payload: map[string]any{"cache_accessed_at": float64(1)}, - Status: model.TaskStatusQueued, - MaxRetries: 3, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(context.Background(), task); err != nil { - t.Fatalf("Create: %v", err) - } - - // Disable planning so the cancelled fixture task is not immediately - // reactivated with a valid payload. - evictor := worker.NewEvictor( - env.repos, - env.cache, - env.cacheGate, - env.accessTracker, - env.sm, - 1, - 10*time.Millisecond, - slog.Default(), - worker.WithCacheEvictionPolicy(cache.EvictionPolicyLRU, 0, 90, 50, 3), - ) - got := runWorkerUntilTask(t, env, evictor, task.ID, 3*time.Second) - if got.Status != model.TaskStatusCancelled { - t.Fatalf("task status = %s, want cancelled", got.Status) - } - if deleteCalls.Load() != 0 { - t.Fatalf("cache delete calls = %d, want 0", deleteCalls.Load()) - } -} diff --git a/internal/worker/evictor_reconcile.go b/internal/worker/evictor_reconcile.go deleted file mode 100644 index dc90c3b..0000000 --- a/internal/worker/evictor_reconcile.go +++ /dev/null @@ -1,144 +0,0 @@ -package worker - -import ( - "context" - "errors" - - "github.com/strahe/synaps3/internal/admin" - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/cacheeviction" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" -) - -const bucketDurabilityBatchSize = 100 - -func (e *Evictor) processBucketDurabilityReconciliation(ctx context.Context, task *model.Task) { - logger := e.taskLogger(task) - processed := 0 - for processed < bucketDurabilityBatchSize { - authorized, err := cacheeviction.DeleteAuthorized(task) - if err != nil { - e.cancelTask(ctx, task, "Cache deletion authorization is invalid") - return - } - if authorized { - if decision := e.resumeBucketDurabilityDeletion(ctx, task); decision != nil { - e.applyEvictionDecision(ctx, task, decision) - return - } - processed++ - continue - } - - candidate, err := e.repos.CacheEvictions.NextBucketDurabilityCandidate(ctx, task.RefID) - if err != nil { - e.applyEvictionDecision(ctx, task, retryEviction(err, "selecting bucket durability candidate")) - return - } - if candidate == nil { - completed, err := e.repos.CacheEvictions.CompleteBucketDurabilityReconciliation(ctx, task) - switch { - case err == nil && completed: - admin.WorkerTasksProcessed.WithLabelValues("evictor", "success").Inc() - logger.Info("bucket cache policy applied", "versions", processed) - return - case err == nil: - continue - case errors.Is(err, cacheeviction.ErrNoLongerEligible): - e.cancelTask(ctx, task, "Bucket no longer exists") - return - default: - e.applyEvictionDecision(ctx, task, retryEviction(err, "completing bucket durability reconciliation")) - return - } - } - - promoted, decision := e.promoteBucketDurabilityCandidate(ctx, task, candidate.VersionID) - if decision != nil { - e.applyEvictionDecision(ctx, task, decision) - return - } - if promoted { - processed++ - } - } - - if err := e.repos.Tasks.ReleaseRunning(ctx, task); err != nil { - e.applyEvictionDecision(ctx, task, retryEviction(err, "requeueing bucket durability reconciliation")) - return - } - admin.WorkerTasksProcessed.WithLabelValues("evictor", "success").Inc() - logger.Debug("requeued bucket cache policy batch", "versions", processed) -} - -func (e *Evictor) promoteBucketDurabilityCandidate( - ctx context.Context, - task *model.Task, - versionID string, -) (bool, *evictionDecision) { - deleteAfterPromotion := e.policy == cache.EvictionPolicyAfterUpload - if !deleteAfterPromotion { - _, err := e.repos.CacheEvictions.PromoteBucketDurabilityCandidate(ctx, task, versionID, false) - switch { - case err == nil: - return true, nil - case errors.Is(err, cacheeviction.ErrNoLongerEligible), - errors.Is(err, cacheeviction.ErrDurabilityThreshold): - return false, nil - default: - return false, retryEviction(err, "promoting bucket durability candidate") - } - } - - var ( - promoted bool - decision *evictionDecision - ) - e.cacheGate.GuardDeletion(versionID, func() { - deletion, err := e.repos.CacheEvictions.PromoteBucketDurabilityCandidate(ctx, task, versionID, true) - switch { - case err == nil: - if err := e.deleteCoordinatorCacheEntry(ctx, task, deletion); err != nil { - decision = retryEviction(err, "deleting reconciled cache entry") - return - } - promoted = true - case errors.Is(err, cacheeviction.ErrNoLongerEligible), - errors.Is(err, cacheeviction.ErrDurabilityThreshold): - return - default: - decision = retryEviction(err, "promoting bucket durability candidate") - } - }) - return promoted, decision -} - -func (e *Evictor) resumeBucketDurabilityDeletion( - ctx context.Context, - task *model.Task, -) *evictionDecision { - var decision *evictionDecision - e.cacheGate.GuardDeletion(task.RefVersionID, func() { - deletion, err := e.repos.CacheEvictions.AuthorizeDeletion(ctx, task, nil) - switch { - case err == nil: - if err := e.deleteCoordinatorCacheEntry(ctx, task, deletion); err != nil { - decision = retryEviction(err, "resuming reconciled cache deletion") - } - case errors.Is(err, repository.ErrNotFound): - switch clearErr := e.repos.CacheEvictions.RecordAuthorizedDeletion(ctx, task); { - case clearErr == nil: - case errors.Is(clearErr, cacheeviction.ErrNoLongerEligible): - decision = cancelEviction("Bucket no longer exists") - default: - decision = retryEviction(clearErr, "clearing completed cache deletion authorization") - } - case errors.Is(err, cacheeviction.ErrNoLongerEligible): - decision = cancelEviction("Authorized cache entry no longer exists") - default: - decision = retryEviction(err, "loading authorized cache deletion") - } - }) - return decision -} diff --git a/internal/worker/evictor_reconcile_internal_test.go b/internal/worker/evictor_reconcile_internal_test.go deleted file mode 100644 index 41a7010..0000000 --- a/internal/worker/evictor_reconcile_internal_test.go +++ /dev/null @@ -1,170 +0,0 @@ -package worker - -import ( - "context" - "fmt" - "log/slog" - "testing" - "time" - - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/cacheaccess" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/state" - "github.com/strahe/synaps3/internal/testutil" - "github.com/uptrace/bun" -) - -func TestEvictorBucketDurabilityReconciliationProcessesOneHundredVersionsPerClaim(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := testutil.SeedBucket(t, db, "durability-batch") - minimum := 1 - if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimum, - }); err != nil { - t.Fatalf("UpdateCopyPolicy: %v", err) - } - - const versionCount = bucketDurabilityBatchSize + 1 - var sourceVersionID string - for index := range versionCount { - version := &model.ObjectVersion{ - VersionID: model.NewVersionID(), - BucketID: bucket.ID, - Key: fmt.Sprintf("shared-%03d.bin", index), - Size: 10, - ETag: "shared-etag", - Checksum: "shared-durability-checksum", - ContentType: "application/octet-stream", - CacheKey: fmt.Sprintf(".versions/shared-%03d", index), - } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(%d): %v", index, err) - } - if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("UpdateVersionState(%d): %v", index, err) - } - if index == 0 { - sourceVersionID = version.VersionID - } - } - - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: sourceVersionID, - ContentSize: 10, - Checksum: "shared-durability-checksum", - RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - providerID := onChainID(t, "101") - dataSetID := onChainID(t, "1001") - pieceID := onChainID(t, "2001") - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: providerID, - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: upload.ID, - DataSetID: dataSetID, - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: providerID, - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "bafk2bzacedurabilitybatch", - PieceID: &pieceID, - RetrievalURL: "https://provider.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - refs, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: 10, - Checksum: "shared-durability-checksum", - }) - if err != nil || len(refs) != versionCount { - t.Fatalf("BindReadableUploadForContent: refs=%d err=%v", len(refs), err) - } - if _, err := repos.CacheEvictions.EnsureBucketDurabilityReconciliation(ctx, bucket.ID, 4); err != nil { - t.Fatalf("EnsureBucketDurabilityReconciliation: %v", err) - } - - evictor := NewEvictor( - repos, - &testutil.MockCache{}, - cacheaccess.NewGate(), - cacheaccess.NewTracker(0, repos.Objects), - state.NewObjectStateMachine(), - 1, - time.Millisecond, - slog.Default(), - WithCacheEvictionPolicy(cache.EvictionPolicyNone, 0, 90, 80, 4), - ) - task, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeEvictCache, time.Minute) - if err != nil || task == nil { - t.Fatalf("ClaimReady first batch: task=%#v err=%v", task, err) - } - evictor.processTask(ctx, task) - assertObjectVersionStateCount(t, db, bucket.ID, model.ObjectStateStored, bucketDurabilityBatchSize) - assertObjectVersionStateCount(t, db, bucket.ID, model.ObjectStateReplicating, 1) - firstBatchTask, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil || firstBatchTask == nil || firstBatchTask.Status != model.TaskStatusQueued { - t.Fatalf("task after first batch = %#v err=%v, want queued", firstBatchTask, err) - } - - task, err = repos.Tasks.ClaimReady(ctx, model.TaskTypeEvictCache, time.Minute) - if err != nil || task == nil { - t.Fatalf("ClaimReady second batch: task=%#v err=%v", task, err) - } - evictor.processTask(ctx, task) - assertObjectVersionStateCount(t, db, bucket.ID, model.ObjectStateStored, versionCount) - assertObjectVersionStateCount(t, db, bucket.ID, model.ObjectStateReplicating, 0) - completed, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil || completed == nil || completed.Status != model.TaskStatusCompleted { - t.Fatalf("task after second batch = %#v err=%v, want completed", completed, err) - } -} - -func assertObjectVersionStateCount( - t *testing.T, - db bun.IDB, - bucketID int64, - wantState model.ObjectState, - wantCount int, -) { - t.Helper() - count, err := db.NewSelect(). - Model((*model.ObjectVersion)(nil)). - Where("bucket_id = ? AND state = ?", bucketID, wantState). - Count(context.Background()) - if err != nil { - t.Fatalf("count %s versions: %v", wantState, err) - } - if count != wantCount { - t.Fatalf("%s version count = %d, want %d", wantState, count, wantCount) - } -} diff --git a/internal/worker/health.go b/internal/worker/health.go deleted file mode 100644 index 51ae009..0000000 --- a/internal/worker/health.go +++ /dev/null @@ -1,39 +0,0 @@ -package worker - -import ( - "sync/atomic" - "time" -) - -// livenessTracker tracks worker liveness based on last tick time. -type livenessTracker struct { - lastTick atomic.Int64 // unix nanos - activeWork atomic.Int64 - pollInterval time.Duration -} - -func newLivenessTracker(pollInterval time.Duration) *livenessTracker { - lt := &livenessTracker{pollInterval: pollInterval} - lt.recordTick() // mark as healthy at creation time - return lt -} - -func (lt *livenessTracker) recordTick() { - lt.lastTick.Store(time.Now().UnixNano()) -} - -func (lt *livenessTracker) recordWorkStarted() { - lt.activeWork.Add(1) -} - -func (lt *livenessTracker) recordWorkFinished() { - lt.activeWork.Add(-1) -} - -func (lt *livenessTracker) healthy() bool { - if lt.activeWork.Load() > 0 { - return true - } - last := time.Unix(0, lt.lastTick.Load()) - return time.Since(last) < 3*lt.pollInterval -} diff --git a/internal/worker/health_test.go b/internal/worker/health_test.go deleted file mode 100644 index 39eaffc..0000000 --- a/internal/worker/health_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package worker - -import ( - "testing" - "time" -) - -func TestLivenessTracker_Healthy(t *testing.T) { - lt := newLivenessTracker(10 * time.Second) - if !lt.healthy() { - t.Fatal("new tracker should be healthy") - } -} - -func TestLivenessTracker_Stale(t *testing.T) { - lt := newLivenessTracker(1 * time.Millisecond) - // Force the lastTick to be far in the past - lt.lastTick.Store(time.Now().Add(-10 * time.Second).UnixNano()) - if lt.healthy() { - t.Fatal("stale tracker should be unhealthy") - } -} - -func TestLivenessTracker_RecordTick(t *testing.T) { - lt := newLivenessTracker(1 * time.Millisecond) - lt.lastTick.Store(time.Now().Add(-10 * time.Second).UnixNano()) - if lt.healthy() { - t.Fatal("should be unhealthy before recordTick") - } - lt.recordTick() - if !lt.healthy() { - t.Fatal("should be healthy after recordTick") - } -} diff --git a/internal/worker/ingress_progress.go b/internal/worker/ingress_progress.go new file mode 100644 index 0000000..e92c3f4 --- /dev/null +++ b/internal/worker/ingress_progress.go @@ -0,0 +1,209 @@ +package worker + +import ( + "context" + "errors" + "log/slog" + "sync" + "time" + + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" +) + +const ( + uploadProgressFlushInterval = time.Second + uploadProgressWriteTimeout = 2 * time.Second +) + +type uploadProgressReporter struct { + ctx context.Context + repos *repository.Repositories + publisher EventPublisher + logger *slog.Logger + + contentID int64 + copyID int64 + generation int64 + taskID int64 + versionID string + bucketName string + objectKey string + attempt int + + mu sync.Mutex + lastFlush time.Time + pendingBytes int64 + pending bool + pendingTimer *time.Timer + closed bool + writes sync.WaitGroup +} + +func (h *TaskHandlers) newIngressProgressReporter( + ctx context.Context, + taskID int64, + generation int64, + copyID int64, + attempt int, + content *model.StorageContent, + bucket *model.Bucket, +) *uploadProgressReporter { + if content == nil || copyID < 1 || generation < 1 || taskID < 1 || attempt < 1 { + return nil + } + reporter := &uploadProgressReporter{ + ctx: ctx, repos: h.deps.Repositories, publisher: h.deps.Events, logger: h.deps.Logger, + contentID: content.ID, copyID: copyID, generation: generation, taskID: taskID, + attempt: attempt, + } + // The transfer belongs to the content; naming a version is only there to + // give the progress event something a reader recognises, so a content with + // no live version still reports progress. + if version, err := h.deps.Repositories.Contents.GetLiveVersionForUpload(ctx, content.ID); err != nil { + h.deps.Logger.Warn("failed to label ingress upload progress", "content_id", content.ID, "error", err) + } else if version != nil { + reporter.versionID = version.VersionID + reporter.objectKey = version.Key + } + if bucket != nil { + reporter.bucketName = bucket.Name + } + reporter.scheduleRecord(0, false) + return reporter +} + +func (r *uploadProgressReporter) OnProgress(bytesUploaded int64) { + if r == nil || r.attempt <= 0 { + return + } + now := time.Now() + r.mu.Lock() + if r.closed { + r.mu.Unlock() + return + } + if r.lastFlush.IsZero() || now.Sub(r.lastFlush) >= uploadProgressFlushInterval { + r.cancelPendingLocked() + r.lastFlush = now + r.scheduleRecordLocked(bytesUploaded, false) + r.mu.Unlock() + return + } + r.pendingBytes = bytesUploaded + r.pending = true + if r.pendingTimer == nil { + r.pendingTimer = time.AfterFunc(uploadProgressFlushInterval-now.Sub(r.lastFlush), r.flushPending) + } + r.mu.Unlock() +} + +func (r *uploadProgressReporter) Flush(bytesUploaded int64, done bool) { + if r == nil || r.attempt <= 0 { + return + } + r.mu.Lock() + if r.closed { + r.mu.Unlock() + return + } + r.cancelPendingLocked() + r.writes.Add(1) + r.mu.Unlock() + r.record(bytesUploaded, done) + r.writes.Done() +} + +func (r *uploadProgressReporter) Close() { + if r == nil { + return + } + r.mu.Lock() + if r.closed { + r.mu.Unlock() + r.writes.Wait() + return + } + r.closed = true + r.cancelPendingLocked() + r.mu.Unlock() + r.writes.Wait() +} + +func (r *uploadProgressReporter) flushPending() { + r.mu.Lock() + if r.closed || !r.pending { + r.pendingTimer = nil + r.mu.Unlock() + return + } + bytesUploaded := r.pendingBytes + r.pending = false + r.pendingTimer = nil + r.lastFlush = time.Now() + r.scheduleRecordLocked(bytesUploaded, false) + r.mu.Unlock() +} + +func (r *uploadProgressReporter) cancelPendingLocked() { + r.pending = false + if r.pendingTimer != nil { + r.pendingTimer.Stop() + r.pendingTimer = nil + } +} + +func (r *uploadProgressReporter) scheduleRecord(bytesUploaded int64, done bool) { + r.mu.Lock() + defer r.mu.Unlock() + if r.closed { + return + } + r.scheduleRecordLocked(bytesUploaded, done) +} + +func (r *uploadProgressReporter) scheduleRecordLocked(bytesUploaded int64, done bool) { + r.writes.Go(func() { + r.record(bytesUploaded, done) + }) +} + +func (r *uploadProgressReporter) record(bytesUploaded int64, done bool) { + ctx, cancel := context.WithTimeout(r.ctx, uploadProgressWriteTimeout) + defer cancel() + upload, err := r.repos.Contents.RecordIngressStoreProgress(ctx, repository.RecordIngressStoreProgressInput{ + CopyID: r.copyID, Generation: r.generation, TaskID: r.taskID, + Attempt: r.attempt, BytesUploaded: bytesUploaded, + }) + if err != nil { + if errors.Is(err, repository.ErrConflict) { + return + } + r.logger.Warn("failed to record ingress upload progress", "content_id", r.contentID, "attempt", r.attempt, "error", err) + return + } + if r.publisher == nil || upload.ProgressUpdatedAt == nil { + return + } + r.publisher.Publish("upload_progress_updated", map[string]any{ + "content_id": r.contentID, "task_id": r.taskID, "version_id": r.versionID, + "bucket_name": r.bucketName, "object_key": r.objectKey, + "progress": uploadProgressEventPayload(upload, done), + }) +} + +func uploadProgressEventPayload(upload *model.StorageCopy, done bool) map[string]any { + uploaded := max(upload.IngressBytesTransferred, 0) + total := max(upload.ContentSize, 0) + uploaded = min(uploaded, total) + payload := map[string]any{ + "scope": "ingress_store", "attempt": upload.IngressStoreAttempt, + "uploaded_bytes": uploaded, "total_bytes": total, + "done": done || (total > 0 && uploaded >= total), + "updated_at": upload.ProgressUpdatedAt.Format(time.RFC3339), + } + if percent := model.UploadProgressPercent(uploaded, total); percent != nil { + payload["percent"] = *percent + } + return payload +} diff --git a/internal/worker/ingress_progress_test.go b/internal/worker/ingress_progress_test.go new file mode 100644 index 0000000..f967006 --- /dev/null +++ b/internal/worker/ingress_progress_test.go @@ -0,0 +1,64 @@ +package worker + +import ( + "context" + "log/slog" + "sync/atomic" + "testing" + "time" + + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" +) + +type blockingProgressRepository struct { + repository.StorageContentRepository + entered chan struct{} + release chan struct{} + calls atomic.Int64 +} + +func (r *blockingProgressRepository) RecordIngressStoreProgress(context.Context, repository.RecordIngressStoreProgressInput) (*model.StorageCopy, error) { + r.calls.Add(1) + select { + case r.entered <- struct{}{}: + default: + } + <-r.release + now := time.Now() + return &model.StorageCopy{ContentSize: 10, IngressStoreAttempt: 1, ProgressUpdatedAt: &now}, nil +} + +func TestUploadProgressCloseWaitsForInflightWrites(t *testing.T) { + store := &blockingProgressRepository{entered: make(chan struct{}, 1), release: make(chan struct{})} + reporter := &uploadProgressReporter{ + ctx: t.Context(), repos: &repository.Repositories{Contents: store}, logger: slog.Default(), + contentID: 1, copyID: 2, generation: 3, taskID: 4, attempt: 1, + } + reporter.OnProgress(5) + select { + case <-store.entered: + case <-time.After(time.Second): + t.Fatal("progress write did not start") + } + closed := make(chan struct{}) + go func() { + reporter.Close() + close(closed) + }() + select { + case <-closed: + t.Fatal("Close returned before the in-flight write completed") + case <-time.After(20 * time.Millisecond): + } + close(store.release) + select { + case <-closed: + case <-time.After(time.Second): + t.Fatal("Close did not return after the in-flight write completed") + } + reporter.OnProgress(9) + if store.calls.Load() != 1 { + t.Fatalf("progress writes after Close = %d, want one total write", store.calls.Load()) + } +} diff --git a/internal/worker/manager.go b/internal/worker/manager.go index 82f35a7..c37f86e 100644 --- a/internal/worker/manager.go +++ b/internal/worker/manager.go @@ -2,661 +2,43 @@ package worker import ( "context" - "errors" - "fmt" "log/slog" - "sync" - "time" - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/cacheeviction" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/state" - "github.com/strahe/synaps3/internal/storagereplacement" + "github.com/ethereum/go-ethereum/common" + ethtypes "github.com/ethereum/go-ethereum/core/types" + taskengine "github.com/strahe/synaps3/internal/task" ) -// Worker defines a background processing unit. -type Worker interface { - // Name returns a human-readable identifier. - Name() string - // Run starts the worker loop; it should block until ctx is cancelled. - Run(ctx context.Context) error - // Healthy returns true if the worker has ticked recently. - Healthy() bool +// WalletReceiptChecker observes a previously broadcast wallet transaction. +type WalletReceiptChecker interface { + TransactionReceipt(context.Context, common.Hash) (*ethtypes.Receipt, error) } -// Manager coordinates the lifecycle of all background workers. +// Manager exposes the single task engine through the application's worker +// lifecycle and health contracts. type Manager struct { - repos *repository.Repositories - workers []Worker - logger *slog.Logger - stateMachine *state.Machine - evictionPolicy cache.EvictionPolicy - uploadMaxRetries int - evictMaxRetries int - providerReplacementMaxRetries int - providerReplacementRecovery time.Duration + engine *taskengine.Engine + logger *slog.Logger } -const ( - defaultUploadMaxRetries = 5 - defaultProviderReplacementRecoveryWait = 5 * time.Second - reconcileBatchSize = 100 -) - -const ( - recoveryStagePrepare = "prepare_upload" - recoveryStageEnsureDataSet = "ensure_dataset" - recoveryStageIngressStore = "ingress_store" - recoveryStageIngressCommit = "ingress_commit" - recoveryStagePeerPull = "peer_pull" - recoveryStagePeerCommit = "peer_commit" -) - -// NewManager creates a new worker manager. -func NewManager(repos *repository.Repositories, logger *slog.Logger, evictionPolicy cache.EvictionPolicy, workers ...Worker) *Manager { - return &Manager{ - repos: repos, - workers: workers, - logger: logger, - stateMachine: state.NewObjectStateMachine(), - evictionPolicy: evictionPolicy, - uploadMaxRetries: defaultUploadMaxRetries, - evictMaxRetries: defaultEvictMaxRetries, - providerReplacementMaxRetries: defaultUploadMaxRetries, - providerReplacementRecovery: defaultProviderReplacementRecoveryWait, +func NewManager(engine *taskengine.Engine, logger *slog.Logger) *Manager { + if logger == nil { + logger = slog.Default() } + return &Manager{engine: engine, logger: logger} } -// WithProviderReplacementRecovery configures provider replacement startup recovery. -func (m *Manager) WithProviderReplacementRecovery(maxRetries int, retryInterval time.Duration) *Manager { - m.providerReplacementMaxRetries = maxRetries - if retryInterval > 0 { - m.providerReplacementRecovery = retryInterval - } - return m -} - -// WithTaskMaxRetries configures max retries for tasks recreated during startup reconciliation. -func (m *Manager) WithTaskMaxRetries(uploadMaxRetries, evictMaxRetries int) *Manager { - m.uploadMaxRetries = uploadMaxRetries - m.evictMaxRetries = evictMaxRetries - return m -} - -// Start launches all registered workers and blocks until ctx is cancelled. +// Start runs the universal task engine until ctx is cancelled. func (m *Manager) Start(ctx context.Context) { - if !m.recoverOnStartup(ctx) { + if m == nil || m.engine == nil { return } - - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - m.recoverProviderReplacementQueue(ctx) - }() - - for _, w := range m.workers { - wg.Add(1) - go func(w Worker) { - defer wg.Done() - m.logger.Info("starting worker", "worker", w.Name()) - if err := w.Run(ctx); err != nil { - m.logger.Error("worker exited with error", "worker", w.Name(), "error", err) - } else { - m.logger.Info("worker stopped", "worker", w.Name()) - } - }(w) - } - - wg.Wait() -} - -// recoverOnStartup releases expired task leases and resets objects stuck -// in intermediate states from a previous crash. -func (m *Manager) recoverOnStartup(ctx context.Context) bool { - released, err := m.repos.Tasks.ReleaseExpiredLeases(ctx) - if err != nil { - m.logger.Error("failed to release expired task leases", "error", err) - } else if released > 0 { - m.logger.Info("released expired task leases", "count", released) - } - keepStage := "" - switch m.evictionPolicy { - case cache.EvictionPolicyLRU: - keepStage = cacheeviction.StageLRU - case cache.EvictionPolicyAfterUpload: - keepStage = cacheeviction.StageAfterUpload - } - cancelled, err := m.repos.CacheEvictions.CancelActiveTasksExcept(ctx, keepStage, "Cancelled because the cache eviction policy changed") - if err != nil { - m.logger.Error("failed to cancel incompatible cache eviction tasks", "error", err) - } else if cancelled > 0 { - m.logger.Info("cancelled incompatible cache eviction tasks", "count", cancelled, "policy", m.evictionPolicy) - } - - // Reconcile unfinished upload work. - m.reconcileTasks(ctx, model.ObjectStateCached, model.TaskTypeUpload, "upload") - m.reconcileStagedUploads(ctx) - m.reconcileIncompleteReadableUploads(ctx) - m.reconcileUnavailableDataSets(ctx) - m.reconcileProviderReplacements(ctx) - - // Log exhausted task count for operator awareness - exhaustedTasks, err := m.repos.Tasks.ListExhausted(ctx, 100) - if err != nil { - m.logger.Error("failed to check exhausted tasks", "error", err) - } else if len(exhaustedTasks) > 0 { - m.logger.Warn("exhausted tasks found on startup, review via GET /admin/exhausted-tasks", "count", len(exhaustedTasks)) - } - return ctx.Err() == nil -} - -func (m *Manager) recoverProviderReplacementQueue(ctx context.Context) bool { - for { - itemBudgets, budgetErr := m.repos.Replacements.InitializeReplacementItemRetryBudgets( - ctx, m.providerReplacementMaxRetries, - ) - if budgetErr != nil { - m.logger.Error("failed to initialize provider replacement item retry budgets", "error", budgetErr) - } else if itemBudgets > 0 { - m.logger.Info("initialized provider replacement item retry budgets", "count", itemBudgets) - } - - releasedItems, leaseErr := m.repos.Replacements.ReleaseExpiredItemLeases(ctx) - if leaseErr != nil { - m.logger.Error("failed to release expired provider replacement item leases", "error", leaseErr) - } else if releasedItems > 0 { - m.logger.Info("released expired provider replacement item leases", "count", releasedItems) - } - if budgetErr == nil && leaseErr == nil { - return true - } - m.logger.Warn("provider replacement startup recovery will retry", - "after", m.providerReplacementRecovery) - if !sleepUntilNextWorkerPoll(ctx, m.providerReplacementRecovery) { - return false - } - } -} - -func (m *Manager) reconcileIncompleteReadableUploads(ctx context.Context) { - afterID := int64(0) - for { - items, err := m.repos.Uploads.ListIncompleteReadableUploads(ctx, afterID, reconcileBatchSize) - if err != nil { - m.logger.Error("failed to list incomplete readable uploads for recovery", "error", err) - return - } - for i := range items { - item := &items[i] - m.enqueueRecoveredUploadRepair(ctx, item.Version, item.Upload.ID) - afterID = item.Upload.ID - } - if len(items) < reconcileBatchSize { - return - } - } -} - -func (m *Manager) reconcileUnavailableDataSets(ctx context.Context) { - afterID := int64(0) - for { - bindings, err := m.repos.Uploads.ListUnavailableDataSetsWithIncompleteCopies(ctx, afterID, reconcileBatchSize) - if err != nil { - m.logger.Error("failed to list unavailable data sets for recovery", "error", err) - return - } - for i := range bindings { - binding := &bindings[i] - if _, err := ensureReplicaRepairTask(ctx, m.repos, binding, m.uploadMaxRetries); err != nil { - m.logger.Error("failed to ensure unavailable data set repair", "dataSetID", binding.ID, "error", err) - } - afterID = binding.ID - } - if len(bindings) < reconcileBatchSize { - return - } + m.logger.Info("starting task engine") + if err := m.engine.Run(ctx); err != nil && ctx.Err() == nil { + m.logger.Error("task engine exited", "error", err) } } -// reconcileProviderReplacements re-issues the coordinator for every replacement -// that still has work. Both coordinator keys are idempotent singletons, so this -// is safe to run on every boot, and it also picks up an abandoned target left by -// a superseded confirmation. -func (m *Manager) reconcileProviderReplacements(ctx context.Context) { - afterID := int64(0) - for { - rows, err := m.repos.Replacements.ListActive(ctx, afterID, reconcileBatchSize) - if err != nil { - m.logger.Error("failed to list active provider replacements for recovery", "error", err) - break - } - for i := range rows { - row := &rows[i] - m.enqueueReplacementCoordinator(ctx, row) - afterID = row.ID - } - if len(rows) < reconcileBatchSize { - break - } - } - - afterID = 0 - for { - rows, err := m.repos.Replacements.ListSupersededCleanupCandidates(ctx, afterID, reconcileBatchSize) - if err != nil { - m.logger.Error("failed to list superseded provider replacements for cleanup", "error", err) - return - } - for i := range rows { - row := &rows[i] - // A superseded replacement's leftover is its unused target, not its - // source; the source still belongs to whoever took over the slot. - task := storagereplacement.NewAbandonedTargetTask(row.ID, row.BucketID, m.uploadMaxRetries, time.Now()) - // Leftover paid services have no Data Sets retry: the replacement is - // superseded. Revive a coordinator that failed or exhausted so an - // unused target does not stay reserved across restarts. - if _, err := m.repos.Tasks.ResumeCoordinator(ctx, task); err != nil { - m.logger.Error("failed to ensure superseded replacement cleanup", "replacementID", row.ID, "error", err) - } - afterID = row.ID - } - if len(rows) < reconcileBatchSize { - return - } - } -} - -// A replacement resumes through whichever coordinator owns its current phase. -func (m *Manager) enqueueReplacementCoordinator(ctx context.Context, row *storagereplacement.Replacement) { - var task *model.Task - // A recorded termination epoch means the old service has already been asked - // to end, so this replacement is past migration even if a wait moved it out - // of the retiring status. - if row.Status == storagereplacement.StatusRetiring || row.TerminationEpoch != nil { - task = storagereplacement.NewRetireTask(row.ID, row.BucketID, m.uploadMaxRetries, time.Now()) - } else { - task = storagereplacement.NewMigrateTask(row.ID, row.BucketID, "", m.uploadMaxRetries, time.Now()) - } - if _, err := m.repos.Tasks.EnsureRecurring(ctx, task); err != nil { - m.logger.Error("failed to ensure provider replacement coordinator", "replacementID", row.ID, "error", err) - } -} - -// reconcileTasks finds object versions in the given state and ensures each has a corresponding -// queued task. Uses idempotency keys to safely skip objects that already have tasks. -// keyPrefix must match the prefix used by the normal task creation path for deduplication. -func (m *Manager) reconcileTasks(ctx context.Context, objState model.ObjectState, taskType model.TaskType, keyPrefix string) { - created := 0 - var cursor versionStateCursor - for { - versions, nextCursor, err := m.listVersionStateBatch(ctx, objState, cursor) - if err != nil { - return - } - if len(versions) == 0 { - break - } - for _, version := range versions { - stage := taskStageForReconcile(taskType) - task := &model.Task{ - Type: taskType, - Stage: stage, - RefType: "object", - RefID: version.ObjectID, - RefVersionID: version.VersionID, - IdempotencyKey: fmt.Sprintf("%s:%s", keyPrefix, version.VersionID), - Status: model.TaskStatusQueued, - MaxRetries: m.maxRetriesForTaskType(taskType), - ScheduledAt: time.Now(), - } - if err := m.repos.Tasks.Create(ctx, task); err != nil { - // Idempotency key collision means task already exists — skip - continue - } - created++ - } - if len(versions) < reconcileBatchSize { - break - } - cursor = nextCursor - } - if created > 0 { - m.logger.Info("reconciled missing tasks", "state", objState, "type", taskType, "created", created) - } -} - -type versionStateCursor struct { - updatedAt time.Time - versionID string -} - -func (m *Manager) listVersionStateBatch(ctx context.Context, objState model.ObjectState, cursor versionStateCursor) ([]model.ObjectVersion, versionStateCursor, error) { - versions, err := m.repos.Objects.ListVersionsByStateAfter(ctx, objState, cursor.updatedAt, cursor.versionID, reconcileBatchSize) - if err != nil { - m.logger.Error("failed to list object versions for reconciliation", "state", objState, "error", err) - return nil, cursor, err - } - if len(versions) == 0 { - return nil, cursor, nil - } - last := versions[len(versions)-1] - return versions, versionStateCursor{updatedAt: last.UpdatedAt, versionID: last.VersionID}, nil -} - -func taskStageForReconcile(taskType model.TaskType) *string { - if taskType != model.TaskTypeUpload { - return nil - } - stage := recoveryStagePrepare - return &stage -} - -func (m *Manager) reconcileStagedUploads(ctx context.Context) { - for _, objState := range []model.ObjectState{ - model.ObjectStateUploading, - model.ObjectStateCommitting, - model.ObjectStateReplicating, - } { - var cursor versionStateCursor - for { - versions, nextCursor, err := m.listVersionStateBatch(ctx, objState, cursor) - if err != nil { - break - } - if len(versions) == 0 { - break - } - for _, version := range versions { - upload, err := m.recoverableUploadForVersion(ctx, version) - if err != nil { - m.logger.Error("failed to load staged upload for reconciliation", "versionID", version.VersionID, "error", err) - continue - } - if upload == nil { - m.reconcileOrphanStagedVersion(ctx, version) - continue - } - if version.State == model.ObjectStateReplicating { - m.reconcileReplicatingUpload(ctx, version, upload) - continue - } - m.reconcileIngressUpload(ctx, version, upload) - } - if len(versions) < reconcileBatchSize { - break - } - cursor = nextCursor - } - } -} - -func (m *Manager) reconcileOrphanStagedVersion(ctx context.Context, version model.ObjectVersion) { - switch version.State { - case model.ObjectStateUploading: - m.enqueueRecoveredUploadStage(ctx, version, 0, recoveryStagePrepare, 0, "") - case model.ObjectStateCommitting: - if err := state.TransitionState(ctx, m.stateMachine, m.repos.Objects, version.VersionID, model.ObjectStateCommitting, model.ObjectStateUploading); err != nil { - m.logger.Error("failed to reset orphan committing version", "versionID", version.VersionID, "error", err) - return - } - version.State = model.ObjectStateUploading - m.enqueueRecoveredUploadStage(ctx, version, 0, recoveryStagePrepare, 0, "") - case model.ObjectStateReplicating: - m.logger.Warn("replicating version has no recoverable storage upload", "versionID", version.VersionID) - } -} - -func (m *Manager) recoverableUploadForVersion(ctx context.Context, version model.ObjectVersion) (*model.StorageUpload, error) { - if version.StorageUploadID != nil { - upload, err := m.repos.Uploads.GetByID(ctx, *version.StorageUploadID) - if err != nil || upload == nil { - return upload, err - } - if isRecoverableUploadStatus(upload.Status) { - return upload, nil - } - return nil, nil - } - return m.repos.Uploads.FindActiveUploadBySourceVersion(ctx, version.VersionID) -} - -func isRecoverableUploadStatus(status model.StorageUploadStatus) bool { - switch status { - case model.StorageUploadStatusRunning, - model.StorageUploadStatusIngressReady, - model.StorageUploadStatusReadable, - model.StorageUploadStatusComplete: - return true - default: - return false - } -} - -func (m *Manager) reconcileIngressUpload(ctx context.Context, version model.ObjectVersion, upload *model.StorageUpload) { - copies, err := m.repos.Uploads.ListCopies(ctx, upload.ID) - if err != nil { - m.logger.Error("failed to list ingress upload copies for reconciliation", "uploadID", upload.ID, "error", err) - return - } - if len(copies) == 0 { - m.enqueueRecoveredUploadStage(ctx, version, upload.ID, recoveryStagePrepare, 0, "") - return - } - var ingress *model.StorageUploadCopy - var committedIngress *model.StorageUploadCopy - for i := range copies { - if copies[i].TransferMethod != model.StorageCopyTransferMethodIngress { - continue - } - if copyCommitted(&copies[i]) { - committedIngress = &copies[i] - continue - } - ingress = &copies[i] - break - } - if ingress == nil { - if committedIngress != nil { - binding, err := m.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, version.BucketID, committedIngress.CopyIndex) - if err != nil { - m.logger.Error("failed to load committed ingress dataset binding for reconciliation", "uploadID", upload.ID, "copyIndex", committedIngress.CopyIndex, "error", err) - return - } - if binding != nil && (dataSetBindingUnavailable(binding) || dataSetBindingWriteBlocked(binding)) { - m.enqueueRecoveredUploadStage(ctx, version, upload.ID, recoveryStageIngressCommit, committedIngress.CopyIndex, committedIngress.TransferMethod) - return - } - } - m.reconcileReplicatingUpload(ctx, version, upload) - return - } - binding, err := m.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, version.BucketID, ingress.CopyIndex) - if err != nil { - m.logger.Error("failed to load ingress dataset binding for reconciliation", "uploadID", upload.ID, "copyIndex", ingress.CopyIndex, "error", err) - return - } - if binding == nil { - m.enqueueRecoveredUploadStage(ctx, version, upload.ID, recoveryStageEnsureDataSet, ingress.CopyIndex, ingress.TransferMethod) - return - } - if binding.Status == model.StorageDataSetStatusUnavailable { - if _, err := ensureReplicaRepairTask(ctx, m.repos, binding, m.uploadMaxRetries); err != nil { - m.logger.Error("failed to ensure recovered ingress repair", "dataSetID", binding.ID, "error", err) - } - } - if copyCommitSubmitted(ingress) && (dataSetBindingUnavailable(binding) || dataSetBindingWriteBlocked(binding)) { - m.enqueueRecoveredUploadStage(ctx, version, upload.ID, recoveryStageIngressCommit, ingress.CopyIndex, ingress.TransferMethod) - return - } - if dataSetBindingUnavailable(binding) || dataSetBindingWriteBlocked(binding) { - reassigned, reassignErr := reassignIngressCopyAndSchedule(ctx, m.repos, m.stateMachine, &version, upload.ID, ingress.CopyIndex, m.uploadMaxRetries, nil) - if reassignErr != nil { - m.logger.Error("failed to reassign recovered ingress copy", "uploadID", upload.ID, "copyIndex", ingress.CopyIndex, "error", reassignErr) - return - } - if reassigned != nil { - return - } - m.enqueueRecoveredUploadStage(ctx, version, upload.ID, recoveryStageEnsureDataSet, ingress.CopyIndex, ingress.TransferMethod) - return - } - if !uploadCanUseDataSetBinding(upload.ID, binding) { - m.enqueueRecoveredUploadStage(ctx, version, upload.ID, recoveryStageEnsureDataSet, ingress.CopyIndex, ingress.TransferMethod) - return - } - if version.State == model.ObjectStateCommitting && copyHasPiece(ingress) { - m.enqueueRecoveredUploadStage(ctx, version, upload.ID, recoveryStageIngressCommit, ingress.CopyIndex, ingress.TransferMethod) - return - } - m.enqueueRecoveredUploadStage(ctx, version, upload.ID, recoveryStageIngressStore, ingress.CopyIndex, ingress.TransferMethod) -} - -func (m *Manager) reconcileReplicatingUpload(ctx context.Context, version model.ObjectVersion, upload *model.StorageUpload) { - copies, err := m.repos.Uploads.ListCopies(ctx, upload.ID) - if err != nil { - m.logger.Error("failed to list peer upload copies for reconciliation", "uploadID", upload.ID, "error", err) - return - } - if len(copies) == 0 { - return - } - readableCopies, err := m.repos.Uploads.ListReadableCommittedCopies(ctx, upload.ID) - if err != nil { - m.logger.Error("failed to list readable upload copies for reconciliation", "uploadID", upload.ID, "error", err) - return - } - if len(readableCopies) > 0 && !versionBoundToUpload(version, upload.ID) { - _, err := m.repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: version.BucketID, - ContentSize: version.Size, - Checksum: version.Checksum, - }) - if err != nil { - m.logger.Error("failed to bind recovered readable upload", "uploadID", upload.ID, "versionID", version.VersionID, "error", err) - return - } - version.State = model.ObjectStateReplicating - version.StorageUploadID = &upload.ID - } - finalized, _, err := m.repos.Uploads.FinalizeUploadIfTargetCopiesMet( - ctx, - repository.NewFinalizeUploadInput( - upload.ID, - m.evictionPolicy.EnqueuesAfterUploadEviction(), - m.evictMaxRetries, - ), - ) - if err != nil { - m.logger.Error("failed to finalize recovered upload", "uploadID", upload.ID, "error", err) - return - } - if finalized { - return - } - assignedSlots := make(map[int]struct{}, len(copies)) - for i := range copies { - copyRow := &copies[i] - assignedSlots[copyRow.CopyIndex] = struct{}{} - if copyRow.TransferMethod != model.StorageCopyTransferMethodPeerPull || copyCommitted(copyRow) || copyRow.Status == model.StorageUploadCopyStatusFailed { - continue - } - stage := recoveryStageEnsureDataSet - binding, err := m.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, version.BucketID, copyRow.CopyIndex) - if err != nil { - m.logger.Error("failed to load peer dataset binding for reconciliation", "uploadID", upload.ID, "copyIndex", copyRow.CopyIndex, "error", err) - continue - } - if binding != nil && binding.Status == model.StorageDataSetStatusUnavailable { - if _, err := ensureReplicaRepairTask(ctx, m.repos, binding, m.uploadMaxRetries); err != nil { - m.logger.Error("failed to ensure recovered peer repair", "dataSetID", binding.ID, "error", err) - } - continue - } - if !uploadCanUseDataSetBinding(upload.ID, binding) { - continue - } - if binding != nil && binding.Status == model.StorageDataSetStatusReady { - stage = recoveryStagePeerPull - if copyHasPiece(copyRow) { - stage = recoveryStagePeerCommit - } - } - m.enqueueRecoveredUploadStage(ctx, version, upload.ID, stage, copyRow.CopyIndex, copyRow.TransferMethod) - } - if upload.RequestedCopies > len(assignedSlots) && len(readableCopies) > 0 { - m.enqueueRecoveredUploadRepair(ctx, version, upload.ID) - } -} - -func versionBoundToUpload(version model.ObjectVersion, uploadID int64) bool { - return version.State == model.ObjectStateReplicating && version.StorageUploadID != nil && *version.StorageUploadID == uploadID -} - -func (m *Manager) enqueueRecoveredUploadStage(ctx context.Context, version model.ObjectVersion, uploadID int64, stage string, copyIndex int, transferMethod model.StorageCopyTransferMethod) { - payload := map[string]interface{}{"upload_id": uploadID} - key := fmt.Sprintf("upload:%s:%s:%d", version.VersionID, stage, uploadID) - if stage == recoveryStagePrepare { - payload = nil - key = fmt.Sprintf("upload:%s", version.VersionID) - } - if transferMethod != "" { - payload["copy_index"] = copyIndex - payload["transfer_method"] = string(transferMethod) - key = fmt.Sprintf("%s:%d", key, copyIndex) - } - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: version.ObjectID, - RefVersionID: version.VersionID, - IdempotencyKey: key, - Payload: payload, - Status: model.TaskStatusQueued, - MaxRetries: m.uploadMaxRetries, - ScheduledAt: time.Now(), - } - if err := m.repos.Tasks.Create(ctx, task); err != nil && !errors.Is(err, repository.ErrAlreadyExists) { - m.logger.Error("failed to enqueue recovered upload stage", "stage", stage, "uploadID", uploadID, "versionID", version.VersionID, "error", err) - } -} - -func (m *Manager) enqueueRecoveredUploadRepair(ctx context.Context, version model.ObjectVersion, uploadID int64) { - stage := recoveryStagePrepare - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: version.ObjectID, - RefVersionID: version.VersionID, - IdempotencyKey: fmt.Sprintf("upload:%s:%s:%d:repair", version.VersionID, stage, uploadID), - Payload: map[string]interface{}{"upload_id": uploadID}, - Status: model.TaskStatusQueued, - MaxRetries: m.uploadMaxRetries, - ScheduledAt: time.Now(), - } - if _, err := m.repos.Tasks.EnsureRecurring(ctx, task); err != nil { - m.logger.Error("failed to enqueue recovered upload repair", "uploadID", uploadID, "versionID", version.VersionID, "error", err) - } -} - -func (m *Manager) maxRetriesForTaskType(taskType model.TaskType) int { - if taskType == model.TaskTypeEvictCache { - return m.evictMaxRetries - } - return m.uploadMaxRetries -} - -// WorkerHealth returns a map of worker name → healthy status. func (m *Manager) WorkerHealth() map[string]bool { - health := make(map[string]bool, len(m.workers)) - for _, w := range m.workers { - health[w.Name()] = w.Healthy() - } - return health + return map[string]bool{"tasks": m != nil && m.engine != nil && m.engine.Healthy()} } diff --git a/internal/worker/manager_test.go b/internal/worker/manager_test.go deleted file mode 100644 index aab7c76..0000000 --- a/internal/worker/manager_test.go +++ /dev/null @@ -1,1462 +0,0 @@ -package worker_test - -import ( - "context" - "errors" - "fmt" - "log/slog" - "sync/atomic" - "testing" - "time" - - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/cacheaccess" - "github.com/strahe/synaps3/internal/cacheeviction" - "github.com/strahe/synaps3/internal/config" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/state" - "github.com/strahe/synaps3/internal/testutil" - "github.com/strahe/synaps3/internal/worker" -) - -type flakyReplacementRecoveryRepo struct { - repository.StorageReplacementRepository - initializeCalls atomic.Int32 - releaseCalls atomic.Int32 - retryAllowed <-chan struct{} - recoveryComplete chan<- struct{} -} - -func (r *flakyReplacementRecoveryRepo) InitializeReplacementItemRetryBudgets( - ctx context.Context, - maxRetries int, -) (int, error) { - if r.initializeCalls.Add(1) == 1 { - return 0, errors.New("injected retry budget recovery failure") - } - if r.retryAllowed != nil { - select { - case <-ctx.Done(): - return 0, ctx.Err() - case <-r.retryAllowed: - } - } - return r.StorageReplacementRepository.InitializeReplacementItemRetryBudgets(ctx, maxRetries) -} - -func (r *flakyReplacementRecoveryRepo) ReleaseExpiredItemLeases(ctx context.Context) (int, error) { - if r.releaseCalls.Add(1) == 1 { - return 0, errors.New("injected item lease recovery failure") - } - count, err := r.StorageReplacementRepository.ReleaseExpiredItemLeases(ctx) - if err == nil && r.recoveryComplete != nil { - close(r.recoveryComplete) - } - return count, err -} - -func TestManager_RecoverOnStartup_ReleasesExpiredLeases(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "mgr-lease-bucket") - objID, versionID := seedManagerVersion(t, repos, bucket, "mgr-lease-key", model.ObjectStateCached) - - // Create a task and claim it, then manually set its lease_until to the past - task := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "object", - RefID: objID, - RefVersionID: versionID, - IdempotencyKey: "upload:" + versionID, - Status: model.TaskStatusRunning, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("creating task: %v", err) - } - - // Set lease_until and claimed_at to past (expired lease) - pastLease := time.Now().Add(-1 * time.Hour) - _, err := db.NewUpdate().Model((*model.Task)(nil)). - Set("claimed_at = ?", pastLease). - Set("lease_until = ?", pastLease). - Set("started_at = ?", pastLease). - Where("id = ?", task.ID). - Exec(ctx) - if err != nil { - t.Fatalf("updating lease: %v", err) - } - - // Start manager with no workers — returns immediately after recovery - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone) - mgr.Start(ctx) - - // Task should be back to queued. - got, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil { - t.Fatalf("getting task: %v", err) - } - if got.Status != model.TaskStatusQueued { - t.Errorf("expected task status queued after lease release, got %s", got.Status) - } -} - -func TestManager_RecoverOnStartup_PreservesActiveLeases(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "mgr-active-lease-bucket") - objID, versionID := seedManagerVersion(t, repos, bucket, "mgr-active-lease-key", model.ObjectStateCached) - now := time.Now() - leaseUntil := now.Add(10 * time.Minute) - task := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "object", - RefID: objID, - RefVersionID: versionID, - IdempotencyKey: "upload:" + versionID, - Status: model.TaskStatusRunning, - MaxRetries: 5, - ScheduledAt: now, - ClaimedAt: &now, - LeaseUntil: &leaseUntil, - StartedAt: &now, - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create running task: %v", err) - } - - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone) - mgr.Start(ctx) - - got, err := repos.Tasks.GetByID(ctx, task.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.Status != model.TaskStatusRunning { - t.Fatalf("task status = %s, want running", got.Status) - } - if got.LeaseUntil == nil || !got.LeaseUntil.After(now) { - t.Fatalf("lease_until = %v, want active lease after %s", got.LeaseUntil, now) - } -} - -func TestManager_RetriesProviderReplacementRecoveryWithoutBlockingWorkers(t *testing.T) { - repos := testutil.NewTestRepos(t) - retryAllowed := make(chan struct{}) - recoveryComplete := make(chan struct{}) - flaky := &flakyReplacementRecoveryRepo{ - StorageReplacementRepository: repos.Replacements, - retryAllowed: retryAllowed, - recoveryComplete: recoveryComplete, - } - repos.Replacements = flaky - var workerRuns atomic.Int32 - workerStarted := make(chan struct{}) - managed := &stubWorker{ - name: "independent-of-recovery", - run: func(ctx context.Context) error { - workerRuns.Add(1) - close(workerStarted) - <-ctx.Done() - return ctx.Err() - }, - } - - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone, managed). - WithProviderReplacementRecovery(5, time.Millisecond). - Start(ctx) - close(done) - }() - - select { - case <-workerStarted: - case <-time.After(time.Second): - t.Fatal("worker did not start while provider replacement recovery was retrying") - } - close(retryAllowed) - select { - case <-recoveryComplete: - case <-time.After(time.Second): - t.Fatal("provider replacement recovery did not complete after retry") - } - cancel() - select { - case <-done: - case <-time.After(time.Second): - t.Fatal("manager did not stop after cancellation") - } - - if flaky.initializeCalls.Load() != 2 || flaky.releaseCalls.Load() != 2 { - t.Fatalf("recovery calls = initialize:%d release:%d, want two attempts each", - flaky.initializeCalls.Load(), flaky.releaseCalls.Load()) - } - if workerRuns.Load() != 1 { - t.Fatalf("worker runs = %d, want one", workerRuns.Load()) - } -} - -func TestManager_RecoverOnStartup_DoesNotResetStagedUploadingState(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "mgr-stale-bucket") - objID, versionID := seedManagerVersion(t, repos, bucket, "mgr-stale-key", model.ObjectStateCached) - - // Transition to uploading - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("transition: %v", err) - } - - // Manually set updated_at to more than 10 minutes ago (stale threshold) - staleTime := time.Now().Add(-15 * time.Minute) - _, err := db.NewUpdate().Model((*model.ObjectVersion)(nil)). - Set("updated_at = ?", staleTime). - Where("version_id = ?", versionID). - Exec(ctx) - if err != nil { - t.Fatalf("setting stale timestamp: %v", err) - } - - // Start manager. Staged uploads keep durable progress in upload/copy rows, - // so startup must not downgrade uploading back to cached. - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone).WithTaskMaxRetries(9, 4) - mgr.Start(ctx) - - got, err := repos.Objects.GetCurrentVersionByObjectID(ctx, objID) - if err != nil { - t.Fatalf("getting object: %v", err) - } - if got.State != model.ObjectStateUploading { - t.Errorf("expected object to remain uploading, got %s", got.State) - } - tasks, total, err := repos.Tasks.List(ctx, string(model.TaskTypeUpload), "prepare_upload", string(model.TaskStatusQueued), 10, 0) - if err != nil { - t.Fatalf("List prepare_upload tasks: %v", err) - } - if total != 1 || len(tasks) != 1 { - t.Fatalf("prepare_upload tasks total=%d tasks=%#v, want one orphan recovery task", total, tasks) - } - if tasks[0].RefID != objID || tasks[0].RefVersionID != versionID || tasks[0].MaxRetries != 9 { - t.Fatalf("prepare_upload task = %#v, want recovered task for uploading version", tasks[0]) - } -} - -func TestManager_RecoverOnStartup_RequeuesOrphanCommittingState(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "mgr-orphan-commit-bucket") - objID, versionID := seedManagerVersion(t, repos, bucket, "mgr-orphan-commit-key", model.ObjectStateCached) - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("committing: %v", err) - } - - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone).WithTaskMaxRetries(9, 4) - mgr.Start(ctx) - - got, err := repos.Objects.GetCurrentVersionByObjectID(ctx, objID) - if err != nil { - t.Fatalf("getting object: %v", err) - } - if got.State != model.ObjectStateUploading { - t.Fatalf("object state = %s, want uploading", got.State) - } - tasks, total, err := repos.Tasks.List(ctx, string(model.TaskTypeUpload), "prepare_upload", string(model.TaskStatusQueued), 10, 0) - if err != nil { - t.Fatalf("List prepare_upload tasks: %v", err) - } - if total != 1 || len(tasks) != 1 { - t.Fatalf("prepare_upload tasks total=%d tasks=%#v, want one orphan recovery task", total, tasks) - } - if tasks[0].RefID != objID || tasks[0].RefVersionID != versionID { - t.Fatalf("prepare_upload task = %#v, want recovered task for committing version", tasks[0]) - } -} - -func TestManager_RecoverOnStartup_ReenqueuesPrimaryCommitStage(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "mgr-primary-commit-recover") - objID, versionID := seedManagerVersion(t, repos, bucket, "recover-primary-commit", model.ObjectStateCached) - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("committing: %v", err) - } - version, err := repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - primary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: primary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "bafk2bzaceprimaryrecover", - RetrievalURL: "https://primary.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone).WithTaskMaxRetries(9, 4) - mgr.Start(ctx) - - tasks, total, err := repos.Tasks.List(ctx, string(model.TaskTypeUpload), "ingress_commit", string(model.TaskStatusQueued), 10, 0) - if err != nil { - t.Fatalf("List ingress_commit tasks: %v", err) - } - if total != 1 || len(tasks) != 1 { - t.Fatalf("ingress_commit tasks total=%d tasks=%#v, want one", total, tasks) - } - if tasks[0].RefID != objID || tasks[0].RefVersionID != versionID || tasks[0].Payload["upload_id"] == nil { - t.Fatalf("ingress_commit task = %#v, want recovered task for source version", tasks[0]) - } -} - -func TestManager_RecoverOnStartup_BindsCommittedIngressBeforeSingleCopyFinalize(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "mgr-single-copy-commit-recover") - objID, versionID := seedManagerVersion(t, repos, bucket, "recover-single-copy", model.ObjectStateCached) - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("committing: %v", err) - } - version, err := repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "bafk2bzacesinglecopyrecover", - PieceID: onChainIDPtr(t, "301"), - RetrievalURL: "https://ingress.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone).WithTaskMaxRetries(9, 4) - mgr.Start(ctx) - - got, err := repos.Objects.GetCurrentVersionByObjectID(ctx, objID) - if err != nil || got == nil { - t.Fatalf("GetCurrentVersionByObjectID: got=%v err=%v", got, err) - } - if got.State != model.ObjectStateStored || got.StorageUploadID == nil || *got.StorageUploadID != upload.ID { - t.Fatalf("object after recovery = state:%s upload:%v, want stored on recovered upload %d", got.State, got.StorageUploadID, upload.ID) - } - gotUpload, err := repos.Uploads.GetByID(ctx, upload.ID) - if err != nil || gotUpload == nil { - t.Fatalf("GetByID(upload): upload=%v err=%v", gotUpload, err) - } - if gotUpload.Status != model.StorageUploadStatusComplete { - t.Fatalf("upload status = %s, want complete", gotUpload.Status) - } -} - -func TestManager_RecoverOnStartup_RequeuesCommittedIngressOnUnavailableDataSet(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "mgr-unavailable-committed-ingress") - objID, versionID := seedManagerVersion(t, repos, bucket, "recover-unavailable-committed-ingress", model.ObjectStateCached) - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("committing: %v", err) - } - version, err := repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: versionID, ContentSize: version.Size, Checksum: version.Checksum, RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001")}); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, CopyIndex: 0, PieceCID: "bafk2bzaceunavailablecommitted", PieceID: onChainIDPtr(t, "301"), RetrievalURL: "https://ingress.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - if err := repos.Uploads.MarkDataSetUnavailable(ctx, binding.ID, "provider unavailable after commit"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone).WithTaskMaxRetries(9, 4) - mgr.Start(ctx) - - tasks, total, err := repos.Tasks.List(ctx, string(model.TaskTypeUpload), "ingress_commit", string(model.TaskStatusQueued), 10, 0) - if err != nil || total != 1 { - t.Fatalf("ingress commit tasks = %#v total=%d err=%v, want one recovery task", tasks, total, err) - } - if tasks[0].RefID != objID || tasks[0].RefVersionID != versionID || taskPayloadInt64ForTest(tasks[0].Payload, "copy_index") != 0 { - t.Fatalf("ingress commit task = %#v, want exact committed copy", tasks[0]) - } - got, err := repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || got == nil || got.State != model.ObjectStateCommitting { - t.Fatalf("version before provider proof = %#v err=%v, want committing", got, err) - } -} - -func TestManager_RecoverOnStartup_MakesExpiredPrimaryCommitTaskClaimable(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "mgr-primary-commit-interrupted") - objID, versionID := seedManagerVersion(t, repos, bucket, "interrupted-primary-commit", model.ObjectStateCached) - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("committing: %v", err) - } - version, err := repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - primary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: primary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "bafk2bzaceprimaryinterrupted", - RetrievalURL: "https://primary.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - stage := "ingress_commit" - now := time.Now() - leaseUntil := now.Add(-time.Minute) - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objID, - RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:ingress_commit:%d", versionID, upload.ID), - Payload: map[string]interface{}{"upload_id": upload.ID, "copy_index": 0, "transfer_method": string(model.StorageCopyTransferMethodIngress)}, - Status: model.TaskStatusRunning, - MaxRetries: 9, - ScheduledAt: now, - ClaimedAt: &now, - LeaseUntil: &leaseUntil, - StartedAt: &now, - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create running task: %v", err) - } - - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone).WithTaskMaxRetries(9, 4) - mgr.Start(ctx) - - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil { - t.Fatalf("ClaimReady: %v", err) - } - if claimed == nil { - t.Fatal("expired ingress_commit task was not claimable after startup recovery") - } - if claimed.ID != task.ID { - t.Fatalf("claimed task ID = %d, want expired task %d", claimed.ID, task.ID) - } -} - -func TestManager_RecoverOnStartup_ReenqueuesReplicatingSecondaryStage(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "mgr-secondary-recover") - objID, versionID := seedManagerVersion(t, repos, bucket, "recover-secondary", model.ObjectStateCached) - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("committing: %v", err) - } - version, err := repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - primary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("primary binding: %v", err) - } - secondary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "202"), CopyIndex: 1, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("secondary binding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: primary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - {StorageDataSetID: secondary.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "bafk2bzacesecondaryrecover", - PieceID: onChainIDPtr(t, "301"), - RetrievalURL: "https://primary.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: version.Size, - Checksum: version.Checksum, - }); err != nil { - t.Fatalf("BindReadableUploadForContent: %v", err) - } - - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone).WithTaskMaxRetries(9, 4) - mgr.Start(ctx) - - tasks, total, err := repos.Tasks.List(ctx, string(model.TaskTypeUpload), "ensure_dataset", string(model.TaskStatusQueued), 10, 0) - if err != nil { - t.Fatalf("List ensure_dataset tasks: %v", err) - } - if total != 1 || len(tasks) != 1 { - t.Fatalf("ensure_dataset tasks total=%d tasks=%#v, want one secondary recovery task", total, tasks) - } - if tasks[0].RefID != objID || tasks[0].RefVersionID != versionID || tasks[0].Payload["copy_index"] == nil { - t.Fatalf("ensure_dataset task = %#v, want recovered secondary task", tasks[0]) - } -} - -func TestManager_RecoverOnStartup_QueuesSingletonRepairIndependentOfObjectState(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "mgr-peer-unavailable-recover") - _, versionID := seedManagerVersion(t, repos, bucket, "recover-peer-unavailable", model.ObjectStateCached) - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("committing: %v", err) - } - version, err := repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - ingress, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("ingress binding: %v", err) - } - peer, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "202"), CopyIndex: 1, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("peer binding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: ingress.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}); err != nil { - t.Fatalf("MarkDataSetReady ingress: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: peer.ID, UploadID: upload.ID, DataSetID: onChainID(t, "2002"), ClientDataSetID: onChainIDPtr(t, "9002")}); err != nil { - t.Fatalf("MarkDataSetReady peer: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: ingress.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - {StorageDataSetID: peer.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "bafk2bzacepeerdeficit", - PieceID: onChainIDPtr(t, "301"), - RetrievalURL: "https://ingress.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: version.Size, - Checksum: version.Checksum, - }); err != nil { - t.Fatalf("BindReadableUploadForContent: %v", err) - } - if err := repos.Uploads.MarkDataSetUnavailable(ctx, peer.ID, "provider dataset retired"); err != nil { - t.Fatalf("MarkDataSetUnavailable peer: %v", err) - } - secondUpload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: "01J000000000000000WAITING2", ContentSize: version.Size, Checksum: "second-waiting-copy", RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt second: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, secondUpload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: peer.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings second: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateReplicating, model.ObjectStateStored); err != nil { - t.Fatalf("mark stored while peer remains incomplete: %v", err) - } - - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone).WithTaskMaxRetries(0, 4) - mgr.Start(ctx) - mgr.Start(ctx) - - ensureTasks, ensureTotal, err := repos.Tasks.List(ctx, string(model.TaskTypeUpload), "ensure_dataset", string(model.TaskStatusQueued), 10, 0) - if err != nil { - t.Fatalf("List ensure_dataset tasks: %v", err) - } - if ensureTotal != 0 || len(ensureTasks) != 0 { - t.Fatalf("ensure_dataset tasks total=%d tasks=%#v, want no recovery for unavailable peer", ensureTotal, ensureTasks) - } - tasks, total, err := repos.Tasks.List(ctx, string(model.TaskTypeUpload), "repair_replica", string(model.TaskStatusQueued), 10, 0) - if err != nil { - t.Fatalf("List repair replica tasks: %v", err) - } - if total != 1 || len(tasks) != 1 { - t.Fatalf("repair_replica tasks total=%d tasks=%#v, want one in-place repair task after repeated recovery", total, tasks) - } - if tasks[0].MaxRetries != 0 { - t.Fatalf("repair task MaxRetries = %d, want configured zero", tasks[0].MaxRetries) - } - peerCopy, err := repos.Uploads.GetUploadCopy(ctx, upload.ID, 1) - if err != nil || peerCopy == nil { - t.Fatalf("GetUploadCopy peer: copy=%v err=%v", peerCopy, err) - } - if tasks[0].RefType != "bucket" || tasks[0].RefID != bucket.ID || tasks[0].RefVersionID != versionID || - taskPayloadInt64ForTest(tasks[0].Payload, "storage_data_set_id") != peer.ID || - taskPayloadInt64ForTest(tasks[0].Payload, "storage_upload_copy_id") != peerCopy.ID { - t.Fatalf("repair task = %#v, want data set %d copy %d", tasks[0], peer.ID, peerCopy.ID) - } -} - -func TestManager_RecoverOnStartup_DoesNotReplaceAssignedFailedPeer(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "mgr-peer-deficit-with-pending-recover") - _, versionID := seedManagerVersion(t, repos, bucket, "recover-peer-deficit-pending", model.ObjectStateCached) - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("committing: %v", err) - } - version, err := repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - ingress, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("ingress binding: %v", err) - } - pendingPeer, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "202"), CopyIndex: 1, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("pending peer binding: %v", err) - } - failedPeer, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "303"), CopyIndex: 2, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("failed peer binding: %v", err) - } - for _, input := range []repository.MarkDataSetReadyInput{ - {ID: ingress.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}, - {ID: pendingPeer.ID, UploadID: upload.ID, DataSetID: onChainID(t, "2002"), ClientDataSetID: onChainIDPtr(t, "9002")}, - {ID: failedPeer.ID, UploadID: upload.ID, DataSetID: onChainID(t, "3003"), ClientDataSetID: onChainIDPtr(t, "9003")}, - } { - if err := repos.Uploads.MarkDataSetReady(ctx, input); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: ingress.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - {StorageDataSetID: pendingPeer.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202")}, - {StorageDataSetID: failedPeer.ID, CopyIndex: 2, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "303")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "bafk2bzacepeerdeficitpending", - PieceID: onChainIDPtr(t, "301"), - RetrievalURL: "https://ingress.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: version.Size, - Checksum: version.Checksum, - }); err != nil { - t.Fatalf("BindReadableUploadForContent: %v", err) - } - if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: upload.ID, CopyIndex: 2, LastError: "peer pull: provider failed"}); err != nil { - t.Fatalf("MarkUploadCopyFailed: %v", err) - } - - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone).WithTaskMaxRetries(9, 4) - mgr.Start(ctx) - - ensureTasks, ensureTotal, err := repos.Tasks.List(ctx, string(model.TaskTypeUpload), "peer_pull", string(model.TaskStatusQueued), 10, 0) - if err != nil { - t.Fatalf("List peer_pull tasks: %v", err) - } - if ensureTotal != 1 || len(ensureTasks) != 1 || taskPayloadInt64ForTest(ensureTasks[0].Payload, "copy_index") != 1 { - t.Fatalf("peer_pull tasks total=%d tasks=%#v, want pending peer recovery", ensureTotal, ensureTasks) - } - repairTasks, repairTotal, err := repos.Tasks.List(ctx, string(model.TaskTypeUpload), "prepare_upload", string(model.TaskStatusQueued), 10, 0) - if err != nil { - t.Fatalf("List repair prepare tasks: %v", err) - } - if repairTotal != 0 || len(repairTasks) != 0 { - t.Fatalf("prepare_upload tasks total=%d tasks=%#v, want no replacement for an assigned failed slot", repairTotal, repairTasks) - } -} - -func TestManager_RecoverOnStartup_RequeuesIncompleteReadableUploadForStoredVersion(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := testutil.SeedBucket(t, db, "mgr-stored-incomplete-recover") - minimum := 1 - if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimum, - }); err != nil { - t.Fatalf("UpdateCopyPolicy: %v", err) - } - _, versionID := seedManagerVersion(t, repos, bucket, "stored-incomplete", model.ObjectStateCached) - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - version, err := repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%#v err=%v", version, err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: upload.ID, - DataSetID: onChainID(t, "1001"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: onChainID(t, "101"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "bafk2bzacemgrstoredrepair", - PieceID: onChainIDPtr(t, "2001"), - RetrievalURL: "https://provider.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: version.Size, - Checksum: version.Checksum, - }); err != nil { - t.Fatalf("BindReadableUploadForContent: %v", err) - } - if done, _, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}); err != nil || done { - t.Fatalf("FinalizeUploadIfTargetCopiesMet = done:%t err:%v, want durable but incomplete", done, err) - } - - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone).WithTaskMaxRetries(9, 4) - mgr.Start(ctx) - tasks, total, err := repos.Tasks.List(ctx, string(model.TaskTypeUpload), "prepare_upload", string(model.TaskStatusQueued), 10, 0) - if err != nil { - t.Fatalf("List repair tasks: %v", err) - } - if total != 1 || len(tasks) != 1 || taskPayloadInt64ForTest(tasks[0].Payload, "upload_id") != upload.ID { - t.Fatalf("repair tasks total=%d tasks=%#v, want stored upload %d", total, tasks, upload.ID) - } -} - -func TestManager_RecoverOnStartup_ReconcilesAllStagedUploads(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "mgr-staged-batch-recover") - for i := 0; i < 101; i++ { - _, versionID := seedManagerVersion(t, repos, bucket, fmt.Sprintf("recover-batch-%03d", i), model.ObjectStateCached) - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading %d: %v", i, err) - } - version, err := repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID %d: version=%v err=%v", i, version, err) - } - if _, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }); err != nil { - t.Fatalf("StartObjectUploadAttempt %d: %v", i, err) - } - } - - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone).WithTaskMaxRetries(9, 4) - mgr.Start(ctx) - - tasks, total, err := repos.Tasks.List(ctx, string(model.TaskTypeUpload), "prepare_upload", string(model.TaskStatusQueued), 200, 0) - if err != nil { - t.Fatalf("List prepare_upload tasks: %v", err) - } - if total != 101 || len(tasks) != 101 { - t.Fatalf("prepare_upload tasks total=%d len=%d, want 101", total, len(tasks)) - } -} - -func TestManager_RecoverOnStartup_UsesBoundedVersionBatches(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - objects := &boundedVersionListRepo{ObjectRepository: repos.Objects} - repos.Objects = objects - - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone).WithTaskMaxRetries(9, 4) - mgr.Start(context.Background()) - - if objects.sawUnbounded { - t.Fatal("startup reconciliation requested an unbounded object version list") - } -} - -func TestManager_ReconcileTasks_CreatesMissingTasks(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "mgr-reconcile-bucket") - objID, versionID := seedManagerVersion(t, repos, bucket, "mgr-reconcile-key", model.ObjectStateCached) - - // No task exists yet — manager should create one during reconciliation - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone).WithTaskMaxRetries(9, 4) - mgr.Start(ctx) - - // Claim the task created by reconciliation - task, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil { - t.Fatalf("claiming task: %v", err) - } - if task == nil { - t.Fatal("expected reconciliation to create missing upload task") - } - if task.RefID != objID || task.RefVersionID != versionID { - t.Errorf("task refs mismatch: got refID=%d version=%s, want %d/%s", task.RefID, task.RefVersionID, objID, versionID) - } - if task.MaxRetries != 9 { - t.Fatalf("task MaxRetries = %d, want 9", task.MaxRetries) - } - if task.Stage == nil || *task.Stage != "prepare_upload" { - t.Fatalf("task Stage = %#v, want prepare_upload", task.Stage) - } -} - -func TestManager_WorkerHealth(t *testing.T) { - repos := testutil.NewTestRepos(t) - w1 := &stubWorker{name: "alpha", isHealthy: true} - w2 := &stubWorker{name: "beta", isHealthy: false} - w3 := &stubWorker{name: "gamma", isHealthy: true} - - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone, w1, w2, w3) - health := mgr.WorkerHealth() - - if len(health) != 3 { - t.Fatalf("expected 3 entries, got %d", len(health)) - } - if !health["alpha"] { - t.Error("expected alpha healthy") - } - if health["beta"] { - t.Error("expected beta unhealthy") - } - if !health["gamma"] { - t.Error("expected gamma healthy") - } -} - -func TestManager_WorkerHealth_Empty(t *testing.T) { - repos := testutil.NewTestRepos(t) - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone) - health := mgr.WorkerHealth() - if len(health) != 0 { - t.Errorf("expected empty map, got %d entries", len(health)) - } -} - -func TestManager_WorkerHealth_RealWorkers(t *testing.T) { - repos := testutil.NewTestRepos(t) - mc := &testutil.MockCache{} - sm := state.NewObjectStateMachine() - logger := slog.Default() - poll := 50 * time.Millisecond - - up := worker.NewUploader(repos, mc, nil, nil, sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, poll, logger) - cacheGate := cacheaccess.NewGate() - accessTracker := cacheaccess.NewTracker(cacheaccess.DefaultPersistenceInterval, repos.Objects) - ev := worker.NewEvictor( - repos, - mc, - cacheGate, - accessTracker, - sm, - 1, - poll, - logger, - worker.WithCacheEvictionPolicy(cache.EvictionPolicyAfterUpload, 0, 90, 80, 3), - ) - - mgr := worker.NewManager(repos, logger, cache.EvictionPolicyAfterUpload, up, ev) - health := mgr.WorkerHealth() - - expected := map[string]bool{ - "uploader": true, - "evictor": true, - } - for name, wantHealthy := range expected { - got, ok := health[name] - if !ok { - t.Errorf("missing worker %q in health map", name) - continue - } - if got != wantHealthy { - t.Errorf("worker %q: expected healthy=%v, got %v", name, wantHealthy, got) - } - } -} - -func TestManager_ReconcileTasks_IdempotencyDedup(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "mgr-dedup-bucket") - objID, versionID := seedManagerVersion(t, repos, bucket, "mgr-dedup-key", model.ObjectStateCached) - - // Pre-create the task with the same idempotency key the manager would use - existingTask := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "object", - RefID: objID, - RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("upload:%s", versionID), - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, existingTask); err != nil { - t.Fatalf("creating existing task: %v", err) - } - - // Start manager — reconciliation should skip (idempotency dedup) - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone) - mgr.Start(ctx) - - // Claim the task — should be exactly one (the pre-existing one) - task, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if err != nil { - t.Fatalf("claiming task: %v", err) - } - if task == nil { - t.Fatal("expected existing task to still be claimable") - } - if task.ID != existingTask.ID { - t.Errorf("expected existing task ID %d, got %d", existingTask.ID, task.ID) - } - - // No second task should be claimable - dup, _ := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) - if dup != nil { - t.Error("expected no duplicate task after idempotency dedup") - } -} - -func TestManager_ReconcileTasks_NoneDoesNotCreateEvictionTask(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "mgr-autoevict-off") - seedManagerVersion(t, repos, bucket, "stored-no-evict", model.ObjectStateStored) - - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone) - mgr.Start(ctx) - - task, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeEvictCache, time.Minute) - if err != nil { - t.Fatalf("claiming evict task: %v", err) - } - if task != nil { - t.Fatal("expected no evict_cache task under none policy") - } -} - -func TestManager_RecoverOnStartup_AfterUploadDoesNotBackfillStoredVersions(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "mgr-autoevict-on") - seedManagerVersion(t, repos, bucket, "stored-without-backfill", model.ObjectStateStored) - - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyAfterUpload).WithTaskMaxRetries(9, 4) - mgr.Start(ctx) - - task, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeEvictCache, time.Minute) - if err != nil { - t.Fatalf("claiming evict task: %v", err) - } - if task != nil { - t.Fatalf("unexpected startup backfill task: %#v", task) - } -} - -func TestManager_RecoverOnStartupSwitchesEvictionPoliciesWithoutBackfill(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := testutil.SeedBucket(t, db, "mgr-eviction-policy-switch") - objectID, versionID := seedManagerVersion(t, repos, bucket, "stored-policy-switch", model.ObjectStateStored) - - activated, err := repos.CacheEvictions.EnsureAfterUploadTask(ctx, objectID, versionID, 4) - if err != nil || !activated { - t.Fatalf("EnsureAfterUploadEvictionTask: activated=%v err=%v", activated, err) - } - afterUploadTasks, total, err := repos.Tasks.List( - ctx, - string(model.TaskTypeEvictCache), - cacheeviction.StageAfterUpload, - "", - 10, - 0, - ) - if err != nil || total != 1 || len(afterUploadTasks) != 1 { - t.Fatalf("after_upload tasks total=%d tasks=%#v err=%v", total, afterUploadTasks, err) - } - afterUploadTask := afterUploadTasks[0] - lruStage := cacheeviction.StageLRU - lruTask := &model.Task{ - Type: model.TaskTypeEvictCache, - Stage: &lruStage, - RefType: "object", - RefID: objectID, - RefVersionID: versionID, - IdempotencyKey: "evict_cache:lru:startup-test:" + versionID, - Status: model.TaskStatusQueued, - MaxRetries: 4, - ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(ctx, lruTask); err != nil { - t.Fatalf("Create LRU task: %v", err) - } - - worker.NewManager(repos, slog.Default(), cache.EvictionPolicyLRU).Start(ctx) - gotAfterUpload, _ := repos.Tasks.GetByID(ctx, afterUploadTask.ID) - gotLRU, _ := repos.Tasks.GetByID(ctx, lruTask.ID) - if gotAfterUpload.Status != model.TaskStatusCancelled || gotLRU.Status != model.TaskStatusQueued { - t.Fatalf("after lru startup after_upload/lru statuses = %s/%s, want cancelled/queued", gotAfterUpload.Status, gotLRU.Status) - } - - worker.NewManager(repos, slog.Default(), cache.EvictionPolicyAfterUpload).Start(ctx) - gotAfterUpload, _ = repos.Tasks.GetByID(ctx, afterUploadTask.ID) - gotLRU, _ = repos.Tasks.GetByID(ctx, lruTask.ID) - if gotAfterUpload.Status != model.TaskStatusCancelled || gotLRU.Status != model.TaskStatusCancelled { - t.Fatalf("after after_upload startup statuses = %s/%s, want cancelled/cancelled", gotAfterUpload.Status, gotLRU.Status) - } - - worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone).Start(ctx) - gotAfterUpload, _ = repos.Tasks.GetByID(ctx, afterUploadTask.ID) - if gotAfterUpload.Status != model.TaskStatusCancelled { - t.Fatalf("after none startup after_upload status = %s, want cancelled", gotAfterUpload.Status) - } - - worker.NewManager(repos, slog.Default(), cache.EvictionPolicyAfterUpload).Start(ctx) - gotAfterUpload, _ = repos.Tasks.GetByID(ctx, afterUploadTask.ID) - if gotAfterUpload.Status != model.TaskStatusCancelled { - t.Fatalf("after_upload task status after restart = %s, want cancelled without backfill", gotAfterUpload.Status) - } -} - -func TestManager_ReconcileTasks_AfterUploadSkipsReplicating(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "mgr-autoevict-replicating") - objID, versionID := seedManagerVersion(t, repos, bucket, "replicating-with-cache", model.ObjectStateCached) - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("committing: %v", err) - } - version, err := repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - primary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("EnsureDataSetBinding primary: %v", err) - } - secondary, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "202"), CopyIndex: 1, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("EnsureDataSetBinding secondary: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: primary.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), ClientDataSetID: onChainIDPtr(t, "9001")}); err != nil { - t.Fatalf("MarkDataSetReady primary: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: primary.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - {StorageDataSetID: secondary.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: "piece-replicating-evict-reconcile", - PieceID: onChainIDPtr(t, "301"), - RetrievalURL: "https://primary.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: version.Size, - Checksum: version.Checksum, - }); err != nil { - t.Fatalf("BindReadableUploadForContent: %v", err) - } - - mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyAfterUpload).WithTaskMaxRetries(9, 4) - mgr.Start(ctx) - - task, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeEvictCache, time.Minute) - if err != nil { - t.Fatalf("claiming evict task: %v", err) - } - if task != nil { - t.Fatalf("unexpected evict_cache task for replicating version %d/%s: %#v", objID, versionID, task) - } -} - -func TestManager_AfterUploadDoesNotRequeueRehydratedCacheEvictedVersion(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := testutil.SeedBucket(t, db, "mgr-after-upload-rehydrated") - _, versionID := seedManagerVersion(t, repos, bucket, "rehydrated.txt", model.ObjectStateCacheEvicted) - if err := repos.Objects.RecordVersionCacheCommit(ctx, versionID, time.Now()); err != nil { - t.Fatalf("RecordVersionCacheCommit: %v", err) - } - - worker.NewManager(repos, slog.Default(), cache.EvictionPolicyAfterUpload).Start(ctx) - tasks, total, err := repos.Tasks.List( - ctx, - string(model.TaskTypeEvictCache), - cacheeviction.StageAfterUpload, - "", - 10, - 0, - ) - if err != nil { - t.Fatalf("List after_upload tasks: %v", err) - } - if total != 0 || len(tasks) != 0 { - t.Fatalf("rehydrated cache_evicted after_upload tasks total=%d tasks=%#v, want none", total, tasks) - } -} - -func seedManagerVersion(t *testing.T, repos *repository.Repositories, bucket *model.Bucket, key string, state model.ObjectState) (int64, string) { - t.Helper() - versionID := model.NewVersionID() - createState := state - if state == model.ObjectStateStored || state == model.ObjectStateCacheEvicted { - createState = model.ObjectStateUploading - } - version := &model.ObjectVersion{ - VersionID: versionID, - BucketID: bucket.ID, - Key: key, - Size: 100, - ETag: "etag-" + key, - Checksum: "checksum-" + key, - ContentType: "text/plain", - CacheKey: ".versions/" + versionID, - State: createState, - } - objID, err := repos.Objects.CreateVersionAndSetCurrent(context.Background(), version) - if err != nil { - t.Fatalf("creating object version: %v", err) - } - if state == model.ObjectStateStored || state == model.ObjectStateCacheEvicted { - acceptManagerVersionUpload(t, repos, versionID) - if state == model.ObjectStateCacheEvicted { - if err := repos.Objects.UpdateVersionState(context.Background(), versionID, model.ObjectStateStored, model.ObjectStateCacheEvicted); err != nil { - t.Fatalf("transition to cache_evicted: %v", err) - } - } - } - return objID, versionID -} - -func acceptManagerVersionUpload(t *testing.T, repos *repository.Repositories, versionID string) { - t.Helper() - ctx := context.Background() - version, err := repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("get version for upload accept: version=%v err=%v", version, err) - } - pieceCID := "piece-" + versionID - providerID := onChainID(t, "101") - dataSetID := onChainID(t, "1001") - pieceID := onChainIDPtr(t, "1") - retrievalURL := "https://provider.example/piece/" + versionID - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: version.BucketID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("start upload attempt: %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: version.BucketID, - ProviderID: providerID, - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("ensure dataset binding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, UploadID: upload.ID, DataSetID: dataSetID}); err != nil { - t.Fatalf("mark dataset ready: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: providerID, - }}); err != nil { - t.Fatalf("create upload copy: %v", err) - } - if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: pieceCID, - PieceID: pieceID, - RetrievalURL: retrievalURL, - }); err != nil { - t.Fatalf("mark copy committed: %v", err) - } - if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: version.BucketID, - ContentSize: version.Size, - Checksum: version.Checksum, - }); err != nil { - t.Fatalf("bind readable upload: %v", err) - } - if finalized, _, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}); err != nil { - t.Fatalf("finalize upload: %v", err) - } else if !finalized { - t.Fatal("finalize upload = false, want true") - } -} - -type boundedVersionListRepo struct { - repository.ObjectRepository - sawUnbounded bool -} - -func (r *boundedVersionListRepo) ListVersionsByState(ctx context.Context, state model.ObjectState, limit int) ([]model.ObjectVersion, error) { - if limit <= 0 { - r.sawUnbounded = true - } - return r.ObjectRepository.ListVersionsByState(ctx, state, limit) -} diff --git a/internal/worker/onchain_id_external_test.go b/internal/worker/onchain_id_external_test.go deleted file mode 100644 index f547a5a..0000000 --- a/internal/worker/onchain_id_external_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package worker_test - -import ( - "testing" - - "github.com/strahe/synaps3/internal/types" -) - -func onChainID(t *testing.T, value string) types.OnChainID { - t.Helper() - id, err := types.ParseOnChainID("test id", value) - if err != nil { - t.Fatalf("parse on-chain id %q: %v", value, err) - } - return id -} - -func onChainIDPtr(t *testing.T, value string) *types.OnChainID { - t.Helper() - id := onChainID(t, value) - return &id -} - -func onChainIDPtrString(id *types.OnChainID) string { - if id == nil { - return "" - } - return id.String() -} diff --git a/internal/worker/onchain_id_internal_test.go b/internal/worker/onchain_id_internal_test.go deleted file mode 100644 index 5836c50..0000000 --- a/internal/worker/onchain_id_internal_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package worker - -import ( - "testing" - - "github.com/strahe/synaps3/internal/types" -) - -func onChainID(t *testing.T, value string) types.OnChainID { - t.Helper() - id, err := types.ParseOnChainID("test id", value) - if err != nil { - t.Fatalf("parse on-chain id %q: %v", value, err) - } - return id -} diff --git a/internal/worker/poll_sleep.go b/internal/worker/poll_sleep.go deleted file mode 100644 index fe8f76b..0000000 --- a/internal/worker/poll_sleep.go +++ /dev/null @@ -1,33 +0,0 @@ -package worker - -import ( - "context" - "math/rand" - "time" -) - -const workerPollJitterDivisor = 5 - -func sleepUntilNextWorkerPoll(ctx context.Context, interval time.Duration) bool { - timer := time.NewTimer(workerPollSleepDuration(interval)) - defer timer.Stop() - - select { - case <-ctx.Done(): - return false - case <-timer.C: - return true - } -} - -func workerPollSleepDuration(interval time.Duration) time.Duration { - if interval <= 0 { - return interval - } - - maxJitter := interval / workerPollJitterDivisor - if maxJitter <= 0 { - return interval - } - return interval + time.Duration(rand.Int63n(int64(maxJitter)+1)) -} diff --git a/internal/worker/poll_sleep_test.go b/internal/worker/poll_sleep_test.go deleted file mode 100644 index 2c90681..0000000 --- a/internal/worker/poll_sleep_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package worker - -import ( - "testing" - "time" -) - -func TestWorkerPollSleepDurationAddsBoundedJitter(t *testing.T) { - interval := 20 * time.Millisecond - maxDuration := interval + interval/workerPollJitterDivisor - - for range 100 { - got := workerPollSleepDuration(interval) - if got < interval || got > maxDuration { - t.Fatalf("worker poll sleep duration = %s, want between %s and %s", got, interval, maxDuration) - } - } -} - -func TestWorkerPollSleepDurationKeepsNonPositiveAndTinyIntervals(t *testing.T) { - for _, interval := range []time.Duration{0, -time.Second, time.Nanosecond} { - if got := workerPollSleepDuration(interval); got != interval { - t.Fatalf("worker poll sleep duration = %s, want %s", got, interval) - } - } -} diff --git a/internal/worker/provider_replacement_worker.go b/internal/worker/provider_replacement_worker.go deleted file mode 100644 index a9e2420..0000000 --- a/internal/worker/provider_replacement_worker.go +++ /dev/null @@ -1,768 +0,0 @@ -package worker - -import ( - "context" - "errors" - "fmt" - "log/slog" - "os" - "sync" - "time" - - "github.com/ipfs/go-cid" - "github.com/strahe/synaps3/internal/admin" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagecommit" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/strahe/synaps3/internal/synapse" - idtypes "github.com/strahe/synaps3/internal/types" - "github.com/strahe/synapse-go/storage" -) - -const ( - replacementItemLeaseTTL = 10 * time.Minute - replacementLeaseRetryDelay = time.Second - replacementSourceRecheckDelay = time.Minute - replacementUploadYieldDelay = time.Second -) - -var ( - errReplacementSourceUnavailable = errors.New("replacement item has no readable source") - errReplacementUploadPrecedes = errors.New("ordinary upload claim precedes replacement item") - errReplacementCommitPending = errors.New("replacement storage confirmation is pending") - errReplacementCommitAttention = errors.New("replacement storage confirmation needs review") - errReplacementCommitObserving = errors.New("replacement storage confirmation remains under observation") - errReplacementOwnerTerminal = errors.New("replacement owner is terminal") -) - -type replacementMigrationPauser interface { - PauseMigration(context.Context, int64, int64, storagereplacement.WaitReason) error -} - -type pulledReplacementPiece struct { - pieceCID cid.Cid - pieceCIDString string - extraHex string -} - -func replacementSourceUnusable(err error) bool { - return synapse.IsProviderUnavailable(err) || - synapse.IsDataSetServiceEnded(err) || - synapse.IsNoProviderCandidates(err) -} - -func orderReplacementSources(copies []repository.ReadableStorageCopy, targetCopyIndex int) []repository.ReadableStorageCopy { - preferred := make([]repository.ReadableStorageCopy, 0, len(copies)) - sameSlot := make([]repository.ReadableStorageCopy, 0, len(copies)) - for i := range copies { - candidate := copies[i] - if candidate.PieceCID == "" || candidate.RetrievalURL == "" { - continue - } - if candidate.CopyIndex == targetCopyIndex { - sameSlot = append(sameSlot, candidate) - continue - } - preferred = append(preferred, candidate) - } - return append(preferred, sameSlot...) -} - -// ProviderReplacementWorker processes provider replacement items. -type ProviderReplacementWorker struct { - repos *repository.Repositories - executor *ReplacementTransferExecutor - concurrency int - pollInterval time.Duration - leaseTTL time.Duration - logger *slog.Logger - *livenessTracker -} - -func NewProviderReplacementWorker( - repos *repository.Repositories, - uploadSupport *Uploader, - concurrency int, - pollInterval time.Duration, - logger *slog.Logger, -) *ProviderReplacementWorker { - registry := newReplacementTargetContextRegistry() - return &ProviderReplacementWorker{ - repos: repos, - executor: NewReplacementTransferExecutor(repos, uploadSupport, registry, logger), - concurrency: concurrency, - pollInterval: pollInterval, - leaseTTL: replacementItemLeaseTTL, - logger: logger, - livenessTracker: newLivenessTracker(pollInterval), - } -} - -func (w *ProviderReplacementWorker) Name() string { return "provider_replacement" } - -func (w *ProviderReplacementWorker) Healthy() bool { return w.healthy() } - -func (w *ProviderReplacementWorker) Run(ctx context.Context) error { - var wg sync.WaitGroup - for range w.concurrency { - wg.Add(1) - go func() { - defer wg.Done() - w.runSlot(ctx) - }() - } - wg.Wait() - return ctx.Err() -} - -func (w *ProviderReplacementWorker) runSlot(ctx context.Context) { - for { - if ctx.Err() != nil { - return - } - w.recordTick() - item, err := w.repos.Replacements.ClaimReadyReplacementItem(ctx, w.leaseTTL) - if err != nil { - if ctx.Err() != nil { - return - } - w.logger.Error("claiming provider replacement item", "error", err) - if !sleepUntilNextWorkerPoll(ctx, w.pollInterval) { - return - } - continue - } - if item == nil { - if !sleepUntilNextWorkerPoll(ctx, w.pollInterval) { - return - } - continue - } - w.processItem(ctx, item) - } -} - -func (w *ProviderReplacementWorker) processItem(parent context.Context, item *storagereplacement.Item) { - if item == nil || item.ClaimedAt == nil { - return - } - w.recordWorkStarted() - defer w.recordWorkFinished() - started := time.Now() - defer func() { - admin.WorkerTaskDuration.WithLabelValues(w.Name()).Observe(time.Since(started).Seconds()) - }() - - token := storagereplacement.ClaimToken{ItemID: item.ID, ClaimedAt: *item.ClaimedAt} - itemCtx, cancel := context.WithCancel(parent) - leaseLost, stopRenewal := w.startLeaseRenewal(itemCtx, cancel, token, item.LeaseUntil) - expectedStateVersion, err := w.executor.Execute(itemCtx, item) - stopRenewal() - cancel() - if leaseLost() || errors.Is(err, repository.ErrItemClaimLost) { - releaseCtx, releaseCancel := context.WithTimeout(context.Background(), taskLeaseOperationTimeout) - releaseErr := w.repos.Replacements.ReleaseReplacementItemClaim(releaseCtx, token) - releaseCancel() - if releaseErr != nil && !errors.Is(releaseErr, repository.ErrItemClaimLost) { - w.logItemTransitionFailure(item, "releasing lost replacement item claim", releaseErr) - } - admin.WorkerTasksProcessed.WithLabelValues(w.Name(), "claim_lost").Inc() - return - } - if parent.Err() != nil { - // Worker shutdown must not leave a live lease behind. Use an independent, - // bounded context because the runtime context is already cancelled. - releaseCtx, releaseCancel := context.WithTimeout(context.Background(), taskLeaseOperationTimeout) - releaseErr := w.repos.Replacements.ReleaseReplacementItemClaim(releaseCtx, token) - releaseCancel() - if releaseErr != nil && !errors.Is(releaseErr, repository.ErrItemClaimLost) { - w.logItemTransitionFailure(item, "releasing replacement item during shutdown", releaseErr) - admin.WorkerTasksProcessed.WithLabelValues(w.Name(), "failure").Inc() - return - } - admin.WorkerTasksProcessed.WithLabelValues(w.Name(), "shutdown_released").Inc() - return - } - - result := "success" - switch { - case err == nil, errors.Is(err, storagereplacement.ErrItemCancelled), errors.Is(err, storagereplacement.ErrItemDeferred): - // The repository already settled the claim. - case errors.Is(err, errReplacementUploadPrecedes): - if transitionErr := w.repos.Replacements.DeferReplacementItemClaim(parent, token, time.Now().Add(replacementUploadYieldDelay)); transitionErr != nil { - w.logItemTransitionFailure(item, "yielding replacement item", transitionErr) - result = "failure" - } - case errors.Is(err, errReplacementCommitPending): - if transitionErr := w.repos.Replacements.DeferReplacementItemClaim(parent, token, time.Now().Add(w.commitPollDelay())); transitionErr != nil { - w.logItemTransitionFailure(item, "waiting for replacement storage confirmation", transitionErr) - result = "failure" - } else { - result = "confirmation_wait" - } - case errors.Is(err, errReplacementCommitObserving): - if transitionErr := w.repos.Replacements.DeferReplacementItemClaim(parent, token, time.Now().Add(commitObservationDelay(w.commitPollDelay()))); transitionErr != nil { - w.logItemTransitionFailure(item, "observing replacement storage confirmation", transitionErr) - result = "failure" - } else { - result = "confirmation_wait" - } - case errors.Is(err, errReplacementCommitAttention): - if transitionErr := w.repos.Replacements.DeferReplacementItemClaim(parent, token, time.Now().Add(storageCommitAttentionDelay)); transitionErr != nil { - w.logItemTransitionFailure(item, "parking replacement storage confirmation for review", transitionErr) - result = "failure" - } else { - result = "confirmation_attention" - } - case errors.Is(err, errReplacementOwnerTerminal): - if transitionErr := w.repos.Replacements.CancelReplacementItemClaim(parent, token); transitionErr != nil { - w.logItemTransitionFailure(item, "cancelling terminal replacement item", transitionErr) - result = "failure" - } else { - result = "cancelled" - } - case errors.Is(err, errReplacementSourceUnavailable): - if transitionErr := w.repos.Replacements.WaitReplacementItemClaim( - parent, token, time.Now().Add(replacementSourceRecheckDelay), safeReplacementItemError(err), - ); transitionErr != nil { - w.logItemTransitionFailure(item, "waiting for replacement source", transitionErr) - result = "failure" - } else { - result = "waiting_source" - } - case synapse.IsProviderUnavailable(err), synapse.IsNoProviderCandidates(err), synapse.IsDataSetServiceEnded(err), dataSetWriteBlockedError(err): - pauseErr := pauseReplacementAtExecutionVersion( - parent, w.repos.Replacements, item.ReplacementID, expectedStateVersion, - ) - if pauseErr != nil && !errors.Is(pauseErr, repository.ErrConflict) { - w.logItemTransitionFailure(item, "pausing unavailable replacement target", pauseErr) - } - if transitionErr := w.repos.Replacements.DeferReplacementItemClaim(parent, token, time.Now().Add(w.pollInterval)); transitionErr != nil && !errors.Is(transitionErr, repository.ErrItemClaimLost) { - w.logItemTransitionFailure(item, "releasing paused replacement item", transitionErr) - } - result = "target_wait" - default: - status, transitionErr := w.repos.Replacements.RetryReplacementItemClaim( - parent, token, time.Now().Add(retryDelay(item.RetryCount)), safeReplacementItemError(err), - ) - if transitionErr != nil { - w.logItemTransitionFailure(item, "scheduling replacement item retry", transitionErr) - result = "failure" - } else { - result = string(status) - } - } - admin.WorkerTasksProcessed.WithLabelValues(w.Name(), result).Inc() -} - -func (w *ProviderReplacementWorker) commitPollDelay() time.Duration { - if w != nil && w.pollInterval > 0 { - return w.pollInterval - } - return storageCommitPollDelay -} - -func pauseReplacementAtExecutionVersion( - ctx context.Context, - pauser replacementMigrationPauser, - replacementID, expectedStateVersion int64, -) error { - if pauser == nil || replacementID <= 0 || expectedStateVersion <= 0 { - return repository.ErrConflict - } - return pauser.PauseMigration(ctx, replacementID, expectedStateVersion, storagereplacement.WaitReasonTarget) -} - -func (w *ProviderReplacementWorker) logItemTransitionFailure(item *storagereplacement.Item, message string, err error) { - w.logger.Error(message, "replacementID", item.ReplacementID, "itemID", item.ID, "error", err) -} - -func safeReplacementItemError(err error) string { - if err == nil { - return "" - } - // Normal APIs do not return this raw diagnostic. - return err.Error() -} - -func (w *ProviderReplacementWorker) startLeaseRenewal( - ctx context.Context, - cancel context.CancelFunc, - token storagereplacement.ClaimToken, - initialLeaseUntil *time.Time, -) (func() bool, func()) { - var mu sync.Mutex - lost := false - done := make(chan struct{}) - stop := make(chan struct{}) - go func() { - defer close(done) - leaseUntil := time.Time{} - if initialLeaseUntil != nil { - leaseUntil = *initialLeaseUntil - } - timer := time.NewTimer(taskLeaseRenewInterval(w.leaseTTL)) - defer timer.Stop() - markLost := func() { - mu.Lock() - lost = true - mu.Unlock() - cancel() - } - for { - select { - case <-ctx.Done(): - return - case <-stop: - return - case <-timer.C: - if leaseUntil.IsZero() || !time.Now().Before(leaseUntil) { - markLost() - return - } - attemptedAt := time.Now() - opCtx, opCancel := context.WithTimeout(context.Background(), taskLeaseOperationTimeout) - err := w.repos.Replacements.RenewReplacementItemLease(opCtx, token, w.leaseTTL) - opCancel() - if err == nil { - leaseUntil = attemptedAt.Add(w.leaseTTL) - timer.Reset(taskLeaseRenewInterval(w.leaseTTL)) - continue - } - if ctx.Err() != nil { - return - } - if errors.Is(err, repository.ErrItemClaimLost) || !time.Now().Before(leaseUntil) { - markLost() - return - } - w.logger.Warn("renewing provider replacement item lease", "itemID", token.ItemID, "error", err) - retryDelay := min(replacementLeaseRetryDelay, taskLeaseRenewInterval(w.leaseTTL), time.Until(leaseUntil)/2) - if retryDelay <= 0 { - markLost() - return - } - timer.Reset(retryDelay) - } - } - }() - var once sync.Once - return func() bool { - mu.Lock() - defer mu.Unlock() - return lost - }, func() { - once.Do(func() { close(stop) }) - <-done - } -} - -// ReplacementTransferExecutor processes one claimed replacement item. -type ReplacementTransferExecutor struct { - repos *repository.Repositories - support *Uploader - registry *replacementTargetContextRegistry - logger *slog.Logger -} - -func NewReplacementTransferExecutor( - repos *repository.Repositories, - uploadSupport *Uploader, - registry *replacementTargetContextRegistry, - logger *slog.Logger, -) *ReplacementTransferExecutor { - return &ReplacementTransferExecutor{repos: repos, support: uploadSupport, registry: registry, logger: logger} -} - -func (e *ReplacementTransferExecutor) Execute(ctx context.Context, item *storagereplacement.Item) (int64, error) { - if item == nil || item.ClaimedAt == nil { - return 0, repository.ErrItemClaimLost - } - token := storagereplacement.ClaimToken{ItemID: item.ID, ClaimedAt: *item.ClaimedAt} - snapshot, err := e.repos.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ - ReplacementID: item.ReplacementID, - ItemID: item.ID, - ItemClaimedAt: token.ClaimedAt, - }) - if err != nil { - return 0, err - } - expectedStateVersion := snapshot.Replacement.StateVersion - copyRow, err := e.repos.Replacements.AttachTargetCopy(ctx, repository.AttachReplacementTargetCopyInput{ - ReplacementID: snapshot.Replacement.ID, - ItemID: item.ID, - UploadID: snapshot.Upload.ID, - ItemClaimedAt: token.ClaimedAt, - }) - if err != nil { - return expectedStateVersion, err - } - // The copy must be visible before claim ordering is checked, or a concurrent - // upload could miss this item. - earlierUpload, err := e.repos.Tasks.HasEarlierRunningUploadCopyClaim( - ctx, token.ClaimedAt, snapshot.Upload.ID, snapshot.Replacement.CopyIndex, - ) - if err != nil { - return expectedStateVersion, err - } - if earlierUpload { - return expectedStateVersion, errReplacementUploadPrecedes - } - if copyCommitted(copyRow) { - return expectedStateVersion, e.finish(ctx, token, &snapshot.Upload) - } - if !snapshot.Replacement.Status.Active() && copyRow.CommitAttemptID != nil && - *copyRow.CommitAttemptID != "" && copyRow.CommitAttemptedAt == nil { - advance, err := (&storagecommit.Advancer{Store: e.repos.Uploads}).ReleaseTerminalReservation( - ctx, *copyRow, snapshot.Target, - ) - if err != nil { - return expectedStateVersion, err - } - if advance.State == storagecommit.AdvanceReleased && advance.ReleaseReason == storagecommit.ReleaseOwnerTerminal { - return expectedStateVersion, errReplacementOwnerTerminal - } - return expectedStateVersion, errors.New("terminal replacement reservation returned an unexpected state") - } - - bucket, err := e.repos.Buckets.GetByID(ctx, snapshot.Replacement.BucketID) - if err != nil { - return expectedStateVersion, err - } - if bucket == nil { - return expectedStateVersion, fmt.Errorf("bucket %d: %w", snapshot.Replacement.BucketID, repository.ErrNotFound) - } - handle, err := e.registry.acquire(ctx, snapshot.Target.ID, func() (synapse.DataSetTarget, error) { - return e.support.contextForReadyBinding(ctx, &snapshot.Target) - }) - if err != nil { - if copyCommitSubmitted(copyRow) { - advance, advanceErr := (&storagecommit.Advancer{Store: e.repos.Uploads}).AdvanceUnavailable( - ctx, *copyRow, snapshot.Target, - ) - if advanceErr != nil { - return expectedStateVersion, advanceErr - } - switch { - case advance.State == storagecommit.AdvancePending: - return expectedStateVersion, errReplacementCommitPending - case advance.State == storagecommit.AdvanceNeedsAttention && advance.Continue: - return expectedStateVersion, errReplacementCommitObserving - case advance.State == storagecommit.AdvanceNeedsAttention: - return expectedStateVersion, errReplacementCommitAttention - default: - return expectedStateVersion, errors.New("unavailable replacement commit returned an unexpected state") - } - } - return expectedStateVersion, err - } - defer handle.release() - return expectedStateVersion, e.copy(ctx, token, snapshot, copyRow, handle, bucket) -} - -func (e *ReplacementTransferExecutor) copy( - ctx context.Context, - token storagereplacement.ClaimToken, - snapshot *repository.ReplacementItemSnapshot, - copyRow *model.StorageUploadCopy, - handle *replacementTargetContextHandle, - bucket *model.Bucket, -) error { - upload := &snapshot.Upload - version := &snapshot.Version - storageCtx := handle.storageCtx - var pieceCID cid.Cid - var pieceCIDString string - extraHex := derefString(copyRow.CommitExtraDataHex) - - if !copyHasPiece(copyRow) { - readable, err := e.repos.Uploads.ListReadableCommittedCopies(ctx, upload.ID) - if err != nil { - return err - } - pulled, err := e.pull(ctx, storageCtx, copyRow, orderReplacementSources(readable, snapshot.Target.CopyIndex)) - if err != nil { - return err - } - if pulled != nil { - pieceCID = pulled.pieceCID - pieceCIDString = pulled.pieceCIDString - extraHex = pulled.extraHex - } else { - stored, storedCID, err := e.storeFromCache(ctx, storageCtx, bucket, version) - if err != nil { - return err - } - if !stored { - return errReplacementSourceUnavailable - } - pieceCID = storedCID - pieceCIDString = pieceCID.String() - _, extraHex, err = e.support.extraDataForCopy(ctx, storageCtx, copyRow, []storage.PieceInput{{PieceCID: pieceCID}}) - if err != nil { - return err - } - } - evidenceCtx, evidenceCancel := providerEvidenceContext(ctx) - err = e.repos.Uploads.MarkUploadCopyPieceReady(evidenceCtx, repository.MarkUploadCopyPieceReadyInput{ - StorageUploadCopyID: copyRow.ID, - RequireEligibleCopy: true, - UploadID: upload.ID, - CopyIndex: copyRow.CopyIndex, - PieceCID: pieceCIDString, - RetrievalURL: storageCtx.PieceURL(pieceCID), - CommitExtraDataHex: extraHex, - }) - evidenceCancel() - if err != nil { - return err - } - copyRow.Status = model.StorageUploadCopyStatusPieceReady - copyRow.CommitExtraDataHex = &extraHex - } else { - if upload.PieceCID == nil || *upload.PieceCID == "" { - return fmt.Errorf("storage upload %d has no piece CID", upload.ID) - } - pieceCIDString = *upload.PieceCID - decoded, err := cid.Decode(pieceCIDString) - if err != nil { - return fmt.Errorf("decode stored piece CID: %w", err) - } - pieceCID = decoded - } - - pieces := []storage.PieceInput{{PieceCID: pieceCID}} - advance, err := e.support.commitReplicaRepairCopy( - ctx, upload, &snapshot.Target, copyRow, storageCtx, pieces, !snapshot.Replacement.Status.Active(), - ) - if err != nil && advance.State == storagecommit.AdvancePending && synapse.IsProviderUnavailable(err) { - return err - } - switch { - case advance.State == storagecommit.AdvanceWaitingCapacity, - advance.State == storagecommit.AdvanceSubmitted, - advance.State == storagecommit.AdvancePending: - if err != nil { - e.logger.Warn("storage commit evidence remains fenced", "stage", "provider replacement commit", "error", err) - } - return errReplacementCommitPending - case advance.State == storagecommit.AdvanceNeedsAttention && advance.Continue: - return errReplacementCommitObserving - case advance.State == storagecommit.AdvanceNeedsAttention: - return errReplacementCommitAttention - case advance.State == storagecommit.AdvanceReleased && advance.ReleaseReason == storagecommit.ReleaseDataSetUnavailable: - return commitReleaseCause(advance) - case advance.State == storagecommit.AdvanceReleased && advance.ReleaseReason == storagecommit.ReleaseOwnerTerminal: - return errReplacementOwnerTerminal - case advance.State == storagecommit.AdvanceReleased: - return errReplacementCommitPending - case advance.State == storagecommit.AdvanceRejected: - return errCommitRejected - case err != nil: - return err - case advance.State != storagecommit.AdvanceConfirmed || advance.Confirmation == nil || len(advance.Confirmation.PieceIDs) == 0: - return errors.New("replacement commit returned no piece ID") - } - result := advance.Confirmation - pieceID := idtypes.OnChainIDFromSDK(result.PieceIDs[0]) - evidenceCtx, evidenceCancel := providerEvidenceContext(ctx) - err = e.repos.Uploads.MarkUploadCopyCommitted(evidenceCtx, repository.MarkUploadCopyCommittedInput{ - StorageUploadCopyID: copyRow.ID, - RequireEligibleCopy: true, - UploadID: upload.ID, - CopyIndex: copyRow.CopyIndex, - PieceCID: pieceCIDString, - PieceID: &pieceID, - RetrievalURL: storageCtx.PieceURL(pieceCID), - CommitExtraDataHex: derefString(copyRow.CommitExtraDataHex), - CommitTransactionID: result.TransactionID, - CommitAttemptID: advance.AttemptID, - CommitConfirmedTransactionID: result.ConfirmedTransactionID, - }) - evidenceCancel() - if err != nil { - return err - } - return e.finish(ctx, token, upload) -} - -func (e *ReplacementTransferExecutor) pull( - ctx context.Context, - storageCtx synapse.DataSetTarget, - copyRow *model.StorageUploadCopy, - sources []repository.ReadableStorageCopy, -) (*pulledReplacementPiece, error) { - for i := range sources { - source := &sources[i] - pieceCID, err := cid.Decode(source.PieceCID) - if err != nil { - continue - } - extraData, extraHex, err := e.support.extraDataForCopy( - ctx, storageCtx, copyRow, []storage.PieceInput{{PieceCID: pieceCID}}, - ) - if err != nil { - return nil, err - } - if _, err := storageCtx.Pull(ctx, storage.PullRequest{ - Pieces: []cid.Cid{pieceCID}, - ExtraData: extraData, - From: func(cid.Cid) string { - return source.RetrievalURL - }, - }); err != nil { - if replacementSourceUnusable(err) { - continue - } - return nil, err - } - return &pulledReplacementPiece{pieceCID: pieceCID, pieceCIDString: source.PieceCID, extraHex: extraHex}, nil - } - return nil, nil -} - -func (e *ReplacementTransferExecutor) storeFromCache( - ctx context.Context, - storageCtx synapse.DataSetTarget, - bucket *model.Bucket, - version *model.ObjectVersion, -) (bool, cid.Cid, error) { - if version == nil || version.CacheKey == "" { - return false, cid.Undef, nil - } - rc, _, err := e.support.cache.Get(ctx, bucket.Name, version.CacheKey) - if err != nil { - if os.IsNotExist(err) { - if version.InCache { - if markErr := e.repos.Objects.SetVersionCachePresence(ctx, version.VersionID, false); markErr != nil { - e.logger.Warn("failed to mark cache location absent", "versionID", version.VersionID, "error", markErr) - } - } - return false, cid.Undef, nil - } - return false, cid.Undef, fmt.Errorf("open retained cache data: %w", err) - } - result, storeErr := storageCtx.Store(ctx, rc, &storage.StoreOptions{}) - closeErr := rc.Close() - if storeErr != nil { - return false, cid.Undef, storeErr - } - if closeErr != nil { - return false, cid.Undef, fmt.Errorf("close retained cache data: %w", closeErr) - } - if result == nil || !result.PieceCID.Defined() { - return false, cid.Undef, errors.New("replacement store returned no piece CID") - } - return true, result.PieceCID, nil -} - -func (e *ReplacementTransferExecutor) finish( - ctx context.Context, - token storagereplacement.ClaimToken, - upload *model.StorageUpload, -) error { - // Finalize first so a crash leaves the claim recoverable. - return finalizeReplacementItem( - func() error { - _, _, err := e.repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, e.support.finalizeUploadInput(upload.ID)) - return err - }, - func() error { return e.repos.Replacements.CompleteReplacementItemClaim(ctx, token) }, - ) -} - -func finalizeReplacementItem(finalizeUpload, completeClaim func() error) error { - if err := finalizeUpload(); err != nil { - return err - } - return completeClaim() -} - -type replacementTargetContextRegistry struct { - mu sync.Mutex - entries map[int64]*replacementTargetContextEntry -} - -type replacementTargetContextEntry struct { - ready chan struct{} - storageCtx synapse.DataSetTarget - err error - refs int -} - -type replacementTargetContextHandle struct { - registry *replacementTargetContextRegistry - targetID int64 - entry *replacementTargetContextEntry - storageCtx synapse.DataSetTarget - once sync.Once -} - -func newReplacementTargetContextRegistry() *replacementTargetContextRegistry { - return &replacementTargetContextRegistry{entries: make(map[int64]*replacementTargetContextEntry)} -} - -func (r *replacementTargetContextRegistry) acquire( - ctx context.Context, - targetID int64, - create func() (synapse.DataSetTarget, error), -) (*replacementTargetContextHandle, error) { - r.mu.Lock() - entry, exists := r.entries[targetID] - if !exists { - entry = &replacementTargetContextEntry{ - ready: make(chan struct{}), - } - r.entries[targetID] = entry - } - // Count creators and waiters before publishing the unlocked entry. A handle - // cannot be evicted while another acquire is still waiting for creation. - entry.refs++ - r.mu.Unlock() - - if !exists { - entry.storageCtx, entry.err = create() - if entry.err != nil { - r.mu.Lock() - if r.entries[targetID] == entry { - delete(r.entries, targetID) - } - r.mu.Unlock() - } - close(entry.ready) - } - select { - case <-ctx.Done(): - r.release(targetID, entry) - return nil, context.Cause(ctx) - case <-entry.ready: - } - if entry.err != nil { - r.release(targetID, entry) - return nil, entry.err - } - return &replacementTargetContextHandle{ - registry: r, targetID: targetID, entry: entry, storageCtx: entry.storageCtx, - }, nil -} - -func (r *replacementTargetContextRegistry) release(targetID int64, entry *replacementTargetContextEntry) { - r.mu.Lock() - defer r.mu.Unlock() - if entry.refs > 0 { - entry.refs-- - } - if entry.refs == 0 && r.entries[targetID] == entry { - delete(r.entries, targetID) - } -} - -func (h *replacementTargetContextHandle) release() { - if h == nil || h.registry == nil || h.entry == nil { - return - } - h.once.Do(func() { - h.registry.release(h.targetID, h.entry) - }) -} diff --git a/internal/worker/provider_replacement_worker_internal_test.go b/internal/worker/provider_replacement_worker_internal_test.go deleted file mode 100644 index 5df1bbb..0000000 --- a/internal/worker/provider_replacement_worker_internal_test.go +++ /dev/null @@ -1,283 +0,0 @@ -package worker - -import ( - "context" - "errors" - "io" - "log/slog" - "sync/atomic" - "testing" - "time" - - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/strahe/synaps3/internal/synapse" -) - -type transientReplacementLeaseRepo struct { - repository.StorageReplacementRepository - calls atomic.Int32 - renewed chan struct{} -} - -func (r *transientReplacementLeaseRepo) RenewReplacementItemLease( - context.Context, - storagereplacement.ClaimToken, - time.Duration, -) error { - if r.calls.Add(1) == 1 { - return errors.New("injected transient renewal failure") - } - if r.calls.Load() == 2 { - close(r.renewed) - } - return nil -} - -func TestReplacementItemLeaseRenewalRetriesTransientFailure(t *testing.T) { - leaseTTL := 60 * time.Millisecond - leaseUntil := time.Now().Add(time.Second) - replacementRepo := &transientReplacementLeaseRepo{renewed: make(chan struct{})} - w := &ProviderReplacementWorker{ - repos: &repository.Repositories{Replacements: replacementRepo}, - leaseTTL: leaseTTL, - logger: slog.New(slog.NewTextHandler(io.Discard, nil)), - } - - itemCtx, cancelItem := context.WithCancel(context.Background()) - leaseLost, stopRenewal := w.startLeaseRenewal( - itemCtx, - cancelItem, - storagereplacement.ClaimToken{ItemID: 42, ClaimedAt: time.Now()}, - &leaseUntil, - ) - defer stopRenewal() - defer cancelItem() - - select { - case <-replacementRepo.renewed: - case <-time.After(time.Second): - t.Fatal("lease renewal did not retry after a transient error") - } - if leaseLost() { - t.Fatal("transient lease renewal error marked the claim as lost") - } - if itemCtx.Err() != nil { - t.Fatalf("transient lease renewal error cancelled item context: %v", itemCtx.Err()) - } -} - -type recordingReplacementPauser struct { - replacementID int64 - expectedStateVersion int64 - reason storagereplacement.WaitReason -} - -func (p *recordingReplacementPauser) PauseMigration( - _ context.Context, - replacementID, expectedStateVersion int64, - reason storagereplacement.WaitReason, -) error { - p.replacementID = replacementID - p.expectedStateVersion = expectedStateVersion - p.reason = reason - return nil -} - -func TestReplacementTargetPauseUsesExecutionSnapshotVersion(t *testing.T) { - t.Parallel() - - pauser := new(recordingReplacementPauser) - if err := pauseReplacementAtExecutionVersion(context.Background(), pauser, 42, 7); err != nil { - t.Fatalf("pauseReplacementAtExecutionVersion: %v", err) - } - if pauser.replacementID != 42 || pauser.expectedStateVersion != 7 || pauser.reason != storagereplacement.WaitReasonTarget { - t.Fatalf("pause request = %#v, want replacement 42 at execution version 7", pauser) - } -} - -func TestFinalizeReplacementItemKeepsClaimRetryableUntilUploadFinalizes(t *testing.T) { - t.Parallel() - - injected := errors.New("injected upload finalization failure") - completed := false - err := finalizeReplacementItem( - func() error { return injected }, - func() error { - completed = true - return nil - }, - ) - if !errors.Is(err, injected) { - t.Fatalf("finalizeReplacementItem error = %v, want injected failure", err) - } - if completed { - t.Fatal("claim completed even though upload finalization failed") - } - - steps := make([]string, 0, 2) - if err := finalizeReplacementItem( - func() error { - steps = append(steps, "finalize") - return nil - }, - func() error { - steps = append(steps, "complete") - return nil - }, - ); err != nil { - t.Fatalf("finalizeReplacementItem success: %v", err) - } - if len(steps) != 2 || steps[0] != "finalize" || steps[1] != "complete" { - t.Fatalf("settlement order = %v, want [finalize complete]", steps) - } -} - -func TestReplacementTargetContextRegistrySharesActiveContextAndRefreshesAfterRelease(t *testing.T) { - t.Parallel() - - registry := newReplacementTargetContextRegistry() - var creates atomic.Int32 - create := func() (synapse.DataSetTarget, error) { - creates.Add(1) - return nil, nil - } - - first, err := registry.acquire(context.Background(), 42, create) - if err != nil { - t.Fatalf("acquire first handle: %v", err) - } - second, err := registry.acquire(context.Background(), 42, create) - if err != nil { - t.Fatalf("acquire second handle: %v", err) - } - first.release() - second.release() - - if got := creates.Load(); got != 1 { - t.Fatalf("context creates = %d, want 1", got) - } - if len(registry.entries) != 0 { - t.Fatalf("registry entries after final release = %d, want 0", len(registry.entries)) - } - - third, err := registry.acquire(context.Background(), 42, create) - if err != nil { - t.Fatalf("acquire refreshed handle: %v", err) - } - third.release() - if got := creates.Load(); got != 2 { - t.Fatalf("context creates after refresh = %d, want 2", got) - } -} - -func TestReplacementTargetContextRegistryDoesNotSerializeCommitsForSameTarget(t *testing.T) { - t.Parallel() - - registry := newReplacementTargetContextRegistry() - create := func() (synapse.DataSetTarget, error) { return nil, nil } - first, err := registry.acquire(context.Background(), 42, create) - if err != nil { - t.Fatalf("acquire first handle: %v", err) - } - defer first.release() - second, err := registry.acquire(context.Background(), 42, create) - if err != nil { - t.Fatalf("acquire second handle: %v", err) - } - defer second.release() - - firstEntered := make(chan struct{}) - releaseFirst := make(chan struct{}) - firstDone := make(chan error, 1) - go func() { - firstDone <- first.commit(context.Background(), func() error { - close(firstEntered) - <-releaseFirst - return nil - }) - }() - <-firstEntered - - secondEntered := make(chan struct{}) - secondDone := make(chan error, 1) - go func() { - secondDone <- second.commit(context.Background(), func() error { - close(secondEntered) - return nil - }) - }() - select { - case <-secondEntered: - case <-time.After(time.Second): - t.Fatal("second commit remained blocked by the removed target gate") - } - close(releaseFirst) - if err := <-firstDone; err != nil { - t.Fatalf("first commit: %v", err) - } - select { - case <-secondEntered: - case <-time.After(time.Second): - t.Fatal("second commit did not enter after the target gate was released") - } - if err := <-secondDone; err != nil { - t.Fatalf("second commit: %v", err) - } -} - -func (h *replacementTargetContextHandle) commit(ctx context.Context, commit func() error) error { - if err := context.Cause(ctx); err != nil { - return err - } - return commit() -} - -func TestReplacementTargetContextRegistryAllowsDifferentTargetsToCommit(t *testing.T) { - t.Parallel() - - registry := newReplacementTargetContextRegistry() - create := func() (synapse.DataSetTarget, error) { return nil, nil } - first, err := registry.acquire(context.Background(), 42, create) - if err != nil { - t.Fatalf("acquire first handle: %v", err) - } - defer first.release() - second, err := registry.acquire(context.Background(), 43, create) - if err != nil { - t.Fatalf("acquire second handle: %v", err) - } - defer second.release() - - entered := make(chan int, 2) - release := make(chan struct{}) - done := make(chan error, 2) - run := func(id int, handle *replacementTargetContextHandle) { - done <- handle.commit(context.Background(), func() error { - entered <- id - <-release - return nil - }) - } - go run(42, first) - go run(43, second) - - seen := make(map[int]bool, 2) - for range 2 { - select { - case id := <-entered: - seen[id] = true - case <-time.After(time.Second): - t.Fatal("different target commits did not enter concurrently") - } - } - close(release) - for range 2 { - if err := <-done; err != nil { - t.Fatalf("commit: %v", err) - } - } - if !seen[42] || !seen[43] { - t.Fatalf("entered targets = %v, want both", seen) - } -} diff --git a/internal/worker/replacement_task_handlers.go b/internal/worker/replacement_task_handlers.go new file mode 100644 index 0000000..1afaf48 --- /dev/null +++ b/internal/worker/replacement_task_handlers.go @@ -0,0 +1,456 @@ +package worker + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "time" + + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagereplacement" + taskengine "github.com/strahe/synaps3/internal/task" +) + +const replacementSeedBatchSize = 100 + +type retirementCheckpoint struct { + AttemptedAt time.Time `json:"attempted_at"` + TerminationEpoch *int64 `json:"termination_epoch,omitempty"` + TransactionHash string `json:"transaction_hash,omitempty"` +} + +func (h *TaskHandlers) replacementCoordinateHandler() taskengine.Handler { + definition := taskengine.Definition{ + Type: model.TaskTypeProviderReplacementCoordinate, InputVersion: 1, + Codec: taskengine.StrictJSONCodec(func(input *storagereplacement.CoordinateInput) error { + return storagereplacement.ValidateCoordinateInput(*input) + }), + RetryLimit: h.retryLimit(), AllowRetry: false, + } + run := func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + input, err := taskengine.DecodeInput[storagereplacement.CoordinateInput](execution) + if err != nil { + return decodeFailure(string(definition.Type), err) + } + row, err := h.deps.Repositories.Replacements.AuthorizeTask(ctx, input.ReplacementID, input.Generation, execution.ID()) + if errors.Is(err, repository.ErrConflict) || errors.Is(err, repository.ErrNotFound) { + return taskengine.Cancel("Provider replacement was superseded", nil) + } + if err != nil { + return h.retryReplacement(execution, input.ReplacementID, err, "replacement_authorization_failed") + } + if row.Status == storagereplacement.StatusSuperseded { + return h.scheduleReplacementRetirement(input, execution.ID(), row, row.TargetDataSetID, "Unused storage service cleanup scheduled") + } + if row.Status == storagereplacement.StatusCompleted { + return taskengine.Complete("Provider replacement completed", func(ctx context.Context, repos *repository.Repositories) error { + return repos.Replacements.CompleteTask(ctx, row.ID, input.Generation, execution.ID()) + }) + } + if row.Status == storagereplacement.StatusFailed || row.Status == storagereplacement.StatusCleanupAttention { + return taskengine.Fail(errors.New("provider replacement requires operator action"), "replacement_attention", nil) + } + if row.Status == storagereplacement.StatusWaiting && row.WaitReason != nil && !row.WaitReason.Valid() { + return taskengine.Fail(fmt.Errorf("provider replacement has unknown wait reason %q", *row.WaitReason), "replacement_unknown_wait_reason", nil) + } + + target, err := h.deps.Repositories.Contents.GetDataSetBindingByID(ctx, row.TargetDataSetID) + if err != nil || target == nil { + if err == nil { + err = repository.ErrNotFound + } + return h.failReplacement(row.ID, err, "replacement_target_missing") + } + if target.Status != model.StorageDataSetStatusReady || target.DataSetID == nil || target.DataSetID.IsZero() { + if target.Status == model.StorageDataSetStatusFailed { + return h.failReplacement(row.ID, errors.New("replacement target storage service failed"), "replacement_target_failed") + } + if target.EnsureTaskID == nil { + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "target", "Preparing replacement storage service", func(ctx context.Context, repos *repository.Repositories) error { + if err := h.enqueueDataSetEnsure(ctx, repos, target); err != nil { + return err + } + return repos.Replacements.MarkWaiting(ctx, row.ID, storagereplacement.WaitReasonTargetCreating) + }) + } + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "target", "Waiting for replacement storage service", func(ctx context.Context, repos *repository.Repositories) error { + return repos.Replacements.MarkWaiting(ctx, row.ID, storagereplacement.WaitReasonTargetCreating) + }) + } + + if !target.IsCurrent { + return taskengine.Suspend(model.TaskResumeModeRecover, 0, "activation", "Activating replacement storage service", func(ctx context.Context, repos *repository.Repositories) error { + return repos.Replacements.Activate(ctx, row.ID) + }) + } + if row.Status == storagereplacement.StatusWaiting { + if err := h.deps.Repositories.Replacements.MarkMigrating(ctx, row.ID); err != nil && !errors.Is(err, repository.ErrConflict) { + return h.retryReplacement(execution, row.ID, err, "replacement_state_failed") + } + } + + inserted, seeded, err := h.deps.Repositories.Replacements.SeedMigrationBatch(ctx, row.ID, replacementSeedBatchSize) + if err != nil { + return h.retryReplacement(execution, row.ID, err, "replacement_seed_failed") + } + if !seeded || inserted > 0 { + return taskengine.Suspend(model.TaskResumeModeRecover, 0, "seeding", "Preparing stored content for migration", nil) + } + item, err := h.deps.Repositories.Replacements.NextPendingReplacementItem(ctx, row.ID) + if err != nil { + return h.retryReplacement(execution, row.ID, err, "replacement_item_scan_failed") + } + if item != nil { + return h.coordinateReplacementItem(ctx, execution, row, item) + } + snapshot, err := h.deps.Repositories.Replacements.ReplacementExecution(ctx, row.ID) + if err != nil { + return h.retryReplacement(execution, row.ID, err, "replacement_progress_failed") + } + if snapshot.HasFailed { + return h.failReplacement(row.ID, errors.New("stored content migration requires attention"), "replacement_item_attention") + } + if !snapshot.SeedingComplete || snapshot.HasPending { + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "copy_work", "Waiting for stored content migration", nil) + } + return h.scheduleReplacementRetirement(input, execution.ID(), row, row.SourceDataSetID, "Old storage service retirement scheduled") + } + return taskHandler{definition: definition, execute: run, recover: run} +} + +func (h *TaskHandlers) coordinateReplacementItem( + ctx context.Context, + execution taskengine.Execution, + replacement *storagereplacement.Replacement, + item *storagereplacement.Item, +) taskengine.Result { + snapshot, err := h.deps.Repositories.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ + ReplacementID: replacement.ID, ItemID: item.ID, + }) + if errors.Is(err, storagereplacement.ErrItemCancelled) { + return taskengine.Suspend(model.TaskResumeModeRecover, 0, "copy_work", "Continuing stored content migration", nil) + } + if errors.Is(err, storagereplacement.ErrItemDeferred) { + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "source", "Waiting for readable stored content", func(ctx context.Context, repos *repository.Repositories) error { + return repos.Replacements.MarkWaiting(ctx, replacement.ID, storagereplacement.WaitReasonReadableSource) + }) + } + if err != nil { + return h.retryReplacement(execution, replacement.ID, err, "replacement_item_load_failed") + } + if snapshot == nil { + return taskengine.Suspend(model.TaskResumeModeRecover, 0, "copy_work", "Continuing stored content migration", nil) + } + copyRow, err := h.deps.Repositories.Replacements.AttachTargetCopy(ctx, repository.AttachReplacementTargetCopyInput{ + ReplacementID: replacement.ID, ItemID: item.ID, ContentID: snapshot.Upload.ID, + }) + if err != nil { + return h.retryReplacement(execution, replacement.ID, err, "replacement_copy_attach_failed") + } + if copyRow.Status == model.StorageCopyStatusCommitted { + return taskengine.Suspend(model.TaskResumeModeRecover, 0, "copy_work", "Stored content migrated", func(ctx context.Context, repos *repository.Repositories) error { + return repos.Replacements.MarkReplacementItemCopied(ctx, replacement.ID, item.ID, copyRow.ID) + }) + } + if copyRow.Status == model.StorageCopyStatusFailed && copyRow.ActiveTaskID == nil { + return taskengine.Suspend(model.TaskResumeModeRecover, 0, "copy_work", "Restarting stored content migration", func(ctx context.Context, repos *repository.Repositories) error { + if err := repos.Contents.ReopenFailedUploadCopy(ctx, copyRow.ID); err != nil { + return err + } + return h.enqueueInitialCopyTask(ctx, repos, copyRow.ID, model.TaskTypeStorageTransferPlan) + }) + } + if copyRow.ActiveTaskID == nil { + return taskengine.Suspend(model.TaskResumeModeRecover, 0, "copy_work", "Migrating stored content", func(ctx context.Context, repos *repository.Repositories) error { + return h.enqueueInitialCopyTask(ctx, repos, copyRow.ID, model.TaskTypeStorageTransferPlan) + }) + } + copyTask, err := h.deps.Repositories.Tasks.GetByID(ctx, *copyRow.ActiveTaskID) + if err != nil { + return h.retryReplacement(execution, replacement.ID, err, "replacement_copy_task_load_failed") + } + if copyTask != nil && copyTask.Status == model.TaskStatusFailed { + message := "stored content migration failed" + if copyTask.LastError != nil { + message = *copyTask.LastError + } + return taskengine.Fail(errors.New(message), "replacement_copy_failed", func(ctx context.Context, repos *repository.Repositories) error { + if err := repos.Replacements.MarkReplacementItemAttention(ctx, replacement.ID, item.ID, message); err != nil { + return err + } + return repos.Replacements.MarkFailed(ctx, replacement.ID, nil, message) + }) + } + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "copy_work", "Waiting for stored content migration", nil) +} + +func (h *TaskHandlers) failReplacement(replacementID int64, err error, reason string) taskengine.Result { + return taskengine.Fail(err, reason, func(ctx context.Context, repos *repository.Repositories) error { + return repos.Replacements.MarkFailed(ctx, replacementID, nil, err.Error()) + }) +} + +func (h *TaskHandlers) retryReplacement( + execution taskengine.Execution, + replacementID int64, + err error, + reason string, +) taskengine.Result { + if !execution.RetryWillFail() { + return retryTask(err, reason) + } + return h.failReplacement(replacementID, err, reason) +} + +func (h *TaskHandlers) scheduleReplacementRetirement( + input storagereplacement.CoordinateInput, + taskID int64, + row *storagereplacement.Replacement, + dataSetID int64, + message string, +) taskengine.Result { + return taskengine.Complete(message, func(ctx context.Context, repos *repository.Repositories) error { + if h.taskService == nil { + return errors.New("task service is unavailable") + } + dataSet, err := repos.Contents.GetDataSetBindingByID(ctx, dataSetID) + if err != nil || dataSet == nil { + return errors.Join(err, repository.ErrNotFound) + } + if dataSet.Status == model.StorageDataSetStatusRetired { + return repos.Replacements.CompleteTask(ctx, row.ID, input.Generation, taskID) + } + if dataSet.RetirementTaskID == nil { + generation, err := repos.Contents.NextDataSetRetirementGeneration(ctx, dataSetID) + if err != nil { + return err + } + retireInput := storagereplacement.RetireInput{ + ReplacementID: row.ID, DataSetID: dataSetID, Generation: generation, + } + retireTask, _, err := h.taskService.EnqueueInTransaction(ctx, repos, taskengine.EnqueueRequest{ + Type: model.TaskTypeStorageDataSetRetire, + IdempotencyKey: storagereplacement.RetireTaskKey(dataSetID, generation), Input: retireInput, + SubjectType: "storage_data_set", SubjectKey: fmt.Sprintf("%d", dataSetID), + }) + if err != nil { + return err + } + if err := repos.Contents.BindDataSetRetirementTask(ctx, dataSetID, generation, retireTask.ID); err != nil { + return err + } + } + if row.Status != storagereplacement.StatusSuperseded { + if err := repos.Replacements.BeginRetirement(ctx, row.ID); err != nil && !errors.Is(err, repository.ErrConflict) { + return err + } + } + return repos.Replacements.CompleteTask(ctx, row.ID, input.Generation, taskID) + }) +} + +func (h *TaskHandlers) dataSetRetireHandler() taskengine.Handler { + definition := taskengine.Definition{ + Type: model.TaskTypeStorageDataSetRetire, InputVersion: 1, + Codec: taskengine.StrictJSONCodec(func(input *storagereplacement.RetireInput) error { + return storagereplacement.ValidateRetireInput(*input) + }), + RetryLimit: h.retryLimit(), AllowRetry: true, + CanManualRetry: func(task *model.Task) bool { + return task == nil || task.FailureReason == nil || *task.FailureReason != "termination_outcome_unknown" + }, + } + return taskHandler{ + definition: definition, + execute: func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + return h.runDataSetRetirement(ctx, execution, true) + }, + recover: func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + return h.runDataSetRetirement(ctx, execution, false) + }, + } +} + +func (h *TaskHandlers) runDataSetRetirement(ctx context.Context, execution taskengine.Execution, mayTerminate bool) taskengine.Result { + input, err := taskengine.DecodeInput[storagereplacement.RetireInput](execution) + if err != nil { + return decodeFailure(string(model.TaskTypeStorageDataSetRetire), err) + } + dataSet, err := h.deps.Repositories.Contents.AuthorizeDataSetRetirementTask(ctx, input.DataSetID, input.Generation, execution.ID()) + if errors.Is(err, repository.ErrConflict) || errors.Is(err, repository.ErrNotFound) { + return taskengine.Cancel("Storage service retirement was superseded", nil) + } + if err != nil { + return h.retryRetirement(execution, input.ReplacementID, err, "retirement_authorization_failed") + } + row, err := h.deps.Repositories.Replacements.GetByID(ctx, input.ReplacementID) + if err != nil || row == nil { + if err == nil { + err = repository.ErrNotFound + } + return taskengine.Fail(err, "replacement_missing", nil) + } + abandoned := row.Status == storagereplacement.StatusSuperseded && input.DataSetID == row.TargetDataSetID + if !abandoned && input.DataSetID != row.SourceDataSetID { + return taskengine.Fail(errors.New("retirement data set does not belong to replacement"), "retirement_identity_mismatch", nil) + } + if dataSet.Status == model.StorageDataSetStatusRetired { + return taskengine.Complete("Storage service retired", func(ctx context.Context, repos *repository.Repositories) error { + return repos.Contents.CompleteDataSetRetirementTask(ctx, input.DataSetID, input.Generation, execution.ID()) + }) + } + if dataSet.DataSetID == nil || dataSet.DataSetID.IsZero() { + return taskengine.Fail(errors.New("storage service has no remote identity"), "retirement_identity_missing", nil) + } + + var terminationEpoch *int64 + if abandoned { + terminationEpoch = row.AbandonedTerminationEpoch + sole, err := h.deps.Repositories.Replacements.CountAbandonedTargetSoleCopies(ctx, row.TargetDataSetID) + if err != nil { + return h.retryRetirement(execution, row.ID, err, "retirement_gate_failed") + } + if sole > 0 { + return taskengine.Fail(fmt.Errorf("unused storage service holds %d sole copies", sole), "retirement_coverage", nil) + } + } else { + terminationEpoch = row.TerminationEpoch + gate, err := h.deps.Repositories.Replacements.EvaluateRetirementGate(ctx, row.ID, nil) + if err != nil { + return h.retryRetirement(execution, row.ID, err, "retirement_gate_failed") + } + if len(gate.Blockers) > 0 { + if containsString(gate.Blockers, "slot_ownership") { + err := fmt.Errorf("retirement safety gate failed: %s", strings.Join(gate.Blockers, ", ")) + return taskengine.Fail(err, "retirement_gate_invalid", func(ctx context.Context, repos *repository.Repositories) error { + return repos.Replacements.MarkCleanupAttention(ctx, row.ID, err.Error()) + }) + } + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "retirement_gate", "Waiting for safe storage service retirement", nil) + } + } + + checkpoint, hasCheckpoint, err := taskengine.DecodeCheckpoint[retirementCheckpoint](execution) + if err != nil { + return taskengine.Fail(err, "invalid_checkpoint", nil) + } + if terminationEpoch == nil && checkpoint.TerminationEpoch != nil { + if *checkpoint.TerminationEpoch < 0 { + return taskengine.Fail(errors.New("storage service retirement checkpoint has an invalid epoch"), "invalid_checkpoint", nil) + } + return taskengine.Suspend(model.TaskResumeModeRecover, 0, "termination_epoch", "Recording storage service retirement", retirementEvidenceSettlement( + abandoned, row.ID, *checkpoint.TerminationEpoch, checkpoint.TransactionHash, + )) + } + if terminationEpoch == nil { + if hasCheckpoint { + err := errors.New("storage service termination outcome could not be recovered") + if abandoned { + return taskengine.Fail(err, "termination_outcome_unknown", nil) + } + return taskengine.Fail(err, "termination_outcome_unknown", func(ctx context.Context, repos *repository.Repositories) error { + return repos.Replacements.MarkCleanupAttention(ctx, row.ID, err.Error()) + }) + } + if !mayTerminate { + return taskengine.Suspend(model.TaskResumeModeExecute, 0, "safe_to_execute", "Storage service is ready to retire", nil) + } + if h.deps.Terminator == nil { + return taskengine.Fail(errors.New("storage service terminator is unavailable"), "dependency_unavailable", nil) + } + checkpoint = retirementCheckpoint{AttemptedAt: time.Now().UTC()} + var terminationEpochValue int64 + var txHash string + attempted, err := execution.WithCheckpointedEffect(ctx, taskengine.ResourceDestructiveMutation, checkpoint, nil, func(ctx context.Context) error { + result, terminateErr := h.deps.Terminator.TerminateService(ctx, dataSet.DataSetID.SDK()) + if result != nil { + terminationEpochValue = result.EndEpoch + txHash = result.TxHash + } + return terminateErr + }) + if err != nil && !attempted { + return retryTask(err, "termination_not_started") + } + if err != nil || terminationEpochValue < 0 { + if err == nil { + err = errors.New("storage service termination returned an invalid epoch") + } + if abandoned { + return taskengine.Fail(err, "termination_outcome_unknown", nil) + } + return taskengine.Fail(err, "termination_outcome_unknown", func(ctx context.Context, repos *repository.Repositories) error { + return repos.Replacements.MarkCleanupAttention(ctx, row.ID, err.Error()) + }) + } + checkpoint.TerminationEpoch = &terminationEpochValue + checkpoint.TransactionHash = txHash + settlement := retirementEvidenceSettlement(abandoned, row.ID, terminationEpochValue, txHash) + if err := execution.WriteCheckpoint(ctx, checkpoint); err != nil { + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "termination_epoch", "Recording storage service retirement", settlement) + } + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "termination_epoch", "Waiting for storage service retirement", settlement) + } + if h.deps.Epochs == nil { + return taskengine.Fail(errors.New("chain epoch reader is unavailable"), "dependency_unavailable", nil) + } + observedEpoch, err := h.deps.Epochs.CurrentEpoch(ctx) + if err != nil { + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "termination_epoch", "Checking storage service retirement", nil) + } + if observedEpoch < *terminationEpoch { + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "termination_epoch", "Waiting for storage service retirement", nil) + } + return taskengine.Complete("Storage service retired", func(ctx context.Context, repos *repository.Repositories) error { + if abandoned { + if err := repos.Replacements.CompleteAbandonedTargetTermination(ctx, row.ID); err != nil { + return err + } + } else if err := repos.Replacements.CompleteRetirement(ctx, row.ID, observedEpoch); err != nil { + return err + } + return repos.Contents.CompleteDataSetRetirementTask(ctx, input.DataSetID, input.Generation, execution.ID()) + }) +} + +func retirementEvidenceSettlement(abandoned bool, replacementID, epoch int64, transactionHash string) taskengine.Settlement { + return func(ctx context.Context, repos *repository.Repositories) error { + evidence := repository.RecordTerminationEpochInput{ + ReplacementID: replacementID, + TxHash: transactionHash, + Epoch: epoch, + } + if abandoned { + return repos.Replacements.RecordAbandonedTerminationEpoch(ctx, evidence) + } + return repos.Replacements.RecordTerminationEpoch(ctx, evidence) + } +} + +func (h *TaskHandlers) retryRetirement( + execution taskengine.Execution, + replacementID int64, + err error, + reason string, +) taskengine.Result { + if !execution.RetryWillFail() { + return retryTask(err, reason) + } + return taskengine.Fail(err, reason, func(ctx context.Context, repos *repository.Repositories) error { + replacement, loadErr := repos.Replacements.GetByID(ctx, replacementID) + if loadErr != nil { + return loadErr + } + if replacement == nil || replacement.Status == storagereplacement.StatusSuperseded { + return nil + } + return repos.Replacements.MarkCleanupAttention(ctx, replacementID, err.Error()) + }) +} + +func containsString(values []string, wanted string) bool { + return slices.Contains(values, wanted) +} diff --git a/internal/worker/revert.go b/internal/worker/revert.go deleted file mode 100644 index c7a1136..0000000 --- a/internal/worker/revert.go +++ /dev/null @@ -1,71 +0,0 @@ -package worker - -import ( - "encoding/hex" - "fmt" - "regexp" - "strings" -) - -// knownSelector maps 4-byte function selectors to human-readable error names -// and a decoder that turns the remaining ABI-encoded data into a string. -var knownSelectors = map[string]struct { - name string - decode func(data []byte) string -}{ - "42d750dc": { - name: "InvalidSignature(address,address)", - decode: func(data []byte) string { - if len(data) < 64 { - return "" - } - expected := formatAddress(data[:32]) - actual := formatAddress(data[32:64]) - return fmt.Sprintf("expected signer %s, recovered %s (possible cause: insufficient payment account balance)", expected, actual) - }, - }, -} - -// revertReasonRe matches "revert reason=[0x]" inside error strings. -var revertReasonRe = regexp.MustCompile(`revert reason=\[0x([0-9a-fA-F]+)\]`) - -// decodeRevertReason inspects errMsg for a revert reason hex pattern and, if a -// known selector is found, returns an enriched error message. If nothing can be -// decoded the original message is returned unchanged. -func decodeRevertReason(errMsg string) string { - m := revertReasonRe.FindStringSubmatch(errMsg) - if len(m) < 2 { - return errMsg - } - - raw, err := hex.DecodeString(m[1]) - if err != nil || len(raw) < 4 { - return errMsg - } - - selector := hex.EncodeToString(raw[:4]) - entry, ok := knownSelectors[selector] - if !ok { - return errMsg - } - - detail := entry.decode(raw[4:]) - var decoded string - if detail != "" { - decoded = fmt.Sprintf("[decoded revert: %s — %s]", entry.name, detail) - } else { - decoded = fmt.Sprintf("[decoded revert: %s]", entry.name) - } - - return errMsg + " " + decoded -} - -// formatAddress extracts an Ethereum address from a 32-byte ABI-encoded word. -func formatAddress(word []byte) string { - if len(word) < 32 { - return "0x" + hex.EncodeToString(word) - } - // ABI address is right-aligned in 32 bytes (last 20 bytes) - addr := word[12:32] - return "0x" + strings.ToLower(hex.EncodeToString(addr)) -} diff --git a/internal/worker/revert_test.go b/internal/worker/revert_test.go deleted file mode 100644 index 4f49a47..0000000 --- a/internal/worker/revert_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package worker - -import ( - "strings" - "testing" -) - -func TestDecodeRevertReason_InvalidSignature(t *testing.T) { - // Real-world error string from task ID 11 (trimmed for readability). - errMsg := `failed to ensure data set: failed to create data set: unexpected status 500: ` + - `chain: message execution failed (exit=[33]), revert reason=[0x42d750dc` + - `0000000000000000000000008c04a1ad757bc02069489dde155fd1379c58980e` + - `000000000000000000000000ee4758fcd302f8eb61977d50d3891c6e66ca64d8]` - - got := decodeRevertReason(errMsg) - - if got == errMsg { - t.Fatal("expected decoded message, got original unchanged") - } - - // Should contain the decoded revert info. - wantSubstrings := []string{ - "InvalidSignature(address,address)", - "0x8c04a1ad757bc02069489dde155fd1379c58980e", - "0xee4758fcd302f8eb61977d50d3891c6e66ca64d8", - "insufficient payment account balance", - } - for _, s := range wantSubstrings { - if !strings.Contains(got, s) { - t.Errorf("decoded message missing %q\ngot: %s", s, got) - } - } -} - -func TestDecodeRevertReason_UnknownSelector(t *testing.T) { - errMsg := `revert reason=[0xdeadbeef0000000000000000000000000000000000000000000000000000000000000001]` - got := decodeRevertReason(errMsg) - if got != errMsg { - t.Errorf("expected original message for unknown selector, got: %s", got) - } -} - -func TestDecodeRevertReason_NoRevertPattern(t *testing.T) { - errMsg := "connection refused" - got := decodeRevertReason(errMsg) - if got != errMsg { - t.Errorf("expected original message when no revert pattern, got: %s", got) - } -} - -func TestDecodeRevertReason_SelectorOnly(t *testing.T) { - // Selector matches but data too short for address decoding. - errMsg := `revert reason=[0x42d750dc00]` - got := decodeRevertReason(errMsg) - - if !strings.Contains(got, "InvalidSignature(address,address)") { - t.Errorf("expected selector name in output, got: %s", got) - } -} diff --git a/internal/worker/storage_cleanup.go b/internal/worker/storage_cleanup.go deleted file mode 100644 index dbd45fe..0000000 --- a/internal/worker/storage_cleanup.go +++ /dev/null @@ -1,317 +0,0 @@ -package worker - -import ( - "context" - "errors" - "fmt" - "log/slog" - "sync" - "time" - - "github.com/ipfs/go-cid" - "github.com/strahe/synaps3/internal/admin" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/strahe/synaps3/internal/synapse" - "github.com/strahe/synapse-go/storage" -) - -type StorageCleanupWorker struct { - repos *repository.Repositories - // terminator and epochs are only used by provider replacement retirement, - // which is the sole destructive service-lifecycle path. - terminator synapse.ServiceTerminator - epochs synapse.ChainEpochReader - storage synapse.StorageClient - concurrency int - pollInterval time.Duration - leaseTTL time.Duration - logger *slog.Logger - *livenessTracker -} - -const ( - storageCleanupConfirmationDelay = time.Minute - storageCleanupReferenceDelay = time.Minute -) - -var errStorageCleanupCopyUnsupported = errors.New("storage cleanup copy unsupported") - -// StorageCleanupOption configures optional cleanup dependencies. -type StorageCleanupOption func(*StorageCleanupWorker) - -// WithServiceTermination enables provider replacement retirement. Without it -// the worker still removes pieces but refuses to end any service. -func WithServiceTermination(terminator synapse.ServiceTerminator, epochs synapse.ChainEpochReader) StorageCleanupOption { - return func(w *StorageCleanupWorker) { - w.terminator = terminator - w.epochs = epochs - } -} - -func NewStorageCleanupWorker(repos *repository.Repositories, storageClient synapse.StorageClient, concurrency int, pollInterval time.Duration, logger *slog.Logger, opts ...StorageCleanupOption) *StorageCleanupWorker { - w := &StorageCleanupWorker{ - repos: repos, - storage: storageClient, - concurrency: concurrency, - pollInterval: pollInterval, - leaseTTL: 5 * time.Minute, - logger: logger, - livenessTracker: newLivenessTracker(pollInterval), - } - for _, opt := range opts { - opt(w) - } - return w -} - -func (w *StorageCleanupWorker) Name() string { return "storage_cleanup" } - -func (w *StorageCleanupWorker) Healthy() bool { return w.healthy() } - -func (w *StorageCleanupWorker) Run(ctx context.Context) error { - var wg sync.WaitGroup - for slot := range w.concurrency { - wg.Add(1) - go func(slot int) { - defer wg.Done() - if !sleepStorageCleanupInitialStagger(ctx, slot) { - return - } - w.runSlot(ctx) - }(slot) - } - wg.Wait() - return ctx.Err() -} - -func sleepStorageCleanupInitialStagger(ctx context.Context, slot int) bool { - if slot <= 0 { - return true - } - timer := time.NewTimer(time.Duration(slot) * time.Second) - defer timer.Stop() - select { - case <-ctx.Done(): - return false - case <-timer.C: - return true - } -} - -func (w *StorageCleanupWorker) runSlot(ctx context.Context) { - if !sleepUntilNextWorkerPoll(ctx, w.pollInterval) { - return - } - for { - if ctx.Err() != nil { - return - } - w.recordTick() - task, err := w.repos.Tasks.ClaimReady(ctx, model.TaskTypeStorageCleanup, w.leaseTTL) - if err != nil { - if ctx.Err() != nil { - return - } - w.logger.Error("claiming storage cleanup task", "error", err) - if !sleepUntilNextWorkerPoll(ctx, w.pollInterval) { - return - } - continue - } - if task == nil { - if !sleepUntilNextWorkerPoll(ctx, w.pollInterval) { - return - } - continue - } - w.recordWorkStarted() - func() { - defer w.recordWorkFinished() - stopLeaseRenewal := startTaskLeaseRenewal(w.logger, w.repos, task, w.leaseTTL) - defer stopLeaseRenewal() - w.processTask(ctx, task) - }() - releaseTaskOnWorkerShutdown(ctx, w.logger, w.repos, task) - } -} - -func (w *StorageCleanupWorker) processTask(ctx context.Context, task *model.Task) { - start := time.Now() - defer func() { - admin.WorkerTaskDuration.WithLabelValues("storage_cleanup").Observe(time.Since(start).Seconds()) - }() - - if task.Stage != nil { - switch *task.Stage { - case storagereplacement.StageRetire, storagereplacement.StageRetireAbandonedTarget: - w.processReplacementRetirementTask(ctx, task) - return - } - } - - logger := w.logger.With("taskID", task.ID, "uploadID", task.RefID) - hasRefs, err := w.repos.StorageCleanup.UploadHasObjectReferences(ctx, task.RefID) - if err != nil { - logger.Error("checking storage cleanup references failed", "error", err) - scheduleTaskRetry(ctx, w.repos, task, "storage_cleanup", logger, err) - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() - return - } - if hasRefs { - if !w.waitForReferences(ctx, task, logger, "Waiting for object references to clear") { - return - } - logger.Info("storage cleanup waiting because object references remain") - return - } - hasRefs, err = w.repos.StorageCleanup.TaskHasObjectReferences(ctx, task.ID, task.RefID) - if err != nil { - logger.Error("checking storage cleanup piece references failed", "error", err) - scheduleTaskRetry(ctx, w.repos, task, "storage_cleanup", logger, err) - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() - return - } - if hasRefs { - if !w.waitForReferences(ctx, task, logger, "Waiting for shared data references to clear") { - return - } - logger.Info("storage cleanup waiting because shared data references remain") - return - } - - copies, err := w.repos.StorageCleanup.ListCopiesForTask(ctx, task.ID) - if err != nil { - logger.Error("loading storage cleanup copies failed", "error", err) - scheduleTaskRetry(ctx, w.repos, task, "storage_cleanup", logger, err) - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() - return - } - if len(copies) == 0 { - if !w.completeTask(ctx, task, logger, "No remote replicas to delete") { - return - } - return - } - - waiting := false - unsupported := false - for _, copy := range copies { - switch copy.Status { - case model.StorageCleanupCopyStatusRemoved: - continue - case model.StorageCleanupCopyStatusUnsupported: - unsupported = true - continue - } - copyWaiting, err := w.processCopy(ctx, copy) - if err != nil { - logger.Warn("storage cleanup copy failed", "copyID", copy.ID, "error", err) - if errors.Is(err, errStorageCleanupCopyUnsupported) { - unsupported = true - continue - } - scheduleTaskRetry(ctx, w.repos, task, "storage_cleanup", logger, err) - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() - return - } - waiting = waiting || copyWaiting - } - if waiting { - if err := w.repos.Tasks.WaitRunning(ctx, task, model.TaskWaitReasonExternalConfirmation, "Waiting for remote replica deletion", storageCleanupConfirmationDelay); err != nil { - logger.Error("waiting for storage cleanup confirmation failed", "error", err) - _ = w.repos.Tasks.FailRunning(ctx, task, err.Error()) - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() - return - } - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "success").Inc() - return - } - if unsupported { - _ = w.repos.Tasks.FailRunning(ctx, task, "Remote replica deletion is not supported for this provider") - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() - return - } - if err := w.repos.StorageCleanup.DeleteUploadProvenanceIfUnreferenced(ctx, task.RefID); err != nil { - logger.Error("deleting storage cleanup provenance failed", "error", err) - scheduleTaskRetry(ctx, w.repos, task, "storage_cleanup", logger, err) - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() - return - } - if !w.completeTask(ctx, task, logger, "Remote replicas deleted") { - return - } -} - -func (w *StorageCleanupWorker) completeTask(ctx context.Context, task *model.Task, logger *slog.Logger, message string) bool { - if err := w.repos.Tasks.CompleteWithMessage(ctx, task, message); err != nil { - if logger != nil { - logger.Error("failed to complete storage cleanup task", "taskID", task.ID, "error", err) - } - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() - return false - } - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "success").Inc() - return true -} - -func (w *StorageCleanupWorker) waitForReferences(ctx context.Context, task *model.Task, logger *slog.Logger, message string) bool { - if err := w.repos.Tasks.WaitRunning(ctx, task, model.TaskWaitReasonDependency, message, storageCleanupReferenceDelay); err != nil { - if logger != nil { - logger.Error("failed to wait storage cleanup task", "taskID", task.ID, "error", err) - } - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() - return false - } - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "success").Inc() - return true -} - -func (w *StorageCleanupWorker) processCopy(ctx context.Context, copy model.StorageCleanupCopy) (bool, error) { - if copy.DataSetID == nil || copy.ClientDataSetID == nil || copy.PieceID == nil || - copy.ProviderID == nil || copy.ProviderID.IsZero() || copy.PieceCID == "" { - msg := "Storage provider details are incomplete for this version" - if err := w.repos.StorageCleanup.MarkCopyUnsupported(ctx, copy.ID, msg); err != nil { - return false, err - } - return false, fmt.Errorf("%w: %s", errStorageCleanupCopyUnsupported, msg) - } - pieceCID, err := cid.Parse(copy.PieceCID) - if err != nil { - msg := "Stored data identifier is invalid" - if markErr := w.repos.StorageCleanup.MarkCopyUnsupported(ctx, copy.ID, msg); markErr != nil { - return false, markErr - } - return false, fmt.Errorf("%w: %v", errStorageCleanupCopyUnsupported, err) - } - providerID := copy.ProviderID.SDK() - cleanupCtx, err := w.storage.OpenCleanupContext(ctx, copy.DataSetID.SDK(), storage.NewDataSetContextOptions{ - ProviderID: &providerID, - }) - if err != nil { - return false, err - } - status, err := cleanupCtx.PieceStatus(ctx, pieceCID) - if err != nil { - return false, err - } - if status == nil || !status.Exists { - return false, w.repos.StorageCleanup.MarkCopyRemoved(ctx, copy.ID) - } - if copy.Status == model.StorageCleanupCopyStatusDeleteScheduled { - return true, nil - } - result, err := cleanupCtx.DeletePieceByID(ctx, copy.PieceID.SDK()) - if err != nil { - return false, err - } - txHash := "" - if result != nil { - txHash = result.Hash.String() - } - if err := w.repos.StorageCleanup.MarkCopyDeleteScheduled(ctx, copy.ID, txHash); err != nil { - return false, err - } - return true, nil -} diff --git a/internal/worker/storage_cleanup_replacement.go b/internal/worker/storage_cleanup_replacement.go deleted file mode 100644 index e2041fc..0000000 --- a/internal/worker/storage_cleanup_replacement.go +++ /dev/null @@ -1,551 +0,0 @@ -package worker - -import ( - "context" - "errors" - "fmt" - "log/slog" - "time" - - "github.com/strahe/synaps3/internal/admin" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/strahe/synaps3/internal/synapse" - "github.com/strahe/synapse-go/storage" -) - -// processReplacementRetirementTask ends a replaced storage service. Every step -// is ordered so the destructive call sits between transactions and can never -// run against a stale view: -// -// tx A evaluate the safety gate, wait if anything still needs the source -// -- terminate the service with the provider -// tx B record the epoch at which the service ends -// -- observe the chain head -// tx C re-evaluate the whole gate, then retire and complete -// -// A blocker that can clear on its own returns to waiting; only a structural -// problem or a payment decision raises operator attention. -func (w *StorageCleanupWorker) processReplacementRetirementTask(ctx context.Context, task *model.Task) { - logger := w.logger.With("taskID", task.ID) - replacementID, err := storagereplacement.ParseRetirePayload(task) - if err != nil { - w.failReplacementRetirement(ctx, task, logger, "parse replacement retirement task", err) - return - } - logger = logger.With("replacementID", replacementID) - - replacement, err := w.repos.Replacements.GetByID(ctx, replacementID) - if err != nil { - w.retryReplacementRetirement(ctx, task, replacementID, logger, "load provider replacement", err) - return - } - if replacement == nil { - w.completeTask(ctx, task, logger, "Replacement already retired") - return - } - if task.Stage != nil && *task.Stage == storagereplacement.StageRetireAbandonedTarget { - w.retireAbandonedTarget(ctx, task, replacement, logger) - return - } - if replacement.Status == storagereplacement.StatusCompleted { - w.completeTask(ctx, task, logger, "Replacement already retired") - return - } - if replacement.Status == storagereplacement.StatusSuperseded { - // A later confirmation owns this slot now. Retiring this replacement's - // source would terminate a service the successor still depends on. - w.completeTask(ctx, task, logger, "Replacement superseded") - return - } - if replacement.Status == storagereplacement.StatusCleanupAttention { - // Automatic retry is suppressed until an operator acts, so the task must - // not keep re-running on its own. - w.completeTask(ctx, task, logger, "Waiting for operator action") - return - } - if w.terminator == nil || w.epochs == nil { - w.retryReplacementRetirement(ctx, task, replacementID, logger, "retire replaced service", - errors.New("service termination is not configured")) - return - } - source, err := w.repos.Uploads.GetDataSetBindingByID(ctx, replacement.SourceDataSetID) - if err != nil { - w.retryReplacementRetirement(ctx, task, replacementID, logger, "load retiring data set", err) - return - } - if source == nil || source.DataSetID == nil || source.DataSetID.IsZero() { - w.retryReplacementRetirement(ctx, task, replacementID, logger, "load retiring data set", - fmt.Errorf("data set %d has no on-chain identity: %w", replacement.SourceDataSetID, repository.ErrNotFound)) - return - } - if source.Status == model.StorageDataSetStatusRetired { - w.completeTask(ctx, task, logger, "Replaced provider already retired") - return - } - - // tx A: nothing may still need the source before it is terminated. - gate, err := w.repos.Replacements.EvaluateRetirementGate(ctx, replacementID, nil) - if err != nil { - w.retryReplacementRetirement(ctx, task, replacementID, logger, "evaluate retirement gate", err) - return - } - if !gate.Passed() { - w.waitForReplacementRetirement(ctx, task, replacementID, gate, logger) - return - } - // Nothing needs the source any more, so the record should say retiring - // rather than waiting while the destructive work runs. - if replacement.Status != storagereplacement.StatusRetiring { - if err := w.repos.Replacements.BeginRetirement(ctx, replacementID); err != nil { - w.retryReplacementRetirement(ctx, task, replacementID, logger, "resume replacement retirement", err) - return - } - replacement.Status = storagereplacement.StatusRetiring - } - - if replacement.TerminationEpoch == nil { - result, err := w.terminator.TerminateService(ctx, source.DataSetID.SDK()) - if err != nil { - w.handleTerminationError(ctx, task, replacementID, logger, err) - return - } - if result == nil { - w.retryReplacementRetirement(ctx, task, replacementID, logger, "terminate replaced service", - errors.New("storage service returned no termination result")) - return - } - // tx B: the end of term is recorded before the service is treated as - // terminated, so a crash here re-reads it instead of terminating twice. - if err := w.repos.Replacements.RecordTerminationEpoch(ctx, repository.RecordTerminationEpochInput{ - ReplacementID: replacementID, - TxHash: result.TxHash, - Epoch: result.EndEpoch, - }); err != nil { - // A conflict means someone recorded an epoch first. Trust the stored - // value rather than the one this attempt just observed, but never - // continue without one. - stored, loadErr := w.repos.Replacements.GetByID(ctx, replacementID) - if loadErr != nil || stored == nil || stored.TerminationEpoch == nil { - w.retryReplacementRetirement(ctx, task, replacementID, logger, "record termination epoch", err) - return - } - replacement.TerminationEpoch = stored.TerminationEpoch - } else { - replacement.TerminationEpoch = &result.EndEpoch - } - logger.Info("replaced storage service termination submitted", - "dataSetID", source.DataSetID.String(), "endEpoch", *replacement.TerminationEpoch) - } - - observed, err := w.epochs.CurrentEpoch(ctx) - if err != nil { - w.retryReplacementRetirement(ctx, task, replacementID, logger, "read chain epoch", err) - return - } - if replacement.TerminationEpoch == nil || observed < *replacement.TerminationEpoch { - w.waitForReplacementEpoch(ctx, task, replacementID, logger) - return - } - - // tx C: the gate is evaluated again from scratch, this time including the - // epoch, and the repository refuses the change if anything regressed. - if err := w.repos.Replacements.CompleteRetirement(ctx, replacementID, observed); err != nil { - w.handleRetirementCompletionError(ctx, task, replacementID, logger, err) - return - } - logger.Info("replaced storage service retired", "dataSetID", source.DataSetID.String()) - w.completeTask(ctx, task, logger, "Replaced provider retired") -} - -// retireAbandonedTarget ends the paid service of a target a later confirmation -// replaced. It asks a narrower question than source retirement: the abandoned -// generation only holds partly migrated data, so the gate is simply that -// nothing depends on it as its last readable copy. The replacement record stays -// superseded throughout. -func (w *StorageCleanupWorker) retireAbandonedTarget( - ctx context.Context, - task *model.Task, - replacement *storagereplacement.Replacement, - logger *slog.Logger, -) { - if w.terminator == nil || w.epochs == nil { - // Abandoned cleanup has no operator retry: the replacement is superseded. - // Waiting keeps the coordinator claimable until termination is configured. - w.waitForAbandonedTarget(ctx, task, logger, "Waiting until unused storage services can be ended") - return - } - target, err := w.repos.Uploads.GetDataSetBindingByID(ctx, replacement.TargetDataSetID) - if err != nil { - w.waitForAbandonedTarget(ctx, task, logger, "Waiting to end the unused storage service") - return - } - if target == nil || target.Status == model.StorageDataSetStatusRetired { - w.completeTask(ctx, task, logger, "Abandoned provider already retired") - return - } - if target.IsCurrent { - // It took over the slot after all, so it is not abandoned. - w.completeTask(ctx, task, logger, "Target is in use") - return - } - if target.DataSetID == nil || target.DataSetID.IsZero() { - if dataSetBindingHasCreationEvidence(target) { - // A creation was submitted and may still land a paid service on this - // provider. Retiring the local row now would release the provider - // while that service exists and nothing is left watching for it, so - // the submission is resolved first. - w.resolveAbandonedTargetCreation(ctx, task, replacement, target, logger) - return - } - // Nothing was ever asked of this provider, but the local generation still - // holds it for this bucket: an unretired generation makes the provider - // unavailable to both automatic selection and a manual choice. Release it. - if err := w.releaseAbandonedTargetWithoutService(ctx, replacement, target, - "provider replacement was superseded before a storage service was created"); err != nil { - w.waitForAbandonedTarget(ctx, task, logger, "Waiting to release the unused storage provider") - return - } - w.completeTask(ctx, task, logger, "Abandoned provider released without a service") - return - } - - sole, err := w.repos.Replacements.CountAbandonedTargetSoleCopies(ctx, target.ID) - if err != nil { - w.waitForAbandonedTarget(ctx, task, logger, "Waiting to end the unused storage service") - return - } - if sole > 0 { - logger.Warn("abandoned target still holds the only readable copy of some content", - "dataSetID", target.DataSetID.String(), "count", sole) - w.waitForAbandonedTarget(ctx, task, logger, "Waiting for content on the abandoned provider to be copied elsewhere") - return - } - activeAttempts, err := w.repos.Uploads.CountActiveCommitAttemptsForDataSet(ctx, target.ID) - if err != nil { - logger.Warn("failed to check abandoned target confirmation attempts", "error", err) - w.waitForAbandonedTarget(ctx, task, logger, "Waiting to end the unused storage service") - return - } - if activeAttempts > 0 { - logger.Info("abandoned target still has active confirmation attempts", "count", activeAttempts) - w.waitForAbandonedTarget(ctx, task, logger, "Waiting for storage confirmations") - return - } - - endEpoch := replacement.AbandonedTerminationEpoch - if endEpoch == nil { - result, terminateErr := w.terminator.TerminateService(ctx, target.DataSetID.SDK()) - if terminateErr != nil { - switch { - case synapse.IsTerminationBlocked(terminateErr): - w.waitForAbandonedTarget(ctx, task, logger, "Waiting for outstanding payment to be settled") - case synapse.IsProviderUnavailable(terminateErr): - w.waitForAbandonedTarget(ctx, task, logger, storagereplacement.WaitReasonProvider.Message()) - default: - w.waitForAbandonedTarget(ctx, task, logger, "Waiting to end the unused storage service") - } - return - } - if result == nil { - w.waitForAbandonedTarget(ctx, task, logger, "Waiting to record the unused storage service end epoch") - return - } - if err := w.repos.Replacements.RecordAbandonedTerminationEpoch(ctx, repository.RecordTerminationEpochInput{ - ReplacementID: replacement.ID, - TxHash: result.TxHash, - Epoch: result.EndEpoch, - }); err != nil { - w.waitForAbandonedTarget(ctx, task, logger, "Waiting to record the unused storage service end epoch") - return - } - endEpoch = &result.EndEpoch - } - observed, err := w.epochs.CurrentEpoch(ctx) - if err != nil { - w.waitForAbandonedTarget(ctx, task, logger, "Waiting to end the unused storage service") - return - } - if observed < *endEpoch { - w.waitForAbandonedTarget(ctx, task, logger, storagereplacement.WaitReasonTerminationEpoch.Message()) - return - } - if err := w.repos.Replacements.CompleteAbandonedTargetTermination(ctx, replacement.ID, time.Now()); err != nil { - w.waitForAbandonedTarget(ctx, task, logger, "Waiting to end the unused storage service") - return - } - logger.Info("abandoned replacement target retired", "dataSetID", target.DataSetID.String()) - w.completeTask(ctx, task, logger, "Abandoned provider retired") -} - -// A recoverable blocker keeps the replacement waiting; the operator sees which -// predicate is holding it. -func (w *StorageCleanupWorker) waitForReplacementRetirement( - ctx context.Context, - task *model.Task, - replacementID int64, - gate repository.RetirementGate, - logger *slog.Logger, -) { - reason := retirementWaitReason(gate) - if reason == storagereplacement.WaitReasonCoverage && !gate.SlotOwned { - // The generations are not in the shape retirement assumes. Retrying - // cannot fix that, so it needs an operator. - w.raiseCleanupAttention(ctx, task, replacementID, logger, - fmt.Sprintf("replica slot ownership is inconsistent: %v", gate.Blockers)) - return - } - if err := w.repos.Replacements.MarkWaiting(ctx, replacementID, reason); err != nil { - logger.Warn("failed to record retirement wait reason", "reason", reason, "error", err) - } - logger.Debug("retirement gate is blocked", "blockers", gate.Blockers) - message := reason.Message() - if gate.ActiveAttempts > 0 { - message = "Waiting for storage confirmations" - } - w.waitForReferences(ctx, task, logger, message) -} - -// retirementWaitReason names the first blocker so the operator sees the reason -// closest to the source of the delay. -func retirementWaitReason(gate repository.RetirementGate) storagereplacement.WaitReason { - if !gate.SlotOwned { - return storagereplacement.WaitReasonCoverage - } - switch { - case gate.ActiveAttempts > 0: - return storagereplacement.WaitReasonProvider - case gate.WaitingItems > 0: - return storagereplacement.WaitReasonReadableSource - case gate.CoverageGaps > 0: - return storagereplacement.WaitReasonCoverage - case gate.SourceWrites > 0: - return storagereplacement.WaitReasonSourceWrites - default: - return storagereplacement.WaitReasonTerminationEpoch - } -} - -func (w *StorageCleanupWorker) waitForReplacementEpoch(ctx context.Context, task *model.Task, replacementID int64, logger *slog.Logger) { - if err := w.repos.Replacements.MarkWaiting(ctx, replacementID, storagereplacement.WaitReasonTerminationEpoch); err != nil { - logger.Warn("failed to record termination epoch wait", "error", err) - } - if err := w.repos.Tasks.WaitRunning(ctx, task, model.TaskWaitReasonExternalConfirmation, - storagereplacement.WaitReasonTerminationEpoch.Message(), storageCleanupConfirmationDelay); err != nil { - logger.Error("failed to wait for termination epoch", "taskID", task.ID, "error", err) - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() - return - } - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "success").Inc() -} - -func (w *StorageCleanupWorker) handleTerminationError(ctx context.Context, task *model.Task, replacementID int64, logger *slog.Logger, err error) { - switch { - case synapse.IsTerminationBlocked(err): - // Settling debt is the operator's decision, never the gateway's. - w.raiseCleanupAttention(ctx, task, replacementID, logger, err.Error()) - case synapse.IsProviderUnavailable(err): - if markErr := w.repos.Replacements.MarkWaiting(ctx, replacementID, storagereplacement.WaitReasonProvider); markErr != nil { - logger.Warn("failed to record provider wait reason", "error", markErr) - } - w.waitForReferences(ctx, task, logger, storagereplacement.WaitReasonProvider.Message()) - default: - w.retryReplacementRetirement(ctx, task, replacementID, logger, "terminate replaced service", err) - } -} - -// CompleteRetirement re-checks everything itself, so a refusal here means the -// world changed between the first gate and the final transaction. -func (w *StorageCleanupWorker) handleRetirementCompletionError(ctx context.Context, task *model.Task, replacementID int64, logger *slog.Logger, err error) { - if errors.Is(err, storagereplacement.ErrPrematureComplete) { - gate, gateErr := w.repos.Replacements.EvaluateRetirementGate(ctx, replacementID, nil) - if gateErr != nil { - w.retryReplacementRetirement(ctx, task, replacementID, logger, "re-evaluate retirement gate", gateErr) - return - } - w.waitForReplacementRetirement(ctx, task, replacementID, gate, logger) - return - } - w.retryReplacementRetirement(ctx, task, replacementID, logger, "complete replacement retirement", err) -} - -// raiseCleanupAttention stops the task in the same step that records why, so -// automatic retry can never resume suppressed cleanup. -func (w *StorageCleanupWorker) raiseCleanupAttention(ctx context.Context, task *model.Task, replacementID int64, logger *slog.Logger, message string) { - err := w.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { - if err := txRepos.Replacements.MarkCleanupAttention(ctx, replacementID, message); err != nil { - return err - } - return txRepos.Tasks.FailRunning(ctx, task, message) - }) - if err != nil { - logger.Error("failed to raise replacement cleanup attention", "error", err) - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() - return - } - logger.Warn("replacement retirement needs operator action", "reason", message) - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() -} - -func (w *StorageCleanupWorker) retryReplacementRetirement( - ctx context.Context, - task *model.Task, - replacementID int64, - logger *slog.Logger, - stage string, - err error, -) { - logger.Error(stage+" failed", "error", err) - status := scheduleTaskRetry(ctx, w.repos, task, "storage_cleanup", logger, err) - // A zero replacement id means the caller owns no replacement state to move, - // which is the case for abandoned target cleanup. - if status == model.TaskStatusExhausted && replacementID > 0 { - // Cleanup that ran out of attempts is an operator decision, not a - // failure to retry, and the record must land even during shutdown. - cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), terminalFailureCleanupTimeout) - defer cancel() - if markErr := w.repos.Replacements.MarkCleanupAttention(cleanupCtx, replacementID, - fmt.Sprintf("%s: %v (max retries reached)", stage, err)); markErr != nil { - logger.Error("failed to record replacement cleanup attention", "error", markErr) - } - } - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() -} - -func (w *StorageCleanupWorker) failReplacementRetirement(ctx context.Context, task *model.Task, logger *slog.Logger, stage string, err error) { - logger.Error(stage+" failed", "error", err) - if failErr := w.repos.Tasks.FailRunning(ctx, task, err.Error()); failErr != nil { - logger.Error("failed to stop replacement retirement task", "error", failErr) - } - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() -} - -// releaseAbandonedTargetWithoutService retires a local generation that never -// received an on-chain identity. Leaving it unretired would block that provider -// from every later confirmation for the bucket. -func (w *StorageCleanupWorker) releaseAbandonedTargetWithoutService( - ctx context.Context, - replacement *storagereplacement.Replacement, - target *model.StorageDataSet, - lastError string, -) error { - if target != nil { - switch target.Status { - case model.StorageDataSetStatusPending, model.StorageDataSetStatusCreating, model.StorageDataSetStatusFailed: - if err := w.repos.Uploads.MarkDataSetFailed(ctx, target.ID, lastError); err != nil && !errors.Is(err, repository.ErrConflict) { - return err - } - } - } - return w.repos.Replacements.RetireAbandonedTarget(ctx, replacement.ID) -} - -// Observation failures never produced a data set id. After the last retry the -// provider must be released: superseded replacements have no Data Sets retry. -func (w *StorageCleanupWorker) retryAbandonedTargetObservation( - ctx context.Context, - task *model.Task, - replacement *storagereplacement.Replacement, - target *model.StorageDataSet, - logger *slog.Logger, - stage string, - err error, -) { - if synapse.IsProviderUnavailable(err) { - w.waitForReferences(ctx, task, logger, storagereplacement.WaitReasonProvider.Message()) - return - } - logger.Error(stage+" failed", "error", err) - status := scheduleTaskRetry(ctx, w.repos, task, "storage_cleanup", logger, err) - if status == model.TaskStatusExhausted { - cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), terminalFailureCleanupTimeout) - defer cancel() - if releaseErr := w.releaseAbandonedTargetWithoutService(cleanupCtx, replacement, target, - fmt.Sprintf("%s: %v (max retries reached)", stage, err)); releaseErr != nil { - logger.Error("failed to release abandoned target after observation retries", "error", releaseErr) - } - } - admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() -} - -// resolveAbandonedTargetCreation finishes a data set creation the superseded -// confirmation left in flight. Until it resolves, the generation has neither a -// service to end nor proof that none exists, and the provider stays reserved. -func (w *StorageCleanupWorker) resolveAbandonedTargetCreation( - ctx context.Context, - task *model.Task, - replacement *storagereplacement.Replacement, - target *model.StorageDataSet, - logger *slog.Logger, -) { - if target.CreateTransactionID == nil || target.CreateStatusURL == nil || target.ClientDataSetID == nil { - // The submission left no way to observe its outcome. Treating it as a - // live service would reserve the provider forever, so it is recorded as - // failed and the generation released. - if err := w.releaseAbandonedTargetWithoutService(ctx, replacement, target, - "provider replacement was superseded before the service could be observed"); err != nil { - w.waitForAbandonedTarget(ctx, task, logger, "Waiting to release the unused storage provider") - return - } - w.completeTask(ctx, task, logger, "Abandoned provider released; its service could not be observed") - return - } - bucket, err := w.repos.Buckets.GetByID(ctx, replacement.BucketID) - if err != nil || bucket == nil { - if err == nil { - err = fmt.Errorf("bucket %d: %w", replacement.BucketID, repository.ErrNotFound) - } - w.retryAbandonedTargetObservation(ctx, task, replacement, target, logger, "load bucket for abandoned target", err) - return - } - if w.storage == nil { - w.retryAbandonedTargetObservation(ctx, task, replacement, target, logger, "open abandoned target context", - errors.New("storage client is not configured")) - return - } - storageCtx, err := w.storage.OpenProviderTarget(ctx, target.ProviderID.SDK(), storage.NewProviderContextOptions{ - DataSetMetadata: map[string]string{"bucket": bucket.Name}, - }) - if err != nil || storageCtx == nil { - if err == nil { - err = errors.New("storage context resolver returned no context") - } - w.retryAbandonedTargetObservation(ctx, task, replacement, target, logger, "open abandoned target context", err) - return - } - result, err := storageCtx.WaitForDataSetCreated(ctx, storage.CreateDataSetSubmission{ - ProviderID: target.ProviderID.SDK(), - TransactionID: *target.CreateTransactionID, - StatusURL: *target.CreateStatusURL, - ClientDataSetID: sdkBigIntPtr(target.ClientDataSetID), - }) - if err != nil { - w.retryAbandonedTargetObservation(ctx, task, replacement, target, logger, "resolve abandoned target creation", err) - return - } - dataSetID, clientDataSetID, err := dataSetResultIDsForBinding(target, result) - if err != nil { - w.retryAbandonedTargetObservation(ctx, task, replacement, target, logger, "validate abandoned target creation", err) - return - } - if err := w.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: target.ID, - DataSetID: dataSetID, - ClientDataSetID: &clientDataSetID, - }); err != nil { - // The chain service exists. Releasing the local row would orphan it, and - // exhausting the task would leave no resume path, so wait and persist again. - w.waitForAbandonedTarget(ctx, task, logger, "Waiting to record the unused storage service") - return - } - // The service now has an identity, so the next run ends it through the gate. - w.waitForAbandonedTarget(ctx, task, logger, "Ending the unused storage service") -} - -// waitForAbandonedTarget parks leftover cleanup without burning retries or -// moving the superseded replacement. Startup can still see a waiting task, and -// ResumeCoordinator revives one that previously failed or exhausted. -func (w *StorageCleanupWorker) waitForAbandonedTarget(ctx context.Context, task *model.Task, logger *slog.Logger, message string) { - w.waitForReferences(ctx, task, logger, message) -} diff --git a/internal/worker/storage_cleanup_replacement_internal_test.go b/internal/worker/storage_cleanup_replacement_internal_test.go deleted file mode 100644 index 56e67b7..0000000 --- a/internal/worker/storage_cleanup_replacement_internal_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package worker - -import ( - "context" - "errors" - "io" - "log/slog" - "testing" - "time" - - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/strahe/synaps3/internal/synapse" - "github.com/strahe/synaps3/internal/testutil" - sdktypes "github.com/strahe/synapse-go/types" -) - -type retirementWaitReplacementRepo struct { - repository.StorageReplacementRepository - reason storagereplacement.WaitReason -} - -func (r *retirementWaitReplacementRepo) MarkWaiting(_ context.Context, _ int64, reason storagereplacement.WaitReason) error { - r.reason = reason - return nil -} - -type retirementWaitTaskRepo struct { - repository.TaskRepository - reason model.TaskWaitReason - message string -} - -func (r *retirementWaitTaskRepo) WaitRunning( - _ context.Context, - _ *model.Task, - reason model.TaskWaitReason, - message string, - _ time.Duration, -) error { - r.reason = reason - r.message = message - return nil -} - -type abandonedTargetUploadRepo struct { - repository.StorageUploadRepository - target *model.StorageDataSet - activeAttempts int - activeErr error -} - -func (r *abandonedTargetUploadRepo) GetDataSetBindingByID(context.Context, int64) (*model.StorageDataSet, error) { - return r.target, nil -} - -func (r *abandonedTargetUploadRepo) CountActiveCommitAttemptsForDataSet(context.Context, int64) (int, error) { - return r.activeAttempts, r.activeErr -} - -type abandonedTargetReplacementRepo struct { - repository.StorageReplacementRepository -} - -func (*abandonedTargetReplacementRepo) CountAbandonedTargetSoleCopies(context.Context, int64) (int, error) { - return 0, nil -} - -func TestReplacementRetirementPrioritizesActiveConfirmations(t *testing.T) { - replacements := new(retirementWaitReplacementRepo) - tasks := new(retirementWaitTaskRepo) - worker := &StorageCleanupWorker{repos: &repository.Repositories{ - Replacements: replacements, - Tasks: tasks, - }} - worker.waitForReplacementRetirement( - t.Context(), - &model.Task{}, - 42, - repository.RetirementGate{SlotOwned: true, ActiveAttempts: 1, WaitingItems: 1}, - slog.New(slog.NewTextHandler(io.Discard, nil)), - ) - - if replacements.reason != storagereplacement.WaitReasonProvider { - t.Fatalf("replacement wait reason = %s, want %s", replacements.reason, storagereplacement.WaitReasonProvider) - } - if tasks.reason != model.TaskWaitReasonDependency || tasks.message != "Waiting for storage confirmations" { - t.Fatalf("task wait = (%s, %q), want dependency confirmation message", tasks.reason, tasks.message) - } -} - -func TestAbandonedTargetRetirementDistinguishesConfirmationCountErrors(t *testing.T) { - countErr := errors.New("count active attempts") - tests := []struct { - name string - activeAttempts int - activeErr error - wantMessage string - }{ - { - name: "database error", - activeErr: countErr, - wantMessage: "Waiting to end the unused storage service", - }, - { - name: "active confirmations", - activeAttempts: 1, - wantMessage: "Waiting for storage confirmations", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - dataSetID := onChainID(t, "2002") - target := &model.StorageDataSet{ - ID: 77, - DataSetID: &dataSetID, - Status: model.StorageDataSetStatusReady, - } - uploads := &abandonedTargetUploadRepo{ - target: target, activeAttempts: tt.activeAttempts, activeErr: tt.activeErr, - } - tasks := new(retirementWaitTaskRepo) - terminationCalls := 0 - terminator := &testutil.MockServiceTerminator{ - TerminateServiceFunc: func(context.Context, sdktypes.BigInt) (*synapse.TerminationResult, error) { - terminationCalls++ - return nil, errors.New("unexpected termination") - }, - } - worker := &StorageCleanupWorker{ - repos: &repository.Repositories{ - Uploads: uploads, Replacements: new(abandonedTargetReplacementRepo), Tasks: tasks, - }, - terminator: terminator, - epochs: new(testutil.MockChainEpochReader), - } - - worker.retireAbandonedTarget( - t.Context(), - &model.Task{}, - &storagereplacement.Replacement{TargetDataSetID: target.ID}, - slog.New(slog.NewTextHandler(io.Discard, nil)), - ) - - if terminationCalls != 0 { - t.Fatalf("termination calls = %d, want 0", terminationCalls) - } - if tasks.reason != model.TaskWaitReasonDependency || tasks.message != tt.wantMessage { - t.Fatalf("task wait = (%s, %q), want dependency with %q", tasks.reason, tasks.message, tt.wantMessage) - } - }) - } -} diff --git a/internal/worker/storage_cleanup_test.go b/internal/worker/storage_cleanup_test.go deleted file mode 100644 index 662a342..0000000 --- a/internal/worker/storage_cleanup_test.go +++ /dev/null @@ -1,608 +0,0 @@ -package worker_test - -import ( - "context" - "log/slog" - "sync/atomic" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ipfs/go-cid" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/synapse" - "github.com/strahe/synaps3/internal/worker" - "github.com/strahe/synapse-go/storage" - sdktypes "github.com/strahe/synapse-go/types" -) - -type fakeCleanupContext struct { - pieceStatusFunc func(context.Context, cid.Cid) (*storage.PieceStatus, error) - deletePieceByIDFunc func(context.Context, sdktypes.BigInt) (*sdktypes.WriteResult, error) -} - -func (f fakeCleanupContext) PieceStatus(ctx context.Context, piece cid.Cid) (*storage.PieceStatus, error) { - if f.pieceStatusFunc != nil { - return f.pieceStatusFunc(ctx, piece) - } - return &storage.PieceStatus{Exists: false}, nil -} - -func (f fakeCleanupContext) DeletePieceByID(ctx context.Context, pieceID sdktypes.BigInt) (*sdktypes.WriteResult, error) { - if f.deletePieceByIDFunc != nil { - return f.deletePieceByIDFunc(ctx, pieceID) - } - return &sdktypes.WriteResult{Hash: common.HexToHash("0xdelete")}, nil -} - -func TestStorageCleanupWorkerMarksMissingPieceRemoved(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - pieceCID := testCID(t).String() - uploadID := int64(77) - task := &model.Task{ - Type: model.TaskTypeStorageCleanup, - RefType: "storage_upload", - RefID: uploadID, - IdempotencyKey: "storage_cleanup:77", - Status: model.TaskStatusQueued, - MaxRetries: 3, - ScheduledAt: time.Now(), - Payload: map[string]interface{}{ - "storage_upload_id": uploadID, - "piece_cid": pieceCID, - }, - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create cleanup task: %v", err) - } - copy := &model.StorageCleanupCopy{ - TaskID: task.ID, - UploadID: uploadID, - CopyIndex: 0, - ProviderID: onChainIDPtr(t, "101"), - DataSetID: onChainIDPtr(t, "1001"), - ClientDataSetID: onChainIDPtr(t, "5001"), - PieceID: onChainIDPtr(t, "2001"), - PieceCID: pieceCID, - Status: model.StorageCleanupCopyStatusPending, - } - if _, err := env.db.NewInsert().Model(copy).Exec(ctx); err != nil { - t.Fatalf("insert cleanup copy: %v", err) - } - - var requestedDataSet string - env.storage.OpenCleanupContextFunc = func(_ context.Context, dataSetID sdktypes.BigInt, opts storage.NewDataSetContextOptions) (synapse.CleanupContext, error) { - if opts.ProviderID == nil || opts.ProviderID.String() != "101" { - t.Fatalf("ProviderID = %v, want 101", opts.ProviderID) - } - requestedDataSet = dataSetID.String() - return fakeCleanupContext{ - pieceStatusFunc: func(context.Context, cid.Cid) (*storage.PieceStatus, error) { - return &storage.PieceStatus{Exists: false}, nil - }, - }, nil - } - - cleanup := worker.NewStorageCleanupWorker(env.repos, env.storage, 1, 20*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, cleanup, task.ID, 5*time.Second) - - gotTask, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if gotTask.Status != model.TaskStatusCompleted { - t.Fatalf("task status = %s, want completed", gotTask.Status) - } - gotCopy := new(model.StorageCleanupCopy) - if err := env.db.NewSelect().Model(gotCopy).Where("id = ?", copy.ID).Scan(ctx); err != nil { - t.Fatalf("select cleanup copy: %v", err) - } - if gotCopy.Status != model.StorageCleanupCopyStatusRemoved || gotCopy.RemovedAt == nil { - t.Fatalf("copy cleanup = status:%s removed:%v, want removed timestamp", gotCopy.Status, gotCopy.RemovedAt) - } - if requestedDataSet != "1001" { - t.Fatalf("cleanup context dataset = %q, want 1001", requestedDataSet) - } -} - -func TestStorageCleanupWorkerMarksZeroProviderUnsupportedAndContinues(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - pieceCID := testCID(t).String() - uploadID := int64(88) - task := &model.Task{ - Type: model.TaskTypeStorageCleanup, - RefType: "storage_upload", - RefID: uploadID, - IdempotencyKey: "storage_cleanup:88", - Status: model.TaskStatusQueued, - MaxRetries: 3, - ScheduledAt: time.Now(), - Payload: map[string]any{ - "storage_upload_id": uploadID, - "piece_cid": pieceCID, - }, - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create cleanup task: %v", err) - } - unsupported := &model.StorageCleanupCopy{ - TaskID: task.ID, - UploadID: uploadID, - CopyIndex: 0, - ProviderID: onChainIDPtr(t, "0"), - DataSetID: onChainIDPtr(t, "1001"), - ClientDataSetID: onChainIDPtr(t, "5001"), - PieceID: onChainIDPtr(t, "2001"), - PieceCID: pieceCID, - Status: model.StorageCleanupCopyStatusPending, - } - removable := &model.StorageCleanupCopy{ - TaskID: task.ID, - UploadID: uploadID, - CopyIndex: 1, - ProviderID: onChainIDPtr(t, "101"), - DataSetID: onChainIDPtr(t, "1001"), - ClientDataSetID: onChainIDPtr(t, "5001"), - PieceID: onChainIDPtr(t, "2001"), - PieceCID: pieceCID, - Status: model.StorageCleanupCopyStatusPending, - } - if _, err := env.db.NewInsert().Model(unsupported).Exec(ctx); err != nil { - t.Fatalf("insert unsupported cleanup copy: %v", err) - } - if _, err := env.db.NewInsert().Model(removable).Exec(ctx); err != nil { - t.Fatalf("insert removable cleanup copy: %v", err) - } - env.storage.OpenCleanupContextFunc = func(context.Context, sdktypes.BigInt, storage.NewDataSetContextOptions) (synapse.CleanupContext, error) { - return fakeCleanupContext{ - pieceStatusFunc: func(context.Context, cid.Cid) (*storage.PieceStatus, error) { - return &storage.PieceStatus{Exists: false}, nil - }, - }, nil - } - - cleanup := worker.NewStorageCleanupWorker(env.repos, env.storage, 1, 20*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, cleanup, task.ID, 5*time.Second) - - gotTask, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if gotTask.Status != model.TaskStatusFailed { - t.Fatalf("task status = %s, want failed after unsupported copy", gotTask.Status) - } - var copies []model.StorageCleanupCopy - if err := env.db.NewSelect().Model(&copies).Where("task_id = ?", task.ID).OrderExpr("copy_index ASC").Scan(ctx); err != nil { - t.Fatalf("select cleanup copies: %v", err) - } - if copies[0].Status != model.StorageCleanupCopyStatusUnsupported { - t.Fatalf("unsupported copy status = %s, want unsupported", copies[0].Status) - } - if copies[1].Status != model.StorageCleanupCopyStatusRemoved { - t.Fatalf("removable copy status = %s, want removed", copies[1].Status) - } -} - -func TestStorageCleanupWorkerWaitsWhenUnsupportedTaskStillHasScheduledDeletion(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - pieceCID := testCID(t).String() - uploadID := int64(89) - task := &model.Task{ - Type: model.TaskTypeStorageCleanup, - RefType: "storage_upload", - RefID: uploadID, - IdempotencyKey: "storage_cleanup:89", - Status: model.TaskStatusQueued, - MaxRetries: 3, - ScheduledAt: time.Now(), - Payload: map[string]interface{}{ - "storage_upload_id": uploadID, - "piece_cid": pieceCID, - }, - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create cleanup task: %v", err) - } - unsupported := &model.StorageCleanupCopy{ - TaskID: task.ID, - UploadID: uploadID, - CopyIndex: 0, - PieceCID: pieceCID, - Status: model.StorageCleanupCopyStatusPending, - } - scheduled := &model.StorageCleanupCopy{ - TaskID: task.ID, - UploadID: uploadID, - CopyIndex: 1, - ProviderID: onChainIDPtr(t, "101"), - DataSetID: onChainIDPtr(t, "1001"), - ClientDataSetID: onChainIDPtr(t, "5001"), - PieceID: onChainIDPtr(t, "2001"), - PieceCID: pieceCID, - Status: model.StorageCleanupCopyStatusPending, - } - if _, err := env.db.NewInsert().Model(unsupported).Exec(ctx); err != nil { - t.Fatalf("insert unsupported cleanup copy: %v", err) - } - if _, err := env.db.NewInsert().Model(scheduled).Exec(ctx); err != nil { - t.Fatalf("insert scheduled cleanup copy: %v", err) - } - env.storage.OpenCleanupContextFunc = func(context.Context, sdktypes.BigInt, storage.NewDataSetContextOptions) (synapse.CleanupContext, error) { - return fakeCleanupContext{ - pieceStatusFunc: func(context.Context, cid.Cid) (*storage.PieceStatus, error) { - return &storage.PieceStatus{Exists: true}, nil - }, - }, nil - } - - cleanup := worker.NewStorageCleanupWorker(env.repos, env.storage, 1, 20*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, env, cleanup, task.ID, model.TaskStatusWaiting, 5*time.Second) - - gotTask, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if gotTask.Status != model.TaskStatusWaiting { - t.Fatalf("task status = %s, want waiting", gotTask.Status) - } - var copies []model.StorageCleanupCopy - if err := env.db.NewSelect().Model(&copies).Where("task_id = ?", task.ID).OrderExpr("copy_index ASC").Scan(ctx); err != nil { - t.Fatalf("select cleanup copies: %v", err) - } - if copies[0].Status != model.StorageCleanupCopyStatusUnsupported { - t.Fatalf("unsupported copy status = %s, want unsupported", copies[0].Status) - } - if copies[1].Status != model.StorageCleanupCopyStatusDeleteScheduled { - t.Fatalf("scheduled copy status = %s, want delete_scheduled", copies[1].Status) - } -} - -func TestStorageCleanupWorkerSchedulesDeletionByPieceID(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - pieceCID := testCID(t).String() - uploadID := int64(90) - task := &model.Task{ - Type: model.TaskTypeStorageCleanup, - RefType: "storage_upload", - RefID: uploadID, - IdempotencyKey: "storage_cleanup:90", - Status: model.TaskStatusQueued, - MaxRetries: 3, - ScheduledAt: time.Now(), - Payload: map[string]interface{}{ - "storage_upload_id": uploadID, - "piece_cid": pieceCID, - }, - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create cleanup task: %v", err) - } - copy := &model.StorageCleanupCopy{ - TaskID: task.ID, - UploadID: uploadID, - CopyIndex: 0, - ProviderID: onChainIDPtr(t, "101"), - DataSetID: onChainIDPtr(t, "1001"), - ClientDataSetID: onChainIDPtr(t, "5001"), - PieceID: onChainIDPtr(t, "2001"), - PieceCID: pieceCID, - Status: model.StorageCleanupCopyStatusPending, - } - if _, err := env.db.NewInsert().Model(copy).Exec(ctx); err != nil { - t.Fatalf("insert cleanup copy: %v", err) - } - - var deletedPieceID string - env.storage.OpenCleanupContextFunc = func(context.Context, sdktypes.BigInt, storage.NewDataSetContextOptions) (synapse.CleanupContext, error) { - return fakeCleanupContext{ - pieceStatusFunc: func(context.Context, cid.Cid) (*storage.PieceStatus, error) { - return &storage.PieceStatus{Exists: true}, nil - }, - deletePieceByIDFunc: func(_ context.Context, pieceID sdktypes.BigInt) (*sdktypes.WriteResult, error) { - deletedPieceID = pieceID.String() - return &sdktypes.WriteResult{Hash: common.HexToHash("0xdelete")}, nil - }, - }, nil - } - - cleanup := worker.NewStorageCleanupWorker(env.repos, env.storage, 1, 20*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, env, cleanup, task.ID, model.TaskStatusWaiting, 5*time.Second) - - if deletedPieceID != "2001" { - t.Fatalf("deleted piece ID = %q, want 2001", deletedPieceID) - } - gotCopy := new(model.StorageCleanupCopy) - if err := env.db.NewSelect().Model(gotCopy).Where("id = ?", copy.ID).Scan(ctx); err != nil { - t.Fatalf("select cleanup copy: %v", err) - } - if gotCopy.Status != model.StorageCleanupCopyStatusDeleteScheduled { - t.Fatalf("copy status = %s, want delete_scheduled", gotCopy.Status) - } -} - -func TestStorageCleanupWorkerWaitsWhenObjectUsesUpload(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - pieceCID := testCID(t).String() - - bucket := &model.Bucket{Name: "storage-cleanup-object-ref-bucket", Status: model.BucketStatusActive} - if err := env.repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create: %v", err) - } - targetUpload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "deleted-version", - ContentSize: 10, - Checksum: "same-checksum", - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt(target): %v", err) - } - seedStorageCleanupWorkerCommittedCopy(t, env, bucket.ID, targetUpload.ID, pieceCID, "101", "1001", "2001") - if finalized, _, err := env.repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: targetUpload.ID}); err != nil { - t.Fatalf("FinalizeUploadIfTargetCopiesMet: %v", err) - } else if !finalized { - t.Fatal("target upload finalized = false, want true") - } - - if _, err := env.repos.Objects.CreateVersionAndSetCurrent(ctx, &model.ObjectVersion{ - BucketID: bucket.ID, - Key: "active-object", - VersionID: "active-object-version", - State: model.ObjectStateStored, - StorageUploadID: &targetUpload.ID, - Size: 10, - ContentType: "text/plain", - ETag: "etag", - Checksum: "same-checksum", - CacheKey: "cache-key", - }); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - - task := &model.Task{ - Type: model.TaskTypeStorageCleanup, - RefType: "storage_upload", - RefID: targetUpload.ID, - IdempotencyKey: "storage_cleanup:object-ref", - Status: model.TaskStatusQueued, - MaxRetries: 3, - ScheduledAt: time.Now(), - Payload: map[string]interface{}{ - "storage_upload_id": targetUpload.ID, - "piece_cid": pieceCID, - }, - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create cleanup task: %v", err) - } - copy := &model.StorageCleanupCopy{ - TaskID: task.ID, - UploadID: targetUpload.ID, - CopyIndex: 0, - ProviderID: onChainIDPtr(t, "101"), - DataSetID: onChainIDPtr(t, "1001"), - ClientDataSetID: onChainIDPtr(t, "5001"), - PieceID: onChainIDPtr(t, "2001"), - PieceCID: pieceCID, - Status: model.StorageCleanupCopyStatusPending, - } - if _, err := env.db.NewInsert().Model(copy).Exec(ctx); err != nil { - t.Fatalf("insert cleanup copy: %v", err) - } - - var cleanupContextCalls atomic.Int32 - env.storage.OpenCleanupContextFunc = func(context.Context, sdktypes.BigInt, storage.NewDataSetContextOptions) (synapse.CleanupContext, error) { - cleanupContextCalls.Add(1) - return fakeCleanupContext{}, nil - } - - cleanup := worker.NewStorageCleanupWorker(env.repos, env.storage, 1, 20*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, env, cleanup, task.ID, model.TaskStatusWaiting, 5*time.Second) - - gotTask, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if gotTask.Status != model.TaskStatusWaiting { - t.Fatalf("task status = %s, want waiting", gotTask.Status) - } - if gotTask.WaitReason == nil || *gotTask.WaitReason != model.TaskWaitReasonDependency { - t.Fatalf("task wait reason = %v, want dependency", gotTask.WaitReason) - } - if gotTask.StatusMessage == nil || *gotTask.StatusMessage != "Waiting for object references to clear" { - t.Fatalf("task status message = %v, want object reference wait message", gotTask.StatusMessage) - } - gotCopy := new(model.StorageCleanupCopy) - if err := env.db.NewSelect().Model(gotCopy).Where("id = ?", copy.ID).Scan(ctx); err != nil { - t.Fatalf("select cleanup copy: %v", err) - } - if gotCopy.Status != model.StorageCleanupCopyStatusPending { - t.Fatalf("copy status = %s, want pending while waiting for references", gotCopy.Status) - } - if cleanupContextCalls.Load() != 0 { - t.Fatalf("cleanup context calls = %d, want 0 while object references upload", cleanupContextCalls.Load()) - } -} - -func TestStorageCleanupWorkerWaitsWhenActiveUploadUsesSamePiece(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - pieceCID := testCID(t).String() - - bucket := &model.Bucket{Name: "storage-cleanup-active-upload-bucket", Status: model.BucketStatusActive} - if err := env.repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Buckets.Create: %v", err) - } - targetUpload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "deleted-version", - ContentSize: 10, - Checksum: "same-checksum", - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt(target): %v", err) - } - seedStorageCleanupWorkerCommittedCopy(t, env, bucket.ID, targetUpload.ID, pieceCID, "101", "1001", "2001") - if finalized, _, err := env.repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: targetUpload.ID}); err != nil { - t.Fatalf("FinalizeUploadIfTargetCopiesMet(target): %v", err) - } else if !finalized { - t.Fatal("target upload finalized = false, want true") - } - - task := &model.Task{ - Type: model.TaskTypeStorageCleanup, - RefType: "storage_upload", - RefID: targetUpload.ID, - IdempotencyKey: "storage_cleanup:active-upload-ref", - Status: model.TaskStatusQueued, - MaxRetries: 3, - ScheduledAt: time.Now(), - Payload: map[string]interface{}{ - "storage_upload_id": targetUpload.ID, - "piece_cid": pieceCID, - }, - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create cleanup task: %v", err) - } - copy := &model.StorageCleanupCopy{ - TaskID: task.ID, - UploadID: targetUpload.ID, - CopyIndex: 0, - ProviderID: onChainIDPtr(t, "101"), - DataSetID: onChainIDPtr(t, "1001"), - ClientDataSetID: onChainIDPtr(t, "5001"), - PieceID: onChainIDPtr(t, "2001"), - PieceCID: pieceCID, - Status: model.StorageCleanupCopyStatusPending, - } - if _, err := env.db.NewInsert().Model(copy).Exec(ctx); err != nil { - t.Fatalf("insert cleanup copy: %v", err) - } - - activeUpload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "active-version", - ContentSize: 10, - Checksum: "same-checksum", - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt(active): %v", err) - } - seedStorageCleanupWorkerCommittedCopy(t, env, bucket.ID, activeUpload.ID, pieceCID, "101", "1001", "2001") - - var cleanupContextCalls atomic.Int32 - env.storage.OpenCleanupContextFunc = func(context.Context, sdktypes.BigInt, storage.NewDataSetContextOptions) (synapse.CleanupContext, error) { - cleanupContextCalls.Add(1) - return fakeCleanupContext{}, nil - } - - cleanup := worker.NewStorageCleanupWorker(env.repos, env.storage, 1, 20*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, env, cleanup, task.ID, model.TaskStatusWaiting, 5*time.Second) - - gotTask, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if gotTask.Status != model.TaskStatusWaiting { - t.Fatalf("task status = %s, want waiting", gotTask.Status) - } - if gotTask.WaitReason == nil || *gotTask.WaitReason != model.TaskWaitReasonDependency { - t.Fatalf("task wait reason = %v, want dependency", gotTask.WaitReason) - } - if gotTask.StatusMessage == nil || *gotTask.StatusMessage != "Waiting for shared data references to clear" { - t.Fatalf("task status message = %v, want shared data reference wait message", gotTask.StatusMessage) - } - gotCopy := new(model.StorageCleanupCopy) - if err := env.db.NewSelect().Model(gotCopy).Where("id = ?", copy.ID).Scan(ctx); err != nil { - t.Fatalf("select cleanup copy: %v", err) - } - if gotCopy.Status != model.StorageCleanupCopyStatusPending { - t.Fatalf("copy status = %s, want pending while waiting for references", gotCopy.Status) - } - if cleanupContextCalls.Load() != 0 { - t.Fatalf("cleanup context calls = %d, want 0 while active upload references piece", cleanupContextCalls.Load()) - } -} - -func seedStorageCleanupWorkerCommittedCopy(t *testing.T, env *testWorkerEnv, bucketID int64, uploadID int64, pieceCID string, providerID string, dataSetID string, pieceID string) { - t.Helper() - ctx := context.Background() - provider := onChainID(t, providerID) - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucketID, - ProviderID: provider, - CopyIndex: 0, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: uploadID, - DataSetID: onChainID(t, dataSetID), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, uploadID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: provider, - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: uploadID, - CopyIndex: 0, - PieceCID: pieceCID, - PieceID: onChainIDPtr(t, pieceID), - RetrievalURL: "https://provider.example/" + pieceCID, - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } -} - -func runWorkerUntilTaskStatus(t *testing.T, env *testWorkerEnv, w worker.Worker, taskID int64, status model.TaskStatus, timeout time.Duration) { - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - done := make(chan struct{}) - go func() { - _ = w.Run(ctx) - close(done) - }() - - deadline := time.After(timeout) - ticker := time.NewTicker(20 * time.Millisecond) - defer ticker.Stop() - - for { - select { - case <-deadline: - cancel() - <-done - t.Fatalf("timed out waiting for task %d to reach status %s", taskID, status) - case <-ticker.C: - task, err := env.repos.Tasks.GetByID(context.Background(), taskID) - if err != nil { - continue - } - if task != nil && task.Status == status { - cancel() - <-done - return - } - } - } -} diff --git a/internal/worker/storage_commit_delay_test.go b/internal/worker/storage_commit_delay_test.go deleted file mode 100644 index e07d083..0000000 --- a/internal/worker/storage_commit_delay_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package worker - -import ( - "context" - "io" - "log/slog" - "testing" - "time" - - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagecommit" -) - -type storageCommitWaitTaskRepo struct { - repository.TaskRepository - delay time.Duration -} - -func (r *storageCommitWaitTaskRepo) WaitRunning( - _ context.Context, - _ *model.Task, - _ model.TaskWaitReason, - _ string, - delay time.Duration, -) error { - r.delay = delay - return nil -} - -func TestCommitObservationDelayUsesMinuteFloor(t *testing.T) { - for _, tc := range []struct { - name string - poll time.Duration - want time.Duration - }{ - {name: "default poll", poll: storageCommitPollDelay, want: time.Minute}, - {name: "long configured poll", poll: 2 * time.Minute, want: 2 * time.Minute}, - } { - t.Run(tc.name, func(t *testing.T) { - if got := commitObservationDelay(tc.poll); got != tc.want { - t.Fatalf("commitObservationDelay(%s) = %s, want %s", tc.poll, got, tc.want) - } - }) - } -} - -func TestUploaderCommitWaitDelays(t *testing.T) { - logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - for _, tc := range []struct { - name string - pollInterval time.Duration - result storagecommit.AdvanceResult - want time.Duration - }{ - { - name: "pending uses normal poll", - result: storagecommit.AdvanceResult{State: storagecommit.AdvancePending}, - want: storageCommitPollDelay, - }, - { - name: "observing attention uses minute floor", - result: storagecommit.AdvanceResult{State: storagecommit.AdvanceNeedsAttention, Continue: true}, - want: time.Minute, - }, - { - name: "observing attention respects longer poll", - pollInterval: 2 * time.Minute, - result: storagecommit.AdvanceResult{State: storagecommit.AdvanceNeedsAttention, Continue: true}, - want: 2 * time.Minute, - }, - { - name: "operator attention remains parked", - result: storagecommit.AdvanceResult{State: storagecommit.AdvanceNeedsAttention}, - want: storageCommitAttentionDelay, - }, - } { - t.Run(tc.name, func(t *testing.T) { - tasks := new(storageCommitWaitTaskRepo) - uploader := &Uploader{ - repos: &repository.Repositories{Tasks: tasks}, - pollInterval: tc.pollInterval, - } - if !uploader.waitForCommitAdvance(t.Context(), &model.Task{}, logger, tc.result) { - t.Fatal("waitForCommitAdvance rejected commit wait state") - } - if tasks.delay != tc.want { - t.Fatalf("wait delay = %s, want %s", tasks.delay, tc.want) - } - }) - } -} diff --git a/internal/worker/storage_task_handlers.go b/internal/worker/storage_task_handlers.go new file mode 100644 index 0000000..fe08b8f --- /dev/null +++ b/internal/worker/storage_task_handlers.go @@ -0,0 +1,1592 @@ +package worker + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "os" + "sort" + "strconv" + "strings" + "time" + + "github.com/ipfs/go-cid" + "github.com/strahe/synaps3/internal/bucketlifecycle" + "github.com/strahe/synaps3/internal/cache" + "github.com/strahe/synaps3/internal/cacheeviction" + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/objectlimits" + "github.com/strahe/synaps3/internal/storagecommit" + "github.com/strahe/synaps3/internal/storagepipeline" + "github.com/strahe/synaps3/internal/synapse" + taskengine "github.com/strahe/synaps3/internal/task" + idtypes "github.com/strahe/synaps3/internal/types" + "github.com/strahe/synapse-go/piece" + "github.com/strahe/synapse-go/storage" + sdktypes "github.com/strahe/synapse-go/types" +) + +const ( + storageDependencyWait = time.Minute + storagePollInterval = 5 * time.Second + dataSetAttentionAfter = 15 * time.Minute + storeAttentionAfter = 30 * time.Minute +) + +type dataSetCreationCheckpoint struct { + AttemptedAt time.Time `json:"attempted_at"` + TransactionID string `json:"transaction_id,omitempty"` + StatusURL string `json:"status_url,omitempty"` + ClientDataSetID string `json:"client_data_set_id,omitempty"` +} + +type storeCheckpoint struct { + AttemptedAt time.Time `json:"attempted_at"` + IntendedPieceCID string `json:"intended_piece_cid"` + ProviderServiceURL string `json:"provider_service_url"` + IngressAttempt int `json:"ingress_attempt,omitempty"` +} + +type pullCheckpoint struct { + AttemptedAt time.Time `json:"attempted_at"` + // AttemptID names the ledger row this request was recorded in, so recovery + // resolves the same row the first execute created. + AttemptID string `json:"attempt_id"` + PieceCID string `json:"piece_cid"` + SourceProviderID string `json:"source_provider_id"` + SourceDataSetID string `json:"source_data_set_id"` + SourcePieceID string `json:"source_piece_id"` + SourceRetrievalURL string `json:"source_retrieval_url"` + CommitExtraDataHex string `json:"commit_extra_data_hex"` +} + +type selectedBinding struct { + copyIndex int + target synapse.StorageTarget +} + +type uploadBindingPlan struct { + copyIndex int + provider idtypes.OnChainID + dataSet *storage.DataSetRef +} + +func (h *TaskHandlers) uploadPlanHandler() taskengine.Handler { + definition := taskengine.Definition{ + Type: model.TaskTypeUploadPlan, InputVersion: 1, + Codec: taskengine.StrictJSONCodec(func(input *storagepipeline.UploadPlanInput) error { + return storagepipeline.ValidateUploadPlanInput(*input) + }), + RetryLimit: h.retryLimit(), AllowRetry: true, + } + run := func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + input, err := taskengine.DecodeInput[storagepipeline.UploadPlanInput](execution) + if err != nil { + return decodeFailure(string(definition.Type), err) + } + if h.deps.Storage == nil || h.taskService == nil { + return taskengine.Fail(errors.New("storage task dependencies are unavailable"), "dependency_unavailable", nil) + } + // Ingest is planned for the bytes, not for one version of them, so the + // content row is the whole subject of this task. + upload, err := h.deps.Repositories.Contents.GetByID(ctx, input.ContentID) + if err != nil { + return retryTask(err, "upload_load_failed") + } + if upload == nil { + return taskengine.Cancel("Storage is no longer required", nil) + } + unreferenced, err := h.deps.Repositories.Objects.ContentIsUnreferenced(ctx, upload.ID) + if err != nil { + return retryTask(err, "upload_reference_check_failed") + } + if unreferenced { + return taskengine.Cancel("Storage is no longer required", nil) + } + bucket, err := h.deps.Repositories.Buckets.GetByID(ctx, upload.BucketID) + if err != nil { + return retryTask(err, "upload_bucket_load_failed") + } + if bucket == nil { + return taskengine.Fail(repository.ErrNotFound, "upload_bucket_missing", nil) + } + if upload.AcceptedAt != nil { + return taskengine.Complete("Storage copies are ready", nil) + } + + plan, err := h.selectUploadBindings(ctx, bucket, upload) + if err != nil { + if synapse.IsProviderUnavailable(err) || synapse.IsNoProviderCandidates(err) { + return taskengine.Suspend(model.TaskResumeModeExecute, storageDependencyWait, "providers", "Waiting for storage providers", nil) + } + return retryTask(err, "storage_selection_failed") + } + if len(plan) < model.ClampStorageCopies(upload.RequestedCopies) { + return taskengine.Suspend(model.TaskResumeModeExecute, storageDependencyWait, "providers", "Waiting for storage providers", nil) + } + targets := make([]synapse.StorageTarget, 0, len(plan)) + for i := range plan { + targets = append(targets, plan[i].target) + } + dataSize := uint64(objectlimits.MinFOCUploadSize) + if upload.ContentSize > int64(dataSize) { + dataSize = uint64(upload.ContentSize) + } + costs, err := h.deps.Storage.PrepareUpload(ctx, dataSize, targets) + if err != nil { + if synapse.IsProviderUnavailable(err) || synapse.IsNoProviderCandidates(err) { + return taskengine.Suspend(model.TaskResumeModeExecute, storageDependencyWait, "funding", "Waiting for storage funding", nil) + } + return retryTask(err, "storage_funding_failed") + } + if costs == nil || !costs.Ready { + return taskengine.Suspend(model.TaskResumeModeExecute, storageDependencyWait, "funding", uploadFundingWaitMessage(costs), nil) + } + bindingPlan := make([]uploadBindingPlan, 0, len(plan)) + for i := range plan { + entry := plan[i] + frozen := uploadBindingPlan{ + copyIndex: entry.copyIndex, + provider: idtypes.OnChainIDFromSDK(entry.target.ProviderID()), + } + if ref, ok := entry.target.DataSetRef(); ok { + frozen.dataSet = &ref + } + bindingPlan = append(bindingPlan, frozen) + } + + return taskengine.Complete("Storage work scheduled", func(ctx context.Context, repos *repository.Repositories) error { + bindings := make([]model.StorageDataSet, 0, len(bindingPlan)) + for i := range bindingPlan { + entry := bindingPlan[i] + binding, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: entry.provider, + CopyIndex: entry.copyIndex, CreatedByContentID: upload.ID, + }) + if err != nil { + return err + } + if entry.dataSet != nil { + dataSetID, clientDataSetID, err := dataSetRefIDs(binding, *entry.dataSet) + if err != nil { + return err + } + if err := repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: binding.ID, ContentID: upload.ID, DataSetID: dataSetID, ClientDataSetID: &clientDataSetID, + }); err != nil { + return err + } + binding.Status = model.StorageDataSetStatusReady + binding.DataSetID = &dataSetID + binding.ClientDataSetID = &clientDataSetID + } + bindings = append(bindings, *binding) + } + sort.Slice(bindings, func(i, j int) bool { return bindings[i].CopyIndex < bindings[j].CopyIndex }) + copyBindings := make([]repository.UploadCopyBindingInput, 0, len(bindings)) + for i := range bindings { + method := model.StorageCopyTransferMethodPeerPull + if i == 0 { + method = model.StorageCopyTransferMethodIngress + } + copyBindings = append(copyBindings, repository.UploadCopyBindingInput{ + StorageDataSetID: bindings[i].ID, CopyIndex: bindings[i].CopyIndex, + TransferMethod: method, ProviderID: bindings[i].ProviderID, + }) + } + if err := repos.Contents.CreateUploadCopiesForBindings(ctx, upload.ID, copyBindings); err != nil { + return err + } + for i := range bindings { + binding := &bindings[i] + if binding.Status != model.StorageDataSetStatusReady { + if binding.EnsureTaskID == nil { + if err := h.enqueueDataSetEnsure(ctx, repos, binding); err != nil && !errors.Is(err, repository.ErrConflict) { + return err + } + } + continue + } + copyRow, err := repos.Contents.GetUploadCopyForDataSet(ctx, upload.ID, binding.ID) + if err != nil { + return err + } + if copyRow != nil && copyRow.ActiveTaskID == nil { + if err := h.enqueueInitialCopyTask(ctx, repos, copyRow.ID, model.TaskTypeStorageTransferPlan); err != nil && !errors.Is(err, repository.ErrConflict) { + return err + } + } + } + return nil + }) + } + return taskHandler{definition: definition, execute: run, recover: run} +} + +func uploadFundingWaitMessage(costs *storage.MultiContextCosts) string { + parts := make([]string, 0, 2) + if costs != nil && costs.DepositNeeded != nil && costs.DepositNeeded.Sign() > 0 { + parts = append(parts, fmt.Sprintf("deposit %s USDFC base units", costs.DepositNeeded.String())) + } + if costs != nil && costs.NeedsFWSSMaxApproval { + parts = append(parts, "approve FWSS spending") + } + if len(parts) == 0 { + return "Waiting for Filecoin payment funding" + } + return "Waiting for Filecoin payment funding: " + strings.Join(parts, "; ") +} + +func (h *TaskHandlers) selectUploadBindings(ctx context.Context, bucket *model.Bucket, upload *model.StorageContent) ([]selectedBinding, error) { + return h.selectBucketBindings(ctx, bucket, model.ClampStorageCopies(upload.RequestedCopies)) +} + +func (h *TaskHandlers) selectBucketBindings(ctx context.Context, bucket *model.Bucket, targetCount int) ([]selectedBinding, error) { + bindings, err := h.deps.Repositories.Contents.ListDataSetBindings(ctx, bucket.ID) + if err != nil { + return nil, err + } + targetCount = model.ClampStorageCopies(targetCount) + selected := make([]selectedBinding, 0, targetCount) + excluded := make([]sdktypes.BigInt, 0, len(bindings)) + usedIndexes := make(map[int]struct{}, len(bindings)) + for i := range bindings { + binding := &bindings[i] + excluded = append(excluded, binding.ProviderID.SDK()) + if !binding.IsCurrent { + continue + } + usedIndexes[binding.CopyIndex] = struct{}{} + if binding.CopyIndex >= targetCount || len(selected) >= targetCount || (binding.Status != model.StorageDataSetStatusReady && binding.Status != model.StorageDataSetStatusPending && binding.Status != model.StorageDataSetStatusCreating) { + continue + } + target, err := h.openBindingTarget(ctx, bucket.Name, binding) + if err != nil { + return nil, err + } + selected = append(selected, selectedBinding{copyIndex: binding.CopyIndex, target: target}) + } + missing := targetCount - len(selected) + if missing <= 0 { + sort.Slice(selected, func(i, j int) bool { return selected[i].copyIndex < selected[j].copyIndex }) + return selected, nil + } + // Free positions come from the bucket's own slot rows, not from the global + // maximum: a data set can only exist on a slot the bucket opened, and the + // foreign key would reject anything else. + slots, err := h.deps.Repositories.Buckets.ActiveReplicaSlots(ctx, bucket.ID) + if err != nil { + return nil, err + } + indexes := make([]int, 0, missing) + for _, copyIndex := range slots { + if len(indexes) >= missing { + break + } + if _, exists := usedIndexes[copyIndex]; !exists { + indexes = append(indexes, copyIndex) + } + } + if len(indexes) == 0 { + sort.Slice(selected, func(i, j int) bool { return selected[i].copyIndex < selected[j].copyIndex }) + return selected, nil + } + if missing > len(indexes) { + missing = len(indexes) + } + targets, selectErr := h.deps.Storage.SelectUploadTargets(ctx, storage.SelectUploadContextsOptions{ + Copies: missing, ExcludeProviderIDs: excluded, DataSetMetadata: map[string]string{"bucket": bucket.Name}, + }) + for i := range targets { + if i >= len(indexes) || targets[i] == nil { + break + } + selected = append(selected, selectedBinding{copyIndex: indexes[i], target: targets[i]}) + } + if selectErr != nil && len(targets) == 0 { + return nil, selectErr + } + sort.Slice(selected, func(i, j int) bool { return selected[i].copyIndex < selected[j].copyIndex }) + return selected, nil +} + +func (h *TaskHandlers) dataSetEnsureHandler() taskengine.Handler { + definition := taskengine.Definition{ + Type: model.TaskTypeStorageDataSetEnsure, InputVersion: 1, + Codec: taskengine.StrictJSONCodec(func(input *storagepipeline.DataSetInput) error { + return storagepipeline.ValidateDataSetInput(*input) + }), + RetryLimit: h.retryLimit(), AllowRetry: true, + } + return taskHandler{ + definition: definition, + execute: func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + return h.runDataSetEnsure(ctx, execution, true) + }, + recover: func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + return h.runDataSetEnsure(ctx, execution, false) + }, + } +} + +func (h *TaskHandlers) runDataSetEnsure(ctx context.Context, execution taskengine.Execution, mayCreate bool) taskengine.Result { + input, err := taskengine.DecodeInput[storagepipeline.DataSetInput](execution) + if err != nil { + return decodeFailure(string(model.TaskTypeStorageDataSetEnsure), err) + } + if h.deps.Storage == nil { + return taskengine.Fail(errors.New("storage client is unavailable"), "dependency_unavailable", nil) + } + binding, err := h.deps.Repositories.Contents.AuthorizeDataSetEnsureTask(ctx, input.DataSetID, execution.ID()) + if errors.Is(err, repository.ErrConflict) || errors.Is(err, repository.ErrNotFound) { + return taskengine.Cancel("Storage service setup was superseded", nil) + } + if err != nil { + return retryTask(err, "dataset_authorization_failed") + } + if binding.Status == model.StorageDataSetStatusReady && binding.DataSetID != nil && !binding.DataSetID.IsZero() { + return taskengine.Complete("Storage service is ready", func(ctx context.Context, repos *repository.Repositories) error { + return h.finishDataSetEnsure(ctx, repos, binding, execution.ID()) + }) + } + bucket, err := h.deps.Repositories.Buckets.GetByID(ctx, binding.BucketID) + if err != nil || bucket == nil { + if err == nil { + err = repository.ErrNotFound + } + return retryTask(err, "dataset_bucket_load_failed") + } + provider, err := h.deps.Storage.OpenProviderTarget(ctx, binding.ProviderID.SDK(), storage.NewProviderContextOptions{ + DataSetMetadata: map[string]string{"bucket": bucket.Name}, + }) + if err != nil { + return taskengine.Suspend(model.TaskResumeModeRecover, storageDependencyWait, "provider", "Waiting for storage provider", nil) + } + matching, err := h.deps.Storage.FindMatchingDataSet(ctx, binding.ProviderID.SDK(), map[string]string{"bucket": bucket.Name}, provider.CDNEnabled()) + if err == nil && matching != nil { + dataSetID, clientDataSetID, identityErr := dataSetRefIDs(binding, *matching) + if identityErr != nil { + return taskengine.Fail(identityErr, "dataset_identity_mismatch", nil) + } + return h.completeDataSetEnsure(binding, execution.ID(), dataSetID, clientDataSetID) + } + if err != nil { + if synapse.IsProviderUnavailable(err) { + return taskengine.Suspend(model.TaskResumeModeRecover, storageDependencyWait, "provider", "Waiting for storage provider", nil) + } + return retryTask(err, "dataset_discovery_failed") + } + + checkpoint, hasCheckpoint, err := taskengine.DecodeCheckpoint[dataSetCreationCheckpoint](execution) + if err != nil { + return taskengine.Fail(err, "invalid_checkpoint", nil) + } + if !hasCheckpoint && binding.CreateTransactionID != nil && binding.CreateStatusURL != nil && binding.ClientDataSetID != nil { + checkpoint = dataSetCreationCheckpoint{ + AttemptedAt: time.Now().UTC(), TransactionID: *binding.CreateTransactionID, + StatusURL: *binding.CreateStatusURL, ClientDataSetID: binding.ClientDataSetID.String(), + } + hasCheckpoint = true + } + if hasCheckpoint && checkpoint.TransactionID != "" { + return h.waitDataSetCreation(ctx, execution, binding, provider, checkpoint) + } + if hasCheckpoint { + if time.Since(checkpoint.AttemptedAt) >= dataSetAttentionAfter { + err := errors.New("storage service creation outcome could not be recovered") + return taskengine.Fail(err, "dataset_creation_unknown", dataSetFailureSettlement(binding.ID, execution.ID(), err.Error(), false)) + } + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Checking storage service creation", nil) + } + if !mayCreate { + return taskengine.Suspend(model.TaskResumeModeExecute, 0, "safe_to_execute", "Storage service is ready to create", nil) + } + + checkpoint = dataSetCreationCheckpoint{AttemptedAt: time.Now().UTC()} + var ( + created *storage.CreateDataSetResult + evidenceErr error + submission storage.CreateDataSetSubmission + ) + createCtx, cancelCreate := context.WithCancel(ctx) + attempted, createErr := execution.WithCheckpointedEffect(ctx, taskengine.ResourceProviderMutation, checkpoint, nil, func(context.Context) error { + var err error + created, err = provider.CreateDataSet(createCtx, &storage.CreateDataSetOptions{OnSubmitted: func(sub storage.CreateDataSetSubmission) { + submission = sub + checkpoint.TransactionID = sub.TransactionID + checkpoint.StatusURL = sub.StatusURL + if sub.ClientDataSetID != nil { + checkpoint.ClientDataSetID = sub.ClientDataSetID.String() + } + evidenceErr = execution.WriteCheckpointWith(ctx, checkpoint, func(ctx context.Context, repos *repository.Repositories) error { + return repos.Contents.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ + ID: binding.ID, ContentID: derefInt64(binding.CreatedByContentID), TransactionID: sub.TransactionID, + StatusURL: sub.StatusURL, ClientDataSetID: onChainIDPtr(sub.ClientDataSetID), + }) + }) + cancelCreate() + }}) + return err + }) + cancelCreate() + if createErr != nil && !attempted { + return retryTask(createErr, "dataset_creation_not_started") + } + if evidenceErr != nil { + clientDataSetID := onChainIDPtr(submission.ClientDataSetID) + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Recording storage service creation", func(ctx context.Context, repos *repository.Repositories) error { + if _, err := repos.Contents.AuthorizeDataSetEnsureTask(ctx, binding.ID, execution.ID()); err != nil { + return err + } + return repos.Contents.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ + ID: binding.ID, ContentID: derefInt64(binding.CreatedByContentID), TransactionID: submission.TransactionID, + StatusURL: submission.StatusURL, ClientDataSetID: clientDataSetID, + }) + }) + } + if created != nil { + dataSetID, clientDataSetID, identityErr := dataSetResultIDs(binding, created) + if identityErr != nil { + return taskengine.Fail(identityErr, "dataset_identity_mismatch", nil) + } + return h.completeDataSetEnsure(binding, execution.ID(), dataSetID, clientDataSetID) + } + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Checking storage service creation", nil) +} + +func (h *TaskHandlers) waitDataSetCreation( + ctx context.Context, + execution taskengine.Execution, + binding *model.StorageDataSet, + provider synapse.ProviderTarget, + checkpoint dataSetCreationCheckpoint, +) taskengine.Result { + clientID, err := idtypes.ParseOnChainID("clientDataSetID", checkpoint.ClientDataSetID) + if err != nil { + return taskengine.Fail(err, "invalid_checkpoint", nil) + } + clientSDK := clientID.SDK() + result, err := provider.WaitForDataSetCreated(ctx, storage.CreateDataSetSubmission{ + ProviderID: binding.ProviderID.SDK(), TransactionID: checkpoint.TransactionID, + StatusURL: checkpoint.StatusURL, ClientDataSetID: &clientSDK, + }) + if err != nil { + if errors.Is(err, synapse.ErrProviderTransactionRejected) { + return taskengine.Fail(err, "dataset_creation_rejected", dataSetFailureSettlement(binding.ID, execution.ID(), err.Error(), true)) + } + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Waiting for storage service", nil) + } + dataSetID, clientDataSetID, err := dataSetResultIDs(binding, result) + if err != nil { + return taskengine.Fail(err, "dataset_identity_mismatch", nil) + } + return h.completeDataSetEnsure(binding, execution.ID(), dataSetID, clientDataSetID) +} + +// dataSetFailureSettlement records that a generation could not be created. +// rejected says the chain refused the creation, which is proof no data set +// exists; an unknown outcome is not, and leaves the row in place so an operator +// still has the provider and the transaction to work from. +func dataSetFailureSettlement(dataSetID, taskID int64, message string, rejected bool) taskengine.Settlement { + return func(ctx context.Context, repos *repository.Repositories) error { + if _, err := repos.Contents.AuthorizeDataSetEnsureTask(ctx, dataSetID, taskID); err != nil { + return err + } + if err := repos.Contents.MarkDataSetFailed(ctx, dataSetID, message); err != nil { + return err + } + if !rejected { + // The outcome is unknown, so the generation may still exist on the + // provider. Retrying this task rediscovers it through + // FindMatchingDataSet and continues the copies bound to it, and + // continuation skips copies that already failed — so they are left + // alone rather than terminated on a guess. + return nil + } + // The chain refused the creation, so nothing will ever serve these + // copies: the data set they name will not exist, and continuation would + // have nothing to rediscover. Failing them keeps their objects from + // sitting at "uploading" forever. + if err := failDataSetCopies(ctx, repos, dataSetID, message); err != nil { + return err + } + // The generation ends here rather than being deleted. Retiring it frees + // the provider for the bucket while keeping the record that this one was + // tried, and it is the only status the provider reservation index lets + // go of. + if _, err := repos.Contents.RetireRejectedDataSet(ctx, dataSetID); err != nil { + return err + } + return nil + } +} + +func failDataSetCopies(ctx context.Context, repos *repository.Repositories, dataSetID int64, message string) error { + copies, err := repos.Contents.ListIncompleteCopiesForDataSet(ctx, dataSetID) + if err != nil { + return err + } + for i := range copies { + copyRow := &copies[i] + if err := repos.Contents.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{ + StorageCopyID: copyRow.ID, ContentID: copyRow.ContentID, CopyIndex: copyRow.CopyIndex, + LastError: message, + }); err != nil { + return err + } + } + return nil +} + +func (h *TaskHandlers) completeDataSetEnsure(binding *model.StorageDataSet, taskID int64, dataSetID, clientDataSetID idtypes.OnChainID) taskengine.Result { + return taskengine.Complete("Storage service is ready", func(ctx context.Context, repos *repository.Repositories) error { + if err := repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: binding.ID, ContentID: derefInt64(binding.CreatedByContentID), DataSetID: dataSetID, ClientDataSetID: &clientDataSetID, + }); err != nil { + return err + } + return h.finishDataSetEnsure(ctx, repos, binding, taskID) + }) +} + +func (h *TaskHandlers) finishDataSetEnsure(ctx context.Context, repos *repository.Repositories, binding *model.StorageDataSet, taskID int64) error { + if err := repos.Contents.CompleteDataSetEnsureTask(ctx, binding.ID, taskID); err != nil { + return err + } + if err := h.continueDataSetCopies(ctx, repos, binding.ID); err != nil { + return err + } + bucket, err := repos.Buckets.GetByID(ctx, binding.BucketID) + if err != nil || bucket == nil { + if err == nil { + err = repository.ErrNotFound + } + return err + } + required := h.effectiveBucketCopies(bucket) + if _, err := repos.Buckets.PromoteReadyIfProvisioned(ctx, bucket.ID, required); err != nil { + return err + } + provisionTask, err := repos.Tasks.GetByIdentity(ctx, model.TaskTypeBucketProvision, bucketlifecycle.ProvisionKey(bucket.ID, bucket.DefaultCopies)) + if err != nil { + return err + } + if provisionTask == nil { + return nil + } + _, err = h.taskService.WakeInTransaction(ctx, repos, []int64{provisionTask.ID}) + return err +} + +func (h *TaskHandlers) continueDataSetCopies(ctx context.Context, repos *repository.Repositories, dataSetID int64) error { + if h.taskService == nil { + return errors.New("task service is unavailable") + } + copies, err := repos.Contents.ListIncompleteCopiesForDataSet(ctx, dataSetID) + if err != nil { + return err + } + wakeIDs := make([]int64, 0, len(copies)) + for i := range copies { + copyRow := &copies[i] + if copyRow.ActiveTaskID != nil { + wakeIDs = append(wakeIDs, *copyRow.ActiveTaskID) + continue + } + if err := h.enqueueInitialCopyTask(ctx, repos, copyRow.ID, model.TaskTypeStorageTransferPlan); err != nil && !errors.Is(err, repository.ErrConflict) { + return err + } + } + _, err = h.taskService.WakeInTransaction(ctx, repos, wakeIDs) + return err +} + +func (h *TaskHandlers) transferPlanHandler() taskengine.Handler { + definition := copyDefinition(model.TaskTypeStorageTransferPlan, h.retryLimit()) + run := func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + input, copyRow, handled, result := h.authorizeCopyTask(ctx, execution) + if handled { + return result + } + if copyRow.Status == model.StorageCopyStatusCommitted { + return h.completeCopyTask(input, execution.ID(), "Storage copy is complete") + } + if copyRow.Status == model.StorageCopyStatusPieceReady || copyRow.Status == model.StorageCopyStatusCommitting { + return h.advanceCopyTask(input, execution.ID(), model.TaskTypeStorageCommitCoordinate, "Storage copy is ready to register") + } + binding, err := h.deps.Repositories.Contents.GetDataSetBindingByID(ctx, copyRow.StorageDataSetID) + if err != nil { + return h.retryCopyTask(execution, input, copyRow, err, "dataset_load_failed") + } + if binding == nil { + return h.failCopyTask(execution, input, copyRow, repository.ErrNotFound, "dataset_missing") + } + if binding.Status != model.StorageDataSetStatusReady || binding.DataSetID == nil || binding.DataSetID.IsZero() { + return taskengine.Suspend(model.TaskResumeModeExecute, storagePollInterval, "dataset", "Waiting for storage service", nil) + } + unreferenced, err := h.deps.Repositories.Objects.ContentIsUnreferenced(ctx, copyRow.ContentID) + if err != nil { + return h.retryCopyTask(execution, input, copyRow, err, "copy_owner_load_failed") + } + if unreferenced { + return h.completeCopyTask(input, execution.ID(), "Storage copy is no longer required") + } + if copyRow.TransferMethod == model.StorageCopyTransferMethodPeerPull { + sources, err := h.deps.Repositories.Contents.ListReadableCommittedCopies(ctx, copyRow.ContentID) + if err != nil { + return h.retryCopyTask(execution, input, copyRow, err, "copy_source_load_failed") + } + if len(sources) > 0 { + return h.advanceCopyTask(input, execution.ID(), model.TaskTypeStoragePull, "Storage copy is ready to transfer") + } + } + // Cached bytes are named by the content, so residency is asked of the + // content's cache entry rather than of a version that happens to exist. + cacheEntry, err := h.deps.Repositories.CacheEvictions.GetCacheEntry(ctx, copyRow.ContentID) + if err != nil { + return h.retryCopyTask(execution, input, copyRow, err, "copy_cache_load_failed") + } + if cacheEntry != nil && cacheEntry.InCache && h.deps.Cache != nil { + bucket, err := h.deps.Repositories.Buckets.GetByID(ctx, copyRow.BucketID) + if err != nil { + return h.retryCopyTask(execution, input, copyRow, err, "copy_bucket_load_failed") + } + if bucket != nil && h.deps.Cache.Exists(ctx, bucket.Name, model.ContentCacheKey(copyRow.ContentID)) { + return h.advanceCopyTask(input, execution.ID(), model.TaskTypeStorageStore, "Storage copy is ready to transfer") + } + } + return taskengine.Suspend(model.TaskResumeModeExecute, storageDependencyWait, "source", "Waiting for a readable storage source", nil) + } + return taskHandler{definition: definition, execute: run, recover: run} +} + +func (h *TaskHandlers) storeHandler() taskengine.Handler { + definition := copyDefinition(model.TaskTypeStorageStore, h.retryLimit()) + definition.AllowRetry = true + definition.CanManualRetry = func(task *model.Task) bool { + if task == nil || task.FailureReason == nil { + return false + } + switch *task.FailureReason { + case "store_not_started": + return len(task.Checkpoint) == 0 + case "store_outcome_unknown", "copy_owner_missing", "copy_context_failed": + return len(task.Checkpoint) > 0 + default: + return false + } + } + return taskHandler{ + definition: definition, + execute: func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + return h.runStore(ctx, execution, true) + }, + recover: func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + return h.runStore(ctx, execution, false) + }, + } +} + +func (h *TaskHandlers) runStore(ctx context.Context, execution taskengine.Execution, mayStore bool) taskengine.Result { + input, copyRow, handled, result := h.authorizeCopyTask(ctx, execution) + if handled { + return result + } + if copyRow.Status == model.StorageCopyStatusCommitted { + return h.completeCopyTask(input, execution.ID(), "Storage copy is complete") + } + if copyRow.Status == model.StorageCopyStatusPieceReady || copyRow.Status == model.StorageCopyStatusCommitting { + return h.advanceCopyTask(input, execution.ID(), model.TaskTypeStorageCommitCoordinate, "Storage copy is ready to register") + } + checkpoint, hasCheckpoint, err := taskengine.DecodeCheckpoint[storeCheckpoint](execution) + if err != nil { + return taskengine.Fail(err, "invalid_checkpoint", nil) + } + _, target, content, bucket, err := h.copyContext(ctx, copyRow) + if err != nil { + return h.copyContextFailure(execution, input, copyRow, err, !hasCheckpoint) + } + if hasCheckpoint { + return h.recoverStore(ctx, execution, input, copyRow, target, checkpoint) + } + if !mayStore { + return taskengine.Suspend(model.TaskResumeModeExecute, 0, "safe_to_execute", "Storage transfer is ready", nil) + } + if h.deps.Cache == nil || h.deps.CacheGate == nil { + return h.failCopyTask(execution, input, copyRow, errors.New("cache reader is unavailable"), "dependency_unavailable") + } + cacheKey := model.ContentCacheKey(content.ID) + releaseCache := h.deps.CacheGate.HoldRead(cacheKey) + defer releaseCache() + calculateReader, _, err := h.deps.Cache.Get(ctx, bucket.Name, cacheKey) + if err != nil { + if os.IsNotExist(err) { + return taskengine.Suspend(model.TaskResumeModeExecute, storageDependencyWait, "source", "Waiting for retained cache data", nil) + } + return h.retryCopyTask(execution, input, copyRow, err, "cache_open_failed") + } + pieceInfo, calculateErr := piece.Calculate(calculateReader) + closeErr := calculateReader.Close() + if calculateErr != nil { + return h.retryCopyTask(execution, input, copyRow, calculateErr, "store_identity_failed") + } + if closeErr != nil { + return h.retryCopyTask(execution, input, copyRow, closeErr, "cache_close_failed") + } + if !pieceInfo.CIDv2.Defined() || content.ContentSize < 0 || pieceInfo.RawSize != uint64(content.ContentSize) { + err := fmt.Errorf("calculated storage identity has size %d, expected %d", pieceInfo.RawSize, content.ContentSize) + return h.failCopyTask(execution, input, copyRow, err, "store_identity_mismatch") + } + storeReader, _, err := h.deps.Cache.Get(ctx, bucket.Name, cacheKey) + if err != nil { + if os.IsNotExist(err) { + return taskengine.Suspend(model.TaskResumeModeExecute, storageDependencyWait, "source", "Waiting for retained cache data", nil) + } + return h.retryCopyTask(execution, input, copyRow, err, "cache_open_failed") + } + defer func() { _ = storeReader.Close() }() + checkpoint = storeCheckpoint{ + AttemptedAt: time.Now().UTC(), IntendedPieceCID: pieceInfo.CIDv2.String(), + ProviderServiceURL: target.ServiceURL(), + } + var checkpointSettlement taskengine.Settlement + if copyRow.TransferMethod == model.StorageCopyTransferMethodIngress { + checkpoint.IngressAttempt = copyRow.IngressStoreAttempt + 1 + checkpointSettlement = func(ctx context.Context, repos *repository.Repositories) error { + _, err := repos.Contents.BeginIngressStoreProgress(ctx, repository.BeginIngressStoreProgressInput{ + CopyID: copyRow.ID, Generation: input.Generation, TaskID: execution.ID(), Attempt: checkpoint.IngressAttempt, + }) + return err + } + } + var stored *storage.StoreResult + var progress *uploadProgressReporter + attempted, err := execution.WithCheckpointedEffect(ctx, taskengine.ResourceProviderMutation, checkpoint, checkpointSettlement, func(ctx context.Context) error { + if copyRow.TransferMethod == model.StorageCopyTransferMethodIngress { + progress = h.newIngressProgressReporter(ctx, execution.ID(), input.Generation, copyRow.ID, checkpoint.IngressAttempt, content, bucket) + } + options := &storage.StoreOptions{PieceCID: pieceInfo.CIDv2} + if progress != nil { + options.OnProgress = progress.OnProgress + } + var storeErr error + stored, storeErr = target.Store(ctx, storeReader, options) + return storeErr + }) + defer progress.Close() + if err != nil { + if !attempted { + return h.retryStoreNotStarted(execution, err) + } + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Checking storage transfer", nil) + } + if stored == nil || !stored.PieceCID.Equals(pieceInfo.CIDv2) || stored.Size != content.ContentSize { + err := errors.New("storage provider returned a mismatched piece identity or size") + return h.failCopyTask(execution, input, copyRow, err, "store_result_invalid") + } + progress.Flush(content.ContentSize, true) + return h.finishPieceTransfer(ctx, execution, input, copyRow, target, stored.PieceCID) +} + +func (h *TaskHandlers) recoverStore( + ctx context.Context, + execution taskengine.Execution, + input storagepipeline.CopyGenerationInput, + copyRow *model.StorageCopy, + target synapse.DataSetTarget, + checkpoint storeCheckpoint, +) taskengine.Result { + if checkpoint.AttemptedAt.IsZero() || checkpoint.IntendedPieceCID == "" || checkpoint.ProviderServiceURL == "" { + return taskengine.Fail(errors.New("storage transfer checkpoint is incomplete"), "invalid_checkpoint", nil) + } + pieceCID, err := cid.Parse(checkpoint.IntendedPieceCID) + if err != nil { + return taskengine.Fail(err, "invalid_checkpoint", nil) + } + pieceInfo, err := piece.ParseV2(pieceCID) + if err != nil || copyRow.ContentSize < 0 || pieceInfo.RawSize != uint64(copyRow.ContentSize) { + if err == nil { + err = fmt.Errorf("checkpointed storage identity has size %d, expected %d", pieceInfo.RawSize, copyRow.ContentSize) + } + return taskengine.Fail(err, "invalid_checkpoint", nil) + } + if h.deps.ParkedPieces == nil { + return taskengine.Suspend(model.TaskResumeModeRecover, storageDependencyWait, "provider", "Waiting for storage provider", nil) + } + state, findErr := h.deps.ParkedPieces.FindParkedPiece(ctx, checkpoint.ProviderServiceURL, pieceCID) + if findErr == nil && state == synapse.ParkedPieceReady { + return h.finishPieceTransfer(ctx, execution, input, copyRow, target, pieceCID) + } + if time.Since(checkpoint.AttemptedAt) >= storeAttentionAfter { + if findErr == nil { + findErr = fmt.Errorf("storage transfer remains %s", state) + } + return taskengine.Fail(findErr, "store_outcome_unknown", nil) + } + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Checking storage transfer", nil) +} + +func (h *TaskHandlers) retryStoreNotStarted(execution taskengine.Execution, err error) taskengine.Result { + if execution.RetryWillFail() { + return taskengine.Fail(err, "store_not_started", nil) + } + return retryTask(err, "store_not_started") +} + +func (h *TaskHandlers) pullHandler() taskengine.Handler { + definition := copyDefinition(model.TaskTypeStoragePull, h.retryLimit()) + return taskHandler{ + definition: definition, + execute: func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + return h.runPull(ctx, execution, true) + }, + recover: func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + return h.runPull(ctx, execution, false) + }, + } +} + +func (h *TaskHandlers) runPull(ctx context.Context, execution taskengine.Execution, mayPull bool) taskengine.Result { + input, copyRow, handled, result := h.authorizeCopyTask(ctx, execution) + if handled { + return result + } + if copyRow.Status == model.StorageCopyStatusCommitted { + return h.completeCopyTask(input, execution.ID(), "Storage copy is complete") + } + if copyRow.Status == model.StorageCopyStatusPieceReady || copyRow.Status == model.StorageCopyStatusCommitting { + return h.advanceCopyTask(input, execution.ID(), model.TaskTypeStorageCommitCoordinate, "Storage copy is ready to register") + } + checkpoint, hasCheckpoint, err := taskengine.DecodeCheckpoint[pullCheckpoint](execution) + if err != nil { + return taskengine.Fail(err, "invalid_checkpoint", nil) + } + if !hasCheckpoint && !mayPull { + return taskengine.Suspend(model.TaskResumeModeExecute, 0, "safe_to_execute", "Storage transfer is ready", nil) + } + _, target, _, _, err := h.copyContext(ctx, copyRow) + if err != nil { + return h.copyContextFailure(execution, input, copyRow, err, !hasCheckpoint) + } + if !hasCheckpoint { + sources, err := h.deps.Repositories.Contents.ListReadableCommittedCopies(ctx, copyRow.ContentID) + if err != nil { + return h.retryCopyTask(execution, input, copyRow, err, "copy_source_load_failed") + } + if len(sources) == 0 { + return taskengine.Suspend(model.TaskResumeModeExecute, storageDependencyWait, "source", "Waiting for a readable storage source", nil) + } + source := sources[0] + pieceCID, err := cid.Parse(source.PieceCID) + if err != nil { + return h.failCopyTask(execution, input, copyRow, err, "source_identity_invalid") + } + extra, err := target.PresignForCommit(ctx, []storage.PieceInput{{PieceCID: pieceCID}}) + if err != nil { + return h.retryCopyTask(execution, input, copyRow, err, "pull_presign_failed") + } + attemptID, err := newAttemptID() + if err != nil { + return h.failCopyTask(execution, input, copyRow, err, "pull_identity_failed") + } + checkpoint = pullCheckpoint{ + AttemptedAt: time.Now().UTC(), AttemptID: attemptID, PieceCID: source.PieceCID, + SourceProviderID: source.ProviderID.String(), SourceDataSetID: source.DataSetID.String(), + SourcePieceID: source.PieceID.String(), SourceRetrievalURL: source.RetrievalURL, + CommitExtraDataHex: hex.EncodeToString(extra), + } + if err := execution.WriteCheckpointWith(ctx, checkpoint, func(ctx context.Context, repos *repository.Repositories) error { + return repos.Contents.ReservePullRequest(ctx, repository.ReservePullRequestInput{ + CopyID: copyRow.ID, Generation: input.Generation, TaskID: execution.ID(), + AttemptID: checkpoint.AttemptID, + SourceProviderID: source.ProviderID, SourceDataSetID: source.DataSetID, SourcePieceID: source.PieceID, + SourcePieceCID: source.PieceCID, SourceRetrievalURL: source.RetrievalURL, + CommitExtraDataHex: checkpoint.CommitExtraDataHex, + }) + }); err != nil { + return h.retryCopyTask(execution, input, copyRow, err, "pull_checkpoint_failed") + } + } + if !mayPull { + pieceCID, parseErr := cid.Parse(checkpoint.PieceCID) + if parseErr != nil { + return taskengine.Fail(parseErr, "invalid_checkpoint", nil) + } + status, statusErr := target.PieceStatus(ctx, pieceCID) + if statusErr == nil && status != nil && status.Exists { + return h.finishPieceTransferWithExtra(execution, input, copyRow, target, pieceCID, checkpoint.CommitExtraDataHex, checkpoint.AttemptID) + } + return taskengine.Suspend(model.TaskResumeModeExecute, storagePollInterval, "provider_confirmation", "Checking storage transfer", nil) + } + pieceCID, err := cid.Parse(checkpoint.PieceCID) + if err != nil { + return taskengine.Fail(err, "invalid_checkpoint", nil) + } + extra, err := hex.DecodeString(checkpoint.CommitExtraDataHex) + if err != nil { + return taskengine.Fail(err, "invalid_checkpoint", nil) + } + err = execution.WithResource(ctx, taskengine.ResourceProviderMutation, func(ctx context.Context) error { + _, pullErr := target.Pull(ctx, storage.PullRequest{ + Pieces: []cid.Cid{pieceCID}, ExtraData: extra, + From: func(cid.Cid) string { return checkpoint.SourceRetrievalURL }, + }) + return pullErr + }) + if err != nil { + switch synapse.ClassifyPullError(err) { + case synapse.PullErrorRetryable: + return taskengine.Suspend(model.TaskResumeModeExecute, storagePollInterval, "provider_confirmation", "Checking storage transfer", nil) + case synapse.PullErrorTerminal: + return h.failPullTask(execution, input, copyRow, checkpoint.AttemptID, err, "pull_failed") + default: + return h.retryPullTask(execution, input, copyRow, checkpoint.AttemptID, err, "pull_request_failed") + } + } + return h.finishPieceTransferWithExtra(execution, input, copyRow, target, pieceCID, checkpoint.CommitExtraDataHex, checkpoint.AttemptID) +} + +func (h *TaskHandlers) commitCoordinateHandler() taskengine.Handler { + definition := copyDefinition(model.TaskTypeStorageCommitCoordinate, h.retryLimit()) + run := func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + input, copyRow, handled, result := h.authorizeCopyTask(ctx, execution) + if handled { + return result + } + if copyRow.Status == model.StorageCopyStatusCommitted { + return h.completeCopyTask(input, execution.ID(), "Storage copy is complete") + } + if copyRow.Status != model.StorageCopyStatusPieceReady && copyRow.Status != model.StorageCopyStatusCommitting { + return h.failCopyTask(execution, input, copyRow, errors.New("storage copy has no transferable piece"), "piece_not_ready") + } + return h.advanceCopyTask(input, execution.ID(), model.TaskTypeStorageCommit, "Storage registration scheduled") + } + return taskHandler{definition: definition, execute: run, recover: run} +} + +func (h *TaskHandlers) commitHandler() taskengine.Handler { + definition := copyDefinition(model.TaskTypeStorageCommit, h.retryLimit()) + return taskHandler{ + definition: definition, + execute: func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + return h.runCommit(ctx, execution, true) + }, + recover: func(ctx context.Context, execution taskengine.Execution) taskengine.Result { + return h.runCommit(ctx, execution, false) + }, + } +} + +func (h *TaskHandlers) runCommit(ctx context.Context, execution taskengine.Execution, maySubmit bool) taskengine.Result { + input, copyRow, handled, result := h.authorizeCopyTask(ctx, execution) + if handled { + return result + } + if copyRow.Status == model.StorageCopyStatusCommitted { + return h.completeCopyTask(input, execution.ID(), "Storage copy is complete") + } + binding, err := h.deps.Repositories.Contents.GetDataSetBindingByID(ctx, copyRow.StorageDataSetID) + if err != nil || binding == nil { + if err == nil { + err = repository.ErrNotFound + } + return h.retryCopyTask(execution, input, copyRow, err, "dataset_load_failed") + } + upload, err := h.deps.Repositories.Contents.GetByID(ctx, copyRow.ContentID) + if err != nil || upload == nil { + if err == nil { + err = repository.ErrNotFound + } + return h.retryCopyTask(execution, input, copyRow, err, "upload_load_failed") + } + if upload.PieceCID == nil || *upload.PieceCID == "" { + return h.failCopyTask(execution, input, copyRow, errors.New("storage upload has no piece identity"), "piece_identity_missing") + } + if copyRow.CommitAttemptedAt == nil && !maySubmit { + return taskengine.Suspend(model.TaskResumeModeExecute, 0, "safe_to_execute", "Storage registration is ready", nil) + } + pieceCID, err := cid.Parse(*upload.PieceCID) + if err != nil { + return h.failCopyTask(execution, input, copyRow, err, "piece_identity_invalid") + } + target, err := h.openReadyDataSet(ctx, binding) + if err != nil { + if copyRow.CommitAttemptedAt != nil && copyRow.CommitAttemptID != nil { + advancer := storagecommit.Advancer{Store: h.deps.Repositories.Contents, StatusChecker: h.deps.CommitStatus} + advanced, advanceErr := advancer.AdvanceUnavailable(ctx, *copyRow, *binding) + if advanceErr != nil { + return h.retryCopyTask(execution, input, copyRow, advanceErr, "commit_recovery_failed") + } + if advanced.State == storagecommit.AdvanceNeedsAttention { + if advanced.Continue && advanced.AttentionCode.Valid() { + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Checking storage registration", nil) + } + return taskengine.Fail(errors.New("storage registration requires attention"), commitAttentionFailureReason(advanced.AttentionCode), nil) + } + } + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Checking storage registration", nil) + } + advancer := storagecommit.Advancer{Store: h.deps.Repositories.Contents, StatusChecker: h.deps.CommitStatus} + advance := func(ctx context.Context) (storagecommit.AdvanceResult, error) { + return advancer.Advance(ctx, storagecommit.AdvanceInput{ + Copy: *copyRow, Binding: *binding, Target: target, + Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, RequireEligibleCopy: true, + }) + } + var advanced storagecommit.AdvanceResult + if copyRow.CommitAttemptedAt == nil { + err = execution.WithResource(ctx, taskengine.ResourceProviderMutation, func(ctx context.Context) error { + var advanceErr error + advanced, advanceErr = advance(ctx) + return advanceErr + }) + } else { + advanced, err = advance(ctx) + } + if err != nil { + if advanced.State == storagecommit.AdvancePending || advanced.State == storagecommit.AdvanceSubmitted { + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Checking storage registration", nil) + } + return h.retryCopyTask(execution, input, copyRow, err, "commit_advance_failed") + } + switch advanced.State { + case storagecommit.AdvanceWaitingCapacity: + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "capacity", "Waiting to register storage", nil) + case storagecommit.AdvanceSubmitted, storagecommit.AdvancePending: + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Waiting for storage registration", nil) + case storagecommit.AdvanceRejected: + return h.retryResolvedCopyTask(execution, input, copyRow, synapse.ErrProviderTransactionRejected, "commit_rejected") + case storagecommit.AdvanceNeedsAttention: + if advanced.Continue && advanced.AttentionCode.Valid() { + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Checking storage registration", nil) + } + return taskengine.Fail(errors.New("storage registration requires attention"), commitAttentionFailureReason(advanced.AttentionCode), nil) + case storagecommit.AdvanceReleased: + if !advanced.ReleaseReason.Valid() { + return taskengine.Fail(fmt.Errorf("storage registration was released with unknown reason %q", advanced.ReleaseReason), "commit_release_reason_unknown", nil) + } + return h.retryResolvedCopyTask(execution, input, copyRow, errors.New("storage registration was released"), string(advanced.ReleaseReason)) + case storagecommit.AdvanceConfirmed: + if advanced.Confirmation == nil || len(advanced.Confirmation.PieceIDs) != 1 { + return taskengine.Fail(errors.New("storage confirmation has no unique piece identity"), "commit_confirmation_invalid", nil) + } + pieceID := idtypes.OnChainIDFromSDK(advanced.Confirmation.PieceIDs[0]) + retrievalURL := target.PieceURL(pieceCID) + extraHex := taskDerefString(copyRow.CommitExtraDataHex) + return taskengine.Complete("Storage copy registered", func(ctx context.Context, repos *repository.Repositories) error { + if err := repos.Contents.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ + StorageCopyID: copyRow.ID, RequireEligibleCopy: true, ContentID: copyRow.ContentID, + CopyIndex: copyRow.CopyIndex, PieceCID: pieceCID.String(), PieceID: &pieceID, + RetrievalURL: retrievalURL, CommitExtraDataHex: extraHex, + CommitTransactionID: advanced.Confirmation.TransactionID, CommitAttemptID: advanced.AttemptID, + CommitConfirmedTransactionID: advanced.Confirmation.ConfirmedTransactionID, + }); err != nil { + return err + } + if err := repos.Contents.CompleteCopyTask(ctx, input.CopyID, input.Generation, execution.ID()); err != nil { + return err + } + // The content is readable once this copy commits, independently of + // which versions currently point at it. + if _, err := repos.Contents.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ + ContentID: copyRow.ContentID, BucketID: copyRow.BucketID, + }); err != nil { + return err + } + _, refs, err := repos.Contents.FinalizeUploadIfTargetCopiesMet(ctx, repository.NewFinalizeUploadInput(copyRow.ContentID)) + if err != nil { + return err + } + return h.enqueueAfterUploadEvictions(ctx, repos, refs) + }) + default: + return taskengine.Fail(fmt.Errorf("unknown storage commit result %q", advanced.State), "commit_result_invalid", nil) + } +} + +func copyDefinition(taskType model.TaskType, retryLimit *int) taskengine.Definition { + return taskengine.Definition{ + Type: taskType, InputVersion: 1, + Codec: taskengine.StrictJSONCodec(func(input *storagepipeline.CopyGenerationInput) error { + return storagepipeline.ValidateCopyGenerationInput(*input) + }), + RetryLimit: retryLimit, AllowRetry: false, + } +} + +func (h *TaskHandlers) authorizeCopyTask( + ctx context.Context, + execution taskengine.Execution, +) (storagepipeline.CopyGenerationInput, *model.StorageCopy, bool, taskengine.Result) { + input, err := taskengine.DecodeInput[storagepipeline.CopyGenerationInput](execution) + if err != nil { + return input, nil, true, decodeFailure(string(execution.Type()), err) + } + copyRow, err := h.deps.Repositories.Contents.AuthorizeCopyTask(ctx, input.CopyID, input.Generation, execution.ID(), execution.ClaimGeneration()) + if errors.Is(err, repository.ErrConflict) || errors.Is(err, repository.ErrNotFound) { + return input, nil, true, taskengine.Cancel("Storage work was superseded", nil) + } + if err != nil { + if execution.RetryWillFail() { + message := err.Error() + return input, nil, true, taskengine.Fail(err, "copy_authorization_failed", func(ctx context.Context, repos *repository.Repositories) error { + copyRow, authorizeErr := repos.Contents.AuthorizeCopyTask(ctx, input.CopyID, input.Generation, execution.ID(), execution.ClaimGeneration()) + if errors.Is(authorizeErr, repository.ErrConflict) || errors.Is(authorizeErr, repository.ErrNotFound) { + return nil + } + if authorizeErr != nil { + return authorizeErr + } + if copyRow.CommitAttemptedAt != nil { + return nil + } + return h.settleCopyFailure(ctx, repos, execution, input, copyRow, message, "") + }) + } + return input, nil, true, retryTask(err, "copy_authorization_failed") + } + return input, copyRow, false, taskengine.Result{} +} + +func (h *TaskHandlers) advanceCopyTask( + input storagepipeline.CopyGenerationInput, + taskID int64, + nextType model.TaskType, + message string, +) taskengine.Result { + return taskengine.Complete(message, func(ctx context.Context, repos *repository.Repositories) error { + return h.enqueueSuccessorCopyTask(ctx, repos, input, taskID, nextType) + }) +} + +func (h *TaskHandlers) completeCopyTask(input storagepipeline.CopyGenerationInput, taskID int64, message string) taskengine.Result { + return taskengine.Complete(message, func(ctx context.Context, repos *repository.Repositories) error { + return repos.Contents.CompleteCopyTask(ctx, input.CopyID, input.Generation, taskID) + }) +} + +func (h *TaskHandlers) enqueueInitialCopyTask(ctx context.Context, repos *repository.Repositories, copyID int64, taskType model.TaskType) error { + if h.taskService == nil { + return errors.New("task service is unavailable") + } + generation, err := repos.Contents.NextCopyWorkGeneration(ctx, copyID) + if err != nil { + return err + } + input := storagepipeline.CopyGenerationInput{CopyID: copyID, Generation: generation} + taskRow, _, err := h.taskService.EnqueueInTransaction(ctx, repos, taskengine.EnqueueRequest{ + Type: taskType, IdempotencyKey: copyTaskKey(taskType, copyID, generation), Input: input, + SubjectType: "storage_copy", SubjectKey: fmt.Sprintf("%d", copyID), + }) + if err != nil { + return err + } + return repos.Contents.BindCopyTask(ctx, copyID, generation, taskRow.ID) +} + +func (h *TaskHandlers) enqueueSuccessorCopyTask( + ctx context.Context, + repos *repository.Repositories, + current storagepipeline.CopyGenerationInput, + currentTaskID int64, + nextType model.TaskType, +) error { + if h.taskService == nil { + return errors.New("task service is unavailable") + } + next := storagepipeline.CopyGenerationInput{CopyID: current.CopyID, Generation: current.Generation + 1} + taskRow, _, err := h.taskService.EnqueueInTransaction(ctx, repos, taskengine.EnqueueRequest{ + Type: nextType, IdempotencyKey: copyTaskKey(nextType, next.CopyID, next.Generation), Input: next, + SubjectType: "storage_copy", SubjectKey: fmt.Sprintf("%d", next.CopyID), + }) + if err != nil { + return err + } + return repos.Contents.ReplaceCopyTask(ctx, current.CopyID, current.Generation, currentTaskID, next.Generation, taskRow.ID) +} + +func (h *TaskHandlers) enqueueDataSetEnsure(ctx context.Context, repos *repository.Repositories, binding *model.StorageDataSet) error { + if h.taskService == nil || binding == nil { + return errors.New("data set task dependencies are unavailable") + } + taskRow, _, err := h.taskService.EnqueueInTransaction(ctx, repos, taskengine.EnqueueRequest{ + Type: model.TaskTypeStorageDataSetEnsure, IdempotencyKey: storagepipeline.DataSetEnsureKey(binding.ID), + Input: storagepipeline.DataSetInput{DataSetID: binding.ID}, + SubjectType: "storage_data_set", SubjectKey: fmt.Sprintf("%d", binding.ID), + }) + if err != nil { + return err + } + return repos.Contents.BindDataSetEnsureTask(ctx, binding.ID, taskRow.ID) +} + +func copyTaskKey(taskType model.TaskType, copyID, generation int64) string { + switch taskType { + case model.TaskTypeStorageTransferPlan: + return storagepipeline.TransferPlanKey(copyID, generation) + case model.TaskTypeStorageStore: + return storagepipeline.StoreKey(copyID, generation) + case model.TaskTypeStoragePull: + return storagepipeline.PullKey(copyID, generation) + case model.TaskTypeStorageCommitCoordinate: + return storagepipeline.CommitCoordinateKey(copyID, generation) + case model.TaskTypeStorageCommit: + return storagepipeline.CommitKey(copyID, generation) + default: + panic(fmt.Sprintf("unsupported copy task type %q", taskType)) + } +} + +// copyContext resolves what a copy task acts on. The unit of work is the +// content: its bytes, its cache file and its size are what the provider +// receives, so no object version needs to exist for the transfer to be valid. +func (h *TaskHandlers) copyContext( + ctx context.Context, + copyRow *model.StorageCopy, +) (*model.StorageDataSet, synapse.DataSetTarget, *model.StorageContent, *model.Bucket, error) { + if copyRow == nil { + return nil, nil, nil, nil, errors.New("storage copy has no data set") + } + binding, err := h.deps.Repositories.Contents.GetDataSetBindingByID(ctx, copyRow.StorageDataSetID) + if err != nil || binding == nil { + if err == nil { + err = repository.ErrNotFound + } + return binding, nil, nil, nil, err + } + target, err := h.openReadyDataSet(ctx, binding) + if err != nil { + return binding, nil, nil, nil, err + } + content, err := h.deps.Repositories.Contents.GetByID(ctx, copyRow.ContentID) + if err != nil || content == nil { + if err == nil { + err = repository.ErrNotFound + } + return binding, target, content, nil, err + } + bucket, err := h.deps.Repositories.Buckets.GetByID(ctx, content.BucketID) + if err != nil || bucket == nil { + if err == nil { + err = repository.ErrNotFound + } + return binding, target, content, bucket, err + } + return binding, target, content, bucket, nil +} + +func (h *TaskHandlers) copyContextFailure( + execution taskengine.Execution, + input storagepipeline.CopyGenerationInput, + copyRow *model.StorageCopy, + err error, + settleSafe bool, +) taskengine.Result { + if errors.Is(err, repository.ErrNotFound) { + if settleSafe { + return h.failCopyTask(execution, input, copyRow, err, "copy_owner_missing") + } + return taskengine.Fail(err, "copy_owner_missing", nil) + } + if synapse.IsProviderUnavailable(err) || errors.Is(err, storage.ErrDataSetUnavailable) { + return taskengine.Suspend(model.TaskResumeModeRecover, storageDependencyWait, "provider", "Waiting for storage provider", nil) + } + if settleSafe { + return h.retryCopyTask(execution, input, copyRow, err, "copy_context_failed") + } + return retryTask(err, "copy_context_failed") +} + +func (h *TaskHandlers) finishPieceTransfer( + ctx context.Context, + execution taskengine.Execution, + input storagepipeline.CopyGenerationInput, + copyRow *model.StorageCopy, + target synapse.DataSetTarget, + pieceCID cid.Cid, +) taskengine.Result { + extra, err := target.PresignForCommit(ctx, []storage.PieceInput{{PieceCID: pieceCID}}) + if err != nil { + return h.retryCopyTask(execution, input, copyRow, err, "commit_presign_failed") + } + return h.finishPieceTransferWithExtra(execution, input, copyRow, target, pieceCID, hex.EncodeToString(extra), "") +} + +// finishPieceTransferWithExtra settles a completed transfer. pullAttemptID is +// empty for a store, which sent no request to a source provider. +func (h *TaskHandlers) finishPieceTransferWithExtra( + execution taskengine.Execution, + input storagepipeline.CopyGenerationInput, + copyRow *model.StorageCopy, + target synapse.DataSetTarget, + pieceCID cid.Cid, + extraHex string, + pullAttemptID string, +) taskengine.Result { + pieceCIDString := pieceCID.String() + retrievalURL := target.PieceURL(pieceCID) + canonicalExtraHex := strings.ToLower(extraHex) + return taskengine.Complete("Storage transfer completed", func(ctx context.Context, repos *repository.Repositories) error { + if err := repos.Contents.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ + StorageCopyID: copyRow.ID, RequireEligibleCopy: true, ContentID: copyRow.ContentID, + CopyIndex: copyRow.CopyIndex, PieceCID: pieceCIDString, RetrievalURL: retrievalURL, + CommitExtraDataHex: canonicalExtraHex, PullAttemptID: pullAttemptID, + }); err != nil { + return err + } + return h.enqueueSuccessorCopyTask(ctx, repos, input, execution.ID(), model.TaskTypeStorageCommitCoordinate) + }) +} + +func (h *TaskHandlers) retryCopyTask( + execution taskengine.Execution, + input storagepipeline.CopyGenerationInput, + copyRow *model.StorageCopy, + err error, + reason string, +) taskengine.Result { + if !execution.RetryWillFail() { + return retryTask(err, reason) + } + return h.failCopyTask(execution, input, copyRow, err, reason) +} + +func (h *TaskHandlers) retryPullTask( + execution taskengine.Execution, + input storagepipeline.CopyGenerationInput, + copyRow *model.StorageCopy, + pullAttemptID string, + err error, + reason string, +) taskengine.Result { + if !execution.RetryWillFail() { + return retryTask(err, reason) + } + return h.failPullTask(execution, input, copyRow, pullAttemptID, err, reason) +} + +func (h *TaskHandlers) failPullTask( + execution taskengine.Execution, + input storagepipeline.CopyGenerationInput, + copyRow *model.StorageCopy, + pullAttemptID string, + err error, + reason string, +) taskengine.Result { + if copyRow == nil || copyRow.CommitAttemptedAt != nil { + return taskengine.Fail(err, reason, nil) + } + message := err.Error() + return taskengine.Fail(err, reason, func(ctx context.Context, repos *repository.Repositories) error { + return h.settleCopyFailure(ctx, repos, execution, input, copyRow, message, pullAttemptID) + }) +} + +func (h *TaskHandlers) retryResolvedCopyTask( + execution taskengine.Execution, + input storagepipeline.CopyGenerationInput, + copyRow *model.StorageCopy, + err error, + reason string, +) taskengine.Result { + if !execution.RetryWillFail() { + return retryTask(err, reason) + } + resolvedCopy := *copyRow + resolvedCopy.CommitAttemptedAt = nil + return h.failCopyTask(execution, input, &resolvedCopy, err, reason) +} + +func (h *TaskHandlers) failCopyTask( + execution taskengine.Execution, + input storagepipeline.CopyGenerationInput, + copyRow *model.StorageCopy, + err error, + reason string, +) taskengine.Result { + if copyRow == nil || copyRow.CommitAttemptedAt != nil { + return taskengine.Fail(err, reason, nil) + } + message := err.Error() + return taskengine.Fail(err, reason, func(ctx context.Context, repos *repository.Repositories) error { + return h.settleCopyFailure(ctx, repos, execution, input, copyRow, message, "") + }) +} + +func (h *TaskHandlers) settleCopyFailure( + ctx context.Context, + repos *repository.Repositories, + execution taskengine.Execution, + input storagepipeline.CopyGenerationInput, + copyRow *model.StorageCopy, + message string, + pullAttemptID string, +) error { + if err := repos.Contents.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{ + StorageCopyID: copyRow.ID, + ContentID: copyRow.ContentID, + CopyIndex: copyRow.CopyIndex, + LastError: message, + PullAttemptID: pullAttemptID, + }); err != nil { + return err + } + if err := repos.Contents.CompleteCopyTask(ctx, input.CopyID, input.Generation, execution.ID()); err != nil { + return err + } + upload, err := repos.Contents.GetByID(ctx, copyRow.ContentID) + if err != nil || upload == nil { + return errors.Join(err, repository.ErrNotFound) + } + // The content row is shared by every version of these bytes, so recording + // the failure once covers all of them; there are no followers to fan out to. + return repos.Contents.RecordContentFailure(ctx, upload.ID, message) +} + +func commitAttentionFailureReason(code storagecommit.AttentionCode) string { + if code == "" { + return "commit_attention_unknown" + } + return string(code) +} + +func (h *TaskHandlers) openBindingTarget(ctx context.Context, bucketName string, binding *model.StorageDataSet) (synapse.StorageTarget, error) { + if binding.DataSetID != nil && !binding.DataSetID.IsZero() { + return h.openReadyDataSet(ctx, binding) + } + return h.deps.Storage.OpenProviderTarget(ctx, binding.ProviderID.SDK(), storage.NewProviderContextOptions{ + DataSetMetadata: map[string]string{"bucket": bucketName}, + }) +} + +func (h *TaskHandlers) openReadyDataSet(ctx context.Context, binding *model.StorageDataSet) (synapse.DataSetTarget, error) { + if binding == nil || binding.DataSetID == nil || binding.DataSetID.IsZero() { + return nil, errors.New("storage data set is not ready") + } + providerID := binding.ProviderID.SDK() + target, err := h.deps.Storage.OpenDataSetTarget(ctx, binding.DataSetID.SDK(), storage.NewDataSetContextOptions{ProviderID: &providerID}) + if err != nil { + return nil, err + } + if target == nil { + return nil, errors.New("storage client returned no data set context") + } + if got := idtypes.OnChainIDFromSDK(target.ProviderID()); !got.Equal(binding.ProviderID) { + return nil, fmt.Errorf("data set resolved provider %s, want %s", got.String(), binding.ProviderID.String()) + } + ref, ok := target.DataSetRef() + if !ok { + return nil, errors.New("storage client did not resolve the requested data set") + } + dataSetID, clientDataSetID, err := dataSetRefIDs(binding, ref) + if err != nil { + return nil, err + } + if !dataSetID.Equal(*binding.DataSetID) { + return nil, fmt.Errorf("data set resolved %s, want %s", dataSetID.String(), binding.DataSetID.String()) + } + if binding.ClientDataSetID == nil { + if err := h.deps.Repositories.Contents.BackfillClientDataSetID(ctx, repository.BackfillClientDataSetIDInput{ + ID: binding.ID, DataSetID: dataSetID, ClientDataSetID: clientDataSetID, + }); err != nil { + return nil, err + } + } else if !binding.ClientDataSetID.Equal(clientDataSetID) { + return nil, errors.New("storage client data set identity changed") + } + return target, nil +} + +func dataSetRefIDs(binding *model.StorageDataSet, ref storage.DataSetRef) (idtypes.OnChainID, idtypes.OnChainID, error) { + if binding == nil { + return idtypes.OnChainID{}, idtypes.OnChainID{}, errors.New("storage data set binding is missing") + } + providerID := idtypes.OnChainIDFromSDK(ref.ProviderID()) + if !providerID.Equal(binding.ProviderID) { + return idtypes.OnChainID{}, idtypes.OnChainID{}, fmt.Errorf("data set resolved provider %s, want %s", providerID.String(), binding.ProviderID.String()) + } + dataSetID := idtypes.OnChainIDFromSDK(ref.DataSetID()) + if dataSetID.IsZero() { + return idtypes.OnChainID{}, idtypes.OnChainID{}, errors.New("data set resolved a zero identity") + } + return dataSetID, idtypes.OnChainIDFromSDK(ref.ClientDataSetID()), nil +} + +func dataSetResultIDs(binding *model.StorageDataSet, result *storage.CreateDataSetResult) (idtypes.OnChainID, idtypes.OnChainID, error) { + if result == nil { + return idtypes.OnChainID{}, idtypes.OnChainID{}, errors.New("storage provider returned no data set result") + } + return dataSetRefIDs(binding, result.DataSet) +} + +func onChainIDPtr(value *sdktypes.BigInt) *idtypes.OnChainID { + if value == nil { + return nil + } + converted := idtypes.OnChainIDFromSDK(*value) + return &converted +} + +func (h *TaskHandlers) enqueueAfterUploadEvictions(ctx context.Context, repos *repository.Repositories, refs []repository.ObjectVersionRef) error { + if h.deps.EvictionPolicy != cache.EvictionPolicyAfterUpload || h.taskService == nil { + return nil + } + // Cache residency is content-addressed, so several refs can name the same + // file. One eviction per content, not per version. + seen := make(map[int64]struct{}, len(refs)) + for _, ref := range refs { + if ref.ContentID == nil { + continue + } + contentID := *ref.ContentID + if _, exists := seen[contentID]; exists { + continue + } + seen[contentID] = struct{}{} + reservation, err := repos.CacheEvictions.PrepareEviction(ctx, contentID) + if err != nil { + return err + } + if reservation.ActiveTaskID != nil { + continue + } + generation := reservation.Generation + taskRow, _, err := h.taskService.EnqueueInTransaction(ctx, repos, taskengine.EnqueueRequest{ + Type: model.TaskTypeCacheEvict, IdempotencyKey: cacheeviction.EvictTaskKey(contentID, generation), + Input: cacheeviction.EvictInput{ContentID: contentID, Generation: generation}, + SubjectType: "storage_content", SubjectKey: strconv.FormatInt(contentID, 10), + }) + if err != nil { + return err + } + if err := repos.CacheEvictions.BindEvictionTask(ctx, contentID, generation, taskRow.ID); err != nil { + return err + } + } + return nil +} + +func newAttemptID() (string, error) { + var value [16]byte + if _, err := rand.Read(value[:]); err != nil { + return "", fmt.Errorf("creating request identity: %w", err) + } + return hex.EncodeToString(value[:]), nil +} + +func derefInt64(value *int64) int64 { + if value == nil { + return 0 + } + return *value +} + +func taskDerefString(value *string) string { + if value == nil { + return "" + } + return *value +} diff --git a/internal/worker/task_handlers.go b/internal/worker/task_handlers.go new file mode 100644 index 0000000..1c7af54 --- /dev/null +++ b/internal/worker/task_handlers.go @@ -0,0 +1,185 @@ +package worker + +import ( + "context" + "errors" + "fmt" + "log/slog" + "sync" + "time" + + "github.com/strahe/synaps3/internal/cache" + "github.com/strahe/synaps3/internal/cacheaccess" + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/observability" + "github.com/strahe/synaps3/internal/storagecommit" + "github.com/strahe/synaps3/internal/synapse" + taskengine "github.com/strahe/synaps3/internal/task" +) + +// TaskHandlerDependencies are the domain services used by registered task +// handlers. The task engine remains the only owner of polling and leases. +type TaskHandlerDependencies struct { + Repositories *repository.Repositories + Events EventPublisher + Cache cache.Cache + CacheGate *cacheaccess.Gate + CacheTracker *cacheaccess.Tracker + Storage synapse.StorageClient + Wallet synapse.WalletOperator + Receipts WalletReceiptChecker + WalletBroadcastTimeout time.Duration + WalletReceiptTimeout time.Duration + Terminator synapse.ServiceTerminator + Epochs synapse.ChainEpochReader + Observability *observability.Service + CommitStatus storagecommit.AddPiecesStatusChecker + ParkedPieces synapse.ParkedPieceChecker + EvictionPolicy cache.EvictionPolicy + MaxCacheBytes int64 + LRUHighPercent int + LRULowPercent int + DefaultCopies int + MaxRetries int + Logger *slog.Logger +} + +type EventPublisher interface { + Publish(topic string, payload map[string]any) +} + +type TaskHandlers struct { + deps TaskHandlerDependencies + taskService *taskengine.Service + + lruCapacityMu sync.Mutex + lruProjectedBytes int64 + lruInFlightDeletes int +} + +func NewTaskHandlers(deps TaskHandlerDependencies) (*TaskHandlers, error) { + if deps.Repositories == nil || deps.Repositories.Tasks == nil { + return nil, errors.New("task handlers require repositories") + } + if deps.MaxRetries < 0 { + return nil, errors.New("task retry limit cannot be negative") + } + if deps.WalletBroadcastTimeout < 0 || deps.WalletReceiptTimeout < 0 { + return nil, errors.New("wallet timeouts cannot be negative") + } + if deps.WalletBroadcastTimeout == 0 { + deps.WalletBroadcastTimeout = 2 * time.Minute + } + if deps.WalletReceiptTimeout == 0 { + deps.WalletReceiptTimeout = 15 * time.Second + } + if deps.EvictionPolicy == cache.EvictionPolicyLRU && + (deps.MaxCacheBytes <= 0 || deps.LRULowPercent < 0 || deps.LRULowPercent > 100 || + deps.LRUHighPercent < 0 || deps.LRUHighPercent > 100 || deps.LRUHighPercent <= deps.LRULowPercent) { + return nil, errors.New("LRU cache capacity requires valid size and watermarks") + } + if deps.Logger == nil { + deps.Logger = slog.Default() + } + return &TaskHandlers{deps: deps}, nil +} + +func (h *TaskHandlers) SetTaskService(service *taskengine.Service) { + h.taskService = service +} + +func (h *TaskHandlers) RegisterCore(registry *taskengine.Registry) error { + if registry == nil { + return errors.New("task registry is required") + } + for _, handler := range []taskengine.Handler{ + h.cacheCapacityHandler(), + h.cacheEvictHandler(), + h.cacheDurabilityHandler(), + h.storageCleanupHandler(), + h.walletHandler(), + h.observabilityHandler(), + h.gcHandler(), + } { + if err := registry.Register(handler); err != nil { + return err + } + } + return nil +} + +// RegisterStorage installs the workflow-neutral storage pipeline. Business +// coordinators may create copy work, but only these handlers mutate a copy's +// provider-side storage state. +func (h *TaskHandlers) RegisterStorage(registry *taskengine.Registry) error { + if registry == nil { + return errors.New("task registry is required") + } + for _, handler := range []taskengine.Handler{ + h.bucketProvisionHandler(), + h.uploadPlanHandler(), + h.dataSetEnsureHandler(), + h.transferPlanHandler(), + h.storeHandler(), + h.pullHandler(), + h.commitCoordinateHandler(), + h.commitHandler(), + } { + if err := registry.Register(handler); err != nil { + return err + } + } + return nil +} + +// RegisterReplacement installs the business coordinator and the destructive +// retirement task. Copy movement remains owned by RegisterStorage handlers. +func (h *TaskHandlers) RegisterReplacement(registry *taskengine.Registry) error { + if registry == nil { + return errors.New("task registry is required") + } + for _, handler := range []taskengine.Handler{ + h.replacementCoordinateHandler(), + h.dataSetRetireHandler(), + } { + if err := registry.Register(handler); err != nil { + return err + } + } + return nil +} + +type taskHandler struct { + definition taskengine.Definition + execute func(context.Context, taskengine.Execution) taskengine.Result + recover func(context.Context, taskengine.Execution) taskengine.Result +} + +func (h taskHandler) Definition() taskengine.Definition { return h.definition } + +func (h taskHandler) Execute(ctx context.Context, execution taskengine.Execution) taskengine.Result { + if h.execute == nil { + return taskengine.Fail(errors.New("execute handler is unavailable"), "handler_unavailable", nil) + } + return h.execute(ctx, execution) +} + +func (h taskHandler) Recover(ctx context.Context, execution taskengine.Execution) taskengine.Result { + if h.recover == nil { + return taskengine.Fail(errors.New("recover handler is unavailable"), "handler_unavailable", nil) + } + return h.recover(ctx, execution) +} + +func (h *TaskHandlers) retryLimit() *int { + value := h.deps.MaxRetries + return &value +} + +func retryTask(err error, reason string) taskengine.Result { + return taskengine.RetryBackoff(err, reason, nil) +} + +func decodeFailure(taskType string, err error) taskengine.Result { + return taskengine.Fail(fmt.Errorf("decoding %s input: %w", taskType, err), "invalid_input", nil) +} diff --git a/internal/worker/task_handlers_test.go b/internal/worker/task_handlers_test.go new file mode 100644 index 0000000..37af314 --- /dev/null +++ b/internal/worker/task_handlers_test.go @@ -0,0 +1,3248 @@ +package worker_test + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "math/big" + "net/http" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + ethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" + "github.com/strahe/synaps3/internal/bucketlifecycle" + "github.com/strahe/synaps3/internal/cache" + "github.com/strahe/synaps3/internal/cacheaccess" + "github.com/strahe/synaps3/internal/cacheeviction" + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagecleanup" + "github.com/strahe/synaps3/internal/storagecommit" + "github.com/strahe/synaps3/internal/storagepipeline" + "github.com/strahe/synaps3/internal/storagepull" + "github.com/strahe/synaps3/internal/storagereplacement" + "github.com/strahe/synaps3/internal/synapse" + "github.com/strahe/synaps3/internal/systemtask" + taskengine "github.com/strahe/synaps3/internal/task" + "github.com/strahe/synaps3/internal/testutil" + idtypes "github.com/strahe/synaps3/internal/types" + "github.com/strahe/synaps3/internal/walletoperation" + "github.com/strahe/synaps3/internal/worker" + "github.com/strahe/synapse-go/pdp" + "github.com/strahe/synapse-go/piece" + "github.com/strahe/synapse-go/storage" + sdktypes "github.com/strahe/synapse-go/types" + "github.com/uptrace/bun" +) + +type handlerTestRuntime struct { + db *bun.DB + repos *repository.Repositories + cache cache.Cache + gate *cacheaccess.Gate + tracker *cacheaccess.Tracker + storage *testutil.MockStorageClient + handlers *worker.TaskHandlers + registry *taskengine.Registry + service *taskengine.Service + engine *taskengine.Engine +} + +type handlerRuntimeOptions struct { + cache cache.Cache + events worker.EventPublisher + storage *testutil.MockStorageClient + wallet synapse.WalletOperator + receipts worker.WalletReceiptChecker + walletBroadcastTimeout time.Duration + walletReceiptTimeout time.Duration + terminator synapse.ServiceTerminator + epochs synapse.ChainEpochReader + parkedPieces synapse.ParkedPieceChecker + policy cache.EvictionPolicy + maxBytes int64 + highPercent int + lowPercent int + concurrency int + maxRetries *int + register func(*worker.TaskHandlers, *taskengine.Registry) error +} + +func newHandlerTestRuntime(t *testing.T, options handlerRuntimeOptions) handlerTestRuntime { + t.Helper() + db := testutil.NewTestFileDB(t) + repos := repository.NewRepositories(db) + cacheStore := options.cache + if cacheStore == nil { + cacheStore = &testutil.MockCache{} + } + storageClient := options.storage + if storageClient == nil { + storageClient = &testutil.MockStorageClient{} + } + gate := cacheaccess.NewGate() + tracker := cacheaccess.NewTracker(cacheaccess.DefaultPersistenceInterval, repos.Objects) + maxRetries := 5 + if options.maxRetries != nil { + maxRetries = *options.maxRetries + } + handlers, err := worker.NewTaskHandlers(worker.TaskHandlerDependencies{ + Repositories: repos, Events: options.events, Cache: cacheStore, CacheGate: gate, CacheTracker: tracker, + Storage: storageClient, Wallet: options.wallet, Receipts: options.receipts, + WalletBroadcastTimeout: options.walletBroadcastTimeout, + WalletReceiptTimeout: options.walletReceiptTimeout, + Terminator: options.terminator, Epochs: options.epochs, + ParkedPieces: options.parkedPieces, + EvictionPolicy: options.policy, MaxCacheBytes: options.maxBytes, + LRUHighPercent: options.highPercent, LRULowPercent: options.lowPercent, + DefaultCopies: 2, MaxRetries: maxRetries, Logger: slog.Default(), + }) + if err != nil { + t.Fatalf("new task handlers: %v", err) + } + registry := taskengine.NewRegistry() + register := options.register + if register == nil { + register = func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterCore(registry) + } + } + if err := register(handlers, registry); err != nil { + t.Fatalf("register handlers: %v", err) + } + service, err := taskengine.NewService(registry, repos, time.Hour) + if err != nil { + t.Fatalf("new task service: %v", err) + } + handlers.SetTaskService(service) + concurrency := max(options.concurrency, 1) + engine, err := taskengine.NewEngine(taskengine.EngineConfig{ + Concurrency: concurrency, PollInterval: 5 * time.Millisecond, LeaseDuration: 300 * time.Millisecond, + Retention: time.Hour, ProviderMutationConcurrency: 4, DestructiveMutationConcurrency: 2, + }, repos, registry, slog.Default()) + if err != nil { + t.Fatalf("new task engine: %v", err) + } + return handlerTestRuntime{ + db: db, repos: repos, cache: cacheStore, gate: gate, tracker: tracker, + storage: storageClient, handlers: handlers, registry: registry, service: service, engine: engine, + } +} + +type testWalletOperator struct { + fund func(context.Context, *big.Int) (string, error) +} + +func (o testWalletOperator) FundUSDFC(ctx context.Context, amount *big.Int) (string, error) { + if o.fund == nil { + return "", errors.New("unexpected fund request") + } + return o.fund(ctx, amount) +} + +func (testWalletOperator) WithdrawUSDFC(context.Context, *big.Int) (string, error) { + return "", errors.New("unexpected withdraw request") +} + +func (testWalletOperator) ApproveFWSS(context.Context) (string, error) { + return "", errors.New("unexpected approval request") +} + +type testReceiptChecker struct { + check func(context.Context, common.Hash) (*ethtypes.Receipt, error) +} + +type testServiceTerminator struct { + calls atomic.Int64 + result *synapse.TerminationResult + err error +} + +func (t *testServiceTerminator) TerminateService(context.Context, sdktypes.BigInt) (*synapse.TerminationResult, error) { + t.calls.Add(1) + return t.result, t.err +} + +type testEpochReader struct { + epoch int64 + err error +} + +type parkedPieceCheckerFunc func(context.Context, string, cid.Cid) (synapse.ParkedPieceState, error) + +func (f parkedPieceCheckerFunc) FindParkedPiece(ctx context.Context, serviceURL string, pieceCID cid.Cid) (synapse.ParkedPieceState, error) { + return f(ctx, serviceURL, pieceCID) +} + +func (r testEpochReader) CurrentEpoch(context.Context) (int64, error) { + return r.epoch, r.err +} + +func (c testReceiptChecker) TransactionReceipt(ctx context.Context, hash common.Hash) (*ethtypes.Receipt, error) { + if c.check == nil { + return nil, errors.New("unexpected receipt request") + } + return c.check(ctx, hash) +} + +func runHandlerEngine(t *testing.T, runtime handlerTestRuntime) (context.CancelFunc, <-chan struct{}) { + t.Helper() + return runEngine(t, runtime.engine) +} + +func runEngine(t *testing.T, engine *taskengine.Engine) (context.CancelFunc, <-chan struct{}) { + t.Helper() + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan struct{}) + go func() { + _ = engine.Run(ctx) + close(done) + }() + return cancel, done +} + +func stopHandlerEngine(t *testing.T, cancel context.CancelFunc, done <-chan struct{}) { + t.Helper() + cancel() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("task engine did not stop") + } +} + +func waitForTask(t *testing.T, repos *repository.Repositories, id int64, predicate func(*model.Task) bool) *model.Task { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + row, err := repos.Tasks.GetByID(t.Context(), id) + if err != nil { + t.Fatalf("get task %d: %v", id, err) + } + if row != nil && predicate(row) { + return row + } + time.Sleep(5 * time.Millisecond) + } + row, _ := repos.Tasks.GetByID(t.Context(), id) + t.Fatalf("task %d did not reach expected state: status=%s resume=%s reason=%s error=%s", + id, taskField(row, func(r *model.Task) *string { s := string(r.Status); return &s }), + taskField(row, func(r *model.Task) *string { s := string(r.ResumeMode); return &s }), + taskField(row, func(r *model.Task) *string { return r.FailureReason }), + taskField(row, func(r *model.Task) *string { return r.LastError })) + return nil +} + +// taskField renders one optional task field for a failure message. +func taskField(row *model.Task, pick func(*model.Task) *string) string { + if row == nil { + return "" + } + if value := pick(row); value != nil { + return *value + } + return "" +} + +func testOnChainID(t *testing.T, value int64) idtypes.OnChainID { + t.Helper() + id, err := idtypes.ParseOnChainID("test id", fmt.Sprintf("%d", value)) + if err != nil { + t.Fatalf("parse on-chain id: %v", err) + } + return id +} + +func testPieceCID(t *testing.T, seed string) cid.Cid { + t.Helper() + hash, err := multihash.Sum([]byte(seed), multihash.SHA2_256, -1) + if err != nil { + t.Fatalf("create piece CID: %v", err) + } + return cid.NewCidV1(cid.Raw, hash) +} + +var storedObjectSequence atomic.Int64 + +func seedStoredCacheObject(t *testing.T, runtime handlerTestRuntime, size int64, accessedAt time.Time) *model.ObjectVersion { + t.Helper() + sequence := storedObjectSequence.Add(1) + ctx := t.Context() + bucket := &model.Bucket{Name: fmt.Sprintf("cache-task-%d", sequence), Status: model.BucketStatusActive, DefaultCopies: 2, MinimumDurableCopies: 2} + if err := runtime.repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("create bucket: %v", err) + } + // Content identity is created before the version that points at it. + upload, err := runtime.repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, ContentSize: size, + Checksum: testutil.StorageChecksum(fmt.Sprintf("checksum-%d", sequence)), RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("ensure storage content: %v", err) + } + version := &model.ObjectVersion{ + VersionID: model.NewVersionID(), BucketID: bucket.ID, Key: "object.bin", Size: size, + ETag: fmt.Sprintf("etag-%d", sequence), ContentType: "application/octet-stream", + ContentID: &upload.ID, + } + if _, err := runtime.repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + t.Fatalf("create object version: %v", err) + } + providerID := testOnChainID(t, 1000+sequence) + dataSetID := testOnChainID(t, 2000+sequence) + clientDataSetID := testOnChainID(t, 3000+sequence) + binding, err := runtime.repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: providerID, CopyIndex: 0, CreatedByContentID: upload.ID, + }) + if err != nil { + t.Fatalf("create data set binding: %v", err) + } + if err := runtime.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: binding.ID, ContentID: upload.ID, DataSetID: dataSetID, ClientDataSetID: &clientDataSetID, + }); err != nil { + t.Fatalf("mark data set ready: %v", err) + } + if err := runtime.repos.Contents.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: providerID, + }}); err != nil { + t.Fatalf("create storage copy: %v", err) + } + copies, err := runtime.repos.Contents.ListCopies(ctx, upload.ID) + if err != nil || len(copies) != 1 { + t.Fatalf("list storage copies = %#v, err=%v", copies, err) + } + pieceCID := testPieceCID(t, fmt.Sprintf("piece-%d", sequence)) + pieceID := testOnChainID(t, 4000+sequence) + testutil.CommitStorageCopy(t, runtime.db, runtime.repos, repository.MarkUploadCopyCommittedInput{ + StorageCopyID: copies[0].ID, ContentID: upload.ID, CopyIndex: 0, + PieceCID: pieceCID.String(), PieceID: &pieceID, RetrievalURL: "https://provider.example/piece/" + pieceCID.String(), + }) + if _, err := runtime.repos.Contents.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ + ContentID: upload.ID, BucketID: bucket.ID, + }); err != nil { + t.Fatalf("bind readable upload: %v", err) + } + finalized, _, err := runtime.repos.Contents.FinalizeUploadIfTargetCopiesMet(ctx, repository.NewFinalizeUploadInput(upload.ID)) + if err != nil || !finalized { + t.Fatalf("finalize storage upload = %v, err=%v", finalized, err) + } + if err := runtime.repos.Objects.RecordContentCacheAccess(ctx, upload.ID, accessedAt); err != nil { + t.Fatalf("record cache access: %v", err) + } + stored, err := runtime.repos.Objects.GetVersionByID(ctx, version.VersionID) + if err != nil || stored == nil || stored.State != model.ObjectStateStored { + t.Fatalf("stored object = %#v, err=%v", stored, err) + } + return stored +} + +func TestLRUDeletionWaitsForOpenReaderAndCancelsAfterNewAccess(t *testing.T) { + var deleteCalls atomic.Int64 + cacheStore := &testutil.MockCache{ + UsedBytesFunc: func() int64 { return 11 }, + DeleteFunc: func(context.Context, string, string) error { + deleteCalls.Add(1) + return nil + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + cache: cacheStore, policy: cache.EvictionPolicyLRU, maxBytes: 10, + highPercent: 90, lowPercent: 50, concurrency: 1, + }) + plannedAt := time.Now().UTC().Add(-time.Hour).Truncate(time.Microsecond) + version := seedStoredCacheObject(t, runtime, 11, plannedAt) + // The gate protects one cache file, and that file belongs to the content. + cacheKey := version.CacheKey() + opened, err := runtime.gate.Open(cacheKey, func() (io.ReadCloser, *cache.ObjectInfo, error) { + return io.NopCloser(strings.NewReader("cached")), &cache.ObjectInfo{Size: 11}, nil + }) + if err != nil { + t.Fatalf("open protected cache entry: %v", err) + } + planner, _, err := runtime.service.Enqueue(t.Context(), taskengine.EnqueueRequest{ + Type: model.TaskTypeCacheCapacityReconcile, IdempotencyKey: systemtask.CacheCapacityKey, + Input: systemtask.Input{}, SubjectType: "system", SubjectKey: "cache-capacity", + }) + if err != nil { + t.Fatalf("enqueue cache capacity task: %v", err) + } + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + var eviction *model.Task + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + page, listErr := runtime.repos.Tasks.List(t.Context(), repository.TaskListFilter{Type: model.TaskTypeCacheEvict, Limit: 10}) + if listErr != nil { + t.Fatalf("list cache tasks: %v", listErr) + } + if len(page.Tasks) == 1 && page.Tasks[0].Status == model.TaskStatusRunning { + eviction = &page.Tasks[0] + break + } + time.Sleep(5 * time.Millisecond) + } + if eviction == nil { + t.Fatalf("cache eviction did not block behind open reader; planner=%d", planner.ID) + } + if err := runtime.tracker.RecordAccess(t.Context(), *version.ContentID, version.CacheAccessedAt); err != nil { + t.Fatalf("record newer cache access: %v", err) + } + if err := opened.Body.Close(); err != nil { + t.Fatalf("close protected cache entry: %v", err) + } + waitForTask(t, runtime.repos, eviction.ID, func(task *model.Task) bool { return task.Status == model.TaskStatusCancelled }) + if deleteCalls.Load() != 0 { + t.Fatalf("cache delete calls = %d, want 0", deleteCalls.Load()) + } + stored, err := runtime.repos.Objects.GetVersionByID(t.Context(), version.VersionID) + if err != nil || stored == nil || !stored.InCache { + t.Fatalf("recently used version = %#v, err=%v", stored, err) + } + entry, err := runtime.repos.CacheEvictions.GetCacheEntry(t.Context(), *version.ContentID) + if err != nil || entry == nil || entry.CacheActiveTaskID != nil { + t.Fatalf("cancelled cache eviction owner = %#v, err=%v", entry, err) + } +} + +type recordedWorkerEvent struct { + topic string + payload map[string]any +} + +type recordingWorkerEvents struct { + events chan recordedWorkerEvent +} + +func (p *recordingWorkerEvents) Publish(topic string, payload map[string]any) { + select { + case p.events <- recordedWorkerEvent{topic: topic, payload: payload}: + default: + } +} + +// TestTerminalStoreFailureSettlesCopyAndContent checks that a store failure +// settles everything the transfer owns: the copy is failed and unbound, ingress +// progress stays on the copy that produced it, and the version's derived +// position follows the copies to failed without a stored column. +func TestTerminalStoreFailureSettlesCopyAndContent(t *testing.T) { + events := &recordingWorkerEvents{events: make(chan recordedWorkerEvent, 8)} + cacheStore := &testutil.MockCache{GetFunc: func(context.Context, string, string) (io.ReadCloser, *cache.ObjectInfo, error) { + return io.NopCloser(strings.NewReader(strings.Repeat("s", 128))), &cache.ObjectInfo{Size: 128}, nil + }} + target := &testutil.MockStorageTarget{ + ServiceURLValue: "https://terminal-store.example", + StoreFunc: func(_ context.Context, _ io.Reader, options *storage.StoreOptions) (*storage.StoreResult, error) { + if options == nil || options.OnProgress == nil || !options.PieceCID.Defined() { + return nil, errors.New("store progress callback or intended piece identity is missing") + } + pieceInfo, err := piece.ParseV2(options.PieceCID) + if err != nil || pieceInfo.RawSize != 128 { + return nil, fmt.Errorf("store piece identity = %#v, err=%v", pieceInfo, err) + } + options.OnProgress(6) + deadline := time.After(time.Second) + for { + select { + case event := <-events.events: + progress, ok := event.payload["progress"].(map[string]any) + if event.topic == "upload_progress_updated" && ok && progress["uploaded_bytes"] == int64(6) { + return &storage.StoreResult{}, nil + } + case <-deadline: + return nil, errors.New("store progress was not published") + } + } + }, + } + storageClient := &testutil.MockStorageClient{} + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + cache: cacheStore, events: events, storage: storageClient, policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + ctx := t.Context() + sequence := storedObjectSequence.Add(1) + bucket := &model.Bucket{Name: fmt.Sprintf("terminal-copy-%d", sequence), Status: model.BucketStatusActive, DefaultCopies: 2, MinimumDurableCopies: 2} + if err := runtime.repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("create terminal copy bucket: %v", err) + } + content, err := runtime.repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, ContentSize: 128, + Checksum: testutil.StorageChecksum(fmt.Sprintf("terminal-%d", sequence)), RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("ensure terminal content: %v", err) + } + version := &model.ObjectVersion{ + VersionID: model.NewVersionID(), BucketID: bucket.ID, Key: "terminal.bin", ContentID: &content.ID, Size: 128, + ETag: "etag", ContentType: "application/octet-stream", + } + if _, err := runtime.repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + t.Fatalf("create terminal copy version: %v", err) + } + providerID := testOnChainID(t, 13000+sequence) + dataSetID := testOnChainID(t, 14000+sequence) + clientDataSetID := testOnChainID(t, 15000+sequence) + binding, err := runtime.repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: providerID, CopyIndex: 0, CreatedByContentID: content.ID, + }) + if err != nil { + t.Fatalf("create terminal copy binding: %v", err) + } + if err := runtime.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: binding.ID, ContentID: content.ID, DataSetID: dataSetID, ClientDataSetID: &clientDataSetID, + }); err != nil { + t.Fatalf("mark terminal copy binding ready: %v", err) + } + if err := runtime.repos.Contents.CreateUploadCopiesForBindings(ctx, content.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: providerID, + }}); err != nil { + t.Fatalf("create terminal copy: %v", err) + } + copies, err := runtime.repos.Contents.ListCopies(ctx, content.ID) + if err != nil || len(copies) != 1 { + t.Fatalf("terminal copies = %#v, err=%v", copies, err) + } + target.ProviderIDValue = providerID.SDK() + targetDataSetID := dataSetID.SDK() + target.DataSetIDValue = &targetDataSetID + target.ClientDataSetIDValue = clientDataSetID.SDK() + storageClient.OpenDataSetTargetFunc = func(context.Context, sdktypes.BigInt, storage.NewDataSetContextOptions) (synapse.DataSetTarget, error) { + return target, nil + } + taskRow := bindCopyTask(t, runtime, &copies[0], model.TaskTypeStorageStore) + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + failedTask := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + if runtime.service.Retryable(failedTask) { + t.Fatal("terminal copy task is unexpectedly retryable") + } + copyRow, err := runtime.repos.Contents.GetUploadCopyByID(ctx, copies[0].ID) + if err != nil || copyRow.Status != model.StorageCopyStatusFailed || copyRow.ActiveTaskID != nil { + t.Fatalf("terminal copy = %#v, err=%v", copyRow, err) + } + // Ingress progress belongs to the transfer, so it survives on the copy. + if copyRow.IngressStoreAttempt != 1 || copyRow.IngressBytesTransferred != 6 || copyRow.ProgressUpdatedAt == nil { + t.Fatalf("terminal copy progress = attempt:%d bytes:%d at:%v", copyRow.IngressStoreAttempt, copyRow.IngressBytesTransferred, copyRow.ProgressUpdatedAt) + } + storedVersion, err := runtime.repos.Objects.GetVersionByID(ctx, version.VersionID) + if err != nil || storedVersion.State != model.ObjectStateFailed { + t.Fatalf("terminal version = %#v, err=%v", storedVersion, err) + } +} + +type testCleanupContext struct { + pieceStatus func(context.Context, cid.Cid) (*storage.PieceStatus, error) + deletePiece func(context.Context, sdktypes.BigInt) (*sdktypes.WriteResult, error) +} + +func (c testCleanupContext) DeletePieceByID(ctx context.Context, pieceID sdktypes.BigInt) (*sdktypes.WriteResult, error) { + return c.deletePiece(ctx, pieceID) +} + +func (c testCleanupContext) PieceStatus(ctx context.Context, pieceCID cid.Cid) (*storage.PieceStatus, error) { + return c.pieceStatus(ctx, pieceCID) +} + +// TestStorageCleanupContinuesPastUnsupportedCopy checks that remote cleanup +// skips a copy the provider cannot delete and still schedules the rest. +func TestStorageCleanupContinuesPastUnsupportedCopy(t *testing.T) { + var deleteCalls atomic.Int64 + cleanupContext := testCleanupContext{ + pieceStatus: func(context.Context, cid.Cid) (*storage.PieceStatus, error) { + return &storage.PieceStatus{Exists: true}, nil + }, + deletePiece: func(context.Context, sdktypes.BigInt) (*sdktypes.WriteResult, error) { + deleteCalls.Add(1) + return &sdktypes.WriteResult{Hash: common.HexToHash("0x1")}, nil + }, + } + storageClient := &testutil.MockStorageClient{ + OpenCleanupContextFunc: func(context.Context, sdktypes.BigInt, storage.NewDataSetContextOptions) (synapse.CleanupContext, error) { + return cleanupContext, nil + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{storage: storageClient}) + ctx := t.Context() + sequence := storedObjectSequence.Add(1) + bucket := &model.Bucket{Name: fmt.Sprintf("cleanup-task-%d", sequence), Status: model.BucketStatusActive, DefaultCopies: 2, MinimumDurableCopies: 2} + if err := runtime.repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("create cleanup bucket: %v", err) + } + content := &model.StorageContent{ + BucketID: bucket.ID, ContentSize: 1, + Checksum: testutil.StorageChecksum(fmt.Sprintf("cleanup-%d", sequence)), + RequestedCopies: 2, CleanupGeneration: 1, + } + if _, err := runtime.db.NewInsert().Model(content).Exec(ctx); err != nil { + t.Fatalf("create cleanup content: %v", err) + } + for copyIndex := range 2 { + providerID := testOnChainID(t, 9000+sequence*10+int64(copyIndex)) + dataSetID := testOnChainID(t, 10000+sequence*10+int64(copyIndex)) + clientDataSetID := testOnChainID(t, 11000+sequence*10+int64(copyIndex)) + binding, err := runtime.repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: providerID, CopyIndex: copyIndex, CreatedByContentID: content.ID, + }) + if err != nil { + t.Fatalf("create cleanup binding %d: %v", copyIndex, err) + } + if err := runtime.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: binding.ID, ContentID: content.ID, DataSetID: dataSetID, ClientDataSetID: &clientDataSetID, + }); err != nil { + t.Fatalf("mark cleanup binding %d ready: %v", copyIndex, err) + } + status := model.StorageCleanupCopyStatusPending + if copyIndex == 0 { + status = model.StorageCleanupCopyStatusUnsupported + } + row := &model.StorageCleanupCopy{ + ContentID: content.ID, BucketID: bucket.ID, CopyIndex: copyIndex, ProviderID: providerID, + StorageDataSetID: binding.ID, DataSetID: &dataSetID, ClientDataSetID: &clientDataSetID, + PieceID: testOnChainID(t, 12000+sequence*10+int64(copyIndex)), + PieceCID: testPieceCID(t, fmt.Sprintf("cleanup-piece-%d-%d", sequence, copyIndex)).String(), Status: status, + } + if _, err := runtime.db.NewInsert().Model(row).Exec(ctx); err != nil { + t.Fatalf("create cleanup copy %d: %v", copyIndex, err) + } + } + taskRow, _, err := runtime.service.Enqueue(ctx, taskengine.EnqueueRequest{ + Type: model.TaskTypeStorageCleanup, IdempotencyKey: storagecleanup.TaskKey(content.ID, 1), + Input: storagecleanup.Input{ContentID: content.ID, Generation: 1}, SubjectType: "storage_content", SubjectKey: fmt.Sprint(content.ID), + }) + if err != nil { + t.Fatalf("enqueue cleanup task: %v", err) + } + if err := runtime.repos.StorageCleanup.BindTask(ctx, content.ID, 1, taskRow.ID); err != nil { + t.Fatalf("bind cleanup task: %v", err) + } + limited := &limitedClaimRepository{TaskRepository: runtime.repos.Tasks, maximum: 1} + runtime.repos.Tasks = limited + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return limited.claims.Load() == 1 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover + }) + if deleteCalls.Load() != 1 { + t.Fatalf("cleanup delete calls = %d, want 1", deleteCalls.Load()) + } + copies, err := runtime.repos.StorageCleanup.AuthorizeTask(ctx, content.ID, 1, taskRow.ID) + if err != nil || len(copies) != 2 || copies[1].Status != model.StorageCleanupCopyStatusDeleteScheduled { + t.Fatalf("cleanup copies = %#v, err=%v", copies, err) + } +} + +func TestStorageCleanupAdmissionFailureDoesNotScheduleDeletion(t *testing.T) { + noRetries := 0 + var deleteCalls atomic.Int64 + cleanupContext := testCleanupContext{ + pieceStatus: func(context.Context, cid.Cid) (*storage.PieceStatus, error) { + return &storage.PieceStatus{Exists: true}, nil + }, + deletePiece: func(context.Context, sdktypes.BigInt) (*sdktypes.WriteResult, error) { + deleteCalls.Add(1) + return nil, errors.New("unexpected remote deletion") + }, + } + storageClient := &testutil.MockStorageClient{ + OpenCleanupContextFunc: func(context.Context, sdktypes.BigInt, storage.NewDataSetContextOptions) (synapse.CleanupContext, error) { + return cleanupContext, nil + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{storage: storageClient, maxRetries: &noRetries}) + bucket := &model.Bucket{Name: "cleanup-admission", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := runtime.repos.Buckets.Create(t.Context(), bucket); err != nil { + t.Fatalf("create cleanup admission bucket: %v", err) + } + content := &model.StorageContent{ + BucketID: bucket.ID, ContentSize: 128, Checksum: testutil.StorageChecksum("cleanup-admission"), + RequestedCopies: 1, CleanupGeneration: 1, + } + if _, err := runtime.db.NewInsert().Model(content).Exec(t.Context()); err != nil { + t.Fatalf("create cleanup admission content: %v", err) + } + providerID := testOnChainID(t, 29101) + dataSetID := testOnChainID(t, 29102) + clientDataSetID := testOnChainID(t, 29103) + binding, err := runtime.repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: providerID, CopyIndex: 0, CreatedByContentID: content.ID, + }) + if err != nil { + t.Fatalf("create cleanup admission binding: %v", err) + } + if err := runtime.repos.Contents.MarkDataSetReady(t.Context(), repository.MarkDataSetReadyInput{ + ID: binding.ID, ContentID: content.ID, DataSetID: dataSetID, ClientDataSetID: &clientDataSetID, + }); err != nil { + t.Fatalf("mark cleanup admission binding ready: %v", err) + } + cleanupCopy := &model.StorageCleanupCopy{ + ContentID: content.ID, BucketID: bucket.ID, CopyIndex: 0, ProviderID: providerID, + StorageDataSetID: binding.ID, DataSetID: &dataSetID, ClientDataSetID: &clientDataSetID, + PieceID: testOnChainID(t, 29104), PieceCID: testPieceCID(t, "cleanup-admission-piece").String(), + Status: model.StorageCleanupCopyStatusPending, + } + if _, err := runtime.db.NewInsert().Model(cleanupCopy).Exec(t.Context()); err != nil { + t.Fatalf("create cleanup admission copy: %v", err) + } + taskRow, _, err := runtime.service.Enqueue(t.Context(), taskengine.EnqueueRequest{ + Type: model.TaskTypeStorageCleanup, IdempotencyKey: storagecleanup.TaskKey(content.ID, 1), + Input: storagecleanup.Input{ContentID: content.ID, Generation: 1}, SubjectType: "storage_content", SubjectKey: fmt.Sprint(content.ID), + }) + if err != nil { + t.Fatalf("enqueue cleanup admission task: %v", err) + } + if err := runtime.repos.StorageCleanup.BindTask(t.Context(), content.ID, 1, taskRow.ID); err != nil { + t.Fatalf("bind cleanup admission task: %v", err) + } + failing := &validateFailureRepository{TaskRepository: runtime.repos.Tasks, err: errors.New("temporary database failure")} + failing.remaining.Store(1) + runtime.repos.Tasks = &limitedClaimRepository{TaskRepository: failing, maximum: 1} + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + failed := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + if failed.FailureReason == nil || *failed.FailureReason != "cleanup_not_started" || !runtime.service.Retryable(failed) || len(failed.Checkpoint) != 0 { + t.Fatalf("cleanup admission task = %#v", failed) + } + copies, err := runtime.repos.StorageCleanup.AuthorizeTask(t.Context(), content.ID, 1, taskRow.ID) + if err != nil || len(copies) != 1 || copies[0].Status != model.StorageCleanupCopyStatusPending || copies[0].ScheduledAt != nil { + t.Fatalf("cleanup copy after admission failure = %#v, err=%v", copies, err) + } + if deleteCalls.Load() != 0 { + t.Fatalf("cleanup admission performed %d remote deletions", deleteCalls.Load()) + } +} + +func TestCacheCapacityTaskStaysPendingWhenLRUDisabled(t *testing.T) { + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{policy: cache.EvictionPolicyNone}) + planner, created, err := runtime.service.Enqueue(t.Context(), taskengine.EnqueueRequest{ + Type: model.TaskTypeCacheCapacityReconcile, IdempotencyKey: systemtask.CacheCapacityKey, + Input: systemtask.Input{}, SubjectType: "system", SubjectKey: "cache-capacity", + }) + if err != nil || !created { + t.Fatalf("enqueue cache capacity task = %#v created=%v err=%v", planner, created, err) + } + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + stored := waitForTask(t, runtime.repos, planner.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusPending && task.StatusMessage != nil && + *task.StatusMessage == "Automatic cache cleanup is disabled" + }) + if stored.FinishedAt != nil || stored.RetentionUntil != nil || stored.RetryCount != 0 { + t.Fatalf("disabled recurring task became terminal: %#v", stored) + } + same, created, err := runtime.service.Enqueue(t.Context(), taskengine.EnqueueRequest{ + Type: model.TaskTypeCacheCapacityReconcile, IdempotencyKey: systemtask.CacheCapacityKey, + Input: systemtask.Input{}, SubjectType: "system", SubjectKey: "cache-capacity", + }) + if err != nil || created || same.ID != planner.ID { + t.Fatalf("reseed disabled cache capacity task = %#v created=%v err=%v", same, created, err) + } +} + +func TestCacheCapacityTaskEvictsLRUItemsOnlyToLowWatermark(t *testing.T) { + var used atomic.Int64 + used.Store(33) + var deletedMu sync.Mutex + var deleted []string + cacheStore := &testutil.MockCache{ + UsedBytesFunc: used.Load, + DeleteFunc: func(_ context.Context, _, key string) error { + deletedMu.Lock() + deleted = append(deleted, key) + deletedMu.Unlock() + used.Add(-11) + return nil + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + cache: cacheStore, policy: cache.EvictionPolicyLRU, maxBytes: 30, + highPercent: 90, lowPercent: 60, concurrency: 1, + }) + base := time.Now().UTC().Add(-3 * time.Hour).Truncate(time.Microsecond) + versions := []*model.ObjectVersion{ + seedStoredCacheObject(t, runtime, 11, base), + seedStoredCacheObject(t, runtime, 11, base.Add(time.Hour)), + seedStoredCacheObject(t, runtime, 11, base.Add(2*time.Hour)), + } + planner, created, err := runtime.service.Enqueue(t.Context(), taskengine.EnqueueRequest{ + Type: model.TaskTypeCacheCapacityReconcile, IdempotencyKey: systemtask.CacheCapacityKey, + Input: systemtask.Input{}, SubjectType: "system", SubjectKey: "cache-capacity", + }) + if err != nil || !created { + t.Fatalf("enqueue cache capacity task = %#v created=%v err=%v", planner, created, err) + } + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) && used.Load() > 11 { + time.Sleep(5 * time.Millisecond) + } + if used.Load() != 11 { + t.Fatalf("cache usage = %d, want 11", used.Load()) + } + deletedMu.Lock() + gotDeleted := append([]string(nil), deleted...) + deletedMu.Unlock() + wantDeleted := []string{versions[0].CacheKey(), versions[1].CacheKey()} + if len(gotDeleted) != len(wantDeleted) || gotDeleted[0] != wantDeleted[0] || gotDeleted[1] != wantDeleted[1] { + t.Fatalf("deleted cache keys = %#v, want %#v", gotDeleted, wantDeleted) + } + for index, version := range versions { + stored, err := runtime.repos.Objects.GetVersionByID(t.Context(), version.VersionID) + if err != nil || stored == nil { + t.Fatalf("load version %d: %#v err=%v", index, stored, err) + } + if index < 2 && stored.InCache { + t.Fatalf("version %d remained in cache", index) + } + if index == 2 && !stored.InCache { + t.Fatal("most recently used version was evicted") + } + } + page, err := runtime.repos.Tasks.List(t.Context(), repository.TaskListFilter{ + Type: model.TaskTypeCacheEvict, Status: model.TaskStatusCompleted, Limit: 10, + }) + if err != nil || len(page.Tasks) != 2 { + t.Fatalf("completed cache tasks = %#v, err=%v", page.Tasks, err) + } +} + +func TestCacheEvictionRecoverObservesBeforeReturningToExecute(t *testing.T) { + var present atomic.Bool + present.Store(true) + var deleteCalls atomic.Int64 + cacheStore := &testutil.MockCache{ + GetFunc: func(context.Context, string, string) (io.ReadCloser, *cache.ObjectInfo, error) { + if !present.Load() { + return nil, nil, os.ErrNotExist + } + return io.NopCloser(strings.NewReader("cached")), &cache.ObjectInfo{Size: 11}, nil + }, + DeleteFunc: func(context.Context, string, string) error { + deleteCalls.Add(1) + present.Store(false) + return nil + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{cache: cacheStore, policy: cache.EvictionPolicyNone}) + version := seedStoredCacheObject(t, runtime, 11, time.Now().UTC().Add(-time.Hour)) + generation, err := runtime.repos.CacheEvictions.NextEvictionGeneration(t.Context(), *version.ContentID) + if err != nil { + t.Fatalf("next eviction generation: %v", err) + } + taskRow, _, err := runtime.service.EnqueueTx(t.Context(), taskengine.EnqueueRequest{ + Type: model.TaskTypeCacheEvict, IdempotencyKey: cacheeviction.EvictTaskKey(*version.ContentID, generation), + Input: cacheeviction.EvictInput{ContentID: *version.ContentID, Generation: generation}, + SubjectType: "storage_content", SubjectKey: fmt.Sprint(*version.ContentID), + }, func(ctx context.Context, repos *repository.Repositories, taskRow *model.Task, _ bool) error { + return repos.CacheEvictions.BindEvictionTask(ctx, *version.ContentID, generation, taskRow.ID) + }) + if err != nil { + t.Fatalf("enqueue cache eviction: %v", err) + } + if _, err := runtime.db.NewUpdate(). + Model((*model.Task)(nil)). + Set("resume_mode = ?", model.TaskResumeModeRecover). + Where("id = ?", taskRow.ID). + Exec(t.Context()); err != nil { + t.Fatalf("force recovery mode: %v", err) + } + + limitedRepos := *runtime.repos + limitedRepos.Tasks = &limitedClaimRepository{TaskRepository: runtime.repos.Tasks, maximum: 1} + firstPass, err := taskengine.NewEngine(taskengine.EngineConfig{ + Concurrency: 1, PollInterval: 5 * time.Millisecond, LeaseDuration: 300 * time.Millisecond, + Retention: time.Hour, ProviderMutationConcurrency: 4, DestructiveMutationConcurrency: 2, + }, &limitedRepos, runtime.registry, slog.Default()) + if err != nil { + t.Fatalf("new limited task engine: %v", err) + } + cancelFirst, doneFirst := runEngine(t, firstPass) + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeExecute + }) + stopHandlerEngine(t, cancelFirst, doneFirst) + if got := deleteCalls.Load(); got != 0 { + t.Fatalf("cache delete calls during recovery = %d, want 0", got) + } + + cancelExecute, doneExecute := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancelExecute, doneExecute) + stored := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusCompleted + }) + if got := deleteCalls.Load(); got != 1 { + t.Fatalf("cache delete calls after execute = %d, want 1", got) + } + if len(stored.Checkpoint) == 0 { + t.Fatal("cache eviction completed without an attempted-effect checkpoint") + } + entry, err := runtime.repos.CacheEvictions.GetCacheEntry(t.Context(), *version.ContentID) + if err != nil || entry == nil { + t.Fatalf("load cache entry after eviction = %#v, err=%v", entry, err) + } + if entry.CacheActiveTaskID != nil || entry.InCache { + t.Fatalf("cache entry after eviction = %#v, want owner released and in_cache=false", entry) + } +} + +func TestCacheEvictionRecoverConvergesAfterDeletionWasAlreadyRecorded(t *testing.T) { + var deleteCalls atomic.Int64 + cacheStore := &testutil.MockCache{ + GetFunc: func(context.Context, string, string) (io.ReadCloser, *cache.ObjectInfo, error) { + return nil, nil, os.ErrNotExist + }, + DeleteFunc: func(context.Context, string, string) error { + deleteCalls.Add(1) + return nil + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{cache: cacheStore, policy: cache.EvictionPolicyNone}) + version := seedStoredCacheObject(t, runtime, 11, time.Now().UTC().Add(-time.Hour)) + generation, err := runtime.repos.CacheEvictions.NextEvictionGeneration(t.Context(), *version.ContentID) + if err != nil { + t.Fatalf("next eviction generation: %v", err) + } + taskRow, _, err := runtime.service.EnqueueTx(t.Context(), taskengine.EnqueueRequest{ + Type: model.TaskTypeCacheEvict, IdempotencyKey: cacheeviction.EvictTaskKey(*version.ContentID, generation), + Input: cacheeviction.EvictInput{ContentID: *version.ContentID, Generation: generation}, + SubjectType: "storage_content", SubjectKey: fmt.Sprint(*version.ContentID), + }, func(ctx context.Context, repos *repository.Repositories, taskRow *model.Task, _ bool) error { + return repos.CacheEvictions.BindEvictionTask(ctx, *version.ContentID, generation, taskRow.ID) + }) + if err != nil { + t.Fatalf("enqueue cache eviction: %v", err) + } + // Simulate a crash after unlink + RecordDeletion committed but before the + // engine persisted the task's completed transition. + if err := runtime.repos.CacheEvictions.RecordDeletion(t.Context(), *version.ContentID, generation, taskRow.ID); err != nil { + t.Fatalf("record cache deletion before recovery: %v", err) + } + if _, err := runtime.db.NewUpdate(). + Model((*model.Task)(nil)). + Set("resume_mode = ?", model.TaskResumeModeRecover). + Where("id = ?", taskRow.ID). + Exec(t.Context()); err != nil { + t.Fatalf("force recovery mode: %v", err) + } + + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusCompleted + }) + if got := deleteCalls.Load(); got != 0 { + t.Fatalf("cache delete calls during recovery = %d, want 0", got) + } + entry, err := runtime.repos.CacheEvictions.GetCacheEntry(t.Context(), *version.ContentID) + if err != nil || entry == nil { + t.Fatalf("load cache entry after recovery = %#v, err=%v", entry, err) + } + if entry.CacheActiveTaskID != nil || entry.InCache { + t.Fatalf("cache entry after recovery = %#v, want owner released and in_cache=false", entry) + } +} + +func TestCacheEvictionRecoverCancelsAfterNewGenerationCompletes(t *testing.T) { + cacheStore := &testutil.MockCache{ + GetFunc: func(context.Context, string, string) (io.ReadCloser, *cache.ObjectInfo, error) { + return nil, nil, os.ErrNotExist + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{cache: cacheStore, policy: cache.EvictionPolicyNone}) + version := seedStoredCacheObject(t, runtime, 11, time.Now().UTC().Add(-time.Hour)) + contentID := *version.ContentID + + firstGeneration, err := runtime.repos.CacheEvictions.NextEvictionGeneration(t.Context(), contentID) + if err != nil { + t.Fatalf("next first eviction generation: %v", err) + } + firstTask, _, err := runtime.service.EnqueueTx(t.Context(), taskengine.EnqueueRequest{ + Type: model.TaskTypeCacheEvict, IdempotencyKey: cacheeviction.EvictTaskKey(contentID, firstGeneration), + Input: cacheeviction.EvictInput{ContentID: contentID, Generation: firstGeneration}, + SubjectType: "storage_content", SubjectKey: fmt.Sprint(contentID), + }, func(ctx context.Context, repos *repository.Repositories, taskRow *model.Task, _ bool) error { + return repos.CacheEvictions.BindEvictionTask(ctx, contentID, firstGeneration, taskRow.ID) + }) + if err != nil { + t.Fatalf("enqueue first cache eviction: %v", err) + } + if err := runtime.repos.CacheEvictions.RecordDeletion(t.Context(), contentID, firstGeneration, firstTask.ID); err != nil { + t.Fatalf("record first cache deletion: %v", err) + } + if err := runtime.repos.Objects.RecordContentCacheCommit(t.Context(), contentID, time.Now()); err != nil { + t.Fatalf("restore cache presence: %v", err) + } + + reservation, err := runtime.repos.CacheEvictions.PrepareEviction(t.Context(), contentID) + if err != nil { + t.Fatalf("prepare replacement eviction: %v", err) + } + if reservation.ActiveTaskID != nil || reservation.Generation <= firstGeneration { + t.Fatalf("replacement eviction reservation = %#v", reservation) + } + secondTask, _, err := runtime.service.EnqueueTx(t.Context(), taskengine.EnqueueRequest{ + Type: model.TaskTypeCacheEvict, IdempotencyKey: cacheeviction.EvictTaskKey(contentID, reservation.Generation), + Input: cacheeviction.EvictInput{ContentID: contentID, Generation: reservation.Generation}, + SubjectType: "storage_content", SubjectKey: fmt.Sprint(contentID), + AvailableAt: time.Now().Add(time.Hour), + }, func(ctx context.Context, repos *repository.Repositories, taskRow *model.Task, _ bool) error { + return repos.CacheEvictions.BindEvictionTask(ctx, contentID, reservation.Generation, taskRow.ID) + }) + if err != nil { + t.Fatalf("enqueue replacement cache eviction: %v", err) + } + if err := runtime.repos.CacheEvictions.RecordDeletion(t.Context(), contentID, reservation.Generation, secondTask.ID); err != nil { + t.Fatalf("record replacement cache deletion: %v", err) + } + if _, err := runtime.db.NewUpdate(). + Model((*model.Task)(nil)). + Set("resume_mode = ?", model.TaskResumeModeRecover). + Where("id = ?", firstTask.ID). + Exec(t.Context()); err != nil { + t.Fatalf("force first task recovery mode: %v", err) + } + + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + stored := waitForTask(t, runtime.repos, firstTask.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusCancelled + }) + if stored.RetryCount != 0 || stored.LastError != nil { + t.Fatalf("superseded task diagnostics = retry_count=%d last_error=%v", stored.RetryCount, stored.LastError) + } + entry, err := runtime.repos.CacheEvictions.GetCacheEntry(t.Context(), contentID) + if err != nil || entry == nil { + t.Fatalf("load cache entry after superseded recovery = %#v, err=%v", entry, err) + } + if entry.CacheOperationGeneration != reservation.Generation || entry.CacheActiveTaskID != nil || entry.InCache { + t.Fatalf("newer cache generation changed by stale recovery: %#v", entry) + } +} + +func TestCacheEvictionPersistentDeleteFailureReleasesOwner(t *testing.T) { + deleteErr := errors.New("cache filesystem is read-only") + cacheStore := &testutil.MockCache{ + DeleteFunc: func(context.Context, string, string) error { + return deleteErr + }, + } + noRetries := 0 + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + cache: cacheStore, policy: cache.EvictionPolicyNone, maxRetries: &noRetries, + }) + version := seedStoredCacheObject(t, runtime, 11, time.Now().UTC().Add(-time.Hour)) + contentID := *version.ContentID + generation, err := runtime.repos.CacheEvictions.NextEvictionGeneration(t.Context(), contentID) + if err != nil { + t.Fatalf("next eviction generation: %v", err) + } + taskRow, _, err := runtime.service.EnqueueTx(t.Context(), taskengine.EnqueueRequest{ + Type: model.TaskTypeCacheEvict, IdempotencyKey: cacheeviction.EvictTaskKey(contentID, generation), + Input: cacheeviction.EvictInput{ContentID: contentID, Generation: generation}, + SubjectType: "storage_content", SubjectKey: fmt.Sprint(contentID), + }, func(ctx context.Context, repos *repository.Repositories, taskRow *model.Task, _ bool) error { + return repos.CacheEvictions.BindEvictionTask(ctx, contentID, generation, taskRow.ID) + }) + if err != nil { + t.Fatalf("enqueue cache eviction: %v", err) + } + + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + stored := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + if stored.FailureReason == nil || *stored.FailureReason != "cache_delete_failed" { + t.Fatalf("failure reason = %v, want cache_delete_failed", stored.FailureReason) + } + if stored.LastError == nil || !strings.Contains(*stored.LastError, deleteErr.Error()) { + t.Fatalf("last error = %v, want %q", stored.LastError, deleteErr) + } + entry, err := runtime.repos.CacheEvictions.GetCacheEntry(t.Context(), contentID) + if err != nil || entry == nil { + t.Fatalf("load cache entry after failed eviction = %#v, err=%v", entry, err) + } + if entry.CacheActiveTaskID != nil || !entry.InCache { + t.Fatalf("cache entry after failed eviction = %#v, want released owner and retained cache", entry) + } +} + +type limitedClaimRepository struct { + repository.TaskRepository + maximum int64 + claims atomic.Int64 +} + +type validateFailureRepository struct { + repository.TaskRepository + remaining atomic.Int64 + err error +} + +func (r *validateFailureRepository) ValidateClaim(ctx context.Context, id, generation int64) error { + if r.remaining.Add(-1) >= 0 { + return r.err + } + return r.TaskRepository.ValidateClaim(ctx, id, generation) +} + +func (r *limitedClaimRepository) ClaimNext(ctx context.Context, lease time.Duration) (*model.Task, error) { + if r.claims.Load() >= r.maximum { + return nil, nil + } + claimed, err := r.TaskRepository.ClaimNext(ctx, lease) + if claimed != nil && err == nil { + r.claims.Add(1) + } + return claimed, err +} + +type seededCopyPipeline struct { + upload *model.StorageContent + source *model.StorageCopy + target *model.StorageCopy + targetSet *model.StorageDataSet + targetClient sdktypes.BigInt + pieceCID cid.Cid +} + +func seedCopyPipeline(t *testing.T, runtime handlerTestRuntime, targetStatus model.StorageCopyStatus) seededCopyPipeline { + t.Helper() + ctx := t.Context() + sequence := storedObjectSequence.Add(1) + bucket := &model.Bucket{Name: fmt.Sprintf("copy-task-%d", sequence), Status: model.BucketStatusActive, DefaultCopies: 2, MinimumDurableCopies: 2} + if err := runtime.repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("create copy bucket: %v", err) + } + // Content identity comes first: a data version cannot exist without it. + upload, err := runtime.repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, ContentSize: 128, + Checksum: testutil.StorageChecksum(fmt.Sprintf("copy-checksum-%d", sequence)), RequestedCopies: 2, + }) + if err != nil { + t.Fatalf("start copy upload: %v", err) + } + bindings := make([]*model.StorageDataSet, 0, 2) + clients := make([]idtypes.OnChainID, 0, 2) + for copyIndex := range 2 { + providerID := testOnChainID(t, 5000+sequence*10+int64(copyIndex)) + dataSetID := testOnChainID(t, 6000+sequence*10+int64(copyIndex)) + clientID := testOnChainID(t, 7000+sequence*10+int64(copyIndex)) + binding, bindErr := runtime.repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: providerID, CopyIndex: copyIndex, CreatedByContentID: upload.ID, + }) + if bindErr != nil { + t.Fatalf("create copy binding %d: %v", copyIndex, bindErr) + } + if readyErr := runtime.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: binding.ID, ContentID: upload.ID, DataSetID: dataSetID, ClientDataSetID: &clientID, + }); readyErr != nil { + t.Fatalf("mark copy binding %d ready: %v", copyIndex, readyErr) + } + binding.DataSetID = &dataSetID + binding.ClientDataSetID = &clientID + bindings = append(bindings, binding) + clients = append(clients, clientID) + } + if err := runtime.repos.Contents.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ + {StorageDataSetID: bindings[0].ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: bindings[0].ProviderID}, + {StorageDataSetID: bindings[1].ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: bindings[1].ProviderID}, + }); err != nil { + t.Fatalf("create copy rows: %v", err) + } + copies, err := runtime.repos.Contents.ListCopies(ctx, upload.ID) + if err != nil || len(copies) != 2 { + t.Fatalf("list copy rows = %#v, err=%v", copies, err) + } + pieceCID := testPieceCID(t, fmt.Sprintf("copy-piece-%d", sequence)) + pieceID := testOnChainID(t, 8000+sequence) + testutil.CommitStorageCopy(t, runtime.db, runtime.repos, repository.MarkUploadCopyCommittedInput{ + StorageCopyID: copies[0].ID, ContentID: upload.ID, CopyIndex: 0, PieceCID: pieceCID.String(), + PieceID: &pieceID, RetrievalURL: "https://source.example/piece/" + pieceCID.String(), + }) + if targetStatus == model.StorageCopyStatusPieceReady { + if err := runtime.repos.Contents.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ + StorageCopyID: copies[1].ID, ContentID: upload.ID, CopyIndex: 1, + PieceCID: pieceCID.String(), RetrievalURL: "https://target.example/piece/" + pieceCID.String(), CommitExtraDataHex: "aabb", + }); err != nil { + t.Fatalf("mark target piece ready: %v", err) + } + } + copies, err = runtime.repos.Contents.ListCopies(ctx, upload.ID) + if err != nil { + t.Fatalf("reload copy rows: %v", err) + } + return seededCopyPipeline{ + upload: upload, source: &copies[0], target: &copies[1], targetSet: bindings[1], + targetClient: clients[1].SDK(), pieceCID: pieceCID, + } +} + +func bindCopyTask(t *testing.T, runtime handlerTestRuntime, copyRow *model.StorageCopy, taskType model.TaskType) *model.Task { + t.Helper() + generation, err := runtime.repos.Contents.NextCopyWorkGeneration(t.Context(), copyRow.ID) + if err != nil { + t.Fatalf("next copy generation: %v", err) + } + input := storagepipeline.CopyGenerationInput{CopyID: copyRow.ID, Generation: generation} + key := storagepipeline.PullKey(copyRow.ID, generation) + switch taskType { + case model.TaskTypeStorageStore: + key = storagepipeline.StoreKey(copyRow.ID, generation) + case model.TaskTypeStorageCommit: + key = storagepipeline.CommitKey(copyRow.ID, generation) + } + taskRow, created, err := runtime.service.Enqueue(t.Context(), taskengine.EnqueueRequest{ + Type: taskType, IdempotencyKey: key, Input: input, + SubjectType: "storage_copy", SubjectKey: fmt.Sprintf("%d", copyRow.ID), + }) + if err != nil || !created { + t.Fatalf("enqueue copy task = %#v created=%v err=%v", taskRow, created, err) + } + if err := runtime.repos.Contents.BindCopyTask(t.Context(), copyRow.ID, generation, taskRow.ID); err != nil { + t.Fatalf("bind copy task: %v", err) + } + return taskRow +} + +func TestCompetingCoordinatorsBindOnlyOneCopyTask(t *testing.T) { + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + pipeline := seedCopyPipeline(t, runtime, model.StorageCopyStatusPending) + generation, err := runtime.repos.Contents.NextCopyWorkGeneration(t.Context(), pipeline.target.ID) + if err != nil { + t.Fatalf("next copy generation: %v", err) + } + input := storagepipeline.CopyGenerationInput{CopyID: pipeline.target.ID, Generation: generation} + bind := func(taskType model.TaskType, key string) (*model.Task, error) { + returnTask, _, enqueueErr := runtime.service.EnqueueTx(t.Context(), taskengine.EnqueueRequest{ + Type: taskType, IdempotencyKey: key, Input: input, + SubjectType: "storage_copy", SubjectKey: fmt.Sprint(pipeline.target.ID), + }, func(ctx context.Context, repos *repository.Repositories, taskRow *model.Task, _ bool) error { + return repos.Contents.BindCopyTask(ctx, pipeline.target.ID, generation, taskRow.ID) + }) + return returnTask, enqueueErr + } + + winner, err := bind(model.TaskTypeStorageTransferPlan, storagepipeline.TransferPlanKey(pipeline.target.ID, generation)) + if err != nil { + t.Fatalf("bind winning copy task: %v", err) + } + if _, err := bind(model.TaskTypeStoragePull, storagepipeline.PullKey(pipeline.target.ID, generation)); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("bind competing copy task error = %v, want conflict", err) + } + + stored, err := runtime.repos.Contents.GetUploadCopyByID(t.Context(), pipeline.target.ID) + if err != nil || stored == nil || stored.ActiveTaskID == nil || *stored.ActiveTaskID != winner.ID || stored.WorkGeneration != generation { + t.Fatalf("copy task fence = %#v, winner=%#v, err=%v", stored, winner, err) + } + count, err := runtime.db.NewSelect(). + Model((*model.Task)(nil)). + Where("subject_type = ? AND subject_key = ?", "storage_copy", fmt.Sprint(pipeline.target.ID)). + Count(t.Context()) + if err != nil { + t.Fatalf("count copy tasks: %v", err) + } + if count != 1 { + t.Fatalf("persisted copy tasks = %d, want one winning task", count) + } +} + +func TestBucketProvisionCreatesDataSetsBeforeMarkingReady(t *testing.T) { + sequence := storedObjectSequence.Add(1) + targets := make([]synapse.StorageTarget, 0, 2) + for i := range 2 { + providerID := testOnChainID(t, 12000+sequence*10+int64(i)).SDK() + dataSetID := testOnChainID(t, 13000+sequence*10+int64(i)).SDK() + clientDataSetID := testOnChainID(t, 14000+sequence*10+int64(i)).SDK() + targets = append(targets, &testutil.MockStorageTarget{ + ProviderIDValue: providerID, DataSetIDValue: &dataSetID, ClientDataSetIDValue: clientDataSetID, + }) + } + storageClient := &testutil.MockStorageClient{ + SelectUploadTargetsFunc: func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { + if opts.Copies != 2 || opts.DataSetMetadata["bucket"] == "" { + t.Fatalf("selection options = %#v", opts) + } + return targets, nil + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: storageClient, policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + bucket := &model.Bucket{Name: fmt.Sprintf("provision-%d", sequence), Status: model.BucketStatusProvisioning, DefaultCopies: 2, MinimumDurableCopies: 2} + if err := runtime.repos.Buckets.Create(t.Context(), bucket); err != nil { + t.Fatalf("create bucket: %v", err) + } + taskRow, created, err := runtime.service.Enqueue(t.Context(), taskengine.EnqueueRequest{ + Type: model.TaskTypeBucketProvision, IdempotencyKey: bucketlifecycle.ProvisionKey(bucket.ID, bucket.DefaultCopies), + Input: bucketlifecycle.ProvisionInput{BucketID: bucket.ID}, SubjectType: "bucket", SubjectKey: fmt.Sprint(bucket.ID), + }) + if err != nil || !created { + t.Fatalf("enqueue bucket provision task = %#v created=%v err=%v", taskRow, created, err) + } + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusCompleted + }) + + stored, err := runtime.repos.Buckets.GetByID(t.Context(), bucket.ID) + if err != nil || stored == nil || stored.Status != model.BucketStatusReady { + t.Fatalf("ready bucket = %#v, err=%v", stored, err) + } + bindings, err := runtime.repos.Contents.ListDataSetBindings(t.Context(), bucket.ID) + if err != nil { + t.Fatalf("list data set bindings: %v", err) + } + if len(bindings) != 2 { + t.Fatalf("data set bindings = %d, want 2", len(bindings)) + } + for i := range bindings { + if !bindings[i].IsCurrent || bindings[i].CopyIndex != i || bindings[i].Status != model.StorageDataSetStatusReady || bindings[i].DataSetID == nil { + t.Fatalf("binding[%d] = %#v", i, bindings[i]) + } + } +} + +func TestDataSetEnsureSchedulesBoundCopyWorkWhenReady(t *testing.T) { + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + sequence := storedObjectSequence.Add(1) + bucket := &model.Bucket{Name: fmt.Sprintf("ensure-ready-%d", sequence), Status: model.BucketStatusActive, DefaultCopies: 2, MinimumDurableCopies: 2} + if err := runtime.repos.Buckets.Create(t.Context(), bucket); err != nil { + t.Fatalf("create ensure bucket: %v", err) + } + // Content identity comes first: a data version cannot exist without it. + upload, err := runtime.repos.Contents.EnsureContent(t.Context(), repository.EnsureContentInput{ + BucketID: bucket.ID, ContentSize: 11, + Checksum: testutil.StorageChecksum(fmt.Sprintf("ensure-checksum-%d", sequence)), RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("start ensure upload: %v", err) + } + version := &model.ObjectVersion{ + VersionID: model.NewVersionID(), BucketID: bucket.ID, Key: "ensure-ready.bin", ContentID: &upload.ID, Size: 11, + ETag: "etag", ContentType: "application/octet-stream", + } + if _, err := runtime.repos.Objects.CreateVersionAndSetCurrent(t.Context(), version); err != nil { + t.Fatalf("create ensure version: %v", err) + } + providerID := testOnChainID(t, 9000+sequence) + binding, err := runtime.repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: providerID, CopyIndex: 0, CreatedByContentID: upload.ID, + }) + if err != nil { + t.Fatalf("create pending data set: %v", err) + } + if err := runtime.repos.Contents.CreateUploadCopiesForBindings(t.Context(), upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: providerID, + }}); err != nil { + t.Fatalf("create bound copy: %v", err) + } + copyRow, err := runtime.repos.Contents.GetUploadCopyForDataSet(t.Context(), upload.ID, binding.ID) + if err != nil || copyRow == nil || copyRow.ActiveTaskID != nil { + t.Fatalf("bound copy = %#v err=%v", copyRow, err) + } + ensureTask, _, err := runtime.service.EnqueueTx(t.Context(), taskengine.EnqueueRequest{ + Type: model.TaskTypeStorageDataSetEnsure, IdempotencyKey: storagepipeline.DataSetEnsureKey(binding.ID), + Input: storagepipeline.DataSetInput{DataSetID: binding.ID}, SubjectType: "storage_data_set", SubjectKey: fmt.Sprint(binding.ID), + }, func(ctx context.Context, repos *repository.Repositories, row *model.Task, _ bool) error { + return repos.Contents.BindDataSetEnsureTask(ctx, binding.ID, row.ID) + }) + if err != nil { + t.Fatalf("enqueue data set ensure: %v", err) + } + dataSetID := testOnChainID(t, 10000+sequence) + clientDataSetID := testOnChainID(t, 11000+sequence) + if err := runtime.repos.Contents.MarkDataSetReady(t.Context(), repository.MarkDataSetReadyInput{ + ID: binding.ID, ContentID: upload.ID, DataSetID: dataSetID, ClientDataSetID: &clientDataSetID, + }); err != nil { + t.Fatalf("mark data set ready: %v", err) + } + limited := &limitedClaimRepository{TaskRepository: runtime.repos.Tasks, maximum: 1} + runtime.repos.Tasks = limited + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + waitForTask(t, runtime.repos, ensureTask.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusCompleted + }) + copyRow, err = runtime.repos.Contents.GetUploadCopyByID(t.Context(), copyRow.ID) + if err != nil || copyRow == nil || copyRow.ActiveTaskID == nil { + t.Fatalf("scheduled copy = %#v err=%v", copyRow, err) + } + transferTask, err := runtime.repos.Tasks.GetByID(t.Context(), *copyRow.ActiveTaskID) + if err != nil || transferTask == nil || transferTask.Type != model.TaskTypeStorageTransferPlan || transferTask.Status != model.TaskStatusPending { + t.Fatalf("scheduled transfer task = %#v err=%v", transferTask, err) + } +} + +func TestDataSetDiscoveryFailureNeverCreatesDataSet(t *testing.T) { + var createCalls atomic.Int64 + provider := &testutil.MockStorageTarget{ + CreateDataSetFunc: func(context.Context, *storage.CreateDataSetOptions) (*storage.CreateDataSetResult, error) { + createCalls.Add(1) + return nil, errors.New("unexpected data set creation") + }, + } + storageClient := &testutil.MockStorageClient{ + OpenProviderTargetFunc: func(_ context.Context, providerID sdktypes.BigInt, _ storage.NewProviderContextOptions) (synapse.ProviderTarget, error) { + provider.ProviderIDValue = providerID.Copy() + return provider, nil + }, + FindMatchingDataSetFunc: func(context.Context, sdktypes.BigInt, map[string]string, bool) (*storage.DataSetRef, error) { + return nil, &synapse.ProviderUnavailableError{Cause: errors.New("data set discovery unavailable")} + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: storageClient, policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + sequence := storedObjectSequence.Add(1) + bucket := &model.Bucket{Name: fmt.Sprintf("ensure-task-%d", sequence), Status: model.BucketStatusActive, DefaultCopies: 2, MinimumDurableCopies: 2} + if err := runtime.repos.Buckets.Create(t.Context(), bucket); err != nil { + t.Fatalf("create ensure bucket: %v", err) + } + // Content identity comes first: a data version cannot exist without it. + upload, err := runtime.repos.Contents.EnsureContent(t.Context(), repository.EnsureContentInput{ + BucketID: bucket.ID, ContentSize: 11, + Checksum: testutil.StorageChecksum(fmt.Sprintf("ensure-task-checksum-%d", sequence)), RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("start ensure upload: %v", err) + } + version := &model.ObjectVersion{ + VersionID: model.NewVersionID(), BucketID: bucket.ID, Key: "ensure.bin", ContentID: &upload.ID, Size: 11, + ETag: "etag", ContentType: "application/octet-stream", + } + if _, err := runtime.repos.Objects.CreateVersionAndSetCurrent(t.Context(), version); err != nil { + t.Fatalf("create ensure version: %v", err) + } + binding, err := runtime.repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: testOnChainID(t, 9000+sequence), CopyIndex: 0, CreatedByContentID: upload.ID, + }) + if err != nil { + t.Fatalf("create pending data set: %v", err) + } + input := storagepipeline.DataSetInput{DataSetID: binding.ID} + taskRow, _, err := runtime.service.EnqueueTx(t.Context(), taskengine.EnqueueRequest{ + Type: model.TaskTypeStorageDataSetEnsure, IdempotencyKey: storagepipeline.DataSetEnsureKey(binding.ID), + Input: input, SubjectType: "storage_data_set", SubjectKey: fmt.Sprint(binding.ID), + }, func(ctx context.Context, repos *repository.Repositories, row *model.Task, _ bool) error { + return repos.Contents.BindDataSetEnsureTask(ctx, binding.ID, row.ID) + }) + if err != nil { + t.Fatalf("enqueue data set ensure: %v", err) + } + limited := &limitedClaimRepository{TaskRepository: runtime.repos.Tasks, maximum: 1} + runtime.repos.Tasks = limited + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return limited.claims.Load() == 1 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover + }) + if createCalls.Load() != 0 { + t.Fatalf("data set creation calls = %d, want zero while discovery is unavailable", createCalls.Load()) + } +} + +func TestDataSetCreationAdmissionFailureRemainsSafeToRecover(t *testing.T) { + noRetries := 0 + var createCalls atomic.Int64 + providerID := testOnChainID(t, 29001) + provider := &testutil.MockStorageTarget{ + ProviderIDValue: providerID.SDK(), + CreateDataSetFunc: func(context.Context, *storage.CreateDataSetOptions) (*storage.CreateDataSetResult, error) { + createCalls.Add(1) + return nil, errors.New("unexpected data set creation") + }, + } + storageClient := &testutil.MockStorageClient{ + OpenProviderTargetFunc: func(context.Context, sdktypes.BigInt, storage.NewProviderContextOptions) (synapse.ProviderTarget, error) { + return provider, nil + }, + FindMatchingDataSetFunc: func(context.Context, sdktypes.BigInt, map[string]string, bool) (*storage.DataSetRef, error) { + return nil, nil + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: storageClient, maxRetries: &noRetries, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + bucket := &model.Bucket{Name: "data-set-admission", Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := runtime.repos.Buckets.Create(t.Context(), bucket); err != nil { + t.Fatalf("create admission bucket: %v", err) + } + binding, err := runtime.repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: providerID, CopyIndex: 0, + }) + if err != nil { + t.Fatalf("create pending data set: %v", err) + } + taskRow, _, err := runtime.service.EnqueueTx(t.Context(), taskengine.EnqueueRequest{ + Type: model.TaskTypeStorageDataSetEnsure, IdempotencyKey: storagepipeline.DataSetEnsureKey(binding.ID), + Input: storagepipeline.DataSetInput{DataSetID: binding.ID}, SubjectType: "storage_data_set", SubjectKey: fmt.Sprint(binding.ID), + }, func(ctx context.Context, repos *repository.Repositories, row *model.Task, _ bool) error { + return repos.Contents.BindDataSetEnsureTask(ctx, binding.ID, row.ID) + }) + if err != nil { + t.Fatalf("enqueue data set ensure: %v", err) + } + failing := &validateFailureRepository{TaskRepository: runtime.repos.Tasks, err: errors.New("temporary database failure")} + failing.remaining.Store(1) + limited := &limitedClaimRepository{TaskRepository: failing, maximum: 2} + runtime.repos.Tasks = limited + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + failed := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + if failed.FailureReason == nil || *failed.FailureReason != "dataset_creation_not_started" || !runtime.service.Retryable(failed) || len(failed.Checkpoint) != 0 { + t.Fatalf("data set admission task = %#v", failed) + } + stored, err := runtime.repos.Contents.GetDataSetBindingByID(t.Context(), binding.ID) + if err != nil || stored.Status != model.StorageDataSetStatusPending || stored.CreateTransactionID != nil || stored.CreateStatusURL != nil { + t.Fatalf("data set after admission failure = %#v, err=%v", stored, err) + } + if createCalls.Load() != 0 { + t.Fatalf("data set creation after admission failure = %d calls", createCalls.Load()) + } + if err := runtime.service.Retry(t.Context(), taskRow.ID); err != nil { + t.Fatalf("retry data set admission task: %v", err) + } + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return limited.claims.Load() == 2 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeExecute + }) + if createCalls.Load() != 0 { + t.Fatalf("data set recovery created %d services before returning to execute", createCalls.Load()) + } +} + +func TestTaskGCLeavesStorageEvidenceIntact(t *testing.T) { + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + pipeline := seedCopyPipeline(t, runtime, model.StorageCopyStatusPending) + taskRow := bindCopyTask(t, runtime, pipeline.target, model.TaskTypeStorageTransferPlan) + claimed, err := runtime.repos.Tasks.ClaimNext(t.Context(), time.Minute) + if err != nil || claimed == nil || claimed.ID != taskRow.ID { + t.Fatalf("claim copy task = %#v, err=%v", claimed, err) + } + if err := runtime.repos.Contents.CompleteCopyTask(t.Context(), pipeline.target.ID, pipeline.target.WorkGeneration+1, taskRow.ID); err != nil { + t.Fatalf("clear copy task fence: %v", err) + } + retentionUntil := time.Now().Add(-time.Minute) + if err := runtime.repos.Tasks.Settle(t.Context(), claimed.ID, claimed.ClaimGeneration, repository.TaskTransition{ + Status: model.TaskStatusCompleted, ResumeMode: model.TaskResumeModeRecover, RetentionUntil: &retentionUntil, + }); err != nil { + t.Fatalf("complete retained task: %v", err) + } + deleted, err := runtime.repos.Tasks.DeleteRetained(t.Context(), time.Now(), 10) + if err != nil || deleted != 1 { + t.Fatalf("delete retained tasks = %d, err=%v", deleted, err) + } + if stored, err := runtime.repos.Tasks.GetByID(t.Context(), taskRow.ID); err != nil || stored != nil { + t.Fatalf("collected task = %#v, err=%v", stored, err) + } + if upload, err := runtime.repos.Contents.GetByID(t.Context(), pipeline.upload.ID); err != nil || upload == nil { + t.Fatalf("storage upload evidence = %#v, err=%v", upload, err) + } + if copyRow, err := runtime.repos.Contents.GetUploadCopyByID(t.Context(), pipeline.target.ID); err != nil || copyRow == nil { + t.Fatalf("storage copy evidence = %#v, err=%v", copyRow, err) + } +} + +func TestStoreTasksReachAndRespectProviderMutationLimit(t *testing.T) { + var running, maximum atomic.Int64 + entered := make(chan struct{}, 6) + release := make(chan struct{}) + cacheStore := &testutil.MockCache{GetFunc: func(context.Context, string, string) (io.ReadCloser, *cache.ObjectInfo, error) { + return io.NopCloser(strings.NewReader(strings.Repeat("s", 128))), &cache.ObjectInfo{Size: 128}, nil + }} + storageClient := &testutil.MockStorageClient{} + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + cache: cacheStore, storage: storageClient, policy: cache.EvictionPolicyNone, concurrency: 8, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + clients := make(map[string]sdktypes.BigInt) + tasks := make([]*model.Task, 0, 6) + for range 6 { + pipeline := seedCopyPipeline(t, runtime, model.StorageCopyStatusPending) + clients[pipeline.targetSet.DataSetID.String()] = pipeline.targetClient.Copy() + tasks = append(tasks, bindCopyTask(t, runtime, pipeline.target, model.TaskTypeStorageStore)) + } + storageClient.OpenDataSetTargetFunc = func(_ context.Context, dataSetID sdktypes.BigInt, opts storage.NewDataSetContextOptions) (synapse.DataSetTarget, error) { + if opts.ProviderID == nil { + return nil, errors.New("missing provider identity") + } + clientID, ok := clients[dataSetID.String()] + if !ok { + return nil, errors.New("unknown data set identity") + } + targetDataSetID := dataSetID.Copy() + return &testutil.MockStorageTarget{ + ProviderIDValue: opts.ProviderID.Copy(), DataSetIDValue: &targetDataSetID, + ClientDataSetIDValue: clientID, ServiceURLValue: "https://store-limit.example", + StoreFunc: func(context.Context, io.Reader, *storage.StoreOptions) (*storage.StoreResult, error) { + current := running.Add(1) + for { + observed := maximum.Load() + if current <= observed || maximum.CompareAndSwap(observed, current) { + break + } + } + entered <- struct{}{} + <-release + running.Add(-1) + return nil, errors.New("injected ambiguous store result") + }, + }, nil + } + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + for range 4 { + select { + case <-entered: + case <-time.After(3 * time.Second): + close(release) + t.Fatal("store tasks did not reach provider mutation concurrency") + } + } + select { + case <-entered: + close(release) + t.Fatal("store tasks exceeded provider mutation concurrency") + case <-time.After(50 * time.Millisecond): + } + close(release) + for _, taskRow := range tasks { + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover + }) + } + if maximum.Load() != 4 { + t.Fatalf("store provider mutation maximum = %d, want 4", maximum.Load()) + } +} + +func TestStoreAdmissionFailureDoesNotStartUploadOrProgress(t *testing.T) { + noRetries := 0 + payload := strings.Repeat("s", 128) + var cacheOpens, storeCalls atomic.Int64 + cacheStore := &testutil.MockCache{GetFunc: func(context.Context, string, string) (io.ReadCloser, *cache.ObjectInfo, error) { + cacheOpens.Add(1) + return io.NopCloser(strings.NewReader(payload)), &cache.ObjectInfo{Size: int64(len(payload))}, nil + }} + target := &testutil.MockStorageTarget{ + ServiceURLValue: "https://store-admission.example", + StoreFunc: func(context.Context, io.Reader, *storage.StoreOptions) (*storage.StoreResult, error) { + storeCalls.Add(1) + return nil, errors.New("unexpected Store call") + }, + } + storageClient := &testutil.MockStorageClient{} + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + cache: cacheStore, storage: storageClient, maxRetries: &noRetries, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + pipeline := seedCopyPipeline(t, runtime, model.StorageCopyStatusPending) + target.ProviderIDValue = pipeline.targetSet.ProviderID.SDK() + targetDataSetID := pipeline.targetSet.DataSetID.SDK() + target.DataSetIDValue = &targetDataSetID + target.ClientDataSetIDValue = pipeline.targetClient + storageClient.OpenDataSetTargetFunc = func(context.Context, sdktypes.BigInt, storage.NewDataSetContextOptions) (synapse.DataSetTarget, error) { + return target, nil + } + taskRow := bindCopyTask(t, runtime, pipeline.target, model.TaskTypeStorageStore) + failing := &validateFailureRepository{TaskRepository: runtime.repos.Tasks, err: errors.New("temporary database failure")} + failing.remaining.Store(1) + limited := &limitedClaimRepository{TaskRepository: failing, maximum: 2} + runtime.repos.Tasks = limited + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + failed := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + if failed.FailureReason == nil || *failed.FailureReason != "store_not_started" || !runtime.service.Retryable(failed) || len(failed.Checkpoint) != 0 { + t.Fatalf("store admission task = %#v", failed) + } + copyRow, err := runtime.repos.Contents.GetUploadCopyByID(t.Context(), pipeline.target.ID) + if err != nil || copyRow.Status != model.StorageCopyStatusPending || copyRow.ActiveTaskID == nil || *copyRow.ActiveTaskID != taskRow.ID || copyRow.IngressStoreAttempt != 0 { + t.Fatalf("copy after store admission failure = %#v, err=%v", copyRow, err) + } + if storeCalls.Load() != 0 || cacheOpens.Load() != 2 { + t.Fatalf("store admission calls = store:%d cache:%d, want 0/2", storeCalls.Load(), cacheOpens.Load()) + } + if err := runtime.service.Retry(t.Context(), taskRow.ID); err != nil { + t.Fatalf("retry store admission task: %v", err) + } + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return limited.claims.Load() == 2 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeExecute + }) + if storeCalls.Load() != 0 || cacheOpens.Load() != 2 { + t.Fatalf("store recovery calls = store:%d cache:%d, want no new calls", storeCalls.Load(), cacheOpens.Load()) + } +} + +func TestStoreManualRetryRequiresUnsettledRecoveryEvidence(t *testing.T) { + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + checkpoint := json.RawMessage(`{"attempted_at":"2026-09-09T00:00:00Z"}`) + tests := []struct { + name string + reason string + checkpoint json.RawMessage + want bool + }{ + {name: "not started", reason: "store_not_started", want: true}, + {name: "not started with checkpoint", reason: "store_not_started", checkpoint: checkpoint}, + {name: "unknown outcome", reason: "store_outcome_unknown", checkpoint: checkpoint, want: true}, + {name: "unknown outcome without checkpoint", reason: "store_outcome_unknown"}, + {name: "checkpointed context failure", reason: "copy_context_failed", checkpoint: checkpoint, want: true}, + {name: "checkpointed owner missing", reason: "copy_owner_missing", checkpoint: checkpoint, want: true}, + {name: "context failure before checkpoint", reason: "copy_context_failed"}, + {name: "invalid checkpoint", reason: "invalid_checkpoint", checkpoint: checkpoint}, + {name: "settled store result failure", reason: "store_result_invalid", checkpoint: checkpoint}, + {name: "settled presign failure", reason: "commit_presign_failed", checkpoint: checkpoint}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reason := tt.reason + taskRow := &model.Task{ + Type: model.TaskTypeStorageStore, Status: model.TaskStatusFailed, + FailureReason: &reason, Checkpoint: tt.checkpoint, + } + if got := runtime.service.Retryable(taskRow); got != tt.want { + t.Fatalf("Retryable() = %t, want %t", got, tt.want) + } + }) + } +} + +func TestStoreUnknownRecoveryOnlyRechecksParkedPiece(t *testing.T) { + payload := strings.Repeat("parked-store", 12) + payload = payload[:128] + info, err := piece.Calculate(strings.NewReader(payload)) + if err != nil { + t.Fatalf("calculate expected piece identity: %v", err) + } + var cacheOpens, storeCalls, parkedCalls atomic.Int64 + var parkedError atomic.Bool + cacheStore := &testutil.MockCache{GetFunc: func(context.Context, string, string) (io.ReadCloser, *cache.ObjectInfo, error) { + cacheOpens.Add(1) + return io.NopCloser(strings.NewReader(payload)), &cache.ObjectInfo{Size: int64(len(payload))}, nil + }} + var parkedState atomic.Value + parkedState.Store(synapse.ParkedPieceMissing) + parked := parkedPieceCheckerFunc(func(_ context.Context, serviceURL string, pieceCID cid.Cid) (synapse.ParkedPieceState, error) { + parkedCalls.Add(1) + if serviceURL != "https://parked.example" || !pieceCID.Equals(info.CIDv2) { + return "", fmt.Errorf("unexpected parked lookup %q %s", serviceURL, pieceCID) + } + if parkedError.Load() { + return "", errors.New("temporary parked-piece lookup failure") + } + return parkedState.Load().(synapse.ParkedPieceState), nil + }) + target := &testutil.MockStorageTarget{ + ServiceURLValue: "https://parked.example", + StoreFunc: func(_ context.Context, reader io.Reader, options *storage.StoreOptions) (*storage.StoreResult, error) { + storeCalls.Add(1) + storedBytes, readErr := io.ReadAll(reader) + if readErr != nil || string(storedBytes) != payload { + return nil, fmt.Errorf("store reader = %d bytes, err=%v", len(storedBytes), readErr) + } + if options == nil || !options.PieceCID.Equals(info.CIDv2) { + return nil, errors.New("store did not receive the intended PieceCIDv2") + } + return nil, errors.New("provider disconnected after accepting the upload") + }, + PresignForCommitFunc: func(_ context.Context, pieces []storage.PieceInput) ([]byte, error) { + if len(pieces) != 1 || !pieces[0].PieceCID.Equals(info.CIDv2) { + return nil, errors.New("presign received the wrong piece") + } + return []byte{0xaa, 0xbb}, nil + }, + } + storageClient := &testutil.MockStorageClient{} + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + cache: cacheStore, storage: storageClient, parkedPieces: parked, policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + pipeline := seedCopyPipeline(t, runtime, model.StorageCopyStatusPending) + version := &model.ObjectVersion{ + VersionID: model.NewVersionID(), BucketID: pipeline.upload.BucketID, Key: "parked-store.bin", + ContentID: &pipeline.upload.ID, Size: pipeline.upload.ContentSize, ETag: "parked-store", + ContentType: "application/octet-stream", + } + if _, err := runtime.repos.Objects.CreateVersionAndSetCurrent(t.Context(), version); err != nil { + t.Fatalf("create parked store version: %v", err) + } + if _, err := runtime.db.NewUpdate().Model((*model.StorageContent)(nil)). + Set("piece_cid = ?", info.CIDv2.String()).Where("id = ?", pipeline.upload.ID).Exec(t.Context()); err != nil { + t.Fatalf("align parked store content identity: %v", err) + } + target.ProviderIDValue = pipeline.targetSet.ProviderID.SDK() + targetDataSetID := pipeline.targetSet.DataSetID.SDK() + target.DataSetIDValue = &targetDataSetID + target.ClientDataSetIDValue = pipeline.targetClient + storageClient.OpenDataSetTargetFunc = func(context.Context, sdktypes.BigInt, storage.NewDataSetContextOptions) (synapse.DataSetTarget, error) { + return target, nil + } + taskRow := bindCopyTask(t, runtime, pipeline.target, model.TaskTypeStorageStore) + limitedRepos := *runtime.repos + limited := &limitedClaimRepository{TaskRepository: runtime.repos.Tasks, maximum: 5} + limitedRepos.Tasks = limited + runtime.repos.Tasks = limited + engine, err := taskengine.NewEngine(taskengine.EngineConfig{ + Concurrency: 1, PollInterval: 5 * time.Millisecond, LeaseDuration: 300 * time.Millisecond, + Retention: time.Hour, ProviderMutationConcurrency: 1, DestructiveMutationConcurrency: 1, + }, &limitedRepos, runtime.registry, slog.Default()) + if err != nil { + t.Fatalf("new limited task engine: %v", err) + } + cancel, done := runEngine(t, engine) + defer stopHandlerEngine(t, cancel, done) + + recovering := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return limited.claims.Load() == 1 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover && len(task.Checkpoint) > 0 + }) + var checkpoint struct { + AttemptedAt time.Time `json:"attempted_at"` + IntendedPieceCID string `json:"intended_piece_cid"` + ProviderServiceURL string `json:"provider_service_url"` + } + if err := json.Unmarshal(recovering.Checkpoint, &checkpoint); err != nil { + t.Fatalf("decode store checkpoint: %v", err) + } + if checkpoint.IntendedPieceCID != info.CIDv2.String() || checkpoint.ProviderServiceURL != target.ServiceURL() { + t.Fatalf("store checkpoint = %#v", checkpoint) + } + parkedState.Store(synapse.ParkedPieceProcessing) + if _, err := runtime.db.NewRaw(`UPDATE tasks SET available_at = ? WHERE id = ?`, time.Now().Add(-time.Second), taskRow.ID).Exec(t.Context()); err != nil { + t.Fatalf("wake processing store recovery: %v", err) + } + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return limited.claims.Load() == 2 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover + }) + parkedError.Store(true) + if _, err := runtime.db.NewRaw(`UPDATE tasks SET available_at = ? WHERE id = ?`, time.Now().Add(-time.Second), taskRow.ID).Exec(t.Context()); err != nil { + t.Fatalf("wake failed store lookup: %v", err) + } + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return limited.claims.Load() == 3 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover + }) + parkedError.Store(false) + parkedState.Store(synapse.ParkedPieceMissing) + checkpoint.AttemptedAt = time.Now().Add(-31 * time.Minute) + checkpointJSON, err := json.Marshal(checkpoint) + if err != nil { + t.Fatalf("encode old store checkpoint: %v", err) + } + if _, err := runtime.db.NewRaw(`UPDATE task_payloads SET checkpoint_json = ? WHERE task_id = ?`, checkpointJSON, taskRow.ID).Exec(t.Context()); err != nil { + t.Fatalf("age store checkpoint: %v", err) + } + if _, err := runtime.db.NewRaw(`UPDATE tasks SET available_at = ? WHERE id = ?`, time.Now().Add(-time.Second), taskRow.ID).Exec(t.Context()); err != nil { + t.Fatalf("wake store recovery: %v", err) + } + failed := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + if failed.FailureReason == nil || *failed.FailureReason != "store_outcome_unknown" || !runtime.service.Retryable(failed) { + t.Fatalf("unknown store task = %#v", failed) + } + copyRow, err := runtime.repos.Contents.GetUploadCopyByID(t.Context(), pipeline.target.ID) + if err != nil || copyRow.Status != model.StorageCopyStatusPending || copyRow.ActiveTaskID == nil || *copyRow.ActiveTaskID != taskRow.ID { + t.Fatalf("unknown store copy = %#v, err=%v", copyRow, err) + } + parkedState.Store(synapse.ParkedPieceReady) + if err := runtime.service.Retry(t.Context(), taskRow.ID); err != nil { + t.Fatalf("retry unknown store: %v", err) + } + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusCompleted + }) + if storeCalls.Load() != 1 || cacheOpens.Load() != 2 || parkedCalls.Load() != 4 { + t.Fatalf("store recovery calls = store:%d cache:%d parked:%d, want 1/2/4", storeCalls.Load(), cacheOpens.Load(), parkedCalls.Load()) + } +} + +func TestPullRecoverObservesThenRepeatsIdenticalRequestInExecute(t *testing.T) { + var pullCalls, statusCalls atomic.Int64 + statusObserved := make(chan struct{}) + type observedPull struct { + piece string + extra string + url string + } + var observedMu sync.Mutex + var observed []observedPull + target := &testutil.MockStorageTarget{ + PresignForCommitFunc: func(context.Context, []storage.PieceInput) ([]byte, error) { return []byte{0xaa, 0xbb}, nil }, + PullFunc: func(_ context.Context, request storage.PullRequest) (*storage.PullResult, error) { + call := pullCalls.Add(1) + observedMu.Lock() + observed = append(observed, observedPull{ + piece: request.Pieces[0].String(), + extra: hex.EncodeToString(request.ExtraData), + url: request.From(request.Pieces[0]), + }) + observedMu.Unlock() + if call == 1 { + return nil, errors.New("ambiguous provider disconnect") + } + return &storage.PullResult{}, nil + }, + PieceStatusFunc: func(context.Context, cid.Cid) (*storage.PieceStatus, error) { + if statusCalls.Add(1) == 1 { + close(statusObserved) + } + return &storage.PieceStatus{Exists: false}, nil + }, + } + storageClient := &testutil.MockStorageClient{} + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: storageClient, policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + pipeline := seedCopyPipeline(t, runtime, model.StorageCopyStatusPending) + version := &model.ObjectVersion{ + VersionID: model.NewVersionID(), BucketID: pipeline.upload.BucketID, Key: "pull-recovery.bin", + ContentID: &pipeline.upload.ID, Size: pipeline.upload.ContentSize, ETag: "pull-recovery", + ContentType: "application/octet-stream", + } + if _, err := runtime.repos.Objects.CreateVersionAndSetCurrent(t.Context(), version); err != nil { + t.Fatalf("create pull recovery version: %v", err) + } + target.ProviderIDValue = pipeline.targetSet.ProviderID.SDK() + dataSetID := pipeline.targetSet.DataSetID.SDK() + target.DataSetIDValue = &dataSetID + target.ClientDataSetIDValue = pipeline.targetClient + storageClient.OpenDataSetTargetFunc = func(context.Context, sdktypes.BigInt, storage.NewDataSetContextOptions) (synapse.DataSetTarget, error) { + return target, nil + } + taskRow := bindCopyTask(t, runtime, pipeline.target, model.TaskTypeStoragePull) + limitedRepos := *runtime.repos + limited := &limitedClaimRepository{TaskRepository: runtime.repos.Tasks, maximum: 3} + limitedRepos.Tasks = limited + engine, err := taskengine.NewEngine(taskengine.EngineConfig{ + Concurrency: 1, PollInterval: 5 * time.Millisecond, LeaseDuration: 300 * time.Millisecond, + Retention: time.Hour, ProviderMutationConcurrency: 4, DestructiveMutationConcurrency: 2, + }, &limitedRepos, runtime.registry, slog.Default()) + if err != nil { + t.Fatalf("new limited task engine: %v", err) + } + cancel, done := runEngine(t, engine) + defer stopHandlerEngine(t, cancel, done) + + recovering := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover + }) + var checkpoint struct { + AttemptID string `json:"attempt_id"` + } + if err := json.Unmarshal(recovering.Checkpoint, &checkpoint); err != nil { + t.Fatalf("decode pull checkpoint: %v", err) + } + if checkpoint.AttemptID == "" { + t.Fatal("pull checkpoint attempt_id is empty") + } + if strings.Contains(string(recovering.Checkpoint), `"request_id"`) { + t.Fatalf("pull checkpoint retains removed request_id: %s", recovering.Checkpoint) + } + attempt := new(storagepull.Attempt) + if err := runtime.db.NewSelect().Model(attempt).Where("attempt_id = ?", checkpoint.AttemptID).Scan(t.Context()); err != nil { + t.Fatalf("load pull attempt %q: %v", checkpoint.AttemptID, err) + } + if _, err := runtime.db.NewRaw(`UPDATE tasks SET available_at = ? WHERE id = ?`, time.Now().Add(-time.Second), taskRow.ID).Exec(t.Context()); err != nil { + t.Fatalf("make pull recovery ready: %v", err) + } + select { + case <-statusObserved: + case <-time.After(3 * time.Second): + t.Fatal("pull recovery did not inspect target piece") + } + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return limited.claims.Load() == 2 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeExecute + }) + if _, err := runtime.db.NewRaw(`UPDATE tasks SET available_at = ? WHERE id = ?`, time.Now().Add(-time.Second), taskRow.ID).Exec(t.Context()); err != nil { + t.Fatalf("make repeated pull ready: %v", err) + } + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusCompleted + }) + if pullCalls.Load() != 2 { + t.Fatalf("pull calls = %d, want 2", pullCalls.Load()) + } + observedMu.Lock() + defer observedMu.Unlock() + if len(observed) != 2 || observed[0] != observed[1] { + t.Fatalf("pull requests = %#v, want two identical requests", observed) + } +} + +func TestPullProviderFailureAtomicallyAbandonsAttemptAndCopy(t *testing.T) { + target := &testutil.MockStorageTarget{ + PresignForCommitFunc: func(context.Context, []storage.PieceInput) ([]byte, error) { return []byte{0xaa, 0xbb}, nil }, + PullFunc: func(context.Context, storage.PullRequest) (*storage.PullResult, error) { + return nil, fmt.Errorf("provider pull: %w", pdp.ErrPullFailed) + }, + } + storageClient := &testutil.MockStorageClient{} + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: storageClient, policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + pipeline := seedCopyPipeline(t, runtime, model.StorageCopyStatusPending) + target.ProviderIDValue = pipeline.targetSet.ProviderID.SDK() + dataSetID := pipeline.targetSet.DataSetID.SDK() + target.DataSetIDValue = &dataSetID + target.ClientDataSetIDValue = pipeline.targetClient + storageClient.OpenDataSetTargetFunc = func(context.Context, sdktypes.BigInt, storage.NewDataSetContextOptions) (synapse.DataSetTarget, error) { + return target, nil + } + taskRow := bindCopyTask(t, runtime, pipeline.target, model.TaskTypeStoragePull) + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + failed := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + if failed.FailureReason == nil || *failed.FailureReason != "pull_failed" { + t.Fatalf("pull task failure = %#v, want pull_failed", failed) + } + copyRow, err := runtime.repos.Contents.GetUploadCopyByID(t.Context(), pipeline.target.ID) + if err != nil || copyRow.Status != model.StorageCopyStatusFailed || copyRow.ActiveTaskID != nil { + t.Fatalf("failed pull copy = %#v, err=%v", copyRow, err) + } + var attempts []storagepull.Attempt + if err := runtime.db.NewSelect().Model(&attempts).Where("content_id = ?", pipeline.target.ContentID).Scan(t.Context()); err != nil { + t.Fatalf("load pull attempts: %v", err) + } + if len(attempts) != 1 || attempts[0].Status != storagepull.AttemptStatusAbandoned || + attempts[0].ResolvedAt == nil || attempts[0].LastError == nil { + t.Fatalf("pull attempts = %#v, want one resolved abandoned attempt", attempts) + } + if err := runtime.repos.Contents.MarkUploadCopyFailed(t.Context(), repository.MarkUploadCopyFailedInput{ + StorageCopyID: pipeline.target.ID, ContentID: pipeline.target.ContentID, + CopyIndex: pipeline.target.CopyIndex, PullAttemptID: attempts[0].AttemptID, + LastError: "replayed pull failure", + }); err != nil { + t.Fatalf("replay abandoned pull settlement: %v", err) + } + if err := runtime.repos.Contents.MarkUploadCopyFailed(t.Context(), repository.MarkUploadCopyFailedInput{ + StorageCopyID: pipeline.target.ID, ContentID: pipeline.target.ContentID, + CopyIndex: pipeline.target.CopyIndex, PullAttemptID: "different-attempt", + LastError: "mismatched pull failure", + }); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("mismatched abandoned pull settlement = %v, want ErrConflict", err) + } +} + +func newPullErrorTask( + t *testing.T, + pullErr error, + maxRetries *int, + pullCalls *atomic.Int64, +) (handlerTestRuntime, seededCopyPipeline, *model.Task) { + t.Helper() + target := &testutil.MockStorageTarget{ + PresignForCommitFunc: func(context.Context, []storage.PieceInput) ([]byte, error) { + return []byte{0xaa, 0xbb}, nil + }, + PullFunc: func(context.Context, storage.PullRequest) (*storage.PullResult, error) { + pullCalls.Add(1) + return nil, pullErr + }, + } + storageClient := &testutil.MockStorageClient{} + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: storageClient, policy: cache.EvictionPolicyNone, maxRetries: maxRetries, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + pipeline := seedCopyPipeline(t, runtime, model.StorageCopyStatusPending) + target.ProviderIDValue = pipeline.targetSet.ProviderID.SDK() + dataSetID := pipeline.targetSet.DataSetID.SDK() + target.DataSetIDValue = &dataSetID + target.ClientDataSetIDValue = pipeline.targetClient + storageClient.OpenDataSetTargetFunc = func(context.Context, sdktypes.BigInt, storage.NewDataSetContextOptions) (synapse.DataSetTarget, error) { + return target, nil + } + return runtime, pipeline, bindCopyTask(t, runtime, pipeline.target, model.TaskTypeStoragePull) +} + +func assertFailedPullSettlement( + t *testing.T, + runtime handlerTestRuntime, + pipeline seededCopyPipeline, + taskRow *model.Task, + wantReason string, +) { + t.Helper() + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + failed := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + if failed.FailureReason == nil || *failed.FailureReason != wantReason { + t.Fatalf("pull task failure = %#v, want %s", failed, wantReason) + } + copyRow, err := runtime.repos.Contents.GetUploadCopyByID(t.Context(), pipeline.target.ID) + if err != nil || copyRow == nil || copyRow.Status != model.StorageCopyStatusFailed || copyRow.ActiveTaskID != nil { + t.Fatalf("failed pull copy = %#v, err=%v", copyRow, err) + } + var attempts []storagepull.Attempt + if err := runtime.db.NewSelect().Model(&attempts).Where("content_id = ?", pipeline.target.ContentID).Scan(t.Context()); err != nil { + t.Fatalf("load pull attempts: %v", err) + } + if len(attempts) != 1 || attempts[0].Status != storagepull.AttemptStatusAbandoned || attempts[0].ResolvedAt == nil { + t.Fatalf("pull attempts = %#v, want one resolved abandoned attempt", attempts) + } +} + +func TestPullTerminalClientErrorsSettleAttemptAndCopy(t *testing.T) { + tests := []struct { + name string + err error + }{ + {name: "bad request", err: &pdp.HTTPError{StatusCode: http.StatusBadRequest}}, + {name: "invalid arguments", err: storage.ErrInvalidArgument}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var pullCalls atomic.Int64 + runtime, pipeline, taskRow := newPullErrorTask(t, tt.err, nil, &pullCalls) + assertFailedPullSettlement(t, runtime, pipeline, taskRow, "pull_failed") + if pullCalls.Load() != 1 { + t.Fatalf("pull calls = %d, want 1", pullCalls.Load()) + } + }) + } +} + +func TestPullRetryableProviderErrorKeepsCopyOpen(t *testing.T) { + limit := 0 + var pullCalls atomic.Int64 + runtime, pipeline, taskRow := newPullErrorTask( + t, + &pdp.HTTPError{StatusCode: http.StatusServiceUnavailable}, + &limit, + &pullCalls, + ) + limitedRepos := *runtime.repos + limitedRepos.Tasks = &limitedClaimRepository{TaskRepository: runtime.repos.Tasks, maximum: 1} + engine, err := taskengine.NewEngine(taskengine.EngineConfig{ + Concurrency: 1, PollInterval: 5 * time.Millisecond, LeaseDuration: 300 * time.Millisecond, + Retention: time.Hour, ProviderMutationConcurrency: 4, DestructiveMutationConcurrency: 2, + }, &limitedRepos, runtime.registry, slog.Default()) + if err != nil { + t.Fatalf("new limited task engine: %v", err) + } + cancel, done := runEngine(t, engine) + defer stopHandlerEngine(t, cancel, done) + pending := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return pullCalls.Load() == 1 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeExecute + }) + if pending.RetryCount != 0 { + t.Fatalf("retry count = %d, want 0 for retryable provider failure", pending.RetryCount) + } + copyRow, err := runtime.repos.Contents.GetUploadCopyByID(t.Context(), pipeline.target.ID) + if err != nil || copyRow == nil || copyRow.Status == model.StorageCopyStatusFailed || copyRow.ActiveTaskID == nil { + t.Fatalf("retryable pull copy = %#v, err=%v", copyRow, err) + } +} + +func TestPullUnknownErrorFailsWhenRetryBudgetIsExhausted(t *testing.T) { + limit := 0 + var pullCalls atomic.Int64 + runtime, pipeline, taskRow := newPullErrorTask(t, errors.New("new sdk pull failure"), &limit, &pullCalls) + assertFailedPullSettlement(t, runtime, pipeline, taskRow, "pull_request_failed") + if pullCalls.Load() != 1 { + t.Fatalf("pull calls = %d, want 1", pullCalls.Load()) + } +} + +func TestPullRecoverWithoutCheckpointReturnsToExecute(t *testing.T) { + var presignCalls, pullCalls, statusCalls atomic.Int64 + target := &testutil.MockStorageTarget{ + PresignForCommitFunc: func(context.Context, []storage.PieceInput) ([]byte, error) { + presignCalls.Add(1) + return []byte{0xaa, 0xbb}, nil + }, + PullFunc: func(context.Context, storage.PullRequest) (*storage.PullResult, error) { + pullCalls.Add(1) + return nil, errors.New("unexpected pull") + }, + PieceStatusFunc: func(context.Context, cid.Cid) (*storage.PieceStatus, error) { + statusCalls.Add(1) + return nil, errors.New("unexpected status lookup") + }, + } + storageClient := &testutil.MockStorageClient{} + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: storageClient, policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + pipeline := seedCopyPipeline(t, runtime, model.StorageCopyStatusPending) + target.ProviderIDValue = pipeline.targetSet.ProviderID.SDK() + dataSetID := pipeline.targetSet.DataSetID.SDK() + target.DataSetIDValue = &dataSetID + target.ClientDataSetIDValue = pipeline.targetClient + storageClient.OpenDataSetTargetFunc = func(context.Context, sdktypes.BigInt, storage.NewDataSetContextOptions) (synapse.DataSetTarget, error) { + return target, nil + } + taskRow := bindCopyTask(t, runtime, pipeline.target, model.TaskTypeStoragePull) + claimed, err := runtime.repos.Tasks.ClaimNext(t.Context(), 100*time.Millisecond) + if err != nil || claimed == nil || claimed.ID != taskRow.ID { + t.Fatalf("seed expired pull claim = %#v, err=%v", claimed, err) + } + if _, err := runtime.db.NewRaw(`UPDATE tasks SET lease_until = ? WHERE id = ?`, time.Now().Add(-time.Second), taskRow.ID).Exec(t.Context()); err != nil { + t.Fatalf("expire pull claim: %v", err) + } + limited := &limitedClaimRepository{TaskRepository: runtime.repos.Tasks, maximum: 1} + runtime.repos.Tasks = limited + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return limited.claims.Load() == 1 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeExecute + }) + if presignCalls.Load() != 0 || pullCalls.Load() != 0 || statusCalls.Load() != 0 { + t.Fatalf("pull recovery calls = presign:%d pull:%d status:%d, want 0/0/0", presignCalls.Load(), pullCalls.Load(), statusCalls.Load()) + } + // A recovery that never sent a request leaves no ledger row behind. + attempts, err := runtime.db.NewSelect(). + Model((*storagepull.Attempt)(nil)). + Where("content_id = ?", pipeline.target.ContentID). + Count(t.Context()) + if err != nil || attempts != 0 { + t.Fatalf("pull attempts = %d, err=%v, want none", attempts, err) + } +} + +func TestCommitRecoverWithoutAttemptCannotSubmit(t *testing.T) { + var submitCalls atomic.Int64 + target := &testutil.MockStorageTarget{ + PresignForCommitFunc: func(context.Context, []storage.PieceInput) ([]byte, error) { return []byte{0xaa, 0xbb}, nil }, + SubmitCommitFunc: func(context.Context, storage.CommitRequest) (*storage.CommitSubmission, error) { + submitCalls.Add(1) + return nil, errors.New("unexpected submit") + }, + } + storageClient := &testutil.MockStorageClient{} + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: storageClient, policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + pipeline := seedCopyPipeline(t, runtime, model.StorageCopyStatusPieceReady) + target.ProviderIDValue = pipeline.targetSet.ProviderID.SDK() + dataSetID := pipeline.targetSet.DataSetID.SDK() + target.DataSetIDValue = &dataSetID + target.ClientDataSetIDValue = pipeline.targetClient + storageClient.OpenDataSetTargetFunc = func(context.Context, sdktypes.BigInt, storage.NewDataSetContextOptions) (synapse.DataSetTarget, error) { + return target, nil + } + taskRow := bindCopyTask(t, runtime, pipeline.target, model.TaskTypeStorageCommit) + claimed, err := runtime.repos.Tasks.ClaimNext(t.Context(), 100*time.Millisecond) + if err != nil || claimed == nil || claimed.ID != taskRow.ID { + t.Fatalf("seed expired commit claim = %#v, err=%v", claimed, err) + } + if _, err := runtime.db.NewRaw(`UPDATE tasks SET lease_until = ? WHERE id = ?`, time.Now().Add(-time.Second), taskRow.ID).Exec(t.Context()); err != nil { + t.Fatalf("expire commit claim: %v", err) + } + limited := &limitedClaimRepository{TaskRepository: runtime.repos.Tasks, maximum: 1} + runtime.repos.Tasks = limited + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return limited.claims.Load() == 1 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeExecute + }) + if submitCalls.Load() != 0 { + t.Fatalf("commit submissions during recovery = %d, want 0", submitCalls.Load()) + } +} + +func TestCommitRecoverableAttentionKeepsObserving(t *testing.T) { + storageClient := &testutil.MockStorageClient{ + OpenDataSetTargetFunc: func(context.Context, sdktypes.BigInt, storage.NewDataSetContextOptions) (synapse.DataSetTarget, error) { + return nil, storage.ErrDataSetUnavailable + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: storageClient, policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + pipeline := seedCopyPipeline(t, runtime, model.StorageCopyStatusPieceReady) + startedAt := time.Now().Add(-16 * time.Minute) + identity := storagecommit.CopyIdentity{ + StorageCopyID: pipeline.target.ID, + ContentID: pipeline.target.ContentID, + CopyIndex: pipeline.target.CopyIndex, + StorageDataSetID: pipeline.target.StorageDataSetID, + } + if _, err := runtime.repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ + Copy: identity, AttemptID: "worker-recoverable-attention", Now: startedAt, + }); err != nil { + t.Fatalf("reserve commit attempt: %v", err) + } + if _, err := runtime.repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ + Copy: identity, AttemptID: "worker-recoverable-attention", ExtraDataHex: "aabb", Now: startedAt, + }); err != nil { + t.Fatalf("mark commit attempted: %v", err) + } + taskRow := bindCopyTask(t, runtime, pipeline.target, model.TaskTypeStorageCommit) + limited := &limitedClaimRepository{TaskRepository: runtime.repos.Tasks, maximum: 1} + runtime.repos.Tasks = limited + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return limited.claims.Load() == 1 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover + }) + copyRow, err := runtime.repos.Contents.GetUploadCopyByID(t.Context(), pipeline.target.ID) + if err != nil || copyRow.ActiveTaskID == nil || *copyRow.ActiveTaskID != taskRow.ID || + copyRow.CommitAttentionCode == nil || *copyRow.CommitAttentionCode != string(storagecommit.AttentionDataSetUnavailable) { + t.Fatalf("commit copy = %#v, err=%v, want fenced recoverable attention", copyRow, err) + } +} + +func TestWalletRecoveryObservesTransactionWithoutRebroadcast(t *testing.T) { + var broadcasts, observations atomic.Int64 + transactionHash := "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + wallet: testWalletOperator{fund: func(_ context.Context, amount *big.Int) (string, error) { + broadcasts.Add(1) + if amount.String() != "100" { + t.Fatalf("wallet amount = %s, want 100", amount) + } + return transactionHash, nil + }}, + receipts: testReceiptChecker{check: func(_ context.Context, hash common.Hash) (*ethtypes.Receipt, error) { + observations.Add(1) + if hash != common.HexToHash(transactionHash) { + t.Fatalf("receipt hash = %s, want %s", hash, transactionHash) + } + return ðtypes.Receipt{Status: ethtypes.ReceiptStatusSuccessful}, nil + }}, + }) + var operation *model.WalletOperation + var taskRow *model.Task + if err := runtime.repos.WithTx(t.Context(), func(repos *repository.Repositories) error { + var err error + operation, _, err = repos.WalletOperations.CreateOrGet(t.Context(), repository.CreateWalletOperationInput{ + Type: model.WalletOperationTypeFund, ClientRequestID: "wallet-recovery", Amount: "100", + }) + if err != nil { + return err + } + taskRow, _, err = runtime.service.EnqueueInTransaction(t.Context(), repos, taskengine.EnqueueRequest{ + Type: model.TaskTypeWalletOperation, IdempotencyKey: walletoperation.TaskKey(operation.ID), + Input: walletoperation.Input{OperationID: operation.ID}, SubjectType: "wallet_operation", SubjectKey: fmt.Sprint(operation.ID), + }) + if err != nil { + return err + } + return repos.WalletOperations.BindTask(t.Context(), operation.ID, taskRow.ID) + }); err != nil { + t.Fatalf("seed wallet task: %v", err) + } + limited := &limitedClaimRepository{TaskRepository: runtime.repos.Tasks, maximum: 2} + runtime.repos.Tasks = limited + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return limited.claims.Load() == 1 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover + }) + if _, err := runtime.db.NewRaw(`UPDATE tasks SET available_at = ? WHERE id = ?`, time.Now().Add(-time.Second), taskRow.ID).Exec(t.Context()); err != nil { + t.Fatalf("make wallet recovery ready: %v", err) + } + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return limited.claims.Load() == 2 && task.Status == model.TaskStatusCompleted + }) + if broadcasts.Load() != 1 || observations.Load() != 1 { + t.Fatalf("wallet calls = broadcasts:%d observations:%d, want 1/1", broadcasts.Load(), observations.Load()) + } + stored, err := runtime.repos.WalletOperations.GetByID(t.Context(), operation.ID) + if err != nil || stored == nil || stored.Status != model.WalletOperationStatusConfirmed || stored.TaskID != nil { + t.Fatalf("wallet operation = %#v, err=%v", stored, err) + } +} + +func TestWalletBroadcastHasIndependentDeadline(t *testing.T) { + var broadcasts atomic.Int64 + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + walletBroadcastTimeout: 20 * time.Millisecond, + wallet: testWalletOperator{fund: func(ctx context.Context, _ *big.Int) (string, error) { + broadcasts.Add(1) + <-ctx.Done() + return "", context.Cause(ctx) + }}, + }) + operation, taskRow := seedWalletTask(t, runtime, "wallet-broadcast-timeout") + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + failed := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + if runtime.service.Retryable(failed) { + t.Fatal("wallet task with an uncertain broadcast is unexpectedly retryable") + } + if broadcasts.Load() != 1 { + t.Fatalf("wallet broadcasts = %d, want 1", broadcasts.Load()) + } + stored, err := runtime.repos.WalletOperations.GetByID(t.Context(), operation.ID) + if err != nil || stored.Status != model.WalletOperationStatusUnknown { + t.Fatalf("wallet operation = %#v, err=%v, want unknown", stored, err) + } +} + +func TestWalletAdmissionFailureRemainsSafeToRecover(t *testing.T) { + limit := 0 + var broadcasts atomic.Int64 + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + maxRetries: &limit, + wallet: testWalletOperator{fund: func(context.Context, *big.Int) (string, error) { + broadcasts.Add(1) + return "", errors.New("unexpected wallet broadcast") + }}, + }) + operation, taskRow := seedWalletTask(t, runtime, "wallet-admission-failure") + failing := &validateFailureRepository{TaskRepository: runtime.repos.Tasks, err: errors.New("temporary database failure")} + failing.remaining.Store(1) + limited := &limitedClaimRepository{TaskRepository: failing, maximum: 2} + runtime.repos.Tasks = limited + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + failed := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + if failed.FailureReason == nil || *failed.FailureReason != "wallet_broadcast_not_started" || !runtime.service.Retryable(failed) { + t.Fatalf("wallet admission task = %#v", failed) + } + storedOperation, err := runtime.repos.WalletOperations.GetByID(t.Context(), operation.ID) + if err != nil || storedOperation.Status != model.WalletOperationStatusPending || storedOperation.TaskID == nil || *storedOperation.TaskID != taskRow.ID || storedOperation.BroadcastAttemptedAt != nil { + t.Fatalf("wallet operation after admission failure = %#v, err=%v", storedOperation, err) + } + if len(failed.Checkpoint) != 0 || broadcasts.Load() != 0 { + t.Fatalf("wallet admission wrote evidence or called effect: checkpoint=%s broadcasts=%d", failed.Checkpoint, broadcasts.Load()) + } + if err := runtime.service.Retry(t.Context(), taskRow.ID); err != nil { + t.Fatalf("retry not-started wallet task: %v", err) + } + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return limited.claims.Load() == 2 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeExecute + }) + if broadcasts.Load() != 0 { + t.Fatalf("wallet recovery broadcast %d times before returning to execute", broadcasts.Load()) + } +} + +func TestWalletReceiptLookupHasIndependentDeadline(t *testing.T) { + var observations atomic.Int64 + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + walletReceiptTimeout: 20 * time.Millisecond, + receipts: testReceiptChecker{check: func(ctx context.Context, _ common.Hash) (*ethtypes.Receipt, error) { + observations.Add(1) + <-ctx.Done() + return nil, context.Cause(ctx) + }}, + }) + operation, taskRow := seedWalletTask(t, runtime, "wallet-receipt-timeout") + transactionHash := "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" + if err := runtime.repos.WalletOperations.MarkBroadcastAttempted(t.Context(), operation.ID, taskRow.ID); err != nil { + t.Fatalf("mark wallet broadcast attempted: %v", err) + } + if err := runtime.repos.WalletOperations.MarkSubmitted(t.Context(), operation.ID, taskRow.ID, transactionHash); err != nil { + t.Fatalf("mark wallet submitted: %v", err) + } + limited := &limitedClaimRepository{TaskRepository: runtime.repos.Tasks, maximum: 1} + runtime.repos.Tasks = limited + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return limited.claims.Load() == 1 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover + }) + if observations.Load() != 1 { + t.Fatalf("wallet receipt observations = %d, want 1", observations.Load()) + } +} + +func seedWalletTask(t *testing.T, runtime handlerTestRuntime, requestID string) (*model.WalletOperation, *model.Task) { + t.Helper() + var operation *model.WalletOperation + var taskRow *model.Task + if err := runtime.repos.WithTx(t.Context(), func(repos *repository.Repositories) error { + var err error + operation, _, err = repos.WalletOperations.CreateOrGet(t.Context(), repository.CreateWalletOperationInput{ + Type: model.WalletOperationTypeFund, ClientRequestID: requestID, Amount: "100", + }) + if err != nil { + return err + } + taskRow, _, err = runtime.service.EnqueueInTransaction(t.Context(), repos, taskengine.EnqueueRequest{ + Type: model.TaskTypeWalletOperation, IdempotencyKey: walletoperation.TaskKey(operation.ID), + Input: walletoperation.Input{OperationID: operation.ID}, SubjectType: "wallet_operation", SubjectKey: fmt.Sprint(operation.ID), + }) + if err != nil { + return err + } + return repos.WalletOperations.BindTask(t.Context(), operation.ID, taskRow.ID) + }); err != nil { + t.Fatalf("seed wallet task: %v", err) + } + return operation, taskRow +} + +func TestReplacementCoordinatorRetiresAfterCancelledItemsAreProcessed(t *testing.T) { + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterReplacement(registry) + }, + }) + ctx := t.Context() + bucket := &model.Bucket{ + Name: "replacement-cancelled-item", Status: model.BucketStatusActive, + DefaultCopies: 1, MinimumDurableCopies: 1, + } + if err := runtime.repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("create bucket: %v", err) + } + content, err := runtime.repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, ContentSize: 11, + Checksum: testutil.StorageChecksum("replacement-cancelled-item"), RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("ensure content: %v", err) + } + version := &model.ObjectVersion{ + VersionID: model.NewVersionID(), BucketID: bucket.ID, Key: "cancelled.bin", Size: content.ContentSize, + ETag: "cancelled", ContentType: "application/octet-stream", ContentID: &content.ID, + } + if _, err := runtime.repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + t.Fatalf("create object version: %v", err) + } + sourceProvider := testOnChainID(t, 7401) + source, err := runtime.repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: sourceProvider, CopyIndex: 0, CreatedByContentID: content.ID, + }) + if err != nil { + t.Fatalf("create source binding: %v", err) + } + sourceClientID := testOnChainID(t, 7402) + if err := runtime.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: source.ID, ContentID: content.ID, DataSetID: testOnChainID(t, 7403), ClientDataSetID: &sourceClientID, + }); err != nil { + t.Fatalf("mark source ready: %v", err) + } + replacement, created, err := runtime.repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: testOnChainID(t, 7404), ClientRequestID: "cancelled-item", + }) + if err != nil || !created { + t.Fatalf("authorize replacement = %#v created=%v err=%v", replacement, created, err) + } + target, err := runtime.repos.Contents.GetDataSetBindingByID(ctx, replacement.TargetDataSetID) + if err != nil || target == nil { + t.Fatalf("load target = %#v err=%v", target, err) + } + targetClientID := testOnChainID(t, 7405) + if err := runtime.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: target.ID, ContentID: content.ID, DataSetID: testOnChainID(t, 7406), ClientDataSetID: &targetClientID, + }); err != nil { + t.Fatalf("mark target ready: %v", err) + } + if err := runtime.repos.Replacements.Activate(ctx, replacement.ID); err != nil { + t.Fatalf("activate replacement: %v", err) + } + if err := runtime.repos.Contents.CreateUploadCopiesForBindings(ctx, content.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: target.ID, CopyIndex: target.CopyIndex, + TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: target.ProviderID, + }}); err != nil { + t.Fatalf("create target copy: %v", err) + } + item := &storagereplacement.Item{ + ReplacementID: replacement.ID, ContentID: content.ID, TargetDataSetID: target.ID, + Status: storagereplacement.ItemStatusCancelled, + } + if _, err := runtime.db.NewInsert().Model(item).Exec(ctx); err != nil { + t.Fatalf("insert cancelled replacement item: %v", err) + } + if _, err := runtime.db.NewUpdate(). + Model((*storagereplacement.Replacement)(nil)). + Set("seeding_complete = ?", true). + Set("items_total = ?", 1). + Where("id = ?", replacement.ID). + Exec(ctx); err != nil { + t.Fatalf("complete replacement seeding: %v", err) + } + coordinateTask, _, err := runtime.service.EnqueueTx(ctx, taskengine.EnqueueRequest{ + Type: model.TaskTypeProviderReplacementCoordinate, + IdempotencyKey: storagereplacement.CoordinateTaskKey(replacement.ID, replacement.TaskGeneration), + Input: storagereplacement.CoordinateInput{ + ReplacementID: replacement.ID, Generation: replacement.TaskGeneration, + }, + SubjectType: "storage_replacement", SubjectKey: fmt.Sprint(replacement.ID), + }, func(ctx context.Context, repos *repository.Repositories, taskRow *model.Task, _ bool) error { + return repos.Replacements.BindTask(ctx, replacement.ID, replacement.TaskGeneration, taskRow.ID) + }) + if err != nil { + t.Fatalf("enqueue replacement coordinator: %v", err) + } + + limitedRepos := *runtime.repos + limitedRepos.Tasks = &limitedClaimRepository{TaskRepository: runtime.repos.Tasks, maximum: 1} + engine, err := taskengine.NewEngine(taskengine.EngineConfig{ + Concurrency: 1, PollInterval: 5 * time.Millisecond, LeaseDuration: 300 * time.Millisecond, + Retention: time.Hour, ProviderMutationConcurrency: 4, DestructiveMutationConcurrency: 2, + }, &limitedRepos, runtime.registry, slog.Default()) + if err != nil { + t.Fatalf("new limited task engine: %v", err) + } + cancel, done := runEngine(t, engine) + defer stopHandlerEngine(t, cancel, done) + waitForTask(t, runtime.repos, coordinateTask.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusCompleted + }) + stored, err := runtime.repos.Replacements.GetByID(ctx, replacement.ID) + if err != nil || stored == nil { + t.Fatalf("load replacement after coordination = %#v err=%v", stored, err) + } + if stored.Status != storagereplacement.StatusRetiring || stored.TaskID != nil { + t.Fatalf("replacement after cancelled item = %#v, want retiring with coordinator released", stored) + } + source, err = runtime.repos.Contents.GetDataSetBindingByID(ctx, source.ID) + if err != nil || source == nil || source.RetirementTaskID == nil { + t.Fatalf("source retirement reservation = %#v err=%v", source, err) + } +} + +func TestRetirementAdmissionFailureDoesNotEnterCleanupAttention(t *testing.T) { + noRetries := 0 + terminator := &testServiceTerminator{err: errors.New("unexpected termination")} + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + maxRetries: &noRetries, terminator: terminator, epochs: testEpochReader{epoch: 80}, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterReplacement(registry) + }, + }) + ctx := t.Context() + bucket := &model.Bucket{Name: "retirement-admission", Status: model.BucketStatusActive, DefaultCopies: 2, MinimumDurableCopies: 2} + if err := runtime.repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("create retirement admission bucket: %v", err) + } + source, err := runtime.repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: testOnChainID(t, 29201), CopyIndex: 0, + }) + if err != nil { + t.Fatalf("create retirement admission source: %v", err) + } + sourceClientID := testOnChainID(t, 29202) + if err := runtime.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: source.ID, DataSetID: testOnChainID(t, 29203), ClientDataSetID: &sourceClientID, + }); err != nil { + t.Fatalf("mark retirement admission source ready: %v", err) + } + first, created, err := runtime.repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: testOnChainID(t, 29204), ClientRequestID: "retirement-admission-first", + }) + if err != nil || !created { + t.Fatalf("authorize retirement admission replacement = %#v, created=%v, err=%v", first, created, err) + } + target, err := runtime.repos.Contents.GetDataSetBindingByID(ctx, first.TargetDataSetID) + if err != nil || target == nil { + t.Fatalf("load retirement admission target = %#v, err=%v", target, err) + } + targetClientID := testOnChainID(t, 29205) + if err := runtime.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: target.ID, DataSetID: testOnChainID(t, 29206), ClientDataSetID: &targetClientID, + }); err != nil { + t.Fatalf("mark retirement admission target ready: %v", err) + } + if _, created, err := runtime.repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: testOnChainID(t, 29207), ClientRequestID: "retirement-admission-successor", + }); err != nil || !created { + t.Fatalf("authorize retirement admission successor: created=%v err=%v", created, err) + } + first, err = runtime.repos.Replacements.GetByID(ctx, first.ID) + if err != nil || first.Status != storagereplacement.StatusSuperseded { + t.Fatalf("superseded retirement admission replacement = %#v, err=%v", first, err) + } + generation, err := runtime.repos.Contents.NextDataSetRetirementGeneration(ctx, target.ID) + if err != nil { + t.Fatalf("next retirement admission generation: %v", err) + } + taskRow, _, err := runtime.service.Enqueue(ctx, taskengine.EnqueueRequest{ + Type: model.TaskTypeStorageDataSetRetire, IdempotencyKey: storagereplacement.RetireTaskKey(target.ID, generation), + Input: storagereplacement.RetireInput{ReplacementID: first.ID, DataSetID: target.ID, Generation: generation}, + SubjectType: "storage_data_set", SubjectKey: fmt.Sprint(target.ID), + }) + if err != nil { + t.Fatalf("enqueue retirement admission task: %v", err) + } + if err := runtime.repos.Contents.BindDataSetRetirementTask(ctx, target.ID, generation, taskRow.ID); err != nil { + t.Fatalf("bind retirement admission task: %v", err) + } + failing := &validateFailureRepository{TaskRepository: runtime.repos.Tasks, err: errors.New("temporary database failure")} + failing.remaining.Store(1) + limited := &limitedClaimRepository{TaskRepository: failing, maximum: 2} + runtime.repos.Tasks = limited + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + failed := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + if failed.FailureReason == nil || *failed.FailureReason != "termination_not_started" || !runtime.service.Retryable(failed) || len(failed.Checkpoint) != 0 { + t.Fatalf("retirement admission task = %#v", failed) + } + stored, err := runtime.repos.Replacements.GetByID(ctx, first.ID) + if err != nil || stored.Status != storagereplacement.StatusSuperseded || stored.AbandonedTerminationEpoch != nil || stored.LastError != nil { + t.Fatalf("replacement after retirement admission failure = %#v, err=%v", stored, err) + } + if terminator.calls.Load() != 0 { + t.Fatalf("retirement admission called terminator %d times", terminator.calls.Load()) + } + if err := runtime.service.Retry(ctx, taskRow.ID); err != nil { + t.Fatalf("retry retirement admission task: %v", err) + } + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return limited.claims.Load() == 2 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeExecute + }) + if terminator.calls.Load() != 0 { + t.Fatalf("retirement recovery terminated service %d times before returning to execute", terminator.calls.Load()) + } +} + +func TestRetirementRecoveryPersistsReturnedEpochWithoutRepeatingTermination(t *testing.T) { + terminator := &testServiceTerminator{result: &synapse.TerminationResult{TxHash: "0xretire", EndEpoch: 84}} + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + terminator: terminator, + epochs: testEpochReader{epoch: 80}, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterReplacement(registry) + }, + }) + ctx := t.Context() + bucket := &model.Bucket{Name: "retirement-recovery", Status: model.BucketStatusActive, DefaultCopies: 2, MinimumDurableCopies: 2} + if err := runtime.repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("create retirement bucket: %v", err) + } + source, err := runtime.repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: testOnChainID(t, 7101), CopyIndex: 0, + }) + if err != nil { + t.Fatalf("create retirement source: %v", err) + } + sourceClientID := testOnChainID(t, 7201) + if err := runtime.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: source.ID, DataSetID: testOnChainID(t, 7301), ClientDataSetID: &sourceClientID, + }); err != nil { + t.Fatalf("mark retirement source ready: %v", err) + } + first, created, err := runtime.repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: testOnChainID(t, 7102), ClientRequestID: "retirement-first", + }) + if err != nil || !created { + t.Fatalf("authorize first replacement = %#v created=%v err=%v", first, created, err) + } + target, err := runtime.repos.Contents.GetDataSetBindingByID(ctx, first.TargetDataSetID) + if err != nil || target == nil { + t.Fatalf("load retirement target = %#v err=%v", target, err) + } + targetClientID := testOnChainID(t, 7202) + if err := runtime.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: target.ID, DataSetID: testOnChainID(t, 7302), ClientDataSetID: &targetClientID, + }); err != nil { + t.Fatalf("mark retirement target ready: %v", err) + } + if _, created, err := runtime.repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: testOnChainID(t, 7103), ClientRequestID: "retirement-successor", + }); err != nil || !created { + t.Fatalf("authorize successor replacement created=%v err=%v", created, err) + } + first, err = runtime.repos.Replacements.GetByID(ctx, first.ID) + if err != nil || first == nil || first.Status != storagereplacement.StatusSuperseded { + t.Fatalf("superseded replacement = %#v err=%v", first, err) + } + generation, err := runtime.repos.Contents.NextDataSetRetirementGeneration(ctx, target.ID) + if err != nil { + t.Fatalf("next retirement generation: %v", err) + } + retirementTask, _, err := runtime.service.Enqueue(ctx, taskengine.EnqueueRequest{ + Type: model.TaskTypeStorageDataSetRetire, + IdempotencyKey: storagereplacement.RetireTaskKey(target.ID, generation), + Input: storagereplacement.RetireInput{ + ReplacementID: first.ID, DataSetID: target.ID, Generation: generation, + }, + SubjectType: "storage_data_set", SubjectKey: fmt.Sprint(target.ID), + }) + if err != nil { + t.Fatalf("enqueue retirement task: %v", err) + } + if err := runtime.repos.Contents.BindDataSetRetirementTask(ctx, target.ID, generation, retirementTask.ID); err != nil { + t.Fatalf("bind retirement task: %v", err) + } + if _, err := runtime.db.ExecContext(ctx, `CREATE TRIGGER fail_retirement_evidence + BEFORE INSERT ON storage_data_set_terminations + WHEN NEW.role = 'abandoned_target' + BEGIN SELECT RAISE(FAIL, 'injected retirement evidence failure'); END`); err != nil { + t.Fatalf("create retirement evidence trigger: %v", err) + } + + cancel, done := runHandlerEngine(t, runtime) + waitForTask(t, runtime.repos, retirementTask.ID, func(task *model.Task) bool { + return strings.Contains(string(task.Checkpoint), `"termination_epoch":84`) + }) + stopHandlerEngine(t, cancel, done) + + storedTask, err := runtime.repos.Tasks.GetByID(ctx, retirementTask.ID) + if err != nil || storedTask == nil || storedTask.Status != model.TaskStatusRunning || storedTask.ResumeMode != model.TaskResumeModeRecover { + t.Fatalf("retirement task after failed settlement = %#v err=%v", storedTask, err) + } + first, err = runtime.repos.Replacements.GetByID(ctx, first.ID) + if err != nil || first == nil || first.AbandonedTerminationEpoch != nil { + t.Fatalf("retirement evidence unexpectedly settled = %#v err=%v", first, err) + } + if _, err := runtime.db.ExecContext(ctx, `DROP TRIGGER fail_retirement_evidence`); err != nil { + t.Fatalf("drop retirement evidence trigger: %v", err) + } + if _, err := runtime.db.NewRaw(`UPDATE tasks SET lease_until = ? WHERE id = ?`, time.Now().Add(-time.Second), retirementTask.ID).Exec(ctx); err != nil { + t.Fatalf("expire retirement task lease: %v", err) + } + + cancel, done = runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + waitForTask(t, runtime.repos, retirementTask.ID, func(task *model.Task) bool { + first, err = runtime.repos.Replacements.GetByID(ctx, first.ID) + return err == nil && first != nil && first.AbandonedTerminationEpoch != nil && + *first.AbandonedTerminationEpoch == 84 && task.Status == model.TaskStatusPending + }) + if terminator.calls.Load() != 1 { + t.Fatalf("retirement termination calls = %d, want 1", terminator.calls.Load()) + } +} + +// A replica index has to name a slot the bucket opened. Allocation used to look +// for a free index by counting up to the global maximum, so a bucket whose low +// slot was held by an unusable generation would reach past its own slots; the +// data set that came back could not be stored, because the slot behind that +// index does not exist. Allocation now reads the bucket's open slots. +func TestUploadPlanAllocatesOnlyWithinTheBucketsOpenSlots(t *testing.T) { + sequence := storedObjectSequence.Add(1) + providerID := testOnChainID(t, 21000+sequence).SDK() + dataSetID := testOnChainID(t, 22000+sequence).SDK() + clientDataSetID := testOnChainID(t, 23000+sequence).SDK() + selection := make(chan storage.SelectUploadContextsOptions, 4) + storageClient := &testutil.MockStorageClient{ + SelectUploadTargetsFunc: func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { + select { + case selection <- opts: + default: + } + return []synapse.StorageTarget{&testutil.MockStorageTarget{ + ProviderIDValue: providerID, DataSetIDValue: &dataSetID, ClientDataSetIDValue: clientDataSetID, + }}, nil + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: storageClient, policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + ctx := t.Context() + + // One open slot, but the content asks for two copies. Allocation has to be + // bounded by the slots the bucket actually opened. + bucket := &model.Bucket{ + Name: fmt.Sprintf("slot-bound-%d", sequence), Status: model.BucketStatusActive, + DefaultCopies: 1, MinimumDurableCopies: 1, + } + if err := runtime.repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("create bucket: %v", err) + } + content, err := runtime.repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, ContentSize: 11, + Checksum: testutil.StorageChecksum(fmt.Sprintf("slot-bound-checksum-%d", sequence)), RequestedCopies: 2, + }) + if err != nil { + t.Fatalf("ensure content: %v", err) + } + version := &model.ObjectVersion{ + VersionID: model.NewVersionID(), BucketID: bucket.ID, Key: "object.bin", Size: 11, + ETag: fmt.Sprintf("slot-bound-etag-%d", sequence), ContentType: "application/octet-stream", + ContentID: &content.ID, + } + if _, err := runtime.repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + t.Fatalf("create object version: %v", err) + } + + taskRow, created, err := runtime.service.Enqueue(ctx, taskengine.EnqueueRequest{ + Type: model.TaskTypeUploadPlan, IdempotencyKey: storagepipeline.UploadPlanKey(content.ID), + Input: storagepipeline.UploadPlanInput{ContentID: content.ID}, + SubjectType: "storage_content", SubjectKey: fmt.Sprint(content.ID), + }) + if err != nil || !created { + t.Fatalf("enqueue upload plan = %#v created=%v err=%v", taskRow, created, err) + } + cancel, done := runHandlerEngine(t, runtime) + var opts storage.SelectUploadContextsOptions + select { + case opts = <-selection: + case <-time.After(5 * time.Second): + stopHandlerEngine(t, cancel, done) + t.Fatal("upload plan never asked for storage targets") + } + stopHandlerEngine(t, cancel, done) + + // Only one slot is open, so one target is what may be asked for. Asking for + // two would hand back a target for an index the bucket never opened. + if opts.Copies != 1 { + t.Fatalf("requested copies = %d, want 1 bounded by the bucket's free open slots", opts.Copies) + } + bindings, err := runtime.repos.Contents.ListDataSetBindings(ctx, bucket.ID) + if err != nil { + t.Fatalf("ListDataSetBindings: %v", err) + } + for i := range bindings { + if bindings[i].CopyIndex >= bucket.DefaultCopies { + t.Fatalf("data set bound to copy index %d, want an index inside the bucket's %d slots", + bindings[i].CopyIndex, bucket.DefaultCopies) + } + } +} + +// A generation that failed to be created must not keep bucket provisioning from +// finishing. The failed row gives up the slot, its provider is excluded from the +// next selection, and the bucket reaches ready on a different provider. +func TestBucketProvisionRecoversFromAFailedDataSetGeneration(t *testing.T) { + sequence := storedObjectSequence.Add(1) + failedProvider := testOnChainID(t, 15000+sequence) + healthyProvider := testOnChainID(t, 16000+sequence) + healthyDataSet := testOnChainID(t, 17000+sequence).SDK() + var excludedFailedProvider atomic.Bool + storageClient := &testutil.MockStorageClient{ + SelectUploadTargetsFunc: func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { + for _, excluded := range opts.ExcludeProviderIDs { + if excluded.String() == failedProvider.String() { + excludedFailedProvider.Store(true) + } + } + clientDataSetID := testOnChainID(t, 18000+sequence).SDK() + return []synapse.StorageTarget{&testutil.MockStorageTarget{ + ProviderIDValue: healthyProvider.SDK(), DataSetIDValue: &healthyDataSet, + ClientDataSetIDValue: clientDataSetID, + }}, nil + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: storageClient, policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + bucket := &model.Bucket{ + Name: fmt.Sprintf("provision-recovery-%d", sequence), Status: model.BucketStatusProvisioning, + DefaultCopies: 1, MinimumDurableCopies: 1, + } + if err := runtime.repos.Buckets.Create(t.Context(), bucket); err != nil { + t.Fatalf("create bucket: %v", err) + } + // The state the real failure path leaves behind: a generation that never + // reached the chain, recorded through the same repository call. + failed, err := runtime.repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: failedProvider, CopyIndex: 0, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + if err := runtime.repos.Contents.MarkDataSetFailed(t.Context(), failed.ID, "creation outcome unknown"); err != nil { + t.Fatalf("MarkDataSetFailed: %v", err) + } + + taskRow, created, err := runtime.service.Enqueue(t.Context(), taskengine.EnqueueRequest{ + Type: model.TaskTypeBucketProvision, IdempotencyKey: bucketlifecycle.ProvisionKey(bucket.ID, bucket.DefaultCopies), + Input: bucketlifecycle.ProvisionInput{BucketID: bucket.ID}, SubjectType: "bucket", SubjectKey: fmt.Sprint(bucket.ID), + }) + if err != nil || !created { + t.Fatalf("enqueue bucket provision task = %#v created=%v err=%v", taskRow, created, err) + } + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusCompleted + }) + + stored, err := runtime.repos.Buckets.GetByID(t.Context(), bucket.ID) + if err != nil || stored == nil || stored.Status != model.BucketStatusReady { + t.Fatalf("bucket after recovery = %#v err=%v, want ready", stored, err) + } + if !excludedFailedProvider.Load() { + t.Fatal("selection did not exclude the failed generation's provider") + } + bindings, err := runtime.repos.Contents.ListDataSetBindings(t.Context(), bucket.ID) + if err != nil { + t.Fatalf("list data set bindings: %v", err) + } + current := 0 + for i := range bindings { + binding := &bindings[i] + if !binding.IsCurrent { + continue + } + current++ + if binding.ProviderID.String() != healthyProvider.String() || binding.Status != model.StorageDataSetStatusReady { + t.Fatalf("current binding = provider:%s status:%s, want the healthy provider ready", binding.ProviderID, binding.Status) + } + } + if current != 1 { + t.Fatalf("current bindings = %d, want exactly 1", current) + } +} + +// A creation the chain refused leaves no data set, so the generation is removed +// and its provider becomes available to the bucket again. Keeping it would +// reserve that provider forever: a failed generation can neither be retired nor +// replaced, and the unique index covers every non-retired row. +func TestDataSetEnsureReleasesTheProviderWhenCreationIsRejected(t *testing.T) { + sequence := storedObjectSequence.Add(1) + providerID := testOnChainID(t, 26000+sequence) + target := &testutil.MockStorageTarget{ + ProviderIDValue: providerID.SDK(), + WaitDataSetFunc: func(context.Context, storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) { + return nil, pdp.ErrTxRejected + }, + } + storageClient := &testutil.MockStorageClient{ + OpenProviderTargetFunc: func(context.Context, sdktypes.BigInt, storage.NewProviderContextOptions) (synapse.ProviderTarget, error) { + return target, nil + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: storageClient, policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + ctx := t.Context() + bucket := &model.Bucket{ + Name: fmt.Sprintf("rejected-%d", sequence), Status: model.BucketStatusProvisioning, + DefaultCopies: 1, MinimumDurableCopies: 1, + } + if err := runtime.repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("create bucket: %v", err) + } + binding, err := runtime.repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: providerID, CopyIndex: 0, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + // The submission was recorded, so recovery resumes by waiting on it. + clientDataSetID := testOnChainID(t, 27000+sequence) + if err := runtime.repos.Contents.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ + ID: binding.ID, TransactionID: "0xrejected", StatusURL: "https://provider.example/status", + ClientDataSetID: &clientDataSetID, + }); err != nil { + t.Fatalf("MarkDataSetCreating: %v", err) + } + ensureTask, _, err := runtime.service.EnqueueTx(ctx, taskengine.EnqueueRequest{ + Type: model.TaskTypeStorageDataSetEnsure, IdempotencyKey: storagepipeline.DataSetEnsureKey(binding.ID), + Input: storagepipeline.DataSetInput{DataSetID: binding.ID}, + SubjectType: "storage_data_set", SubjectKey: fmt.Sprint(binding.ID), + }, func(ctx context.Context, repos *repository.Repositories, row *model.Task, _ bool) error { + return repos.Contents.BindDataSetEnsureTask(ctx, binding.ID, row.ID) + }) + if err != nil { + t.Fatalf("enqueue data set ensure: %v", err) + } + + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + waitForTask(t, runtime.repos, ensureTask.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + + ended, err := runtime.repos.Contents.GetDataSetBindingByID(ctx, binding.ID) + if err != nil || ended == nil || ended.Status != model.StorageDataSetStatusRetired { + t.Fatalf("rejected generation = %#v err=%v, want it retired rather than deleted", ended, err) + } + // The provider is what retirement buys back. + reused, err := runtime.repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: providerID, CopyIndex: 0, + }) + if err != nil || reused == nil || reused.ID == binding.ID { + t.Fatalf("rebinding the freed provider = %#v err=%v", reused, err) + } +} + +// An unknown creation outcome must not terminate the copies bound to the +// generation. Retrying the ensure task rediscovers a data set the provider did +// create but never reported, and continuation only picks up copies that are +// still in transfer states — so failing them here would destroy the recovery +// that keeping the row is for. +func TestDataSetEnsureKeepsCopiesWhenTheCreationOutcomeIsUnknown(t *testing.T) { + sequence := storedObjectSequence.Add(1) + providerID := testOnChainID(t, 28000+sequence) + storageClient := &testutil.MockStorageClient{ + OpenProviderTargetFunc: func(context.Context, sdktypes.BigInt, storage.NewProviderContextOptions) (synapse.ProviderTarget, error) { + return &testutil.MockStorageTarget{ProviderIDValue: providerID.SDK()}, nil + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: storageClient, policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + ctx := t.Context() + bucket := &model.Bucket{ + Name: fmt.Sprintf("unknown-%d", sequence), Status: model.BucketStatusActive, + DefaultCopies: 1, MinimumDurableCopies: 1, + } + if err := runtime.repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("create bucket: %v", err) + } + content, err := runtime.repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, ContentSize: 11, + Checksum: testutil.StorageChecksum(fmt.Sprintf("unknown-checksum-%d", sequence)), RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("ensure content: %v", err) + } + version := &model.ObjectVersion{ + VersionID: model.NewVersionID(), BucketID: bucket.ID, Key: "unknown.bin", ContentID: &content.ID, + Size: 11, ETag: fmt.Sprintf("unknown-etag-%d", sequence), ContentType: "application/octet-stream", + } + if _, err := runtime.repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + t.Fatalf("create object version: %v", err) + } + binding, err := runtime.repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: providerID, CopyIndex: 0, CreatedByContentID: content.ID, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + if err := runtime.repos.Contents.CreateUploadCopiesForBindings(ctx, content.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: binding.ID, CopyIndex: 0, ProviderID: providerID, + TransferMethod: model.StorageCopyTransferMethodIngress, + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings: %v", err) + } + ensureTask, _, err := runtime.service.EnqueueTx(ctx, taskengine.EnqueueRequest{ + Type: model.TaskTypeStorageDataSetEnsure, IdempotencyKey: storagepipeline.DataSetEnsureKey(binding.ID), + Input: storagepipeline.DataSetInput{DataSetID: binding.ID}, + SubjectType: "storage_data_set", SubjectKey: fmt.Sprint(binding.ID), + }, func(ctx context.Context, repos *repository.Repositories, row *model.Task, _ bool) error { + return repos.Contents.BindDataSetEnsureTask(ctx, binding.ID, row.ID) + }) + if err != nil { + t.Fatalf("enqueue data set ensure: %v", err) + } + // A creation was attempted long enough ago that the outcome is given up on, + // and no transaction was ever learned. + attempted := time.Now().UTC().Add(-30 * time.Minute).Format(time.RFC3339Nano) + if _, err := runtime.db.NewUpdate(). + Model((*model.TaskPayload)(nil)). + Set("checkpoint_json = ?", fmt.Sprintf(`{"attempted_at":%q}`, attempted)). + Where("task_id = ?", ensureTask.ID). + Exec(ctx); err != nil { + t.Fatalf("write attempted checkpoint: %v", err) + } + + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + failed := waitForTask(t, runtime.repos, ensureTask.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + if failed.FailureReason == nil || *failed.FailureReason != "dataset_creation_unknown" { + t.Fatalf("failure reason = %v, want dataset_creation_unknown", failed.FailureReason) + } + + kept, err := runtime.repos.Contents.GetDataSetBindingByID(ctx, binding.ID) + if err != nil || kept == nil || kept.Status != model.StorageDataSetStatusFailed { + t.Fatalf("unknown generation = %#v err=%v, want it kept as failed", kept, err) + } + // The copy stays in a transfer state, which is what continuation picks up. + incomplete, err := runtime.repos.Contents.ListIncompleteCopiesForDataSet(ctx, binding.ID) + if err != nil || len(incomplete) != 1 { + t.Fatalf("incomplete copies = %#v err=%v, want the bound copy still recoverable", incomplete, err) + } +} diff --git a/internal/worker/task_lease.go b/internal/worker/task_lease.go deleted file mode 100644 index 8947fa5..0000000 --- a/internal/worker/task_lease.go +++ /dev/null @@ -1,68 +0,0 @@ -package worker - -import ( - "context" - "log/slog" - "time" - - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" -) - -const ( - taskLeaseOperationTimeout = 2 * time.Second -) - -func startTaskLeaseRenewal(logger *slog.Logger, repos *repository.Repositories, task *model.Task, leaseTTL time.Duration) func() { - if repos == nil || repos.Tasks == nil || task == nil || task.ID == 0 || leaseTTL <= 0 { - return func() {} - } - taskID := task.ID - renewCtx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - defer close(done) - ticker := time.NewTicker(taskLeaseRenewInterval(leaseTTL)) - defer ticker.Stop() - for { - select { - case <-renewCtx.Done(): - return - case <-ticker.C: - opCtx, opCancel := context.WithTimeout(context.Background(), taskLeaseOperationTimeout) - err := repos.Tasks.RenewLease(opCtx, task, leaseTTL) - opCancel() - if err != nil && logger != nil { - logger.Warn("failed to renew task lease", "taskID", taskID, "error", err) - } - } - } - }() - return func() { - cancel() - <-done - } -} - -func taskLeaseRenewInterval(leaseTTL time.Duration) time.Duration { - if leaseTTL <= 0 { - return time.Second - } - interval := leaseTTL / 3 - if interval <= 0 { - return leaseTTL - } - return interval -} - -func releaseTaskOnWorkerShutdown(ctx context.Context, logger *slog.Logger, repos *repository.Repositories, task *model.Task) { - if ctx.Err() == nil || repos == nil || repos.Tasks == nil || task == nil || task.ID == 0 { - return - } - taskID := task.ID - opCtx, cancel := context.WithTimeout(context.Background(), taskLeaseOperationTimeout) - defer cancel() - if err := repos.Tasks.ReleaseRunning(opCtx, task); err != nil && logger != nil { - logger.Debug("skipped task release on worker shutdown", "taskID", taskID, "error", err) - } -} diff --git a/internal/worker/task_lease_test.go b/internal/worker/task_lease_test.go deleted file mode 100644 index 859c359..0000000 --- a/internal/worker/task_lease_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package worker - -import ( - "context" - "testing" - "time" - - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/testutil" -) - -type renewLeaseNotifyRepo struct { - repository.TaskRepository - renewed chan struct{} -} - -func (r *renewLeaseNotifyRepo) RenewLease(ctx context.Context, task *model.Task, leaseDuration time.Duration) error { - err := r.TaskRepository.RenewLease(ctx, task, leaseDuration) - if err == nil { - select { - case r.renewed <- struct{}{}: - default: - } - } - return err -} - -func TestTaskLeaseRenewalExtendsLeaseUntilStopped(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - task := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "object", - RefID: 1, - RefVersionID: "01J0000000000000000000LEASE", - IdempotencyKey: "upload:lease-renewal", - Status: model.TaskStatusQueued, - } - if err := repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create: %v", err) - } - claimed, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, 30*time.Millisecond) - if err != nil { - t.Fatalf("ClaimReady: %v", err) - } - if claimed == nil || claimed.LeaseUntil == nil { - t.Fatal("expected claimed task with lease") - } - oldLeaseUntil := *claimed.LeaseUntil - - notify := &renewLeaseNotifyRepo{ - TaskRepository: repos.Tasks, - renewed: make(chan struct{}, 1), - } - repos.Tasks = notify - - stop := startTaskLeaseRenewal(nil, repos, claimed, 30*time.Millisecond) - defer stop() - - select { - case <-notify.renewed: - case <-time.After(300 * time.Millisecond): - t.Fatal("lease renewal was not observed") - } - - got, err := repos.Tasks.GetByID(ctx, claimed.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.LeaseUntil == nil || !got.LeaseUntil.After(oldLeaseUntil) { - t.Fatal("lease_until was not renewed") - } -} diff --git a/internal/worker/task_status.go b/internal/worker/task_status.go deleted file mode 100644 index a231e3b..0000000 --- a/internal/worker/task_status.go +++ /dev/null @@ -1,42 +0,0 @@ -package worker - -import ( - "context" - "log/slog" - - "github.com/strahe/synaps3/internal/admin" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" -) - -func scheduleTaskRetry(ctx context.Context, repos *repository.Repositories, task *model.Task, workerName string, logger *slog.Logger, err error) model.TaskStatus { - if repos == nil || repos.Tasks == nil || task == nil || err == nil { - return "" - } - status, retryErr := repos.Tasks.ScheduleRetryRunning(ctx, task, err.Error(), retryDelay(task.RetryCount)) - if retryErr != nil { - if logger != nil { - logger.Error("failed to schedule task retry", "error", retryErr) - } - return "" - } - if status == model.TaskStatusExhausted { - admin.TasksExhaustedTotal.WithLabelValues(workerName, string(task.Type)).Inc() - } - return status -} - -func completeWorkerTask(ctx context.Context, repos *repository.Repositories, task *model.Task, workerName string, logger *slog.Logger) bool { - if repos == nil || repos.Tasks == nil || task == nil { - return false - } - if err := repos.Tasks.Complete(ctx, task); err != nil { - if logger != nil { - logger.Error("failed to complete task", "taskID", task.ID, "error", err) - } - admin.WorkerTasksProcessed.WithLabelValues(workerName, "failure").Inc() - return false - } - admin.WorkerTasksProcessed.WithLabelValues(workerName, "success").Inc() - return true -} diff --git a/internal/worker/testutil_test.go b/internal/worker/testutil_test.go deleted file mode 100644 index 8b6be0f..0000000 --- a/internal/worker/testutil_test.go +++ /dev/null @@ -1,178 +0,0 @@ -package worker_test - -import ( - "context" - "strconv" - "testing" - - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/cacheaccess" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/state" - "github.com/strahe/synaps3/internal/testutil" - "github.com/uptrace/bun" -) - -// testWorkerEnv holds all components needed to test workers. -type testWorkerEnv struct { - repos *repository.Repositories - cache cache.Cache - cacheGate *cacheaccess.Gate - accessTracker *cacheaccess.Tracker - sm *state.Machine - storage *testutil.MockStorageClient - db *bun.DB -} - -// newTestWorkerEnv constructs a test environment with file-backed SQLite -// and a real filesystem cache. -func newTestWorkerEnv(t *testing.T) *testWorkerEnv { - t.Helper() - db := testutil.NewTestFileDB(t) - repos := repository.NewRepositories(db) - fsCache := newWorkerTestCache(t, 1<<30) - sm := state.NewObjectStateMachine() - sc := &testutil.MockStorageClient{} - cacheGate := cacheaccess.NewGate() - accessTracker := cacheaccess.NewTracker(cacheaccess.DefaultPersistenceInterval, repos.Objects) - - return &testWorkerEnv{ - repos: repos, - cache: fsCache, - cacheGate: cacheGate, - accessTracker: accessTracker, - sm: sm, - storage: sc, - db: db, - } -} - -// newTestWorkerEnvWithMockCache constructs a test environment with mock cache. -func newTestWorkerEnvWithMockCache(t *testing.T, mc *testutil.MockCache) *testWorkerEnv { - t.Helper() - db := testutil.NewTestFileDB(t) - repos := repository.NewRepositories(db) - sm := state.NewObjectStateMachine() - sc := &testutil.MockStorageClient{} - cacheGate := cacheaccess.NewGate() - accessTracker := cacheaccess.NewTracker(cacheaccess.DefaultPersistenceInterval, repos.Objects) - - return &testWorkerEnv{ - repos: repos, - cache: mc, - cacheGate: cacheGate, - accessTracker: accessTracker, - sm: sm, - storage: sc, - db: db, - } -} - -// stubWorker is a minimal Worker implementation for manager-level tests. -type stubWorker struct { - name string - isHealthy bool - run func(context.Context) error -} - -func (s *stubWorker) Name() string { return s.name } -func (s *stubWorker) Run(ctx context.Context) error { - if s.run != nil { - return s.run(ctx) - } - return nil -} -func (s *stubWorker) Healthy() bool { return s.isHealthy } - -func taskPayloadInt64ForTest(payload map[string]interface{}, key string) int64 { - raw, ok := payload[key] - if !ok { - return 0 - } - switch v := raw.(type) { - case int: - return int64(v) - case int64: - return v - case float64: - return int64(v) - default: - return 0 - } -} - -func newWorkerTestCache(t *testing.T, maxBytes int64) cache.Cache { - t.Helper() - dir := t.TempDir() - c, err := cache.NewFilesystem(dir, maxBytes) - if err != nil { - t.Fatalf("creating test cache: %v", err) - } - return c -} - -func acceptWorkerVersionUpload(t *testing.T, env *testWorkerEnv, versionID string, pieceCID string, retrievalURL string) *model.StorageUpload { - t.Helper() - ctx := context.Background() - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("get version for upload accept: version=%v err=%v", version, err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: version.BucketID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("start upload attempt: %v", err) - } - providerID := onChainID(t, strconv.FormatInt(100+upload.ID, 10)) - dataSetID := onChainID(t, strconv.FormatInt(1000+upload.ID, 10)) - pieceID := onChainIDPtr(t, strconv.FormatInt(2000+upload.ID, 10)) - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: version.BucketID, - ProviderID: providerID, - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("ensure dataset binding: %v", err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, UploadID: upload.ID, DataSetID: dataSetID}); err != nil { - t.Fatalf("mark dataset ready: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: providerID, - }}); err != nil { - t.Fatalf("create upload copy: %v", err) - } - if err := env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: pieceCID, - PieceID: pieceID, - RetrievalURL: retrievalURL, - }); err != nil { - t.Fatalf("mark copy committed: %v", err) - } - if _, err := env.repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: version.BucketID, - ContentSize: version.Size, - Checksum: version.Checksum, - }); err != nil { - t.Fatalf("bind readable upload: %v", err) - } - if finalized, _, err := env.repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}); err != nil { - t.Fatalf("finalize upload: %v", err) - } else if !finalized { - t.Fatal("finalize upload = false, want true") - } - return upload -} diff --git a/internal/worker/uploader.go b/internal/worker/uploader.go deleted file mode 100644 index 4440801..0000000 --- a/internal/worker/uploader.go +++ /dev/null @@ -1,3409 +0,0 @@ -package worker - -import ( - "context" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "log/slog" - "math/rand" - "os" - "sort" - "strconv" - "strings" - "sync" - "time" - - "github.com/ipfs/go-cid" - "github.com/strahe/synaps3/internal/admin" - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/objectlimits" - "github.com/strahe/synaps3/internal/state" - "github.com/strahe/synaps3/internal/storagecommit" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/strahe/synaps3/internal/synapse" - idtypes "github.com/strahe/synaps3/internal/types" - "github.com/strahe/synapse-go/pdp" - "github.com/strahe/synapse-go/storage" - sdktypes "github.com/strahe/synapse-go/types" -) - -const ( - terminalFailureCleanupTimeout = 5 * time.Second - uploadFundingWaitDelay = time.Minute - uploadDependencyWaitDelay = time.Minute - storageCommitPollDelay = 5 * time.Second - storageCommitObservationDelay = time.Minute - storageCommitAttentionDelay = 24 * time.Hour -) - -var ( - errCommitRejected = errors.New("commit transaction rejected") - errDataSetCreationIncomplete = errors.New("data set creation submission is incomplete") -) - -var submittedCommitRequestTimeout = storagecommit.DefaultRequestTimeout - -// Uploader claims upload tasks, persists upload provenance, and accepts complete -// uploads for object versions. -type Uploader struct { - repos *repository.Repositories - cache cache.Cache - storage synapse.StorageClient - statusChecker *synapse.PDPStatusChecker - wallet synapse.WalletQuerier // optional; nil skips balance pre-check - stateMachine *state.Machine - evictionPolicy cache.EvictionPolicy - evictMaxRetries int - targetCopies int - eventPublisher admin.EventPublisher - replacementMaxRetries int - concurrency int - pollInterval time.Duration - leaseTTL time.Duration - logger *slog.Logger - *livenessTracker -} - -const ( - defaultEvictMaxRetries = 3 - uploadPollJitterDivisor = 5 - uploadProgressTimeout = 2 * time.Second - - uploadStagePrepare = "prepare_upload" - uploadStageEnsureDataSet = "ensure_dataset" - uploadStageIngressStore = "ingress_store" - uploadStageIngressCommit = "ingress_commit" - uploadStagePeerPull = "peer_pull" - uploadStagePeerCommit = "peer_commit" - uploadStageRepairReplica = "repair_replica" - uploadStageReplaceProvider = storagereplacement.StageMigrate -) - -// UploaderOption configures uploader behavior. -type UploaderOption func(*Uploader) - -// WithEvictMaxRetries configures max retries for cache eviction tasks created after upload. -func WithEvictMaxRetries(maxRetries int) UploaderOption { - return func(u *Uploader) { - u.evictMaxRetries = maxRetries - } -} - -func WithEventPublisher(publisher admin.EventPublisher) UploaderOption { - return func(u *Uploader) { - u.eventPublisher = publisher - } -} - -// WithPDPStatusChecker configures the client used to resume submitted commits. -func WithPDPStatusChecker(checker *synapse.PDPStatusChecker) UploaderOption { - return func(u *Uploader) { - if checker != nil { - u.statusChecker = checker - } - } -} - -// WithProviderReplacementMaxRetries sets the provider replacement retry limit. -func WithProviderReplacementMaxRetries(maxRetries int) UploaderOption { - return func(u *Uploader) { - u.replacementMaxRetries = maxRetries - } -} - -// providerEvidenceContext lets monotonic evidence writes outlive lease cancellation. -func providerEvidenceContext(parent context.Context) (context.Context, context.CancelFunc) { - return context.WithTimeout(context.WithoutCancel(parent), taskLeaseOperationTimeout) -} - -func boundedTargetCopies(copies int) int { - return model.ClampStorageCopies(copies) -} - -// NewUploader creates a new upload worker. -func NewUploader(repos *repository.Repositories, c cache.Cache, sc synapse.StorageClient, wallet synapse.WalletQuerier, sm *state.Machine, evictionPolicy cache.EvictionPolicy, targetCopies int, concurrency int, pollInterval time.Duration, logger *slog.Logger, opts ...UploaderOption) *Uploader { - u := &Uploader{ - repos: repos, - cache: c, - storage: sc, - statusChecker: synapse.NewPDPStatusChecker(synapse.PDPStatusCheckerOptions{Timeout: submittedCommitRequestTimeout}), - wallet: wallet, - stateMachine: sm, - evictionPolicy: evictionPolicy, - evictMaxRetries: defaultEvictMaxRetries, - targetCopies: boundedTargetCopies(targetCopies), - replacementMaxRetries: defaultUploadMaxRetries, - concurrency: concurrency, - pollInterval: pollInterval, - leaseTTL: 10 * time.Minute, - logger: logger, - livenessTracker: newLivenessTracker(pollInterval), - } - for _, opt := range opts { - opt(u) - } - return u -} - -type uploadProgressReporter struct { - ctx context.Context - repos *repository.Repositories - publisher admin.EventPublisher - logger *slog.Logger - uploadID int64 - taskID int64 - versionID string - bucketName string - objectKey string - attempt int - totalBytes int64 - flushInterval time.Duration - - mu sync.Mutex - lastFlush time.Time - pendingBytes int64 - pending bool - pendingTimer *time.Timer -} - -func (u *Uploader) beginIngressProgressReporter(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, uploadID int64, logger *slog.Logger) *uploadProgressReporter { - if u == nil || u.repos == nil || u.repos.Uploads == nil || uploadID == 0 { - return nil - } - upload, err := u.repos.Uploads.BeginIngressStoreProgress(ctx, uploadID) - if err != nil { - logger.Warn("failed to begin ingress upload progress", "uploadID", uploadID, "error", err) - return nil - } - reporter := &uploadProgressReporter{ - ctx: ctx, - repos: u.repos, - publisher: u.eventPublisher, - logger: logger, - uploadID: uploadID, - versionID: upload.SourceVersionID, - attempt: upload.IngressStoreAttempt, - totalBytes: upload.ContentSize, - flushInterval: time.Second, - } - if task != nil { - reporter.taskID = task.ID - if reporter.versionID == "" { - reporter.versionID = task.RefVersionID - } - } - if version != nil { - if reporter.versionID == "" { - reporter.versionID = version.VersionID - } - reporter.objectKey = version.Key - if reporter.totalBytes == 0 { - reporter.totalBytes = version.Size - } - } - if bucket != nil { - reporter.bucketName = bucket.Name - } - reporter.record(0, false) - return reporter -} - -func (r *uploadProgressReporter) OnProgress(bytesUploaded int64) { - if r == nil || r.attempt <= 0 { - return - } - now := time.Now() - r.mu.Lock() - if r.flushInterval <= 0 || r.lastFlush.IsZero() || now.Sub(r.lastFlush) >= r.flushInterval { - r.cancelPendingLocked() - r.lastFlush = now - r.mu.Unlock() - go r.record(bytesUploaded, false) - return - } - r.pendingBytes = bytesUploaded - r.pending = true - if r.pendingTimer == nil { - delay := r.flushInterval - now.Sub(r.lastFlush) - r.pendingTimer = time.AfterFunc(delay, r.flushPendingProgress) - } - r.mu.Unlock() -} - -func (r *uploadProgressReporter) Flush(bytesUploaded int64, done bool) { - if r == nil || r.attempt <= 0 { - return - } - r.mu.Lock() - r.cancelPendingLocked() - r.mu.Unlock() - r.record(bytesUploaded, done) -} - -func (r *uploadProgressReporter) flushPendingProgress() { - r.mu.Lock() - if !r.pending { - r.pendingTimer = nil - r.mu.Unlock() - return - } - bytesUploaded := r.pendingBytes - r.pending = false - r.pendingTimer = nil - r.lastFlush = time.Now() - r.mu.Unlock() - r.record(bytesUploaded, false) -} - -func (r *uploadProgressReporter) cancelPendingLocked() { - r.pending = false - if r.pendingTimer != nil { - r.pendingTimer.Stop() - r.pendingTimer = nil - } -} - -func (r *uploadProgressReporter) record(bytesUploaded int64, done bool) { - if r == nil || r.repos == nil || r.repos.Uploads == nil { - return - } - ctx, cancel := r.recordContext() - defer cancel() - upload, err := r.repos.Uploads.RecordIngressStoreProgress(ctx, repository.RecordIngressStoreProgressInput{ - UploadID: r.uploadID, - Attempt: r.attempt, - BytesUploaded: bytesUploaded, - }) - if err != nil { - if r.logger != nil { - r.logger.Warn("failed to record ingress upload progress", "uploadID", r.uploadID, "attempt", r.attempt, "error", err) - } - return - } - if r.publisher == nil || upload == nil || upload.ProgressUpdatedAt == nil { - return - } - r.publisher.Publish("upload_progress_updated", map[string]any{ - "upload_id": r.uploadID, - "task_id": nullableTaskID(r.taskID), - "version_id": r.versionID, - "bucket_name": r.bucketName, - "object_key": r.objectKey, - "progress": uploadProgressEventPayload(upload, done), - }) -} - -func (r *uploadProgressReporter) recordContext() (context.Context, context.CancelFunc) { - baseCtx := context.Background() - if r != nil && r.ctx != nil { - baseCtx = r.ctx - } - return context.WithTimeout(baseCtx, uploadProgressTimeout) -} - -func nullableTaskID(taskID int64) any { - if taskID == 0 { - return nil - } - return taskID -} - -func uploadProgressEventPayload(upload *model.StorageUpload, done bool) map[string]any { - uploaded := upload.IngressBytesTransferred - if uploaded < 0 { - uploaded = 0 - } - total := upload.ContentSize - if total < 0 { - total = 0 - } - if uploaded > total { - uploaded = total - } - progress := map[string]any{ - "scope": "ingress_store", - "attempt": upload.IngressStoreAttempt, - "uploaded_bytes": uploaded, - "total_bytes": total, - "done": done || (total > 0 && uploaded >= total), - "updated_at": upload.ProgressUpdatedAt.Format(time.RFC3339), - } - if percent := model.UploadProgressPercent(uploaded, total); percent != nil { - progress["percent"] = *percent - } - return progress -} - -func (u *Uploader) Name() string { return "uploader" } - -func (u *Uploader) finalizeUploadInput(uploadID int64) repository.FinalizeUploadInput { - return repository.NewFinalizeUploadInput( - uploadID, - u.evictionPolicy.EnqueuesAfterUploadEviction(), - u.evictMaxRetries, - ) -} - -func (u *Uploader) Run(ctx context.Context) error { - var wg sync.WaitGroup - for range u.concurrency { - wg.Add(1) - go func() { - defer wg.Done() - u.runSlot(ctx) - }() - } - - wg.Wait() - return ctx.Err() -} - -func (u *Uploader) runSlot(ctx context.Context) { - if !sleepUntilNextUploadPoll(ctx, u.pollInterval) { - return - } - - for { - if ctx.Err() != nil { - return - } - - u.recordTick() - task, err := u.repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, u.leaseTTL) - if err != nil { - if ctx.Err() != nil { - return - } - u.logger.Error("claiming upload task", "error", err) - if !sleepUntilNextUploadPoll(ctx, u.pollInterval) { - return - } - continue - } - if task == nil { - if !sleepUntilNextUploadPoll(ctx, u.pollInterval) { - return - } - continue - } - - u.recordWorkStarted() - func() { - defer u.recordWorkFinished() - stopLeaseRenewal := startTaskLeaseRenewal(u.logger, u.repos, task, u.leaseTTL) - defer stopLeaseRenewal() - u.processTask(ctx, task) - }() - releaseTaskOnWorkerShutdown(ctx, u.logger, u.repos, task) - } -} - -func sleepUntilNextUploadPoll(ctx context.Context, interval time.Duration) bool { - timer := time.NewTimer(uploadPollSleepDuration(interval)) - defer timer.Stop() - - select { - case <-ctx.Done(): - return false - case <-timer.C: - return true - } -} - -func uploadPollSleepDuration(interval time.Duration) time.Duration { - if interval <= 0 { - return interval - } - - maxJitter := interval / uploadPollJitterDivisor - if maxJitter <= 0 { - return interval - } - return interval + time.Duration(rand.Int63n(int64(maxJitter)+1)) -} - -// Healthy returns true if the worker has ticked recently. -func (u *Uploader) Healthy() bool { return u.healthy() } - -func (u *Uploader) processTask(ctx context.Context, task *model.Task) { - start := time.Now() - defer func() { - admin.WorkerTaskDuration.WithLabelValues("uploader").Observe(time.Since(start).Seconds()) - }() - - logger := u.logger.With("taskID", task.ID, "objectID", task.RefID, "versionID", task.RefVersionID) - - if u.storage == nil { - logger.Warn("storage client not configured, failing task") - _ = u.repos.Tasks.FailRunning(ctx, task, "storage client not configured") - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() - return - } - switch uploadTaskStage(task) { - case uploadStageRepairReplica: - u.processReplicaRepairTask(ctx, task, logger) - return - case uploadStageReplaceProvider: - // The coordinator tracks a bucket slot rather than one object version, - // so it is dispatched before any version lookup. - u.processReplacementTask(ctx, task, logger) - return - } - - version, err := u.repos.Objects.GetVersionByID(ctx, task.RefVersionID) - if err != nil || version == nil { - logger.Warn("object version not found for upload task", "error", err) - _ = u.repos.Tasks.FailRunning(ctx, task, "object not found") - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() - return - } - if task.ClaimedAt == nil { - return - } - uploadID, _ := taskUploadID(task) - err = u.repos.Uploads.AcquireUploadTask(ctx, repository.AcquireUploadTaskInput{ - TaskID: task.ID, - TaskClaimedAt: *task.ClaimedAt, - UploadID: uploadID, - VersionID: version.VersionID, - }) - if err != nil { - switch { - case errors.Is(err, repository.ErrUploadTaskCancelled): - completeWorkerTask(ctx, u.repos, task, "uploader", logger) - case errors.Is(err, repository.ErrTaskClaimLost): - default: - u.handleTaskFailure(ctx, task, logger, "acquire upload task", err) - } - return - } - - bucket, err := u.repos.Buckets.GetByID(ctx, version.BucketID) - if err != nil || bucket == nil { - logger.Error("bucket not found", "bucketID", version.BucketID, "error", err) - _ = u.repos.Tasks.FailRunning(ctx, task, "bucket not found") - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() - return - } - defer u.publishUploadStateChanged(task, version, bucket) - - if durableObjectState(version.State) { - u.processDurableUploadTask(ctx, task, version, bucket, logger) - return - } - - u.processStagedTask(ctx, task, version, bucket, uploadTaskStage(task), logger) -} - -func durableObjectState(state model.ObjectState) bool { - return state == model.ObjectStateStored || state == model.ObjectStateCacheEvicted -} - -func (u *Uploader) processDurableUploadTask( - ctx context.Context, - task *model.Task, - version *model.ObjectVersion, - bucket *model.Bucket, - logger *slog.Logger, -) { - uploadID, ok := taskUploadID(task) - if !ok && version.StorageUploadID != nil { - uploadID = *version.StorageUploadID - ok = uploadID > 0 - } - if !ok { - completeWorkerTask(ctx, u.repos, task, "uploader", logger) - return - } - upload, err := u.repos.Uploads.GetByID(ctx, uploadID) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "load durable upload", err) - return - } - if upload == nil || upload.Status == model.StorageUploadStatusComplete { - completeWorkerTask(ctx, u.repos, task, "uploader", logger) - return - } - - stage := uploadTaskStage(task) - if stage == uploadStagePrepare { - u.prepareReadableUploadRepair(ctx, task, version, bucket, uploadID, logger) - return - } - uploadID, copyIndex, err := uploadStageIDs(task, true) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "parse durable upload task payload", err) - return - } - binding, err := u.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, copyIndex) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "load durable upload data set", err) - return - } - if binding == nil { - u.prepareReadableUploadRepair(ctx, task, version, bucket, uploadID, logger) - return - } - if binding.Status == model.StorageDataSetStatusFailed && !dataSetBindingEstablished(binding) { - u.prepareReadableUploadRepair(ctx, task, version, bucket, uploadID, logger) - return - } - if binding.Status == model.StorageDataSetStatusPending || binding.Status == model.StorageDataSetStatusCreating { - u.ensureUploadDataSet(ctx, task, version, bucket, uploadID, copyIndex, logger) - return - } - if binding.Status == model.StorageDataSetStatusReady { - u.processStagedTask(ctx, task, version, bucket, stage, logger) - return - } - queued, err := ensureReplicaRepairTask(ctx, u.repos, binding, task.MaxRetries) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "handoff durable upload repair", err) - return - } - if !queued { - u.waitForStorageDependency(ctx, task, logger, "Waiting for in-place replica recovery") - return - } - completeWorkerTask(ctx, u.repos, task, "uploader", logger) -} - -func (u *Uploader) publishUploadStateChanged(task *model.Task, version *model.ObjectVersion, bucket *model.Bucket) { - if u == nil || u.eventPublisher == nil || task == nil { - return - } - payload := map[string]any{ - "task_id": task.ID, - "version_id": task.RefVersionID, - } - if uploadID, err := payloadInt64(task.Payload, "upload_id"); err == nil && uploadID != 0 { - payload["upload_id"] = uploadID - } - if version != nil { - payload["version_id"] = version.VersionID - payload["object_key"] = version.Key - } - if bucket != nil { - payload["bucket_name"] = bucket.Name - } - u.eventPublisher.Publish("upload_state_changed", payload) -} - -func (u *Uploader) processStagedTask(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, stage string, logger *slog.Logger) { - switch stage { - case uploadStagePrepare: - u.prepareStagedUpload(ctx, task, version, bucket, logger) - case uploadStageEnsureDataSet: - uploadID, copyIndex, err := uploadStageIDs(task, true) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "parse upload task payload", err) - return - } - u.ensureUploadDataSet(ctx, task, version, bucket, uploadID, copyIndex, logger) - case uploadStageIngressStore: - uploadID, copyIndex, err := uploadStageIDs(task, true) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "parse upload task payload", err) - return - } - if u.deferToReplicaRepair(ctx, task, bucket.ID, uploadID, copyIndex, logger) { - return - } - if u.deferToReplacement(ctx, task, bucket.ID, uploadID, copyIndex, logger) { - return - } - u.ingressStore(ctx, task, version, bucket, uploadID, copyIndex, logger) - case uploadStageIngressCommit: - uploadID, copyIndex, err := uploadStageIDs(task, true) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "parse upload task payload", err) - return - } - if u.deferToReplicaRepair(ctx, task, bucket.ID, uploadID, copyIndex, logger) { - return - } - if u.deferToReplacement(ctx, task, bucket.ID, uploadID, copyIndex, logger) { - return - } - u.ingressCommit(ctx, task, version, bucket, uploadID, copyIndex, logger) - case uploadStagePeerPull: - uploadID, copyIndex, err := uploadStageIDs(task, true) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "parse upload task payload", err) - return - } - if u.deferToReplicaRepair(ctx, task, bucket.ID, uploadID, copyIndex, logger) { - return - } - if u.deferToReplacement(ctx, task, bucket.ID, uploadID, copyIndex, logger) { - return - } - u.peerPull(ctx, task, version, bucket, uploadID, copyIndex, logger) - case uploadStagePeerCommit: - uploadID, copyIndex, err := uploadStageIDs(task, true) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "parse upload task payload", err) - return - } - if u.deferToReplicaRepair(ctx, task, bucket.ID, uploadID, copyIndex, logger) { - return - } - if u.deferToReplacement(ctx, task, bucket.ID, uploadID, copyIndex, logger) { - return - } - u.peerCommit(ctx, task, version, bucket, uploadID, copyIndex, logger) - default: - u.handleTaskFailure(ctx, task, logger, "parse upload task payload", fmt.Errorf("unknown upload stage %q", stage)) - } -} - -func (u *Uploader) deferToReplicaRepair(ctx context.Context, task *model.Task, bucketID, uploadID int64, copyIndex int, logger *slog.Logger) bool { - binding, err := u.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucketID, copyIndex) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "load upload data set", err) - return true - } - if binding == nil { - return false - } - repairTask, err := u.repos.Tasks.GetByIdempotencyKey(ctx, replicaRepairTaskKey(binding.ID)) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "check replica repair task", err) - return true - } - if repairTask == nil || repairTask.Status != model.TaskStatusRunning { - return false - } - // A task queued before copy ids were recorded still resolves through its - // replica slot, which now names the current generation. - claimedCopyID := uploadStageCopyID(task) - if claimedCopyID == 0 { - copyRow, err := u.taskUploadCopy(ctx, task, uploadID, copyIndex) - if err != nil || copyRow == nil { - if err == nil { - err = fmt.Errorf("upload copy %d not found", copyIndex) - } - u.handleTaskFailure(ctx, task, logger, "load upload copy for replica repair coordination", err) - return true - } - claimedCopyID = copyRow.ID - } - repairCopyID, err := payloadInt64(repairTask.Payload, replicaRepairCopyIDKey) - if err == nil && repairCopyID != claimedCopyID { - return false - } - if !taskClaimPrecedes(repairTask, task) { - return false - } - u.waitForStorageDependency(ctx, task, logger, "Waiting for in-place replica recovery") - return true -} - -func taskClaimPrecedes(first, second *model.Task) bool { - if first == nil || second == nil || first.ClaimedAt == nil || second.ClaimedAt == nil { - return true - } - if first.ClaimedAt.Equal(*second.ClaimedAt) { - return first.ID < second.ID - } - return first.ClaimedAt.Before(*second.ClaimedAt) -} - -func (u *Uploader) prepareStagedUpload(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, logger *slog.Logger) { - if uploadID, ok := taskUploadID(task); ok { - u.prepareReadableUploadRepair(ctx, task, version, bucket, uploadID, logger) - return - } - if version.State == model.ObjectStateReplicating && version.StorageUploadID != nil { - u.prepareReadableUploadRepair(ctx, task, version, bucket, *version.StorageUploadID, logger) - return - } - if version.State == model.ObjectStateCached { - if err := state.TransitionState(ctx, u.stateMachine, u.repos.Objects, version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - u.handleTaskFailure(ctx, task, logger, "state transition cached→uploading", err) - return - } - } else if version.State != model.ObjectStateUploading { - u.handleTaskFailure(ctx, task, logger, "prepare upload", fmt.Errorf("object state %s is not uploadable", version.State)) - return - } - - targetCopies := u.targetCopiesForBucket(bucket) - upload, err := u.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: version.BucketID, - SourceTaskID: task.ID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: targetCopies, - }) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "start upload attempt", err) - return - } - targetCopies = boundedTargetCopies(upload.RequestedCopies) - plan, planErr := u.ensureBucketProviderBindings(ctx, bucket, upload.ID, targetCopies) - if planErr != nil && !synapse.IsNoProviderCandidates(planErr) { - u.handleTaskFailure(ctx, task, logger, "ensure provider bindings", planErr) - return - } - plan, err = u.preserveInFlightUploadBindings(ctx, upload.ID, plan) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "preserve in-flight provider bindings", err) - return - } - copyInputs, err := u.uploadCopyInputs(ctx, upload.ID, plan.bindings) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "plan upload copy rows", err) - return - } - if err := u.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, copyInputs); err != nil { - u.handleTaskFailure(ctx, task, logger, "create upload copy rows", err) - return - } - for _, input := range copyInputs { - binding := plan.byID[input.StorageDataSetID] - if binding != nil && binding.Status == model.StorageDataSetStatusUnavailable { - if err := u.ensureReplicaRepairTask(ctx, binding, task.MaxRetries); err != nil { - u.handleTaskFailure(ctx, task, logger, "ensure unavailable replica repair", err) - return - } - } - } - if len(plan.bindings) == 0 { - u.waitForStorageDependency(ctx, task, logger, "Waiting for an assigned storage provider") - return - } - fundedBindings, fundingReady := u.ensureUploadFundingReady(ctx, task, version.Size, bucket, upload.ID, &plan, logger) - if !fundingReady { - return - } - ingress, err := u.ensureWritableIngressCopy(ctx, upload.ID, fundedBindings) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "select writable ingress copy", err) - return - } - if ingress == nil { - u.waitForStorageDependency(ctx, task, logger, "Waiting for an assigned storage provider to recover") - return - } - if err := u.enqueueUploadStageForCopy(ctx, task, uploadStageEnsureDataSet, upload.ID, ingress.CopyIndex, model.StorageCopyTransferMethodIngress, ingress.ID); err != nil { - u.handleTaskFailure(ctx, task, logger, "enqueue ingress dataset task", err) - return - } - completeWorkerTask(ctx, u.repos, task, "uploader", logger) -} - -func (u *Uploader) uploadCopyInputs(ctx context.Context, uploadID int64, bindings []model.StorageDataSet) ([]repository.UploadCopyBindingInput, error) { - existingCopies, err := u.repos.Uploads.ListCopies(ctx, uploadID) - if err != nil { - return nil, err - } - existingByIndex := make(map[int]model.StorageUploadCopy, len(existingCopies)) - // A slot can hold a copy per generation during a replacement, so the copy - // that matches a binding is found by data set rather than by slot. - existingByDataSet := make(map[int64]model.StorageUploadCopy, len(existingCopies)) - ingressCopyIndex := -1 - for _, copyRow := range existingCopies { - existingByIndex[copyRow.CopyIndex] = copyRow - if copyRow.StorageDataSetID != nil { - existingByDataSet[*copyRow.StorageDataSetID] = copyRow - } - if copyRow.TransferMethod == model.StorageCopyTransferMethodIngress { - ingressCopyIndex = copyRow.CopyIndex - } - } - if ingressCopyIndex < 0 { - for i := range bindings { - if uploadCanUseDataSetBinding(uploadID, &bindings[i]) { - ingressCopyIndex = bindings[i].CopyIndex - break - } - } - } - if ingressCopyIndex < 0 { - for i := range bindings { - if uploadTracksDataSetBinding(uploadID, &bindings[i]) { - ingressCopyIndex = bindings[i].CopyIndex - break - } - } - } - inputs := make([]repository.UploadCopyBindingInput, 0, len(bindings)) - for _, binding := range bindings { - existingCopy, hasCopy := existingByDataSet[binding.ID] - if !hasCopy { - existingCopy, hasCopy = existingByIndex[binding.CopyIndex] - } - if !hasCopy && !uploadTracksDataSetBinding(uploadID, &binding) { - continue - } - transferMethod := model.StorageCopyTransferMethodPeerPull - if hasCopy { - transferMethod = existingCopy.TransferMethod - } else if binding.CopyIndex == ingressCopyIndex { - transferMethod = model.StorageCopyTransferMethodIngress - } - inputs = append(inputs, repository.UploadCopyBindingInput{ - StorageDataSetID: binding.ID, - CopyIndex: binding.CopyIndex, - TransferMethod: transferMethod, - ProviderID: binding.ProviderID, - }) - } - return inputs, nil -} - -func taskUploadID(task *model.Task) (int64, bool) { - if task == nil || task.Payload == nil { - return 0, false - } - uploadID, err := payloadInt64(task.Payload, "upload_id") - return uploadID, err == nil && uploadID > 0 -} - -func (u *Uploader) prepareReadableUploadRepair(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, uploadID int64, logger *slog.Logger) { - if (version.State != model.ObjectStateReplicating && !durableObjectState(version.State)) || - version.StorageUploadID == nil || *version.StorageUploadID != uploadID { - u.handleTaskFailure(ctx, task, logger, "prepare upload repair", fmt.Errorf("object state %s is not repairable for upload %d", version.State, uploadID)) - return - } - upload, err := u.repos.Uploads.GetByID(ctx, uploadID) - if err != nil || upload == nil { - if err == nil { - err = fmt.Errorf("storage upload %d not found", uploadID) - } - u.handleTaskFailure(ctx, task, logger, "load repair upload", err) - return - } - if upload.BucketID != version.BucketID || upload.ContentSize != version.Size || upload.Checksum != version.Checksum { - u.handleTaskFailure(ctx, task, logger, "prepare upload repair", fmt.Errorf("upload %d does not match object version %s", uploadID, version.VersionID)) - return - } - readableCopies, err := u.repos.Uploads.ListReadableCommittedCopies(ctx, uploadID) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "list readable repair copies", err) - return - } - if len(readableCopies) == 0 && !version.InCache { - u.waitForStorageDependency(ctx, task, logger, "Waiting for a readable replica or retained cache data") - return - } - finalized, _, err := u.repos.Uploads.FinalizeUploadIfTargetCopiesMet( - ctx, - u.finalizeUploadInput(uploadID), - ) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "finalize repaired upload", err) - return - } - if finalized { - completeWorkerTask(ctx, u.repos, task, "uploader", logger) - return - } - plan, planErr := u.ensureBucketProviderBindings(ctx, bucket, uploadID, boundedTargetCopies(upload.RequestedCopies)) - if planErr != nil && !synapse.IsNoProviderCandidates(planErr) { - u.handleTaskFailure(ctx, task, logger, "ensure repair provider bindings", planErr) - return - } - plan, err = u.preserveInFlightUploadBindings(ctx, uploadID, plan) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "preserve in-flight repair bindings", err) - return - } - inputs, err := u.uploadCopyInputs(ctx, uploadID, plan.bindings) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "plan repair upload copy rows", err) - return - } - if err := u.repos.Uploads.CreateUploadCopiesForBindings(ctx, uploadID, inputs); err != nil { - u.handleTaskFailure(ctx, task, logger, "create repair upload copies", err) - return - } - fundedBindings := make(map[int]*model.StorageDataSet, len(plan.writable)) - fundingCandidates := make([]model.StorageDataSet, 0, len(plan.writable)) - for i := range plan.writable { - binding := plan.byID[plan.writable[i].ID] - if binding == nil { - continue - } - if binding.Status == model.StorageDataSetStatusReady { - fundedBindings[binding.CopyIndex] = binding - continue - } - fundingCandidates = append(fundingCandidates, *binding) - } - if len(fundedBindings) == 0 && len(fundingCandidates) == 0 { - u.waitForStorageDependency(ctx, task, logger, "Waiting for an assigned storage provider") - return - } - deferredContext := false - if len(fundingCandidates) > 0 { - fundingPlan := newBucketBindingPlan(fundingCandidates, uploadID) - candidateBindings, fundingReady := u.ensureUploadFundingReady(ctx, task, version.Size, bucket, uploadID, &fundingPlan, logger) - if !fundingReady { - return - } - for copyIndex, binding := range candidateBindings { - fundedBindings[copyIndex] = binding - } - deferredContext = fundingPlan.deferredContext - } - copies, err := u.repos.Uploads.ListCopies(ctx, uploadID) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "reload repair upload copies", err) - return - } - for i := range copies { - copyRow := &copies[i] - if copyCommitted(copyRow) || copyRow.Status == model.StorageUploadCopyStatusFailed { - continue - } - binding := plan.byCopyIndex[copyRow.CopyIndex] - switch { - case fundedBindings[copyRow.CopyIndex] != nil && uploadCanUseDataSetBinding(uploadID, binding): - if err := u.enqueueUploadStageForCopy(ctx, task, uploadStageEnsureDataSet, uploadID, copyRow.CopyIndex, copyRow.TransferMethod, copyRow.ID); err != nil { - u.handleTaskFailure(ctx, task, logger, "enqueue existing repair copy", err) - return - } - case binding != nil && binding.Status == model.StorageDataSetStatusUnavailable: - if err := u.ensureReplicaRepairTask(ctx, binding, task.MaxRetries); err != nil { - u.handleTaskFailure(ctx, task, logger, "ensure unavailable replica repair", err) - return - } - } - } - if !plan.complete || deferredContext { - u.waitForStorageDependency(ctx, task, logger, "Waiting for an eligible storage provider") - return - } - completeWorkerTask(ctx, u.repos, task, "uploader", logger) -} - -func (u *Uploader) targetCopiesForBucket(bucket *model.Bucket) int { - if bucket != nil && bucket.DefaultCopies != nil { - return boundedTargetCopies(*bucket.DefaultCopies) - } - return boundedTargetCopies(u.targetCopies) -} - -type bucketBindingPlan struct { - bindings []model.StorageDataSet - writable []model.StorageDataSet - byID map[int64]*model.StorageDataSet - byCopyIndex map[int]*model.StorageDataSet - complete bool - deferredContext bool -} - -func newBucketBindingPlan(bindings []model.StorageDataSet, uploadID int64) bucketBindingPlan { - plan := bucketBindingPlan{ - bindings: bindings, - byID: make(map[int64]*model.StorageDataSet, len(bindings)), - byCopyIndex: make(map[int]*model.StorageDataSet, len(bindings)), - } - for i := range plan.bindings { - binding := &plan.bindings[i] - plan.byID[binding.ID] = binding - plan.byCopyIndex[binding.CopyIndex] = binding - if uploadCanUseDataSetBinding(uploadID, binding) { - plan.writable = append(plan.writable, *binding) - } - } - return plan -} - -// preserveInFlightUploadBindings keeps unfinished copy rows on the generation -// they were created for. A provider replacement may have moved the slot to a -// new current generation while funding or task enqueueing was waiting. -func (u *Uploader) preserveInFlightUploadBindings( - ctx context.Context, - uploadID int64, - plan bucketBindingPlan, -) (bucketBindingPlan, error) { - copies, err := u.repos.Uploads.ListCopies(ctx, uploadID) - if err != nil { - return bucketBindingPlan{}, err - } - chosen := make(map[int]model.StorageUploadCopy) - for i := range copies { - copyRow := copies[i] - if copyRow.StorageDataSetID == nil || copyRow.Status == model.StorageUploadCopyStatusFailed || copyCommitted(©Row) { - continue - } - previous, ok := chosen[copyRow.CopyIndex] - if !ok || (copyRow.TransferMethod == model.StorageCopyTransferMethodIngress && previous.TransferMethod != model.StorageCopyTransferMethodIngress) || - (copyRow.TransferMethod == previous.TransferMethod && copyRow.ID < previous.ID) { - chosen[copyRow.CopyIndex] = copyRow - } - } - if len(chosen) == 0 { - return plan, nil - } - authoritative := make(map[int]model.StorageDataSet, len(chosen)) - for copyIndex, copyRow := range chosen { - binding, err := u.repos.Uploads.GetDataSetBindingByID(ctx, *copyRow.StorageDataSetID) - if err != nil { - return bucketBindingPlan{}, err - } - if binding == nil || binding.CopyIndex != copyIndex || - (binding.Status != model.StorageDataSetStatusReady && binding.Status != model.StorageDataSetStatusDraining) { - continue - } - authoritative[copyIndex] = *binding - } - if len(authoritative) == 0 { - return plan, nil - } - bindings := make([]model.StorageDataSet, 0, len(plan.bindings)+len(authoritative)) - seen := make(map[int]struct{}, len(plan.bindings)+len(authoritative)) - for i := range plan.bindings { - binding := plan.bindings[i] - if inFlight, ok := authoritative[binding.CopyIndex]; ok { - binding = inFlight - } - if _, ok := seen[binding.CopyIndex]; ok { - continue - } - seen[binding.CopyIndex] = struct{}{} - bindings = append(bindings, binding) - } - for copyIndex, binding := range authoritative { - if _, ok := seen[copyIndex]; ok { - continue - } - bindings = append(bindings, binding) - } - out := newBucketBindingPlan(bindings, uploadID) - out.complete = plan.complete - out.deferredContext = plan.deferredContext - for i := range out.bindings { - binding := &out.bindings[i] - if binding.Status == model.StorageDataSetStatusDraining { - out.writable = append(out.writable, *binding) - } - } - return out, nil -} - -func (u *Uploader) ensureWritableIngressCopy(ctx context.Context, uploadID int64, fundedBindings map[int]*model.StorageDataSet) (*model.StorageUploadCopy, error) { - copies, err := u.repos.Uploads.ListCopies(ctx, uploadID) - if err != nil { - return nil, err - } - var ingress *model.StorageUploadCopy - for i := range copies { - copyRow := &copies[i] - if copyRow.TransferMethod != model.StorageCopyTransferMethodIngress || copyCommitted(copyRow) { - continue - } - ingress = copyRow - if fundedBindings[copyRow.CopyIndex] != nil { - return ingress, nil - } - break - } - if ingress == nil { - return nil, nil - } - return u.repos.Uploads.ReassignIngressCopy(ctx, uploadID, ingress.CopyIndex) -} - -func (u *Uploader) waitForStorageDependency(ctx context.Context, task *model.Task, logger *slog.Logger, message string) { - if err := u.repos.Tasks.WaitRunning(ctx, task, model.TaskWaitReasonDependency, message, uploadDependencyWaitDelay); err != nil { - logger.Error("failed to wait for storage dependency", "error", err) - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() - return - } - admin.WorkerTasksProcessed.WithLabelValues("uploader", "success").Inc() -} - -func (u *Uploader) ensureBucketProviderBindings(ctx context.Context, bucket *model.Bucket, uploadID int64, targetCopies int) (bucketBindingPlan, error) { - bindings, err := u.repos.Uploads.ListDataSetBindings(ctx, bucket.ID) - if err != nil { - return bucketBindingPlan{}, err - } - existing := make(map[int]model.StorageDataSet, len(bindings)) - targetCopies = boundedTargetCopies(targetCopies) - selected := make([]model.StorageDataSet, 0, targetCopies) - excluded := make([]sdktypes.BigInt, 0, len(bindings)) - inFlight := 0 - for i := range bindings { - binding := bindings[i] - // Automatic selection excludes every provider this bucket has ever used, - // including retired generations, so a replacement never lands back on a - // provider the operator already moved away from. - excluded = append(excluded, binding.ProviderID.SDK()) - if !binding.IsCurrent { - continue - } - existing[binding.CopyIndex] = binding - if dataSetBindingCanOccupyUploadSlot(uploadID, &binding) { - if len(selected) < targetCopies { - selected = append(selected, binding) - } - continue - } - if dataSetBindingInFlightForOtherUpload(uploadID, &binding) { - inFlight++ - } - } - if len(selected) >= targetCopies { - plan := newBucketBindingPlan(selected, uploadID) - plan.complete = true - return plan, nil - } - if inFlight > 0 { - return newBucketBindingPlan(selected, uploadID), nil - } - need := targetCopies - len(selected) - missingIndexes := make([]int, 0, need) - for copyIndex := 0; len(missingIndexes) < need && copyIndex < model.StorageCopiesMax; copyIndex++ { - if _, occupied := existing[copyIndex]; occupied { - continue - } - missingIndexes = append(missingIndexes, copyIndex) - } - if len(missingIndexes) == 0 { - return newBucketBindingPlan(selected, uploadID), nil - } - targets, selectErr := u.storage.SelectUploadTargets(ctx, storage.SelectUploadContextsOptions{ - Copies: len(missingIndexes), - ExcludeProviderIDs: excluded, - DataSetMetadata: map[string]string{"bucket": bucket.Name}, - }) - for i, target := range targets { - if i >= len(missingIndexes) { - break - } - if target == nil { - return newBucketBindingPlan(selected, uploadID), errors.New("storage target selector returned a nil target") - } - copyIndex := missingIndexes[i] - binding, err := u.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: idtypes.OnChainIDFromSDK(target.ProviderID()), - CopyIndex: copyIndex, - CreatedByUploadID: uploadID, - }) - if err != nil { - return newBucketBindingPlan(selected, uploadID), err - } - if ref, bound := target.DataSetRef(); bound { - dataSetID, clientDataSetID, err := dataSetRefIDsForBinding(binding, ref) - if err != nil { - return newBucketBindingPlan(selected, uploadID), err - } - if err := u.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: uploadID, - DataSetID: dataSetID, - ClientDataSetID: &clientDataSetID, - }); err != nil { - return newBucketBindingPlan(selected, uploadID), err - } - binding.Status = model.StorageDataSetStatusReady - binding.DataSetID = &dataSetID - binding.ClientDataSetID = &clientDataSetID - } - selected = append(selected, *binding) - existing[copyIndex] = *binding - } - sort.Slice(selected, func(i, j int) bool { return selected[i].CopyIndex < selected[j].CopyIndex }) - if selectErr != nil { - return newBucketBindingPlan(selected, uploadID), selectErr - } - if len(targets) != len(missingIndexes) { - return newBucketBindingPlan(selected, uploadID), &synapse.NoProviderCandidatesError{ - Cause: fmt.Errorf("SelectUploadTargets returned %d targets, want %d", len(targets), len(missingIndexes)), - } - } - plan := newBucketBindingPlan(selected, uploadID) - plan.complete = true - return plan, nil -} - -func (u *Uploader) ensureUploadFundingReady( - ctx context.Context, - task *model.Task, - contentSize int64, - bucket *model.Bucket, - uploadID int64, - plan *bucketBindingPlan, - logger *slog.Logger, -) (map[int]*model.StorageDataSet, bool) { - if plan == nil { - u.handleTaskFailure(ctx, task, logger, "prepare upload funding contexts", errors.New("missing bucket binding plan")) - return nil, false - } - targets := make([]synapse.StorageTarget, 0, len(plan.writable)) - fundedBindings := make(map[int]*model.StorageDataSet, len(plan.writable)) - for i := range plan.writable { - binding := plan.byID[plan.writable[i].ID] - if binding == nil { - u.handleTaskFailure(ctx, task, logger, "prepare upload funding contexts", errors.New("storage data set binding is missing from plan")) - return nil, false - } - var ( - storageTarget synapse.StorageTarget - err error - ) - if binding.DataSetID != nil && !binding.DataSetID.IsZero() { - storageTarget, err = u.contextForReadyBinding(ctx, binding) - } else { - storageTarget, err = u.contextForBindingProvider(ctx, binding, bucket.Name) - } - if err == nil { - targets = append(targets, storageTarget) - fundedBindings[binding.CopyIndex] = binding - continue - } - switch { - case binding.Status == model.StorageDataSetStatusReady && dataSetFailureEnded(err, binding): - latest, markErr := u.markDataSetStatus(ctx, binding, model.StorageDataSetStatusDraining, err.Error()) - if markErr != nil { - u.handleTaskFailure(ctx, task, logger, "mark funding data set draining", markErr) - return nil, false - } - binding = latest - plan.byID[binding.ID] = binding - if binding.Status == model.StorageDataSetStatusUnavailable { - if repairErr := u.ensureReplicaRepairTask(ctx, binding, task.MaxRetries); repairErr != nil { - u.handleTaskFailure(ctx, task, logger, "ensure unavailable replica repair", repairErr) - return nil, false - } - } else if binding.Status == model.StorageDataSetStatusReady { - plan.deferredContext = true - } else if !dataSetBindingWriteBlocked(binding) { - u.handleTaskFailure(ctx, task, logger, "mark funding data set draining", fmt.Errorf("data set status changed to %s", binding.Status)) - return nil, false - } - case binding.Status == model.StorageDataSetStatusReady && dataSetFailureUnavailable(err, binding): - latest, markErr := u.markDataSetStatus(ctx, binding, model.StorageDataSetStatusUnavailable, err.Error()) - if markErr != nil { - u.handleTaskFailure(ctx, task, logger, "mark funding data set unavailable", markErr) - return nil, false - } - binding = latest - plan.byID[binding.ID] = binding - switch binding.Status { - case model.StorageDataSetStatusUnavailable: - if repairErr := u.ensureReplicaRepairTask(ctx, binding, task.MaxRetries); repairErr != nil { - u.handleTaskFailure(ctx, task, logger, "ensure unavailable replica repair", repairErr) - return nil, false - } - case model.StorageDataSetStatusReady: - plan.deferredContext = true - case model.StorageDataSetStatusDraining, model.StorageDataSetStatusRetired: - default: - u.handleTaskFailure(ctx, task, logger, "mark funding data set unavailable", fmt.Errorf("data set status changed to %s", binding.Status)) - return nil, false - } - case synapse.IsNoProviderCandidates(err): - plan.deferredContext = true - case synapse.IsProviderUnavailable(err): - plan.deferredContext = true - default: - u.handleTaskFailure(ctx, task, logger, "prepare upload funding contexts", err) - return nil, false - } - } - if len(targets) == 0 { - u.waitForStorageDependency(ctx, task, logger, "Waiting for an assigned storage provider to recover") - return nil, false - } - dataSize := uint64(objectlimits.MinFOCUploadSize) - if contentSize > int64(dataSize) { - dataSize = uint64(contentSize) - } - costs, err := u.storage.PrepareUpload(ctx, dataSize, targets) - if err != nil { - if synapse.IsProviderUnavailable(err) || synapse.IsNoProviderCandidates(err) { - u.waitForStorageDependency(ctx, task, logger, "Waiting for storage providers to become available") - return nil, false - } - u.handleTaskFailure(ctx, task, logger, "prepare upload funding", err) - return nil, false - } - if costs == nil { - u.handleTaskFailure(ctx, task, logger, "prepare upload funding", errors.New("missing storage cost estimate")) - return nil, false - } - if costs.Ready { - return fundedBindings, true - } - message := uploadFundingWaitMessage(costs) - if err := u.repos.Tasks.WaitRunning(ctx, task, model.TaskWaitReasonDependency, message, uploadFundingWaitDelay); err != nil { - logger.Error("failed to wait for upload funding", "uploadID", uploadID, "error", err) - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() - return nil, false - } - admin.WorkerTasksProcessed.WithLabelValues("uploader", "success").Inc() - logger.Info("upload funding deferred", "uploadID", uploadID, "message", message) - return nil, false -} - -func uploadFundingWaitMessage(costs *storage.MultiContextCosts) string { - parts := make([]string, 0, 2) - if costs != nil && costs.DepositNeeded != nil && costs.DepositNeeded.Sign() > 0 { - parts = append(parts, fmt.Sprintf("deposit %s USDFC base units", costs.DepositNeeded.String())) - } - if costs != nil && costs.NeedsFWSSMaxApproval { - parts = append(parts, "approve FWSS spending") - } - if len(parts) == 0 { - return "Waiting for Filecoin payment funding" - } - return "Waiting for Filecoin payment funding: " + strings.Join(parts, "; ") -} - -func (u *Uploader) ensureUploadDataSet(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, uploadID int64, copyIndex int, logger *slog.Logger) { - copyRow, err := u.taskUploadCopy(ctx, task, uploadID, copyIndex) - if err != nil || copyRow == nil { - if err == nil { - err = fmt.Errorf("upload copy %d not found", copyIndex) - } - u.handleTaskFailure(ctx, task, logger, "load upload copy", err) - return - } - binding, err := u.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, copyIndex) - if err != nil || binding == nil { - if err == nil { - err = fmt.Errorf("dataset binding for copy_index %d not found", copyIndex) - } - u.handleTaskFailure(ctx, task, logger, "load dataset binding", err) - return - } - if dataSetBindingUnavailable(binding) || dataSetBindingWriteBlocked(binding) { - u.handleAssignedDataSetDependency(ctx, task, version, uploadID, copyRow, binding, logger, "ensure dataset", fmt.Errorf("data set status is %s", binding.Status)) - return - } - if !uploadCanUseDataSetBinding(uploadID, binding) { - u.waitForStorageDependency(ctx, task, logger, "Waiting for the assigned storage service to become writable") - return - } - if binding.Status != model.StorageDataSetStatusReady { - providerTarget, err := u.contextForBindingProvider(ctx, binding, bucket.Name) - if err != nil { - u.markDataSetStageFailed(ctx, task, version, bucket, uploadID, copyIndex, binding, logger, "create dataset context", err) - return - } - switch binding.Status { - case model.StorageDataSetStatusPending, model.StorageDataSetStatusFailed: - matchingRef, err := u.storage.FindMatchingDataSet( - ctx, - binding.ProviderID.SDK(), - map[string]string{"bucket": bucket.Name}, - providerTarget.CDNEnabled(), - ) - if err != nil { - u.markDataSetStageFailed(ctx, task, version, bucket, uploadID, copyIndex, binding, logger, "find existing dataset", err) - return - } - if matchingRef != nil { - dataSetID, clientDataSetID, err := dataSetRefIDsForBinding(binding, *matchingRef) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "validate existing dataset", err) - return - } - if err := u.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: uploadID, - DataSetID: dataSetID, - ClientDataSetID: &clientDataSetID, - }); err != nil { - u.handleTaskFailure(ctx, task, logger, "mark existing dataset ready", err) - return - } - } else { - var submitted storage.CreateDataSetSubmission - var submitErr error - result, err := providerTarget.CreateDataSet(ctx, &storage.CreateDataSetOptions{ - OnSubmitted: func(sub storage.CreateDataSetSubmission) { - submitted = sub - submitErr = u.repos.Uploads.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ - ID: binding.ID, - UploadID: uploadID, - TransactionID: sub.TransactionID, - StatusURL: sub.StatusURL, - ClientDataSetID: onChainIDPtrFromSDKPtr(sub.ClientDataSetID), - }) - }, - }) - if err != nil { - if submitted.TransactionID != "" { - if submitErr != nil { - u.handleTaskFailure(ctx, task, logger, "save dataset submission", submitErr) - return - } - if synapse.IsProviderUnavailable(err) { - u.waitForStorageDependency(ctx, task, logger, "Waiting for storage service creation") - return - } - u.handleTaskFailure(ctx, task, logger, "wait dataset", err) - return - } - u.markDataSetStageFailed(ctx, task, version, bucket, uploadID, copyIndex, binding, logger, "create dataset", err) - return - } - if submitted.TransactionID != "" { - binding.CreateTransactionID = &submitted.TransactionID - } - dataSetID, clientDataSetID, refErr := dataSetResultIDsForBinding(binding, result) - if refErr != nil { - u.handleTaskFailure(ctx, task, logger, "validate created dataset", refErr) - return - } - if err := u.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: uploadID, - DataSetID: dataSetID, - ClientDataSetID: &clientDataSetID, - }); err != nil { - u.handleTaskFailure(ctx, task, logger, "mark dataset ready", err) - return - } - } - case model.StorageDataSetStatusCreating: - if binding.CreateTransactionID == nil || binding.CreateStatusURL == nil || binding.ClientDataSetID == nil { - u.markDataSetStageFailed(ctx, task, version, bucket, uploadID, copyIndex, binding, logger, "wait dataset", errDataSetCreationIncomplete) - return - } - clientDataSetIDSDK := sdkBigIntPtr(binding.ClientDataSetID) - result, err := providerTarget.WaitForDataSetCreated(ctx, storage.CreateDataSetSubmission{ - ProviderID: binding.ProviderID.SDK(), - TransactionID: *binding.CreateTransactionID, - StatusURL: *binding.CreateStatusURL, - ClientDataSetID: clientDataSetIDSDK, - }) - if err != nil { - if errors.Is(err, pdp.ErrTxRejected) { - u.markDataSetStageFailed(ctx, task, version, bucket, uploadID, copyIndex, binding, logger, "wait dataset", err) - return - } - if synapse.IsProviderUnavailable(err) { - u.waitForStorageDependency(ctx, task, logger, "Waiting for storage service creation") - return - } - u.handleTaskFailure(ctx, task, logger, "wait dataset", err) - return - } - dataSetID, clientDataSetID, refErr := dataSetResultIDsForBinding(binding, result) - if refErr != nil { - u.handleTaskFailure(ctx, task, logger, "validate recovered dataset", refErr) - return - } - if err := u.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: uploadID, - DataSetID: dataSetID, - ClientDataSetID: &clientDataSetID, - }); err != nil { - u.handleTaskFailure(ctx, task, logger, "mark dataset ready", err) - return - } - default: - u.handleTaskFailure(ctx, task, logger, "ensure dataset", fmt.Errorf("dataset binding status %s cannot be ensured", binding.Status)) - return - } - } - nextStage := uploadStagePeerPull - if copyRow.TransferMethod == model.StorageCopyTransferMethodIngress { - nextStage = uploadStageIngressStore - } - if err := u.enqueueUploadStageForCopy(ctx, task, nextStage, uploadID, copyIndex, copyRow.TransferMethod, copyRow.ID); err != nil { - u.handleTaskFailure(ctx, task, logger, "enqueue next upload stage", err) - return - } - completeWorkerTask(ctx, u.repos, task, "uploader", logger) -} - -func (u *Uploader) ingressStore(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, uploadID int64, copyIndex int, logger *slog.Logger) { - binding, storageCtx, err := u.readyContextForCopy(ctx, task, bucket, uploadID, copyIndex) - if err != nil { - u.markDataSetStageFailed(ctx, task, version, bucket, uploadID, copyIndex, binding, logger, "ingress context", err) - return - } - if binding == nil { - u.handleTaskFailure(ctx, task, logger, "ingress context", errors.New("ingress dataset binding not found")) - return - } - copyRow, err := u.taskUploadCopy(ctx, task, uploadID, copyIndex) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "load ingress copy", err) - return - } - if copyHasPiece(copyRow) { - if version.State == model.ObjectStateUploading { - if err := state.TransitionState(ctx, u.stateMachine, u.repos.Objects, version.VersionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - u.handleTaskFailure(ctx, task, logger, "state transition uploading→committing", err) - return - } - } - if err := u.enqueueUploadStageForCopy(ctx, task, uploadStageIngressCommit, uploadID, copyIndex, copyRow.TransferMethod, copyRow.ID); err != nil { - u.handleTaskFailure(ctx, task, logger, "enqueue ingress commit", err) - return - } - completeWorkerTask(ctx, u.repos, task, "uploader", logger) - return - } - rc, _, err := u.cache.Get(ctx, bucket.Name, version.CacheKey) - if err != nil { - if os.IsNotExist(err) && version.InCache { - if markErr := u.repos.Objects.SetVersionCachePresence(ctx, version.VersionID, false); markErr != nil { - logger.Warn("failed to mark cache location absent", "error", markErr) - } - } - u.handleIngressFailure(ctx, task, version, uploadID, copyIndex, logger, "cache read", err) - return - } - defer func() { _ = rc.Close() }() - progress := u.beginIngressProgressReporter(ctx, task, version, bucket, uploadID, logger) - result, err := storageCtx.Store(ctx, rc, &storage.StoreOptions{ - OnProgress: func(bytesUploaded int64) { - progress.OnProgress(bytesUploaded) - }, - }) - if err != nil { - u.handleIngressDataSetFailure(ctx, task, version, uploadID, copyIndex, binding.ID, logger, "ingress store", err) - return - } - progress.Flush(version.Size, true) - pieceCID := result.PieceCID.String() - if err := u.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - StorageUploadCopyID: uploadStageCopyID(task), - UploadID: uploadID, - CopyIndex: copyIndex, - PieceCID: pieceCID, - RetrievalURL: storageCtx.PieceURL(result.PieceCID), - }); err != nil { - u.handleTaskFailure(ctx, task, logger, "mark ingress piece ready", err) - return - } - if version.State == model.ObjectStateUploading { - if err := state.TransitionState(ctx, u.stateMachine, u.repos.Objects, version.VersionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - u.handleTaskFailure(ctx, task, logger, "state transition uploading→committing", err) - return - } - } - if err := u.enqueueUploadStageForCopy(ctx, task, uploadStageIngressCommit, uploadID, copyIndex, copyRow.TransferMethod, copyRow.ID); err != nil { - u.handleTaskFailure(ctx, task, logger, "enqueue ingress commit", err) - return - } - completeWorkerTask(ctx, u.repos, task, "uploader", logger) -} - -func (u *Uploader) ingressCommit(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, uploadID int64, copyIndex int, logger *slog.Logger) { - copyRow, err := u.taskUploadCopy(ctx, task, uploadID, copyIndex) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "load ingress copy", err) - return - } - if copyCommitted(copyRow) { - u.finishCommittedIngress(ctx, task, version, bucket, uploadID, copyRow, logger) - return - } - binding, storageCtx, err := u.readyContextForCopy(ctx, task, bucket, uploadID, copyIndex) - if err != nil { - if binding != nil && u.handleUnavailableCommitContext(ctx, task, binding, copyRow, logger, "ingress commit context") { - return - } - u.markDataSetStageFailed(ctx, task, version, bucket, uploadID, copyIndex, binding, logger, "ingress commit context", err) - return - } - upload, err := u.repos.Uploads.GetByID(ctx, uploadID) - if err != nil || upload == nil || upload.PieceCID == nil { - if err == nil { - err = errors.New("upload has no ingress piece cid") - } - u.handleTaskFailure(ctx, task, logger, "load ingress upload", err) - return - } - pieceCID, err := cid.Decode(*upload.PieceCID) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "decode ingress piece cid", err) - return - } - pieces := []storage.PieceInput{{PieceCID: pieceCID}} - advance, err := u.advanceStorageCommit(ctx, binding, copyRow, storageCtx, pieces, false, false) - if err != nil && advance.State == storagecommit.AdvancePending && synapse.IsProviderUnavailable(err) { - u.handleIngressDataSetFailure(ctx, task, version, uploadID, copyIndex, binding.ID, logger, "ingress commit", err) - return - } - if u.waitForCommitAdvance(ctx, task, logger, advance) { - if err != nil { - logger.Warn("storage commit evidence remains fenced", "stage", "ingress commit", "error", err) - } - return - } - if err != nil { - u.handleCommitTaskFailure(ctx, task, copyRow, logger, "advance ingress commit", err) - return - } - switch { - case advance.State == storagecommit.AdvanceReleased && advance.ReleaseReason == storagecommit.ReleaseDataSetUnavailable: - u.handleIngressDataSetFailure(ctx, task, version, uploadID, copyIndex, binding.ID, logger, "ingress commit", commitReleaseCause(advance)) - return - case advance.State == storagecommit.AdvanceReleased: - return - case advance.State == storagecommit.AdvanceRejected: - u.handleIngressFailure(ctx, task, version, uploadID, copyIndex, logger, "ingress commit", errCommitRejected) - return - case advance.State != storagecommit.AdvanceConfirmed || advance.Confirmation == nil || len(advance.Confirmation.PieceIDs) == 0: - u.handleTaskFailure(ctx, task, logger, "advance ingress commit", errors.New("storage commit returned no confirmation")) - return - } - result := advance.Confirmation - pieceID := onChainIDPtrFromSDK(result.PieceIDs[0]) - if err := u.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - StorageUploadCopyID: copyRow.ID, - UploadID: uploadID, - CopyIndex: copyIndex, - PieceCID: *upload.PieceCID, - PieceID: pieceID, - RetrievalURL: storageCtx.PieceURL(pieceCID), - CommitExtraDataHex: derefString(copyRow.CommitExtraDataHex), - CommitTransactionID: result.TransactionID, - CommitAttemptID: advance.AttemptID, - CommitConfirmedTransactionID: result.ConfirmedTransactionID, - }); err != nil { - u.handleTaskFailure(ctx, task, logger, "mark ingress committed", err) - return - } - u.finishReadable(ctx, task, version, uploadID, logger) -} - -func (u *Uploader) finishCommittedIngress(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, uploadID int64, copyRow *model.StorageUploadCopy, logger *slog.Logger) { - readable, err := u.repos.Uploads.HasReadableCommittedCopy(ctx, uploadID) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "check committed ingress readability", err) - return - } - if readable { - u.finishReadable(ctx, task, version, uploadID, logger) - return - } - if copyRow == nil || copyRow.StorageDataSetID == nil { - u.handleTaskFailure(ctx, task, logger, "load committed ingress data set", repository.ErrNotFound) - return - } - binding, err := u.repos.Uploads.GetDataSetBindingByID(ctx, *copyRow.StorageDataSetID) - if err != nil || binding == nil { - if err == nil { - err = repository.ErrNotFound - } - u.handleTaskFailure(ctx, task, logger, "load committed ingress data set", err) - return - } - switch binding.Status { - case model.StorageDataSetStatusReady: - u.finishReadable(ctx, task, version, uploadID, logger) - return - case model.StorageDataSetStatusUnavailable: - case model.StorageDataSetStatusDraining, model.StorageDataSetStatusRetired: - u.waitForStorageDependency(ctx, task, logger, "Waiting for the storage service to be replaced") - return - default: - u.handleTaskFailure(ctx, task, logger, "recover committed ingress data set", fmt.Errorf("data set status %s cannot be recovered in place", binding.Status)) - return - } - if binding.DataSetID == nil || binding.DataSetID.IsZero() { - u.handleTaskFailure(ctx, task, logger, "recover committed ingress data set", errors.New("established data set has no data set ID")) - return - } - if _, err := u.contextForReadyBinding(ctx, binding); err != nil { - u.handleCommittedIngressDataSetFailure(ctx, task, binding, logger, err) - return - } - if err := u.ensureReplicaRepairTask(ctx, binding, task.MaxRetries); err != nil { - u.handleTaskFailure(ctx, task, logger, "schedule data set recovery finalization", err) - return - } - recovered, err := u.repos.Uploads.RecoverDataSet(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: uploadID, - DataSetID: *binding.DataSetID, - }) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "recover committed ingress data set", err) - return - } - if !recovered { - latest, loadErr := u.repos.Uploads.GetDataSetBindingByID(ctx, binding.ID) - if loadErr != nil || latest == nil { - if loadErr == nil { - loadErr = repository.ErrNotFound - } - u.handleTaskFailure(ctx, task, logger, "reload committed ingress data set", loadErr) - return - } - switch latest.Status { - case model.StorageDataSetStatusReady: - case model.StorageDataSetStatusUnavailable: - u.waitForStorageDependency(ctx, task, logger, "Waiting for the assigned storage provider to recover") - return - case model.StorageDataSetStatusDraining, model.StorageDataSetStatusRetired: - u.waitForStorageDependency(ctx, task, logger, "Waiting for the storage service to be replaced") - return - default: - u.handleTaskFailure(ctx, task, logger, "recover committed ingress data set", fmt.Errorf("data set status changed to %s", latest.Status)) - return - } - } - u.finishReadable(ctx, task, version, uploadID, logger) -} - -func (u *Uploader) handleCommittedIngressDataSetFailure(ctx context.Context, task *model.Task, binding *model.StorageDataSet, logger *slog.Logger, err error) { - status := model.StorageDataSetStatusUnavailable - if dataSetFailureEnded(err, binding) { - status = model.StorageDataSetStatusDraining - } else if !dataSetFailureUnavailable(err, binding) { - u.handleTaskFailure(ctx, task, logger, "verify committed ingress context", err) - return - } - latest, markErr := u.markDataSetStatus(ctx, binding, status, err.Error()) - if markErr != nil { - u.handleTaskFailure(ctx, task, logger, "mark committed ingress data set", markErr) - return - } - var message string - if latest.Status == model.StorageDataSetStatusReady { - message = "Waiting to retry the storage operation" - } else if latest.Status == model.StorageDataSetStatusUnavailable { - message = "Waiting for the assigned storage provider to recover" - } else if dataSetBindingWriteBlocked(latest) { - message = "Waiting for the storage service to be replaced" - } else { - u.handleTaskFailure(ctx, task, logger, "mark committed ingress data set", fmt.Errorf("data set status changed to %s", latest.Status)) - return - } - u.waitForStorageDependency(ctx, task, logger, message) -} - -func (u *Uploader) finishReadable(ctx context.Context, task *model.Task, version *model.ObjectVersion, uploadID int64, logger *slog.Logger) { - refs, err := u.repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: uploadID, - BucketID: version.BucketID, - ContentSize: version.Size, - Checksum: version.Checksum, - }) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "bind readable upload", err) - return - } - ref := repository.ObjectVersionRef{ObjectID: version.ObjectID, VersionID: version.VersionID} - _, needsPreparation, err := u.scheduleRemainingPeerCopies(ctx, ref, version.BucketID, uploadID, task.MaxRetries) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "schedule remaining upload copies", err) - return - } - if needsPreparation { - if err := u.enqueueRepairUploadForVersion(ctx, ref, task.MaxRetries, uploadID); err != nil { - u.handleTaskFailure(ctx, task, logger, "schedule upload preparation", err) - return - } - } - _, _, err = u.repos.Uploads.FinalizeUploadIfTargetCopiesMet( - ctx, - u.finalizeUploadInput(uploadID), - ) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "finalize readable upload", err) - return - } - if !completeWorkerTask(ctx, u.repos, task, "uploader", logger) { - return - } - logger.Info("upload readable copy committed", "uploadID", uploadID, "versions", len(refs)) -} - -func (u *Uploader) scheduleRemainingPeerCopies( - ctx context.Context, - ref repository.ObjectVersionRef, - bucketID int64, - uploadID int64, - maxRetries int, -) ([]model.StorageUploadCopy, bool, error) { - copies, err := u.repos.Uploads.ListCopies(ctx, uploadID) - if err != nil { - return nil, false, err - } - upload, err := u.repos.Uploads.GetByID(ctx, uploadID) - if err != nil { - return nil, false, err - } - if upload == nil { - return nil, false, fmt.Errorf("storage upload %d not found", uploadID) - } - currentSlots, err := u.repos.Uploads.CountCurrentGenerationCopySlots(ctx, uploadID) - if err != nil { - return nil, false, err - } - needsPreparation := currentSlots < boundedTargetCopies(upload.RequestedCopies) - for _, copyRow := range copies { - if copyRow.TransferMethod != model.StorageCopyTransferMethodPeerPull || copyCommitted(©Row) || copyRow.Status == model.StorageUploadCopyStatusFailed { - continue - } - var binding *model.StorageDataSet - if copyRow.StorageDataSetID != nil { - binding, err = u.repos.Uploads.GetDataSetBindingByID(ctx, *copyRow.StorageDataSetID) - } else { - binding, err = u.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucketID, copyRow.CopyIndex) - } - if err != nil { - return nil, false, err - } - switch { - case binding != nil && (binding.Status == model.StorageDataSetStatusReady || - (copyRow.StorageDataSetID != nil && binding.Status == model.StorageDataSetStatusDraining)): - if err := u.enqueueUploadStageForVersionCopy(ctx, ref, maxRetries, uploadStageEnsureDataSet, uploadID, copyRow.CopyIndex, copyRow.TransferMethod, copyRow.ID); err != nil { - return nil, false, err - } - case binding != nil && binding.Status == model.StorageDataSetStatusUnavailable: - if err := u.ensureReplicaRepairTask(ctx, binding, maxRetries); err != nil { - return nil, false, err - } - case uploadCanUseDataSetBinding(uploadID, binding): - needsPreparation = true - } - } - return copies, needsPreparation, nil -} - -func (u *Uploader) repairReadableBinding(ctx context.Context, task *model.Task, version *model.ObjectVersion, uploadID int64, logger *slog.Logger, stage string) bool { - _, err := u.repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: uploadID, - BucketID: version.BucketID, - ContentSize: version.Size, - Checksum: version.Checksum, - }) - if err != nil { - u.handleTaskFailure(ctx, task, logger, stage, err) - return false - } - return true -} - -func (u *Uploader) peerPull(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, uploadID int64, copyIndex int, logger *slog.Logger) { - binding, storageCtx, err := u.readyContextForCopy(ctx, task, bucket, uploadID, copyIndex) - if err != nil { - u.markDataSetStageFailed(ctx, task, version, bucket, uploadID, copyIndex, binding, logger, "peer pull context", err) - return - } - copyRow, err := u.taskUploadCopy(ctx, task, uploadID, copyIndex) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "load peer copy", err) - return - } - if copyCommitted(copyRow) { - u.finishPeerCopy(ctx, task, version, uploadID, logger) - return - } - if copyHasPiece(copyRow) { - if err := u.enqueueUploadStageForCopy(ctx, task, uploadStagePeerCommit, uploadID, copyIndex, copyRow.TransferMethod, copyRow.ID); err != nil { - u.handleTaskFailure(ctx, task, logger, "enqueue peer commit", err) - return - } - completeWorkerTask(ctx, u.repos, task, "uploader", logger) - return - } - readableCopies, err := u.repos.Uploads.ListReadableCommittedCopies(ctx, uploadID) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "load readable source copy", err) - return - } - var pieceCID cid.Cid - var pieceCIDString string - var extraHex string - if len(readableCopies) > 0 { - if !u.repairReadableBinding(ctx, task, version, uploadID, logger, "repair readable binding") { - return - } - sourceCopy := readableCopies[0] - pieceCID, err = cid.Decode(sourceCopy.PieceCID) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "decode readable piece cid", err) - return - } - pieceCIDString = sourceCopy.PieceCID - pieces := []storage.PieceInput{{PieceCID: pieceCID}} - extraData, encodedExtra, err := u.extraDataForCopy(ctx, storageCtx, copyRow, pieces) - if err != nil { - u.handlePeerDataSetFailure(ctx, task, bucket, uploadID, copyIndex, binding.ID, logger, "peer presign", err) - return - } - extraHex = encodedExtra - if _, err := storageCtx.Pull(ctx, storage.PullRequest{ - Pieces: []cid.Cid{pieceCID}, - ExtraData: extraData, - From: func(cid.Cid) string { - return sourceCopy.RetrievalURL - }, - }); err != nil { - u.handlePeerDataSetFailure(ctx, task, bucket, uploadID, copyIndex, binding.ID, logger, "peer pull", err) - return - } - } else { - rc, _, err := u.cache.Get(ctx, bucket.Name, version.CacheKey) - if err != nil { - if os.IsNotExist(err) { - if version.InCache { - if markErr := u.repos.Objects.SetVersionCachePresence(ctx, version.VersionID, false); markErr != nil { - logger.Warn("failed to mark cache location absent", "versionID", version.VersionID, "error", markErr) - } - } - u.waitForStorageDependency(ctx, task, logger, "Waiting for a readable replica or retained cache data") - return - } - u.handleTaskFailure(ctx, task, logger, "open retained cache data", err) - return - } - result, storeErr := storageCtx.Store(ctx, rc, &storage.StoreOptions{}) - closeErr := rc.Close() - if storeErr != nil { - u.handlePeerDataSetFailure(ctx, task, bucket, uploadID, copyIndex, binding.ID, logger, "peer cache store", storeErr) - return - } - if closeErr != nil { - u.handleTaskFailure(ctx, task, logger, "close retained cache data", closeErr) - return - } - if result == nil || !result.PieceCID.Defined() { - u.handleTaskFailure(ctx, task, logger, "peer cache store", errors.New("store returned no piece CID")) - return - } - pieceCID = result.PieceCID - pieceCIDString = pieceCID.String() - _, extraHex, err = u.extraDataForCopy(ctx, storageCtx, copyRow, []storage.PieceInput{{PieceCID: pieceCID}}) - if err != nil { - u.handlePeerDataSetFailure(ctx, task, bucket, uploadID, copyIndex, binding.ID, logger, "peer presign", err) - return - } - } - if err := u.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - StorageUploadCopyID: uploadStageCopyID(task), - UploadID: uploadID, - CopyIndex: copyIndex, - PieceCID: pieceCIDString, - RetrievalURL: storageCtx.PieceURL(pieceCID), - CommitExtraDataHex: extraHex, - }); err != nil { - u.handleTaskFailure(ctx, task, logger, "mark peer piece ready", err) - return - } - if err := u.enqueueUploadStageForCopy(ctx, task, uploadStagePeerCommit, uploadID, copyIndex, copyRow.TransferMethod, copyRow.ID); err != nil { - u.handleTaskFailure(ctx, task, logger, "enqueue peer commit", err) - return - } - completeWorkerTask(ctx, u.repos, task, "uploader", logger) -} - -func (u *Uploader) peerCommit(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, uploadID int64, copyIndex int, logger *slog.Logger) { - copyRow, err := u.taskUploadCopy(ctx, task, uploadID, copyIndex) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "load peer copy", err) - return - } - if copyCommitted(copyRow) { - u.finishPeerCopy(ctx, task, version, uploadID, logger) - return - } - binding, storageCtx, err := u.readyContextForCopy(ctx, task, bucket, uploadID, copyIndex) - if err != nil { - if binding != nil && u.handleUnavailableCommitContext(ctx, task, binding, copyRow, logger, "peer commit context") { - return - } - u.markDataSetStageFailed(ctx, task, version, bucket, uploadID, copyIndex, binding, logger, "peer commit context", err) - return - } - readableCopies, err := u.repos.Uploads.ListReadableCommittedCopies(ctx, uploadID) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "load readable source copy", err) - return - } - pieceCIDString := "" - if len(readableCopies) > 0 { - pieceCIDString = readableCopies[0].PieceCID - } else if copyHasPiece(copyRow) { - upload, loadErr := u.repos.Uploads.GetByID(ctx, uploadID) - if loadErr != nil { - u.handleTaskFailure(ctx, task, logger, "load peer upload piece", loadErr) - return - } - if upload != nil && upload.PieceCID != nil { - pieceCIDString = *upload.PieceCID - } - } - if pieceCIDString == "" { - u.handleTaskFailure(ctx, task, logger, "load peer piece", errors.New("peer copy has no persisted piece CID")) - return - } - pieceCID, err := cid.Decode(pieceCIDString) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "decode readable piece cid", err) - return - } - pieces := []storage.PieceInput{{PieceCID: pieceCID}} - advance, err := u.advanceStorageCommit(ctx, binding, copyRow, storageCtx, pieces, false, false) - if err != nil && advance.State == storagecommit.AdvancePending && synapse.IsProviderUnavailable(err) { - u.handlePeerDataSetFailure(ctx, task, bucket, uploadID, copyIndex, binding.ID, logger, "peer commit", err) - return - } - if u.waitForCommitAdvance(ctx, task, logger, advance) { - if err != nil { - logger.Warn("storage commit evidence remains fenced", "stage", "peer commit", "error", err) - } - return - } - if err != nil { - u.handleCommitTaskFailure(ctx, task, copyRow, logger, "advance peer commit", err) - return - } - switch { - case advance.State == storagecommit.AdvanceReleased && advance.ReleaseReason == storagecommit.ReleaseDataSetUnavailable: - u.handlePeerDataSetFailure(ctx, task, bucket, uploadID, copyIndex, binding.ID, logger, "peer commit", commitReleaseCause(advance)) - return - case advance.State == storagecommit.AdvanceReleased: - return - case advance.State == storagecommit.AdvanceRejected: - u.handleTaskFailure(ctx, task, logger, "peer commit", errCommitRejected) - return - case advance.State != storagecommit.AdvanceConfirmed || advance.Confirmation == nil || len(advance.Confirmation.PieceIDs) == 0: - u.handleTaskFailure(ctx, task, logger, "advance peer commit", errors.New("storage commit returned no confirmation")) - return - } - result := advance.Confirmation - pieceID := onChainIDPtrFromSDK(result.PieceIDs[0]) - if err := u.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - StorageUploadCopyID: copyRow.ID, - UploadID: uploadID, - CopyIndex: copyIndex, - PieceCID: pieceCIDString, - PieceID: pieceID, - RetrievalURL: storageCtx.PieceURL(pieceCID), - CommitExtraDataHex: derefString(copyRow.CommitExtraDataHex), - CommitTransactionID: result.TransactionID, - CommitAttemptID: advance.AttemptID, - CommitConfirmedTransactionID: result.ConfirmedTransactionID, - }); err != nil { - u.handleTaskFailure(ctx, task, logger, "mark peer committed", err) - return - } - u.finishPeerCopy(ctx, task, version, uploadID, logger) -} - -func (u *Uploader) finishPeerCopy(ctx context.Context, task *model.Task, version *model.ObjectVersion, uploadID int64, logger *slog.Logger) { - if !u.repairReadableBinding(ctx, task, version, uploadID, logger, "repair readable binding") { - return - } - ref := repository.ObjectVersionRef{ObjectID: version.ObjectID, VersionID: version.VersionID} - _, needsPreparation, err := u.scheduleRemainingPeerCopies(ctx, ref, version.BucketID, uploadID, task.MaxRetries) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "schedule remaining upload copies", err) - return - } - if needsPreparation { - if err := u.enqueueRepairUploadForVersion(ctx, ref, task.MaxRetries, uploadID); err != nil { - u.handleTaskFailure(ctx, task, logger, "schedule upload preparation", err) - return - } - } - if _, _, err := u.repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, u.finalizeUploadInput(uploadID)); err != nil { - u.handleTaskFailure(ctx, task, logger, "finalize upload", err) - return - } - completeWorkerTask(ctx, u.repos, task, "uploader", logger) -} - -func (u *Uploader) enqueueUploadStageForCopy( - ctx context.Context, - parent *model.Task, - stage string, - uploadID int64, - copyIndex int, - transferMethod model.StorageCopyTransferMethod, - copyID int64, -) error { - if parent == nil || copyID <= 0 { - return fmt.Errorf("enqueueing concrete upload copy: %w", repository.ErrInvalidInput) - } - ref := repository.ObjectVersionRef{ObjectID: parent.RefID, VersionID: parent.RefVersionID} - return u.enqueueUploadStageForVersionCopy(ctx, ref, parent.MaxRetries, stage, uploadID, copyIndex, transferMethod, copyID) -} - -func (u *Uploader) enqueueUploadStageForVersionCopy( - ctx context.Context, - ref repository.ObjectVersionRef, - maxRetries int, - stage string, - uploadID int64, - copyIndex int, - transferMethod model.StorageCopyTransferMethod, - copyID int64, -) error { - if copyID <= 0 { - return fmt.Errorf("enqueueing concrete upload copy: %w", repository.ErrInvalidInput) - } - task := newUploadStageTask(ref, maxRetries, stage, uploadID, copyIndex, transferMethod, copyID) - if err := u.repos.Tasks.Create(ctx, task); err != nil && !errors.Is(err, repository.ErrAlreadyExists) { - return err - } - return nil -} - -// Recording the concrete copy lets the replacement safety gate tell a write -// bound for the retiring generation from one bound for its replacement. A task -// queued before this existed simply carries no copy id. -func stageTaskCopyID(ctx context.Context, repos *repository.Repositories, uploadID int64, copyIndex int, transferMethod model.StorageCopyTransferMethod) (int64, error) { - if transferMethod == "" { - return 0, nil - } - copyRow, err := repos.Uploads.GetUploadCopy(ctx, uploadID, copyIndex) - if err != nil { - return 0, err - } - if copyRow == nil { - return 0, nil - } - return copyRow.ID, nil -} - -func newUploadStageTask(ref repository.ObjectVersionRef, maxRetries int, stage string, uploadID int64, copyIndex int, transferMethod model.StorageCopyTransferMethod, copyID int64) *model.Task { - payload := map[string]interface{}{ - "upload_id": uploadID, - } - // The key stays keyed by slot so tasks queued before copy ids existed keep - // their identity. - key := fmt.Sprintf("upload:%s:%s:%d", ref.VersionID, stage, uploadID) - if transferMethod != "" { - payload["copy_index"] = copyIndex - payload["transfer_method"] = string(transferMethod) - key = fmt.Sprintf("%s:%d", key, copyIndex) - if copyID > 0 { - payload[replicaRepairCopyIDKey] = copyID - } - } - return &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: ref.ObjectID, - RefVersionID: ref.VersionID, - IdempotencyKey: key, - Payload: payload, - Status: model.TaskStatusQueued, - MaxRetries: maxRetries, - ScheduledAt: time.Now(), - } -} - -func ensureIngressHandoffTask(ctx context.Context, repos *repository.Repositories, ref repository.ObjectVersionRef, maxRetries int, uploadID int64, copyIndex int) error { - copyID, err := stageTaskCopyID(ctx, repos, uploadID, copyIndex, model.StorageCopyTransferMethodIngress) - if err != nil { - return err - } - task := newUploadStageTask(ref, maxRetries, uploadStageEnsureDataSet, uploadID, copyIndex, model.StorageCopyTransferMethodIngress, copyID) - created, err := repos.Tasks.EnsureRecurring(ctx, task) - if err != nil || created { - return err - } - existing, err := repos.Tasks.GetByIdempotencyKey(ctx, task.IdempotencyKey) - if err != nil { - return err - } - if existing == nil || existing.RefType != task.RefType || existing.RefID != task.RefID || existing.RefVersionID != task.RefVersionID || uploadTaskStage(existing) != uploadStageEnsureDataSet { - return fmt.Errorf("ensuring reassigned ingress task: conflicting task identity: %w", repository.ErrConflict) - } - existingUploadID, uploadErr := payloadInt64(existing.Payload, "upload_id") - existingCopyIndex, copyErr := payloadInt64(existing.Payload, "copy_index") - if uploadErr != nil || copyErr != nil || existingUploadID != uploadID || int(existingCopyIndex) != copyIndex { - return fmt.Errorf("ensuring reassigned ingress task: conflicting task payload: %w", repository.ErrConflict) - } - switch existing.Status { - case model.TaskStatusQueued, model.TaskStatusScheduled, model.TaskStatusWaiting: - return nil - case model.TaskStatusRunning: - transferMethod, _ := existing.Payload["transfer_method"].(string) - if transferMethod == string(model.StorageCopyTransferMethodIngress) { - return nil - } - return fmt.Errorf("ensuring reassigned ingress task: running task owns a different transfer method: %w", repository.ErrConflict) - default: - return fmt.Errorf("ensuring reassigned ingress task: task status %s: %w", existing.Status, repository.ErrConflict) - } -} - -func reassignIngressCopyAndSchedule( - ctx context.Context, - repos *repository.Repositories, - stateMachine *state.Machine, - version *model.ObjectVersion, - uploadID int64, - unavailableCopyIndex int, - maxRetries int, - runningTask *model.Task, -) (*model.StorageUploadCopy, error) { - if version == nil { - return nil, fmt.Errorf("reassigning ingress copy: missing object version: %w", repository.ErrInvalidInput) - } - var reassigned *model.StorageUploadCopy - err := repos.WithTx(ctx, func(txRepos *repository.Repositories) error { - wasCommitting := version.State == model.ObjectStateCommitting - if wasCommitting { - if err := state.TransitionState(ctx, stateMachine, txRepos.Objects, version.VersionID, model.ObjectStateCommitting, model.ObjectStateUploading); err != nil { - return fmt.Errorf("lock committing version before ingress reassignment: %w", err) - } - } - selected, err := txRepos.Uploads.ReassignIngressCopy(ctx, uploadID, unavailableCopyIndex) - if err != nil { - return err - } - if wasCommitting && (selected == nil || copyHasPiece(selected)) { - if err := state.TransitionState(ctx, stateMachine, txRepos.Objects, version.VersionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - return fmt.Errorf("preserve committing state after ingress reassignment: %w", err) - } - } - if selected == nil { - return nil - } - if runningTask != nil { - if err := txRepos.Tasks.LockRunningClaim(ctx, runningTask); err != nil { - return err - } - } - ref := repository.ObjectVersionRef{ObjectID: version.ObjectID, VersionID: version.VersionID} - if err := ensureIngressHandoffTask(ctx, txRepos, ref, maxRetries, uploadID, selected.CopyIndex); err != nil { - return fmt.Errorf("enqueue reassigned ingress: %w", err) - } - if runningTask != nil { - if err := txRepos.Tasks.Complete(ctx, runningTask); err != nil { - return err - } - } - reassigned = selected - return nil - }) - return reassigned, err -} - -func uploadTaskStage(task *model.Task) string { - if task == nil { - return uploadStagePrepare - } - if task.Stage != nil && *task.Stage != "" { - return *task.Stage - } - if task.Payload == nil { - return uploadStagePrepare - } - stage, _ := task.Payload["stage"].(string) - if stage == "" { - return uploadStagePrepare - } - return stage -} - -func uploadStageIDs(task *model.Task, needsCopyIndex bool) (int64, int, error) { - uploadID, err := payloadInt64(task.Payload, "upload_id") - if err != nil { - return 0, 0, err - } - copyIndex := 0 - if needsCopyIndex { - v, err := payloadInt64(task.Payload, "copy_index") - if err != nil { - return 0, 0, err - } - copyIndex = int(v) - } - return uploadID, copyIndex, nil -} - -// taskUploadCopy loads the copy a staged task is bound to. When the task records -// a concrete copy it wins over the replica slot, so reads and writes stay on the -// same generation after a replacement activates. -func (u *Uploader) taskUploadCopy(ctx context.Context, task *model.Task, uploadID int64, copyIndex int) (*model.StorageUploadCopy, error) { - if copyID := uploadStageCopyID(task); copyID > 0 { - copyRow, err := u.repos.Uploads.GetUploadCopyByID(ctx, copyID) - if err != nil { - return nil, err - } - if copyRow == nil { - return nil, fmt.Errorf("storage upload copy %d: %w", copyID, repository.ErrNotFound) - } - transferMethod, _ := task.Payload["transfer_method"].(string) - if copyRow.UploadID != uploadID || copyRow.CopyIndex != copyIndex || - (transferMethod != "" && string(copyRow.TransferMethod) != transferMethod) { - return nil, fmt.Errorf("storage upload copy %d does not match task payload: %w", copyID, repository.ErrConflict) - } - return copyRow, nil - } - return u.repos.Uploads.GetUploadCopy(ctx, uploadID, copyIndex) -} - -// taskCopyDataSet resolves the data set a task's copy is bound to. Tasks queued -// before copy ids existed, and copies not yet bound to a data set, fall back to -// the slot's current generation, which is where they belong. -func (u *Uploader) taskCopyDataSet(ctx context.Context, task *model.Task, bucketID, uploadID int64, copyIndex int) (*model.StorageDataSet, error) { - copyRow, err := u.taskUploadCopy(ctx, task, uploadID, copyIndex) - if err != nil { - return nil, err - } - if copyRow != nil && copyRow.StorageDataSetID != nil && *copyRow.StorageDataSetID > 0 { - return u.repos.Uploads.GetDataSetBindingByID(ctx, *copyRow.StorageDataSetID) - } - return u.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucketID, copyIndex) -} - -// uploadStageCopyID reports the concrete copy a staged task targets. Tasks -// queued before copy ids were recorded return zero and stay resolvable through -// their replica slot. -func uploadStageCopyID(task *model.Task) int64 { - if task == nil || task.Payload == nil { - return 0 - } - if _, ok := task.Payload[replicaRepairCopyIDKey]; !ok { - return 0 - } - copyID, err := payloadInt64(task.Payload, replicaRepairCopyIDKey) - if err != nil || copyID < 0 { - return 0 - } - return copyID -} - -func payloadInt64(payload map[string]interface{}, key string) (int64, error) { - raw, ok := payload[key] - if !ok { - return 0, fmt.Errorf("missing %s", key) - } - switch v := raw.(type) { - case int: - return int64(v), nil - case int64: - return v, nil - case float64: - return int64(v), nil - case json.Number: - return v.Int64() - case string: - return strconv.ParseInt(v, 10, 64) - default: - return 0, fmt.Errorf("%s has unsupported type %T", key, raw) - } -} - -func (u *Uploader) markDataSetStageFailed(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, uploadID int64, copyIndex int, binding *model.StorageDataSet, logger *slog.Logger, stage string, err error) { - copyRow, copyErr := u.taskUploadCopy(ctx, task, uploadID, copyIndex) - if copyErr != nil { - u.handleTaskFailure(ctx, task, logger, "load failed upload copy", copyErr) - return - } - if copyRow == nil { - u.handleTaskFailure(ctx, task, logger, stage, err) - return - } - if dataSetFailureEnded(err, binding) { - if dataSetBindingEstablished(binding) { - latest, markErr := u.markDataSetStatus(ctx, binding, model.StorageDataSetStatusDraining, err.Error()) - if markErr != nil { - u.handleTaskFailure(ctx, task, logger, "mark data set draining", markErr) - return - } - binding = latest - } - u.handleAssignedDataSetDependency(ctx, task, version, uploadID, copyRow, binding, logger, stage, err) - return - } - if dataSetFailureUnavailable(err, binding) { - if dataSetBindingEstablished(binding) { - latest, markErr := u.markDataSetStatus(ctx, binding, model.StorageDataSetStatusUnavailable, err.Error()) - if markErr != nil { - u.handleTaskFailure(ctx, task, logger, "mark data set unavailable", markErr) - return - } - binding = latest - } - u.handleAssignedDataSetDependency(ctx, task, version, uploadID, copyRow, binding, logger, stage, err) - return - } - if dataSetCreationRejected(err) && binding != nil { - latest, markErr := u.markDataSetStatus(ctx, binding, model.StorageDataSetStatusFailed, err.Error()) - if markErr != nil { - u.handleTaskFailure(ctx, task, logger, "mark data set creation failed", markErr) - return - } - if latest.Status != model.StorageDataSetStatusFailed { - if latest.Status == model.StorageDataSetStatusReady || latest.Status == model.StorageDataSetStatusUnavailable || dataSetBindingWriteBlocked(latest) { - u.waitForStorageDependency(ctx, task, logger, "Waiting for the storage data set state to settle") - return - } - u.handleTaskFailure(ctx, task, logger, "mark data set creation failed", fmt.Errorf("data set status changed to %s", latest.Status)) - return - } - binding = latest - u.handleDataSetCreationFailure(ctx, task, version, uploadID, copyRow, binding, logger, stage, err) - return - } - if copyRow.TransferMethod == model.StorageCopyTransferMethodPeerPull { - u.markPeerFailed(ctx, task, uploadID, copyIndex, 0, logger, stage, err) - return - } - u.handleIngressFailure(ctx, task, version, uploadID, copyIndex, logger, stage, err) -} - -func (u *Uploader) markPeerFailed(ctx context.Context, task *model.Task, uploadID int64, copyIndex int, discardDataSetID int64, logger *slog.Logger, stage string, err error) { - if appendErr := u.repos.Uploads.AppendUploadFailure(ctx, repository.AppendUploadFailureInput{ - UploadID: uploadID, - CopyIndex: copyIndex, - TransferMethod: string(model.StorageCopyTransferMethodPeerPull), - Stage: stage, - ErrorMessage: err.Error(), - }); appendErr != nil { - logger.Warn("failed to append peer upload failure", "uploadID", uploadID, "copyIndex", copyIndex, "error", appendErr) - } - logger.Error(stage+" failed", "error", err) - status := scheduleTaskRetry(ctx, u.repos, task, "uploader", logger, err) - if status == model.TaskStatusExhausted { - cleanupCtx, cancel := context.WithTimeout(context.Background(), terminalFailureCleanupTimeout) - defer cancel() - if markErr := u.repos.Uploads.MarkUploadCopyFailed(cleanupCtx, repository.MarkUploadCopyFailedInput{ - StorageUploadCopyID: uploadStageCopyID(task), - UploadID: uploadID, - CopyIndex: copyIndex, - LastError: fmt.Sprintf("%s: %v", stage, err), - }); markErr != nil { - logger.Warn("failed to mark peer upload copy failed", "uploadID", uploadID, "copyIndex", copyIndex, "error", markErr) - } - discarded := false - if discardDataSetID > 0 { - var discardErr error - discarded, discardErr = u.repos.Uploads.DiscardFailedDataSetCandidate(cleanupCtx, uploadID, copyIndex, discardDataSetID) - if discardErr != nil { - logger.Warn("failed to discard failed dataset candidate", "uploadID", uploadID, "copyIndex", copyIndex, "dataSetID", discardDataSetID, "error", discardErr) - } - } - if discarded { - if repairErr := u.enqueueRepairUpload(cleanupCtx, task, uploadID); repairErr != nil { - logger.Warn("failed to enqueue authorized slot retry", "uploadID", uploadID, "copyIndex", copyIndex, "error", repairErr) - } - } - } - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() -} - -func (u *Uploader) handlePeerDataSetFailure(ctx context.Context, task *model.Task, bucket *model.Bucket, uploadID int64, copyIndex int, dataSetID int64, logger *slog.Logger, stage string, err error) { - var binding *model.StorageDataSet - if dataSetID > 0 { - var loadErr error - binding, loadErr = u.repos.Uploads.GetDataSetBindingByID(ctx, dataSetID) - if loadErr != nil { - u.handleTaskFailure(ctx, task, logger, "load failed peer data set", loadErr) - return - } - } - if dataSetFailureEnded(err, binding) || dataSetFailureUnavailable(err, binding) { - copyRow, loadErr := u.taskUploadCopy(ctx, task, uploadID, copyIndex) - if loadErr != nil || copyRow == nil { - if loadErr == nil { - loadErr = errors.New("peer upload copy not found") - } - u.handleTaskFailure(ctx, task, logger, "load failed peer copy", loadErr) - return - } - if dataSetFailureEnded(err, binding) { - latest, markErr := u.markDataSetStatus(ctx, binding, model.StorageDataSetStatusDraining, err.Error()) - if markErr != nil { - u.handleTaskFailure(ctx, task, logger, "mark peer data set draining", markErr) - return - } - binding = latest - } else { - latest, markErr := u.markDataSetStatus(ctx, binding, model.StorageDataSetStatusUnavailable, err.Error()) - if markErr != nil { - u.handleTaskFailure(ctx, task, logger, "mark peer data set unavailable", markErr) - return - } - binding = latest - } - u.handleAssignedDataSetDependency(ctx, task, nil, uploadID, copyRow, binding, logger, stage, err) - return - } - u.markPeerFailed(ctx, task, uploadID, copyIndex, 0, logger, stage, err) -} - -func (u *Uploader) handleAssignedDataSetDependency( - ctx context.Context, - task *model.Task, - version *model.ObjectVersion, - uploadID int64, - copyRow *model.StorageUploadCopy, - binding *model.StorageDataSet, - logger *slog.Logger, - stage string, - err error, -) { - if copyRow == nil { - u.handleTaskFailure(ctx, task, logger, stage, err) - return - } - if dataSetBindingEstablished(binding) { - switch binding.Status { - case model.StorageDataSetStatusReady: - u.waitForStorageDependency(ctx, task, logger, "Waiting to retry the storage operation") - return - case model.StorageDataSetStatusUnavailable, model.StorageDataSetStatusDraining, model.StorageDataSetStatusRetired: - default: - u.handleTaskFailure(ctx, task, logger, stage, fmt.Errorf("data set status changed to %s: %w", binding.Status, err)) - return - } - } - if appendErr := u.repos.Uploads.AppendUploadFailure(ctx, repository.AppendUploadFailureInput{ - UploadID: uploadID, - CopyIndex: copyRow.CopyIndex, - TransferMethod: string(copyRow.TransferMethod), - Stage: stage, - ErrorMessage: err.Error(), - }); appendErr != nil { - logger.Warn("failed to append storage dependency failure", "uploadID", uploadID, "copyIndex", copyRow.CopyIndex, "error", appendErr) - } - if binding != nil && binding.Status == model.StorageDataSetStatusUnavailable { - if repairErr := u.ensureReplicaRepairTask(ctx, binding, task.MaxRetries); repairErr != nil { - u.handleTaskFailure(ctx, task, logger, "ensure unavailable replica repair", repairErr) - return - } - } - if copyRow.TransferMethod == model.StorageCopyTransferMethodPeerPull { - if binding != nil && (dataSetBindingUnavailable(binding) || dataSetBindingWriteBlocked(binding)) { - completeWorkerTask(ctx, u.repos, task, "uploader", logger) - return - } - u.waitForStorageDependency(ctx, task, logger, "Waiting for the assigned storage provider to recover") - return - } - if copyCommitSubmitted(copyRow) { - u.waitForStorageDependency(ctx, task, logger, "Waiting for storage confirmation") - return - } - reassigned, reassignErr := reassignIngressCopyAndSchedule(ctx, u.repos, u.stateMachine, version, uploadID, copyRow.CopyIndex, task.MaxRetries, task) - if reassignErr != nil { - u.handleTaskFailure(ctx, task, logger, "reassign ingress copy", reassignErr) - return - } - if reassigned == nil { - if binding != nil && binding.Status == model.StorageDataSetStatusUnavailable && dataSetBindingEstablished(binding) { - completeWorkerTask(ctx, u.repos, task, "uploader", logger) - return - } - message := "Waiting for the assigned storage provider to recover" - if binding != nil && dataSetBindingWriteBlocked(binding) { - message = "Waiting for the storage service to be replaced" - } - u.waitForStorageDependency(ctx, task, logger, message) - return - } - admin.WorkerTasksProcessed.WithLabelValues("uploader", "success").Inc() -} - -func (u *Uploader) handleDataSetCreationFailure( - ctx context.Context, - task *model.Task, - version *model.ObjectVersion, - uploadID int64, - copyRow *model.StorageUploadCopy, - binding *model.StorageDataSet, - logger *slog.Logger, - stage string, - err error, -) { - if appendErr := u.repos.Uploads.AppendUploadFailure(ctx, repository.AppendUploadFailureInput{ - UploadID: uploadID, - CopyIndex: copyRow.CopyIndex, - TransferMethod: string(copyRow.TransferMethod), - Stage: stage, - ErrorMessage: err.Error(), - }); appendErr != nil { - logger.Warn("failed to append data set creation failure", "uploadID", uploadID, "copyIndex", copyRow.CopyIndex, "error", appendErr) - } - logger.Error(stage+" failed", "error", err) - status := scheduleTaskRetry(ctx, u.repos, task, "uploader", logger, err) - if status != model.TaskStatusExhausted { - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() - return - } - cleanupCtx, cancel := context.WithTimeout(context.Background(), terminalFailureCleanupTimeout) - defer cancel() - if markErr := u.repos.Uploads.MarkUploadCopyFailed(cleanupCtx, repository.MarkUploadCopyFailedInput{ - StorageUploadCopyID: copyRow.ID, - UploadID: uploadID, - CopyIndex: copyRow.CopyIndex, - LastError: fmt.Sprintf("%s: %v", stage, err), - }); markErr != nil { - logger.Warn("failed to mark rejected data set copy failed", "uploadID", uploadID, "copyIndex", copyRow.CopyIndex, "error", markErr) - } - discarded, discardErr := u.repos.Uploads.DiscardFailedDataSetCandidate(cleanupCtx, uploadID, copyRow.CopyIndex, binding.ID) - if discardErr != nil { - logger.Warn("failed to discard rejected data set candidate", "uploadID", uploadID, "copyIndex", copyRow.CopyIndex, "dataSetID", binding.ID, "error", discardErr) - } - if discarded { - if version != nil && version.State == model.ObjectStateCommitting { - if stateErr := state.TransitionState(cleanupCtx, u.stateMachine, u.repos.Objects, version.VersionID, model.ObjectStateCommitting, model.ObjectStateUploading); stateErr != nil { - logger.Warn("failed to resume upload after rejected data set", "versionID", version.VersionID, "error", stateErr) - } - } - var enqueueErr error - if version != nil && version.State == model.ObjectStateReplicating { - enqueueErr = u.enqueueRepairUpload(cleanupCtx, task, uploadID) - } else if version != nil { - enqueueErr = u.enqueuePrepareUpload(cleanupCtx, task, version.VersionID, uploadID) - } - if enqueueErr != nil { - logger.Warn("failed to retry authorized replica slot", "uploadID", uploadID, "copyIndex", copyRow.CopyIndex, "error", enqueueErr) - } - } else if copyRow.TransferMethod == model.StorageCopyTransferMethodIngress { - reassigned, reassignErr := u.repos.Uploads.ReassignIngressCopy(cleanupCtx, uploadID, copyRow.CopyIndex) - if reassignErr != nil { - logger.Warn("failed to reassign ingress after data set rejection", "uploadID", uploadID, "copyIndex", copyRow.CopyIndex, "error", reassignErr) - } else if reassigned != nil { - if enqueueErr := u.enqueueUploadStageForCopy(cleanupCtx, task, uploadStageEnsureDataSet, uploadID, reassigned.CopyIndex, reassigned.TransferMethod, reassigned.ID); enqueueErr != nil { - logger.Warn("failed to enqueue ingress after data set rejection", "uploadID", uploadID, "copyIndex", reassigned.CopyIndex, "error", enqueueErr) - } - } - } - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() -} - -func (u *Uploader) enqueueRepairUpload(ctx context.Context, parent *model.Task, uploadID int64) error { - if parent == nil { - return errors.New("parent task is required for upload repair") - } - ref := repository.ObjectVersionRef{ObjectID: parent.RefID, VersionID: parent.RefVersionID} - return u.enqueueRepairUploadForVersion(ctx, ref, parent.MaxRetries, uploadID) -} - -func (u *Uploader) enqueueRepairUploadForVersion(ctx context.Context, ref repository.ObjectVersionRef, maxRetries int, uploadID int64) error { - stage := uploadStagePrepare - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: ref.ObjectID, - RefVersionID: ref.VersionID, - IdempotencyKey: fmt.Sprintf("upload:%s:%s:%d:repair", ref.VersionID, stage, uploadID), - Payload: map[string]interface{}{"upload_id": uploadID}, - Status: model.TaskStatusQueued, - MaxRetries: maxRetries, - ScheduledAt: time.Now(), - } - _, err := u.repos.Tasks.EnsureRecurring(ctx, task) - return err -} - -func (u *Uploader) enqueuePrepareUpload(ctx context.Context, parent *model.Task, versionID string, failedUploadID int64) error { - stage := uploadStagePrepare - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: parent.RefID, - RefVersionID: parent.RefVersionID, - IdempotencyKey: fmt.Sprintf("upload:%s:%s:%d", versionID, stage, failedUploadID), - Status: model.TaskStatusQueued, - MaxRetries: parent.MaxRetries, - ScheduledAt: time.Now(), - } - if err := u.repos.Tasks.Create(ctx, task); err != nil && !errors.Is(err, repository.ErrAlreadyExists) { - return err - } - return nil -} - -func dataSetWriteBlockedError(err error) bool { - return synapse.IsDataSetWriteBlocked(err) -} - -func dataSetBindingWriteBlocked(binding *model.StorageDataSet) bool { - if binding == nil { - return false - } - return binding.Status == model.StorageDataSetStatusDraining || binding.Status == model.StorageDataSetStatusRetired -} - -func dataSetBindingUnavailable(binding *model.StorageDataSet) bool { - return binding != nil && binding.Status == model.StorageDataSetStatusUnavailable -} - -func dataSetBindingEstablished(binding *model.StorageDataSet) bool { - return binding != nil && binding.DataSetID != nil && !binding.DataSetID.IsZero() -} - -func dataSetFailureEnded(err error, binding *model.StorageDataSet) bool { - return dataSetBindingWriteBlocked(binding) || dataSetWriteBlockedError(err) || synapse.IsDataSetServiceEnded(err) -} - -func dataSetFailureUnavailable(err error, binding *model.StorageDataSet) bool { - return dataSetBindingUnavailable(binding) || synapse.IsProviderUnavailable(err) || synapse.IsNoProviderCandidates(err) -} - -func dataSetCreationRejected(err error) bool { - return errors.Is(err, pdp.ErrTxRejected) || errors.Is(err, errDataSetCreationIncomplete) -} - -func (u *Uploader) markDataSetStatus(ctx context.Context, binding *model.StorageDataSet, status model.StorageDataSetStatus, lastError string) (*model.StorageDataSet, error) { - if binding == nil || binding.ID <= 0 { - return nil, fmt.Errorf("marking storage data set status: %w", repository.ErrInvalidInput) - } - var err error - switch status { - case model.StorageDataSetStatusDraining: - err = u.repos.Uploads.MarkDataSetDraining(ctx, binding.ID, lastError) - case model.StorageDataSetStatusUnavailable: - err = u.repos.Uploads.MarkDataSetUnavailable(ctx, binding.ID, lastError) - case model.StorageDataSetStatusFailed: - err = u.repos.Uploads.MarkDataSetFailed(ctx, binding.ID, lastError) - default: - return nil, fmt.Errorf("marking storage data set status %s: %w", status, repository.ErrInvalidInput) - } - if err == nil { - binding.Status = status - binding.LastError = &lastError - return binding, nil - } - if !errors.Is(err, repository.ErrConflict) { - return nil, err - } - latest, loadErr := u.repos.Uploads.GetDataSetBindingByID(ctx, binding.ID) - if loadErr != nil { - return nil, loadErr - } - if latest == nil { - return nil, fmt.Errorf("loading storage data set %d after status conflict: %w", binding.ID, repository.ErrNotFound) - } - return latest, nil -} - -// readyContextForCopy opens the provider context for the data set this task's -// copy is actually bound to, not for whichever generation currently owns the -// slot. A provider replacement can take the slot while an upload is mid-flight; -// resolving by slot would then store the piece on the new provider while the -// copy row being updated still belongs to the old one. -func (u *Uploader) readyContextForCopy(ctx context.Context, task *model.Task, bucket *model.Bucket, uploadID int64, copyIndex int) (*model.StorageDataSet, synapse.DataSetTarget, error) { - binding, err := u.taskCopyDataSet(ctx, task, bucket.ID, uploadID, copyIndex) - if err != nil { - return nil, nil, err - } - if binding == nil { - return nil, nil, nil - } - concrete := uploadStageCopyID(task) > 0 - usableStatus := binding.Status == model.StorageDataSetStatusReady || - concrete && binding.Status == model.StorageDataSetStatusDraining - if !usableStatus || - binding.DataSetID == nil || binding.DataSetID.IsZero() { - return binding, nil, fmt.Errorf("dataset binding %d is not ready", binding.ID) - } - storageCtx, err := u.contextForReadyBinding(ctx, binding) - if err != nil { - return binding, nil, err - } - return binding, storageCtx, nil -} - -func (u *Uploader) contextForBindingProvider(ctx context.Context, binding *model.StorageDataSet, bucketName string) (synapse.ProviderTarget, error) { - storageCtx, err := u.storage.OpenProviderTarget(ctx, binding.ProviderID.SDK(), storage.NewProviderContextOptions{ - DataSetMetadata: map[string]string{"bucket": bucketName}, - }) - if err != nil { - return nil, err - } - if storageCtx == nil { - return nil, errors.New("storage context resolver returned no context") - } - return storageCtx, nil -} - -func (u *Uploader) contextForReadyBinding(ctx context.Context, binding *model.StorageDataSet) (synapse.DataSetTarget, error) { - if binding == nil || binding.DataSetID == nil || binding.DataSetID.IsZero() { - return nil, errors.New("storage data set binding is not ready") - } - providerID := binding.ProviderID.SDK() - storageCtx, err := u.storage.OpenDataSetTarget(ctx, binding.DataSetID.SDK(), storage.NewDataSetContextOptions{ - ProviderID: &providerID, - }) - if err != nil { - return nil, err - } - if storageCtx == nil { - return nil, errors.New("storage context resolver returned no context") - } - if got := idtypes.OnChainIDFromSDK(storageCtx.ProviderID()); !got.Equal(binding.ProviderID) { - return nil, fmt.Errorf("dataset %s resolved provider %s, want %s", binding.DataSetID, got.String(), binding.ProviderID.String()) - } - ref, bound := storageCtx.DataSetRef() - if !bound { - return nil, fmt.Errorf("storage context did not resolve requested data set %s", binding.DataSetID.String()) - } - resolvedDataSetID, resolvedClientDataSetID, err := dataSetRefIDsForBinding(binding, ref) - if err != nil { - return nil, err - } - if !resolvedDataSetID.Equal(*binding.DataSetID) { - return nil, fmt.Errorf("storage context resolved data set %s, want %s", resolvedDataSetID.String(), binding.DataSetID.String()) - } - if binding.ClientDataSetID == nil { - if err := u.repos.Uploads.BackfillClientDataSetID(ctx, repository.BackfillClientDataSetIDInput{ - ID: binding.ID, - DataSetID: resolvedDataSetID, - ClientDataSetID: resolvedClientDataSetID, - }); err != nil { - return nil, err - } - binding.ClientDataSetID = &resolvedClientDataSetID - } else if !binding.ClientDataSetID.Equal(resolvedClientDataSetID) { - return nil, fmt.Errorf("dataset %s resolved client data set ID %s, want %s", - binding.DataSetID.String(), resolvedClientDataSetID.String(), binding.ClientDataSetID.String()) - } - return storageCtx, nil -} - -func dataSetRefIDsForBinding(binding *model.StorageDataSet, ref storage.DataSetRef) (idtypes.OnChainID, idtypes.OnChainID, error) { - if binding == nil { - return idtypes.OnChainID{}, idtypes.OnChainID{}, errors.New("storage data set binding is missing") - } - providerID := idtypes.OnChainIDFromSDK(ref.ProviderID()) - if !providerID.Equal(binding.ProviderID) { - return idtypes.OnChainID{}, idtypes.OnChainID{}, fmt.Errorf("dataset resolved provider %s, want %s", providerID.String(), binding.ProviderID.String()) - } - dataSetID := idtypes.OnChainIDFromSDK(ref.DataSetID()) - if dataSetID.IsZero() { - return idtypes.OnChainID{}, idtypes.OnChainID{}, errors.New("dataset resolved a zero data set ID") - } - return dataSetID, idtypes.OnChainIDFromSDK(ref.ClientDataSetID()), nil -} - -func dataSetResultIDsForBinding(binding *model.StorageDataSet, result *storage.CreateDataSetResult) (idtypes.OnChainID, idtypes.OnChainID, error) { - if result == nil { - return idtypes.OnChainID{}, idtypes.OnChainID{}, errors.New("storage provider returned no data set creation result") - } - return dataSetRefIDsForBinding(binding, result.DataSet) -} - -// extraDataForCopy takes the concrete copy rather than a replica slot: the -// cached presign blob belongs to one generation, and resolving it by slot would -// hand back the replacement's blob after an activation. -func (u *Uploader) extraDataForCopy(ctx context.Context, storageCtx synapse.DataSetTarget, copyRow *model.StorageUploadCopy, pieces []storage.PieceInput) ([]byte, string, error) { - if copyRow != nil && copyRow.CommitExtraDataHex != nil && *copyRow.CommitExtraDataHex != "" { - extraData, err := hex.DecodeString(*copyRow.CommitExtraDataHex) - return extraData, strings.ToLower(*copyRow.CommitExtraDataHex), err - } - extraData, err := storageCtx.PresignForCommit(ctx, pieces) - if err != nil { - return nil, "", err - } - return extraData, strings.ToLower(hex.EncodeToString(extraData)), nil -} - -func (u *Uploader) advanceStorageCommit( - ctx context.Context, - binding *model.StorageDataSet, - copyRow *model.StorageUploadCopy, - storageCtx synapse.DataSetTarget, - pieces []storage.PieceInput, - requireEligibleCopy bool, - ownerTerminal bool, -) (storagecommit.AdvanceResult, error) { - if binding == nil || copyRow == nil { - return storagecommit.AdvanceResult{}, errors.New("storage commit binding or copy is missing") - } - advancer := storagecommit.Advancer{ - Store: u.repos.Uploads, - StatusChecker: u.statusChecker, - RequestTimeout: submittedCommitRequestTimeout, - } - return advancer.Advance(ctx, storagecommit.AdvanceInput{ - Copy: *copyRow, - Binding: *binding, - Target: storageCtx, - Pieces: pieces, - RequireEligibleCopy: requireEligibleCopy, - OwnerTerminal: ownerTerminal, - }) -} - -func (u *Uploader) handleUnavailableCommitContext( - ctx context.Context, - task *model.Task, - binding *model.StorageDataSet, - copyRow *model.StorageUploadCopy, - logger *slog.Logger, - stage string, -) bool { - if !copyCommitSubmitted(copyRow) || binding == nil { - return false - } - advancer := storagecommit.Advancer{Store: u.repos.Uploads} - result, err := advancer.AdvanceUnavailable(ctx, *copyRow, *binding) - if err != nil { - u.handleTaskFailure(ctx, task, logger, stage, err) - return true - } - if !u.waitForCommitAdvance(ctx, task, logger, result) { - u.handleTaskFailure(ctx, task, logger, stage, errors.New("unavailable storage commit returned an unexpected state")) - } - return true -} - -func (u *Uploader) waitForCommitAdvance( - ctx context.Context, - task *model.Task, - logger *slog.Logger, - result storagecommit.AdvanceResult, -) bool { - var ( - delay time.Duration - message string - ) - switch result.State { - case storagecommit.AdvanceWaitingCapacity: - delay = u.commitPollDelay() - message = "Waiting to submit stored content" - // Once every slot is flagged for attention this queue cannot be relied on - // to move by itself, so point at the confirmations instead of letting it - // read like an ordinary queue. Partial holds keep the ordinary wording, - // because the unflagged attempts are still working through. - // - // This is the state that persists until someone acts on it, and the - // branch runs once per poll for every blocked task, so the log stays at - // debug. The wait message above is the durable, operator-visible record. - if result.AttentionHeld >= storagecommit.MaxActiveAttemptsPerDataSet { - message = "Waiting for storage confirmations that need attention" - logger.Debug("storage commit capacity is fully held by flagged confirmations", - "attentionHeld", result.AttentionHeld, "capacity", storagecommit.MaxActiveAttemptsPerDataSet) - } - case storagecommit.AdvanceSubmitted, storagecommit.AdvancePending: - delay = u.commitPollDelay() - message = "Waiting for storage confirmation" - case storagecommit.AdvanceNeedsAttention: - if result.Continue { - delay = commitObservationDelay(u.commitPollDelay()) - message = "Waiting for storage confirmation" - } else { - delay = storageCommitAttentionDelay - message = "Storage confirmation needs review" - } - case storagecommit.AdvanceReleased: - if result.ReleaseReason != storagecommit.ReleaseBeforeSubmitCanceled { - return false - } - delay = u.commitPollDelay() - message = "Waiting to retry storage submission" - default: - return false - } - waitCtx := ctx - cancel := func() {} - if result.State == storagecommit.AdvanceReleased { - waitCtx, cancel = providerEvidenceContext(ctx) - } - defer cancel() - if err := u.repos.Tasks.WaitRunning(waitCtx, task, model.TaskWaitReasonExternalConfirmation, message, delay); err != nil { - logger.Error("failed to wait for storage confirmation", "error", err) - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() - return true - } - admin.WorkerTasksProcessed.WithLabelValues("uploader", "success").Inc() - return true -} - -func (u *Uploader) commitPollDelay() time.Duration { - if u != nil && u.pollInterval > 0 { - return u.pollInterval - } - return storageCommitPollDelay -} - -func commitObservationDelay(pollInterval time.Duration) time.Duration { - return max(pollInterval, storageCommitObservationDelay) -} - -// commitReleaseCause names why a released commit gave up on its data set. The -// advancer carries the provider error when it has one, which keeps the recorded -// reason specific; the sentinel preserves the classification when it does not. -func commitReleaseCause(result storagecommit.AdvanceResult) error { - if result.Cause != nil { - return result.Cause - } - return storage.ErrDataSetUnavailable -} - -func onChainIDPtrFromSDK(value sdktypes.BigInt) *idtypes.OnChainID { - id := idtypes.OnChainIDFromSDK(value) - return &id -} - -func onChainIDPtrFromSDKPtr(value *sdktypes.BigInt) *idtypes.OnChainID { - if value == nil { - return nil - } - return onChainIDPtrFromSDK(*value) -} - -func sdkBigIntPtr(value *idtypes.OnChainID) *sdktypes.BigInt { - if value == nil { - return nil - } - id := value.SDK() - return &id -} - -func derefString(value *string) string { - if value == nil { - return "" - } - return *value -} - -func copyHasPiece(copyRow *model.StorageUploadCopy) bool { - if copyRow == nil { - return false - } - switch copyRow.Status { - case model.StorageUploadCopyStatusPieceReady, model.StorageUploadCopyStatusCommitting, model.StorageUploadCopyStatusCommitted: - return true - default: - return false - } -} - -func copyCommitSubmitted(copyRow *model.StorageUploadCopy) bool { - return copyRow != nil && copyRow.Status == model.StorageUploadCopyStatusCommitting && - copyRow.CommitAttemptID != nil && *copyRow.CommitAttemptID != "" && - copyRow.CommitAttemptedAt != nil -} - -func copyCommitted(copyRow *model.StorageUploadCopy) bool { - return copyRow != nil && copyRow.Status == model.StorageUploadCopyStatusCommitted -} - -func (u *Uploader) submittedCommitPresence(ctx context.Context, uploadID int64, copyIndex int) (bool, bool, error) { - copies, err := u.repos.Uploads.ListCopies(ctx, uploadID) - if err != nil { - return false, false, err - } - currentSubmitted := false - otherSubmitted := false - for i := range copies { - if !copyCommitSubmitted(&copies[i]) { - continue - } - if copies[i].CopyIndex == copyIndex { - currentSubmitted = true - } else { - otherSubmitted = true - } - } - return currentSubmitted, otherSubmitted, nil -} - -func dataSetBindingCanEnsureWrite(binding *model.StorageDataSet) bool { - return binding != nil && dataSetStatusCanEnsureWrite(binding.Status) -} - -func uploadCanUseDataSetBinding(uploadID int64, binding *model.StorageDataSet) bool { - if binding == nil { - return false - } - if binding.Status == model.StorageDataSetStatusReady { - return true - } - if binding.Status == model.StorageDataSetStatusFailed && dataSetBindingHasCreationEvidence(binding) { - return false - } - return binding.CreatedByUploadID != nil && - *binding.CreatedByUploadID == uploadID && - dataSetBindingCanEnsureWrite(binding) -} - -func dataSetBindingCanOccupyUploadSlot(uploadID int64, binding *model.StorageDataSet) bool { - if binding == nil { - return false - } - switch binding.Status { - case model.StorageDataSetStatusReady: - return true - case model.StorageDataSetStatusUnavailable: - return dataSetBindingEstablished(binding) - case model.StorageDataSetStatusPending, model.StorageDataSetStatusCreating, model.StorageDataSetStatusFailed: - return uploadCanUseDataSetBinding(uploadID, binding) - default: - return false - } -} - -func dataSetBindingInFlightForOtherUpload(uploadID int64, binding *model.StorageDataSet) bool { - if binding == nil || dataSetBindingCanOccupyUploadSlot(uploadID, binding) { - return false - } - switch binding.Status { - case model.StorageDataSetStatusPending, model.StorageDataSetStatusCreating: - return true - default: - return false - } -} - -func uploadTracksDataSetBinding(uploadID int64, binding *model.StorageDataSet) bool { - if binding == nil { - return false - } - if dataSetBindingHasCreationEvidence(binding) { - return true - } - return binding.CreatedByUploadID != nil && *binding.CreatedByUploadID == uploadID -} - -func dataSetBindingHasCreationEvidence(binding *model.StorageDataSet) bool { - if binding == nil { - return false - } - return dataSetBindingEstablished(binding) || - (binding.ClientDataSetID != nil && !binding.ClientDataSetID.IsZero()) || - (binding.CreateTransactionID != nil && *binding.CreateTransactionID != "") || - (binding.CreateStatusURL != nil && *binding.CreateStatusURL != "") -} - -func dataSetStatusCanEnsureWrite(status model.StorageDataSetStatus) bool { - switch status { - case model.StorageDataSetStatusPending, - model.StorageDataSetStatusCreating, - model.StorageDataSetStatusReady, - model.StorageDataSetStatusFailed: - return true - default: - return false - } -} - -func (u *Uploader) handleTaskFailure(ctx context.Context, task *model.Task, logger *slog.Logger, stage string, err error) { - logger.Error(stage+" failed", "error", err) - scheduleTaskRetry(ctx, u.repos, task, "uploader", logger, err) - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() -} - -func (u *Uploader) handleCommitTaskFailure( - ctx context.Context, - task *model.Task, - copyRow *model.StorageUploadCopy, - logger *slog.Logger, - stage string, - err error, -) { - logger.Error(stage+" failed", "error", err) - if task == nil || task.RetryCount+1 < task.MaxRetries || copyRow == nil || copyRow.StorageDataSetID == nil { - scheduleTaskRetry(ctx, u.repos, task, "uploader", logger, err) - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() - return - } - - terminalCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), terminalFailureCleanupTimeout) - defer cancel() - identity := storagecommit.CopyIdentity{ - StorageUploadCopyID: copyRow.ID, - UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, - StorageDataSetID: *copyRow.StorageDataSetID, - } - status := model.TaskStatus("") - retryErr := u.repos.WithTx(terminalCtx, func(txRepos *repository.Repositories) error { - fresh, loadErr := txRepos.Uploads.GetUploadCopyByID(terminalCtx, copyRow.ID) - if loadErr != nil { - return loadErr - } - if fresh == nil || fresh.StorageDataSetID == nil || *fresh.StorageDataSetID != identity.StorageDataSetID { - return repository.ErrConflict - } - if fresh.CommitAttemptID != nil && *fresh.CommitAttemptID != "" { - if fresh.CommitAttemptedAt != nil { - return errCommitAttemptBecameActive - } - if releaseErr := txRepos.Uploads.ReleaseCommitAttempt(terminalCtx, storagecommit.ReleaseInput{ - Copy: identity, - AttemptID: *fresh.CommitAttemptID, - ClearReadyAt: true, - ClearExtraData: true, - }); releaseErr != nil { - return releaseErr - } - } else if releaseErr := txRepos.Uploads.ReleaseCommitReservation(terminalCtx, storagecommit.ReservationReleaseInput{ - Copy: identity, - ClearReadyAt: true, - ClearExtraData: true, - }); releaseErr != nil { - return releaseErr - } - var scheduleErr error - status, scheduleErr = txRepos.Tasks.ScheduleRetryRunning( - terminalCtx, task, err.Error(), retryDelay(task.RetryCount), - ) - if scheduleErr != nil { - return scheduleErr - } - if status != model.TaskStatusExhausted { - return errCommitRetryDidNotExhaust - } - return nil - }) - if retryErr != nil { - fresh, loadErr := u.repos.Uploads.GetUploadCopyByID(terminalCtx, copyRow.ID) - if loadErr == nil && fresh != nil && fresh.CommitAttemptID != nil && *fresh.CommitAttemptID != "" && - fresh.CommitAttemptedAt != nil { - logger.Warn("storage commit became externally observable before task exhaustion", - "stage", stage, "attemptID", *fresh.CommitAttemptID) - u.waitForCommitAdvance(terminalCtx, task, logger, storagecommit.AdvanceResult{ - State: storagecommit.AdvancePending, AttemptID: *fresh.CommitAttemptID, - }) - return - } - logger.Error("failed to exhaust commit task and release its reservation", "error", retryErr) - // The settlement rolled back, so this task still holds its claim. Park it - // briefly rather than leaving it running: ordinary task leases are only - // reclaimed when the process starts, so a task abandoned here stays stuck - // until the next restart. - if waitErr := u.repos.Tasks.WaitRunning( - terminalCtx, task, model.TaskWaitReasonExternalConfirmation, - "Waiting for storage confirmation", u.commitPollDelay(), - ); waitErr != nil { - logger.Error("failed to park commit task after settlement rollback", "error", waitErr) - } - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() - return - } - admin.TasksExhaustedTotal.WithLabelValues("uploader", string(task.Type)).Inc() - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() -} - -var ( - errCommitAttemptBecameActive = errors.New("storage commit attempt became active") - errCommitRetryDidNotExhaust = errors.New("storage commit retry did not exhaust") -) - -func (u *Uploader) handleFailure(ctx context.Context, task *model.Task, version *model.ObjectVersion, logger *slog.Logger, stage string, err error) model.TaskStatus { - logger.Error(stage+" failed", "error", err) - status := scheduleTaskRetry(ctx, u.repos, task, "uploader", logger, err) - if status == model.TaskStatusExhausted { - cleanupCtx, cancel := context.WithTimeout(context.Background(), terminalFailureCleanupTimeout) - defer cancel() - u.failUploadingContent(cleanupCtx, task, version, logger, fmt.Sprintf("%s: %v (max retries reached)", stage, err)) - } - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() - return status -} - -func (u *Uploader) handleIngressFailure(ctx context.Context, task *model.Task, version *model.ObjectVersion, uploadID int64, copyIndex int, logger *slog.Logger, stage string, err error) { - if stage != "cache read" { - if appendErr := u.repos.Uploads.AppendUploadFailure(ctx, repository.AppendUploadFailureInput{ - UploadID: uploadID, - CopyIndex: copyIndex, - TransferMethod: string(model.StorageCopyTransferMethodIngress), - Stage: stage, - ErrorMessage: err.Error(), - }); appendErr != nil { - logger.Warn("failed to append ingress upload failure", "uploadID", uploadID, "copyIndex", copyIndex, "error", appendErr) - } - } - currentSubmitted, otherSubmitted, checkErr := u.submittedCommitPresence(ctx, uploadID, copyIndex) - if checkErr != nil { - u.handleTaskFailure(ctx, task, logger, "check submitted ingress commits", checkErr) - return - } - if currentSubmitted || otherSubmitted { - logger.Error(stage+" failed while a commit remains submitted", "error", err) - status := scheduleTaskRetry(ctx, u.repos, task, "uploader", logger, err) - if status == model.TaskStatusExhausted && !currentSubmitted { - cleanupCtx, cancel := context.WithTimeout(context.Background(), terminalFailureCleanupTimeout) - defer cancel() - if markErr := u.repos.Uploads.MarkUploadCopyFailed(cleanupCtx, repository.MarkUploadCopyFailedInput{ - StorageUploadCopyID: uploadStageCopyID(task), - UploadID: uploadID, - CopyIndex: copyIndex, - LastError: fmt.Sprintf("%s: %v", stage, err), - }); markErr != nil { - logger.Warn("failed to mark alternate ingress upload copy failed", "uploadID", uploadID, "copyIndex", copyIndex, "error", markErr) - } - } - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() - return - } - status := u.handleFailure(ctx, task, version, logger, stage, err) - if status == model.TaskStatusExhausted { - cleanupCtx, cancel := context.WithTimeout(context.Background(), terminalFailureCleanupTimeout) - defer cancel() - if markErr := u.repos.Uploads.MarkUploadCopyFailed(cleanupCtx, repository.MarkUploadCopyFailedInput{ - StorageUploadCopyID: uploadStageCopyID(task), - UploadID: uploadID, - CopyIndex: copyIndex, - LastError: fmt.Sprintf("%s: %v", stage, err), - }); markErr != nil { - logger.Warn("failed to mark ingress upload copy failed", "uploadID", uploadID, "copyIndex", copyIndex, "error", markErr) - } - } -} - -func (u *Uploader) handleIngressDataSetFailure(ctx context.Context, task *model.Task, version *model.ObjectVersion, uploadID int64, copyIndex int, dataSetID int64, logger *slog.Logger, stage string, err error) { - var binding *model.StorageDataSet - if dataSetID > 0 { - var loadErr error - binding, loadErr = u.repos.Uploads.GetDataSetBindingByID(ctx, dataSetID) - if loadErr != nil { - u.handleTaskFailure(ctx, task, logger, "load failed ingress data set", loadErr) - return - } - } - if dataSetFailureEnded(err, binding) || dataSetFailureUnavailable(err, binding) { - copyRow, loadErr := u.taskUploadCopy(ctx, task, uploadID, copyIndex) - if loadErr != nil || copyRow == nil { - if loadErr == nil { - loadErr = errors.New("ingress upload copy not found") - } - u.handleTaskFailure(ctx, task, logger, "load failed ingress copy", loadErr) - return - } - if dataSetFailureEnded(err, binding) { - latest, markErr := u.markDataSetStatus(ctx, binding, model.StorageDataSetStatusDraining, err.Error()) - if markErr != nil { - u.handleTaskFailure(ctx, task, logger, "mark ingress data set draining", markErr) - return - } - binding = latest - } else { - latest, markErr := u.markDataSetStatus(ctx, binding, model.StorageDataSetStatusUnavailable, err.Error()) - if markErr != nil { - u.handleTaskFailure(ctx, task, logger, "mark ingress data set unavailable", markErr) - return - } - binding = latest - } - u.handleAssignedDataSetDependency(ctx, task, version, uploadID, copyRow, binding, logger, stage, err) - return - } - u.handleIngressFailure(ctx, task, version, uploadID, copyIndex, logger, stage, err) -} - -func (u *Uploader) failUploadingContent(ctx context.Context, task *model.Task, version *model.ObjectVersion, logger *slog.Logger, lastError string) { - refs, err := u.repos.Objects.FailUploadingContentFollowers(ctx, version.BucketID, version.Size, version.Checksum, task.RefVersionID, lastError) - if err == nil { - logger.Info("marked matching active upload versions failed", "count", len(refs)) - return - } - logger.Warn("failed to mark matching active upload versions failed", "error", err) - from := version.State - if from != model.ObjectStateUploading && from != model.ObjectStateCommitting { - logger.Warn("cannot transition non-ingress upload state to failed", "state", from) - return - } - _ = state.TransitionToFailed(ctx, u.stateMachine, u.repos.Objects, task.RefVersionID, from, lastError) -} diff --git a/internal/worker/uploader_internal_test.go b/internal/worker/uploader_internal_test.go deleted file mode 100644 index a29990f..0000000 --- a/internal/worker/uploader_internal_test.go +++ /dev/null @@ -1,565 +0,0 @@ -package worker - -import ( - "context" - "errors" - "fmt" - "io" - "log/slog" - "testing" - "time" - - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagecommit" - "github.com/strahe/synaps3/internal/synapse" - "github.com/strahe/synaps3/internal/testutil" - "github.com/strahe/synapse-go/storage" - sdktypes "github.com/strahe/synapse-go/types" -) - -func seedCommitFailureCopy( - t *testing.T, - repos *repository.Repositories, - bucket *model.Bucket, - dataSet *model.StorageDataSet, - sourceVersionID string, -) *model.StorageUploadCopy { - t.Helper() - upload, err := repos.Uploads.StartObjectUploadAttempt(t.Context(), repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: sourceVersionID, ContentSize: 1, - Checksum: "checksum-" + sourceVersionID, RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(t.Context(), upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: dataSet.ID, - CopyIndex: dataSet.CopyIndex, - TransferMethod: model.StorageCopyTransferMethodPeerPull, - ProviderID: dataSet.ProviderID, - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - copyRow, err := repos.Uploads.GetUploadCopyForDataSet(t.Context(), upload.ID, dataSet.ID) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopyForDataSet: copy=%#v err=%v", copyRow, err) - } - if err := repos.Uploads.MarkUploadCopyPieceReady(t.Context(), repository.MarkUploadCopyPieceReadyInput{ - StorageUploadCopyID: copyRow.ID, UploadID: upload.ID, CopyIndex: copyRow.CopyIndex, - PieceCID: "bafkqaaa", RetrievalURL: "https://provider.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - copyRow, err = repos.Uploads.GetUploadCopyByID(t.Context(), copyRow.ID) - if err != nil { - t.Fatalf("reload copy: %v", err) - } - return copyRow -} - -func seedCommitFailureFixture( - t *testing.T, - maxRetries int, -) (*repository.Repositories, *model.StorageUploadCopy, *model.StorageDataSet, *model.Bucket, *model.Task) { - t.Helper() - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - bucket := testutil.SeedBucket(t, db, "commit-failure-"+model.NewVersionID()) - owner, err := repos.Uploads.StartObjectUploadAttempt(t.Context(), repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: model.NewVersionID(), ContentSize: 1, - Checksum: "commit-failure-owner", RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt owner: %v", err) - } - dataSet, err := repos.Uploads.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, "901"), CopyIndex: 0, CreatedByUploadID: owner.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(t.Context(), repository.MarkDataSetReadyInput{ - ID: dataSet.ID, UploadID: owner.ID, DataSetID: onChainID(t, "9001"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - dataSet, err = repos.Uploads.GetDataSetBindingByID(t.Context(), dataSet.ID) - if err != nil || dataSet == nil { - t.Fatalf("GetDataSetBindingByID: dataSet=%#v err=%v", dataSet, err) - } - copyRow := seedCommitFailureCopy(t, repos, bucket, dataSet, model.NewVersionID()) - identity := storagecommit.CopyIdentity{ - StorageUploadCopyID: copyRow.ID, UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, StorageDataSetID: dataSet.ID, - } - if _, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: identity, AttemptID: "commit-failure-reservation", - }); err != nil { - t.Fatalf("ReserveCommitAttempt: %v", err) - } - stage := uploadStagePeerCommit - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "object", RefID: bucket.ID, - RefVersionID: model.NewVersionID(), IdempotencyKey: "commit-failure-task-" + model.NewVersionID(), - Payload: map[string]interface{}{}, Status: model.TaskStatusQueued, - MaxRetries: maxRetries, ScheduledAt: time.Now(), - } - if err := repos.Tasks.Create(t.Context(), task); err != nil { - t.Fatalf("Create task: %v", err) - } - claimed, err := repos.Tasks.ClaimReady(t.Context(), model.TaskTypeUpload, time.Minute) - if err != nil || claimed == nil { - t.Fatalf("ClaimReady: task=%#v err=%v", claimed, err) - } - return repos, copyRow, dataSet, bucket, claimed -} - -func TestCommitTaskFailureReservationLifecycle(t *testing.T) { - logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - - t.Run("transient retry retains FIFO", func(t *testing.T) { - repos, copyRow, _, _, task := seedCommitFailureFixture(t, 2) - identity := storagecommit.CopyIdentity{ - StorageUploadCopyID: copyRow.ID, UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, StorageDataSetID: *copyRow.StorageDataSetID, - } - if err := repos.Uploads.ReleaseCommitAttempt(t.Context(), storagecommit.ReleaseInput{ - Copy: identity, AttemptID: "commit-failure-reservation", - }); err != nil { - t.Fatalf("seed ready-only reservation: %v", err) - } - (&Uploader{repos: repos}).handleCommitTaskFailure(t.Context(), task, copyRow, logger, "commit", errors.New("presign failed")) - persisted, err := repos.Uploads.GetUploadCopyByID(t.Context(), copyRow.ID) - if err != nil || persisted.CommitReadyAt == nil || persisted.CommitAttemptID != nil { - t.Fatalf("transient copy = %#v err=%v, want retained FIFO", persisted, err) - } - gotTask, err := repos.Tasks.GetByID(t.Context(), task.ID) - if err != nil || gotTask.Status != model.TaskStatusScheduled || gotTask.RetryCount != 1 { - t.Fatalf("transient task = %#v err=%v", gotTask, err) - } - }) - - t.Run("terminal retry releases FIFO for successor", func(t *testing.T) { - repos, copyRow, dataSet, bucket, task := seedCommitFailureFixture(t, 1) - identity := storagecommit.CopyIdentity{ - StorageUploadCopyID: copyRow.ID, UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, StorageDataSetID: *copyRow.StorageDataSetID, - } - if err := repos.Uploads.ReleaseCommitAttempt(t.Context(), storagecommit.ReleaseInput{ - Copy: identity, AttemptID: "commit-failure-reservation", - }); err != nil { - t.Fatalf("seed ready-only reservation: %v", err) - } - follower := seedCommitFailureCopy(t, repos, bucket, dataSet, model.NewVersionID()) - followerIdentity := storagecommit.CopyIdentity{ - StorageUploadCopyID: follower.ID, UploadID: follower.UploadID, - CopyIndex: follower.CopyIndex, StorageDataSetID: *follower.StorageDataSetID, - } - waiting, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: followerIdentity, AttemptID: "follower-waiting", - }) - if err != nil || waiting.State != storagecommit.ReservationWaiting { - t.Fatalf("follower before cleanup = %#v err=%v, want waiting", waiting, err) - } - (&Uploader{repos: repos}).handleCommitTaskFailure(t.Context(), task, copyRow, logger, "commit", errors.New("presign failed")) - persisted, err := repos.Uploads.GetUploadCopyByID(t.Context(), copyRow.ID) - if err != nil || persisted.CommitReadyAt != nil || persisted.CommitAttemptID != nil { - t.Fatalf("terminal copy = %#v err=%v, want cleared FIFO", persisted, err) - } - gotTask, err := repos.Tasks.GetByID(t.Context(), task.ID) - if err != nil || gotTask.Status != model.TaskStatusExhausted || gotTask.RetryCount != 1 { - t.Fatalf("terminal task = %#v err=%v", gotTask, err) - } - admitted, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: followerIdentity, AttemptID: "follower-admitted", - }) - if err != nil || admitted.State != storagecommit.ReservationAcquired { - t.Fatalf("follower after cleanup = %#v err=%v, want acquired", admitted, err) - } - }) - - t.Run("attempted fence parks without exhaustion", func(t *testing.T) { - repos, copyRow, _, _, task := seedCommitFailureFixture(t, 1) - identity := storagecommit.CopyIdentity{ - StorageUploadCopyID: copyRow.ID, UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, StorageDataSetID: *copyRow.StorageDataSetID, - } - if _, err := repos.Uploads.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ - Copy: identity, AttemptID: "commit-failure-reservation", ExtraDataHex: "abcd", - }); err != nil { - t.Fatalf("MarkCommitAttempted: %v", err) - } - (&Uploader{repos: repos, pollInterval: time.Second}).handleCommitTaskFailure( - t.Context(), task, copyRow, logger, "commit", errors.New("persist evidence failed"), - ) - persisted, err := repos.Uploads.GetUploadCopyByID(t.Context(), copyRow.ID) - if err != nil || persisted.CommitAttemptID == nil || *persisted.CommitAttemptID != "commit-failure-reservation" || - persisted.CommitAttemptedAt == nil { - t.Fatalf("attempted copy = %#v err=%v, want preserved fence", persisted, err) - } - gotTask, err := repos.Tasks.GetByID(t.Context(), task.ID) - if err != nil || gotTask.Status != model.TaskStatusWaiting || gotTask.RetryCount != 0 || - gotTask.WaitReason == nil || *gotTask.WaitReason != model.TaskWaitReasonExternalConfirmation { - t.Fatalf("attempted task = %#v err=%v, want confirmation wait", gotTask, err) - } - }) - - t.Run("settlement rollback parks instead of stranding the claim", func(t *testing.T) { - repos, copyRow, _, _, task := seedCommitFailureFixture(t, 1) - identity := storagecommit.CopyIdentity{ - StorageUploadCopyID: copyRow.ID, UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, StorageDataSetID: *copyRow.StorageDataSetID, - } - if err := repos.Uploads.ReleaseCommitAttempt(t.Context(), storagecommit.ReleaseInput{ - Copy: identity, AttemptID: "commit-failure-reservation", - }); err != nil { - t.Fatalf("seed ready-only reservation: %v", err) - } - // Failing the copy leaves nothing for the terminal settlement to release, so - // its transaction rolls back without the attempt ever becoming observable. - if err := repos.Uploads.MarkUploadCopyFailed(t.Context(), repository.MarkUploadCopyFailedInput{ - StorageUploadCopyID: copyRow.ID, UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, LastError: "copy failed before settlement", - }); err != nil { - t.Fatalf("MarkUploadCopyFailed: %v", err) - } - (&Uploader{repos: repos, pollInterval: time.Second}).handleCommitTaskFailure( - t.Context(), task, copyRow, logger, "commit", errors.New("presign failed"), - ) - gotTask, err := repos.Tasks.GetByID(t.Context(), task.ID) - if err != nil || gotTask.Status != model.TaskStatusWaiting || - gotTask.WaitReason == nil || *gotTask.WaitReason != model.TaskWaitReasonExternalConfirmation { - t.Fatalf("rolled back task = %#v err=%v, want a parked task rather than a held claim", gotTask, err) - } - }) -} - -func TestEnsureBucketProviderBindingsPersistsPartialSelectionBeforeWaiting(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := &model.Bucket{Name: "partial-selection-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Create bucket: %v", err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J0000000000000PARTIAL01", - ContentSize: 1024, - Checksum: "partial-selection-checksum", - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - providerTarget := testutil.NewMockProviderTarget(sdktypes.NewBigInt(101), storage.NewProviderContextOptions{}) - dataSetTarget := testutil.NewMockDataSetTarget(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), nil) - dataSetTarget.ClientDataSetIDValue = sdktypes.NewBigInt(9002) - client := &testutil.MockStorageClient{ - SelectUploadTargetsFunc: func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - if opts.Copies != 3 { - t.Fatalf("selection copies = %d, want 3", opts.Copies) - } - return []synapse.StorageTarget{providerTarget, dataSetTarget}, &synapse.NoProviderCandidatesError{ - Cause: &storage.InsufficientUploadContextsError{Requested: 3, Available: 2}, - } - }, - } - uploader := &Uploader{repos: repos, storage: client} - - plan, err := uploader.ensureBucketProviderBindings(ctx, bucket, upload.ID, 3) - if !synapse.IsNoProviderCandidates(err) || plan.complete || len(plan.bindings) != 2 { - t.Fatalf("plan = %#v, error = %T %v; want two persisted partial bindings", plan, err, err) - } - pending := plan.byCopyIndex[0] - ready := plan.byCopyIndex[1] - if pending == nil || pending.ProviderID.String() != "101" || pending.Status != model.StorageDataSetStatusPending { - t.Fatalf("provider binding = %#v, want pending provider 101", pending) - } - if ready == nil || ready.ProviderID.String() != "202" || ready.Status != model.StorageDataSetStatusReady || - ready.DataSetID == nil || ready.DataSetID.String() != "2002" || - ready.ClientDataSetID == nil || ready.ClientDataSetID.String() != "9002" { - t.Fatalf("data set binding = %#v, want complete ready identity", ready) - } -} - -func TestContextForReadyBindingBackfillsClientDataSetIDAndRejectsConflict(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := &model.Bucket{Name: "ready-binding-backfill-bucket", Status: model.BucketStatusActive} - if err := repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("Create bucket: %v", err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J00000000000BACKFILL01", - ContentSize: 1024, - Checksum: "ready-binding-backfill-checksum", - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - dataSetID := onChainID(t, "1001") - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, UploadID: upload.ID, DataSetID: dataSetID, - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - binding, err = repos.Uploads.GetDataSetBindingByID(ctx, binding.ID) - if err != nil || binding == nil { - t.Fatalf("GetDataSetBindingByID: binding=%#v err=%v", binding, err) - } - target := testutil.NewMockDataSetTarget(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), nil) - target.ClientDataSetIDValue = sdktypes.NewBigInt(9001) - client := &testutil.MockStorageClient{ - OpenDataSetTargetFunc: func(_ context.Context, gotDataSetID sdktypes.BigInt, opts storage.NewDataSetContextOptions) (synapse.DataSetTarget, error) { - if !gotDataSetID.Equal(sdktypes.NewBigInt(1001)) || opts.ProviderID == nil || !opts.ProviderID.Equal(sdktypes.NewBigInt(101)) { - t.Fatalf("OpenDataSetTarget inputs = dataSet:%s provider:%v", gotDataSetID.String(), opts.ProviderID) - } - return target, nil - }, - } - uploader := &Uploader{repos: repos, storage: client} - - if _, err := uploader.contextForReadyBinding(ctx, binding); err != nil { - t.Fatalf("contextForReadyBinding: %v", err) - } - if binding.ClientDataSetID == nil || binding.ClientDataSetID.String() != "9001" { - t.Fatalf("in-memory client data set ID = %v, want 9001", binding.ClientDataSetID) - } - stored, err := repos.Uploads.GetDataSetBindingByID(ctx, binding.ID) - if err != nil || stored == nil || stored.ClientDataSetID == nil || stored.ClientDataSetID.String() != "9001" { - t.Fatalf("stored binding = %#v err=%v, want client data set ID 9001", stored, err) - } - - target.ClientDataSetIDValue = sdktypes.NewBigInt(9002) - if _, err := uploader.contextForReadyBinding(ctx, binding); err == nil { - t.Fatal("contextForReadyBinding accepted a conflicting client data set ID") - } -} - -func TestDataSetResultIDsForBindingRejectsMissingResult(t *testing.T) { - t.Parallel() - - binding := &model.StorageDataSet{ProviderID: onChainID(t, "101")} - if _, _, err := dataSetResultIDsForBinding(binding, nil); err == nil { - t.Fatal("dataSetResultIDsForBinding accepted a missing creation result") - } -} - -func TestUploadProgressEventPayloadUsesOverflowSafePercent(t *testing.T) { - updatedAt := time.Now() - const huge = int64(1 << 62) - payload := uploadProgressEventPayload(&model.StorageUpload{ - IngressStoreAttempt: 1, - IngressBytesTransferred: huge, - ContentSize: huge, - ProgressUpdatedAt: &updatedAt, - }, true) - - percent, ok := payload["percent"].(int) - if !ok { - t.Fatalf("percent missing or wrong type: %#v", payload["percent"]) - } - if percent != 100 { - t.Fatalf("percent = %d, want 100", percent) - } -} - -func TestUploadProgressEventPayloadMarksDoneWhenBytesReachTotal(t *testing.T) { - updatedAt := time.Now() - payload := uploadProgressEventPayload(&model.StorageUpload{ - IngressStoreAttempt: 1, - IngressBytesTransferred: 100, - ContentSize: 100, - ProgressUpdatedAt: &updatedAt, - }, false) - - done, ok := payload["done"].(bool) - if !ok { - t.Fatalf("done missing or wrong type: %#v", payload["done"]) - } - if !done { - t.Fatal("done = false, want true when uploaded bytes reach total bytes") - } -} - -func TestUploadProgressReporterRecordRespectsCanceledContext(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "progress-context") - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J0000000000000000CTXPRG", - ContentSize: 100, - Checksum: "sha256:progress-context", - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - upload, err = repos.Uploads.BeginIngressStoreProgress(ctx, upload.ID) - if err != nil { - t.Fatalf("BeginIngressStoreProgress: %v", err) - } - - canceledCtx, cancel := context.WithCancel(context.Background()) - cancel() - reporter := &uploadProgressReporter{ - ctx: canceledCtx, - repos: repos, - uploadID: upload.ID, - attempt: upload.IngressStoreAttempt, - } - reporter.record(50, false) - - got, err := repos.Uploads.GetByID(context.Background(), upload.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.IngressBytesTransferred != 0 { - t.Fatalf("ingress_bytes_transferred = %d, want 0 after canceled reporter context", got.IngressBytesTransferred) - } -} - -func TestUploadProgressReporterCoalescesThrottledProgress(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - - bucket := testutil.SeedBucket(t, db, "progress-coalesce") - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J0000000000000000COALES", - ContentSize: 100, - Checksum: "sha256:progress-coalesce", - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - upload, err = repos.Uploads.BeginIngressStoreProgress(ctx, upload.ID) - if err != nil { - t.Fatalf("BeginIngressStoreProgress: %v", err) - } - - reporter := &uploadProgressReporter{ - ctx: context.Background(), - repos: repos, - uploadID: upload.ID, - attempt: upload.IngressStoreAttempt, - flushInterval: 50 * time.Millisecond, - } - reporter.OnProgress(10) - reporter.OnProgress(40) - reporter.OnProgress(70) - - deadline := time.Now().Add(time.Second) - for time.Now().Before(deadline) { - got, err := repos.Uploads.GetByID(context.Background(), upload.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.IngressBytesTransferred == 70 { - return - } - time.Sleep(10 * time.Millisecond) - } - - got, err := repos.Uploads.GetByID(context.Background(), upload.ID) - if err != nil { - t.Fatalf("GetByID final: %v", err) - } - t.Fatalf("ingress_bytes_transferred = %d, want coalesced latest progress 70", got.IngressBytesTransferred) -} - -func TestCommitReleaseCauseKeepsDataSetEndedClassification(t *testing.T) { - // Both the uploader and the replacement worker decide between "draining" and - // "unavailable" from this error, so every cause a release can carry must still - // read as an ended data set. - for _, tc := range []struct { - name string - cause error - }{ - {name: "missing cause falls back to the sentinel"}, - {name: "service ended", cause: storage.ErrDataSetUnavailable}, - {name: "write blocked", cause: fmt.Errorf("submit commit: %w", &storage.DataSetPDPPaymentTerminatedError{ - DataSetID: sdktypes.NewBigInt(1001), PDPEndEpoch: sdktypes.Epoch(42), - })}, - } { - t.Run(tc.name, func(t *testing.T) { - cause := commitReleaseCause(storagecommit.AdvanceResult{ - State: storagecommit.AdvanceReleased, - ReleaseReason: storagecommit.ReleaseDataSetUnavailable, - Cause: tc.cause, - }) - if cause == nil { - t.Fatal("release cause is nil, want an error the failure handlers can classify") - } - if !dataSetFailureEnded(cause, nil) { - t.Fatalf("release cause %v is not an ended data set, so it would be marked unavailable", cause) - } - }) - } -} - -func TestCapacityWaitNamesAttentionHeldSlots(t *testing.T) { - logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - cases := []struct { - name string - attentionHeld int - wantMessage string - }{ - { - name: "queued behind work that still moves", - attentionHeld: 0, - wantMessage: "Waiting to submit stored content", - }, - { - // The unflagged attempts are still working through, so the queue moves. - name: "partial attention hold keeps the ordinary wording", - attentionHeld: storagecommit.MaxActiveAttemptsPerDataSet - 1, - wantMessage: "Waiting to submit stored content", - }, - { - name: "every slot flagged for attention", - attentionHeld: storagecommit.MaxActiveAttemptsPerDataSet, - wantMessage: "Waiting for storage confirmations that need attention", - }, - } - for _, testCase := range cases { - t.Run(testCase.name, func(t *testing.T) { - repos, _, _, _, task := seedCommitFailureFixture(t, 2) - handled := (&Uploader{repos: repos}).waitForCommitAdvance(t.Context(), task, logger, storagecommit.AdvanceResult{ - State: storagecommit.AdvanceWaitingCapacity, AttentionHeld: testCase.attentionHeld, - }) - if !handled { - t.Fatal("capacity wait was not handled") - } - gotTask, err := repos.Tasks.GetByID(t.Context(), task.ID) - if err != nil || gotTask == nil || gotTask.StatusMessage == nil { - t.Fatalf("task after capacity wait = %#v err=%v", gotTask, err) - } - if *gotTask.StatusMessage != testCase.wantMessage { - t.Fatalf("capacity wait message = %q, want %q", *gotTask.StatusMessage, testCase.wantMessage) - } - if gotTask.Status != model.TaskStatusWaiting || gotTask.RetryCount != 0 { - t.Fatalf("task after capacity wait = %#v, want waiting without retry", gotTask) - } - }) - } -} diff --git a/internal/worker/uploader_replacement.go b/internal/worker/uploader_replacement.go deleted file mode 100644 index 2bbc21b..0000000 --- a/internal/worker/uploader_replacement.go +++ /dev/null @@ -1,615 +0,0 @@ -package worker - -import ( - "context" - "errors" - "fmt" - "log/slog" - "time" - - "github.com/strahe/synaps3/internal/admin" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/objectlimits" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/strahe/synaps3/internal/synapse" - idtypes "github.com/strahe/synaps3/internal/types" - "github.com/strahe/synapse-go/storage" -) - -// One bounded window of upload history per seeding pass. Small enough that the -// write stays short on SQLite's single writer, large enough that a long history -// still drains in a reasonable number of passes. -const replacementSeedBatchSize = 200 - -func (u *Uploader) processReplacementTask(ctx context.Context, task *model.Task, logger *slog.Logger) { - if task == nil || task.ClaimedAt == nil { - return - } - payload, err := storagereplacement.ParseMigratePayload(task) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "parse replacement migration task", err) - return - } - replacement, err := u.repos.Replacements.GetByID(ctx, payload.ReplacementID) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "load provider replacement", err) - return - } - if replacement == nil || replacement.Status == storagereplacement.StatusCompleted || replacement.Status.Retryable() { - // A finished or operator-owned replacement has no work the coordinator - // may do on its own. - completeWorkerTask(ctx, u.repos, task, "uploader", logger) - return - } - if replacement.Status == storagereplacement.StatusSuperseded { - // The successor owns the slot. This coordinator's leftover is the unused - // target, which must still be ended even if confirmation already queued it. - if err := u.ensureAbandonedTargetTask(ctx, replacement.ID, replacement.BucketID, task.MaxRetries); err != nil { - u.handleReplacementTaskFailure(ctx, task, 0, logger, "queue abandoned replacement cleanup", err) - return - } - completeWorkerTask(ctx, u.repos, task, "uploader", logger) - return - } - target, err := u.repos.Uploads.GetDataSetBindingByID(ctx, replacement.TargetDataSetID) - if err != nil { - u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "load replacement target", err) - return - } - if target == nil { - u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "load replacement target", - fmt.Errorf("target data set %d: %w", replacement.TargetDataSetID, repository.ErrNotFound)) - return - } - - switch replacementPhase(replacement.Status, target) { - case storagereplacement.PhasePrepare: - u.prepareReplacementTarget(ctx, task, replacement, target, logger) - case storagereplacement.PhaseMigrate: - u.migrateReplacementItem(ctx, task, replacement, target, payload, logger) - case storagereplacement.PhaseRetire: - // Retirement belongs to the cleanup worker; the migration coordinator's - // job is finished once it has handed over. - if err := u.ensureReplacementRetirementTask(ctx, replacement.ID, replacement.BucketID, task.MaxRetries); err != nil { - u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "queue replacement retirement", err) - return - } - completeWorkerTask(ctx, u.repos, task, "uploader", logger) - default: - // An unrecognised phase is a bug, not an outage. Stop the task without - // touching the replacement so the record still describes reality. - u.failReplacementTaskWithoutMutation(ctx, task, logger, - fmt.Errorf("replacement %d has no coordinator work in status %s", replacement.ID, replacement.Status)) - } -} - -// A waiting replacement resumes at the phase it was waiting in, which is -// recovered from the data rather than remembered in the status. -func replacementPhase(status storagereplacement.Status, target *model.StorageDataSet) storagereplacement.Phase { - if phase := storagereplacement.PhaseFor(status); phase != storagereplacement.PhaseNone { - return phase - } - if status != storagereplacement.StatusWaiting { - return storagereplacement.PhaseNone - } - if target != nil && target.IsCurrent { - return storagereplacement.PhaseMigrate - } - return storagereplacement.PhasePrepare -} - -// prepareReplacementTarget creates the approved service and, once it is -// writable, switches the slot over. Until that switch every write still goes to -// the source, so a failure here costs nothing but time. -func (u *Uploader) prepareReplacementTarget( - ctx context.Context, - task *model.Task, - replacement *storagereplacement.Replacement, - target *model.StorageDataSet, - logger *slog.Logger, -) { - bucket, err := u.repos.Buckets.GetByID(ctx, replacement.BucketID) - if err != nil || bucket == nil { - if err == nil { - err = fmt.Errorf("bucket %d: %w", replacement.BucketID, repository.ErrNotFound) - } - u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "load replacement bucket", err) - return - } - if target.Status != model.StorageDataSetStatusReady { - if target.Status == model.StorageDataSetStatusPending || target.Status == model.StorageDataSetStatusFailed { - if !u.ensureReplacementFundingReady(ctx, task, replacement, target, bucket, logger) { - return - } - } - ready, err := u.createReplacementDataSet(ctx, replacement, target, bucket) - if err != nil { - if errors.Is(err, storagereplacement.ErrTargetInUse) { - // Retrying cannot change this: the operator has to pick a - // provider that is free. The source still owns the replica at - // this point, so a new confirmation is available to them. - u.failReplacementTarget(ctx, task, replacement.ID, logger, err.Error()) - return - } - u.handleReplacementProviderFailure(ctx, task, replacement, logger, "create replacement service", err) - return - } - if !ready { - u.waitForReplacementDependency(ctx, task, replacement, storagereplacement.WaitReasonTargetCreating, logger, - storagereplacement.WaitReasonTargetCreating.Message()) - return - } - } - if err := u.repos.Replacements.Activate(ctx, replacement.ID); err != nil { - if errors.Is(err, repository.ErrConflict) { - u.waitForReplacementDependency(ctx, task, replacement, storagereplacement.WaitReasonTargetWritable, logger, - storagereplacement.WaitReasonTargetWritable.Message()) - return - } - u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "activate replacement target", err) - return - } - logger.Info("provider replacement activated", - "replacementID", replacement.ID, "bucketID", replacement.BucketID, "copyIndex", replacement.CopyIndex) - u.continueReplacementTask(ctx, task, replacement.ID, "", logger) -} - -// createReplacementDataSet drives one data set creation step and reports -// whether the service is ready. It mirrors the ordinary upload path but does -// not touch any copy row, because the target holds no data yet. -func (u *Uploader) createReplacementDataSet( - ctx context.Context, - replacement *storagereplacement.Replacement, - target *model.StorageDataSet, - bucket *model.Bucket, -) (bool, error) { - storageCtx, err := u.contextForBindingProvider(ctx, target, bucket.Name) - if err != nil { - return false, err - } - switch target.Status { - case model.StorageDataSetStatusPending, model.StorageDataSetStatusFailed: - // Nothing has been submitted for this generation yet, so a context that - // already carries a data set can only be somebody else's: this provider - // still runs a live service for this bucket, a generation released - // locally without being terminated on chain. A replacement must open its - // own paid service -- attaching here would leave it paying for, and - // later retiring, a service it does not own. - // - // A creation this replacement did submit is resumed by the creating - // branch below, which resolves the recorded transaction instead. - matchingRef, err := u.storage.FindMatchingDataSet( - ctx, - target.ProviderID.SDK(), - map[string]string{"bucket": bucket.Name}, - storageCtx.CDNEnabled(), - ) - if err != nil { - return false, err - } - if matchingRef != nil { - return false, fmt.Errorf("provider %s already runs data set %s for this bucket: %w", - target.ProviderID.String(), idtypes.OnChainIDFromSDK(matchingRef.DataSetID()).String(), - storagereplacement.ErrTargetInUse) - } - var submitted storage.CreateDataSetSubmission - var submitErr error - result, err := storageCtx.CreateDataSet(ctx, &storage.CreateDataSetOptions{ - OnSubmitted: func(sub storage.CreateDataSetSubmission) { - submitted = sub - submitErr = u.repos.Uploads.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ - ID: target.ID, - TransactionID: sub.TransactionID, - StatusURL: sub.StatusURL, - ClientDataSetID: onChainIDPtrFromSDKPtr(sub.ClientDataSetID), - }) - }, - }) - if submitted.TransactionID != "" && submitErr != nil { - return false, fmt.Errorf("save replacement service submission: %w", submitErr) - } - if err != nil { - return false, err - } - dataSetID, clientDataSetID, err := dataSetResultIDsForBinding(target, result) - if err != nil { - return false, err - } - return true, u.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: target.ID, - DataSetID: dataSetID, - ClientDataSetID: &clientDataSetID, - }) - case model.StorageDataSetStatusCreating: - if target.CreateTransactionID == nil || target.CreateStatusURL == nil || target.ClientDataSetID == nil { - return false, errDataSetCreationIncomplete - } - result, err := storageCtx.WaitForDataSetCreated(ctx, storage.CreateDataSetSubmission{ - ProviderID: target.ProviderID.SDK(), - TransactionID: *target.CreateTransactionID, - StatusURL: *target.CreateStatusURL, - ClientDataSetID: sdkBigIntPtr(target.ClientDataSetID), - }) - if err != nil { - return false, err - } - dataSetID, clientDataSetID, err := dataSetResultIDsForBinding(target, result) - if err != nil { - return false, err - } - return true, u.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: target.ID, - DataSetID: dataSetID, - ClientDataSetID: &clientDataSetID, - }) - default: - return false, fmt.Errorf("replacement %d target status %s cannot be prepared", replacement.ID, target.Status) - } -} - -func (u *Uploader) migrateReplacementItem( - ctx context.Context, - task *model.Task, - replacement *storagereplacement.Replacement, - target *model.StorageDataSet, - _ storagereplacement.MigratePayload, - logger *slog.Logger, -) { - if replacement.Status == storagereplacement.StatusWaiting && - (replacement.WaitReason == nil || *replacement.WaitReason != storagereplacement.WaitReasonReadableSource) { - bucket, err := u.repos.Buckets.GetByID(ctx, replacement.BucketID) - if err != nil || bucket == nil { - if err == nil { - err = fmt.Errorf("bucket %d: %w", replacement.BucketID, repository.ErrNotFound) - } - u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "load replacement bucket", err) - return - } - if _, err := u.contextForReadyBinding(ctx, target); err != nil { - u.handleReplacementProviderFailure(ctx, task, replacement, logger, "check replacement target", err) - return - } - if err := u.repos.Replacements.MarkMigrating(ctx, replacement.ID); err != nil { - u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "resume replacement migration", err) - return - } - } - if !replacement.SeedingComplete { - if _, _, err := u.repos.Replacements.SeedMigrationBatchWithBudget( - ctx, replacement.ID, replacementSeedBatchSize, u.replacementMaxRetries, - ); err != nil { - u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "seed replacement migration", err) - return - } - } - execution, err := u.repos.Replacements.ReplacementExecution(ctx, replacement.ID) - if err != nil { - u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "inspect replacement migration", err) - return - } - if execution.SeedingComplete && !execution.HasPending && !execution.HasActive && - !execution.HasRetrying && !execution.HasWaitingSource { - if execution.HasFailed { - message := "Stored content needs attention" - u.failReplacementCoordinator(ctx, task, replacement.ID, nil, logger, message) - return - } - u.finishReplacementMigration(ctx, task, replacement, logger) - return - } - if execution.SeedingComplete && !execution.HasPending && !execution.HasActive && - !execution.HasRetrying && execution.HasWaitingSource { - u.waitForReplacementDependencyAfter(ctx, task, replacement, storagereplacement.WaitReasonReadableSource, - logger, storagereplacement.WaitReasonReadableSource.Message(), 5*time.Second) - return - } - u.waitForReplacementCoordinatorPoll(ctx, task, logger, execution) -} - -func (u *Uploader) waitForReplacementCoordinatorPoll( - ctx context.Context, - task *model.Task, - logger *slog.Logger, - execution storagereplacement.ExecutionSnapshot, -) { - message := "Discovering stored content to move" - if execution.SeedingComplete { - message = fmt.Sprintf("Moving stored content (%d of %d copied)", execution.ItemsCopied, execution.ItemsTotal) - } - if err := u.repos.Tasks.WaitRunning(ctx, task, model.TaskWaitReasonDependency, message, 5*time.Second); err != nil { - u.handleReplacementTaskFailure(ctx, task, execution.ReplacementID, logger, "wait for replacement items", err) - return - } - admin.WorkerTasksProcessed.WithLabelValues("uploader", "success").Inc() -} - -func (u *Uploader) finishReplacementMigration( - ctx context.Context, - task *model.Task, - replacement *storagereplacement.Replacement, - logger *slog.Logger, -) { - if err := u.repos.Replacements.BeginRetirement(ctx, replacement.ID); err != nil { - u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "begin replacement retirement", err) - return - } - if err := u.ensureReplacementRetirementTask(ctx, replacement.ID, replacement.BucketID, task.MaxRetries); err != nil { - u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "queue replacement retirement", err) - return - } - logger.Info("provider replacement migration complete", "replacementID", replacement.ID) - completeWorkerTask(ctx, u.repos, task, "uploader", logger) -} - -func (u *Uploader) ensureReplacementRetirementTask(ctx context.Context, replacementID, bucketID int64, maxRetries int) error { - _, err := u.repos.Tasks.EnsureRecurring(ctx, storagereplacement.NewRetireTask(replacementID, bucketID, maxRetries, time.Now())) - if err != nil { - return fmt.Errorf("ensure replacement retirement task for replacement %d: %w", replacementID, err) - } - return nil -} - -func (u *Uploader) ensureAbandonedTargetTask(ctx context.Context, replacementID, bucketID int64, maxRetries int) error { - _, err := u.repos.Tasks.EnsureRecurring(ctx, storagereplacement.NewAbandonedTargetTask(replacementID, bucketID, maxRetries, time.Now())) - if err != nil { - return fmt.Errorf("ensure abandoned replacement cleanup for replacement %d: %w", replacementID, err) - } - return nil -} - -// ensureReplacementFundingReady waits, without burning retries, until the wallet -// can pay for the approved service. Ordinary uploads already do this; creating -// a replacement service is the same kind of paid work. -func (u *Uploader) ensureReplacementFundingReady( - ctx context.Context, - task *model.Task, - replacement *storagereplacement.Replacement, - target *model.StorageDataSet, - bucket *model.Bucket, - logger *slog.Logger, -) bool { - storageCtx, err := u.contextForBindingProvider(ctx, target, bucket.Name) - if err != nil { - u.handleReplacementProviderFailure(ctx, task, replacement, logger, "open replacement funding context", err) - return false - } - costs, err := u.storage.PrepareUpload(ctx, uint64(objectlimits.MinFOCUploadSize), []synapse.StorageTarget{storageCtx}) - if err != nil { - u.handleReplacementProviderFailure(ctx, task, replacement, logger, "prepare replacement funding", err) - return false - } - if costs == nil { - u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "prepare replacement funding", - errors.New("missing storage cost estimate")) - return false - } - if costs.Ready { - return true - } - u.waitForReplacementDependency(ctx, task, replacement, storagereplacement.WaitReasonFunding, logger, - uploadFundingWaitMessage(costs)) - return false -} - -func (u *Uploader) continueReplacementTask( - ctx context.Context, - task *model.Task, - replacementID int64, - versionID string, - logger *slog.Logger, -) { - err := u.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { - if err := txRepos.Tasks.LockRunningClaim(ctx, task); err != nil { - return err - } - return txRepos.Tasks.ContinueRunning(ctx, task, versionID, storagereplacement.NewMigratePayload(replacementID)) - }) - if err != nil { - u.handleReplacementTaskFailure(ctx, task, replacementID, logger, "advance replacement task", err) - return - } - admin.WorkerTasksProcessed.WithLabelValues("uploader", "success").Inc() -} - -// waitForReplacementDependency records why progress paused without consuming -// retry budget. Waiting is never a failure. -func (u *Uploader) waitForReplacementDependency( - ctx context.Context, - task *model.Task, - replacement *storagereplacement.Replacement, - reason storagereplacement.WaitReason, - logger *slog.Logger, - message string, -) { - u.waitForReplacementDependencyAfter(ctx, task, replacement, reason, logger, message, uploadDependencyWaitDelay) -} - -func (u *Uploader) waitForReplacementDependencyAfter( - ctx context.Context, - task *model.Task, - replacement *storagereplacement.Replacement, - reason storagereplacement.WaitReason, - logger *slog.Logger, - message string, - delay time.Duration, -) { - err := u.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { - if err := txRepos.Replacements.MarkWaiting(ctx, replacement.ID, reason); err != nil { - return err - } - if err := context.Cause(ctx); err != nil { - return err - } - if err := txRepos.Tasks.WaitRunning(ctx, task, model.TaskWaitReasonDependency, message, delay); err != nil { - return err - } - return context.Cause(ctx) - }) - if err != nil { - logger.Error("failed to wait for replacement dependency", - "replacementID", replacement.ID, "taskID", task.ID, "reason", reason, "error", err) - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() - return - } - admin.WorkerTasksProcessed.WithLabelValues("uploader", "success").Inc() -} - -// handleReplacementProviderFailure keeps a recoverable provider problem in -// waiting and reserves retries for genuinely unknown failures. -func (u *Uploader) handleReplacementProviderFailure( - ctx context.Context, - task *model.Task, - replacement *storagereplacement.Replacement, - logger *slog.Logger, - stage string, - err error, -) { - switch { - case synapse.IsProviderUnavailable(err), synapse.IsNoProviderCandidates(err): - u.waitForReplacementDependency(ctx, task, replacement, storagereplacement.WaitReasonTarget, logger, - storagereplacement.WaitReasonTarget.Message()) - case dataSetWriteBlockedError(err), synapse.IsDataSetServiceEnded(err): - // The approved target itself ended. That needs a new confirmation, so it - // is operator work rather than another attempt. - u.failReplacementCoordinator(ctx, task, replacement.ID, nil, logger, fmt.Sprintf("%s: %v", stage, err)) - default: - u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, stage, err) - } -} - -// handleReplacementTaskFailure retries, and records the replacement as failed -// only once the task has genuinely run out of attempts. -func (u *Uploader) handleReplacementTaskFailure( - ctx context.Context, - task *model.Task, - replacementID int64, - logger *slog.Logger, - stage string, - err error, -) { - logger.Error(stage+" failed", "replacementID", replacementID, "error", err) - if replacementID <= 0 { - scheduleTaskRetry(ctx, u.repos, task, "uploader", logger, err) - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() - return - } - message := fmt.Sprintf("%s: %v", stage, err) - status, retryErr := u.repos.Replacements.ScheduleCoordinatorRetry(ctx, repository.ReplacementCoordinatorRetryInput{ - ReplacementID: replacementID, - Task: task, - LastError: message, - Backoff: retryDelay(task.RetryCount), - }) - if retryErr != nil { - logger.Error("failed to schedule provider replacement coordinator retry", - "replacementID", replacementID, "error", retryErr) - } else if status == model.TaskStatusExhausted { - admin.TasksExhaustedTotal.WithLabelValues("uploader", string(task.Type)).Inc() - } - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() -} - -func (u *Uploader) failReplacementCoordinator( - ctx context.Context, - task *model.Task, - replacementID int64, - reason *storagereplacement.FailureReason, - logger *slog.Logger, - message string, -) { - err := u.repos.Replacements.FailCoordinator(ctx, repository.ReplacementCoordinatorFailureInput{ - ReplacementID: replacementID, - Task: task, - FailureReason: reason, - LastError: message, - }) - if err != nil { - logger.Error("failed to stop provider replacement coordinator", - "replacementID", replacementID, "error", err) - } - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() -} - -// failReplacementTarget records an approved target that can never work and -// stops its coordinator in the same transaction, so the record never shows work -// in progress with nothing queued to do it. Retrying is pointless here, so the -// retry budget is not spent first. -func (u *Uploader) failReplacementTarget(ctx context.Context, task *model.Task, replacementID int64, logger *slog.Logger, message string) { - reason := storagereplacement.FailureReasonTargetInUse - err := u.repos.Replacements.FailCoordinator(ctx, repository.ReplacementCoordinatorFailureInput{ - ReplacementID: replacementID, - Task: task, - FailureReason: &reason, - LastError: message, - }) - if err != nil { - logger.Error("failed to record unusable replacement target", - "replacementID", replacementID, "error", err) - } else { - logger.Warn("provider replacement needs a different provider", - "replacementID", replacementID, "reason", message) - } - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() -} - -// failReplacementTaskWithoutMutation stops a task that must not retry while -// leaving the replacement record exactly as it is. -func (u *Uploader) failReplacementTaskWithoutMutation(ctx context.Context, task *model.Task, logger *slog.Logger, err error) { - logger.Error("replacement coordinator stopped", "taskID", task.ID, "error", err) - if failErr := u.repos.Tasks.FailRunning(ctx, task, err.Error()); failErr != nil { - logger.Error("failed to stop replacement coordinator", "taskID", task.ID, "error", failErr) - } - admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() -} - -// deferToReplacement yields an ordinary upload stage to the replacement -// coordinator when both target the same copy. It mirrors the recovered-replica -// gate so the two coordinators and normal uploads never write one row at once. -func (u *Uploader) deferToReplacement(ctx context.Context, task *model.Task, bucketID, uploadID int64, copyIndex int, logger *slog.Logger) bool { - copyRow, err := u.taskUploadCopy(ctx, task, uploadID, copyIndex) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "load upload copy for replacement coordination", err) - return true - } - // The write belongs to the generation its copy is bound to. Asking the slot - // instead would stop deferring the moment the replacement takes the slot, - // which is exactly when the two writers overlap. - binding, err := u.taskCopyDataSet(ctx, task, bucketID, uploadID, copyIndex) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "load upload data set", err) - return true - } - if binding == nil { - return false - } - replacement, err := u.repos.Replacements.GetActiveForDataSet(ctx, binding.ID) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "check provider replacement", err) - return true - } - if replacement == nil { - return false - } - if binding.ID != replacement.TargetDataSetID { - return false - } - if copyRow == nil { - u.handleTaskFailure(ctx, task, logger, "load upload copy for replacement coordination", - fmt.Errorf("upload copy %d not found", copyIndex)) - return true - } - // The item claim is visible before AttachTargetCopy and is compared by claim time. - itemClaim, err := u.repos.Replacements.RunningReplacementItemClaimForUpload(ctx, replacement.ID, uploadID) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "check replacement item in progress", err) - return true - } - // Ordinary uploads win an exact timestamp tie. The replacement side uses - // claimed_at <= its own claim, so both paths now apply the same total order. - if itemClaim == nil || task.ClaimedAt == nil || !itemClaim.ClaimedAt.Before(*task.ClaimedAt) { - return false - } - u.waitForStorageDependency(ctx, task, logger, "Waiting for the approved provider replacement") - return true -} diff --git a/internal/worker/uploader_replacement_internal_test.go b/internal/worker/uploader_replacement_internal_test.go deleted file mode 100644 index 359b3ad..0000000 --- a/internal/worker/uploader_replacement_internal_test.go +++ /dev/null @@ -1,450 +0,0 @@ -package worker - -import ( - "context" - "log/slog" - "testing" - "time" - - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/strahe/synaps3/internal/testutil" - "github.com/strahe/synaps3/internal/types" -) - -type replacementGateUploadRepo struct { - repository.StorageUploadRepository - copyRow *model.StorageUploadCopy - target *model.StorageDataSet -} - -func (r *replacementGateUploadRepo) GetUploadCopyByID(context.Context, int64) (*model.StorageUploadCopy, error) { - return r.copyRow, nil -} - -func (r *replacementGateUploadRepo) GetDataSetBindingByID(context.Context, int64) (*model.StorageDataSet, error) { - return r.target, nil -} - -type replacementGateReplacementRepo struct { - repository.StorageReplacementRepository - replacement *storagereplacement.Replacement - claim *storagereplacement.ClaimToken -} - -func (r *replacementGateReplacementRepo) GetActiveForDataSet(context.Context, int64) (*storagereplacement.Replacement, error) { - return r.replacement, nil -} - -func (r *replacementGateReplacementRepo) RunningReplacementItemClaimForUpload( - context.Context, - int64, - int64, -) (*storagereplacement.ClaimToken, error) { - return r.claim, nil -} - -type replacementGateTaskRepo struct { - repository.TaskRepository - waits int -} - -func (r *replacementGateTaskRepo) WaitRunning( - context.Context, - *model.Task, - model.TaskWaitReason, - string, - time.Duration, -) error { - r.waits++ - return nil -} - -func TestOrdinaryUploadDefersToReplacementClaimBeforeTargetCopyAttach(t *testing.T) { - claimedAt := time.Now() - taskClaimedAt := claimedAt.Add(time.Millisecond) - leaseUntil := taskClaimedAt.Add(time.Minute) - targetID := int64(42) - copyRow := &model.StorageUploadCopy{ - ID: 11, - UploadID: 22, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - StorageDataSetID: &targetID, - } - uploads := &replacementGateUploadRepo{ - copyRow: copyRow, - target: &model.StorageDataSet{ID: targetID}, - } - replacements := &replacementGateReplacementRepo{ - replacement: &storagereplacement.Replacement{ID: 33, TargetDataSetID: targetID}, - claim: &storagereplacement.ClaimToken{ItemID: 44, ClaimedAt: claimedAt}, - } - tasks := new(replacementGateTaskRepo) - u := &Uploader{repos: &repository.Repositories{ - Uploads: uploads, - Replacements: replacements, - Tasks: tasks, - }} - task := newUploadStageTask( - repository.ObjectVersionRef{ObjectID: 1, VersionID: "version"}, - 5, - uploadStageIngressStore, - copyRow.UploadID, - copyRow.CopyIndex, - copyRow.TransferMethod, - copyRow.ID, - ) - task.Status = model.TaskStatusRunning - task.ClaimedAt = &taskClaimedAt - task.LeaseUntil = &leaseUntil - - if !u.deferToReplacement(context.Background(), task, 1, copyRow.UploadID, copyRow.CopyIndex, slog.Default()) { - t.Fatal("ordinary upload did not defer to the earlier replacement claim") - } - if tasks.waits != 1 { - t.Fatalf("dependency waits = %d, want 1", tasks.waits) - } - - replacements.claim.ClaimedAt = taskClaimedAt - if u.deferToReplacement(context.Background(), task, 1, copyRow.UploadID, copyRow.CopyIndex, slog.Default()) { - t.Fatal("ordinary upload deferred on an exact claim timestamp tie") - } - if tasks.waits != 1 { - t.Fatalf("dependency waits after exact tie = %d, want 1", tasks.waits) - } - - uploads.target = &model.StorageDataSet{ID: targetID - 1} - if u.deferToReplacement(context.Background(), task, 1, copyRow.UploadID, copyRow.CopyIndex, slog.Default()) { - t.Fatal("ordinary upload deferred while writing the replacement source generation") - } - if tasks.waits != 1 { - t.Fatalf("dependency waits after source write = %d, want 1", tasks.waits) - } -} - -// In-place recovery and an approved replacement both want to finish the same -// generation's work. The replacement wins while it is running, otherwise the -// two would race each other over one copy row. -func TestEnsureReplicaRepairTaskStandsDownForApprovedReplacement(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := testutil.SeedBucket(t, db, "repair-vs-replacement") - - version := &model.ObjectVersion{ - VersionID: model.NewVersionID(), - BucketID: bucket.ID, - Key: "file.txt", - Size: 11, - ETag: "etag", - Checksum: "sum", - CacheKey: ".versions/repair-vs-replacement", - } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - providerID := mustOnChainID(t, "101") - binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: providerID, - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: upload.ID, - DataSetID: mustOnChainID(t, "1001"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: providerID, - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - established, err := repos.Uploads.GetDataSetBindingByID(ctx, binding.ID) - if err != nil || established == nil { - t.Fatalf("GetDataSetBindingByID: %#v err=%v", established, err) - } - - // Without a replacement the generation repairs itself as usual. - created, err := ensureReplicaRepairTask(ctx, repos, established, 5) - if err != nil { - t.Fatalf("ensureReplicaRepairTask: %v", err) - } - if !created { - t.Fatal("recovery did not queue repair work for an unfinished copy") - } - if _, err := db.NewDelete(). - Model((*model.Task)(nil)). - Where("idempotency_key = ?", replicaRepairTaskKey(established.ID)). - Exec(ctx); err != nil { - t.Fatalf("clear repair task: %v", err) - } - - replacement, _, err := repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ - BucketID: bucket.ID, - SourceDataSetID: established.ID, - SelectionMode: storagereplacement.SelectionModeManual, - TargetProviderID: mustOnChainID(t, "202"), - ClientRequestID: "internal-replacement-1", - MaxRetries: 5, - }) - if err != nil { - t.Fatalf("Authorize: %v", err) - } - created, err = ensureReplicaRepairTask(ctx, repos, established, 5) - if err != nil { - t.Fatalf("ensureReplicaRepairTask during replacement: %v", err) - } - if created { - t.Fatal("recovery queued repair work while an approved replacement owns the generation") - } - - // The generation being written *to* needs the same protection: repairing the - // target in place would race the coordinator over one copy row, and the two - // gates key off different task payloads so neither would see the other. - target, err := repos.Uploads.GetDataSetBindingByID(ctx, replacement.TargetDataSetID) - if err != nil || target == nil { - t.Fatalf("GetDataSetBindingByID target = %#v err=%v", target, err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: target.ID, DataSetID: mustOnChainID(t, "2002"), - }); err != nil { - t.Fatalf("MarkDataSetReady target: %v", err) - } - if err := repos.Uploads.MarkDataSetUnavailable(ctx, target.ID, "target provider unreachable"); err != nil { - t.Fatalf("MarkDataSetUnavailable target: %v", err) - } - unavailableTarget, err := repos.Uploads.GetDataSetBindingByID(ctx, target.ID) - if err != nil || unavailableTarget == nil { - t.Fatalf("reload target = %#v err=%v", unavailableTarget, err) - } - created, err = ensureReplicaRepairTask(ctx, repos, unavailableTarget, 5) - if err != nil { - t.Fatalf("ensureReplicaRepairTask for target: %v", err) - } - if created { - t.Fatal("recovery queued repair work on the generation the replacement is writing to") - } - - // A replacement that has given up must not hold the slot hostage. - if err := repos.Replacements.MarkFailed(ctx, replacement.ID, nil, "target creation exhausted"); err != nil { - t.Fatalf("MarkFailed: %v", err) - } - created, err = ensureReplicaRepairTask(ctx, repos, established, 5) - if err != nil { - t.Fatalf("ensureReplicaRepairTask after failure: %v", err) - } - if !created { - t.Fatal("recovery stayed blocked after the replacement terminally failed") - } -} - -func mustOnChainID(t *testing.T, value string) types.OnChainID { - t.Helper() - id, err := types.ParseOnChainID("test id", value) - if err != nil { - t.Fatalf("parse on-chain id %q: %v", value, err) - } - return id -} - -// An upload already in flight belongs to the generation its copy was bound to. -// Resolving the data set by replica slot instead would follow the slot to the -// replacement target the moment it activates, storing the piece on the new -// provider while the copy row being updated still points at the old one. -func TestTaskCopyDataSetFollowsTheCopyNotTheSlot(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - bucket := testutil.SeedBucket(t, db, "in-flight-across-activation") - - version := &model.ObjectVersion{ - VersionID: model.NewVersionID(), - BucketID: bucket.ID, - Key: "file.txt", - Size: 11, - ETag: "etag", - Checksum: "sum", - CacheKey: ".versions/in-flight-across-activation", - } - if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent: %v", err) - } - upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: version.VersionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - source, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: mustOnChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: source.ID, UploadID: upload.ID, DataSetID: mustOnChainID(t, "1001"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: source.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: mustOnChainID(t, "101"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - inFlight, err := repos.Uploads.GetUploadCopy(ctx, upload.ID, 0) - if err != nil || inFlight == nil { - t.Fatalf("GetUploadCopy = %#v err=%v", inFlight, err) - } - - // The task was queued while the source still owned the slot. - task := newUploadStageTask( - repository.ObjectVersionRef{ObjectID: version.ObjectID, VersionID: version.VersionID}, - 5, uploadStageIngressStore, upload.ID, 0, model.StorageCopyTransferMethodIngress, inFlight.ID) - - replacement, _, err := repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ - BucketID: bucket.ID, - SourceDataSetID: source.ID, - SelectionMode: storagereplacement.SelectionModeManual, - TargetProviderID: mustOnChainID(t, "202"), - ClientRequestID: "internal-replacement-2", - MaxRetries: 5, - }) - if err != nil { - t.Fatalf("Authorize: %v", err) - } - if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: replacement.TargetDataSetID, DataSetID: mustOnChainID(t, "2002"), - }); err != nil { - t.Fatalf("MarkDataSetReady target: %v", err) - } - if err := repos.Replacements.Activate(ctx, replacement.ID); err != nil { - t.Fatalf("Activate: %v", err) - } - - current, err := repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) - if err != nil || current == nil || current.ID != replacement.TargetDataSetID { - t.Fatalf("slot owner after activation = %#v err=%v, want the target", current, err) - } - - u := &Uploader{repos: repos} - // A retry after activation must keep the source binding whether it resumes - // immediately after copy creation, after storing the piece, or while commit - // confirmation is in progress. - for _, status := range []model.StorageUploadCopyStatus{ - model.StorageUploadCopyStatusPending, - model.StorageUploadCopyStatusPieceReady, - model.StorageUploadCopyStatusCommitting, - } { - if _, err := db.NewUpdate().Model((*model.StorageUploadCopy)(nil)). - Set("status = ?", status). - Where("id = ?", inFlight.ID). - Exec(ctx); err != nil { - t.Fatalf("set in-flight status %s: %v", status, err) - } - preserved, err := u.preserveInFlightUploadBindings( - ctx, - upload.ID, - newBucketBindingPlan([]model.StorageDataSet{*current}, upload.ID), - ) - if err != nil { - t.Fatalf("preserve binding at %s: %v", status, err) - } - if got := preserved.byCopyIndex[0]; got == nil || got.ID != source.ID { - t.Fatalf("binding at %s = %#v, want source generation %d", status, got, source.ID) - } - } - resolved, err := u.taskCopyDataSet(ctx, task, bucket.ID, upload.ID, 0) - if err != nil || resolved == nil { - t.Fatalf("taskCopyDataSet = %#v err=%v", resolved, err) - } - if resolved.ID != source.ID { - t.Fatalf("resolved data set %d, want the retiring generation %d the copy is bound to", resolved.ID, source.ID) - } - - // A task queued before copy ids existed has nothing to anchor to and stays - // resolvable through the slot, which is where it belongs. - legacy := newUploadStageTask( - repository.ObjectVersionRef{ObjectID: version.ObjectID, VersionID: version.VersionID}, - 5, uploadStageIngressStore, upload.ID, 0, model.StorageCopyTransferMethodIngress, 0) - viaSlot, err := u.taskCopyDataSet(ctx, legacy, bucket.ID, upload.ID, 0) - if err != nil || viaSlot == nil { - t.Fatalf("taskCopyDataSet legacy = %#v err=%v", viaSlot, err) - } - if viaSlot.ID != replacement.TargetDataSetID { - t.Fatalf("legacy task resolved to %d, want the current generation %d", viaSlot.ID, replacement.TargetDataSetID) - } -} - -// A replacement that has already asked the old service to end is past -// migration, even when a dependency wait has moved it out of the retiring -// status. Restarting it as a migration would re-run the copy pass and, worse, -// leave nothing driving the termination it already started. -func TestEnqueueReplacementCoordinatorResumesRetirementAfterTermination(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - m := &Manager{repos: repos, logger: slog.Default(), uploadMaxRetries: 5} - - epoch := int64(4200) - for _, tc := range []struct { - name string - row storagereplacement.Replacement - wantKey func(int64) string - }{ - {"migrating", storagereplacement.Replacement{ID: 1, BucketID: 9, Status: storagereplacement.StatusMigrating}, storagereplacement.MigrateTaskKey}, - {"waiting before termination", storagereplacement.Replacement{ID: 2, BucketID: 9, Status: storagereplacement.StatusWaiting}, storagereplacement.MigrateTaskKey}, - {"retiring", storagereplacement.Replacement{ID: 3, BucketID: 9, Status: storagereplacement.StatusRetiring}, storagereplacement.RetireTaskKey}, - {"waiting after termination", storagereplacement.Replacement{ID: 4, BucketID: 9, Status: storagereplacement.StatusWaiting, TerminationEpoch: &epoch}, storagereplacement.RetireTaskKey}, - } { - t.Run(tc.name, func(t *testing.T) { - row := tc.row - m.enqueueReplacementCoordinator(ctx, &row) - task, err := repos.Tasks.GetByIdempotencyKey(ctx, tc.wantKey(row.ID)) - if err != nil || task == nil { - t.Fatalf("coordinator for %s = %#v err=%v, want %s", tc.name, task, err, tc.wantKey(row.ID)) - } - }) - } - - // The retirement cases must not also have queued a migration pass. - for _, id := range []int64{3, 4} { - task, err := repos.Tasks.GetByIdempotencyKey(ctx, storagereplacement.MigrateTaskKey(id)) - if err != nil { - t.Fatalf("GetByIdempotencyKey: %v", err) - } - if task != nil { - t.Fatalf("replacement %d was restarted as a migration after its service was already ended", id) - } - } -} diff --git a/internal/worker/uploader_replacement_test.go b/internal/worker/uploader_replacement_test.go deleted file mode 100644 index a4d444e..0000000 --- a/internal/worker/uploader_replacement_test.go +++ /dev/null @@ -1,1115 +0,0 @@ -package worker_test - -import ( - "context" - "errors" - "fmt" - "log/slog" - "math/big" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagereplacement" - "github.com/strahe/synaps3/internal/synapse" - "github.com/strahe/synaps3/internal/testutil" - "github.com/strahe/synaps3/internal/worker" - "github.com/strahe/synapse-go/storage" - sdktypes "github.com/strahe/synapse-go/types" - "github.com/uptrace/bun" -) - -type replacementEnv struct { - env *testWorkerEnv - bucket *model.Bucket - upload *model.StorageUpload - versionID string - source *model.StorageDataSet - sourceCtx *fakeUploadContext - targetCtx *fakeUploadContext -} - -// seedReplacementEnv stores one object on a single replica slot and prepares -// fake contexts for both the retiring provider and its replacement. -func seedReplacementEnv(t *testing.T) *replacementEnv { - t.Helper() - env := newTestWorkerEnv(t) - ctx := context.Background() - bucket, _, versionID := seedCachedObject(t, env) - - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: %v", err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - source := seedReadyBinding(t, env, bucket.ID, upload.ID, 0, "101", "1001") - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: source.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: onChainID(t, "101"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: testCID(t).String(), - PieceID: onChainIDPtr(t, "2001"), - RetrievalURL: "https://source.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - if _, err := env.repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: version.Size, - Checksum: version.Checksum, - }); err != nil { - t.Fatalf("BindReadableUploadForContent: %v", err) - } - - sourceCtx := readyFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(2001), testCID(t)) - targetCtx := newFakeUploadContext(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(3002), testCID(t)) - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - switch { - case createContextDataSetIDEqual(opts, sdktypes.NewBigInt(1001)): - return sourceCtx, nil - case createContextDataSetIDEqual(opts, sdktypes.NewBigInt(2002)), createContextProviderIDEqual(opts, sdktypes.NewBigInt(202)): - return targetCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - return &replacementEnv{ - env: env, bucket: bucket, upload: upload, versionID: versionID, - source: source, sourceCtx: sourceCtx, targetCtx: targetCtx, - } -} - -func (r *replacementEnv) authorize(t *testing.T, provider string) *storagereplacement.Replacement { - t.Helper() - row, _, err := r.env.repos.Replacements.Authorize(context.Background(), repository.AuthorizeReplacementInput{ - BucketID: r.bucket.ID, - SourceDataSetID: r.source.ID, - SelectionMode: storagereplacement.SelectionModeManual, - TargetProviderID: onChainID(t, provider), - ClientRequestID: "worker-replacement-" + provider, - MaxRetries: 5, - }) - if err != nil { - t.Fatalf("Authorize: %v", err) - } - return row -} - -func (r *replacementEnv) coordinatorTask(t *testing.T, replacementID int64) *model.Task { - t.Helper() - task, err := r.env.repos.Tasks.GetByIdempotencyKey(context.Background(), storagereplacement.MigrateTaskKey(replacementID)) - if err != nil || task == nil { - t.Fatalf("migration coordinator = %#v err=%v", task, err) - } - return task -} - -func (r *replacementEnv) abandonedTargetTask(t *testing.T, replacementID int64) *model.Task { - t.Helper() - task, err := r.env.repos.Tasks.GetByIdempotencyKey(context.Background(), storagereplacement.AbandonedTargetTaskKey(replacementID)) - if err != nil || task == nil { - t.Fatalf("abandoned-target cleanup = %#v err=%v, want it queued with the later confirmation", task, err) - } - return task -} - -// runUploaderUntil drives the uploader until cond holds. Waiting and retrying -// both leave the task in an active status, which runWorkerUntilTask never -// returns on. -func (r *replacementEnv) runUploaderUntil(t *testing.T, cond func() bool, timeout time.Duration) { - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - uploader := r.newUploader() - replacementWorker := r.newProviderReplacementWorker(uploader) - done := make(chan struct{}, 2) - go func() { - _ = uploader.Run(ctx) - done <- struct{}{} - }() - go func() { - _ = replacementWorker.Run(ctx) - done <- struct{}{} - }() - deadline := time.Now().Add(timeout) - for time.Now().Before(deadline) { - if cond() { - cancel() - <-done - <-done - return - } - time.Sleep(20 * time.Millisecond) - } - cancel() - <-done - <-done - t.Fatal("uploader did not reach the expected state before the timeout") -} - -func (r *replacementEnv) newUploader() *worker.Uploader { - return worker.NewUploader(r.env.repos, r.env.cache, r.env.storage, nil, r.env.sm, - cache.EvictionPolicyAfterUpload, 1, 1, 10*time.Millisecond, slog.Default(), - worker.WithProviderReplacementMaxRetries(5)) -} - -func (r *replacementEnv) newProviderReplacementWorker(uploader *worker.Uploader) *worker.ProviderReplacementWorker { - return worker.NewProviderReplacementWorker(r.env.repos, uploader, 4, 10*time.Millisecond, slog.Default()) -} - -func (r *replacementEnv) runReplacementUntilTask(t *testing.T, taskID int64, timeout time.Duration) *model.Task { - t.Helper() - var final *model.Task - r.runUploaderUntil(t, func() bool { - task, err := r.env.repos.Tasks.GetByID(context.Background(), taskID) - if err != nil || task == nil { - return false - } - if task.Status == model.TaskStatusQueued || task.Status == model.TaskStatusScheduled || - task.Status == model.TaskStatusWaiting || task.Status == model.TaskStatusRunning { - return false - } - final = task - return true - }, timeout) - return final -} - -// The whole approved flow: prepare the new service, switch the slot, copy the -// stored content across, and hand over to retirement. -func TestUploader_ReplacementPreparesActivatesAndMigrates(t *testing.T) { - fixture := seedReplacementEnv(t) - ctx := context.Background() - row := fixture.authorize(t, "202") - task := fixture.coordinatorTask(t, row.ID) - - final := fixture.runReplacementUntilTask(t, task.ID, 20*time.Second) - if final == nil || final.Status != model.TaskStatusCompleted { - t.Fatalf("coordinator task = %#v, want completed", final) - } - - current, err := fixture.env.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, fixture.bucket.ID, 0) - if err != nil || current == nil || current.ID != row.TargetDataSetID { - t.Fatalf("current binding = %#v err=%v, want the replacement target", current, err) - } - source, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.source.ID) - if err != nil || source == nil || source.IsCurrent || source.Status != model.StorageDataSetStatusDraining { - t.Fatalf("source = %#v err=%v, want draining and not current", source, err) - } - - // The content must actually exist on the new provider, pulled rather than - // re-uploaded from this node. - targetCopy, err := fixture.env.repos.Uploads.GetUploadCopyForDataSet(ctx, fixture.upload.ID, row.TargetDataSetID) - if err != nil || targetCopy == nil || targetCopy.Status != model.StorageUploadCopyStatusCommitted { - t.Fatalf("target copy = %#v err=%v, want committed", targetCopy, err) - } - if fixture.targetCtx.pullCalls.Load() == 0 { - t.Fatal("migration did not pull from a remote replica") - } - if fixture.targetCtx.storeCalls.Load() != 0 { - t.Fatal("migration uploaded from local cache while a readable replica existed") - } - - got, err := fixture.env.repos.Replacements.GetByID(ctx, row.ID) - if err != nil || got == nil || got.Status != storagereplacement.StatusRetiring { - t.Fatalf("replacement = %#v err=%v, want retiring", got, err) - } - if got.ItemsCopied != got.ItemsTotal || got.ItemsTotal != 1 { - t.Fatalf("progress = %d/%d, want 1/1", got.ItemsCopied, got.ItemsTotal) - } - retire, err := fixture.env.repos.Tasks.GetByIdempotencyKey(ctx, storagereplacement.RetireTaskKey(row.ID)) - if err != nil || retire == nil { - t.Fatalf("retirement coordinator = %#v err=%v, want it queued", retire, err) - } -} - -// With no readable replica and no retained cache data the item is parked and -// the replacement waits, without burning retry budget. -func TestUploader_ReplacementWaitsWhenNoSourceIsReadable(t *testing.T) { - fixture := seedReplacementEnv(t) - ctx := context.Background() - row := fixture.authorize(t, "202") - task := fixture.coordinatorTask(t, row.ID) - - // The retiring provider can no longer serve reads and the cached copy is - // gone, so this content has nowhere to come from. - fixture.targetCtx.pullErr = &synapse.ProviderUnavailableError{Cause: errors.New("source provider unreachable")} - if err := fixture.env.cache.Delete(ctx, fixture.bucket.Name, ".versions/"+fixture.versionID); err != nil { - t.Fatalf("cache delete: %v", err) - } - - fixture.runUploaderUntil(t, func() bool { - got, err := fixture.env.repos.Replacements.GetByID(ctx, row.ID) - return err == nil && got != nil && got.Status == storagereplacement.StatusWaiting - }, 20*time.Second) - - final, err := fixture.env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || final == nil { - t.Fatalf("GetByID task: %#v err=%v", final, err) - } - if final.RetryCount != 0 { - t.Fatalf("retry count = %d, want waiting to leave the retry budget untouched", final.RetryCount) - } - got, err := fixture.env.repos.Replacements.GetByID(ctx, row.ID) - if err != nil || got == nil { - t.Fatalf("GetByID: %#v err=%v", got, err) - } - if got.WaitReason == nil || *got.WaitReason != storagereplacement.WaitReasonReadableSource { - t.Fatalf("wait reason = %v, want readable_source", got.WaitReason) - } - if got.LastError != nil { - t.Fatalf("last_error = %v, want waiting to record no failure", *got.LastError) - } - var item storagereplacement.Item - if err := fixture.env.db.NewSelect().Model(&item). - Where("replacement_id = ?", row.ID). - Scan(ctx); err != nil || item.Status != storagereplacement.ItemStatusWaitingSource { - t.Fatalf("parked item = %#v err=%v, want waiting_source", item, err) - } -} - -func TestUploader_ReplacementWaitsForFundingBeforeCreatingTheService(t *testing.T) { - fixture := seedReplacementEnv(t) - ctx := context.Background() - row := fixture.authorize(t, "202") - task := fixture.coordinatorTask(t, row.ID) - var createCalls atomic.Int32 - fixture.targetCtx.createCalls = &createCalls - fixture.env.storage.PrepareUploadFunc = func(context.Context, uint64, []synapse.StorageTarget) (*storage.MultiContextCosts, error) { - return &storage.MultiContextCosts{Ready: false}, nil - } - - fixture.runUploaderUntil(t, func() bool { - got, err := fixture.env.repos.Replacements.GetByID(ctx, row.ID) - return err == nil && got != nil && got.Status == storagereplacement.StatusWaiting - }, 20*time.Second) - - final, err := fixture.env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || final == nil { - t.Fatalf("GetByID task: %#v err=%v", final, err) - } - if final.RetryCount != 0 { - t.Fatalf("retry count = %d, want funding wait to leave the retry budget untouched", final.RetryCount) - } - got, err := fixture.env.repos.Replacements.GetByID(ctx, row.ID) - if err != nil || got == nil { - t.Fatalf("GetByID: %#v err=%v", got, err) - } - if got.WaitReason == nil || *got.WaitReason != storagereplacement.WaitReasonFunding { - t.Fatalf("wait reason = %v, want funding", got.WaitReason) - } - if createCalls.Load() != 0 { - t.Fatalf("CreateDataSet calls = %d, want none while funding is not ready", createCalls.Load()) - } -} - -// A coordinator that cannot make progress retries the task, but must leave the -// replacement record describing reality rather than inventing a state. -func TestUploader_ReplacementRetriesWithoutRewritingTheRecord(t *testing.T) { - fixture := seedReplacementEnv(t) - ctx := context.Background() - row := fixture.authorize(t, "202") - task := fixture.coordinatorTask(t, row.ID) - - // Put the approved target into a state no preparation step can advance. - if _, err := fixture.env.db.NewUpdate(). - Model((*model.StorageDataSet)(nil)). - Set("status = ?", model.StorageDataSetStatusRetired). - Where("id = ?", row.TargetDataSetID). - Exec(ctx); err != nil { - t.Fatalf("retire target: %v", err) - } - - fixture.runUploaderUntil(t, func() bool { - got, err := fixture.env.repos.Tasks.GetByID(ctx, task.ID) - return err == nil && got != nil && (got.RetryCount > 0 || got.Status == model.TaskStatusFailed) - }, 20*time.Second) - - got, err := fixture.env.repos.Replacements.GetByID(ctx, row.ID) - if err != nil || got == nil { - t.Fatalf("GetByID: %#v err=%v", got, err) - } - if got.Status != storagereplacement.StatusPreparingTarget { - t.Fatalf("replacement status = %s, want it untouched at preparing_target", got.Status) - } -} - -// retirementFixture runs migration to completion so retirement has a realistic -// starting point: a drained source and a target that already holds the data. -type retirementFixture struct { - *replacementEnv - replacement *storagereplacement.Replacement - terminator *testutil.MockServiceTerminator - epochs *testutil.MockChainEpochReader - epoch atomic.Int64 - terminated atomic.Int32 -} - -func seedRetirementFixture(t *testing.T) *retirementFixture { - t.Helper() - base := seedReplacementEnv(t) - row := base.authorize(t, "202") - migrate := base.coordinatorTask(t, row.ID) - if final := base.runReplacementUntilTask(t, migrate.ID, 20*time.Second); final == nil || - final.Status != model.TaskStatusCompleted { - t.Fatalf("migration coordinator = %#v, want completed", final) - } - - fixture := &retirementFixture{replacementEnv: base, replacement: row} - fixture.epoch.Store(1000) - fixture.terminator = &testutil.MockServiceTerminator{ - TerminateServiceFunc: func(context.Context, sdktypes.BigInt) (*synapse.TerminationResult, error) { - fixture.terminated.Add(1) - return &synapse.TerminationResult{TxHash: "0xterminate", EndEpoch: fixture.epoch.Load() + 5}, nil - }, - } - fixture.epochs = &testutil.MockChainEpochReader{ - CurrentEpochFunc: func(context.Context) (int64, error) { return fixture.epoch.Load(), nil }, - } - return fixture -} - -func (f *retirementFixture) newCleanupWorker() *worker.StorageCleanupWorker { - return worker.NewStorageCleanupWorker(f.env.repos, f.env.storage, 1, 10*time.Millisecond, slog.Default(), - worker.WithServiceTermination(f.terminator, f.epochs)) -} - -func (f *retirementFixture) runCleanupUntil(t *testing.T, cond func() bool, timeout time.Duration) { - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - done := make(chan struct{}) - w := f.newCleanupWorker() - go func() { - defer close(done) - _ = w.Run(ctx) - }() - deadline := time.Now().Add(timeout) - for time.Now().Before(deadline) { - if cond() { - cancel() - <-done - return - } - time.Sleep(20 * time.Millisecond) - } - cancel() - <-done - t.Fatal("storage cleanup did not reach the expected state before the timeout") -} - -// releaseWaitingTasks brings a parked task's schedule forward so a test does not -// have to sleep through the real wait interval. -func (f *retirementFixture) releaseWaitingTasks(t *testing.T) { - t.Helper() - if _, err := f.env.db.NewUpdate(). - Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("status = ?", model.TaskStatusWaiting). - Exec(context.Background()); err != nil { - t.Fatalf("release waiting tasks: %v", err) - } -} - -func (f *retirementFixture) reload(t *testing.T) *storagereplacement.Replacement { - t.Helper() - got, err := f.env.repos.Replacements.GetByID(context.Background(), f.replacement.ID) - if err != nil || got == nil { - t.Fatalf("GetByID: %#v err=%v", got, err) - } - return got -} - -// The old service is only treated as gone once the chain has actually reached -// the epoch it ends at. -func TestStorageCleanup_RetirementWaitsForTheTerminationEpoch(t *testing.T) { - fixture := seedRetirementFixture(t) - ctx := context.Background() - - fixture.runCleanupUntil(t, func() bool { - got, err := fixture.env.repos.Replacements.GetByID(ctx, fixture.replacement.ID) - return err == nil && got != nil && got.WaitReason != nil && - *got.WaitReason == storagereplacement.WaitReasonTerminationEpoch - }, 20*time.Second) - - waiting := fixture.reload(t) - if waiting.Status != storagereplacement.StatusWaiting { - t.Fatalf("status = %s, want waiting for the end of term", waiting.Status) - } - if waiting.TerminationEpoch == nil { - t.Fatal("termination epoch was not recorded before waiting for it") - } - source, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.source.ID) - if err != nil || source == nil || source.Status == model.StorageDataSetStatusRetired { - t.Fatalf("source = %#v err=%v, want it not retired before the epoch is reached", source, err) - } - - // Let the chain catch up. The task is parked for the epoch re-check delay, - // so bring it forward rather than sleeping through it. - fixture.epoch.Store(*waiting.TerminationEpoch) - fixture.releaseWaitingTasks(t) - fixture.runCleanupUntil(t, func() bool { - got, err := fixture.env.repos.Replacements.GetByID(ctx, fixture.replacement.ID) - return err == nil && got != nil && got.Status == storagereplacement.StatusCompleted - }, 20*time.Second) - - done := fixture.reload(t) - if done.TerminationObservedAt == nil { - t.Fatal("completed replacement recorded no observation time") - } - source, err = fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.source.ID) - if err != nil || source == nil || source.Status != model.StorageDataSetStatusRetired || source.IsCurrent { - t.Fatalf("source = %#v err=%v, want retired", source, err) - } - if got := fixture.terminated.Load(); got != 1 { - t.Fatalf("termination calls = %d, want the service terminated exactly once", got) - } -} - -// A version the new provider cannot serve must keep the old one alive. -func TestStorageCleanup_RetirementRefusesWhileCoverageIsIncomplete(t *testing.T) { - fixture := seedRetirementFixture(t) - ctx := context.Background() - - // Take the migrated copy away, leaving the retained version with no home on - // the new provider. - if _, err := fixture.env.db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("status = ?", model.StorageUploadCopyStatusFailed). - Where("storage_data_set_id = ?", fixture.replacement.TargetDataSetID). - Exec(ctx); err != nil { - t.Fatalf("break target coverage: %v", err) - } - - fixture.runCleanupUntil(t, func() bool { - got, err := fixture.env.repos.Replacements.GetByID(ctx, fixture.replacement.ID) - return err == nil && got != nil && got.WaitReason != nil - }, 20*time.Second) - - got := fixture.reload(t) - if got.Status != storagereplacement.StatusWaiting { - t.Fatalf("status = %s, want waiting", got.Status) - } - if *got.WaitReason != storagereplacement.WaitReasonCoverage { - t.Fatalf("wait reason = %s, want coverage", *got.WaitReason) - } - if fixture.terminated.Load() != 0 { - t.Fatal("terminated the old service while a version was still uncovered") - } - source, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.source.ID) - if err != nil || source == nil || source.Status == model.StorageDataSetStatusRetired { - t.Fatalf("source = %#v err=%v, want it kept alive", source, err) - } -} - -// Settling payment debt is the operator's call. The replacement must stop and -// say so instead of retrying, and the two facts must be recorded together. -func TestStorageCleanup_RetirementRaisesAttentionOnPaymentDebt(t *testing.T) { - fixture := seedRetirementFixture(t) - ctx := context.Background() - fixture.terminator.TerminateServiceFunc = func(context.Context, sdktypes.BigInt) (*synapse.TerminationResult, error) { - fixture.terminated.Add(1) - return nil, &synapse.TerminationBlockedError{Reason: "payment_debt", Shortfall: big.NewInt(500)} - } - - fixture.runCleanupUntil(t, func() bool { - got, err := fixture.env.repos.Replacements.GetByID(ctx, fixture.replacement.ID) - return err == nil && got != nil && got.Status == storagereplacement.StatusCleanupAttention - }, 20*time.Second) - - got := fixture.reload(t) - if got.LastError == nil || !strings.Contains(*got.LastError, "payment_debt") { - t.Fatalf("last_error = %v, want it to name the payment debt", got.LastError) - } - retire, err := fixture.env.repos.Tasks.GetByIdempotencyKey(ctx, storagereplacement.RetireTaskKey(fixture.replacement.ID)) - if err != nil || retire == nil { - t.Fatalf("retirement task = %#v err=%v", retire, err) - } - if retire.Status != model.TaskStatusFailed { - t.Fatalf("retirement task status = %s, want it stopped rather than retrying", retire.Status) - } - source, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.source.ID) - if err != nil || source == nil || source.Status == model.StorageDataSetStatusRetired { - t.Fatalf("source = %#v err=%v, want it kept alive", source, err) - } -} - -// The abandoned target coordinator existed but was never dispatched: the -// cleanup worker only routed the source retirement stage, so the task fell into -// ordinary replica cleanup, found nothing, and completed without ending the -// paid service it was created to end. -func TestStorageCleanup_AbandonedTargetIsDispatchedAndTerminated(t *testing.T) { - fixture := seedReplacementEnv(t) - ctx := context.Background() - first := fixture.authorize(t, "202") - - // Bring the first target's service into existence, then supersede it. - if err := fixture.env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: first.TargetDataSetID, - DataSetID: onChainID(t, "2002"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - second := fixture.authorize(t, "303") - if second.ID == first.ID { - t.Fatal("the later confirmation reused the record") - } - - var terminated atomic.Int64 - terminator := &testutil.MockServiceTerminator{ - TerminateServiceFunc: func(_ context.Context, dataSetID sdktypes.BigInt) (*synapse.TerminationResult, error) { - if dataSetID.String() != "2002" { - t.Errorf("terminated data set %s, want the abandoned target 2002", dataSetID.String()) - } - terminated.Add(1) - return &synapse.TerminationResult{TxHash: "0xabandon", EndEpoch: 1000}, nil - }, - } - epochs := &testutil.MockChainEpochReader{ - CurrentEpochFunc: func(context.Context) (int64, error) { return 5000, nil }, - } - task := fixture.abandonedTargetTask(t, first.ID) - - worker := worker.NewStorageCleanupWorker(fixture.env.repos, fixture.env.storage, 1, 10*time.Millisecond, - slog.Default(), worker.WithServiceTermination(terminator, epochs)) - runWorkerUntilTask(t, fixture.env, worker, task.ID, 20*time.Second) - - if terminated.Load() != 1 { - t.Fatalf("termination calls = %d, want the abandoned service ended exactly once", terminated.Load()) - } - abandoned, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, first.TargetDataSetID) - if err != nil || abandoned == nil || abandoned.Status != model.StorageDataSetStatusRetired { - t.Fatalf("abandoned target = %#v err=%v, want retired", abandoned, err) - } - source, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.source.ID) - if err != nil || source == nil || !source.IsCurrent || source.Status == model.StorageDataSetStatusRetired { - t.Fatalf("source = %#v err=%v, want it untouched and still current", source, err) - } -} - -func TestStorageCleanup_AbandonedTargetRestartObservesStoredTermination(t *testing.T) { - fixture := seedReplacementEnv(t) - ctx := context.Background() - first := fixture.authorize(t, "202") - if err := fixture.env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: first.TargetDataSetID, DataSetID: onChainID(t, "2002"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - fixture.authorize(t, "303") - - var terminationCalls atomic.Int64 - terminator := &testutil.MockServiceTerminator{ - TerminateServiceFunc: func(context.Context, sdktypes.BigInt) (*synapse.TerminationResult, error) { - terminationCalls.Add(1) - return &synapse.TerminationResult{TxHash: "0xpersisted", EndEpoch: 5000}, nil - }, - } - var observedEpoch atomic.Int64 - observedEpoch.Store(1000) - epochs := &testutil.MockChainEpochReader{ - CurrentEpochFunc: func(context.Context) (int64, error) { return observedEpoch.Load(), nil }, - } - task := fixture.abandonedTargetTask(t, first.ID) - cleanup := worker.NewStorageCleanupWorker(fixture.env.repos, fixture.env.storage, 1, 10*time.Millisecond, - slog.Default(), worker.WithServiceTermination(terminator, epochs)) - runWorkerUntilTaskStatus(t, fixture.env, cleanup, task.ID, model.TaskStatusWaiting, 20*time.Second) - if terminationCalls.Load() != 1 { - t.Fatalf("termination calls before restart = %d, want 1", terminationCalls.Load()) - } - replacement, err := fixture.env.repos.Replacements.GetByID(ctx, first.ID) - if err != nil || replacement == nil || replacement.AbandonedTerminationEpoch == nil || - *replacement.AbandonedTerminationEpoch != 5000 { - t.Fatalf("stored abandoned termination = %#v err=%v, want epoch 5000", replacement, err) - } - - observedEpoch.Store(6000) - if _, err := fixture.env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now()).Where("id = ?", task.ID).Exec(ctx); err != nil { - t.Fatalf("release epoch wait: %v", err) - } - cleanup = worker.NewStorageCleanupWorker(fixture.env.repos, fixture.env.storage, 1, 10*time.Millisecond, - slog.Default(), worker.WithServiceTermination(terminator, epochs)) - runWorkerUntilTaskStatus(t, fixture.env, cleanup, task.ID, model.TaskStatusCompleted, 20*time.Second) - if terminationCalls.Load() != 1 { - t.Fatalf("termination calls after restart = %d, want the stored epoch to prevent a second call", terminationCalls.Load()) - } - replacement, err = fixture.env.repos.Replacements.GetByID(ctx, first.ID) - if err != nil || replacement == nil || replacement.AbandonedTerminationObservedAt == nil { - t.Fatalf("observed abandoned termination = %#v err=%v", replacement, err) - } -} - -func TestStorageCleanup_AbandonedTargetWaitsOnPaymentDebtThenRetires(t *testing.T) { - fixture := seedReplacementEnv(t) - ctx := context.Background() - first := fixture.authorize(t, "202") - if err := fixture.env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: first.TargetDataSetID, - DataSetID: onChainID(t, "2002"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if second := fixture.authorize(t, "303"); second.ID == first.ID { - t.Fatal("the later confirmation reused the record") - } - - var blocked atomic.Bool - blocked.Store(true) - var terminated atomic.Int64 - terminator := &testutil.MockServiceTerminator{ - TerminateServiceFunc: func(context.Context, sdktypes.BigInt) (*synapse.TerminationResult, error) { - terminated.Add(1) - if blocked.Load() { - return nil, &synapse.TerminationBlockedError{Reason: "payment_debt", Shortfall: big.NewInt(500)} - } - return &synapse.TerminationResult{TxHash: "0xabandon", EndEpoch: 1000}, nil - }, - } - epochs := &testutil.MockChainEpochReader{ - CurrentEpochFunc: func(context.Context) (int64, error) { return 5000, nil }, - } - task := fixture.abandonedTargetTask(t, first.ID) - - cleanup := worker.NewStorageCleanupWorker(fixture.env.repos, fixture.env.storage, 1, 10*time.Millisecond, - slog.Default(), worker.WithServiceTermination(terminator, epochs)) - runWorkerUntilTaskStatus(t, fixture.env, cleanup, task.ID, model.TaskStatusWaiting, 20*time.Second) - - got, err := fixture.env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || got == nil || got.Status != model.TaskStatusWaiting { - t.Fatalf("task = %#v err=%v, want waiting so cleanup can resume after the debt is settled", got, err) - } - abandoned, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, first.TargetDataSetID) - if err != nil || abandoned == nil || abandoned.Status == model.StorageDataSetStatusRetired { - t.Fatalf("abandoned target = %#v err=%v, want it kept until termination succeeds", abandoned, err) - } - - blocked.Store(false) - if _, err := fixture.env.db.NewUpdate(). - Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now()). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - t.Fatalf("release wait schedule: %v", err) - } - cleanup = worker.NewStorageCleanupWorker(fixture.env.repos, fixture.env.storage, 1, 10*time.Millisecond, - slog.Default(), worker.WithServiceTermination(terminator, epochs)) - runWorkerUntilTaskStatus(t, fixture.env, cleanup, task.ID, model.TaskStatusCompleted, 20*time.Second) - - abandoned, err = fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, first.TargetDataSetID) - if err != nil || abandoned == nil || abandoned.Status != model.StorageDataSetStatusRetired { - t.Fatalf("abandoned target = %#v err=%v, want retired after the debt cleared", abandoned, err) - } - if terminated.Load() < 2 { - t.Fatalf("termination calls = %d, want a blocked attempt and a successful one", terminated.Load()) - } -} - -func TestStorageCleanup_AbandonedTargetWithoutOnChainServiceReleasesTheProvider(t *testing.T) { - for _, tc := range []struct { - name string - maxRetries int - prepare func(*testing.T, *replacementEnv, *storagereplacement.Replacement) - }{ - { - name: "service was never created", - maxRetries: 5, - }, - { - name: "creation observation exhausts", - maxRetries: 1, - prepare: func(t *testing.T, fixture *replacementEnv, first *storagereplacement.Replacement) { - t.Helper() - clientDataSetID := onChainIDPtr(t, "9202") - if err := fixture.env.repos.Uploads.MarkDataSetCreating(context.Background(), repository.MarkDataSetCreatingInput{ - ID: first.TargetDataSetID, - TransactionID: "0xabandoned-create", - StatusURL: "https://provider-202.example/status/create", - ClientDataSetID: clientDataSetID, - }); err != nil { - t.Fatalf("MarkDataSetCreating: %v", err) - } - fixture.targetCtx.waitErr = errors.New("rpc timeout") - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - fixture := seedReplacementEnv(t) - ctx := context.Background() - first := fixture.authorize(t, "202") - if tc.prepare != nil { - tc.prepare(t, fixture, first) - } - second := fixture.authorize(t, "303") - if second.ID == first.ID { - t.Fatal("the later confirmation reused the record") - } - - var terminated atomic.Int64 - terminator := &testutil.MockServiceTerminator{ - TerminateServiceFunc: func(context.Context, sdktypes.BigInt) (*synapse.TerminationResult, error) { - terminated.Add(1) - return &synapse.TerminationResult{TxHash: "0xabandon", EndEpoch: 1000}, nil - }, - } - epochs := &testutil.MockChainEpochReader{ - CurrentEpochFunc: func(context.Context) (int64, error) { return 5000, nil }, - } - task := fixture.abandonedTargetTask(t, first.ID) - if tc.maxRetries != task.MaxRetries { - if _, err := fixture.env.db.NewUpdate(). - Model((*model.Task)(nil)). - Set("max_retries = ?", tc.maxRetries). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - t.Fatalf("set abandoned-target max retries: %v", err) - } - task.MaxRetries = tc.maxRetries - } - - cleanup := worker.NewStorageCleanupWorker(fixture.env.repos, fixture.env.storage, 1, 10*time.Millisecond, - slog.Default(), worker.WithServiceTermination(terminator, epochs)) - final := runWorkerUntilTask(t, fixture.env, cleanup, task.ID, 20*time.Second) - if final == nil { - t.Fatal("abandoned target task was not processed") - } - - if terminated.Load() != 0 { - t.Fatalf("termination calls = %d, want none when no on-chain service exists", terminated.Load()) - } - abandoned, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, first.TargetDataSetID) - if err != nil || abandoned == nil || abandoned.Status != model.StorageDataSetStatusRetired { - t.Fatalf("abandoned target = %#v err=%v, want retired", abandoned, err) - } - if _, _, err := fixture.env.repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ - BucketID: fixture.bucket.ID, - SourceDataSetID: fixture.source.ID, - SelectionMode: storagereplacement.SelectionModeManual, - TargetProviderID: onChainID(t, "202"), - ClientRequestID: "worker-reuse-202", - MaxRetries: 5, - }); err != nil { - t.Fatalf("reusing the released provider: %v", err) - } - }) - } -} - -// Parked work is only reached once nothing executable is left. Re-queueing the -// coordinator there spins it against the same item at queue-tail rate and -// starves ordinary uploads, so it has to wait for the source instead. -func TestUploader_OnlyParkedItemsPutTheCoordinatorIntoAWait(t *testing.T) { - fixture := seedReplacementEnv(t) - ctx := context.Background() - replacement := fixture.authorize(t, "202") - if err := fixture.env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: replacement.TargetDataSetID, - DataSetID: onChainID(t, "2002"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - - // The retiring generation is still writing this content, so there is - // nothing to copy from yet and nothing else to do. - if _, err := fixture.env.db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("status = ?", model.StorageUploadCopyStatusPending). - Where("upload_id = ? AND storage_data_set_id = ?", fixture.upload.ID, fixture.source.ID). - Exec(ctx); err != nil { - t.Fatalf("put the source copy back in flight: %v", err) - } - - task := fixture.coordinatorTask(t, replacement.ID) - fixture.runUploaderUntil(t, func() bool { - row, err := fixture.env.repos.Replacements.GetByID(ctx, replacement.ID) - return err == nil && row != nil && row.Status == storagereplacement.StatusWaiting - }, 20*time.Second) - - row, err := fixture.env.repos.Replacements.GetByID(ctx, replacement.ID) - if err != nil || row == nil { - t.Fatalf("GetByID = %#v err=%v", row, err) - } - if row.WaitReason == nil || *row.WaitReason != storagereplacement.WaitReasonReadableSource { - t.Fatalf("wait reason = %v, want readable_source", row.WaitReason) - } - settled, err := fixture.env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || settled == nil { - t.Fatalf("GetByID task = %#v err=%v", settled, err) - } - // Waiting must not burn the retry budget; the item is rechecked next tick. - if settled.RetryCount != 0 { - t.Fatalf("retry count = %d, want waiting to leave the budget untouched", settled.RetryCount) - } - if settled.Status != model.TaskStatusWaiting { - t.Fatalf("task status = %s, want waiting", settled.Status) - } - if settled.ScheduledAt.Before(time.Now()) { - t.Fatal("the coordinator was re-queued immediately instead of waiting for the source") - } -} - -type cancelReplacementWaitHook struct { - cancel context.CancelFunc - fired atomic.Bool -} - -func (h *cancelReplacementWaitHook) BeforeQuery(ctx context.Context, _ *bun.QueryEvent) context.Context { - return ctx -} - -func (h *cancelReplacementWaitHook) AfterQuery(_ context.Context, event *bun.QueryEvent) { - query := strings.ToLower(event.Query) - if event.Err != nil || event.Operation() != "UPDATE" || - !strings.Contains(query, "storage_replacements") || - !strings.Contains(query, "wait_reason") || !strings.Contains(query, "waiting") { - return - } - if h.fired.CompareAndSwap(false, true) { - h.cancel() - } -} - -func TestUploader_ReplacementWaitCancellationDoesNotPartiallyPauseLifecycle(t *testing.T) { - fixture := seedReplacementEnv(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - replacement := fixture.authorize(t, "202") - if err := fixture.env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: replacement.TargetDataSetID, - DataSetID: onChainID(t, "2002"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - - if _, err := fixture.env.db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("status = ?", model.StorageUploadCopyStatusPending). - Where("upload_id = ? AND storage_data_set_id = ?", fixture.upload.ID, fixture.source.ID). - Exec(ctx); err != nil { - t.Fatalf("put the source copy back in flight: %v", err) - } - - task := fixture.coordinatorTask(t, replacement.ID) - hook := &cancelReplacementWaitHook{cancel: cancel} - fixture.env.db.AddQueryHook(hook) - uploader := fixture.newUploader() - replacementWorker := fixture.newProviderReplacementWorker(uploader) - done := make(chan struct{}, 2) - go func() { - _ = uploader.Run(ctx) - done <- struct{}{} - }() - go func() { - _ = replacementWorker.Run(ctx) - done <- struct{}{} - }() - - select { - case <-done: - <-done - case <-time.After(20 * time.Second): - cancel() - <-done - <-done - t.Fatal("uploader did not reach the replacement wait before the timeout") - } - if !hook.fired.Load() { - t.Fatal("replacement wait cancellation hook did not run") - } - - row, err := fixture.env.repos.Replacements.GetByID(context.Background(), replacement.ID) - if err != nil || row == nil { - t.Fatalf("GetByID replacement = %#v err=%v", row, err) - } - if row.Status == storagereplacement.StatusWaiting || row.WaitReason != nil { - t.Fatalf("replacement = %#v, want the cancelled wait rolled back", row) - } - settled, err := fixture.env.repos.Tasks.GetByID(context.Background(), task.ID) - if err != nil || settled == nil { - t.Fatalf("GetByID task = %#v err=%v", settled, err) - } - terminal := settled.Status == model.TaskStatusCompleted || settled.Status == model.TaskStatusFailed || - settled.Status == model.TaskStatusExhausted || settled.Status == model.TaskStatusCancelled - if settled.RetryCount != 0 || terminal { - t.Fatalf("task = %#v, want an active coordinator with retry budget untouched", settled) - } -} - -// A replacement has to open its own paid service. If the chosen provider still -// runs a live service for this bucket, adopting it would leave the replacement -// paying for, and later retiring, a service it does not own. -func TestUploader_ReplacementNeverAttachesToAnExistingService(t *testing.T) { - fixture := seedReplacementEnv(t) - ctx := context.Background() - - // The target provider already runs a data set carrying this bucket's - // metadata: a generation released locally without being terminated on chain. - var createCalls atomic.Int32 - existing := readyFakeUploadContext( - sdktypes.NewBigInt(202), sdktypes.NewBigInt(9002), sdktypes.NewBigInt(3002), testCID(t)) - existing.createCalls = &createCalls - existingRef, ok := existing.DataSetRef() - if !ok { - t.Fatal("existing target has no data set reference") - } - fixture.env.storage.FindMatchingDataSetFunc = func(_ context.Context, providerID sdktypes.BigInt, metadata map[string]string, withCDN bool) (*storage.DataSetRef, error) { - if !providerID.Equal(sdktypes.NewBigInt(202)) || metadata["bucket"] != fixture.bucket.Name || withCDN { - t.Fatalf("FindMatchingDataSet inputs = provider:%s metadata:%v withCDN:%t", providerID.String(), metadata, withCDN) - } - return &existingRef, nil - } - fixture.env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextProviderIDEqual(opts, sdktypes.NewBigInt(202)) { - return fixture.targetCtx, nil - } - return fixture.sourceCtx, nil - } - - replacement := fixture.authorize(t, "202") - task := fixture.coordinatorTask(t, replacement.ID) - fixture.runUploaderUntil(t, func() bool { - row, err := fixture.env.repos.Replacements.GetByID(ctx, replacement.ID) - return err == nil && row != nil && row.Status == storagereplacement.StatusFailed - }, 20*time.Second) - - if calls := createCalls.Load(); calls != 0 { - t.Fatalf("CreateDataSet calls = %d, want the replacement to refuse rather than reuse", calls) - } - target, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, replacement.TargetDataSetID) - if err != nil || target == nil { - t.Fatalf("GetDataSetBindingByID = %#v err=%v", target, err) - } - if target.DataSetID != nil { - t.Fatalf("target adopted data set %s, want no service recorded", target.DataSetID.String()) - } - if target.Status == model.StorageDataSetStatusReady { - t.Fatal("target was marked ready without a service of its own") - } - - row, err := fixture.env.repos.Replacements.GetByID(ctx, replacement.ID) - if err != nil || row == nil || row.LastError == nil { - t.Fatalf("replacement = %#v err=%v, want a recorded reason", row, err) - } - // The operator has to be able to act on it: name the provider and the service. - if !strings.Contains(*row.LastError, "202") || !strings.Contains(*row.LastError, "9002") { - t.Fatalf("last error = %q, want the provider and data set named", *row.LastError) - } - - // The source still owns the replica, so a new confirmation is the way out. - source, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.source.ID) - if err != nil || source == nil || !source.IsCurrent { - t.Fatalf("source = %#v err=%v, want it to still own the replica", source, err) - } - // Retrying cannot make a taken provider free, so the budget is not spent. - settled, err := fixture.env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || settled == nil { - t.Fatalf("GetByID task = %#v err=%v", settled, err) - } - if settled.RetryCount != 0 { - t.Fatalf("retry count = %d, want no retries spent on an unusable provider", settled.RetryCount) - } - if settled.Status != model.TaskStatusFailed { - t.Fatalf("task status = %s, want failed", settled.Status) - } -} - -// Refusing a bound context must not refuse this replacement's own service. Once -// a creation has been submitted, the recorded transaction is what identifies -// the service -- and by then the SDK resolves a bound context for it, which is -// exactly the state a crash between submitting and recording leaves behind. -func TestUploader_ReplacementResumesItsOwnSubmittedCreation(t *testing.T) { - fixture := seedReplacementEnv(t) - ctx := context.Background() - replacement := fixture.authorize(t, "202") - - if err := fixture.env.repos.Uploads.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ - ID: replacement.TargetDataSetID, - TransactionID: "0xcreate2002", - StatusURL: "https://provider-202.example/status/create", - ClientDataSetID: onChainIDPtr(t, "12002"), - }); err != nil { - t.Fatalf("MarkDataSetCreating: %v", err) - } - // The service this replacement created is now visible to the resolver. - bound := readyFakeUploadContext( - sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(3002), testCID(t)) - fixture.env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextProviderIDEqual(opts, sdktypes.NewBigInt(202)) || - createContextDataSetIDEqual(opts, sdktypes.NewBigInt(2002)) { - return bound, nil - } - return fixture.sourceCtx, nil - } - - fixture.runUploaderUntil(t, func() bool { - target, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, replacement.TargetDataSetID) - return err == nil && target != nil && target.Status == model.StorageDataSetStatusReady - }, 20*time.Second) - - target, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, replacement.TargetDataSetID) - if err != nil || target == nil || target.DataSetID == nil { - t.Fatalf("target = %#v err=%v, want the submitted service recorded", target, err) - } - if target.DataSetID.String() != "2002" { - t.Fatalf("target recorded data set %s, want the one it submitted", target.DataSetID.String()) - } - row, err := fixture.env.repos.Replacements.GetByID(ctx, replacement.ID) - if err != nil || row == nil { - t.Fatalf("GetByID = %#v err=%v", row, err) - } - if row.Status == storagereplacement.StatusFailed { - t.Fatalf("replacement failed on its own submitted service: %v", row.LastError) - } -} - -// A migration whose target refuses writes must release its commit attempt and -// pause on the target, the same failover the ordinary upload paths take. -func TestUploader_ReplacementWriteBlockedTargetPausesOnTheTarget(t *testing.T) { - fixture := seedReplacementEnv(t) - ctx := context.Background() - row := fixture.authorize(t, "202") - - fixture.targetCtx.submitPreflightErr = fmt.Errorf("storage.DataSetContext.SubmitCommit: %w", - &storage.DataSetPDPPaymentTerminatedError{DataSetID: sdktypes.NewBigInt(2002), PDPEndEpoch: 3778900}) - - fixture.runUploaderUntil(t, func() bool { - got, err := fixture.env.repos.Replacements.GetByID(ctx, row.ID) - return err == nil && got != nil && got.WaitReason != nil && *got.WaitReason == storagereplacement.WaitReasonTarget - }, 20*time.Second) - - // The coordinator owns generation lifecycle here, so a refused target pauses - // the replacement instead of draining the generation it is migrating into. - // What the commit path owes is releasing its own attempt. - targetCopy, err := fixture.env.repos.Uploads.GetUploadCopyForDataSet(ctx, fixture.upload.ID, row.TargetDataSetID) - if err != nil || targetCopy == nil || targetCopy.Status != model.StorageUploadCopyStatusPieceReady || - targetCopy.CommitAttemptID != nil || targetCopy.CommitReadyAt != nil || - targetCopy.CommitExtraDataHex != nil || targetCopy.CommitTransactionID != nil { - t.Fatalf("target copy = %#v err=%v, want its commit attempt released", targetCopy, err) - } - var item storagereplacement.Item - if err := fixture.env.db.NewSelect().Model(&item). - Where("replacement_id = ?", row.ID). - Scan(ctx); err != nil || item.RetryCount != 0 { - t.Fatalf("paused item = %#v err=%v, want the retry budget untouched", item, err) - } -} diff --git a/internal/worker/uploader_replica_repair.go b/internal/worker/uploader_replica_repair.go deleted file mode 100644 index 86cfe47..0000000 --- a/internal/worker/uploader_replica_repair.go +++ /dev/null @@ -1,572 +0,0 @@ -package worker - -import ( - "context" - "errors" - "fmt" - "log/slog" - "os" - "time" - - "github.com/ipfs/go-cid" - "github.com/strahe/synaps3/internal/admin" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagecommit" - "github.com/strahe/synaps3/internal/synapse" - idtypes "github.com/strahe/synaps3/internal/types" - "github.com/strahe/synapse-go/storage" -) - -const ( - replicaRepairDataSetIDKey = "storage_data_set_id" - replicaRepairCopyIDKey = "storage_upload_copy_id" -) - -type replicaRepairPayload struct { - dataSetID int64 - copyID int64 -} - -func replicaRepairTaskKey(dataSetID int64) string { - return fmt.Sprintf("upload:repair-data-set:%d", dataSetID) -} - -func newReplicaRepairPayload(dataSetID, copyID int64) map[string]interface{} { - return map[string]interface{}{ - replicaRepairDataSetIDKey: dataSetID, - replicaRepairCopyIDKey: copyID, - } -} - -func parseReplicaRepairPayload(task *model.Task) (replicaRepairPayload, error) { - if task == nil { - return replicaRepairPayload{}, errors.New("replica repair task is required") - } - dataSetID, err := payloadInt64(task.Payload, replicaRepairDataSetIDKey) - if err != nil { - return replicaRepairPayload{}, fmt.Errorf("invalid %s: %w", replicaRepairDataSetIDKey, err) - } - if dataSetID <= 0 { - return replicaRepairPayload{}, fmt.Errorf("invalid %s: must be positive", replicaRepairDataSetIDKey) - } - copyID, err := payloadInt64(task.Payload, replicaRepairCopyIDKey) - if err != nil { - return replicaRepairPayload{}, fmt.Errorf("invalid %s: %w", replicaRepairCopyIDKey, err) - } - if copyID <= 0 { - return replicaRepairPayload{}, fmt.Errorf("invalid %s: must be positive", replicaRepairCopyIDKey) - } - return replicaRepairPayload{dataSetID: dataSetID, copyID: copyID}, nil -} - -func (u *Uploader) ensureReplicaRepairTask(ctx context.Context, binding *model.StorageDataSet, maxRetries int) error { - _, err := ensureReplicaRepairTask(ctx, u.repos, binding, maxRetries) - return err -} - -func ensureReplicaRepairTask(ctx context.Context, repos *repository.Repositories, binding *model.StorageDataSet, maxRetries int) (bool, error) { - if binding == nil || binding.ID <= 0 || !dataSetBindingEstablished(binding) { - return false, nil - } - if binding.Status != model.StorageDataSetStatusUnavailable && binding.Status != model.StorageDataSetStatusReady { - return false, nil - } - // An approved replacement already owns this generation's remaining work. - // Repairing it in place would fight the migration, so recovery stands down - // until the replacement finishes or terminally fails. - replacing, err := repos.Replacements.HasInProgressForDataSet(ctx, binding.ID) - if err != nil { - return false, fmt.Errorf("check provider replacement for data set %d: %w", binding.ID, err) - } - if replacing { - return false, nil - } - copyRow, err := repos.Uploads.NextFinalizableCopyForDataSet(ctx, binding.ID) - if err != nil { - return false, fmt.Errorf("select replica finalization copy for data set %d: %w", binding.ID, err) - } - if copyRow == nil { - copyRow, err = repos.Uploads.NextIncompleteCopyForDataSet(ctx, binding.ID) - if err != nil { - return false, fmt.Errorf("select replica repair copy for data set %d: %w", binding.ID, err) - } - } - if copyRow == nil { - return false, nil - } - upload, err := repos.Uploads.GetByID(ctx, copyRow.UploadID) - if err != nil { - return false, fmt.Errorf("load replica repair upload %d: %w", copyRow.UploadID, err) - } - if upload == nil || upload.SourceVersionID == "" { - return false, fmt.Errorf("load replica repair upload %d: %w", copyRow.UploadID, repository.ErrNotFound) - } - stage := uploadStageRepairReplica - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "bucket", - RefID: binding.BucketID, - RefVersionID: upload.SourceVersionID, - IdempotencyKey: replicaRepairTaskKey(binding.ID), - Payload: newReplicaRepairPayload(binding.ID, copyRow.ID), - Status: model.TaskStatusQueued, - MaxRetries: maxRetries, - ScheduledAt: time.Now(), - } - if _, err := repos.Tasks.EnsureRecurring(ctx, task); err != nil { - return false, fmt.Errorf("ensure replica repair task for data set %d: %w", binding.ID, err) - } - return true, nil -} - -func (u *Uploader) processReplicaRepairTask(ctx context.Context, task *model.Task, logger *slog.Logger) { - payload, err := parseReplicaRepairPayload(task) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "parse replica repair task", err) - return - } - if task == nil || task.ClaimedAt == nil { - return - } - item, err := u.repos.Uploads.AcquireReplicaRepairItem(ctx, repository.AcquireReplicaRepairItemInput{ - TaskID: task.ID, - TaskClaimedAt: *task.ClaimedAt, - StorageDataSetID: payload.dataSetID, - StorageUploadCopyID: payload.copyID, - BucketID: task.RefID, - }) - if err != nil { - switch { - case errors.Is(err, repository.ErrReplicaRepairItemCancelled): - u.advanceReplicaRepairTask(ctx, task, payload.dataSetID, logger) - case errors.Is(err, repository.ErrTaskClaimLost): - return - default: - u.handleTaskFailure(ctx, task, logger, "acquire replica repair item", err) - } - return - } - binding := &item.DataSet - copyRow := &item.Copy - upload := &item.Upload - version := &item.Version - if !copyCommitted(copyRow) { - ordinaryTaskRunning, err := u.repos.Tasks.HasEarlierRunningUploadCopyTask(ctx, task, copyRow.UploadID, copyRow.CopyIndex) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "check ordinary upload copy task", err) - return - } - if ordinaryTaskRunning { - u.waitForStorageDependency(ctx, task, logger, "Waiting for the active upload operation to finish") - return - } - } - if binding.DataSetID == nil || binding.DataSetID.IsZero() { - u.handleTaskFailure(ctx, task, logger, "validate replica repair data set", errors.New("established data set has no data set ID")) - return - } - alreadyCommitted := copyCommitted(copyRow) - switch binding.Status { - case model.StorageDataSetStatusReady, model.StorageDataSetStatusUnavailable: - case model.StorageDataSetStatusDraining: - if !alreadyCommitted { - completeWorkerTask(ctx, u.repos, task, "uploader", logger) - return - } - case model.StorageDataSetStatusRetired, model.StorageDataSetStatusFailed: - completeWorkerTask(ctx, u.repos, task, "uploader", logger) - return - default: - u.handleTaskFailure(ctx, task, logger, "validate replica repair data set", fmt.Errorf("data set status %s cannot be recovered in place", binding.Status)) - return - } - bucket, err := u.repos.Buckets.GetByID(ctx, upload.BucketID) - if err != nil || bucket == nil { - if err == nil { - err = fmt.Errorf("bucket %d not found", upload.BucketID) - } - u.handleTaskFailure(ctx, task, logger, "load replica repair bucket", err) - return - } - if alreadyCommitted { - if binding.Status == model.StorageDataSetStatusUnavailable { - if _, err := u.contextForReadyBinding(ctx, binding); err != nil { - u.handleReplicaRepairDataSetFailure(ctx, task, binding, logger, "verify committed replica context", err) - return - } - recovered, err := u.repos.Uploads.RecoverDataSet(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: upload.ID, - DataSetID: *binding.DataSetID, - }) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "recover committed replica data set", err) - return - } - if !recovered { - latest, loadErr := u.repos.Uploads.GetDataSetBindingByID(ctx, binding.ID) - if loadErr != nil { - u.handleTaskFailure(ctx, task, logger, "reload committed replica data set", loadErr) - return - } - if latest == nil { - u.handleTaskFailure(ctx, task, logger, "reload committed replica data set", repository.ErrNotFound) - return - } - switch latest.Status { - case model.StorageDataSetStatusReady: - binding = latest - case model.StorageDataSetStatusUnavailable: - u.waitForStorageDependency(ctx, task, logger, "Waiting for the assigned storage provider to recover") - return - case model.StorageDataSetStatusDraining: - binding = latest - case model.StorageDataSetStatusRetired, model.StorageDataSetStatusFailed: - completeWorkerTask(ctx, u.repos, task, "uploader", logger) - return - default: - u.handleTaskFailure(ctx, task, logger, "recover committed replica data set", fmt.Errorf("data set status changed to %s", latest.Status)) - return - } - } else { - binding.Status = model.StorageDataSetStatusReady - } - } - if err := u.finishReplicaRepairItem(ctx, task, upload, version, binding.ID, logger); err != nil { - u.handleTaskFailure(ctx, task, logger, "finalize committed replica repair", err) - } - return - } - storageCtx, err := u.contextForReadyBinding(ctx, binding) - if err != nil { - if u.handleUnavailableCommitContext(ctx, task, binding, copyRow, logger, "restore replica context") { - return - } - u.handleReplicaRepairDataSetFailure(ctx, task, binding, logger, "restore replica context", err) - return - } - if err := u.repairReplicaCopy(ctx, task, upload, version, bucket, binding, copyRow, storageCtx, logger); err != nil { - u.handleReplicaRepairDataSetFailure(ctx, task, binding, logger, "repair replica copy", err) - } -} - -func (u *Uploader) repairReplicaCopy( - ctx context.Context, - task *model.Task, - upload *model.StorageUpload, - version *model.ObjectVersion, - bucket *model.Bucket, - binding *model.StorageDataSet, - copyRow *model.StorageUploadCopy, - storageCtx synapse.DataSetTarget, - logger *slog.Logger, -) error { - readableCopies, err := u.repos.Uploads.ListReadableCommittedCopies(ctx, upload.ID) - if err != nil { - return err - } - var pieceCID cid.Cid - var pieceCIDString string - if !copyHasPiece(copyRow) { - var extraHex string - if len(readableCopies) > 0 { - sourceCopy := readableCopies[0] - pieceCID, err = cid.Decode(sourceCopy.PieceCID) - if err != nil { - return fmt.Errorf("decode source piece CID: %w", err) - } - pieceCIDString = sourceCopy.PieceCID - pieces := []storage.PieceInput{{PieceCID: pieceCID}} - extraData, encodedExtra, err := u.extraDataForCopy(ctx, storageCtx, copyRow, pieces) - if err != nil { - return err - } - extraHex = encodedExtra - if _, err := storageCtx.Pull(ctx, storage.PullRequest{ - Pieces: []cid.Cid{pieceCID}, - ExtraData: extraData, - From: func(cid.Cid) string { - return sourceCopy.RetrievalURL - }, - }); err != nil { - return err - } - } else { - if version == nil { - u.waitForReplicaRepairSource(ctx, task, logger) - return nil - } - rc, _, err := u.cache.Get(ctx, bucket.Name, version.CacheKey) - if err != nil { - if os.IsNotExist(err) { - if version.InCache { - if markErr := u.repos.Objects.SetVersionCachePresence(ctx, version.VersionID, false); markErr != nil { - logger.Warn("failed to mark cache location absent", "versionID", version.VersionID, "error", markErr) - } - } - u.waitForReplicaRepairSource(ctx, task, logger) - return nil - } - return fmt.Errorf("open retained cache data: %w", err) - } - result, storeErr := storageCtx.Store(ctx, rc, &storage.StoreOptions{}) - closeErr := rc.Close() - if storeErr != nil { - return storeErr - } - if closeErr != nil { - return fmt.Errorf("close retained cache data: %w", closeErr) - } - if result == nil || !result.PieceCID.Defined() { - return errors.New("replica repair store returned no piece CID") - } - pieceCID = result.PieceCID - pieceCIDString = pieceCID.String() - _, extraHex, err = u.extraDataForCopy( - ctx, - storageCtx, - copyRow, - []storage.PieceInput{{PieceCID: pieceCID}}, - ) - if err != nil { - return err - } - } - if err := u.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - StorageUploadCopyID: copyRow.ID, - RequireEligibleCopy: true, - UploadID: upload.ID, - CopyIndex: copyRow.CopyIndex, - PieceCID: pieceCIDString, - RetrievalURL: storageCtx.PieceURL(pieceCID), - CommitExtraDataHex: extraHex, - }); err != nil { - return err - } - copyRow.Status = model.StorageUploadCopyStatusPieceReady - copyRow.CommitExtraDataHex = &extraHex - } else { - if upload.PieceCID == nil || *upload.PieceCID == "" { - u.waitForReplicaRepairSource(ctx, task, logger) - return nil - } - pieceCIDString = *upload.PieceCID - pieceCID, err = cid.Decode(pieceCIDString) - if err != nil { - return fmt.Errorf("decode stored piece CID: %w", err) - } - } - pieces := []storage.PieceInput{{PieceCID: pieceCID}} - advance, err := u.commitReplicaRepairCopy(ctx, upload, binding, copyRow, storageCtx, pieces, false) - if err != nil && advance.State == storagecommit.AdvancePending && synapse.IsProviderUnavailable(err) { - return err - } - if u.waitForCommitAdvance(ctx, task, logger, advance) { - if err != nil { - logger.Warn("storage commit evidence remains fenced", "stage", "replica repair commit", "error", err) - } - return nil - } - if err != nil { - // Commit failures own their task transition here so final exhaustion can - // clear an unsubmitted FIFO reservation in the same transaction. - u.handleCommitTaskFailure(ctx, task, copyRow, logger, "advance replica repair commit", err) - return nil - } - switch { - case advance.State == storagecommit.AdvanceReleased && advance.ReleaseReason == storagecommit.ReleaseDataSetUnavailable: - return commitReleaseCause(advance) - case advance.State == storagecommit.AdvanceReleased: - return nil - case advance.State == storagecommit.AdvanceRejected: - return errCommitRejected - case advance.State != storagecommit.AdvanceConfirmed || advance.Confirmation == nil || len(advance.Confirmation.PieceIDs) == 0: - return errors.New("replica repair commit returned no piece ID") - } - result := advance.Confirmation - pieceID := idtypes.OnChainIDFromSDK(result.PieceIDs[0]) - if err := u.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - StorageUploadCopyID: copyRow.ID, - RequireEligibleCopy: true, - UploadID: upload.ID, - CopyIndex: copyRow.CopyIndex, - PieceCID: pieceCIDString, - PieceID: &pieceID, - RetrievalURL: storageCtx.PieceURL(pieceCID), - CommitExtraDataHex: derefString(copyRow.CommitExtraDataHex), - CommitTransactionID: result.TransactionID, - CommitAttemptID: advance.AttemptID, - CommitConfirmedTransactionID: result.ConfirmedTransactionID, - }); err != nil { - return err - } - if binding.Status == model.StorageDataSetStatusUnavailable { - recovered, err := u.repos.Uploads.RecoverDataSet(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: upload.ID, - DataSetID: *binding.DataSetID, - }) - if err != nil { - return fmt.Errorf("mark replica data set recovered: %w", err) - } - if !recovered { - latest, loadErr := u.repos.Uploads.GetDataSetBindingByID(ctx, binding.ID) - if loadErr != nil { - return fmt.Errorf("reload replica data set after recovery conflict: %w", loadErr) - } - if latest == nil { - return fmt.Errorf("reload replica data set after recovery conflict: %w", repository.ErrNotFound) - } - switch latest.Status { - case model.StorageDataSetStatusReady: - binding = latest - case model.StorageDataSetStatusUnavailable: - u.waitForStorageDependency(ctx, task, logger, "Waiting for the assigned storage provider to recover") - return nil - case model.StorageDataSetStatusDraining: - binding = latest - case model.StorageDataSetStatusRetired, model.StorageDataSetStatusFailed: - completeWorkerTask(ctx, u.repos, task, "uploader", logger) - return nil - default: - return fmt.Errorf("replica data set status changed to %s during recovery", latest.Status) - } - } else { - binding.Status = model.StorageDataSetStatusReady - } - } - return u.finishReplicaRepairItem(ctx, task, upload, version, binding.ID, logger) -} - -func (u *Uploader) finishReplicaRepairItem( - ctx context.Context, - task *model.Task, - upload *model.StorageUpload, - version *model.ObjectVersion, - dataSetID int64, - logger *slog.Logger, -) error { - if _, err := u.repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: upload.BucketID, - ContentSize: upload.ContentSize, - Checksum: upload.Checksum, - }); err != nil { - return err - } - if version == nil { - return fmt.Errorf("load live version for storage upload %d: %w", upload.ID, repository.ErrNotFound) - } - ref := repository.ObjectVersionRef{ObjectID: version.ObjectID, VersionID: version.VersionID} - _, needsPreparation, err := u.scheduleRemainingPeerCopies(ctx, ref, upload.BucketID, upload.ID, task.MaxRetries) - if err != nil { - return err - } - if needsPreparation { - if err := u.enqueueRepairUploadForVersion(ctx, ref, task.MaxRetries, upload.ID); err != nil { - return err - } - } - if _, _, err := u.repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, u.finalizeUploadInput(upload.ID)); err != nil { - return err - } - u.advanceReplicaRepairTask(ctx, task, dataSetID, logger) - return nil -} - -func (u *Uploader) waitForReplicaRepairSource(ctx context.Context, task *model.Task, logger *slog.Logger) { - u.waitForStorageDependency(ctx, task, logger, "Waiting for a readable replica or retained cache data") -} - -func (u *Uploader) commitReplicaRepairCopy( - ctx context.Context, - upload *model.StorageUpload, - binding *model.StorageDataSet, - copyRow *model.StorageUploadCopy, - storageCtx synapse.DataSetTarget, - pieces []storage.PieceInput, - ownerTerminal bool, -) (storagecommit.AdvanceResult, error) { - if upload == nil || copyRow == nil || copyRow.UploadID != upload.ID { - return storagecommit.AdvanceResult{}, errors.New("replica repair commit identity mismatch") - } - return u.advanceStorageCommit(ctx, binding, copyRow, storageCtx, pieces, true, ownerTerminal) -} - -func (u *Uploader) handleReplicaRepairDataSetFailure(ctx context.Context, task *model.Task, binding *model.StorageDataSet, logger *slog.Logger, stage string, err error) { - switch { - case dataSetWriteBlockedError(err), synapse.IsDataSetServiceEnded(err): - latest, markErr := u.markDataSetStatus(ctx, binding, model.StorageDataSetStatusDraining, err.Error()) - if markErr != nil { - u.handleTaskFailure(ctx, task, logger, "mark replica data set draining", markErr) - return - } - switch latest.Status { - case model.StorageDataSetStatusDraining, model.StorageDataSetStatusRetired, model.StorageDataSetStatusFailed: - completeWorkerTask(ctx, u.repos, task, "uploader", logger) - case model.StorageDataSetStatusReady, model.StorageDataSetStatusUnavailable: - u.waitForStorageDependency(ctx, task, logger, "Waiting to retry the storage operation") - default: - u.handleTaskFailure(ctx, task, logger, stage, fmt.Errorf("data set status changed to %s: %w", latest.Status, err)) - } - case synapse.IsProviderUnavailable(err), synapse.IsNoProviderCandidates(err): - latest, markErr := u.markDataSetStatus(ctx, binding, model.StorageDataSetStatusUnavailable, err.Error()) - if markErr != nil { - u.handleTaskFailure(ctx, task, logger, "mark replica data set unavailable", markErr) - return - } - switch latest.Status { - case model.StorageDataSetStatusUnavailable: - u.waitForStorageDependency(ctx, task, logger, "Waiting for the assigned storage provider to recover") - case model.StorageDataSetStatusReady: - u.waitForStorageDependency(ctx, task, logger, "Waiting to retry the storage operation") - case model.StorageDataSetStatusDraining, model.StorageDataSetStatusRetired, model.StorageDataSetStatusFailed: - completeWorkerTask(ctx, u.repos, task, "uploader", logger) - default: - u.handleTaskFailure(ctx, task, logger, stage, fmt.Errorf("data set status changed to %s: %w", latest.Status, err)) - } - default: - u.handleTaskFailure(ctx, task, logger, stage, err) - } -} - -func (u *Uploader) advanceReplicaRepairTask(ctx context.Context, task *model.Task, dataSetID int64, logger *slog.Logger) { - var nextCopyID int64 - err := u.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { - if err := txRepos.Tasks.LockRunningClaim(ctx, task); err != nil { - return err - } - next, err := txRepos.Uploads.NextFinalizableCopyForDataSet(ctx, dataSetID) - if err != nil { - return err - } - if next == nil { - next, err = txRepos.Uploads.NextIncompleteCopyForDataSet(ctx, dataSetID) - if err != nil { - return err - } - } - if next == nil { - return txRepos.Tasks.Complete(ctx, task) - } - upload, err := txRepos.Uploads.GetByID(ctx, next.UploadID) - if err != nil { - return err - } - if upload == nil || upload.SourceVersionID == "" { - return fmt.Errorf("load next replica repair upload %d: %w", next.UploadID, repository.ErrNotFound) - } - nextCopyID = next.ID - return txRepos.Tasks.ContinueRunning(ctx, task, upload.SourceVersionID, newReplicaRepairPayload(dataSetID, next.ID)) - }) - if err != nil { - u.handleTaskFailure(ctx, task, logger, "advance replica repair task", err) - return - } - if nextCopyID > 0 { - logger.Debug("queued next replica repair item", "dataSetID", dataSetID, "copyID", nextCopyID) - } - admin.WorkerTasksProcessed.WithLabelValues("uploader", "success").Inc() -} diff --git a/internal/worker/uploader_test.go b/internal/worker/uploader_test.go deleted file mode 100644 index bede8ce..0000000 --- a/internal/worker/uploader_test.go +++ /dev/null @@ -1,6663 +0,0 @@ -package worker_test - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "log/slog" - "math/big" - "net/http" - "net/http/httptest" - "os" - "strconv" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/ipfs/go-cid" - "github.com/multiformats/go-multihash" - "github.com/strahe/synaps3/internal/cache" - "github.com/strahe/synaps3/internal/cacheeviction" - "github.com/strahe/synaps3/internal/config" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/storagecommit" - "github.com/strahe/synaps3/internal/synapse" - "github.com/strahe/synaps3/internal/testutil" - "github.com/strahe/synaps3/internal/worker" - "github.com/strahe/synapse-go/pdp" - "github.com/strahe/synapse-go/storage" - sdktypes "github.com/strahe/synapse-go/types" -) - -const fakeSubmittedCommitTxHash = "0x7890abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456" - -func testCID(t *testing.T) cid.Cid { - t.Helper() - mh, err := multihash.Sum([]byte("test-data"), multihash.SHA2_256, -1) - if err != nil { - t.Fatalf("creating test multihash: %v", err) - } - return cid.NewCidV1(cid.Raw, mh) -} - -func createContextProviderIDEqual(opts *testutil.OpenTargetOptions, want sdktypes.BigInt) bool { - return opts.ProviderID != nil && opts.ProviderID.Equal(want) -} - -func createContextDataSetIDEqual(opts *testutil.OpenTargetOptions, want sdktypes.BigInt) bool { - return opts.DataSetID != nil && opts.DataSetID.Equal(want) -} - -func copyCommitSubmittedForTest(copyRow *model.StorageUploadCopy) bool { - return copyRow != nil && copyRow.Status == model.StorageUploadCopyStatusCommitting && - copyRow.CommitAttemptID != nil && *copyRow.CommitAttemptID != "" && - copyRow.CommitAttemptedAt != nil && - copyRow.CommitTransactionID != nil && *copyRow.CommitTransactionID != "" -} - -func seedSubmittedCommitAttempt( - t *testing.T, - repos *repository.Repositories, - copyRow *model.StorageUploadCopy, - attemptID string, - extraDataHex string, - transactionID string, -) { - t.Helper() - if copyRow == nil || copyRow.StorageDataSetID == nil { - t.Fatal("submitted commit fixture has no concrete copy binding") - } - identity := storagecommit.CopyIdentity{ - StorageUploadCopyID: copyRow.ID, - UploadID: copyRow.UploadID, - CopyIndex: copyRow.CopyIndex, - StorageDataSetID: *copyRow.StorageDataSetID, - } - if _, err := repos.Uploads.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: identity, AttemptID: attemptID, - }); err != nil { - t.Fatalf("ReserveCommitAttempt: %v", err) - } - if _, err := repos.Uploads.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ - Copy: identity, AttemptID: attemptID, ExtraDataHex: extraDataHex, - }); err != nil { - t.Fatalf("MarkCommitAttempted: %v", err) - } - if err := repos.Uploads.RecordCommitTransaction(t.Context(), storagecommit.EvidenceInput{ - Copy: identity, AttemptID: attemptID, TransactionID: transactionID, - }); err != nil { - t.Fatalf("RecordCommitTransaction: %v", err) - } -} - -// seedCachedObject creates a bucket, writes a file into the filesystem cache, -// and inserts an object in "cached" state. -func seedCachedObject(t *testing.T, env *testWorkerEnv) (*model.Bucket, int64, string) { - t.Helper() - ctx := context.Background() - - bucket := &model.Bucket{Name: fmt.Sprintf("b-%d", time.Now().UnixNano()), Status: model.BucketStatusActive} - if err := env.repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("creating bucket: %v", err) - } - - key := "hello.txt" - data := []byte("hello world") - versionID := model.NewVersionID() - cacheKey := ".versions/" + versionID - - info, err := env.cache.Put(ctx, bucket.Name, cacheKey, bytes.NewReader(data)) - if err != nil { - t.Fatalf("cache put: %v", err) - } - - version := &model.ObjectVersion{ - VersionID: versionID, - BucketID: bucket.ID, - Key: key, - Size: int64(len(data)), - ETag: info.ETag, - Checksum: info.Checksum, - ContentType: "text/plain", - CacheKey: cacheKey, - } - objID, err := env.repos.Objects.CreateVersionAndSetCurrent(ctx, version) - if err != nil { - t.Fatalf("creating object version: %v", err) - } - return bucket, objID, versionID -} - -// seedObjectInDB inserts a bucket and object into the DB only (no cache write). -func seedObjectInDB(t *testing.T, env *testWorkerEnv, bucketStatus model.BucketStatus) (*model.Bucket, int64, string) { - t.Helper() - ctx := context.Background() - - bucket := &model.Bucket{Name: fmt.Sprintf("b-%d", time.Now().UnixNano()), Status: bucketStatus} - if err := env.repos.Buckets.Create(ctx, bucket); err != nil { - t.Fatalf("creating bucket: %v", err) - } - - versionID := model.NewVersionID() - version := &model.ObjectVersion{ - VersionID: versionID, - BucketID: bucket.ID, - Key: "hello.txt", - Size: 11, - ETag: "abc123", - Checksum: "sha256-test", - ContentType: "text/plain", - CacheKey: ".versions/" + versionID, - } - objID, err := env.repos.Objects.CreateVersionAndSetCurrent(ctx, version) - if err != nil { - t.Fatalf("creating object version: %v", err) - } - return bucket, objID, versionID -} - -// seedTask creates a queued task of the given type. -func seedTask(t *testing.T, env *testWorkerEnv, taskType model.TaskType, refID int64, versionID string, maxRetries, retryCount int) *model.Task { - t.Helper() - ctx := context.Background() - task := &model.Task{ - Type: taskType, - RefType: "object", - RefID: refID, - RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("%s:%s", taskType, versionID), - Status: model.TaskStatusQueued, - RetryCount: retryCount, - MaxRetries: maxRetries, - ScheduledAt: time.Now(), - } - switch taskType { - case model.TaskTypeUpload: - task.Payload = map[string]interface{}{"stage": ""} - case model.TaskTypeEvictCache: - stage := cacheeviction.StageAfterUpload - task.Stage = &stage - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("creating task: %v", err) - } - return task -} - -func seedStagedUploadTask(t *testing.T, env *testWorkerEnv, refID int64, versionID string, maxRetries int) *model.Task { - t.Helper() - stage := "prepare_upload" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: refID, - RefVersionID: versionID, - IdempotencyKey: "upload:" + versionID, - Status: model.TaskStatusQueued, - MaxRetries: maxRetries, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(context.Background(), task); err != nil { - t.Fatalf("creating staged task: %v", err) - } - return task -} - -func waitForObjectState(t *testing.T, env *testWorkerEnv, versionID string, state model.ObjectState, timeout time.Duration) { - t.Helper() - deadline := time.After(timeout) - ticker := time.NewTicker(10 * time.Millisecond) - defer ticker.Stop() - - for { - select { - case <-deadline: - version, err := env.repos.Objects.GetVersionByID(context.Background(), versionID) - if err != nil { - t.Fatalf("timed out waiting for version %s state %s; last lookup error: %v", versionID, state, err) - } - if version == nil { - t.Fatalf("timed out waiting for version %s state %s; version not found", versionID, state) - } - t.Fatalf("timed out waiting for version %s state %s; current state %s", versionID, state, version.State) - case <-ticker.C: - version, err := env.repos.Objects.GetVersionByID(context.Background(), versionID) - if err != nil || version == nil { - continue - } - if version.State == state { - return - } - } - } -} - -// runWorkerUntilTask runs a worker and waits until the given task -// leaves active queue states, or times out. -func runWorkerUntilTask(t *testing.T, env *testWorkerEnv, w worker.Worker, taskID int64, timeout time.Duration) *model.Task { - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - baseTasks := env.repos.Tasks - env.repos.Tasks = stopAfterTaskRepo{TaskRepository: baseTasks, stopTaskID: taskID} - defer func() { env.repos.Tasks = baseTasks }() - - done := make(chan struct{}) - go func() { - _ = w.Run(ctx) - close(done) - }() - - deadline := time.After(timeout) - ticker := time.NewTicker(20 * time.Millisecond) - defer ticker.Stop() - - for { - select { - case <-deadline: - cancel() - <-done - t.Fatalf("timed out waiting for task %d to be processed", taskID) - case <-ticker.C: - task, err := env.repos.Tasks.GetByID(context.Background(), taskID) - if err != nil { - continue - } - if task != nil && task.Status != model.TaskStatusQueued && task.Status != model.TaskStatusScheduled && task.Status != model.TaskStatusWaiting && task.Status != model.TaskStatusRunning { - cancel() - <-done - return task - } - } - } -} - -type stopAfterTaskRepo struct { - repository.TaskRepository - stopTaskID int64 -} - -func (r stopAfterTaskRepo) ClaimReady(ctx context.Context, taskType model.TaskType, leaseDuration time.Duration) (*model.Task, error) { - task, err := r.GetByID(ctx, r.stopTaskID) - if err != nil { - return nil, err - } - if task != nil && !taskStatusActive(task.Status) { - return nil, nil - } - return r.TaskRepository.ClaimReady(ctx, taskType, leaseDuration) -} - -func taskStatusActive(status model.TaskStatus) bool { - switch status { - case model.TaskStatusQueued, model.TaskStatusScheduled, model.TaskStatusWaiting, model.TaskStatusRunning: - return true - default: - return false - } -} - -// runWorkerUntilTaskRetryCount runs a worker until the task has recorded at -// least the requested retry count, then cancels the worker. -func runWorkerUntilTaskRetryCount(t *testing.T, env *testWorkerEnv, w worker.Worker, taskID int64, retryCount int, timeout time.Duration) { - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - done := make(chan struct{}) - go func() { - _ = w.Run(ctx) - close(done) - }() - - deadline := time.After(timeout) - ticker := time.NewTicker(20 * time.Millisecond) - defer ticker.Stop() - - for { - select { - case <-deadline: - cancel() - <-done - t.Fatalf("timed out waiting for task %d retry_count >= %d", taskID, retryCount) - case <-ticker.C: - task, err := env.repos.Tasks.GetByID(context.Background(), taskID) - if err != nil { - continue - } - if task != nil && task.RetryCount >= retryCount && task.Status != model.TaskStatusRunning && task.Status != model.TaskStatusFailed { - cancel() - <-done - return - } - } - } -} - -func waitForSignal(t *testing.T, ch <-chan struct{}, timeout time.Duration, name string) { - t.Helper() - select { - case <-ch: - case <-time.After(timeout): - t.Fatalf("timed out waiting for %s", name) - } -} - -func waitForTaskStatus(t *testing.T, env *testWorkerEnv, taskID int64, status model.TaskStatus, timeout time.Duration) { - t.Helper() - deadline := time.After(timeout) - ticker := time.NewTicker(10 * time.Millisecond) - defer ticker.Stop() - - for { - select { - case <-deadline: - task, err := env.repos.Tasks.GetByID(context.Background(), taskID) - if err != nil { - t.Fatalf("timed out waiting for task %d to reach %s; last lookup error: %v", taskID, status, err) - } - if task == nil { - t.Fatalf("timed out waiting for task %d to reach %s; task not found", taskID, status) - } - t.Fatalf("timed out waiting for task %d to reach %s; current status %s", taskID, status, task.Status) - case <-ticker.C: - task, err := env.repos.Tasks.GetByID(context.Background(), taskID) - if err != nil || task == nil { - continue - } - if task.Status == status { - return - } - } - } -} - -type publishedAdminEvent struct { - topic string - payload map[string]any -} - -type fakeAdminEventPublisher struct { - mu sync.Mutex - events []publishedAdminEvent -} - -func (p *fakeAdminEventPublisher) Publish(topic string, payload map[string]any) { - p.mu.Lock() - defer p.mu.Unlock() - p.events = append(p.events, publishedAdminEvent{topic: topic, payload: payload}) -} - -func (p *fakeAdminEventPublisher) hasTopic(topic string) bool { - p.mu.Lock() - defer p.mu.Unlock() - for _, event := range p.events { - if event.topic == topic { - return true - } - } - return false -} - -type fakeUploadContext struct { - providerID sdktypes.BigInt - dataSetID sdktypes.BigInt - dataSetMu sync.RWMutex - boundDataSet *sdktypes.BigInt - pieceID sdktypes.BigInt - pieceCID cid.Cid - clientDataID sdktypes.BigInt - pullEntered chan struct{} - releasePull chan struct{} - pullOnce sync.Once - pullErr error - createCalls *atomic.Int32 - waitCalls *atomic.Int32 - skipCreateSubmission bool - createErr error - waitErr error - storeErr error - storeProgress []int64 - storeCalls atomic.Int32 - commitErr error - // submitPreflightErr fails a commit the way the SDK does when it refuses the - // data set: contextCore.submitCommit validates writability before it presigns - // or adds pieces, so the provider is never contacted and no transaction is - // recorded. commitErr fails after the submission callback instead, which is - // the ambiguous shape. - submitPreflightErr error - serviceURL string - presignCalls atomic.Int32 - commitCalls atomic.Int32 - commitMu sync.Mutex - commitExtras [][]byte - pullCalls atomic.Int32 -} - -func newFakeUploadContext(providerID sdktypes.BigInt, dataSetID sdktypes.BigInt, pieceID sdktypes.BigInt, pieceCID cid.Cid) *fakeUploadContext { - dataSetUint64, _ := dataSetID.Uint64() - return &fakeUploadContext{ - providerID: providerID, - dataSetID: dataSetID, - pieceID: pieceID, - pieceCID: pieceCID, - clientDataID: sdktypes.NewBigInt(dataSetUint64 + 10000), - } -} - -func readyFakeUploadContext(providerID sdktypes.BigInt, dataSetID sdktypes.BigInt, pieceID sdktypes.BigInt, pieceCID cid.Cid) *fakeUploadContext { - ctx := newFakeUploadContext(providerID, dataSetID, pieceID, pieceCID) - id := dataSetID.Copy() - ctx.boundDataSet = &id - return ctx -} - -func seedReadyBinding(t *testing.T, env *testWorkerEnv, bucketID, uploadID int64, copyIndex int, providerID, dataSetID string) *model.StorageDataSet { - t.Helper() - ctx := context.Background() - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucketID, - ProviderID: onChainID(t, providerID), - CopyIndex: copyIndex, - CreatedByUploadID: uploadID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding(%d): %v", copyIndex, err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: uploadID, - DataSetID: onChainID(t, dataSetID), - }); err != nil { - t.Fatalf("MarkDataSetReady(%d): %v", copyIndex, err) - } - return binding -} - -func seedDeadUnestablishedBinding(t *testing.T, env *testWorkerEnv, bucketID, uploadID int64, copyIndex int, providerID string) *model.StorageDataSet { - t.Helper() - ctx := context.Background() - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucketID, - ProviderID: onChainID(t, providerID), - CopyIndex: copyIndex, - CreatedByUploadID: uploadID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding(%d): %v", copyIndex, err) - } - if _, err := env.db.NewUpdate(). - Model((*model.StorageDataSet)(nil)). - Set("status = ?", model.StorageDataSetStatusUnavailable). - Set("last_error = ?", "create dataset: HTTP 404"). - Where("id = ?", binding.ID). - Exec(ctx); err != nil { - t.Fatalf("mark unestablished unavailable(%d): %v", copyIndex, err) - } - binding.Status = model.StorageDataSetStatusUnavailable - return binding -} - -func (f *fakeUploadContext) ProviderID() sdktypes.BigInt { return f.providerID.Copy() } - -func (f *fakeUploadContext) DataSetRef() (storage.DataSetRef, bool) { - f.dataSetMu.RLock() - defer f.dataSetMu.RUnlock() - if f.boundDataSet == nil { - return storage.DataSetRef{}, false - } - ref, err := storage.NewDataSetRef(f.providerID, *f.boundDataSet, f.clientDataID) - return ref, err == nil -} - -func (f *fakeUploadContext) ClientDataSetID() sdktypes.BigInt { return f.clientDataID.Copy() } - -func (f *fakeUploadContext) GetProviderInfo() storage.Provider { - return storage.Provider{ - ID: f.providerID.Copy(), - ServiceURL: f.ServiceURL(), - } -} - -func (f *fakeUploadContext) CDNEnabled() bool { return false } - -func (f *fakeUploadContext) PieceURL(piece cid.Cid) string { - return fmt.Sprintf("https://provider-%s.example/piece/%s", f.providerID.String(), piece.String()) -} - -func (f *fakeUploadContext) ServiceURL() string { - if f.serviceURL != "" { - return f.serviceURL - } - return fmt.Sprintf("https://provider-%s.example", f.providerID.String()) -} - -func (f *fakeUploadContext) CreateDataSet(_ context.Context, opts *storage.CreateDataSetOptions) (*storage.CreateDataSetResult, error) { - if f.createCalls != nil { - f.createCalls.Add(1) - } - submission := storage.CreateDataSetSubmission{ - ProviderID: f.providerID.Copy(), - TransactionID: fmt.Sprintf("0xcreate%s", f.dataSetID.String()), - StatusURL: fmt.Sprintf("https://provider-%s.example/status/create", f.providerID.String()), - ClientDataSetID: sdkBigIntTestPtr(f.clientDataID), - } - if !f.skipCreateSubmission && opts != nil && opts.OnSubmitted != nil { - opts.OnSubmitted(submission) - } - if f.createErr != nil { - return nil, f.createErr - } - ref, err := storage.NewDataSetRef(f.providerID, f.dataSetID, f.clientDataID) - if err != nil { - return nil, err - } - return &storage.CreateDataSetResult{ - TransactionID: submission.TransactionID, - DataSet: ref, - }, nil -} - -func (f *fakeUploadContext) WaitForDataSetCreated(_ context.Context, submission storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) { - if f.waitCalls != nil { - f.waitCalls.Add(1) - } - if f.waitErr != nil { - return nil, f.waitErr - } - if !submission.ProviderID.Equal(f.providerID) { - return nil, fmt.Errorf("submission provider %s does not match target provider %s", submission.ProviderID.String(), f.providerID.String()) - } - if submission.ClientDataSetID == nil { - return nil, errors.New("submission is missing client data set ID") - } - ref, err := storage.NewDataSetRef(f.providerID, f.dataSetID, *submission.ClientDataSetID) - if err != nil { - return nil, err - } - return &storage.CreateDataSetResult{ - TransactionID: submission.TransactionID, - DataSet: ref, - }, nil -} - -func (f *fakeUploadContext) Store(_ context.Context, r io.Reader, opts *storage.StoreOptions) (*storage.StoreResult, error) { - f.storeCalls.Add(1) - if _, err := io.ReadAll(r); err != nil { - return nil, err - } - if opts != nil && opts.OnProgress != nil { - for _, n := range f.storeProgress { - opts.OnProgress(n) - } - } - if f.storeErr != nil { - return nil, f.storeErr - } - return &storage.StoreResult{PieceCID: f.pieceCID, Size: 11}, nil -} - -func (f *fakeUploadContext) PresignForCommit(_ context.Context, _ []storage.PieceInput) ([]byte, error) { - f.presignCalls.Add(1) - return []byte(fmt.Sprintf("extra-%s", f.providerID.String())), nil -} - -func (f *fakeUploadContext) Pull(ctx context.Context, _ storage.PullRequest) (*storage.PullResult, error) { - f.pullCalls.Add(1) - if f.pullEntered != nil { - f.pullOnce.Do(func() { close(f.pullEntered) }) - } - if f.releasePull != nil { - select { - case <-f.releasePull: - case <-ctx.Done(): - return nil, ctx.Err() - } - } - if f.pullErr != nil { - return nil, f.pullErr - } - return &storage.PullResult{Status: storage.PullStatusComplete}, nil -} - -func (f *fakeUploadContext) SubmitCommit(_ context.Context, req storage.CommitRequest) (*storage.CommitSubmission, error) { - if f.submitPreflightErr != nil { - return nil, f.submitPreflightErr - } - f.commitCalls.Add(1) - f.commitMu.Lock() - f.commitExtras = append(f.commitExtras, append([]byte(nil), req.ExtraData...)) - f.commitMu.Unlock() - if req.OnSubmitted != nil { - req.OnSubmitted(fakeSubmittedCommitTxHash) - } - if f.commitErr != nil { - return nil, f.commitErr - } - ref, err := storage.NewDataSetRef(f.providerID, f.dataSetID, f.clientDataID) - if err != nil { - return nil, err - } - pieceCIDs := make([]cid.Cid, 0, len(req.Pieces)) - for _, piece := range req.Pieces { - pieceCIDs = append(pieceCIDs, piece.PieceCID) - } - return &storage.CommitSubmission{ - Kind: storage.CommitKindAddPieces, TransactionID: fakeSubmittedCommitTxHash, - StatusURL: f.ServiceURL() + "/status/commit", ProviderID: f.providerID.Copy(), - DataSet: &ref, PieceCIDs: pieceCIDs, - }, nil -} - -func (f *fakeUploadContext) GetCommitStatus(_ context.Context, submission storage.CommitSubmission) (*storage.CommitStatus, error) { - if f.commitErr != nil { - return nil, f.commitErr - } - ref, err := storage.NewDataSetRef(f.providerID, f.dataSetID, f.clientDataID) - if err != nil { - return nil, err - } - return &storage.CommitStatus{ - Kind: storage.CommitKindAddPieces, State: storage.CommitStateConfirmed, - TransactionID: submission.TransactionID, DataSet: &ref, - PieceIDs: []sdktypes.BigInt{f.pieceID.Copy()}, - }, nil -} - -func (f *fakeUploadContext) PieceStatus(context.Context, cid.Cid) (*storage.PieceStatus, error) { - return &storage.PieceStatus{}, nil -} - -func sdkBigIntTestPtr(id sdktypes.BigInt) *sdktypes.BigInt { - cp := id.Copy() - return &cp -} - -func newFakeUploadContexts(t *testing.T, copies int, base uint64) []synapse.StorageTarget { - t.Helper() - contexts := make([]synapse.StorageTarget, 0, copies) - for i := 0; i < copies; i++ { - offset := base + uint64(i) - contexts = append(contexts, newFakeUploadContext( - sdktypes.NewBigInt(100+offset), - sdktypes.NewBigInt(1000+offset), - sdktypes.NewBigInt(2000+offset), - testCID(t), - )) - } - return contexts -} - -func seedReadyPrimaryStoreTask(t *testing.T, env *testWorkerEnv) (*model.StorageUpload, *model.Task, *fakeUploadContext) { - t.Helper() - bucket, objID, versionID := seedCachedObject(t, env) - ctx := context.Background() - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark uploading: %v", err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: upload.ID, - DataSetID: onChainID(t, "1001"), - ClientDataSetID: onChainIDPtr(t, "11001"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - stage := "ingress_store" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objID, - RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:ingress_store:%d", versionID, upload.ID), - Payload: map[string]interface{}{"upload_id": upload.ID, "copy_index": 0, "transfer_method": string(model.StorageCopyTransferMethodIngress)}, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("create primary store task: %v", err) - } - primaryCtx := newFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(2001), testCID(t)) - dataSetID := sdktypes.NewBigInt(1001) - primaryCtx.boundDataSet = &dataSetID - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextDataSetIDEqual(opts, sdktypes.NewBigInt(1001)) { - return primaryCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - return upload, task, primaryCtx -} - -func TestUploader_WaitsPollIntervalBeforeInitialClaim(t *testing.T) { - env := newTestWorkerEnv(t) - _, objID, versionID := seedCachedObject(t, env) - task := seedStagedUploadTask(t, env, objID, versionID, 5) - - uploadStarted := make(chan struct{}) - var closeStarted sync.Once - pollInterval := 100 * time.Millisecond - - env.storage.SelectUploadTargetsFunc = func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - closeStarted.Do(func() { close(uploadStarted) }) - contexts := make([]synapse.StorageTarget, 0, opts.Copies) - for i := 0; i < opts.Copies; i++ { - contexts = append(contexts, newFakeUploadContext( - sdktypes.NewBigInt(uint64(100+i)), - sdktypes.NewBigInt(uint64(1000+i)), - sdktypes.NewBigInt(uint64(2000+i)), - testCID(t), - )) - } - return contexts, nil - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, pollInterval, slog.Default()) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = uploader.Run(ctx) - close(done) - }() - defer func() { - cancel() - waitForSignal(t, done, time.Second, "uploader shutdown") - }() - - select { - case <-uploadStarted: - t.Fatal("uploader claimed upload task before initial poll interval elapsed") - case <-time.After(pollInterval / 2): - } - - waitForTaskStatus(t, env, task.ID, model.TaskStatusCompleted, time.Second) -} - -func TestUploader_CompletesRetryWhenObjectIsAlreadyStored(t *testing.T) { - env := newTestWorkerEnv(t) - fixture := seedReadableUploadWithPendingPeer(t, env) - stage := "peer_commit" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: fixture.objID, - RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:peer_commit:%d:1", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{"upload_id": fixture.upload.ID, "copy_index": 1, "transfer_method": string(model.StorageCopyTransferMethodPeerPull)}, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(context.Background(), task); err != nil { - t.Fatalf("create peer commit task: %v", err) - } - - claimed, err := env.repos.Tasks.ClaimReady(context.Background(), model.TaskTypeUpload, time.Minute) - if err != nil || claimed == nil { - t.Fatalf("ClaimReady: task=%v err=%v", claimed, err) - } - if err := env.repos.Uploads.MarkUploadCopyCommitted(context.Background(), repository.MarkUploadCopyCommittedInput{ - UploadID: fixture.upload.ID, - CopyIndex: 1, - PieceCID: testCID(t).String(), - PieceID: onChainIDPtr(t, "302"), - RetrievalURL: "https://peer.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - finalized, _, err := env.repos.Uploads.FinalizeUploadIfTargetCopiesMet(context.Background(), repository.FinalizeUploadInput{UploadID: fixture.upload.ID}) - if err != nil { - t.Fatalf("FinalizeUploadIfTargetCopiesMet: %v", err) - } - if !finalized { - t.Fatal("FinalizeUploadIfTargetCopiesMet did not finalize the upload") - } - status, err := env.repos.Tasks.ScheduleRetryRunning(context.Background(), claimed, "late worker failure", 0) - if err != nil { - t.Fatalf("ScheduleRetryRunning: %v", err) - } - if status != model.TaskStatusScheduled { - t.Fatalf("retry status = %s, want scheduled", status) - } - - var storageCalls atomic.Int32 - env.storage.SelectUploadTargetsFunc = func(context.Context, storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - storageCalls.Add(1) - return nil, errors.New("storage must not be called for a stored object") - } - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 10*time.Millisecond, slog.Default()) - got := runWorkerUntilTask(t, env, uploader, task.ID, time.Second) - - if got.Status != model.TaskStatusCompleted { - t.Fatalf("task status = %s, want completed", got.Status) - } - if got.RetryCount != 1 { - t.Fatalf("retry count = %d, want 1", got.RetryCount) - } - if calls := storageCalls.Load(); calls != 0 { - t.Fatalf("storage calls = %d, want 0", calls) - } -} - -func TestUploader_DurableIncompleteUploadKeepsRepairingOriginalReplica(t *testing.T) { - for _, tc := range []struct { - name string - cacheState model.ObjectState - inCache bool - }{ - {name: "stored with retained cache", cacheState: model.ObjectStateStored, inCache: true}, - {name: "cache evicted", cacheState: model.ObjectStateCacheEvicted}, - } { - t.Run(tc.name, func(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - bucket, err := env.repos.Buckets.GetByID(ctx, fixture.upload.BucketID) - if err != nil || bucket == nil { - t.Fatalf("GetByID bucket: bucket=%#v err=%v", bucket, err) - } - minimum := 1 - if _, err := env.repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimum, - }); err != nil { - t.Fatalf("UpdateCopyPolicy: %v", err) - } - if complete, _, err := env.repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: fixture.upload.ID}); err != nil || complete { - t.Fatalf("FinalizeUploadIfTargetCopiesMet = complete:%t err:%v, want durable before target", complete, err) - } - if tc.cacheState == model.ObjectStateCacheEvicted { - if err := env.repos.Objects.UpdateVersionState(ctx, fixture.versionID, model.ObjectStateStored, model.ObjectStateCacheEvicted); err != nil { - t.Fatalf("mark cache evicted: %v", err) - } - if err := env.repos.Objects.SetVersionCachePresence(ctx, fixture.versionID, false); err != nil { - t.Fatalf("clear cache presence: %v", err) - } - } - if err := env.repos.Uploads.MarkDataSetUnavailable(ctx, fixture.peer.ID, "temporary outage"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - - var selectTargetsCalls atomic.Int32 - env.storage.SelectUploadTargetsFunc = func(context.Context, storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - return nil, errors.New("durable repair must not select a replacement provider") - } - stage := "prepare_upload" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: fixture.objID, - RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:prepare_upload:%d:durable", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{"upload_id": fixture.upload.ID}, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create durable repair task: %v", err) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("durable repair task = %#v, want completed without retry", gotTask) - } - if selectTargetsCalls.Load() != 0 { - t.Fatalf("SelectUploadTargets calls = %d, want 0", selectTargetsCalls.Load()) - } - peerCopy, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, fixture.peer.CopyIndex) - if err != nil || peerCopy == nil { - t.Fatalf("GetUploadCopy peer: copy=%#v err=%v", peerCopy, err) - } - repairTasks, total, err := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "repair_replica", "", 10, 0) - if err != nil || total != 1 || len(repairTasks) != 1 || - taskPayloadInt64ForTest(repairTasks[0].Payload, "storage_data_set_id") != fixture.peer.ID || - taskPayloadInt64ForTest(repairTasks[0].Payload, "storage_upload_copy_id") != peerCopy.ID { - t.Fatalf("repair tasks = %#v total=%d err=%v, want original data set %d copy %d", repairTasks, total, err, fixture.peer.ID, peerCopy.ID) - } - version, err := env.repos.Objects.GetVersionByID(ctx, fixture.versionID) - if err != nil || version == nil || version.State != tc.cacheState || version.InCache != tc.inCache { - t.Fatalf("durable version after repair handoff = %#v err=%v", version, err) - } - }) - } -} - -func TestUploader_DurableEnsureTaskDoesNotReassignIngress(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - bucket, err := env.repos.Buckets.GetByID(ctx, fixture.upload.BucketID) - if err != nil || bucket == nil { - t.Fatalf("GetByID bucket: bucket=%#v err=%v", bucket, err) - } - if _, err := env.db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("requested_copies = ?", 3). - Where("id = ?", fixture.upload.ID). - Exec(ctx); err != nil { - t.Fatalf("raise frozen target for fixture: %v", err) - } - if _, err := env.db.NewUpdate(). - Model((*model.StorageUploadCopy)(nil)). - Set("status = ?", model.StorageUploadCopyStatusPending). - Where("upload_id = ? AND copy_index = ?", fixture.upload.ID, fixture.ingress.CopyIndex). - Exec(ctx); err != nil { - t.Fatalf("make ingress copy pending: %v", err) - } - if err := env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: fixture.upload.ID, - CopyIndex: fixture.peer.CopyIndex, - PieceCID: testCID(t).String(), - PieceID: onChainIDPtr(t, "302"), - RetrievalURL: "https://peer.example/piece", - }); err != nil { - t.Fatalf("commit peer copy: %v", err) - } - spare, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "303"), - CopyIndex: 2, - CreatedByUploadID: fixture.upload.ID, - }) - if err != nil { - t.Fatalf("create spare binding: %v", err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: spare.ID, - UploadID: fixture.upload.ID, - DataSetID: onChainID(t, "3003"), - }); err != nil { - t.Fatalf("mark spare binding ready: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, fixture.upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: spare.ID, - CopyIndex: spare.CopyIndex, - TransferMethod: model.StorageCopyTransferMethodPeerPull, - ProviderID: spare.ProviderID, - }}); err != nil { - t.Fatalf("create spare copy: %v", err) - } - minimum := 1 - if _, err := env.repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimum, - }); err != nil { - t.Fatalf("set minimum: %v", err) - } - if complete, _, err := env.repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: fixture.upload.ID}); err != nil || complete { - t.Fatalf("FinalizeUploadIfTargetCopiesMet = complete:%t err:%v, want durable before target", complete, err) - } - if err := env.repos.Uploads.MarkDataSetUnavailable(ctx, fixture.ingress.ID, "temporary outage"); err != nil { - t.Fatalf("mark ingress unavailable: %v", err) - } - - stage := "ensure_dataset" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: fixture.objID, - RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:ensure_dataset:%d:0:durable", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{ - "upload_id": fixture.upload.ID, - "copy_index": fixture.ingress.CopyIndex, - "transfer_method": string(model.StorageCopyTransferMethodIngress), - }, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("create stale ensure task: %v", err) - } - env.storage.SelectUploadTargetsFunc = func(context.Context, storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - return nil, errors.New("durable ensure task must not choose a provider") - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 3, 1, 10*time.Millisecond, slog.Default()) - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("stale ensure task = %#v, want completed without retry", gotTask) - } - spareCopy, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, spare.CopyIndex) - if err != nil || spareCopy == nil { - t.Fatalf("GetUploadCopy spare: copy=%#v err=%v", spareCopy, err) - } - if spareCopy.TransferMethod != model.StorageCopyTransferMethodPeerPull { - t.Fatalf("spare transfer method = %s, want peer pull", spareCopy.TransferMethod) - } - ingressCopy, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, fixture.ingress.CopyIndex) - if err != nil || ingressCopy == nil { - t.Fatalf("GetUploadCopy ingress: copy=%#v err=%v", ingressCopy, err) - } - repairTasks, total, err := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "repair_replica", "", 10, 0) - if err != nil || total != 1 || len(repairTasks) != 1 || - taskPayloadInt64ForTest(repairTasks[0].Payload, "storage_data_set_id") != fixture.ingress.ID || - taskPayloadInt64ForTest(repairTasks[0].Payload, "storage_upload_copy_id") != ingressCopy.ID { - t.Fatalf("repair tasks = %#v total=%d err=%v, want original ingress data set %d copy %d", repairTasks, total, err, fixture.ingress.ID, ingressCopy.ID) - } -} - -func TestUploader_DurableReadyCopyContinuesStagedPipeline(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - bucket, err := env.repos.Buckets.GetByID(ctx, fixture.upload.BucketID) - if err != nil || bucket == nil { - t.Fatalf("GetByID bucket: bucket=%#v err=%v", bucket, err) - } - if _, err := env.db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("requested_copies = ?", 3). - Where("id = ?", fixture.upload.ID). - Exec(ctx); err != nil { - t.Fatalf("raise frozen target for fixture: %v", err) - } - if err := env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: fixture.upload.ID, - CopyIndex: fixture.peer.CopyIndex, - PieceCID: testCID(t).String(), - PieceID: onChainIDPtr(t, "302"), - RetrievalURL: "https://peer.example/piece", - }); err != nil { - t.Fatalf("commit peer copy: %v", err) - } - remaining, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "303"), - CopyIndex: 2, - CreatedByUploadID: fixture.upload.ID, - }) - if err != nil { - t.Fatalf("create remaining binding: %v", err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: remaining.ID, - UploadID: fixture.upload.ID, - DataSetID: onChainID(t, "3003"), - }); err != nil { - t.Fatalf("mark remaining binding ready: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, fixture.upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: remaining.ID, - CopyIndex: remaining.CopyIndex, - TransferMethod: model.StorageCopyTransferMethodPeerPull, - ProviderID: remaining.ProviderID, - }}); err != nil { - t.Fatalf("create remaining copy: %v", err) - } - minimum := 2 - if _, err := env.repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimum, - }); err != nil { - t.Fatalf("set minimum: %v", err) - } - if complete, _, err := env.repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: fixture.upload.ID}); err != nil || complete { - t.Fatalf("FinalizeUploadIfTargetCopiesMet = complete:%t err=%v, want stored before target", complete, err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, fixture.versionID) - if err != nil || version == nil || version.State != model.ObjectStateStored { - t.Fatalf("version after minimum = %#v err=%v, want stored", version, err) - } - - stage := "ensure_dataset" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: fixture.objID, - RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:ensure_dataset:%d:2:remaining", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{ - "upload_id": fixture.upload.ID, - "copy_index": remaining.CopyIndex, - "transfer_method": string(model.StorageCopyTransferMethodPeerPull), - }, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("create remaining ensure task: %v", err) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 3, 1, 10*time.Millisecond, slog.Default()) - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("remaining ensure task = %#v, want completed without retry", gotTask) - } - - peerPull, err := env.repos.Tasks.GetByIdempotencyKey(ctx, fmt.Sprintf("upload:%s:peer_pull:%d:2", fixture.versionID, fixture.upload.ID)) - if err != nil || peerPull == nil || peerPull.Status != model.TaskStatusQueued { - t.Fatalf("peer_pull task = %#v err=%v, want queued staged copy work", peerPull, err) - } - repairTasks, total, err := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "repair_replica", "", 10, 0) - if err != nil || total != 0 { - t.Fatalf("repair tasks = %#v total=%d err=%v, want none for a ready remaining copy", repairTasks, total, err) - } - upload, err := env.repos.Uploads.GetByID(ctx, fixture.upload.ID) - if err != nil || upload == nil || upload.Status != model.StorageUploadStatusReadable { - t.Fatalf("upload = %#v err=%v, want readable until the target is met", upload, err) - } -} - -func TestUploader_PeerCopyCommitSchedulesRemainingTargetCopies(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - bucket, err := env.repos.Buckets.GetByID(ctx, fixture.upload.BucketID) - if err != nil || bucket == nil { - t.Fatalf("GetByID bucket: bucket=%#v err=%v", bucket, err) - } - if _, err := env.db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("requested_copies = ?", 3). - Where("id = ?", fixture.upload.ID). - Exec(ctx); err != nil { - t.Fatalf("raise frozen target for fixture: %v", err) - } - if err := env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: fixture.upload.ID, - CopyIndex: fixture.peer.CopyIndex, - PieceCID: testCID(t).String(), - PieceID: onChainIDPtr(t, "302"), - RetrievalURL: "https://peer.example/piece", - }); err != nil { - t.Fatalf("commit peer copy: %v", err) - } - remaining, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "303"), - CopyIndex: 2, - CreatedByUploadID: fixture.upload.ID, - }) - if err != nil { - t.Fatalf("create remaining binding: %v", err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: remaining.ID, - UploadID: fixture.upload.ID, - DataSetID: onChainID(t, "3003"), - }); err != nil { - t.Fatalf("mark remaining binding ready: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, fixture.upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: remaining.ID, - CopyIndex: remaining.CopyIndex, - TransferMethod: model.StorageCopyTransferMethodPeerPull, - ProviderID: remaining.ProviderID, - }}); err != nil { - t.Fatalf("create remaining copy: %v", err) - } - minimum := 2 - if _, err := env.repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimum, - }); err != nil { - t.Fatalf("set minimum: %v", err) - } - - stage := "peer_commit" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: fixture.objID, - RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:peer_commit:%d:1:committed", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{ - "upload_id": fixture.upload.ID, - "copy_index": fixture.peer.CopyIndex, - "transfer_method": string(model.StorageCopyTransferMethodPeerPull), - }, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("create peer commit task: %v", err) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 3, 1, 10*time.Millisecond, slog.Default()) - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("peer commit task = %#v, want completed without retry", gotTask) - } - - remainingEnsure, err := env.repos.Tasks.GetByIdempotencyKey(ctx, fmt.Sprintf("upload:%s:ensure_dataset:%d:2", fixture.versionID, fixture.upload.ID)) - if err != nil || remainingEnsure == nil || remainingEnsure.Status != model.TaskStatusQueued { - t.Fatalf("remaining copy task = %#v err=%v, want queued ensure_dataset", remainingEnsure, err) - } - upload, err := env.repos.Uploads.GetByID(ctx, fixture.upload.ID) - if err != nil || upload == nil || upload.Status != model.StorageUploadStatusReadable { - t.Fatalf("upload = %#v err=%v, want readable until the target is met", upload, err) - } -} - -func TestUploader_ThreeReplicasWithMinimumTwoDurableCopiesCompletesAllCopies(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - bucket, objID, versionID := seedCachedObject(t, env) - minimum := 2 - three := 3 - if _, err := env.repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetDefaultCopies: true, - DefaultCopies: &three, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimum, - }); err != nil { - t.Fatalf("set minimum durable copies: %v", err) - } - - task := seedStagedUploadTask(t, env, objID, versionID, 5) - pieceCID := testCID(t) - - c1 := newFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(2001), pieceCID) - c2 := newFakeUploadContext(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(3001), pieceCID) - c3 := newFakeUploadContext(sdktypes.NewBigInt(303), sdktypes.NewBigInt(3003), sdktypes.NewBigInt(4001), pieceCID) - - contextsByProvider := map[string]*fakeUploadContext{ - c1.providerID.String(): c1, - c2.providerID.String(): c2, - c3.providerID.String(): c3, - } - contextsByDataSet := map[string]*fakeUploadContext{ - c1.dataSetID.String(): c1, - c2.dataSetID.String(): c2, - c3.dataSetID.String(): c3, - } - - env.storage.SelectUploadTargetsFunc = func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - if opts.Copies != 3 { - t.Fatalf("SelectUploadTargets copies = %d, want 3", opts.Copies) - } - return []synapse.StorageTarget{c1, c2, c3}, nil - } - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if opts.ProviderID != nil { - return contextsByProvider[opts.ProviderID.String()], nil - } - if opts.DataSetID != nil { - return contextsByDataSet[opts.DataSetID.String()], nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyNone, 3, 2, 10*time.Millisecond, slog.Default()) - runCtx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = uploader.Run(runCtx) - close(done) - }() - defer func() { - cancel() - waitForSignal(t, done, time.Second, "uploader shutdown") - }() - - // Wait until version becomes stored (at 2 copies) - waitForObjectState(t, env, versionID, model.ObjectStateStored, 5*time.Second) - - // Wait until upload status becomes complete (all 3 copies finished) - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - upload, err := env.repos.Uploads.FindActiveUploadBySourceVersion(ctx, versionID) - if err == nil && upload == nil { - latest, loadErr := env.repos.Uploads.FindLatestUploadBySourceVersion(ctx, versionID) - if loadErr == nil && latest != nil && latest.Status == model.StorageUploadStatusComplete { - break - } - } - time.Sleep(20 * time.Millisecond) - } - - upload, err := env.repos.Uploads.FindLatestUploadBySourceVersion(ctx, versionID) - if err != nil || upload == nil { - t.Fatalf("FindLatestUploadBySourceVersion: upload=%#v err=%v", upload, err) - } - if upload.Status != model.StorageUploadStatusComplete { - t.Fatalf("upload status = %s, want %s", upload.Status, model.StorageUploadStatusComplete) - } - - copies, err := env.repos.Uploads.ListCopies(ctx, upload.ID) - if err != nil { - t.Fatalf("ListCopies: %v", err) - } - if len(copies) != 3 { - t.Fatalf("len(copies) = %d, want 3", len(copies)) - } - for i, c := range copies { - if c.Status != model.StorageUploadCopyStatusCommitted { - t.Errorf("copy[%d] status = %s, want committed", i, c.Status) - } - } - _ = task -} - -func TestUploader_UsesReadySpareWhenLowerIndexSlotIsDead(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - bucket, objID, versionID := seedCachedObject(t, env) - minimum := 2 - three := 3 - if _, err := env.repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetDefaultCopies: true, - DefaultCopies: &three, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimum, - }); err != nil { - t.Fatalf("set copy policy: %v", err) - } - seedUpload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: "01J000000000000000DEADSLOT", - ContentSize: 1, - Checksum: "dead-slot", - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt seed: %v", err) - } - seedReadyBinding(t, env, bucket.ID, seedUpload.ID, 0, "101", "1001") - seedReadyBinding(t, env, bucket.ID, seedUpload.ID, 1, "202", "2002") - seedDeadUnestablishedBinding(t, env, bucket.ID, seedUpload.ID, 2, "303") - seedReadyBinding(t, env, bucket.ID, seedUpload.ID, 3, "404", "4004") - - task := seedStagedUploadTask(t, env, objID, versionID, 5) - pieceCID := testCID(t) - c0 := readyFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(2001), pieceCID) - c1 := readyFakeUploadContext(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(3001), pieceCID) - c3 := readyFakeUploadContext(sdktypes.NewBigInt(404), sdktypes.NewBigInt(4004), sdktypes.NewBigInt(5001), pieceCID) - contextsByProvider := map[string]*fakeUploadContext{ - c0.providerID.String(): c0, - c1.providerID.String(): c1, - c3.providerID.String(): c3, - } - var selectTargetsCalls atomic.Int32 - env.storage.SelectUploadTargetsFunc = func(context.Context, storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - return nil, errors.New("dead lower-index slot must not force a new provider") - } - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if opts.ProviderID != nil { - if ctx := contextsByProvider[opts.ProviderID.String()]; ctx != nil { - return ctx, nil - } - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyNone, 3, 2, 10*time.Millisecond, slog.Default()) - runCtx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = uploader.Run(runCtx) - close(done) - }() - defer func() { - cancel() - waitForSignal(t, done, time.Second, "uploader shutdown") - }() - - waitForObjectState(t, env, versionID, model.ObjectStateStored, 5*time.Second) - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - latest, loadErr := env.repos.Uploads.FindLatestUploadBySourceVersion(ctx, versionID) - if loadErr == nil && latest != nil && latest.Status == model.StorageUploadStatusComplete { - break - } - time.Sleep(20 * time.Millisecond) - } - - upload, err := env.repos.Uploads.FindLatestUploadBySourceVersion(ctx, versionID) - if err != nil || upload == nil { - t.Fatalf("FindLatestUploadBySourceVersion: upload=%#v err=%v", upload, err) - } - if upload.Status != model.StorageUploadStatusComplete { - t.Fatalf("upload status = %s, want %s", upload.Status, model.StorageUploadStatusComplete) - } - if got := selectTargetsCalls.Load(); got != 0 { - t.Fatalf("SelectUploadTargets calls = %d, want 0", got) - } - copies, err := env.repos.Uploads.ListCopies(ctx, upload.ID) - if err != nil { - t.Fatalf("ListCopies: %v", err) - } - if len(copies) != 3 { - t.Fatalf("len(copies) = %d, want 3 using the ready spare", len(copies)) - } - gotIndexes := make([]int, 0, len(copies)) - for _, copyRow := range copies { - gotIndexes = append(gotIndexes, copyRow.CopyIndex) - if copyRow.Status != model.StorageUploadCopyStatusCommitted { - t.Errorf("copy %d status = %s, want committed", copyRow.CopyIndex, copyRow.Status) - } - } - if gotIndexes[0] != 0 || gotIndexes[1] != 1 || gotIndexes[2] != 3 { - t.Fatalf("copy indexes = %v, want [0 1 3]", gotIndexes) - } - _ = task -} - -func TestUploader_RepairPrepareFillsTargetFromReadySpareWhenAssignedSlotIsDead(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - bucket, err := env.repos.Buckets.GetByID(ctx, fixture.upload.BucketID) - if err != nil || bucket == nil { - t.Fatalf("GetByID bucket: bucket=%#v err=%v", bucket, err) - } - if err := env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: fixture.upload.ID, - CopyIndex: fixture.peer.CopyIndex, - PieceCID: testCID(t).String(), - PieceID: onChainIDPtr(t, "302"), - RetrievalURL: "https://peer.example/piece", - }); err != nil { - t.Fatalf("commit peer copy: %v", err) - } - if _, err := env.db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("requested_copies = ?", 3). - Where("id = ?", fixture.upload.ID). - Exec(ctx); err != nil { - t.Fatalf("raise frozen target: %v", err) - } - minimum := 2 - if _, err := env.repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimum, - }); err != nil { - t.Fatalf("set minimum: %v", err) - } - if complete, _, err := env.repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: fixture.upload.ID}); err != nil || complete { - t.Fatalf("FinalizeUploadIfTargetCopiesMet = complete:%t err=%v, want stored before target", complete, err) - } - seedDeadUnestablishedBinding(t, env, bucket.ID, fixture.upload.ID, 2, "303") - spare := seedReadyBinding(t, env, bucket.ID, fixture.upload.ID, 3, "404", "4004") - - var selectTargetsCalls atomic.Int32 - env.storage.SelectUploadTargetsFunc = func(context.Context, storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - return nil, errors.New("ready spare must be used before selecting a new provider") - } - spareCtx := readyFakeUploadContext(sdktypes.NewBigInt(404), sdktypes.NewBigInt(4004), sdktypes.NewBigInt(5001), testCID(t)) - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextProviderIDEqual(opts, spareCtx.providerID) || createContextDataSetIDEqual(opts, spareCtx.dataSetID) { - return spareCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - stage := "prepare_upload" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: fixture.objID, - RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:prepare_upload:%d:repair", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{"upload_id": fixture.upload.ID}, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create repair task: %v", err) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyNone, 3, 1, 10*time.Millisecond, slog.Default()) - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("repair task = %#v, want completed without retry", gotTask) - } - if got := selectTargetsCalls.Load(); got != 0 { - t.Fatalf("SelectUploadTargets calls = %d, want 0", got) - } - copies, err := env.repos.Uploads.ListCopies(ctx, fixture.upload.ID) - if err != nil { - t.Fatalf("ListCopies: %v", err) - } - if len(copies) != 3 || copies[2].CopyIndex != 3 || copies[2].StorageDataSetID == nil || *copies[2].StorageDataSetID != spare.ID { - t.Fatalf("copies after repair = %#v, want spare copy index 3", copies) - } - if copies[2].Status != model.StorageUploadCopyStatusPending || copies[2].TransferMethod != model.StorageCopyTransferMethodPeerPull { - t.Fatalf("spare copy = %#v, want pending peer pull", copies[2]) - } - remaining, err := env.repos.Tasks.GetByIdempotencyKey(ctx, fmt.Sprintf("upload:%s:ensure_dataset:%d:3", fixture.versionID, fixture.upload.ID)) - if err != nil || remaining == nil || remaining.Status != model.TaskStatusQueued { - t.Fatalf("remaining copy task = %#v err=%v, want queued ensure_dataset for spare slot", remaining, err) - } -} - -func TestUploader_FinishPeerCopySchedulesRemainingPeerCopy(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - bucket, err := env.repos.Buckets.GetByID(ctx, fixture.upload.BucketID) - if err != nil || bucket == nil { - t.Fatalf("GetByID bucket: bucket=%#v err=%v", bucket, err) - } - // Increase requested copies to 3 - if _, err := env.db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("requested_copies = ?", 3). - Where("id = ?", fixture.upload.ID). - Exec(ctx); err != nil { - t.Fatalf("raise target copies: %v", err) - } - // Add 3rd copy (CopyIndex 2) - tertiary, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "303"), - CopyIndex: 2, - CreatedByUploadID: fixture.upload.ID, - }) - if err != nil { - t.Fatalf("create tertiary binding: %v", err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: tertiary.ID, - UploadID: fixture.upload.ID, - DataSetID: onChainID(t, "3003"), - }); err != nil { - t.Fatalf("mark tertiary binding ready: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, fixture.upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: tertiary.ID, - CopyIndex: tertiary.CopyIndex, - TransferMethod: model.StorageCopyTransferMethodPeerPull, - ProviderID: tertiary.ProviderID, - }}); err != nil { - t.Fatalf("create tertiary copy: %v", err) - } - - minimum := 2 - if _, err := env.repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ - Name: bucket.Name, - SetMinimumDurableCopies: true, - MinimumDurableCopies: &minimum, - }); err != nil { - t.Fatalf("set minimum: %v", err) - } - - // Prepare Copy 1 (the 2nd copy) with piece ready for peer_commit - pieceCID := testCID(t) - if err := env.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: fixture.upload.ID, - CopyIndex: fixture.peer.CopyIndex, - PieceCID: pieceCID.String(), - }); err != nil { - t.Fatalf("mark piece ready: %v", err) - } - // Make sure no task exists for Copy 2 - copy2Task, _ := env.repos.Tasks.GetByIdempotencyKey(ctx, fmt.Sprintf("upload:%s:ensure_dataset:%d:2", fixture.versionID, fixture.upload.ID)) - if copy2Task != nil { - t.Fatalf("expected no copy 2 task before peer commit runs") - } - - // Seed peer_commit task for Copy 1 - stage := "peer_commit" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: fixture.objID, - RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:peer_commit:%d:1", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{ - "upload_id": fixture.upload.ID, - "copy_index": fixture.peer.CopyIndex, - "transfer_method": string(model.StorageCopyTransferMethodPeerPull), - }, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("create peer_commit task: %v", err) - } - - peerDataSetID := sdktypes.NewBigInt(2002) - secondaryCtx := newFakeUploadContext(fixture.peer.ProviderID.SDK(), peerDataSetID, sdktypes.NewBigInt(3001), pieceCID) - secondaryCtx.boundDataSet = &peerDataSetID - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if opts.ProviderID != nil && opts.ProviderID.String() == fixture.peer.ProviderID.String() { - return secondaryCtx, nil - } - if opts.DataSetID != nil && opts.DataSetID.String() == "2002" { - return secondaryCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyNone, 3, 1, 10*time.Millisecond, slog.Default()) - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted { - t.Fatalf("peer_commit task = %#v, want completed", gotTask) - } - - // Version should be stored (met minimum of 2) - version, err := env.repos.Objects.GetVersionByID(ctx, fixture.versionID) - if err != nil || version == nil || version.State != model.ObjectStateStored { - t.Fatalf("version state = %v, want stored", version) - } - - // BUT finishPeerCopy MUST have scheduled Copy 2 ensure_dataset task! - copy2Ensure, err := env.repos.Tasks.GetByIdempotencyKey(ctx, fmt.Sprintf("upload:%s:ensure_dataset:%d:2", fixture.versionID, fixture.upload.ID)) - if err != nil || copy2Ensure == nil || copy2Ensure.Status != model.TaskStatusQueued { - t.Fatalf("copy 2 ensure task = %#v err=%v, want queued by finishPeerCopy", copy2Ensure, err) - } -} - -func TestUploader_ClaimsLaterPendingTaskWhileAnotherUploadRuns(t *testing.T) { - env := newTestWorkerEnv(t) - _, firstObjID, firstVersionID := seedCachedObject(t, env) - firstTask := seedStagedUploadTask(t, env, firstObjID, firstVersionID, 5) - - firstUploadEntered := make(chan struct{}) - releaseFirstUpload := make(chan struct{}) - var releaseOnce sync.Once - var uploadCalls atomic.Int32 - - env.storage.SelectUploadTargetsFunc = func(ctx context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - call := uploadCalls.Add(1) - if call == 1 { - close(firstUploadEntered) - select { - case <-releaseFirstUpload: - case <-ctx.Done(): - return nil, ctx.Err() - } - } - contexts := make([]synapse.StorageTarget, 0, opts.Copies) - for i := 0; i < opts.Copies; i++ { - offset := uint64(call*100 + int32(i)) - contexts = append(contexts, newFakeUploadContext( - sdktypes.NewBigInt(100+offset), - sdktypes.NewBigInt(1000+offset), - sdktypes.NewBigInt(2000+offset), - testCID(t), - )) - } - return contexts, nil - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 2, 20*time.Millisecond, slog.Default()) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = uploader.Run(ctx) - close(done) - }() - defer func() { - releaseOnce.Do(func() { close(releaseFirstUpload) }) - cancel() - waitForSignal(t, done, time.Second, "uploader shutdown") - }() - - waitForSignal(t, firstUploadEntered, time.Second, "first upload to start") - - _, secondObjID, secondVersionID := seedCachedObject(t, env) - secondTask := seedStagedUploadTask(t, env, secondObjID, secondVersionID, 5) - - waitForTaskStatus(t, env, secondTask.ID, model.TaskStatusCompleted, 500*time.Millisecond) - - got, err := env.repos.Tasks.GetByID(context.Background(), firstTask.ID) - if err != nil { - t.Fatalf("get first task: %v", err) - } - if got.Status != model.TaskStatusRunning { - t.Fatalf("first task status = %s, want running while second task completed", got.Status) - } -} - -func TestUploader_HealthyWhileUploadTaskIsActive(t *testing.T) { - env := newTestWorkerEnv(t) - _, objID, versionID := seedCachedObject(t, env) - _ = seedStagedUploadTask(t, env, objID, versionID, 5) - - uploadEntered := make(chan struct{}) - releaseUpload := make(chan struct{}) - var releaseOnce sync.Once - pollInterval := 20 * time.Millisecond - - env.storage.SelectUploadTargetsFunc = func(ctx context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - close(uploadEntered) - select { - case <-releaseUpload: - case <-ctx.Done(): - return nil, ctx.Err() - } - contexts := make([]synapse.StorageTarget, 0, opts.Copies) - for i := 0; i < opts.Copies; i++ { - contexts = append(contexts, newFakeUploadContext( - sdktypes.NewBigInt(uint64(100+i)), - sdktypes.NewBigInt(uint64(1000+i)), - sdktypes.NewBigInt(uint64(2000+i)), - testCID(t), - )) - } - return contexts, nil - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, pollInterval, slog.Default()) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = uploader.Run(ctx) - close(done) - }() - defer func() { - releaseOnce.Do(func() { close(releaseUpload) }) - cancel() - waitForSignal(t, done, time.Second, "uploader shutdown") - }() - - waitForSignal(t, uploadEntered, time.Second, "upload to start") - time.Sleep(4 * pollInterval) - - if !uploader.Healthy() { - t.Fatal("uploader should remain healthy while upload task is active") - } -} - -func TestUploader_StagedPrimaryStoreRecordsTransferProgress(t *testing.T) { - env := newTestWorkerEnv(t) - upload, task, primaryCtx := seedReadyPrimaryStoreTask(t, env) - primaryCtx.storeProgress = []int64{5, 11} - publisher := &fakeAdminEventPublisher{} - before, err := env.repos.Objects.GetVersionByID(context.Background(), task.RefVersionID) - if err != nil || before == nil || before.CacheAccessedAt == nil { - t.Fatalf("GetVersionByID before upload: version=%v err=%v", before, err) - } - objects := &cacheAccessCountingObjectRepo{ObjectRepository: env.repos.Objects} - env.repos.Objects = objects - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 50*time.Millisecond, slog.Default(), worker.WithEventPublisher(publisher)) - runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - - got, err := env.repos.Uploads.GetByID(context.Background(), upload.ID) - if err != nil || got == nil { - t.Fatalf("GetByID(upload): upload=%v err=%v", got, err) - } - if got.IngressStoreAttempt != 1 || got.IngressBytesTransferred != 11 || got.ProgressUpdatedAt == nil { - t.Fatalf("staged progress = bytes:%d attempt:%d updated:%v, want completed primary transfer", got.IngressBytesTransferred, got.IngressStoreAttempt, got.ProgressUpdatedAt) - } - if !publisher.hasTopic("upload_progress_updated") { - t.Fatal("expected upload_progress_updated event") - } - if objects.writes != 0 { - t.Fatalf("background upload cache access writes = %d, want 0", objects.writes) - } - after, err := env.repos.Objects.GetVersionByID(context.Background(), task.RefVersionID) - if err != nil || after == nil || after.CacheAccessedAt == nil { - t.Fatalf("GetVersionByID after upload: version=%v err=%v", after, err) - } - if !after.CacheAccessedAt.Equal(*before.CacheAccessedAt) { - t.Fatalf("cache_accessed_at changed from %v to %v during background upload", before.CacheAccessedAt, after.CacheAccessedAt) - } -} - -type cacheAccessCountingObjectRepo struct { - repository.ObjectRepository - writes int -} - -func (r *cacheAccessCountingObjectRepo) RecordVersionCacheAccess(ctx context.Context, versionID string, accessedAt time.Time) error { - r.writes++ - return r.ObjectRepository.RecordVersionCacheAccess(ctx, versionID, accessedAt) -} - -func TestUploader_StagedPrimaryCommitKeepsCacheUntilAllCopiesCommitted(t *testing.T) { - env := newTestWorkerEnv(t) - bucket, objID, versionID := seedCachedObject(t, env) - task := seedStagedUploadTask(t, env, objID, versionID, 5) - - pieceCID := testCID(t) - peerPullEntered := make(chan struct{}) - releaseSecondaryPull := make(chan struct{}) - var releaseSecondaryPullOnce sync.Once - - primary := newFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(2001), pieceCID) - secondary := newFakeUploadContext(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(3001), pieceCID) - secondary.pullEntered = peerPullEntered - secondary.releasePull = releaseSecondaryPull - contextsByProvider := map[string]*fakeUploadContext{ - primary.providerID.String(): primary, - secondary.providerID.String(): secondary, - } - contextsByDataSet := map[string]*fakeUploadContext{ - primary.dataSetID.String(): primary, - secondary.dataSetID.String(): secondary, - } - - env.storage.SelectUploadTargetsFunc = func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - if opts.Copies != 2 { - t.Fatalf("SelectUploadTargets copies = %d, want 2", opts.Copies) - } - return []synapse.StorageTarget{primary, secondary}, nil - } - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if opts.ProviderID != nil { - return contextsByProvider[opts.ProviderID.String()], nil - } - if opts.DataSetID != nil { - return contextsByDataSet[opts.DataSetID.String()], nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - runCtx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = uploader.Run(runCtx) - close(done) - }() - defer func() { - cancel() - releaseSecondaryPullOnce.Do(func() { close(releaseSecondaryPull) }) - waitForSignal(t, done, time.Second, "uploader shutdown") - }() - - waitForObjectState(t, env, versionID, model.ObjectStateReplicating, 3*time.Second) - waitForSignal(t, peerPullEntered, time.Second, "secondary pull to start") - - obj, err := env.repos.Objects.GetVersionByID(context.Background(), versionID) - if err != nil || obj == nil { - t.Fatalf("GetVersionByID: obj=%v err=%v", obj, err) - } - if obj.StorageUploadID == nil || !obj.InFilecoin { - t.Fatalf("replicating object storage = upload:%v in_filecoin:%v, want readable upload", obj.StorageUploadID, obj.InFilecoin) - } - if !obj.InCache { - t.Fatal("replicating object in_cache = false, want cache retained before all copies are committed") - } - upload, err := env.repos.Uploads.GetByID(context.Background(), *obj.StorageUploadID) - if err != nil || upload == nil { - t.Fatalf("GetByID(upload): upload=%v err=%v", upload, err) - } - if upload.Status != model.StorageUploadStatusReadable { - t.Fatalf("upload status = %s, want readable while peer copy is blocked", upload.Status) - } - evict, err := env.repos.Tasks.ClaimReady(context.Background(), model.TaskTypeEvictCache, time.Minute) - if err != nil { - t.Fatalf("ClaimReady(evict): %v", err) - } - if evict != nil { - t.Fatalf("unexpected evict task before all copies are committed: %#v", evict) - } - - releaseSecondaryPullOnce.Do(func() { close(releaseSecondaryPull) }) - waitForObjectState(t, env, versionID, model.ObjectStateStored, 3*time.Second) - - evict, err = env.repos.Tasks.ClaimReady(context.Background(), model.TaskTypeEvictCache, time.Minute) - if err != nil { - t.Fatalf("ClaimReady(evict after stored): %v", err) - } - if evict == nil { - t.Fatal("expected evict task after all copies are committed") - } - if evict.RefID != objID || evict.RefVersionID != versionID { - t.Fatalf("evict task refs after stored = (%d,%s), want (%d,%s)", evict.RefID, evict.RefVersionID, objID, versionID) - } - if task.ID == 0 || bucket.ID == 0 { - t.Fatal("seeded task and bucket should be persisted") - } -} - -func TestUploader_EmptyPayloadStartsStagedPrepare(t *testing.T) { - env := newTestWorkerEnv(t) - _, objID, versionID := seedCachedObject(t, env) - task := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "object", - RefID: objID, - RefVersionID: versionID, - IdempotencyKey: "upload:" + versionID, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if _, err := env.db.NewInsert().Model(task).Exec(context.Background()); err != nil { - t.Fatalf("creating upload task without payload: %v", err) - } - - var legacyUploadCalled atomic.Bool - env.storage.UploadFunc = func(_ context.Context, _ io.Reader, _ *storage.UploadOptions) (*storage.UploadResult, error) { - legacyUploadCalled.Store(true) - return nil, errors.New("legacy upload should not be called") - } - var selectTargetsCalls atomic.Int32 - env.storage.SelectUploadTargetsFunc = func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - if opts.Copies != 3 { - t.Fatalf("SelectUploadTargets copies = %d, want 3", opts.Copies) - } - return []synapse.StorageTarget{ - newFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(2001), testCID(t)), - newFakeUploadContext(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(3001), testCID(t)), - newFakeUploadContext(sdktypes.NewBigInt(303), sdktypes.NewBigInt(3003), sdktypes.NewBigInt(4001), testCID(t)), - }, nil - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 100*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - - if legacyUploadCalled.Load() { - t.Fatal("empty upload payload used legacy upload path") - } - got, err := env.repos.Objects.GetVersionByID(context.Background(), versionID) - if err != nil || got == nil { - t.Fatalf("GetVersionByID: got=%v err=%v", got, err) - } - if got.State != model.ObjectStateUploading || got.StorageUploadID != nil || got.InFilecoin { - t.Fatalf("version after prepare = state:%s upload:%v in_filecoin:%v, want uploading without FOC binding", got.State, got.StorageUploadID, got.InFilecoin) - } - var uploads []model.StorageUpload - if err := env.db.NewSelect().Model(&uploads).Where("source_version_id = ?", versionID).Scan(context.Background()); err != nil { - t.Fatalf("list uploads: %v", err) - } - if len(uploads) != 1 || uploads[0].Status != model.StorageUploadStatusRunning || uploads[0].RequestedCopies != 3 { - t.Fatalf("uploads after prepare = %#v, want one running upload with three requested copies", uploads) - } - tasks, _, err := env.repos.Tasks.List(context.Background(), string(model.TaskTypeUpload), "", "", 10, 0) - if err != nil { - t.Fatalf("list upload tasks: %v", err) - } - foundEnsurePrimary := false - for _, task := range tasks { - if task.RefVersionID == versionID && strings.Contains(task.IdempotencyKey, "ensure_dataset") && strings.HasSuffix(task.IdempotencyKey, ":0") { - if task.Stage == nil || *task.Stage != "ensure_dataset" { - t.Fatalf("ensure dataset task stage = %#v, want ensure_dataset", task.Stage) - } - if _, ok := task.Payload["stage"]; ok { - t.Fatalf("ensure dataset task payload kept stage: %#v", task.Payload) - } - foundEnsurePrimary = true - } - } - if !foundEnsurePrimary { - t.Fatalf("upload tasks = %#v, want primary ensure_dataset task", tasks) - } -} - -func TestUploader_StagedPrepareUsesConfiguredCopyCount(t *testing.T) { - env := newTestWorkerEnv(t) - _, objID, versionID := seedCachedObject(t, env) - task := seedStagedUploadTask(t, env, objID, versionID, 5) - - env.storage.SelectUploadTargetsFunc = func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - if opts.Copies != 4 { - t.Fatalf("SelectUploadTargets copies = %d, want 4", opts.Copies) - } - return []synapse.StorageTarget{ - newFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(2001), testCID(t)), - newFakeUploadContext(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(3001), testCID(t)), - newFakeUploadContext(sdktypes.NewBigInt(303), sdktypes.NewBigInt(3003), sdktypes.NewBigInt(4001), testCID(t)), - newFakeUploadContext(sdktypes.NewBigInt(404), sdktypes.NewBigInt(4004), sdktypes.NewBigInt(5001), testCID(t)), - }, nil - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 4, 1, 100*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - - ctx := context.Background() - var uploads []model.StorageUpload - if err := env.db.NewSelect().Model(&uploads).Where("source_version_id = ?", versionID).Scan(ctx); err != nil { - t.Fatalf("list uploads: %v", err) - } - if len(uploads) != 1 || uploads[0].RequestedCopies != 4 { - t.Fatalf("uploads after prepare = %#v, want one upload with four requested copies", uploads) - } - copies, err := env.repos.Uploads.ListCopies(ctx, uploads[0].ID) - if err != nil { - t.Fatalf("ListCopies: %v", err) - } - if len(copies) != 4 { - t.Fatalf("copy rows = %d, want 4", len(copies)) - } -} - -func TestUploader_StagedPrepareUsesBucketCopyOverride(t *testing.T) { - env := newTestWorkerEnv(t) - bucket, objID, versionID := seedCachedObject(t, env) - task := seedStagedUploadTask(t, env, objID, versionID, 5) - ctx := context.Background() - bucketCopies := 4 - if err := env.repos.Buckets.SetDefaultCopies(ctx, bucket.Name, &bucketCopies); err != nil { - t.Fatalf("SetDefaultCopies: %v", err) - } - - env.storage.SelectUploadTargetsFunc = func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - if opts.Copies != 4 { - t.Fatalf("SelectUploadTargets copies = %d, want 4", opts.Copies) - } - contexts := make([]synapse.StorageTarget, 0, opts.Copies) - for i := 0; i < opts.Copies; i++ { - contexts = append(contexts, newFakeUploadContext( - sdktypes.NewBigInt(uint64(100+i)), - sdktypes.NewBigInt(uint64(1000+i)), - sdktypes.NewBigInt(uint64(2000+i)), - testCID(t), - )) - } - return contexts, nil - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 100*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - - var uploads []model.StorageUpload - if err := env.db.NewSelect().Model(&uploads).Where("source_version_id = ?", versionID).Scan(ctx); err != nil { - t.Fatalf("list uploads: %v", err) - } - if len(uploads) != 1 || uploads[0].RequestedCopies != 4 { - t.Fatalf("uploads after prepare = %#v, want one upload with four requested copies", uploads) - } -} - -func TestUploader_StagedPrepareReusesExistingUploadRequestedCopies(t *testing.T) { - env := newTestWorkerEnv(t) - bucket, objID, versionID := seedCachedObject(t, env) - ctx := context.Background() - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark uploading: %v", err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - contextsByDataSet := make(map[string]*fakeUploadContext, 4) - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 4, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - for copyIndex := 0; copyIndex < 4; copyIndex++ { - providerID := onChainID(t, fmt.Sprintf("%d", 100+copyIndex)) - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: providerID, - CopyIndex: copyIndex, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding(%d): %v", copyIndex, err) - } - dataSetID := onChainID(t, fmt.Sprintf("%d", 1000+copyIndex)) - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: upload.ID, - DataSetID: dataSetID, - }); err != nil { - t.Fatalf("MarkDataSetReady(%d): %v", copyIndex, err) - } - storageCtx := newFakeUploadContext( - providerID.SDK(), - dataSetID.SDK(), - sdktypes.NewBigInt(uint64(2000+copyIndex)), - testCID(t), - ) - boundDataSetID := dataSetID.SDK() - storageCtx.boundDataSet = &boundDataSetID - contextsByDataSet[dataSetID.String()] = storageCtx - transferMethod := model.StorageCopyTransferMethodPeerPull - if copyIndex == 0 { - transferMethod = model.StorageCopyTransferMethodIngress - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, - CopyIndex: copyIndex, - TransferMethod: transferMethod, - ProviderID: providerID, - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings(%d): %v", copyIndex, err) - } - } - - newBucketCopies := 7 - if err := env.repos.Buckets.SetDefaultCopies(ctx, bucket.Name, &newBucketCopies); err != nil { - t.Fatalf("SetDefaultCopies after upload: %v", err) - } - var selectTargetsCopies []int - env.storage.SelectUploadTargetsFunc = func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCopies = append(selectTargetsCopies, opts.Copies) - return newFakeUploadContexts(t, opts.Copies, 10000), nil - } - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if opts.DataSetID != nil { - if uploadCtx := contextsByDataSet[opts.DataSetID.String()]; uploadCtx != nil { - return uploadCtx, nil - } - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - stage := "prepare_upload" - retryTask := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objID, - RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:retry", versionID), - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, retryTask); err != nil { - t.Fatalf("create retry prepare task: %v", err) - } - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 100*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, uploader, retryTask.ID, 5*time.Second) - - if len(selectTargetsCopies) != 0 { - t.Fatalf("SelectUploadTargets copies after retry = %v, want none", selectTargetsCopies) - } - copies, err := env.repos.Uploads.ListCopies(ctx, upload.ID) - if err != nil { - t.Fatalf("ListCopies after retry: %v", err) - } - if len(copies) != 4 { - t.Fatalf("copy rows after retry = %d, want historical 4", len(copies)) - } -} - -func TestUploader_StagedPrepareCapsConfiguredCopyCount(t *testing.T) { - env := newTestWorkerEnv(t) - _, objID, versionID := seedCachedObject(t, env) - task := seedStagedUploadTask(t, env, objID, versionID, 5) - - env.storage.SelectUploadTargetsFunc = func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - if opts.Copies != 8 { - t.Fatalf("SelectUploadTargets copies = %d, want 8", opts.Copies) - } - contexts := make([]synapse.StorageTarget, 0, opts.Copies) - for i := 0; i < opts.Copies; i++ { - contexts = append(contexts, newFakeUploadContext( - sdktypes.NewBigInt(uint64(100+i)), - sdktypes.NewBigInt(uint64(1000+i)), - sdktypes.NewBigInt(uint64(2000+i)), - testCID(t), - )) - } - return contexts, nil - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 99, 1, 100*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - - ctx := context.Background() - var uploads []model.StorageUpload - if err := env.db.NewSelect().Model(&uploads).Where("source_version_id = ?", versionID).Scan(ctx); err != nil { - t.Fatalf("list uploads: %v", err) - } - if len(uploads) != 1 || uploads[0].RequestedCopies != 8 { - t.Fatalf("uploads after prepare = %#v, want one upload with eight requested copies", uploads) - } - copies, err := env.repos.Uploads.ListCopies(ctx, uploads[0].ID) - if err != nil { - t.Fatalf("ListCopies: %v", err) - } - if len(copies) != 8 { - t.Fatalf("copy rows = %d, want 8", len(copies)) - } -} - -func TestUploader_EnsureDatasetAdoptsMatchingDataSet(t *testing.T) { - env := newTestWorkerEnv(t) - bucket, objID, versionID := seedCachedObject(t, env) - ctx := context.Background() - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark uploading: %v", err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - task := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "object", - RefID: objID, - RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:ensure_dataset:%d:0", versionID, upload.ID), - Payload: map[string]interface{}{ - "stage": "ensure_dataset", - "upload_id": upload.ID, - "copy_index": 0, - }, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("create ensure task: %v", err) - } - - var createCalls atomic.Int32 - existingID := sdktypes.NewBigInt(13236) - providerCtx := newFakeUploadContext(sdktypes.NewBigInt(101), existingID, sdktypes.NewBigInt(2001), testCID(t)) - providerCtx.createCalls = &createCalls - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextProviderIDEqual(opts, sdktypes.NewBigInt(101)) { - return providerCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - env.storage.FindMatchingDataSetFunc = func(_ context.Context, providerID sdktypes.BigInt, metadata map[string]string, withCDN bool) (*storage.DataSetRef, error) { - if !providerID.Equal(sdktypes.NewBigInt(101)) || metadata["bucket"] != bucket.Name || withCDN { - return nil, fmt.Errorf("unexpected matching data set request: provider=%s metadata=%v withCDN=%t", providerID.String(), metadata, withCDN) - } - ref, err := storage.NewDataSetRef(providerID, existingID, providerCtx.clientDataID) - if err != nil { - return nil, err - } - return &ref, nil - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 50*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - - if got := createCalls.Load(); got != 0 { - t.Fatalf("CreateDataSet calls = %d, want 0 when a matching data set exists", got) - } - gotBinding, err := env.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) - if err != nil || gotBinding == nil { - t.Fatalf("GetDataSetBindingByCopyIndex: binding=%v err=%v", gotBinding, err) - } - if gotBinding.Status != model.StorageDataSetStatusReady || onChainIDPtrString(gotBinding.DataSetID) != "13236" { - t.Fatalf("binding after ensure = status:%s dataSet:%v, want ready/13236", gotBinding.Status, gotBinding.DataSetID) - } -} - -func TestUploader_EnsureDatasetContextTimeoutKeepsBindingPendingForRetry(t *testing.T) { - env := newTestWorkerEnv(t) - bucket, objID, versionID := seedCachedObject(t, env) - ctx := context.Background() - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark uploading: %v", err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - stage := "ensure_dataset" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objID, - RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:ensure_dataset:%d:0", versionID, upload.ID), - Payload: map[string]interface{}{"upload_id": upload.ID, "copy_index": 0}, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("create ensure task: %v", err) - } - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextProviderIDEqual(opts, sdktypes.NewBigInt(101)) { - return nil, context.DeadlineExceeded - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 50*time.Millisecond, slog.Default()) - runWorkerUntilTaskRetryCount(t, env, uploader, task.ID, 1, 5*time.Second) - - gotBinding, err := env.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) - if err != nil || gotBinding == nil { - t.Fatalf("GetDataSetBindingByCopyIndex: binding=%v err=%v", gotBinding, err) - } - if gotBinding.Status != model.StorageDataSetStatusPending { - t.Fatalf("binding status after context timeout = %s, want pending", gotBinding.Status) - } - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, upload.ID, 0) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%v err=%v", copyRow, err) - } - if copyRow.Status != model.StorageUploadCopyStatusPending { - t.Fatalf("copy status after context timeout = %s, want pending", copyRow.Status) - } -} - -func TestUploader_EnsureDatasetSubmittedProviderUnavailableWaitsWithoutRetry(t *testing.T) { - env := newTestWorkerEnv(t) - bucket, objID, versionID := seedCachedObject(t, env) - ctx := context.Background() - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark uploading: %v", err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - stage := "ensure_dataset" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objID, - RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:ensure_dataset:%d:0", versionID, upload.ID), - Payload: map[string]interface{}{"upload_id": upload.ID, "copy_index": 0}, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("create ensure task: %v", err) - } - - var createCalls atomic.Int32 - var waitCalls atomic.Int32 - providerCtx := newFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(2001), testCID(t)) - providerCtx.createCalls = &createCalls - providerCtx.waitCalls = &waitCalls - providerCtx.createErr = &synapse.ProviderUnavailableError{Cause: context.DeadlineExceeded} - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextProviderIDEqual(opts, sdktypes.NewBigInt(101)) { - return providerCtx, nil - } - if createContextDataSetIDEqual(opts, sdktypes.NewBigInt(1001)) { - return providerCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 50*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, env, uploader, task.ID, model.TaskStatusWaiting, 5*time.Second) - gotTask, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || gotTask.RetryCount != 0 || gotTask.WaitReason == nil || *gotTask.WaitReason != model.TaskWaitReasonDependency { - t.Fatalf("submitted create task = %#v err=%v, want dependency wait without retry", gotTask, err) - } - - gotBinding, err := env.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) - if err != nil || gotBinding == nil { - t.Fatalf("GetDataSetBindingByCopyIndex after create error: binding=%v err=%v", gotBinding, err) - } - if gotBinding.Status != model.StorageDataSetStatusCreating || gotBinding.CreateTransactionID == nil || gotBinding.CreateStatusURL == nil { - t.Fatalf("binding after submitted create error = %#v, want creating with submission", gotBinding) - } - - providerCtx.createErr = nil - if _, err := env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - t.Fatalf("reschedule task retry: %v", err) - } - runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - - if createCalls.Load() != 1 || waitCalls.Load() != 1 { - t.Fatalf("create calls=%d wait calls=%d, want create once then wait once", createCalls.Load(), waitCalls.Load()) - } - gotBinding, err = env.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) - if err != nil || gotBinding == nil { - t.Fatalf("GetDataSetBindingByCopyIndex after retry: binding=%v err=%v", gotBinding, err) - } - if gotBinding.Status != model.StorageDataSetStatusReady || onChainIDPtrString(gotBinding.DataSetID) != "1001" { - t.Fatalf("binding after retry = %#v, want ready dataset 1001", gotBinding) - } -} - -func TestUploader_EnsureDatasetCreatingProviderUnavailableWaitsWithoutRetry(t *testing.T) { - env := newTestWorkerEnv(t) - bucket, objID, versionID := seedCachedObject(t, env) - ctx := context.Background() - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark uploading: %v", err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := env.repos.Uploads.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ - ID: binding.ID, - UploadID: upload.ID, - TransactionID: "0xcreate3e9", - StatusURL: "https://provider.example/status/create", - ClientDataSetID: onChainIDPtr(t, "11001"), - }); err != nil { - t.Fatalf("MarkDataSetCreating: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - stage := "ensure_dataset" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objID, - RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:ensure_dataset:%d:0", versionID, upload.ID), - Payload: map[string]interface{}{"upload_id": upload.ID, "copy_index": 0}, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("create ensure task: %v", err) - } - - var createCalls atomic.Int32 - var waitCalls atomic.Int32 - providerCtx := newFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(2001), testCID(t)) - providerCtx.createCalls = &createCalls - providerCtx.waitCalls = &waitCalls - providerCtx.waitErr = &synapse.ProviderUnavailableError{Cause: context.DeadlineExceeded} - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextProviderIDEqual(opts, sdktypes.NewBigInt(101)) { - return providerCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 50*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, env, uploader, task.ID, model.TaskStatusWaiting, 5*time.Second) - gotTask, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || gotTask.RetryCount != 0 || gotTask.WaitReason == nil || *gotTask.WaitReason != model.TaskWaitReasonDependency { - t.Fatalf("creating data set task = %#v err=%v, want dependency wait without retry", gotTask, err) - } - - gotBinding, err := env.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) - if err != nil || gotBinding == nil { - t.Fatalf("GetDataSetBindingByCopyIndex: binding=%v err=%v", gotBinding, err) - } - if gotBinding.Status != model.StorageDataSetStatusCreating { - t.Fatalf("binding status after wait error = %s, want creating", gotBinding.Status) - } - if gotBinding.CreateTransactionID == nil || *gotBinding.CreateTransactionID != "0xcreate3e9" { - t.Fatalf("binding transaction after wait error = %v, want original transaction", gotBinding.CreateTransactionID) - } - if createCalls.Load() != 0 || waitCalls.Load() != 1 { - t.Fatalf("create calls=%d wait calls=%d, want only one wait", createCalls.Load(), waitCalls.Load()) - } -} - -func TestUploader_EnsureDatasetCreatingRejectedMarksBindingFailed(t *testing.T) { - env := newTestWorkerEnv(t) - bucket, objID, versionID := seedCachedObject(t, env) - ctx := context.Background() - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark uploading: %v", err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := env.repos.Uploads.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ - ID: binding.ID, - UploadID: upload.ID, - TransactionID: "0xcreate3e9", - StatusURL: "https://provider.example/status/create", - ClientDataSetID: onChainIDPtr(t, "11001"), - }); err != nil { - t.Fatalf("MarkDataSetCreating: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - stage := "ensure_dataset" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objID, - RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:ensure_dataset:%d:0", versionID, upload.ID), - Payload: map[string]interface{}{"upload_id": upload.ID, "copy_index": 0}, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("create ensure task: %v", err) - } - - var createCalls atomic.Int32 - var waitCalls atomic.Int32 - providerCtx := newFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(2001), testCID(t)) - providerCtx.createCalls = &createCalls - providerCtx.waitCalls = &waitCalls - providerCtx.waitErr = fmt.Errorf("wait rejected: %w", pdp.ErrTxRejected) - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextProviderIDEqual(opts, sdktypes.NewBigInt(101)) { - return providerCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 50*time.Millisecond, slog.Default()) - runWorkerUntilTaskRetryCount(t, env, uploader, task.ID, 1, 5*time.Second) - - gotBinding, err := env.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) - if err != nil || gotBinding == nil { - t.Fatalf("GetDataSetBindingByCopyIndex: binding=%v err=%v", gotBinding, err) - } - if gotBinding.Status != model.StorageDataSetStatusFailed { - t.Fatalf("binding status after rejected wait = %s, want failed", gotBinding.Status) - } - if createCalls.Load() != 0 || waitCalls.Load() != 1 { - t.Fatalf("create calls=%d wait calls=%d, want only one wait", createCalls.Load(), waitCalls.Load()) - } - - if _, err := env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - t.Fatalf("reschedule rejected create task: %v", err) - } - workerCtx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = uploader.Run(workerCtx) - close(done) - }() - waitForTaskStatus(t, env, task.ID, model.TaskStatusWaiting, 5*time.Second) - cancel() - waitForSignal(t, done, time.Second, "uploader shutdown") - - gotTask, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || gotTask == nil { - t.Fatalf("GetByID task after retry: task=%#v err=%v", gotTask, err) - } - if gotTask.RetryCount != 1 { - t.Fatalf("retry count after evidence-backed failure = %d, want 1", gotTask.RetryCount) - } - if createCalls.Load() != 0 || waitCalls.Load() != 1 { - t.Fatalf("create calls=%d wait calls=%d after retry, want no repeated service creation", createCalls.Load(), waitCalls.Load()) - } -} - -func TestUploader_EnsureDatasetCreationRejectionPreservesEstablishedEvidence(t *testing.T) { - env := newTestWorkerEnv(t) - bucket, objID, versionID := seedCachedObject(t, env) - ctx := context.Background() - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark uploading: %v", err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := env.repos.Uploads.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ - ID: binding.ID, - UploadID: upload.ID, - TransactionID: "0xcreate3e9", - StatusURL: "https://provider.example/status/create", - ClientDataSetID: onChainIDPtr(t, "11001"), - }); err != nil { - t.Fatalf("MarkDataSetCreating: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - stage := "ensure_dataset" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objID, - RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:ensure_dataset:%d:0", versionID, upload.ID), - Payload: map[string]interface{}{"upload_id": upload.ID, "copy_index": 0}, - Status: model.TaskStatusQueued, - MaxRetries: 1, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("create ensure task: %v", err) - } - - providerCtx := newFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(2001), testCID(t)) - providerCtx.waitErr = fmt.Errorf("wait rejected: %w", pdp.ErrTxRejected) - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextProviderIDEqual(opts, sdktypes.NewBigInt(101)) { - return providerCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 50*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - - gotTask, _ := env.repos.Tasks.GetByID(ctx, task.ID) - if gotTask.Status != model.TaskStatusExhausted { - t.Fatalf("task status = %s, want exhausted", gotTask.Status) - } - gotVersion, _ := env.repos.Objects.GetVersionByID(ctx, versionID) - if gotVersion.State != model.ObjectStateUploading || gotVersion.StorageUploadID != nil { - t.Fatalf("version after rejected dataset creation = state:%s upload:%v, want recoverable uploading state", gotVersion.State, gotVersion.StorageUploadID) - } - gotUpload, err := env.repos.Uploads.GetByID(ctx, upload.ID) - if err != nil || gotUpload == nil { - t.Fatalf("GetByID(upload): upload=%v err=%v", gotUpload, err) - } - if gotUpload.Status != model.StorageUploadStatusFailed { - t.Fatalf("upload status = %s, want failed creation attempt retained for diagnosis", gotUpload.Status) - } - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, upload.ID, 0) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%v err=%v", copyRow, err) - } - if copyRow.Status != model.StorageUploadCopyStatusFailed { - t.Fatalf("primary copy status = %s, want failed", copyRow.Status) - } - provenance, err := env.repos.Uploads.GetUploadProvenance(ctx, upload.ID) - if err != nil { - t.Fatalf("GetUploadProvenance: %v", err) - } - if provenance == nil || len(provenance.Failures) != 1 { - t.Fatalf("provenance failures = %#v, want one primary failure", provenance) - } - failure := provenance.Failures[0] - if failure.ProviderID == nil || failure.ProviderID.String() != "101" || failure.TransferMethod != string(model.StorageCopyTransferMethodIngress) || failure.Stage == nil || *failure.Stage != "wait dataset" || failure.ErrorMessage == nil || *failure.ErrorMessage != "wait rejected: pdp: transaction rejected" { - t.Fatalf("failure = %#v, want provider 101 ingress wait dataset failure", failure) - } - gotBinding, err := env.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) - if err != nil || gotBinding == nil { - t.Fatalf("GetDataSetBindingByCopyIndex: binding=%v err=%v", gotBinding, err) - } - if gotBinding.Status != model.StorageDataSetStatusFailed || gotBinding.CreateTransactionID == nil || gotBinding.ClientDataSetID == nil { - t.Fatalf("binding after rejected creation = %#v, want failed binding with service evidence retained", gotBinding) - } - _, prepareTotal, err := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "prepare_upload", "", 10, 0) - if err != nil { - t.Fatalf("List prepare tasks: %v", err) - } - if prepareTotal != 0 { - t.Fatalf("prepare tasks = %d, want no automatic provider replacement", prepareTotal) - } -} - -// ingressCommitFixture is a stored primary copy with its ingress_commit task -// queued and its provider context wired, which is where the commit-path tests -// start. -type ingressCommitFixture struct { - env *testWorkerEnv - bucket *model.Bucket - objectID int64 - versionID string - upload *model.StorageUpload - binding *model.StorageDataSet - copyRow *model.StorageUploadCopy - pieceCID cid.Cid - task *model.Task - primary *fakeUploadContext -} - -func seedReadyIngressCommitTask(t *testing.T, env *testWorkerEnv, requestedCopies, maxRetries int) ingressCommitFixture { - t.Helper() - bucket, objID, versionID := seedCachedObject(t, env) - ctx := context.Background() - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark uploading: %v", err) - } - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("mark committing: %v", err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: requestedCopies, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - primary, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: primary.ID, - UploadID: upload.ID, - DataSetID: onChainID(t, "1001"), - ClientDataSetID: onChainIDPtr(t, "9001"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: primary.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: onChainID(t, "101"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - pieceCID := testCID(t) - if err := env.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: pieceCID.String(), - RetrievalURL: fmt.Sprintf("https://provider-101.example/piece/%s", pieceCID.String()), - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, upload.ID, 0) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - stage := "ingress_commit" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objID, - RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:ingress_commit:%d", versionID, upload.ID), - Payload: map[string]interface{}{"upload_id": upload.ID, "copy_index": 0, "transfer_method": string(model.StorageCopyTransferMethodIngress)}, - Status: model.TaskStatusQueued, - MaxRetries: maxRetries, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("create primary commit task: %v", err) - } - primaryCtx := newFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(2001), pieceCID) - primaryDataSetID := sdktypes.NewBigInt(1001) - primaryCtx.boundDataSet = &primaryDataSetID - primaryCtx.clientDataID = sdktypes.NewBigInt(9001) - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextDataSetIDEqual(opts, primaryDataSetID) { - return primaryCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - return ingressCommitFixture{ - env: env, bucket: bucket, objectID: objID, versionID: versionID, upload: upload, - binding: primary, copyRow: copyRow, pieceCID: pieceCID, task: task, primary: primaryCtx, - } -} - -func TestUploader_PrimaryCommitPendingDoesNotConsumeRetryBudget(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadyIngressCommitTask(t, env, 3, 1) - upload, versionID, task := fixture.upload, fixture.versionID, fixture.task - primaryCtx := fixture.primary - primaryCtx.commitErr = errors.New("commit status poll timeout") - statusRequests := atomic.Int32{} - statusConfirmed := atomic.Bool{} - statusServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - statusRequests.Add(1) - if r.URL.Path != "/pdp/data-sets/1001/pieces/added/"+fakeSubmittedCommitTxHash { - t.Fatalf("commit status path = %q, want submitted transaction status path", r.URL.Path) - } - w.Header().Set("Content-Type", "application/json") - if statusConfirmed.Load() { - _, _ = fmt.Fprintf(w, `{"txHash":%q,"txStatus":"confirmed","dataSetId":1001,"pieceCount":1,"addMessageOk":true,"piecesAdded":true,"confirmedPieceIds":[2001]}`, fakeSubmittedCommitTxHash) - return - } - _, _ = fmt.Fprintf(w, `{"txHash":%q,"txStatus":"pending","dataSetId":1001,"pieceCount":1,"piecesAdded":false}`, fakeSubmittedCommitTxHash) - })) - defer statusServer.Close() - primaryCtx.serviceURL = statusServer.URL - - uploader := worker.NewUploader( - env.repos, env.cache, env.storage, nil, env.sm, - cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 50*time.Millisecond, slog.Default(), - worker.WithPDPStatusChecker(synapse.NewPDPStatusChecker(synapse.PDPStatusCheckerOptions{AllowPrivateNetworks: true})), - ) - runWorkerUntilTaskStatus(t, env, uploader, task.ID, model.TaskStatusWaiting, 5*time.Second) - gotTask, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || gotTask == nil || gotTask.Status != model.TaskStatusWaiting || gotTask.RetryCount != 0 { - t.Fatalf("submitted commit task = %#v err=%v, want waiting without retry consumption", gotTask, err) - } - - gotVersion, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || gotVersion == nil { - t.Fatalf("GetVersionByID after commit error: version=%v err=%v", gotVersion, err) - } - if gotVersion.State != model.ObjectStateCommitting || gotVersion.FailedAtState != nil || gotVersion.LastError != nil { - t.Fatalf("version after submitted commit error = state:%s failed_at:%v last_error:%v, want committing without failure", gotVersion.State, gotVersion.FailedAtState, gotVersion.LastError) - } - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, upload.ID, 0) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%v err=%v", copyRow, err) - } - if copyRow.Status != model.StorageUploadCopyStatusCommitting || copyRow.CommitTransactionID == nil || *copyRow.CommitTransactionID == "" { - t.Fatalf("primary copy after submitted commit error = %#v, want committing with tx", copyRow) - } - if copyRow.CommitExtraDataHex == nil || *copyRow.CommitExtraDataHex == "" { - t.Fatalf("primary copy commit extra data = %#v, want persisted payload", copyRow.CommitExtraDataHex) - } - - primaryCtx.commitErr = nil - statusConfirmed.Store(true) - if _, err := env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - t.Fatalf("reschedule pending commit task: %v", err) - } - runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - - primaryCtx.commitMu.Lock() - commitExtras := append([][]byte(nil), primaryCtx.commitExtras...) - primaryCtx.commitMu.Unlock() - if primaryCtx.presignCalls.Load() != 1 { - t.Fatalf("presign calls = %d, want one persisted extra data payload reused", primaryCtx.presignCalls.Load()) - } - if primaryCtx.commitCalls.Load() != 1 { - t.Fatalf("commit calls = %d, want retry to wait on submitted transaction without resubmitting", primaryCtx.commitCalls.Load()) - } - if statusRequests.Load() < 1 { - t.Fatalf("status requests = %d, want submitted transaction observation", statusRequests.Load()) - } - if len(commitExtras) != 1 || string(commitExtras[0]) != "extra-101" { - t.Fatalf("commit extras = %q, want only original commit payload", commitExtras) - } - copyRow, err = env.repos.Uploads.GetUploadCopy(ctx, upload.ID, 0) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy after retry: copy=%v err=%v", copyRow, err) - } - if copyRow.Status != model.StorageUploadCopyStatusCommitted { - t.Fatalf("primary copy after retry = %s, want committed", copyRow.Status) - } -} - -func TestUploader_RejectedSubmittedIngressCommitIsResubmitted(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadyIngressCommitTask(t, env, 1, 2) - upload, versionID, task := fixture.upload, fixture.versionID, fixture.task - copyRow := fixture.copyRow - seedSubmittedCommitAttempt(t, env.repos, copyRow, "rejected-ingress", "01", fakeSubmittedCommitTxHash) - - statusRequests := atomic.Int32{} - statusServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - statusRequests.Add(1) - w.Header().Set("Content-Type", "application/json") - _, _ = fmt.Fprintf(w, `{"txHash":%q,"txStatus":"rejected","dataSetId":1001,"pieceCount":1,"piecesAdded":false}`, fakeSubmittedCommitTxHash) - })) - defer statusServer.Close() - primaryCtx := fixture.primary - primaryCtx.serviceURL = statusServer.URL - - uploader := worker.NewUploader( - env.repos, env.cache, env.storage, nil, env.sm, - cache.EvictionPolicyAfterUpload, 1, 1, 10*time.Millisecond, slog.Default(), - worker.WithPDPStatusChecker(synapse.NewPDPStatusChecker(synapse.PDPStatusCheckerOptions{AllowPrivateNetworks: true})), - ) - runWorkerUntilTaskRetryCount(t, env, uploader, task.ID, 1, 5*time.Second) - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, upload.ID, 0) - if err != nil || copyRow.Status != model.StorageUploadCopyStatusPieceReady || copyRow.CommitTransactionID != nil || copyRow.CommitExtraDataHex != nil { - t.Fatalf("copy after rejected status = %#v err=%v, want resubmittable piece", copyRow, err) - } - if _, err := env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - t.Fatalf("reschedule rejected commit task: %v", err) - } - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 1 { - t.Fatalf("rejected commit task = %#v, want completed after one retry", gotTask) - } - if statusRequests.Load() != 1 || primaryCtx.commitCalls.Load() != 1 || primaryCtx.presignCalls.Load() != 1 { - t.Fatalf("rejected commit recovery calls = status:%d commit:%d presign:%d, want 1/1/1", statusRequests.Load(), primaryCtx.commitCalls.Load(), primaryCtx.presignCalls.Load()) - } - copyRow, err = env.repos.Uploads.GetUploadCopy(ctx, upload.ID, 0) - if err != nil || copyRow.Status != model.StorageUploadCopyStatusCommitted { - t.Fatalf("copy after rejected commit retry = %#v err=%v, want committed", copyRow, err) - } - gotVersion, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || gotVersion == nil || gotVersion.State != model.ObjectStateStored { - t.Fatalf("version after rejected commit retry = %#v err=%v, want stored", gotVersion, err) - } -} - -func TestUploader_SubmittedPeerMismatchedStatusRemainsRecoverableAfterExhaustion(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - pieceCID := testCID(t) - if err := env.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: fixture.upload.ID, - CopyIndex: 1, - PieceCID: pieceCID.String(), - RetrievalURL: fmt.Sprintf("https://provider-202.example/piece/%s", pieceCID.String()), - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, 1) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - seedSubmittedCommitAttempt(t, env.repos, copyRow, "mismatched-peer", "02", fakeSubmittedCommitTxHash) - stage := "peer_commit" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "object", RefID: fixture.objID, RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:peer_commit:%d:1", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{"upload_id": fixture.upload.ID, "copy_index": 1, "transfer_method": string(model.StorageCopyTransferMethodPeerPull)}, - Status: model.TaskStatusQueued, MaxRetries: 1, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create peer commit task: %v", err) - } - - statusServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = fmt.Fprintf(w, `{"txHash":%q,"txStatus":"confirmed","dataSetId":9999,"pieceCount":1,"addMessageOk":true,"piecesAdded":true,"confirmedPieceIds":[302]}`, fakeSubmittedCommitTxHash) - })) - defer statusServer.Close() - peerCtx := newFakeUploadContext(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(302), pieceCID) - peerDataSetID := sdktypes.NewBigInt(2002) - peerCtx.boundDataSet = &peerDataSetID - peerCtx.serviceURL = statusServer.URL - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextDataSetIDEqual(opts, peerDataSetID) { - return peerCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - uploader := worker.NewUploader( - env.repos, env.cache, env.storage, nil, env.sm, - cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default(), - worker.WithPDPStatusChecker(synapse.NewPDPStatusChecker(synapse.PDPStatusCheckerOptions{AllowPrivateNetworks: true})), - ) - runWorkerUntilTaskStatus(t, env, uploader, task.ID, model.TaskStatusWaiting, 5*time.Second) - gotTask, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || gotTask.Status != model.TaskStatusWaiting || gotTask.RetryCount != 0 { - t.Fatalf("peer commit task = %#v err=%v, want attention wait without retry", gotTask, err) - } - copyRow, err = env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, 1) - if err != nil || !copyCommitSubmittedForTest(copyRow) || *copyRow.CommitTransactionID != fakeSubmittedCommitTxHash || - copyRow.CommitAttentionCode == nil || *copyRow.CommitAttentionCode != string(storagecommit.AttentionSubmissionMismatch) { - t.Fatalf("peer copy after mismatched status = %#v err=%v, want recoverable attention", copyRow, err) - } - if err := env.repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: fixture.upload.ID, CopyIndex: 1, LastError: "late generic failure"}); !errors.Is(err, repository.ErrConflict) { - t.Fatalf("MarkUploadCopyFailed submitted peer error = %v, want conflict", err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, fixture.versionID) - if err != nil || version == nil || version.State != model.ObjectStateReplicating { - t.Fatalf("version after submitted peer uncertainty = %#v err=%v, want replicating", version, err) - } -} - -func TestUploader_UnavailablePeerKeepsObjectReadableAndWaitsForInPlaceRepair(t *testing.T) { - env := newTestWorkerEnv(t) - _, objID, versionID := seedCachedObject(t, env) - _ = seedStagedUploadTask(t, env, objID, versionID, 1) - - pieceCID := testCID(t) - ingress := newFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(2001), pieceCID) - peer := newFakeUploadContext(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(3001), pieceCID) - peer.pullErr = &synapse.ProviderUnavailableError{Cause: errors.New("provider pull failed")} - contextsByProvider := map[string]*fakeUploadContext{ - ingress.providerID.String(): ingress, - peer.providerID.String(): peer, - } - contextsByDataSet := map[string]*fakeUploadContext{ - ingress.dataSetID.String(): ingress, - peer.dataSetID.String(): peer, - } - var selectTargetsCalls atomic.Int32 - env.storage.SelectUploadTargetsFunc = func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - if opts.Copies == 2 { - return []synapse.StorageTarget{ingress, peer}, nil - } - t.Fatalf("unexpected SelectUploadTargets copies = %d; established slots must not be replaced", opts.Copies) - return nil, nil - } - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if opts.ProviderID != nil { - return contextsByProvider[opts.ProviderID.String()], nil - } - if opts.DataSetID != nil { - return contextsByDataSet[opts.DataSetID.String()], nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - runCtx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = uploader.Run(runCtx) - close(done) - }() - defer func() { - cancel() - waitForSignal(t, done, time.Second, "uploader shutdown") - }() - - var obj *model.ObjectVersion - var repairTask *model.Task - deadline := time.After(3 * time.Second) - ticker := time.NewTicker(20 * time.Millisecond) - defer ticker.Stop() - for { - select { - case <-deadline: - t.Fatal("timed out waiting for peer failure to enter in-place repair") - case <-ticker.C: - got, err := env.repos.Objects.GetVersionByID(context.Background(), versionID) - if err != nil || got == nil || got.StorageUploadID == nil { - continue - } - upload, err := env.repos.Uploads.GetByID(context.Background(), *got.StorageUploadID) - if err != nil || upload == nil { - continue - } - copies, err := env.repos.Uploads.ListCopies(context.Background(), upload.ID) - if err != nil { - continue - } - exhaustedTasks, err := env.repos.Tasks.ListExhausted(context.Background(), 10) - if err != nil || len(exhaustedTasks) != 0 { - continue - } - repairTasks, _, err := env.repos.Tasks.List(context.Background(), string(model.TaskTypeUpload), "repair_replica", string(model.TaskStatusWaiting), 10, 0) - if err != nil { - continue - } - for i := range repairTasks { - if taskPayloadInt64ForTest(repairTasks[i].Payload, "storage_data_set_id") > 0 { - repairTask = &repairTasks[i] - break - } - } - if got.State == model.ObjectStateReplicating && upload.Status == model.StorageUploadStatusReadable && len(copies) == 2 && repairTask != nil { - obj = got - goto doneWaiting - } - } - } - -doneWaiting: - if obj == nil || obj.StorageUploadID == nil || !obj.InFilecoin { - t.Fatalf("object after secondary failure = %#v, want readable replicating version", obj) - } - copies, err := env.repos.Uploads.ListCopies(context.Background(), *obj.StorageUploadID) - if err != nil { - t.Fatalf("ListCopies: %v", err) - } - if len(copies) != 2 || copies[0].Status != model.StorageUploadCopyStatusCommitted || copies[1].Status != model.StorageUploadCopyStatusPending { - t.Fatalf("copies after peer failure = %#v, want committed ingress and pending original peer", copies) - } - if repairTask.RefType != "bucket" || taskPayloadInt64ForTest(repairTask.Payload, "storage_upload_copy_id") != copies[1].ID { - t.Fatalf("repair task = %#v, want exact original peer copy %d", repairTask, copies[1].ID) - } - binding, err := env.repos.Uploads.GetDataSetBindingByCopyIndex(context.Background(), obj.BucketID, 1) - if err != nil || binding == nil || binding.Status != model.StorageDataSetStatusUnavailable { - t.Fatalf("peer binding after outage = %#v err=%v, want unavailable", binding, err) - } - evict, err := env.repos.Tasks.ClaimReady(context.Background(), model.TaskTypeEvictCache, time.Minute) - if err != nil { - t.Fatalf("ClaimReady(evict): %v", err) - } - if evict != nil { - t.Fatalf("unexpected evict task for partial upload: %#v", evict) - } - peer.pullErr = nil - if _, err := env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("id = ?", repairTask.ID). - Exec(context.Background()); err != nil { - t.Fatalf("reschedule replica repair: %v", err) - } - waitForObjectState(t, env, versionID, model.ObjectStateStored, 5*time.Second) - repairedCopy, err := env.repos.Uploads.GetUploadCopy(context.Background(), *obj.StorageUploadID, 1) - if err != nil || repairedCopy == nil || repairedCopy.Status != model.StorageUploadCopyStatusCommitted { - t.Fatalf("repaired copy = %#v err=%v, want committed original slot", repairedCopy, err) - } - recoveredBinding, err := env.repos.Uploads.GetDataSetBindingByCopyIndex(context.Background(), obj.BucketID, 1) - if err != nil || recoveredBinding == nil || recoveredBinding.Status != model.StorageDataSetStatusReady { - t.Fatalf("recovered binding = %#v err=%v, want ready", recoveredBinding, err) - } - if got := selectTargetsCalls.Load(); got != 1 { - t.Fatalf("SelectUploadTargets calls = %d, want only initial slot provisioning", got) - } -} - -func TestUploader_ReplicaRepairDoesNotRecoverDataSetBeforeStorageOperationSucceeds(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - if err := env.repos.Uploads.MarkDataSetUnavailable(ctx, fixture.peer.ID, "temporary outage"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, 1) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - peerCtx := newFakeUploadContext(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(3001), testCID(t)) - dataSetID := sdktypes.NewBigInt(2002) - peerCtx.boundDataSet = &dataSetID - peerCtx.pullErr = errors.New("unexpected pull response") - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextDataSetIDEqual(opts, dataSetID) { - return peerCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - stage := "repair_replica" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "bucket", RefID: fixture.upload.BucketID, - RefVersionID: fixture.versionID, IdempotencyKey: fmt.Sprintf("upload:repair-data-set:%d", fixture.peer.ID), - Payload: map[string]interface{}{"storage_data_set_id": fixture.peer.ID, "storage_upload_copy_id": copyRow.ID}, - Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create repair task: %v", err) - } - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTaskRetryCount(t, env, uploader, task.ID, 1, 5*time.Second) - - gotTask, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || gotTask.Status != model.TaskStatusScheduled || gotTask.RetryCount != 1 { - t.Fatalf("repair task = %#v err=%v, want bounded retry", gotTask, err) - } - binding, err := env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.peer.ID) - if err != nil || binding == nil || binding.Status != model.StorageDataSetStatusUnavailable { - t.Fatalf("binding before successful storage operation = %#v err=%v, want unavailable", binding, err) - } - gotCopy, err := env.repos.Uploads.GetUploadCopyByID(ctx, copyRow.ID) - if err != nil || gotCopy == nil || gotCopy.Status != model.StorageUploadCopyStatusPending { - t.Fatalf("copy after failed repair operation = %#v err=%v, want pending", gotCopy, err) - } -} - -func TestUploader_ReplicaRepairResubmitsRejectedCommit(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - if err := env.repos.Uploads.MarkDataSetUnavailable(ctx, fixture.peer.ID, "temporary outage"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - pieceCID := testCID(t) - if err := env.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: fixture.upload.ID, - CopyIndex: 1, - PieceCID: pieceCID.String(), - RetrievalURL: fmt.Sprintf("https://provider-202.example/piece/%s", pieceCID.String()), - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, 1) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - seedSubmittedCommitAttempt(t, env.repos, copyRow, "rejected-repair", "02", fakeSubmittedCommitTxHash) - stage := "repair_replica" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "bucket", RefID: fixture.upload.BucketID, - RefVersionID: fixture.versionID, IdempotencyKey: fmt.Sprintf("upload:repair-data-set:%d", fixture.peer.ID), - Payload: map[string]interface{}{"storage_data_set_id": fixture.peer.ID, "storage_upload_copy_id": copyRow.ID}, - Status: model.TaskStatusQueued, MaxRetries: 2, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create repair task: %v", err) - } - - statusRequests := atomic.Int32{} - statusServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - statusRequests.Add(1) - w.Header().Set("Content-Type", "application/json") - _, _ = fmt.Fprintf(w, `{"txHash":%q,"txStatus":"rejected","dataSetId":2002,"pieceCount":1,"piecesAdded":false}`, fakeSubmittedCommitTxHash) - })) - defer statusServer.Close() - peerCtx := newFakeUploadContext(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(302), pieceCID) - peerDataSetID := sdktypes.NewBigInt(2002) - peerCtx.boundDataSet = &peerDataSetID - peerCtx.serviceURL = statusServer.URL - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextDataSetIDEqual(opts, peerDataSetID) { - return peerCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - uploader := worker.NewUploader( - env.repos, env.cache, env.storage, nil, env.sm, - cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default(), - worker.WithPDPStatusChecker(synapse.NewPDPStatusChecker(synapse.PDPStatusCheckerOptions{AllowPrivateNetworks: true})), - ) - runWorkerUntilTaskRetryCount(t, env, uploader, task.ID, 1, 5*time.Second) - copyRow, err = env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, 1) - if err != nil || copyRow == nil || copyRow.Status != model.StorageUploadCopyStatusPieceReady || copyRow.CommitTransactionID != nil || copyRow.CommitExtraDataHex != nil { - t.Fatalf("repair copy after rejected commit = %#v err=%v, want resubmittable piece", copyRow, err) - } - if _, err := env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - t.Fatalf("reschedule repair task: %v", err) - } - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted { - t.Fatalf("repair task = %#v, want completed after rejected commit retry", gotTask) - } - if statusRequests.Load() != 1 || peerCtx.commitCalls.Load() != 1 { - t.Fatalf("repair recovery calls = status:%d commit:%d, want 1/1", statusRequests.Load(), peerCtx.commitCalls.Load()) - } - copyRow, err = env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, 1) - if err != nil || copyRow == nil || copyRow.Status != model.StorageUploadCopyStatusCommitted { - t.Fatalf("repair copy after retry = %#v err=%v, want committed", copyRow, err) - } - binding, err := env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.peer.ID) - if err != nil || binding == nil || binding.Status != model.StorageDataSetStatusReady { - t.Fatalf("data set after rejected repair retry = %#v err=%v, want ready", binding, err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, fixture.versionID) - if err != nil || version == nil || version.State != model.ObjectStateStored { - t.Fatalf("version after rejected repair retry = %#v err=%v, want stored", version, err) - } -} - -func TestUploader_ReplicaRepairResumesAfterCopyCommitBeforeDataSetRecovery(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - if err := env.repos.Uploads.MarkDataSetUnavailable(ctx, fixture.peer.ID, "temporary outage"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, 1) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - if err := env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - StorageUploadCopyID: copyRow.ID, - UploadID: fixture.upload.ID, - CopyIndex: 1, - PieceCID: testCID(t).String(), - PieceID: onChainIDPtr(t, "302"), - RetrievalURL: "https://peer.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - stage := "repair_replica" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "bucket", RefID: fixture.upload.BucketID, - IdempotencyKey: fmt.Sprintf("upload:repair-data-set:%d", fixture.peer.ID), - Payload: map[string]interface{}{"storage_data_set_id": fixture.peer.ID, "storage_upload_copy_id": copyRow.ID}, - Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create repair task: %v", err) - } - peerBinding, err := env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.peer.ID) - if err != nil || peerBinding == nil || peerBinding.DataSetID == nil { - t.Fatalf("GetDataSetBindingByID: binding=%#v err=%v", peerBinding, err) - } - peerCtx := newFakeUploadContext(peerBinding.ProviderID.SDK(), peerBinding.DataSetID.SDK(), sdktypes.NewBigInt(302), testCID(t)) - boundDataSetID := peerBinding.DataSetID.SDK() - peerCtx.boundDataSet = &boundDataSetID - var contextCalls atomic.Int32 - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - contextCalls.Add(1) - if !createContextDataSetIDEqual(opts, boundDataSetID) { - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - return peerCtx, nil - } - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("repair task = %#v, want idempotent completion", gotTask) - } - binding, err := env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.peer.ID) - if err != nil || binding == nil || binding.Status != model.StorageDataSetStatusReady { - t.Fatalf("recovered data set = %#v err=%v, want ready", binding, err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, fixture.versionID) - if err != nil || version == nil || version.State != model.ObjectStateStored { - t.Fatalf("finalized version = %#v err=%v, want stored", version, err) - } - if contextCalls.Load() != 1 || peerCtx.storeCalls.Load() != 0 || peerCtx.commitCalls.Load() != 0 { - t.Fatalf("committed resume calls: context=%d store=%d commit=%d, want one context validation without rewriting", contextCalls.Load(), peerCtx.storeCalls.Load(), peerCtx.commitCalls.Load()) - } -} - -func TestUploader_DataSetRecoveryFinalizesOtherCommittedUploads(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - ingress, err := env.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, fixture.upload.BucketID, 0) - if err != nil || ingress == nil { - t.Fatalf("GetDataSetBindingByCopyIndex ingress: binding=%#v err=%v", ingress, err) - } - otherUpload, otherVersionID := seedCommittedReplicatingUploadOnBindings(t, env, fixture.upload.BucketID, ingress, fixture.peer) - - currentCopy, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, 1) - if err != nil || currentCopy == nil { - t.Fatalf("GetUploadCopy current: copy=%#v err=%v", currentCopy, err) - } - if err := env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - StorageUploadCopyID: currentCopy.ID, - UploadID: fixture.upload.ID, - CopyIndex: 1, - PieceCID: testCID(t).String(), - PieceID: onChainIDPtr(t, "302"), - RetrievalURL: "https://peer.example/current-piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted current: %v", err) - } - if err := env.repos.Uploads.MarkDataSetUnavailable(ctx, fixture.peer.ID, "temporary outage"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - stage := "repair_replica" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "bucket", RefID: fixture.upload.BucketID, - RefVersionID: fixture.versionID, IdempotencyKey: fmt.Sprintf("upload:repair-data-set:%d", fixture.peer.ID), - Payload: map[string]interface{}{"storage_data_set_id": fixture.peer.ID, "storage_upload_copy_id": currentCopy.ID}, - Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create repair task: %v", err) - } - peerDataSetID := sdktypes.NewBigInt(2002) - peerCtx := newFakeUploadContext(sdktypes.NewBigInt(202), peerDataSetID, sdktypes.NewBigInt(302), testCID(t)) - peerCtx.boundDataSet = &peerDataSetID - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextDataSetIDEqual(opts, peerDataSetID) { - return peerCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("repair task = %#v, want one coordinator to finish every affected upload", gotTask) - } - for _, versionID := range []string{fixture.versionID, otherVersionID} { - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil || version.State != model.ObjectStateStored { - t.Fatalf("version %s after data set recovery = %#v err=%v, want stored", versionID, version, err) - } - } - other, err := env.repos.Uploads.GetByID(ctx, otherUpload.ID) - if err != nil || other == nil || other.Status != model.StorageUploadStatusComplete { - t.Fatalf("other upload after data set recovery = %#v err=%v, want complete", other, err) - } -} - -func TestUploader_ReplicaRepairFinalizesCommittedCopyWhenDataSetStartsDraining(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - if err := env.repos.Uploads.MarkDataSetUnavailable(ctx, fixture.peer.ID, "temporary outage"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, 1) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - if _, err := env.db.ExecContext(ctx, fmt.Sprintf(`CREATE TRIGGER drain_repair_data_set_after_commit - AFTER UPDATE OF status ON storage_upload_copies - WHEN NEW.id = %d AND NEW.status = '%s' - BEGIN - UPDATE storage_data_sets - SET status = '%s', updated_at = CURRENT_TIMESTAMP - WHERE id = %d; - END`, copyRow.ID, model.StorageUploadCopyStatusCommitted, model.StorageDataSetStatusDraining, fixture.peer.ID)); err != nil { - t.Fatalf("create draining race trigger: %v", err) - } - - peerBinding, err := env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.peer.ID) - if err != nil || peerBinding == nil || peerBinding.DataSetID == nil { - t.Fatalf("GetDataSetBindingByID: binding=%#v err=%v", peerBinding, err) - } - peerCtx := newFakeUploadContext(peerBinding.ProviderID.SDK(), peerBinding.DataSetID.SDK(), sdktypes.NewBigInt(302), testCID(t)) - boundDataSetID := peerBinding.DataSetID.SDK() - peerCtx.boundDataSet = &boundDataSetID - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if !createContextDataSetIDEqual(opts, boundDataSetID) { - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - return peerCtx, nil - } - stage := "repair_replica" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "bucket", RefID: fixture.upload.BucketID, - IdempotencyKey: fmt.Sprintf("upload:repair-data-set:%d", fixture.peer.ID), - Payload: map[string]interface{}{"storage_data_set_id": fixture.peer.ID, "storage_upload_copy_id": copyRow.ID}, - Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create repair task: %v", err) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("repair task = %#v, want completed without retry", gotTask) - } - binding, err := env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.peer.ID) - if err != nil || binding == nil || binding.Status != model.StorageDataSetStatusDraining { - t.Fatalf("data set after concurrent transition = %#v err=%v, want draining", binding, err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, fixture.versionID) - if err != nil || version == nil || version.State != model.ObjectStateStored { - t.Fatalf("finalized version = %#v err=%v, want stored", version, err) - } -} - -func TestUploader_CommittedReplicaRepairWaitsForLiveProviderEvidence(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - if err := env.repos.Uploads.MarkDataSetUnavailable(ctx, fixture.peer.ID, "temporary outage"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, 1) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - if err := env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - StorageUploadCopyID: copyRow.ID, - UploadID: fixture.upload.ID, - CopyIndex: 1, - PieceCID: testCID(t).String(), - PieceID: onChainIDPtr(t, "302"), - RetrievalURL: "https://peer.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - stage := "repair_replica" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "bucket", RefID: fixture.upload.BucketID, - IdempotencyKey: fmt.Sprintf("upload:repair-data-set:%d", fixture.peer.ID), - Payload: map[string]interface{}{"storage_data_set_id": fixture.peer.ID, "storage_upload_copy_id": copyRow.ID}, - Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create repair task: %v", err) - } - env.storage.OpenTargetFunc = func(_ context.Context, _ *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - return nil, &synapse.ProviderUnavailableError{Cause: context.DeadlineExceeded} - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, env, uploader, task.ID, model.TaskStatusWaiting, 5*time.Second) - gotTask, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || gotTask.RetryCount != 0 || gotTask.WaitReason == nil || *gotTask.WaitReason != model.TaskWaitReasonDependency { - t.Fatalf("repair task = %#v err=%v, want dependency wait without retry", gotTask, err) - } - binding, err := env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.peer.ID) - if err != nil || binding == nil || binding.Status != model.StorageDataSetStatusUnavailable { - t.Fatalf("data set without live evidence = %#v err=%v, want unavailable", binding, err) - } -} - -func TestUploader_ReplicaRepairUsesRetainedCacheWhenNoRemoteCopyIsReadable(t *testing.T) { - env := newTestWorkerEnv(t) - bucket, _, versionID := seedCachedObject(t, env) - ctx := context.Background() - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark version uploading: %v", err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%#v err=%v", version, err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: upload.ID, - DataSetID: onChainID(t, "1001"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: onChainID(t, "101"), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := env.repos.Uploads.MarkDataSetUnavailable(ctx, binding.ID, "temporary outage"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, upload.ID, 0) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - stage := "repair_replica" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "bucket", RefID: bucket.ID, - RefVersionID: versionID, IdempotencyKey: fmt.Sprintf("upload:repair-data-set:%d", binding.ID), - Payload: map[string]interface{}{"storage_data_set_id": binding.ID, "storage_upload_copy_id": copyRow.ID}, - Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create repair task: %v", err) - } - repairedCtx := newFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(2001), testCID(t)) - dataSetID := sdktypes.NewBigInt(1001) - repairedCtx.boundDataSet = &dataSetID - var createContextCalls atomic.Int32 - var selectTargetsCalls atomic.Int32 - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - createContextCalls.Add(1) - if !createContextProviderIDEqual(opts, sdktypes.NewBigInt(101)) || !createContextDataSetIDEqual(opts, dataSetID) { - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - return repairedCtx, nil - } - env.storage.SelectUploadTargetsFunc = func(_ context.Context, _ storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - return nil, errors.New("replica repair must not select a new provider") - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 1, 1, 10*time.Millisecond, slog.Default()) - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("repair task = %#v, want completed without retry", gotTask) - } - if got := createContextCalls.Load(); got != 2 { - t.Fatalf("CreateContext calls = %d, want one submission run and one confirmation run on the original context", got) - } - if got := selectTargetsCalls.Load(); got != 0 { - t.Fatalf("SelectUploadTargets calls = %d, want no provider selection", got) - } - if got := repairedCtx.storeCalls.Load(); got != 1 { - t.Fatalf("Store calls = %d, want one retained-cache upload", got) - } - if got := repairedCtx.pullCalls.Load(); got != 0 { - t.Fatalf("Pull calls = %d, want no remote source pull", got) - } - repairedCopy, err := env.repos.Uploads.GetUploadCopyByID(ctx, copyRow.ID) - if err != nil || repairedCopy == nil || repairedCopy.Status != model.StorageUploadCopyStatusCommitted { - t.Fatalf("repaired copy = %#v err=%v, want committed original copy", repairedCopy, err) - } - recoveredBinding, err := env.repos.Uploads.GetDataSetBindingByID(ctx, binding.ID) - if err != nil || recoveredBinding == nil || recoveredBinding.Status != model.StorageDataSetStatusReady { - t.Fatalf("recovered binding = %#v err=%v, want ready", recoveredBinding, err) - } - storedVersion, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || storedVersion == nil || storedVersion.State != model.ObjectStateStored { - t.Fatalf("stored version = %#v err=%v, want stored", storedVersion, err) - } -} - -func TestUploader_ReplicaRepairUsesSharedVersionAfterSourceDelete(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedStoredUploadWithPendingExtraCopy(t, env) - - followerVersionID := model.NewVersionID() - followerCacheKey := ".versions/" + followerVersionID - info, err := env.cache.Put(ctx, fixture.bucket.Name, followerCacheKey, bytes.NewReader([]byte("hello world"))) - if err != nil { - t.Fatalf("cache follower: %v", err) - } - follower := &model.ObjectVersion{ - VersionID: followerVersionID, BucketID: fixture.bucket.ID, Key: "follower.txt", - Size: info.Size, ETag: info.ETag, Checksum: info.Checksum, ContentType: "text/plain", CacheKey: followerCacheKey, - } - if _, err := env.repos.Objects.CreateVersionAndSetCurrent(ctx, follower); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(follower): %v", err) - } - if err := env.repos.Objects.UpdateVersionState(ctx, follower.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark follower uploading: %v", err) - } - if _, err := env.repos.Uploads.BindReadableUploadForVersion(ctx, repository.BindReadableUploadForVersionInput{ - UploadID: fixture.upload.ID, BucketID: fixture.bucket.ID, ContentSize: follower.Size, - Checksum: follower.Checksum, VersionID: follower.VersionID, - }); err != nil { - t.Fatalf("BindReadableUploadForVersion(follower): %v", err) - } - if _, _, err := env.repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.NewFinalizeUploadInput(fixture.upload.ID, false, 0)); err != nil { - t.Fatalf("FinalizeUploadIfTargetCopiesMet(follower): %v", err) - } - - task := seedReplicaRepairTaskWithStatus(t, env, fixture, model.TaskStatusExhausted, "shared-source-delete") - if _, err := env.repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: fixture.bucket.ID, Key: fixture.sourceKey, VersionID: fixture.sourceVersionID, - }); err != nil { - t.Fatalf("DeleteObjectVersionPermanently(source): %v", err) - } - if err := env.repos.Tasks.RetryExhausted(ctx, task.ID); err != nil { - t.Fatalf("RetryExhausted: %v", err) - } - - peerBinding, err := env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.peer.ID) - if err != nil || peerBinding == nil || peerBinding.DataSetID == nil { - t.Fatalf("GetDataSetBindingByID(peer): binding=%#v err=%v", peerBinding, err) - } - peerDataSetID := peerBinding.DataSetID.SDK() - peerCtx := newFakeUploadContext(peerBinding.ProviderID.SDK(), peerDataSetID, sdktypes.NewBigInt(302), testCID(t)) - peerCtx.boundDataSet = &peerDataSetID - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if !createContextDataSetIDEqual(opts, peerDataSetID) { - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - return peerCtx, nil - } - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 1, 1, 10*time.Millisecond, slog.Default()) - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("repair task after source delete = %#v, want completed without retry", gotTask) - } - copyRow, err := env.repos.Uploads.GetUploadCopyByID(ctx, fixture.repairCopy.ID) - if err != nil || copyRow == nil || copyRow.Status != model.StorageUploadCopyStatusCommitted { - t.Fatalf("repair copy after source delete = %#v err=%v, want committed", copyRow, err) - } - gotFollower, err := env.repos.Objects.GetVersionByID(ctx, follower.VersionID) - if err != nil || gotFollower == nil || gotFollower.State != model.ObjectStateStored { - t.Fatalf("surviving shared version = %#v err=%v, want stored", gotFollower, err) - } -} - -func TestUploader_ReplicaRepairDoesNotReachStorageAfterPermanentDeleteWins(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedStoredUploadWithPendingExtraCopy(t, env) - task := seedReplicaRepairTaskWithStatus(t, env, fixture, model.TaskStatusExhausted, "delete-wins") - if _, err := env.repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ - BucketID: fixture.bucket.ID, Key: fixture.sourceKey, VersionID: fixture.sourceVersionID, - }); err != nil { - t.Fatalf("DeleteObjectVersionPermanently: %v", err) - } - if err := env.repos.Tasks.RetryExhausted(ctx, task.ID); err != nil { - t.Fatalf("RetryExhausted: %v", err) - } - var createContextCalls atomic.Int32 - var selectTargetsCalls atomic.Int32 - env.storage.OpenTargetFunc = func(_ context.Context, _ *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - createContextCalls.Add(1) - return nil, errors.New("deleted repair must not create a storage context") - } - env.storage.SelectUploadTargetsFunc = func(_ context.Context, _ storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - return nil, errors.New("deleted repair must not select storage providers") - } - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 1, 1, 10*time.Millisecond, slog.Default()) - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("cancelled repair task = %#v, want completed without retry", gotTask) - } - if createContextCalls.Load() != 0 || selectTargetsCalls.Load() != 0 { - t.Fatalf("storage context calls after delete = exact:%d selection:%d, want zero", createContextCalls.Load(), selectTargetsCalls.Load()) - } - copyRow, err := env.repos.Uploads.GetUploadCopyByID(ctx, fixture.repairCopy.ID) - if err != nil || copyRow == nil || copyRow.Status != model.StorageUploadCopyStatusFailed { - t.Fatalf("cancelled repair copy = %#v err=%v, want failed", copyRow, err) - } -} - -type storedUploadWithPendingExtraCopyFixture struct { - bucket *model.Bucket - sourceKey string - sourceVersionID string - upload *model.StorageUpload - peer *model.StorageDataSet - repairCopy *model.StorageUploadCopy -} - -func seedStoredUploadWithPendingExtraCopy(t *testing.T, env *testWorkerEnv) storedUploadWithPendingExtraCopyFixture { - t.Helper() - bucket, _, sourceVersionID := seedCachedObject(t, env) - ctx := context.Background() - source, err := env.repos.Objects.GetVersionByID(ctx, sourceVersionID) - if err != nil || source == nil { - t.Fatalf("GetVersionByID(source): version=%#v err=%v", source, err) - } - if err := env.repos.Objects.UpdateVersionState(ctx, sourceVersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark source uploading: %v", err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: sourceVersionID, ContentSize: source.Size, Checksum: source.Checksum, RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - ingress, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding(ingress): %v", err) - } - peer, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, "202"), CopyIndex: 1, CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding(peer): %v", err) - } - for _, input := range []repository.MarkDataSetReadyInput{ - {ID: ingress.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001")}, - {ID: peer.ID, UploadID: upload.ID, DataSetID: onChainID(t, "2002")}, - } { - if err := env.repos.Uploads.MarkDataSetReady(ctx, input); err != nil { - t.Fatalf("MarkDataSetReady(%d): %v", input.ID, err) - } - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: ingress.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: ingress.ProviderID}, - {StorageDataSetID: peer.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: peer.ProviderID}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, CopyIndex: 0, PieceCID: testCID(t).String(), PieceID: onChainIDPtr(t, "301"), RetrievalURL: "https://ingress.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted(ingress): %v", err) - } - if _, err := env.repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, BucketID: bucket.ID, ContentSize: source.Size, Checksum: source.Checksum, - }); err != nil { - t.Fatalf("BindReadableUploadForContent: %v", err) - } - if _, _, err := env.repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.NewFinalizeUploadInput(upload.ID, false, 0)); err != nil { - t.Fatalf("FinalizeUploadIfTargetCopiesMet: %v", err) - } - if err := env.repos.Uploads.MarkDataSetUnavailable(ctx, peer.ID, "temporary outage"); err != nil { - t.Fatalf("MarkDataSetUnavailable(peer): %v", err) - } - repairCopy, err := env.repos.Uploads.GetUploadCopy(ctx, upload.ID, 1) - if err != nil || repairCopy == nil { - t.Fatalf("GetUploadCopy(repair): copy=%#v err=%v", repairCopy, err) - } - return storedUploadWithPendingExtraCopyFixture{ - bucket: bucket, sourceKey: source.Key, sourceVersionID: sourceVersionID, - upload: upload, peer: peer, repairCopy: repairCopy, - } -} - -func seedReplicaRepairTaskWithStatus( - t *testing.T, - env *testWorkerEnv, - fixture storedUploadWithPendingExtraCopyFixture, - status model.TaskStatus, - suffix string, -) *model.Task { - t.Helper() - stage := "repair_replica" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "bucket", RefID: fixture.bucket.ID, - RefVersionID: fixture.sourceVersionID, IdempotencyKey: "upload:repair-data-set:" + suffix, - Payload: map[string]interface{}{"storage_data_set_id": fixture.peer.ID, "storage_upload_copy_id": fixture.repairCopy.ID}, - Status: status, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(context.Background(), task); err != nil { - t.Fatalf("Create replica repair task: %v", err) - } - return task -} - -func TestUploader_PeerTransientFailureKeepsCopyRetryable(t *testing.T) { - env := newTestWorkerEnv(t) - bucket, objID, versionID := seedCachedObject(t, env) - ctx := context.Background() - - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("committing: %v", err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - ingress, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("ingress binding: %v", err) - } - peer, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "202"), CopyIndex: 1, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("peer binding: %v", err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: ingress.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001")}); err != nil { - t.Fatalf("MarkDataSetReady ingress: %v", err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: peer.ID, UploadID: upload.ID, DataSetID: onChainID(t, "2002")}); err != nil { - t.Fatalf("MarkDataSetReady peer: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: ingress.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - {StorageDataSetID: peer.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: testCID(t).String(), - PieceID: onChainIDPtr(t, "301"), - RetrievalURL: "https://ingress.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - if _, err := env.repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: version.Size, - Checksum: version.Checksum, - }); err != nil { - t.Fatalf("BindReadableUploadForContent: %v", err) - } - - peerCtx := newFakeUploadContext(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(3001), testCID(t)) - peerDataSetID := sdktypes.NewBigInt(2002) - peerCtx.boundDataSet = &peerDataSetID - peerCtx.pullErr = errors.New("temporary provider pull failed") - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextDataSetIDEqual(opts, sdktypes.NewBigInt(2002)) { - return peerCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - stage := "peer_pull" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objID, - RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:peer_pull:%d:1", versionID, upload.ID), - Payload: map[string]interface{}{"upload_id": upload.ID, "copy_index": 1, "transfer_method": string(model.StorageCopyTransferMethodPeerPull)}, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create peer task: %v", err) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTaskRetryCount(t, env, uploader, task.ID, 1, 5*time.Second) - - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, upload.ID, 1) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy peer: copy=%v err=%v", copyRow, err) - } - if copyRow.Status != model.StorageUploadCopyStatusPending { - t.Fatalf("peer copy status = %s, want pending while task remains retryable", copyRow.Status) - } - copies, err := env.repos.Uploads.ListCopies(ctx, upload.ID) - if err != nil { - t.Fatalf("ListCopies: %v", err) - } - if len(copies) != 2 { - t.Fatalf("copy count = %d, want no replacement before retry exhaustion", len(copies)) - } -} - -type readableUploadWithPendingPeerFixture struct { - objID int64 - versionID string - upload *model.StorageUpload - ingress *model.StorageDataSet - peer *model.StorageDataSet -} - -func seedReadableUploadWithPendingPeer(t *testing.T, env *testWorkerEnv) readableUploadWithPendingPeerFixture { - t.Helper() - bucket, objID, versionID := seedCachedObject(t, env) - ctx := context.Background() - - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("uploading: %v", err) - } - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("committing: %v", err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - ingress, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("ingress binding: %v", err) - } - peer, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "202"), CopyIndex: 1, CreatedByUploadID: upload.ID}) - if err != nil { - t.Fatalf("peer binding: %v", err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: ingress.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001")}); err != nil { - t.Fatalf("MarkDataSetReady ingress: %v", err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: peer.ID, UploadID: upload.ID, DataSetID: onChainID(t, "2002")}); err != nil { - t.Fatalf("MarkDataSetReady peer: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: ingress.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - {StorageDataSetID: peer.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - if err := env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: 0, - PieceCID: testCID(t).String(), - PieceID: onChainIDPtr(t, "301"), - RetrievalURL: "https://ingress.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - if _, err := env.repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: bucket.ID, - ContentSize: version.Size, - Checksum: version.Checksum, - }); err != nil { - t.Fatalf("BindReadableUploadForContent: %v", err) - } - return readableUploadWithPendingPeerFixture{ - objID: objID, - versionID: versionID, - upload: upload, - ingress: ingress, - peer: peer, - } -} - -func seedCommittedReplicatingUploadOnBindings( - t *testing.T, - env *testWorkerEnv, - bucketID int64, - ingress *model.StorageDataSet, - peer *model.StorageDataSet, -) (*model.StorageUpload, string) { - t.Helper() - ctx := context.Background() - versionID := model.NewVersionID() - version := &model.ObjectVersion{ - VersionID: versionID, - BucketID: bucketID, - Key: "other.txt", - Size: 12, - ETag: "other-etag", - Checksum: "other-checksum", - ContentType: "text/plain", - CacheKey: ".versions/" + versionID, - } - if _, err := env.repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { - t.Fatalf("CreateVersionAndSetCurrent(other): %v", err) - } - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark other upload uploading: %v", err) - } - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("mark other upload committing: %v", err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucketID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt(other): %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: ingress.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: ingress.ProviderID}, - {StorageDataSetID: peer.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: peer.ProviderID}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings(other): %v", err) - } - pieceCID := testCID(t).String() - for copyIndex, input := range []struct { - pieceID string - retrievalURL string - }{ - {pieceID: "401", retrievalURL: "https://ingress.example/other-piece"}, - {pieceID: "402", retrievalURL: "https://peer.example/other-piece"}, - } { - if err := env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: upload.ID, - CopyIndex: copyIndex, - PieceCID: pieceCID, - PieceID: onChainIDPtr(t, input.pieceID), - RetrievalURL: input.retrievalURL, - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted(other %d): %v", copyIndex, err) - } - } - if _, err := env.repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ - UploadID: upload.ID, - BucketID: bucketID, - ContentSize: version.Size, - Checksum: version.Checksum, - }); err != nil { - t.Fatalf("BindReadableUploadForContent(other): %v", err) - } - return upload, versionID -} - -func TestUploader_PeerUsesRetainedCacheWhenReadableProviderBecomesUnavailable(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - source, err := env.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, fixture.upload.BucketID, 0) - if err != nil || source == nil { - t.Fatalf("GetDataSetBindingByCopyIndex source: binding=%#v err=%v", source, err) - } - if err := env.repos.Uploads.MarkDataSetUnavailable(ctx, source.ID, "temporary outage"); err != nil { - t.Fatalf("MarkDataSetUnavailable source: %v", err) - } - - pieceCID := testCID(t) - peerCtx := newFakeUploadContext(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(3002), pieceCID) - peerDataSetID := sdktypes.NewBigInt(2002) - peerCtx.boundDataSet = &peerDataSetID - env.storage.SelectUploadTargetsFunc = func(context.Context, storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - return nil, errors.New("existing replica slots must not select another provider") - } - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextProviderIDEqual(opts, sdktypes.NewBigInt(202)) && createContextDataSetIDEqual(opts, sdktypes.NewBigInt(2002)) { - return peerCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - stage := "peer_pull" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "object", RefID: fixture.objID, RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:peer_pull:%d:1", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{"upload_id": fixture.upload.ID, "copy_index": 1, "transfer_method": string(model.StorageCopyTransferMethodPeerPull)}, - Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create peer task: %v", err) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - runCtx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = uploader.Run(runCtx) - close(done) - }() - defer func() { - cancel() - waitForSignal(t, done, time.Second, "uploader shutdown") - }() - - deadline := time.After(5 * time.Second) - ticker := time.NewTicker(20 * time.Millisecond) - defer ticker.Stop() - for { - select { - case <-deadline: - t.Fatal("timed out waiting for cache-backed peer commit") - case <-ticker.C: - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, 1) - if err == nil && copyRow != nil && copyRow.Status == model.StorageUploadCopyStatusCommitted { - goto committed - } - } - } - -committed: - if peerCtx.storeCalls.Load() != 1 || peerCtx.pullCalls.Load() != 0 { - t.Fatalf("peer transfer calls = store:%d pull:%d, want cache store only", peerCtx.storeCalls.Load(), peerCtx.pullCalls.Load()) - } - exhausted, err := env.repos.Tasks.ListExhausted(ctx, 10) - if err != nil { - t.Fatalf("ListExhausted: %v", err) - } - if len(exhausted) != 0 { - t.Fatalf("exhausted tasks = %#v, want none", exhausted) - } - version, err := env.repos.Objects.GetVersionByID(ctx, fixture.versionID) - if err != nil || version == nil || version.State != model.ObjectStateReplicating { - t.Fatalf("version after one available replica = %#v err=%v, want strict replicating", version, err) - } -} - -func TestUploader_QueuedPeerStageHandsUnavailableDataSetToInPlaceRepair(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - - if err := env.repos.Uploads.MarkDataSetUnavailable(ctx, fixture.peer.ID, "provider dataset retired"); err != nil { - t.Fatalf("MarkDataSetUnavailable peer: %v", err) - } - - var selectTargetsCalls atomic.Int32 - env.storage.SelectUploadTargetsFunc = func(_ context.Context, _ storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - return nil, errors.New("established slots must not select another provider") - } - - stage := "peer_pull" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: fixture.objID, - RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:peer_pull:%d:1", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{"upload_id": fixture.upload.ID, "copy_index": 1, "transfer_method": string(model.StorageCopyTransferMethodPeerPull)}, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create ensure task: %v", err) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - - gotTask, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil { - t.Fatalf("GetByID(task): %v", err) - } - if gotTask.Status != model.TaskStatusCompleted { - t.Fatalf("ensure task status = %s, want completed", gotTask.Status) - } - copies, err := env.repos.Uploads.ListCopies(ctx, fixture.upload.ID) - if err != nil { - t.Fatalf("ListCopies: %v", err) - } - if len(copies) != 2 || copies[1].CopyIndex != 1 || copies[1].Status != model.StorageUploadCopyStatusPending || copies[1].StorageDataSetID == nil || *copies[1].StorageDataSetID != fixture.peer.ID { - t.Fatalf("copies after unavailable peer ensure = %#v, want original pending peer copy", copies) - } - if got := selectTargetsCalls.Load(); got != 0 { - t.Fatalf("SelectUploadTargets calls = %d, want no provider replacement", got) - } - repairTasks, total, err := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "repair_replica", "", 10, 0) - if err != nil { - t.Fatalf("List repair tasks: %v", err) - } - if total != 1 || len(repairTasks) != 1 || repairTasks[0].RefVersionID != fixture.versionID || taskPayloadInt64ForTest(repairTasks[0].Payload, "storage_data_set_id") != fixture.peer.ID || taskPayloadInt64ForTest(repairTasks[0].Payload, "storage_upload_copy_id") != copies[1].ID { - t.Fatalf("repair tasks = %#v, want one exact in-place repair", repairTasks) - } -} - -func TestUploader_ReplicaRepairWaitsForRunningPeerOperation(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, fixture.peer.CopyIndex) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - - dataSetID := sdktypes.NewBigInt(2002) - peerCtx := newFakeUploadContext(sdktypes.NewBigInt(202), dataSetID, sdktypes.NewBigInt(3002), testCID(t)) - peerCtx.boundDataSet = &dataSetID - peerCtx.pullEntered = make(chan struct{}) - peerCtx.releasePull = make(chan struct{}) - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if !createContextDataSetIDEqual(opts, dataSetID) { - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - return peerCtx, nil - } - - peerStage := "peer_pull" - peerTask := &model.Task{ - Type: model.TaskTypeUpload, Stage: &peerStage, RefType: "object", RefID: fixture.objID, RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:peer_pull:%d:1", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{"upload_id": fixture.upload.ID, "copy_index": fixture.peer.CopyIndex, "transfer_method": string(model.StorageCopyTransferMethodPeerPull)}, - Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, peerTask); err != nil { - t.Fatalf("Create peer task: %v", err) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 2, 2, 10*time.Millisecond, slog.Default()) - workerCtx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = uploader.Run(workerCtx) - close(done) - }() - released := false - defer func() { - if !released { - close(peerCtx.releasePull) - } - cancel() - waitForSignal(t, done, time.Second, "uploader shutdown") - }() - waitForSignal(t, peerCtx.pullEntered, 5*time.Second, "ordinary peer pull") - - if err := env.repos.Uploads.MarkDataSetUnavailable(ctx, fixture.peer.ID, "temporary outage"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - repairStage := "repair_replica" - repairTask := &model.Task{ - Type: model.TaskTypeUpload, Stage: &repairStage, RefType: "bucket", RefID: fixture.upload.BucketID, - RefVersionID: fixture.versionID, IdempotencyKey: fmt.Sprintf("upload:repair-data-set:%d", fixture.peer.ID), - Payload: map[string]interface{}{"storage_data_set_id": fixture.peer.ID, "storage_upload_copy_id": copyRow.ID}, - Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, repairTask); err != nil { - t.Fatalf("Create repair task: %v", err) - } - waitForTaskStatus(t, env, repairTask.ID, model.TaskStatusWaiting, 5*time.Second) - if got := peerCtx.pullCalls.Load(); got != 1 { - t.Fatalf("pull calls while ordinary task is running = %d, want 1", got) - } - - close(peerCtx.releasePull) - released = true - waitForTaskStatus(t, env, peerTask.ID, model.TaskStatusCompleted, 5*time.Second) - peerCommitDeadline := time.After(5 * time.Second) - peerCommitTicker := time.NewTicker(10 * time.Millisecond) - defer peerCommitTicker.Stop() - for { - select { - case <-peerCommitDeadline: - t.Fatal("timed out waiting for peer commit handoff to finish") - case <-peerCommitTicker.C: - _, total, listErr := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "peer_commit", string(model.TaskStatusCompleted), 10, 0) - if listErr == nil && total == 1 { - goto peerCommitFinished - } - } - } - -peerCommitFinished: - if _, err := env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("id = ?", repairTask.ID). - Exec(ctx); err != nil { - t.Fatalf("reschedule replica repair: %v", err) - } - waitForTaskStatus(t, env, repairTask.ID, model.TaskStatusCompleted, 5*time.Second) - - if got := peerCtx.pullCalls.Load(); got != 1 { - t.Fatalf("total pull calls = %d, want one transfer for the concrete copy", got) - } - if got := peerCtx.commitCalls.Load(); got != 1 { - t.Fatalf("total commit calls = %d, want one commit for the concrete copy", got) - } -} - -func TestUploader_PeerOperationDefersWhileRecoveredDataSetHasActiveRepair(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, fixture.peer.CopyIndex) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - repairStage := "repair_replica" - repairTask := &model.Task{ - Type: model.TaskTypeUpload, Stage: &repairStage, RefType: "bucket", RefID: fixture.upload.BucketID, - RefVersionID: fixture.versionID, IdempotencyKey: fmt.Sprintf("upload:repair-data-set:%d", fixture.peer.ID), - Payload: map[string]interface{}{"storage_data_set_id": fixture.peer.ID, "storage_upload_copy_id": copyRow.ID}, - Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, repairTask); err != nil { - t.Fatalf("Create repair task: %v", err) - } - claimedRepair, err := env.repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Hour) - if err != nil || claimedRepair == nil || claimedRepair.ID != repairTask.ID { - t.Fatalf("ClaimReady repair: task=%#v err=%v", claimedRepair, err) - } - peerStage := "peer_pull" - peerTask := &model.Task{ - Type: model.TaskTypeUpload, Stage: &peerStage, RefType: "object", RefID: fixture.objID, RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:peer_pull:%d:1", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{"upload_id": fixture.upload.ID, "copy_index": fixture.peer.CopyIndex, "transfer_method": string(model.StorageCopyTransferMethodPeerPull)}, - Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, peerTask); err != nil { - t.Fatalf("Create peer task: %v", err) - } - var createContextCalls atomic.Int32 - env.storage.OpenTargetFunc = func(context.Context, *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - createContextCalls.Add(1) - return nil, errors.New("ordinary peer operation must defer to active repair") - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, env, uploader, peerTask.ID, model.TaskStatusWaiting, 5*time.Second) - gotTask, err := env.repos.Tasks.GetByID(ctx, peerTask.ID) - if err != nil || gotTask == nil || gotTask.RetryCount != 0 { - t.Fatalf("deferred peer task = %#v err=%v, want dependency wait without retry", gotTask, err) - } - if got := createContextCalls.Load(); got != 0 { - t.Fatalf("CreateContext calls = %d, want no competing storage operation", got) - } -} - -func TestUploader_IngressOperationDefersWhileDataSetHasActiveRepair(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - upload, ingressTask, _ := seedReadyPrimaryStoreTask(t, env) - binding, err := env.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, upload.BucketID, 0) - if err != nil || binding == nil { - t.Fatalf("GetDataSetBindingByCopyIndex: binding=%#v err=%v", binding, err) - } - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, upload.ID, 0) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - if _, err := env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(time.Hour)). - Where("id = ?", ingressTask.ID). - Exec(ctx); err != nil { - t.Fatalf("defer ingress task claim: %v", err) - } - repairStage := "repair_replica" - repairTask := &model.Task{ - Type: model.TaskTypeUpload, Stage: &repairStage, RefType: "bucket", RefID: upload.BucketID, - RefVersionID: upload.SourceVersionID, IdempotencyKey: fmt.Sprintf("upload:repair-data-set:%d", binding.ID), - Payload: map[string]interface{}{"storage_data_set_id": binding.ID, "storage_upload_copy_id": copyRow.ID}, - Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, repairTask); err != nil { - t.Fatalf("Create repair task: %v", err) - } - claimedRepair, err := env.repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Hour) - if err != nil || claimedRepair == nil || claimedRepair.ID != repairTask.ID { - t.Fatalf("ClaimReady repair: task=%#v err=%v", claimedRepair, err) - } - if _, err := env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("id = ?", ingressTask.ID). - Exec(ctx); err != nil { - t.Fatalf("release ingress task claim: %v", err) - } - var createContextCalls atomic.Int32 - env.storage.OpenTargetFunc = func(context.Context, *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - createContextCalls.Add(1) - return nil, errors.New("ordinary ingress operation must defer to active repair") - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 3, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, env, uploader, ingressTask.ID, model.TaskStatusWaiting, 5*time.Second) - gotTask, err := env.repos.Tasks.GetByID(ctx, ingressTask.ID) - if err != nil || gotTask == nil || gotTask.RetryCount != 0 { - t.Fatalf("deferred ingress task = %#v err=%v, want dependency wait without retry", gotTask, err) - } - if got := createContextCalls.Load(); got != 0 { - t.Fatalf("CreateContext calls = %d, want no competing storage operation", got) - } -} - -func TestUploader_RepairPreparePreservesAssignedPeerSlots(t *testing.T) { - cases := []struct { - name string - mark func(context.Context, *testing.T, *testWorkerEnv, readableUploadWithPendingPeerFixture) - wantRepair bool - }{ - { - name: "failed copy", - mark: func(ctx context.Context, t *testing.T, env *testWorkerEnv, fixture readableUploadWithPendingPeerFixture) { - t.Helper() - if err := env.repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: fixture.upload.ID, CopyIndex: 1, LastError: "peer pull: provider failed"}); err != nil { - t.Fatalf("MarkUploadCopyFailed: %v", err) - } - }, - }, - { - name: "unavailable dataset", - wantRepair: true, - mark: func(ctx context.Context, t *testing.T, env *testWorkerEnv, fixture readableUploadWithPendingPeerFixture) { - t.Helper() - if err := env.repos.Uploads.MarkDataSetUnavailable(ctx, fixture.peer.ID, "provider dataset retired"); err != nil { - t.Fatalf("MarkDataSetUnavailable peer: %v", err) - } - }, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - tc.mark(ctx, t, env, fixture) - - var selectTargetsCalls atomic.Int32 - env.storage.SelectUploadTargetsFunc = func(_ context.Context, _ storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - return nil, errors.New("assigned slots must not select another provider") - } - - stage := "prepare_upload" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: fixture.objID, - RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:prepare_upload:%d:repair", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{"upload_id": fixture.upload.ID}, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create repair task: %v", err) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - - gotTask, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil { - t.Fatalf("GetByID(task): %v", err) - } - if gotTask.Status != model.TaskStatusCompleted { - t.Fatalf("repair task status = %s, want completed", gotTask.Status) - } - copies, err := env.repos.Uploads.ListCopies(ctx, fixture.upload.ID) - if err != nil { - t.Fatalf("ListCopies: %v", err) - } - if len(copies) != 2 || copies[1].CopyIndex != 1 { - t.Fatalf("copies after repair prepare = %#v, want original assigned slots only", copies) - } - if got := selectTargetsCalls.Load(); got != 0 { - t.Fatalf("SelectUploadTargets calls = %d, want no provider replacement", got) - } - repairTasks, total, err := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "repair_replica", "", 10, 0) - if err != nil { - t.Fatalf("List repair tasks: %v", err) - } - if tc.wantRepair { - if total != 1 || len(repairTasks) != 1 || taskPayloadInt64ForTest(repairTasks[0].Payload, "storage_data_set_id") != fixture.peer.ID { - t.Fatalf("repair tasks = %#v, want one in-place repair", repairTasks) - } - } else if total != 0 { - t.Fatalf("repair tasks = %#v, want failed copy to remain operator-visible", repairTasks) - } - }) - } -} - -func TestUploader_FailedNewPeerDataSetWithServiceEvidenceIsNotReselected(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - if _, err := env.db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("requested_copies = ?", 3). - Where("id = ?", fixture.upload.ID). - Exec(ctx); err != nil { - t.Fatalf("set requested copies: %v", err) - } - failedProviderID := onChainID(t, "303") - failedBinding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: fixture.upload.BucketID, - ProviderID: failedProviderID, - CopyIndex: 2, - CreatedByUploadID: fixture.upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding failed candidate: %v", err) - } - if err := env.repos.Uploads.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ - ID: failedBinding.ID, - UploadID: fixture.upload.ID, - TransactionID: "0xcreate303", - StatusURL: "https://provider-303.example/status/create", - ClientDataSetID: onChainIDPtr(t, "10303"), - }); err != nil { - t.Fatalf("MarkDataSetCreating failed candidate: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, fixture.upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: failedBinding.ID, - CopyIndex: 2, - TransferMethod: model.StorageCopyTransferMethodPeerPull, - ProviderID: failedProviderID, - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings failed candidate: %v", err) - } - - failedCtx := newFakeUploadContext(sdktypes.NewBigInt(303), sdktypes.NewBigInt(3003), sdktypes.NewBigInt(4001), testCID(t)) - failedCtx.waitErr = fmt.Errorf("wait rejected: %w", pdp.ErrTxRejected) - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextProviderIDEqual(opts, failedCtx.providerID) { - return failedCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - var selectTargetsCalls atomic.Int32 - env.storage.SelectUploadTargetsFunc = func(_ context.Context, _ storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - return nil, errors.New("service evidence must prevent provider reselection") - } - - stage := "ensure_dataset" - ensureTask := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: fixture.objID, - RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:ensure_dataset:%d:2", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{"upload_id": fixture.upload.ID, "copy_index": 2, "transfer_method": string(model.StorageCopyTransferMethodPeerPull)}, - Status: model.TaskStatusQueued, - MaxRetries: 1, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, ensureTask); err != nil { - t.Fatalf("Create ensure task: %v", err) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, uploader, ensureTask.ID, 5*time.Second) - - repairTasks, repairTotal, err := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "prepare_upload", "", 10, 0) - if err != nil { - t.Fatalf("list repair tasks: %v", err) - } - if repairTotal != 0 || len(repairTasks) != 0 { - t.Fatalf("repair tasks = %#v, want no automatic provider replacement", repairTasks) - } - if got := selectTargetsCalls.Load(); got != 0 { - t.Fatalf("SelectUploadTargets calls = %d, want 0", got) - } - retained, err := env.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, fixture.upload.BucketID, 2) - if err != nil || retained == nil { - t.Fatalf("GetDataSetBindingByCopyIndex: binding=%v err=%v", retained, err) - } - if retained.ProviderID.String() != failedProviderID.String() || retained.Status != model.StorageDataSetStatusFailed || retained.CreateTransactionID == nil || retained.ClientDataSetID == nil { - t.Fatalf("retained binding = %#v, want failed provider 303 with service evidence", retained) - } - - copies, err := env.repos.Uploads.ListCopies(ctx, fixture.upload.ID) - if err != nil { - t.Fatalf("ListCopies: %v", err) - } - if len(copies) != 3 || copies[2].CopyIndex != 2 || copies[2].ProviderID == nil || copies[2].ProviderID.String() != failedProviderID.String() || copies[2].Status != model.StorageUploadCopyStatusFailed { - t.Fatalf("copies after rejection = %#v, want failed original provider copy retained", copies) - } - provenance, err := env.repos.Uploads.GetUploadProvenance(ctx, fixture.upload.ID) - if err != nil { - t.Fatalf("GetUploadProvenance: %v", err) - } - if len(provenance.Failures) != 1 || provenance.Failures[0].ProviderID == nil || provenance.Failures[0].ProviderID.String() != failedProviderID.String() { - t.Fatalf("provenance failures = %#v, want failed provider attempt retained", provenance.Failures) - } -} - -func TestUploader_RepairReusesAuthorizedBindingWithoutUploadCopy(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - if _, err := env.db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("requested_copies = ?", 3). - Where("id = ?", fixture.upload.ID). - Exec(ctx); err != nil { - t.Fatalf("set requested copies: %v", err) - } - - replacementProviderID := onChainID(t, "404") - if _, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: fixture.upload.BucketID, - ProviderID: replacementProviderID, - CopyIndex: 2, - CreatedByUploadID: fixture.upload.ID, - }); err != nil { - t.Fatalf("EnsureDataSetBinding replacement: %v", err) - } - - replacement := newFakeUploadContext(sdktypes.NewBigInt(404), sdktypes.NewBigInt(4004), sdktypes.NewBigInt(5001), testCID(t)) - var selectTargetsCalls atomic.Int32 - env.storage.SelectUploadTargetsFunc = func(_ context.Context, _ storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - return []synapse.StorageTarget{replacement}, nil - } - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextProviderIDEqual(opts, replacement.providerID) { - return replacement, nil - } - if createContextDataSetIDEqual(opts, replacement.dataSetID) { - return replacement, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - stage := "prepare_upload" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: fixture.objID, - RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:prepare_upload:%d:repair", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{"upload_id": fixture.upload.ID}, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create repair task: %v", err) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - - if got := selectTargetsCalls.Load(); got != 0 { - t.Fatalf("SelectUploadTargets calls = %d, want 0 when an authorized slot binding exists", got) - } - copies, err := env.repos.Uploads.ListCopies(ctx, fixture.upload.ID) - if err != nil { - t.Fatalf("ListCopies: %v", err) - } - if len(copies) != 3 || copies[2].CopyIndex != 2 || copies[2].ProviderID == nil || copies[2].ProviderID.String() != "404" { - t.Fatalf("copies after repair = %#v, want reused replacement copy index 2 from provider 404", copies) - } -} - -func TestUploader_FailedCandidateWithoutServiceEvidenceCanBeReselected(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - failedProviderID := onChainID(t, "303") - failedBinding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: fixture.upload.BucketID, - ProviderID: failedProviderID, - CopyIndex: 2, - CreatedByUploadID: fixture.upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding failed candidate: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, fixture.upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: failedBinding.ID, - CopyIndex: 2, - TransferMethod: model.StorageCopyTransferMethodPeerPull, - ProviderID: failedProviderID, - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings failed candidate: %v", err) - } - - prepareStage := "prepare_upload" - completedAt := time.Now() - completedRepair := &model.Task{ - Type: model.TaskTypeUpload, Stage: &prepareStage, RefType: "object", RefID: fixture.objID, RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:prepare_upload:%d:repair", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{"upload_id": fixture.upload.ID}, - Status: model.TaskStatusCompleted, MaxRetries: 5, ScheduledAt: completedAt, CompletedAt: &completedAt, - } - if err := env.repos.Tasks.Create(ctx, completedRepair); err != nil { - t.Fatalf("Create completed repair task: %v", err) - } - - baseUploads := env.repos.Uploads - env.repos.Uploads = &appendFailureUploadRepo{ - StorageUploadRepository: baseUploads, - err: errors.New("append failure unavailable"), - } - - failedCtx := newFakeUploadContext(sdktypes.NewBigInt(303), sdktypes.NewBigInt(3003), sdktypes.NewBigInt(4001), testCID(t)) - failedCtx.skipCreateSubmission = true - failedCtx.createErr = fmt.Errorf("create rejected: %w", pdp.ErrTxRejected) - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextProviderIDEqual(opts, failedCtx.providerID) { - return failedCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - stage := "ensure_dataset" - ensureTask := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: fixture.objID, - RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:ensure_dataset:%d:2", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{"upload_id": fixture.upload.ID, "copy_index": 2, "transfer_method": string(model.StorageCopyTransferMethodPeerPull)}, - Status: model.TaskStatusQueued, - MaxRetries: 1, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, ensureTask); err != nil { - t.Fatalf("Create ensure task: %v", err) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, uploader, ensureTask.ID, 5*time.Second) - - binding, err := baseUploads.GetDataSetBindingByCopyIndex(ctx, fixture.upload.BucketID, 2) - if err != nil { - t.Fatalf("GetDataSetBindingByCopyIndex: %v", err) - } - if binding != nil { - t.Fatalf("failed dataset candidate binding = %#v, want evidence-free candidate discarded", binding) - } - repairTasks, total, err := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "prepare_upload", "", 10, 0) - if err != nil { - t.Fatalf("List repair tasks: %v", err) - } - if total != 1 || len(repairTasks) != 1 || repairTasks[0].Status != model.TaskStatusQueued || !strings.Contains(repairTasks[0].IdempotencyKey, ":repair") { - t.Fatalf("repair tasks = %#v, want completed coordinator reactivated for the authorized slot", repairTasks) - } -} - -func TestUploader_EvidenceFreeDataSetCandidateIsNotSharedAcrossUploads(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - bucket, _, firstVersionID := seedCachedObject(t, env) - firstVersion, err := env.repos.Objects.GetVersionByID(ctx, firstVersionID) - if err != nil || firstVersion == nil { - t.Fatalf("GetVersionByID(first): version=%#v err=%v", firstVersion, err) - } - firstUpload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: firstVersionID, - ContentSize: firstVersion.Size, - Checksum: firstVersion.Checksum, - RequestedCopies: 1, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt(first): %v", err) - } - candidate, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "303"), - CopyIndex: 0, - CreatedByUploadID: firstUpload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding(candidate): %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, firstUpload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: candidate.ID, - CopyIndex: 0, - TransferMethod: model.StorageCopyTransferMethodIngress, - ProviderID: candidate.ProviderID, - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings(first): %v", err) - } - - secondVersionID := model.NewVersionID() - cacheKey := ".versions/" + secondVersionID - info, err := env.cache.Put(ctx, bucket.Name, cacheKey, strings.NewReader("second object")) - if err != nil { - t.Fatalf("cache second object: %v", err) - } - secondVersion := &model.ObjectVersion{ - VersionID: secondVersionID, - BucketID: bucket.ID, - Key: "second.txt", - Size: int64(len("second object")), - ETag: info.ETag, - Checksum: info.Checksum, - CacheKey: cacheKey, - } - secondObjectID, err := env.repos.Objects.CreateVersionAndSetCurrent(ctx, secondVersion) - if err != nil { - t.Fatalf("CreateVersionAndSetCurrent(second): %v", err) - } - prepareTask := seedStagedUploadTask(t, env, secondObjectID, secondVersionID, 5) - var selectTargetsCalls atomic.Int32 - env.storage.SelectUploadTargetsFunc = func(context.Context, storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - return nil, errors.New("occupied slot must wait for its creating upload") - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 1, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, env, uploader, prepareTask.ID, model.TaskStatusWaiting, 5*time.Second) - if got := selectTargetsCalls.Load(); got != 0 { - t.Fatalf("SelectUploadTargets calls while candidate owns slot = %d, want 0", got) - } - secondUpload, err := env.repos.Uploads.FindLatestUploadBySourceVersion(ctx, secondVersionID) - if err != nil || secondUpload == nil { - t.Fatalf("FindLatestUploadBySourceVersion(second): upload=%#v err=%v", secondUpload, err) - } - secondCopies, err := env.repos.Uploads.ListCopies(ctx, secondUpload.ID) - if err != nil || len(secondCopies) != 0 { - t.Fatalf("second upload copies before candidate resolves = %#v err=%v, want none", secondCopies, err) - } - gotTask, err := env.repos.Tasks.GetByID(ctx, prepareTask.ID) - if err != nil || gotTask == nil || gotTask.RetryCount != 0 { - t.Fatalf("second prepare task = %#v err=%v, want dependency wait without retry", gotTask, err) - } - - if err := env.repos.Uploads.MarkDataSetFailed(ctx, candidate.ID, "creation rejected"); err != nil { - t.Fatalf("MarkDataSetFailed(candidate): %v", err) - } - if err := env.repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: firstUpload.ID, CopyIndex: 0, LastError: "creation rejected"}); err != nil { - t.Fatalf("MarkUploadCopyFailed(first): %v", err) - } - discarded, err := env.repos.Uploads.DiscardFailedDataSetCandidate(ctx, firstUpload.ID, 0, candidate.ID) - if err != nil || !discarded { - t.Fatalf("DiscardFailedDataSetCandidate: discarded=%t err=%v, want safe discard", discarded, err) - } - - replacement := newFakeUploadContext(sdktypes.NewBigInt(404), sdktypes.NewBigInt(4004), sdktypes.NewBigInt(5004), testCID(t)) - env.storage.SelectUploadTargetsFunc = func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - if opts.Copies != 1 { - t.Fatalf("SelectUploadTargets copies = %d, want 1", opts.Copies) - } - return []synapse.StorageTarget{replacement}, nil - } - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextProviderIDEqual(opts, replacement.providerID) { - return replacement, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - if _, err := env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("id = ?", prepareTask.ID). - Exec(ctx); err != nil { - t.Fatalf("reschedule second prepare: %v", err) - } - runWorkerUntilTask(t, env, uploader, prepareTask.ID, 5*time.Second) - if got := selectTargetsCalls.Load(); got != 1 { - t.Fatalf("SelectUploadTargets calls after safe discard = %d, want 1", got) - } - secondCopies, err = env.repos.Uploads.ListCopies(ctx, secondUpload.ID) - if err != nil || len(secondCopies) != 1 || secondCopies[0].ProviderID == nil || secondCopies[0].ProviderID.String() != "404" { - t.Fatalf("second upload copies after reselection = %#v err=%v, want provider 404", secondCopies, err) - } -} - -type appendFailureUploadRepo struct { - repository.StorageUploadRepository - err error -} - -func (r *appendFailureUploadRepo) AppendUploadFailure(context.Context, repository.AppendUploadFailureInput) error { - return r.err -} - -func TestUploader_MissingVersion(t *testing.T) { - env := newTestWorkerEnv(t) - _, objID, _ := seedCachedObject(t, env) - - task := &model.Task{ - Type: model.TaskTypeUpload, - RefType: "object", - RefID: objID, - RefVersionID: "01J000000000000000MISSING1", - IdempotencyKey: fmt.Sprintf("upload:%d:missing", objID), - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(context.Background(), task); err != nil { - t.Fatalf("creating task: %v", err) - } - - env.storage.UploadFunc = func(_ context.Context, _ io.Reader, _ *storage.UploadOptions) (*storage.UploadResult, error) { - t.Error("upload should not be called for missing version") - return nil, errors.New("should not be called") - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 50*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - - got, _ := env.repos.Tasks.GetByID(context.Background(), task.ID) - if got.Status != model.TaskStatusFailed { - t.Errorf("expected task failed, got %s", got.Status) - } - if got.LastError == nil || !strings.Contains(*got.LastError, "object not found") { - t.Errorf("expected object not found error, got %v", got.LastError) - } -} - -func TestUploader_NilStorageClient(t *testing.T) { - env := newTestWorkerEnv(t) - _, objID, versionID := seedCachedObject(t, env) - task := seedTask(t, env, model.TaskTypeUpload, objID, versionID, 5, 0) - - uploader := worker.NewUploader(env.repos, env.cache, nil, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 50*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - - got, _ := env.repos.Tasks.GetByID(context.Background(), task.ID) - if got.Status != model.TaskStatusFailed { - t.Errorf("expected task failed, got %s", got.Status) - } - if got.LastError == nil || !strings.Contains(*got.LastError, "storage client not configured") { - t.Errorf("expected storage client error, got %v", got.LastError) - } -} - -func TestUploader_StagedPrimaryStoreCacheMissMarksCacheLocationAbsent(t *testing.T) { - mc := &testutil.MockCache{ - GetFunc: func(_ context.Context, _, _ string) (io.ReadCloser, *cache.ObjectInfo, error) { - return nil, nil, os.ErrNotExist - }, - } - env := newTestWorkerEnvWithMockCache(t, mc) - bucket, objID, versionID := seedObjectInDB(t, env, model.BucketStatusActive) - ctx := context.Background() - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark uploading: %v", err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, - SourceVersionID: versionID, - ContentSize: version.Size, - Checksum: version.Checksum, - RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, - ProviderID: onChainID(t, "101"), - CopyIndex: 0, - CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding: %v", err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, - UploadID: upload.ID, - DataSetID: onChainID(t, "1001"), - ClientDataSetID: onChainIDPtr(t, "11001"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{ - {StorageDataSetID: binding.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101")}, - }); err != nil { - t.Fatalf("CreateUploadCopiesForBindings: %v", err) - } - stage := "ingress_store" - task := &model.Task{ - Type: model.TaskTypeUpload, - Stage: &stage, - RefType: "object", - RefID: objID, - RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:ingress_store:%d", versionID, upload.ID), - Payload: map[string]interface{}{"upload_id": upload.ID, "copy_index": 0, "transfer_method": string(model.StorageCopyTransferMethodIngress)}, - Status: model.TaskStatusQueued, - MaxRetries: 5, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("create primary store task: %v", err) - } - primaryCtx := newFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(2001), testCID(t)) - dataSetID := sdktypes.NewBigInt(1001) - primaryCtx.boundDataSet = &dataSetID - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextDataSetIDEqual(opts, sdktypes.NewBigInt(1001)) { - return primaryCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 50*time.Millisecond, slog.Default()) - runWorkerUntilTaskRetryCount(t, env, uploader, task.ID, 1, 5*time.Second) - - got, _ := env.repos.Tasks.GetByID(ctx, task.ID) - if got.Status != model.TaskStatusScheduled { - t.Errorf("expected task scheduled for retry, got %s", got.Status) - } - obj, _ := env.repos.Objects.GetCurrentVersionByObjectID(ctx, objID) - if obj.State != model.ObjectStateUploading { - t.Errorf("expected object state uploading after cache miss retry, got %s", obj.State) - } - if obj.InCache { - t.Error("expected current object cache location to be false after staged cache miss") - } - version, _ = env.repos.Objects.GetVersionByID(ctx, versionID) - if version.InCache { - t.Error("expected version cache location to be false after staged cache miss") - } -} - -func TestUploader_IngressProviderFailureClassification(t *testing.T) { - type ingressFixture struct { - env *testWorkerEnv - bucket *model.Bucket - objectID int64 - versionID string - upload *model.StorageUpload - task *model.Task - bindings []*model.StorageDataSet - contexts []*fakeUploadContext - } - seed := func(t *testing.T, copies, readyCopies int) ingressFixture { - t.Helper() - env := newTestWorkerEnv(t) - bucket, objID, versionID := seedCachedObject(t, env) - ctx := context.Background() - if err := env.repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { - t.Fatalf("mark uploading: %v", err) - } - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%v err=%v", version, err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: versionID, ContentSize: version.Size, Checksum: version.Checksum, RequestedCopies: copies, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - fixture := ingressFixture{env: env, bucket: bucket, objectID: objID, versionID: versionID, upload: upload} - for copyIndex := range copies { - providerID := uint64(101 + copyIndex*101) - dataSetID := uint64(1001 + copyIndex*1001) - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, strconv.FormatUint(providerID, 10)), CopyIndex: copyIndex, CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding(%d): %v", copyIndex, err) - } - if copyIndex < readyCopies { - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, UploadID: upload.ID, DataSetID: onChainID(t, strconv.FormatUint(dataSetID, 10)), - }); err != nil { - t.Fatalf("MarkDataSetReady(%d): %v", copyIndex, err) - } - } - method := model.StorageCopyTransferMethodPeerPull - if copyIndex == 0 { - method = model.StorageCopyTransferMethodIngress - } - if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ - StorageDataSetID: binding.ID, CopyIndex: copyIndex, TransferMethod: method, ProviderID: onChainID(t, strconv.FormatUint(providerID, 10)), - }}); err != nil { - t.Fatalf("CreateUploadCopiesForBindings(%d): %v", copyIndex, err) - } - storageCtx := newFakeUploadContext(sdktypes.NewBigInt(providerID), sdktypes.NewBigInt(dataSetID), sdktypes.NewBigInt(uint64(2001+copyIndex)), testCID(t)) - boundID := sdktypes.NewBigInt(dataSetID) - storageCtx.boundDataSet = &boundID - fixture.bindings = append(fixture.bindings, binding) - fixture.contexts = append(fixture.contexts, storageCtx) - } - stage := "ingress_store" - fixture.task = &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "object", RefID: objID, RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:ingress_store:%d", versionID, upload.ID), - Payload: map[string]interface{}{"upload_id": upload.ID, "copy_index": 0, "transfer_method": string(model.StorageCopyTransferMethodIngress)}, - Status: model.TaskStatusQueued, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, fixture.task); err != nil { - t.Fatalf("create ingress store task: %v", err) - } - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - for _, storageCtx := range fixture.contexts { - if createContextDataSetIDEqual(opts, storageCtx.dataSetID) || createContextProviderIDEqual(opts, storageCtx.providerID) { - return storageCtx, nil - } - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - return fixture - } - - t.Run("temporary outage reassigns ingress without replacing provider", func(t *testing.T) { - fixture := seed(t, 2, 2) - fixture.contexts[0].storeErr = &synapse.ProviderUnavailableError{Cause: errors.New("provider unavailable")} - var selectTargetsCalls atomic.Int32 - fixture.env.storage.SelectUploadTargetsFunc = func(_ context.Context, _ storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - return nil, errors.New("existing slots must not be replaced") - } - uploader := worker.NewUploader(fixture.env.repos, fixture.env.cache, fixture.env.storage, nil, fixture.env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, fixture.env, uploader, fixture.task.ID, 5*time.Second) - - gotTask, _ := fixture.env.repos.Tasks.GetByID(context.Background(), fixture.task.ID) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("ingress task = %#v, want completed without retry", gotTask) - } - binding, _ := fixture.env.repos.Uploads.GetDataSetBindingByID(context.Background(), fixture.bindings[0].ID) - if binding.Status != model.StorageDataSetStatusUnavailable { - t.Fatalf("failed ingress binding status = %s, want unavailable", binding.Status) - } - copies, err := fixture.env.repos.Uploads.ListCopies(context.Background(), fixture.upload.ID) - if err != nil || len(copies) != 2 || copies[0].TransferMethod != model.StorageCopyTransferMethodPeerPull || copies[1].TransferMethod != model.StorageCopyTransferMethodIngress { - t.Fatalf("reassigned copies = %#v err=%v", copies, err) - } - repairTasks, total, err := fixture.env.repos.Tasks.List(context.Background(), string(model.TaskTypeUpload), "repair_replica", "", 10, 0) - if err != nil || total != 1 || taskPayloadInt64ForTest(repairTasks[0].Payload, "storage_upload_copy_id") != copies[0].ID { - t.Fatalf("repair tasks = %#v total=%d err=%v, want original ingress copy", repairTasks, total, err) - } - ensureTasks, total, err := fixture.env.repos.Tasks.List(context.Background(), string(model.TaskTypeUpload), "ensure_dataset", "", 10, 0) - if err != nil || total != 1 || taskPayloadInt64ForTest(ensureTasks[0].Payload, "copy_index") != 1 { - t.Fatalf("ensure tasks = %#v total=%d err=%v, want reassigned slot 1", ensureTasks, total, err) - } - if got := selectTargetsCalls.Load(); got != 0 { - t.Fatalf("SelectUploadTargets calls = %d, want no topology change", got) - } - }) - - t.Run("committed ingress recovers after live provider evidence", func(t *testing.T) { - fixture := seed(t, 1, 1) - ctx := context.Background() - if err := fixture.env.repos.Objects.UpdateVersionState(ctx, fixture.versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("mark committing: %v", err) - } - if err := fixture.env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ - UploadID: fixture.upload.ID, CopyIndex: 0, PieceCID: testCID(t).String(), PieceID: onChainIDPtr(t, "2001"), RetrievalURL: "https://primary.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyCommitted: %v", err) - } - if err := fixture.env.repos.Uploads.MarkDataSetUnavailable(ctx, fixture.bindings[0].ID, "crashed after commit"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - stage := "ingress_commit" - if _, err := fixture.env.db.NewUpdate().Model((*model.Task)(nil)). - Set("stage = ?", stage). - Set("payload = ?", map[string]interface{}{"upload_id": fixture.upload.ID, "copy_index": 0, "transfer_method": string(model.StorageCopyTransferMethodIngress)}). - Where("id = ?", fixture.task.ID). - Exec(ctx); err != nil { - t.Fatalf("prepare committed ingress task: %v", err) - } - providerAvailable := atomic.Bool{} - fixture.env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if !createContextDataSetIDEqual(opts, fixture.contexts[0].dataSetID) { - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - if !providerAvailable.Load() { - return nil, &synapse.ProviderUnavailableError{Cause: context.DeadlineExceeded} - } - return fixture.contexts[0], nil - } - uploader := worker.NewUploader(fixture.env.repos, fixture.env.cache, fixture.env.storage, nil, fixture.env.sm, cache.EvictionPolicyAfterUpload, 1, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, fixture.env, uploader, fixture.task.ID, model.TaskStatusWaiting, 5*time.Second) - gotTask, err := fixture.env.repos.Tasks.GetByID(ctx, fixture.task.ID) - if err != nil || gotTask.RetryCount != 0 { - t.Fatalf("committed ingress while unavailable = %#v err=%v, want wait without retry", gotTask, err) - } - providerAvailable.Store(true) - if _, err := fixture.env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("id = ?", fixture.task.ID). - Exec(ctx); err != nil { - t.Fatalf("reschedule committed ingress: %v", err) - } - gotTask = runWorkerUntilTask(t, fixture.env, uploader, fixture.task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("recovered committed ingress task = %#v, want completed without retry", gotTask) - } - version, err := fixture.env.repos.Objects.GetVersionByID(ctx, fixture.versionID) - if err != nil || version == nil || version.State != model.ObjectStateStored { - t.Fatalf("recovered committed version = %#v err=%v, want stored", version, err) - } - binding, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.bindings[0].ID) - if err != nil || binding == nil || binding.Status != model.StorageDataSetStatusReady { - t.Fatalf("recovered committed binding = %#v err=%v, want ready", binding, err) - } - if fixture.contexts[0].storeCalls.Load() != 0 || fixture.contexts[0].commitCalls.Load() != 0 { - t.Fatalf("committed recovery repeated storage operation: store=%d commit=%d", fixture.contexts[0].storeCalls.Load(), fixture.contexts[0].commitCalls.Load()) - } - }) - - t.Run("submitted ingress waits without handoff", func(t *testing.T) { - fixture := seed(t, 2, 2) - ctx := context.Background() - if err := fixture.env.repos.Objects.UpdateVersionState(ctx, fixture.versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("mark committing: %v", err) - } - if err := fixture.env.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: fixture.upload.ID, CopyIndex: 0, PieceCID: testCID(t).String(), RetrievalURL: "https://primary.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - copyRow, err := fixture.env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, 0) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - seedSubmittedCommitAttempt(t, fixture.env.repos, copyRow, "submitted-ingress", "01", fakeSubmittedCommitTxHash) - if err := fixture.env.repos.Uploads.MarkDataSetUnavailable(ctx, fixture.bindings[0].ID, "provider timeout after submit"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - stage := "ingress_commit" - if _, err := fixture.env.db.NewUpdate().Model((*model.Task)(nil)). - Set("stage = ?", stage). - Set("payload = ?", map[string]interface{}{"upload_id": fixture.upload.ID, "copy_index": 0, "transfer_method": string(model.StorageCopyTransferMethodIngress)}). - Where("id = ?", fixture.task.ID). - Exec(ctx); err != nil { - t.Fatalf("prepare submitted commit task: %v", err) - } - uploader := worker.NewUploader(fixture.env.repos, fixture.env.cache, fixture.env.storage, nil, fixture.env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, fixture.env, uploader, fixture.task.ID, model.TaskStatusWaiting, 5*time.Second) - - gotTask, err := fixture.env.repos.Tasks.GetByID(ctx, fixture.task.ID) - if err != nil || gotTask.RetryCount != 0 || gotTask.WaitReason == nil || *gotTask.WaitReason != model.TaskWaitReasonExternalConfirmation { - t.Fatalf("submitted ingress task = %#v err=%v, want confirmation wait without retry", gotTask, err) - } - copies, err := fixture.env.repos.Uploads.ListCopies(ctx, fixture.upload.ID) - if err != nil || len(copies) != 2 || copies[0].TransferMethod != model.StorageCopyTransferMethodIngress || copies[1].TransferMethod != model.StorageCopyTransferMethodPeerPull { - t.Fatalf("copies with submitted ingress = %#v err=%v, want unchanged roles", copies, err) - } - }) - - t.Run("alternate failure preserves earlier submitted commit", func(t *testing.T) { - fixture := seed(t, 2, 2) - ctx := context.Background() - if err := fixture.env.repos.Objects.UpdateVersionState(ctx, fixture.versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("mark committing: %v", err) - } - if err := fixture.env.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: fixture.upload.ID, CopyIndex: 0, PieceCID: testCID(t).String(), RetrievalURL: "https://primary.example/piece", - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - copyRow, err := fixture.env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, 0) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - seedSubmittedCommitAttempt(t, fixture.env.repos, copyRow, "preserved-ingress", "01", fakeSubmittedCommitTxHash) - if err := fixture.env.repos.Uploads.MarkDataSetUnavailable(ctx, fixture.bindings[0].ID, "legacy handoff state"); err != nil { - t.Fatalf("MarkDataSetUnavailable: %v", err) - } - if _, err := fixture.env.db.NewUpdate().Model((*model.StorageUploadCopy)(nil)). - Set("transfer_method = CASE WHEN copy_index = 0 THEN ? ELSE ? END", model.StorageCopyTransferMethodPeerPull, model.StorageCopyTransferMethodIngress). - Where("upload_id = ?", fixture.upload.ID). - Exec(ctx); err != nil { - t.Fatalf("seed legacy handoff roles: %v", err) - } - if _, err := fixture.env.db.NewUpdate().Model((*model.Task)(nil)). - Set("payload = ?", map[string]interface{}{"upload_id": fixture.upload.ID, "copy_index": 1, "transfer_method": string(model.StorageCopyTransferMethodIngress)}). - Set("max_retries = 1"). - Where("id = ?", fixture.task.ID). - Exec(ctx); err != nil { - t.Fatalf("prepare alternate ingress task: %v", err) - } - fixture.contexts[1].storeErr = errors.New("unexpected alternate provider response") - uploader := worker.NewUploader(fixture.env.repos, fixture.env.cache, fixture.env.storage, nil, fixture.env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - gotTask := runWorkerUntilTask(t, fixture.env, uploader, fixture.task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusExhausted { - t.Fatalf("alternate ingress task = %#v, want exhausted", gotTask) - } - version, err := fixture.env.repos.Objects.GetVersionByID(ctx, fixture.versionID) - if err != nil || version == nil || version.State != model.ObjectStateCommitting || version.FailedAtState != nil { - t.Fatalf("version with submitted commit = %#v err=%v, want committing", version, err) - } - upload, err := fixture.env.repos.Uploads.GetByID(ctx, fixture.upload.ID) - if err != nil || upload == nil || upload.Status == model.StorageUploadStatusFailed { - t.Fatalf("upload with submitted commit = %#v err=%v, must not fail", upload, err) - } - copies, err := fixture.env.repos.Uploads.ListCopies(ctx, fixture.upload.ID) - if err != nil || len(copies) != 2 || !copyCommitSubmittedForTest(&copies[0]) || copies[1].Status != model.StorageUploadCopyStatusFailed { - t.Fatalf("copies after alternate exhaustion = %#v err=%v, want submitted source and failed alternate", copies, err) - } - }) - - t.Run("handoff rollback keeps ingress retryable", func(t *testing.T) { - fixture := seed(t, 2, 2) - ctx := context.Background() - if err := fixture.env.repos.Objects.UpdateVersionState(ctx, fixture.versionID, model.ObjectStateUploading, model.ObjectStateCommitting); err != nil { - t.Fatalf("mark committing: %v", err) - } - var changed atomic.Bool - fixture.env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextDataSetIDEqual(opts, fixture.contexts[0].dataSetID) { - if changed.CompareAndSwap(false, true) { - if err := fixture.env.repos.Objects.UpdateVersionState(ctx, fixture.versionID, model.ObjectStateCommitting, model.ObjectStateUploading); err != nil { - t.Fatalf("concurrent state change: %v", err) - } - } - return nil, &synapse.ProviderUnavailableError{Cause: context.DeadlineExceeded} - } - for _, storageCtx := range fixture.contexts { - if createContextDataSetIDEqual(opts, storageCtx.dataSetID) || createContextProviderIDEqual(opts, storageCtx.providerID) { - return storageCtx, nil - } - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - uploader := worker.NewUploader(fixture.env.repos, fixture.env.cache, fixture.env.storage, nil, fixture.env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTaskRetryCount(t, fixture.env, uploader, fixture.task.ID, 1, 5*time.Second) - - copies, err := fixture.env.repos.Uploads.ListCopies(ctx, fixture.upload.ID) - if err != nil || len(copies) != 2 || copies[0].TransferMethod != model.StorageCopyTransferMethodIngress || copies[1].TransferMethod != model.StorageCopyTransferMethodPeerPull { - t.Fatalf("copies after rolled-back handoff = %#v err=%v, want original roles", copies, err) - } - ensureTasks, total, err := fixture.env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "ensure_dataset", "", 10, 0) - if err != nil || total != 0 { - t.Fatalf("ensure tasks after rolled-back handoff = %#v total=%d err=%v, want none", ensureTasks, total, err) - } - staleStage := "ensure_dataset" - staleTask := &model.Task{ - Type: model.TaskTypeUpload, Stage: &staleStage, RefType: "object", RefID: fixture.objectID, RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:ensure_dataset:%d:1", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{"upload_id": fixture.upload.ID, "copy_index": 1, "transfer_method": string(model.StorageCopyTransferMethodPeerPull)}, - Status: model.TaskStatusCompleted, MaxRetries: 5, ScheduledAt: time.Now(), - } - if err := fixture.env.repos.Tasks.Create(ctx, staleTask); err != nil { - t.Fatalf("create stale completed ensure task: %v", err) - } - if _, err := fixture.env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("id = ?", fixture.task.ID). - Exec(ctx); err != nil { - t.Fatalf("reschedule ingress retry: %v", err) - } - gotTask := runWorkerUntilTask(t, fixture.env, uploader, fixture.task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 1 { - t.Fatalf("retried ingress task = %#v, want completed after one bounded retry", gotTask) - } - copies, err = fixture.env.repos.Uploads.ListCopies(ctx, fixture.upload.ID) - if err != nil || len(copies) != 2 || copies[0].TransferMethod != model.StorageCopyTransferMethodPeerPull || copies[1].TransferMethod != model.StorageCopyTransferMethodIngress { - t.Fatalf("copies after retried handoff = %#v err=%v, want one alternate ingress", copies, err) - } - ensureTasks, total, err = fixture.env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "ensure_dataset", "", 10, 0) - if err != nil || total != 1 || taskPayloadInt64ForTest(ensureTasks[0].Payload, "copy_index") != 1 || ensureTasks[0].Payload["transfer_method"] != string(model.StorageCopyTransferMethodIngress) { - t.Fatalf("ensure tasks after retried handoff = %#v total=%d err=%v, want alternate ingress", ensureTasks, total, err) - } - }) - - t.Run("temporary outage without alternate hands off to one coordinator", func(t *testing.T) { - fixture := seed(t, 1, 1) - fixture.contexts[0].storeErr = &synapse.ProviderUnavailableError{Cause: errors.New("provider unavailable")} - uploader := worker.NewUploader(fixture.env.repos, fixture.env.cache, fixture.env.storage, nil, fixture.env.sm, cache.EvictionPolicyAfterUpload, 1, 1, 10*time.Millisecond, slog.Default()) - gotTask := runWorkerUntilTask(t, fixture.env, uploader, fixture.task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("ingress task = %#v, want handoff without retry", gotTask) - } - copyRow, err := fixture.env.repos.Uploads.GetUploadCopy(context.Background(), fixture.upload.ID, 0) - if err != nil || copyRow == nil || copyRow.Status != model.StorageUploadCopyStatusPending { - t.Fatalf("original copy = %#v err=%v, want pending for in-place repair", copyRow, err) - } - repairTasks, total, err := fixture.env.repos.Tasks.List(context.Background(), string(model.TaskTypeUpload), "repair_replica", "", 10, 0) - if err != nil || total != 1 || taskPayloadInt64ForTest(repairTasks[0].Payload, "storage_upload_copy_id") != copyRow.ID { - t.Fatalf("repair tasks = %#v total=%d err=%v, want the only remaining writer", repairTasks, total, err) - } - }) - - t.Run("recovered ingress schedules remaining pending peer", func(t *testing.T) { - fixture := seed(t, 2, 1) - ctx := context.Background() - fixture.contexts[0].storeErr = &synapse.ProviderUnavailableError{Cause: errors.New("provider unavailable")} - var selectTargetsCalls atomic.Int32 - fixture.env.storage.SelectUploadTargetsFunc = func(_ context.Context, _ storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - return nil, errors.New("existing slots must not be replaced") - } - uploader := worker.NewUploader(fixture.env.repos, fixture.env.cache, fixture.env.storage, nil, fixture.env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - gotIngress := runWorkerUntilTask(t, fixture.env, uploader, fixture.task.ID, 5*time.Second) - if gotIngress.Status != model.TaskStatusCompleted || gotIngress.RetryCount != 0 { - t.Fatalf("ingress task = %#v, want coordinator handoff without retry", gotIngress) - } - - repairTasks, total, err := fixture.env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "repair_replica", "", 10, 0) - if err != nil || total != 1 { - t.Fatalf("repair tasks = %#v total=%d err=%v, want one ingress coordinator", repairTasks, total, err) - } - peerBinding, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.bindings[1].ID) - if err != nil || peerBinding == nil || peerBinding.Status != model.StorageDataSetStatusPending { - t.Fatalf("peer binding before recovery = %#v err=%v, want pending", peerBinding, err) - } - - fixture.contexts[0].storeErr = nil - if _, err := fixture.env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("id = ?", repairTasks[0].ID). - Exec(ctx); err != nil { - t.Fatalf("reschedule recovered ingress: %v", err) - } - runCtx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = uploader.Run(runCtx) - close(done) - }() - defer func() { - cancel() - waitForSignal(t, done, time.Second, "uploader shutdown") - }() - waitForObjectState(t, fixture.env, fixture.versionID, model.ObjectStateStored, 5*time.Second) - - peerCopy, err := fixture.env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, 1) - if err != nil || peerCopy == nil || peerCopy.Status != model.StorageUploadCopyStatusCommitted { - t.Fatalf("peer copy after ingress recovery = %#v err=%v, want committed", peerCopy, err) - } - ensureTasks, total, err := fixture.env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "ensure_dataset", "", 10, 0) - if err != nil || total != 1 { - t.Fatalf("peer ensure tasks = %#v total=%d err=%v, want one", ensureTasks, total, err) - } - if ensureTasks[0].RefType != "object" || ensureTasks[0].RefID != fixture.objectID || ensureTasks[0].RefVersionID != fixture.versionID || taskPayloadInt64ForTest(ensureTasks[0].Payload, "copy_index") != 1 { - t.Fatalf("peer ensure task = %#v, want exact source object and pending slot", ensureTasks[0]) - } - if got := selectTargetsCalls.Load(); got != 0 { - t.Fatalf("SelectUploadTargets calls = %d, want no topology change", got) - } - }) - - t.Run("ended service waits for operator replacement", func(t *testing.T) { - fixture := seed(t, 1, 1) - fixture.contexts[0].storeErr = &storage.DataSetPDPPaymentTerminatedError{DataSetID: sdktypes.NewBigInt(1001), PDPEndEpoch: 3778900} - uploader := worker.NewUploader(fixture.env.repos, fixture.env.cache, fixture.env.storage, nil, fixture.env.sm, cache.EvictionPolicyAfterUpload, 1, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, fixture.env, uploader, fixture.task.ID, model.TaskStatusWaiting, 5*time.Second) - gotTask, _ := fixture.env.repos.Tasks.GetByID(context.Background(), fixture.task.ID) - if gotTask.RetryCount != 0 || gotTask.WaitReason == nil || *gotTask.WaitReason != model.TaskWaitReasonDependency { - t.Fatalf("ended-service task = %#v, want dependency wait without retry", gotTask) - } - binding, _ := fixture.env.repos.Uploads.GetDataSetBindingByID(context.Background(), fixture.bindings[0].ID) - if binding.Status != model.StorageDataSetStatusDraining { - t.Fatalf("ended service binding status = %s, want draining", binding.Status) - } - }) - - t.Run("concurrent lifecycle change converges without retry", func(t *testing.T) { - fixture := seed(t, 1, 1) - var changed atomic.Bool - fixture.env.storage.OpenTargetFunc = func(_ context.Context, _ *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if changed.CompareAndSwap(false, true) { - if err := fixture.env.repos.Uploads.MarkDataSetDraining(context.Background(), fixture.bindings[0].ID, "service ended concurrently"); err != nil { - t.Fatalf("MarkDataSetDraining: %v", err) - } - } - return nil, &synapse.ProviderUnavailableError{Cause: context.DeadlineExceeded} - } - uploader := worker.NewUploader(fixture.env.repos, fixture.env.cache, fixture.env.storage, nil, fixture.env.sm, cache.EvictionPolicyAfterUpload, 1, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, fixture.env, uploader, fixture.task.ID, model.TaskStatusWaiting, 5*time.Second) - - gotTask, err := fixture.env.repos.Tasks.GetByID(context.Background(), fixture.task.ID) - if err != nil || gotTask.RetryCount != 0 || gotTask.WaitReason == nil || *gotTask.WaitReason != model.TaskWaitReasonDependency { - t.Fatalf("task after lifecycle conflict = %#v err=%v, want dependency wait without retry", gotTask, err) - } - binding, err := fixture.env.repos.Uploads.GetDataSetBindingByID(context.Background(), fixture.bindings[0].ID) - if err != nil || binding == nil || binding.Status != model.StorageDataSetStatusDraining { - t.Fatalf("binding after lifecycle conflict = %#v err=%v, want concurrent draining state", binding, err) - } - }) - - t.Run("unknown error uses bounded retry without lifecycle change", func(t *testing.T) { - fixture := seed(t, 1, 1) - fixture.contexts[0].storeErr = errors.New("unexpected provider response") - uploader := worker.NewUploader(fixture.env.repos, fixture.env.cache, fixture.env.storage, nil, fixture.env.sm, cache.EvictionPolicyAfterUpload, 1, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTaskRetryCount(t, fixture.env, uploader, fixture.task.ID, 1, 5*time.Second) - gotTask, _ := fixture.env.repos.Tasks.GetByID(context.Background(), fixture.task.ID) - if gotTask.Status != model.TaskStatusScheduled || gotTask.RetryCount != 1 { - t.Fatalf("unknown-error task = %#v, want bounded retry", gotTask) - } - binding, _ := fixture.env.repos.Uploads.GetDataSetBindingByID(context.Background(), fixture.bindings[0].ID) - if binding.Status != model.StorageDataSetStatusReady { - t.Fatalf("unknown-error binding status = %s, want ready", binding.Status) - } - }) -} - -func TestUploader_UnestablishedCandidateOutageWaitsWithoutReplicaRepair(t *testing.T) { - env := newTestWorkerEnv(t) - _, objID, versionID := seedCachedObject(t, env) - ctx := context.Background() - task := seedStagedUploadTask(t, env, objID, versionID, 5) - candidate := newFakeUploadContext( - sdktypes.NewBigInt(101), - sdktypes.NewBigInt(1001), - sdktypes.NewBigInt(2001), - testCID(t), - ) - env.storage.SelectUploadTargetsFunc = func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - if opts.Copies != 1 { - t.Fatalf("SelectUploadTargets copies = %d, want one authorized slot", opts.Copies) - } - return []synapse.StorageTarget{candidate}, nil - } - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if !createContextProviderIDEqual(opts, sdktypes.NewBigInt(101)) || opts.DataSetID != nil { - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - return nil, &synapse.ProviderUnavailableError{Cause: errors.New("candidate provider unavailable")} - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 1, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, env, uploader, task.ID, model.TaskStatusWaiting, 5*time.Second) - gotTask, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || gotTask.RetryCount != 0 || gotTask.WaitReason == nil || *gotTask.WaitReason != model.TaskWaitReasonDependency { - t.Fatalf("candidate task = %#v err=%v, want dependency wait without retry", gotTask, err) - } - upload, err := env.repos.Uploads.FindLatestUploadBySourceVersion(ctx, versionID) - if err != nil || upload == nil { - t.Fatalf("FindLatestUploadBySourceVersion: upload=%#v err=%v", upload, err) - } - bindings, err := env.repos.Uploads.ListDataSetBindings(ctx, upload.BucketID) - if err != nil || len(bindings) != 1 || bindings[0].Status != model.StorageDataSetStatusPending || bindings[0].DataSetID != nil { - t.Fatalf("candidate bindings = %#v err=%v, want one unestablished pending slot", bindings, err) - } - copies, err := env.repos.Uploads.ListCopies(ctx, upload.ID) - if err != nil || len(copies) != 1 || copies[0].Status != model.StorageUploadCopyStatusPending || copies[0].StorageDataSetID == nil || *copies[0].StorageDataSetID != bindings[0].ID { - t.Fatalf("candidate copies = %#v err=%v, want one pending assigned copy", copies, err) - } - repairTasks, total, err := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "repair_replica", "", 10, 0) - if err != nil || total != 0 || len(repairTasks) != 0 { - t.Fatalf("repair tasks = %#v total=%d err=%v, want none for unestablished candidate", repairTasks, total, err) - } -} - -func TestUploader_SPUploadFailure_Retry(t *testing.T) { - env := newTestWorkerEnv(t) - _, objID, versionID := seedCachedObject(t, env) - task := seedStagedUploadTask(t, env, objID, versionID, 5) - - env.storage.SelectUploadTargetsFunc = func(_ context.Context, _ storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - return nil, errors.New("SP unavailable") - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 50*time.Millisecond, slog.Default()) - runWorkerUntilTaskRetryCount(t, env, uploader, task.ID, 1, time.Second) - - got, _ := env.repos.Tasks.GetByID(context.Background(), task.ID) - if got == nil { - t.Fatal("expected task after retry") - } - if got.Status != model.TaskStatusScheduled { - t.Errorf("expected task scheduled for retry, got %s", got.Status) - } - if got.RetryCount != 1 { - t.Errorf("expected retry_count=1, got %d", got.RetryCount) - } -} - -func TestUploader_SPUploadFailure_MaxRetries(t *testing.T) { - env := newTestWorkerEnv(t) - _, objID, versionID := seedCachedObject(t, env) - - task := seedStagedUploadTask(t, env, objID, versionID, 5) - if _, err := env.db.NewUpdate(). - Model((*model.Task)(nil)). - Set("retry_count = ?", 4). - Where("id = ?", task.ID). - Exec(context.Background()); err != nil { - t.Fatalf("set retry count: %v", err) - } - task.RetryCount = 4 - - env.storage.SelectUploadTargetsFunc = func(_ context.Context, _ storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - return nil, errors.New("SP permanent failure") - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 50*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - - ctx := context.Background() - got, _ := env.repos.Tasks.GetByID(ctx, task.ID) - if got.Status != model.TaskStatusExhausted { - t.Errorf("expected task exhausted, got %s", got.Status) - } - - obj, _ := env.repos.Objects.GetCurrentVersionByObjectID(ctx, objID) - if obj.State != model.ObjectStateUploading { - t.Errorf("expected object state uploading after prepare exhaustion, got %s", obj.State) - } -} - -func TestUploader_AllAssignedProvidersUnavailableWaitsWithoutRetry(t *testing.T) { - env := newTestWorkerEnv(t) - bucket, objID, versionID := seedCachedObject(t, env) - ctx := context.Background() - seedUpload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: "01J000000000000000SEEDSLOT", ContentSize: 1, Checksum: "seed-slots", RequestedCopies: 3, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt seed: %v", err) - } - bindings := make([]*model.StorageDataSet, 0, 3) - for copyIndex, ids := range [][2]string{{"101", "1001"}, {"202", "2002"}, {"303", "3003"}} { - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, ids[0]), CopyIndex: copyIndex, CreatedByUploadID: seedUpload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding(%d): %v", copyIndex, err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, UploadID: seedUpload.ID, DataSetID: onChainID(t, ids[1])}); err != nil { - t.Fatalf("MarkDataSetReady(%d): %v", copyIndex, err) - } - if err := env.repos.Uploads.MarkDataSetUnavailable(ctx, binding.ID, "temporary outage"); err != nil { - t.Fatalf("MarkDataSetUnavailable(%d): %v", copyIndex, err) - } - bindings = append(bindings, binding) - } - task := seedStagedUploadTask(t, env, objID, versionID, 5) - var selectTargetsCalls atomic.Int32 - var providerRecovered atomic.Bool - recoveredContext := newFakeUploadContext( - sdktypes.NewBigInt(101), - sdktypes.NewBigInt(1001), - sdktypes.NewBigInt(2001), - testCID(t), - ) - recoveredDataSetID := sdktypes.NewBigInt(1001) - recoveredContext.boundDataSet = &recoveredDataSetID - env.storage.SelectUploadTargetsFunc = func(_ context.Context, _ storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - return nil, errors.New("assigned slots must not select replacement providers") - } - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if providerRecovered.Load() && createContextDataSetIDEqual(opts, recoveredDataSetID) { - return recoveredContext, nil - } - return nil, &synapse.ProviderUnavailableError{Cause: errors.New("provider still unavailable")} - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 3, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, env, uploader, task.ID, model.TaskStatusWaiting, 5*time.Second) - gotTask, err := env.repos.Tasks.GetByID(ctx, task.ID) - if err != nil || gotTask.RetryCount != 0 || gotTask.WaitReason == nil || *gotTask.WaitReason != model.TaskWaitReasonDependency { - t.Fatalf("waiting task = %#v err=%v", gotTask, err) - } - if got := selectTargetsCalls.Load(); got != 0 { - t.Fatalf("SelectUploadTargets calls = %d, want no topology change", got) - } - upload, err := env.repos.Uploads.FindLatestUploadBySourceVersion(ctx, versionID) - if err != nil || upload == nil { - t.Fatalf("FindLatestUploadBySourceVersion: upload=%#v err=%v", upload, err) - } - copies, err := env.repos.Uploads.ListCopies(ctx, upload.ID) - if err != nil || len(copies) != 3 { - t.Fatalf("assigned copies = %#v err=%v, want all three slots", copies, err) - } - for _, copyRow := range copies { - if copyRow.Status != model.StorageUploadCopyStatusPending || copyRow.StorageDataSetID == nil { - t.Fatalf("assigned copy = %#v, want pending original slot", copyRow) - } - } - exhausted, err := env.repos.Tasks.ListExhausted(ctx, 10) - if err != nil || len(exhausted) != 0 { - t.Fatalf("exhausted tasks = %#v err=%v, want none", exhausted, err) - } - repairTasks, repairTotal, err := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "repair_replica", "", 10, 0) - if err != nil || repairTotal != 3 { - t.Fatalf("repair tasks = %#v total=%d err=%v, want one per assigned slot", repairTasks, repairTotal, err) - } - var recoveredTask *model.Task - for i := range repairTasks { - if taskPayloadInt64ForTest(repairTasks[i].Payload, "storage_data_set_id") == bindings[0].ID { - recoveredTask = &repairTasks[i] - } - if _, err := env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(time.Hour)). - Where("id = ?", repairTasks[i].ID). - Exec(ctx); err != nil { - t.Fatalf("defer repair task %d: %v", repairTasks[i].ID, err) - } - } - if recoveredTask == nil { - t.Fatal("repair task for recovered slot not found") - } - if _, err := env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(time.Hour)). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - t.Fatalf("defer original prepare task: %v", err) - } - providerRecovered.Store(true) - if _, err := env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("id = ?", recoveredTask.ID). - Exec(ctx); err != nil { - t.Fatalf("reschedule recovered slot: %v", err) - } - recoveryUploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 3, 1, 10*time.Millisecond, slog.Default()) - gotRepair := runWorkerUntilTask(t, env, recoveryUploader, recoveredTask.ID, 5*time.Second) - if gotRepair.Status != model.TaskStatusCompleted || gotRepair.RetryCount != 0 { - t.Fatalf("recovered repair task = %#v, want completed without retry", gotRepair) - } - if got := recoveredContext.storeCalls.Load(); got != 1 { - t.Fatalf("recovered provider Store calls = %d, want retained-cache repair", got) - } - recoveredVersion, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || recoveredVersion == nil || recoveredVersion.State != model.ObjectStateReplicating { - t.Fatalf("version after one recovered slot = %#v err=%v, want readable replicating", recoveredVersion, err) - } - if _, err := env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - t.Fatalf("resume original prepare task: %v", err) - } - resumeUploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 3, 1, 10*time.Millisecond, slog.Default()) - gotPrepare := runWorkerUntilTask(t, env, resumeUploader, task.ID, 5*time.Second) - if gotPrepare.Status != model.TaskStatusCompleted || gotPrepare.RetryCount != 0 { - t.Fatalf("resumed prepare task = %#v, want completed without retry", gotPrepare) - } - if got := selectTargetsCalls.Load(); got != 0 { - t.Fatalf("SelectUploadTargets calls after recovery = %d, want no topology change", got) - } -} - -func TestUploader_PrepareSkipsUnavailableAssignedContextAndReassignsIngress(t *testing.T) { - env := newTestWorkerEnv(t) - bucket, objID, versionID := seedCachedObject(t, env) - ctx := context.Background() - seedUpload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: "01J000000000000000SEEDCTX1", ContentSize: 1, Checksum: "seed-contexts", RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt seed: %v", err) - } - bindings := make([]*model.StorageDataSet, 0, 2) - for copyIndex, ids := range [][2]string{{"101", "1001"}, {"202", "2002"}} { - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, ids[0]), CopyIndex: copyIndex, CreatedByUploadID: seedUpload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding(%d): %v", copyIndex, err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ID: binding.ID, UploadID: seedUpload.ID, DataSetID: onChainID(t, ids[1])}); err != nil { - t.Fatalf("MarkDataSetReady(%d): %v", copyIndex, err) - } - bindings = append(bindings, binding) - } - task := seedStagedUploadTask(t, env, objID, versionID, 5) - healthyCtx := newFakeUploadContext(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(3001), testCID(t)) - dataSetID := sdktypes.NewBigInt(2002) - healthyCtx.boundDataSet = &dataSetID - var selectTargetsCalls atomic.Int32 - env.storage.SelectUploadTargetsFunc = func(_ context.Context, _ storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - return nil, errors.New("assigned slots must not select replacement providers") - } - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - switch { - case createContextDataSetIDEqual(opts, sdktypes.NewBigInt(1001)): - return nil, &synapse.ProviderUnavailableError{Cause: errors.New("provider 101 unavailable")} - case createContextDataSetIDEqual(opts, dataSetID): - return healthyCtx, nil - default: - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - } - env.storage.PrepareUploadFunc = func(_ context.Context, _ uint64, contexts []synapse.StorageTarget) (*storage.MultiContextCosts, error) { - if len(contexts) != 1 || !contexts[0].ProviderID().Equal(sdktypes.NewBigInt(202)) { - t.Fatalf("funding contexts = %#v, want only provider 202", contexts) - } - return &storage.MultiContextCosts{Ready: true}, nil - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if got := selectTargetsCalls.Load(); got != 0 { - t.Fatalf("SelectUploadTargets calls = %d, want no topology change", got) - } - failedBinding, err := env.repos.Uploads.GetDataSetBindingByID(ctx, bindings[0].ID) - if err != nil || failedBinding == nil || failedBinding.Status != model.StorageDataSetStatusUnavailable { - t.Fatalf("provider 101 binding = %#v err=%v, want unavailable", failedBinding, err) - } - upload, err := env.repos.Uploads.FindLatestUploadBySourceVersion(ctx, versionID) - if err != nil || upload == nil { - t.Fatalf("FindLatestUploadBySourceVersion: upload=%#v err=%v", upload, err) - } - copies, err := env.repos.Uploads.ListCopies(ctx, upload.ID) - if err != nil || len(copies) != 2 || copies[0].TransferMethod != model.StorageCopyTransferMethodPeerPull || copies[1].TransferMethod != model.StorageCopyTransferMethodIngress { - t.Fatalf("copies after context outage = %#v err=%v", copies, err) - } - repairTasks, repairTotal, err := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "repair_replica", "", 10, 0) - if err != nil || repairTotal != 1 || taskPayloadInt64ForTest(repairTasks[0].Payload, "storage_upload_copy_id") != copies[0].ID { - t.Fatalf("repair tasks = %#v total=%d err=%v", repairTasks, repairTotal, err) - } - ensureTasks, ensureTotal, err := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "ensure_dataset", "", 10, 0) - if err != nil || ensureTotal != 1 || taskPayloadInt64ForTest(ensureTasks[0].Payload, "copy_index") != 1 { - t.Fatalf("ensure tasks = %#v total=%d err=%v, want ingress slot 1", ensureTasks, ensureTotal, err) - } -} - -func TestUploader_NewSlotProviderExhaustionWaitsWithoutRetry(t *testing.T) { - env := newTestWorkerEnv(t) - _, objID, versionID := seedCachedObject(t, env) - task := seedStagedUploadTask(t, env, objID, versionID, 5) - env.storage.SelectUploadTargetsFunc = func(_ context.Context, _ storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - return nil, &synapse.NoProviderCandidatesError{Cause: errors.New("no remaining providers")} - } - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 3, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, env, uploader, task.ID, model.TaskStatusWaiting, 5*time.Second) - gotTask, err := env.repos.Tasks.GetByID(context.Background(), task.ID) - if err != nil || gotTask.RetryCount != 0 || gotTask.WaitReason == nil || *gotTask.WaitReason != model.TaskWaitReasonDependency { - t.Fatalf("provider exhaustion task = %#v err=%v, want dependency wait", gotTask, err) - } - exhausted, err := env.repos.Tasks.ListExhausted(context.Background(), 10) - if err != nil || len(exhausted) != 0 { - t.Fatalf("exhausted tasks = %#v err=%v, want none", exhausted, err) - } -} - -func TestUploader_NewSlotProviderExhaustionUsesExistingReadySlots(t *testing.T) { - env := newTestWorkerEnv(t) - bucket, objID, versionID := seedCachedObject(t, env) - ctx := context.Background() - seedUpload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: "01J000000000000000SEEDGAP1", ContentSize: 1, Checksum: "seed-gap", RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt seed: %v", err) - } - readyContexts := make(map[string]*fakeUploadContext, 2) - for copyIndex, ids := range [][2]string{{"101", "1001"}, {"202", "2002"}} { - binding, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, ids[0]), CopyIndex: copyIndex, CreatedByUploadID: seedUpload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding(%d): %v", copyIndex, err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: binding.ID, UploadID: seedUpload.ID, DataSetID: onChainID(t, ids[1]), - }); err != nil { - t.Fatalf("MarkDataSetReady(%d): %v", copyIndex, err) - } - providerID, _ := strconv.ParseUint(ids[0], 10, 64) - dataSetID, _ := strconv.ParseUint(ids[1], 10, 64) - storageCtx := newFakeUploadContext(sdktypes.NewBigInt(providerID), sdktypes.NewBigInt(dataSetID), sdktypes.NewBigInt(uint64(3001+copyIndex)), testCID(t)) - boundID := sdktypes.NewBigInt(dataSetID) - storageCtx.boundDataSet = &boundID - readyContexts[ids[1]] = storageCtx - } - task := seedStagedUploadTask(t, env, objID, versionID, 5) - var selectTargetsCalls atomic.Int32 - env.storage.SelectUploadTargetsFunc = func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalls.Add(1) - if opts.Copies != 1 { - t.Fatalf("SelectUploadTargets copies = %d, want one missing slot", opts.Copies) - } - return nil, &synapse.NoProviderCandidatesError{Cause: errors.New("no remaining providers")} - } - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if opts.DataSetID != nil { - if storageCtx := readyContexts[opts.DataSetID.String()]; storageCtx != nil { - return storageCtx, nil - } - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - env.storage.PrepareUploadFunc = func(_ context.Context, _ uint64, contexts []synapse.StorageTarget) (*storage.MultiContextCosts, error) { - if len(contexts) != 2 { - t.Fatalf("PrepareUpload contexts = %d, want two existing ready slots", len(contexts)) - } - return &storage.MultiContextCosts{Ready: true}, nil - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 3, 1, 10*time.Millisecond, slog.Default()) - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("prepare task = %#v, want existing slots to continue without retry", gotTask) - } - if got := selectTargetsCalls.Load(); got != 1 { - t.Fatalf("SelectUploadTargets calls = %d, want one bounded attempt for the missing slot", got) - } - upload, err := env.repos.Uploads.FindLatestUploadBySourceVersion(ctx, versionID) - if err != nil || upload == nil { - t.Fatalf("FindLatestUploadBySourceVersion: upload=%#v err=%v", upload, err) - } - copies, err := env.repos.Uploads.ListCopies(ctx, upload.ID) - if err != nil || len(copies) != 2 { - t.Fatalf("partial upload copies = %#v err=%v, want existing two slots", copies, err) - } - ensureTasks, total, err := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "ensure_dataset", string(model.TaskStatusQueued), 10, 0) - if err != nil || total != 1 || taskPayloadInt64ForTest(ensureTasks[0].Payload, "copy_index") != 0 { - t.Fatalf("ensure tasks = %#v total=%d err=%v, want ingress on existing slot 0", ensureTasks, total, err) - } - - workerCtx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = uploader.Run(workerCtx) - close(done) - }() - defer func() { - cancel() - waitForSignal(t, done, time.Second, "uploader shutdown") - }() - deadline := time.After(5 * time.Second) - ticker := time.NewTicker(20 * time.Millisecond) - defer ticker.Stop() - for { - select { - case <-deadline: - t.Fatal("timed out waiting for missing-slot repair dependency") - case <-ticker.C: - version, versionErr := env.repos.Objects.GetVersionByID(ctx, versionID) - repairTasks, repairTotal, listErr := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "prepare_upload", string(model.TaskStatusWaiting), 10, 0) - if versionErr == nil && version != nil && version.State == model.ObjectStateReplicating && listErr == nil && repairTotal == 1 { - if repairTasks[0].RetryCount != 0 || repairTasks[0].WaitReason == nil || *repairTasks[0].WaitReason != model.TaskWaitReasonDependency { - t.Fatalf("missing-slot repair task = %#v, want dependency wait without retry", repairTasks[0]) - } - return - } - } - } -} - -func TestUploader_FundingSkipsUnavailableCandidateAndUsesReadySlot(t *testing.T) { - env := newTestWorkerEnv(t) - bucket, objID, versionID := seedCachedObject(t, env) - ctx := context.Background() - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil { - t.Fatalf("GetVersionByID: version=%#v err=%v", version, err) - } - upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ - BucketID: bucket.ID, SourceVersionID: versionID, ContentSize: version.Size, Checksum: version.Checksum, RequestedCopies: 2, - }) - if err != nil { - t.Fatalf("StartObjectUploadAttempt: %v", err) - } - candidate, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding candidate: %v", err) - } - ready, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ - BucketID: bucket.ID, ProviderID: onChainID(t, "202"), CopyIndex: 1, CreatedByUploadID: upload.ID, - }) - if err != nil { - t.Fatalf("EnsureDataSetBinding ready: %v", err) - } - if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ - ID: ready.ID, UploadID: upload.ID, DataSetID: onChainID(t, "2002"), - }); err != nil { - t.Fatalf("MarkDataSetReady: %v", err) - } - task := seedStagedUploadTask(t, env, objID, versionID, 5) - readyCtx := newFakeUploadContext(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(3002), testCID(t)) - readyDataSetID := sdktypes.NewBigInt(2002) - readyCtx.boundDataSet = &readyDataSetID - candidateCtx := newFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(3001), testCID(t)) - var candidateRecovered atomic.Bool - var candidateCreateCalls atomic.Int32 - candidateCtx.createCalls = &candidateCreateCalls - env.storage.SelectUploadTargetsFunc = func(context.Context, storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - return nil, errors.New("all requested slots are already assigned") - } - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - switch { - case createContextProviderIDEqual(opts, sdktypes.NewBigInt(101)): - if candidateRecovered.Load() { - return candidateCtx, nil - } - return nil, &synapse.ProviderUnavailableError{Cause: errors.New("candidate provider unavailable")} - case createContextDataSetIDEqual(opts, readyDataSetID): - return readyCtx, nil - default: - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - } - var prepareCalls atomic.Int32 - env.storage.PrepareUploadFunc = func(_ context.Context, _ uint64, contexts []synapse.StorageTarget) (*storage.MultiContextCosts, error) { - call := prepareCalls.Add(1) - wantProvider := sdktypes.NewBigInt(202) - if call > 1 { - wantProvider = sdktypes.NewBigInt(101) - } - if len(contexts) != 1 || !contexts[0].ProviderID().Equal(wantProvider) { - t.Fatalf("funding contexts on call %d = %#v, want only provider %s", call, contexts, wantProvider.String()) - } - return &storage.MultiContextCosts{Ready: true}, nil - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("prepare task = %#v, want ready slot to continue without retry", gotTask) - } - retainedCandidate, err := env.repos.Uploads.GetDataSetBindingByID(ctx, candidate.ID) - if err != nil || retainedCandidate == nil || retainedCandidate.Status != model.StorageDataSetStatusPending { - t.Fatalf("candidate binding = %#v err=%v, want pending without lifecycle change", retainedCandidate, err) - } - copies, err := env.repos.Uploads.ListCopies(ctx, upload.ID) - if err != nil || len(copies) != 2 || copies[0].TransferMethod != model.StorageCopyTransferMethodPeerPull || copies[1].TransferMethod != model.StorageCopyTransferMethodIngress { - t.Fatalf("reassigned ingress copies = %#v err=%v, want ready slot 1 as ingress", copies, err) - } - - workerCtx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = uploader.Run(workerCtx) - close(done) - }() - defer func() { - cancel() - waitForSignal(t, done, time.Second, "uploader shutdown") - }() - var repairTask *model.Task - deadline := time.After(5 * time.Second) - ticker := time.NewTicker(20 * time.Millisecond) - defer ticker.Stop() - for repairTask == nil { - select { - case <-deadline: - t.Fatal("timed out waiting for deferred candidate repair") - case <-ticker.C: - version, versionErr := env.repos.Objects.GetVersionByID(ctx, versionID) - tasks, total, listErr := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "prepare_upload", string(model.TaskStatusWaiting), 10, 0) - if versionErr == nil && version != nil && version.State == model.ObjectStateReplicating && listErr == nil && total == 1 { - repairTask = &tasks[0] - } - } - } - if candidateCreateCalls.Load() != 0 { - t.Fatalf("CreateDataSet calls before candidate recovery = %d, want 0", candidateCreateCalls.Load()) - } - candidateRecovered.Store(true) - if _, err := env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("id = ?", repairTask.ID). - Exec(ctx); err != nil { - t.Fatalf("reschedule candidate repair: %v", err) - } - waitForObjectState(t, env, versionID, model.ObjectStateStored, 5*time.Second) - if candidateCreateCalls.Load() != 1 { - t.Fatalf("CreateDataSet calls after candidate recovery = %d, want 1", candidateCreateCalls.Load()) - } - if prepareCalls.Load() != 2 { - t.Fatalf("PrepareUpload calls = %d, want initial ready slot and delayed candidate preflight", prepareCalls.Load()) - } -} - -func TestUploader_EvictTaskIdempotency(t *testing.T) { - env := newTestWorkerEnv(t) - _, objID, versionID := seedCachedObject(t, env) - task := seedStagedUploadTask(t, env, objID, versionID, 5) - - pieceCID := testCID(t) - ingress := newFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(2001), pieceCID) - env.storage.SelectUploadTargetsFunc = func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - if opts.Copies != 1 { - t.Fatalf("SelectUploadTargets copies = %d, want 1", opts.Copies) - } - return []synapse.StorageTarget{ingress}, nil - } - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextProviderIDEqual(opts, ingress.providerID) { - return ingress, nil - } - if createContextDataSetIDEqual(opts, ingress.dataSetID) { - return ingress, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - // Pre-create conflicting evict_cache task to trigger idempotency collision - evictionStage := cacheeviction.StageAfterUpload - conflict := &model.Task{ - Type: model.TaskTypeEvictCache, - Stage: &evictionStage, - RefType: "object", - RefID: objID, - RefVersionID: versionID, - IdempotencyKey: fmt.Sprintf("evict_cache:%s", versionID), - Status: model.TaskStatusQueued, - MaxRetries: 3, - ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(context.Background(), conflict); err != nil { - t.Fatalf("creating conflict task: %v", err) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 1, 1, 10*time.Millisecond, slog.Default()) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - _ = uploader.Run(ctx) - close(done) - }() - waitForObjectState(t, env, versionID, model.ObjectStateStored, 5*time.Second) - cancel() - waitForSignal(t, done, time.Second, "uploader shutdown") - - // ErrAlreadyExists is treated as idempotent success — task completes - got, _ := env.repos.Tasks.GetByID(context.Background(), task.ID) - if got.Status != model.TaskStatusCompleted { - t.Errorf("expected task completed (evict task already exists = idempotent), got %s", got.Status) - } - - obj, _ := env.repos.Objects.GetCurrentVersionByObjectID(context.Background(), objID) - if obj.State != model.ObjectStateStored { - t.Errorf("expected object in stored state, got %s", obj.State) - } -} - -func TestUploader_UploadFundingWaitsWithoutRetry(t *testing.T) { - env := newTestWorkerEnv(t) - _, objID, versionID := seedCachedObject(t, env) - task := seedStagedUploadTask(t, env, objID, versionID, 5) - - var selectTargetsCalled atomic.Int32 - env.storage.SelectUploadTargetsFunc = func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalled.Add(1) - return newFakeUploadContexts(t, opts.Copies, 0), nil - } - env.storage.PrepareUploadFunc = func(_ context.Context, dataSize uint64, contexts []synapse.StorageTarget) (*storage.MultiContextCosts, error) { - if dataSize == 0 { - t.Fatal("PrepareUpload data size must be positive") - } - if len(contexts) != config.DefaultFilecoinCopies { - t.Fatalf("PrepareUpload contexts = %d, want %d", len(contexts), config.DefaultFilecoinCopies) - } - return &storage.MultiContextCosts{ - DepositNeeded: big.NewInt(123), - NeedsFWSSMaxApproval: true, - Ready: false, - }, nil - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 50*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, env, uploader, task.ID, model.TaskStatusWaiting, 5*time.Second) - - got, _ := env.repos.Tasks.GetByID(context.Background(), task.ID) - if got.Status != model.TaskStatusWaiting { - t.Errorf("expected task waiting for funding, got %s", got.Status) - } - if got.RetryCount != 0 { - t.Errorf("expected retry_count=0, got %d", got.RetryCount) - } - if got.WaitReason == nil || *got.WaitReason != model.TaskWaitReasonDependency { - t.Fatalf("wait_reason = %v, want dependency", got.WaitReason) - } - if got.StatusMessage == nil || !strings.Contains(*got.StatusMessage, "deposit 123") || !strings.Contains(*got.StatusMessage, "approve FWSS") { - message := "" - if got.StatusMessage != nil { - message = *got.StatusMessage - } - t.Fatalf("status_message = %q, want deposit and approval guidance", message) - } - if selectTargetsCalled.Load() != 1 { - t.Fatalf("SelectUploadTargets calls = %d, want 1", selectTargetsCalled.Load()) - } - upload, err := env.repos.Uploads.FindLatestUploadBySourceVersion(context.Background(), versionID) - if err != nil || upload == nil { - t.Fatalf("expected upload attempt before funding wait, upload=%v err=%v", upload, err) - } - copies, err := env.repos.Uploads.ListCopies(context.Background(), upload.ID) - if err != nil { - t.Fatalf("ListCopies: %v", err) - } - if len(copies) != config.DefaultFilecoinCopies { - t.Fatalf("copy rows before funding ready = %d, want one pending row per assigned slot", len(copies)) - } - for _, copyRow := range copies { - if copyRow.Status != model.StorageUploadCopyStatusPending || copyRow.StorageDataSetID == nil { - t.Fatalf("copy before funding = %#v, want pending assigned copy", copyRow) - } - } -} - -func TestUploader_UploadFundingRetryReusesBindings(t *testing.T) { - env := newTestWorkerEnv(t) - bucket, objID, versionID := seedCachedObject(t, env) - task := seedStagedUploadTask(t, env, objID, versionID, 5) - ctx := context.Background() - - var ( - selectTargetsCalled atomic.Int32 - prepareCalled atomic.Int32 - uploadContexts []synapse.StorageTarget - ) - env.storage.SelectUploadTargetsFunc = func(_ context.Context, _ storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - selectTargetsCalled.Add(1) - uploadContexts = newFakeUploadContexts(t, config.DefaultFilecoinCopies, 0) - return uploadContexts, nil - } - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - for _, uploadCtx := range uploadContexts { - fakeCtx := uploadCtx.(*fakeUploadContext) - if createContextProviderIDEqual(opts, fakeCtx.providerID) || createContextDataSetIDEqual(opts, fakeCtx.dataSetID) { - return fakeCtx, nil - } - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - env.storage.PrepareUploadFunc = func(_ context.Context, _ uint64, contexts []synapse.StorageTarget) (*storage.MultiContextCosts, error) { - if len(contexts) != config.DefaultFilecoinCopies { - t.Fatalf("PrepareUpload contexts = %d, want %d", len(contexts), config.DefaultFilecoinCopies) - } - if prepareCalled.Add(1) == 1 { - return &storage.MultiContextCosts{DepositNeeded: big.NewInt(10), Ready: false}, nil - } - return &storage.MultiContextCosts{DepositNeeded: big.NewInt(0), Ready: true}, nil - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 50*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, env, uploader, task.ID, model.TaskStatusWaiting, 5*time.Second) - - got, _ := env.repos.Tasks.GetByID(ctx, task.ID) - if got.RetryCount != 0 { - t.Fatalf("retry_count after funding wait = %d, want 0", got.RetryCount) - } - if _, err := env.db.NewUpdate().Model((*model.Task)(nil)). - Set("scheduled_at = ?", time.Now().Add(-time.Second)). - Where("id = ?", task.ID). - Exec(ctx); err != nil { - t.Fatalf("reschedule funding wait: %v", err) - } - runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - - got, _ = env.repos.Tasks.GetByID(ctx, task.ID) - if got.Status != model.TaskStatusCompleted { - t.Fatalf("expected retried task completed, got %s", got.Status) - } - if selectTargetsCalled.Load() != 1 { - t.Fatalf("SelectUploadTargets calls = %d, want 1", selectTargetsCalled.Load()) - } - if prepareCalled.Load() != 2 { - t.Fatalf("PrepareUpload calls = %d, want 2", prepareCalled.Load()) - } - bindings, err := env.repos.Uploads.ListDataSetBindings(ctx, bucket.ID) - if err != nil { - t.Fatalf("ListDataSetBindings: %v", err) - } - if len(bindings) != config.DefaultFilecoinCopies { - t.Fatalf("bindings = %d, want %d", len(bindings), config.DefaultFilecoinCopies) - } - upload, err := env.repos.Uploads.FindLatestUploadBySourceVersion(ctx, versionID) - if err != nil || upload == nil { - t.Fatalf("expected upload attempt, upload=%v err=%v", upload, err) - } - copies, err := env.repos.Uploads.ListCopies(ctx, upload.ID) - if err != nil { - t.Fatalf("ListCopies: %v", err) - } - if len(copies) != config.DefaultFilecoinCopies { - t.Fatalf("copy rows = %d, want %d", len(copies), config.DefaultFilecoinCopies) - } -} - -func TestUploader_PrepareFundingDoesNotQueryWallet(t *testing.T) { - env := newTestWorkerEnv(t) - _, objID, versionID := seedCachedObject(t, env) - task := seedStagedUploadTask(t, env, objID, versionID, 5) - - var walletCalled atomic.Bool - wallet := &testutil.MockWalletQuerier{ - GetWalletInfoFunc: func(_ context.Context) (*synapse.WalletInfo, error) { - walletCalled.Store(true) - return nil, errors.New("wallet should not be queried during upload funding preparation") - }, - } - env.storage.SelectUploadTargetsFunc = func(_ context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { - return newFakeUploadContexts(t, opts.Copies, 0), nil - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, wallet, env.sm, cache.EvictionPolicyAfterUpload, config.DefaultFilecoinCopies, 1, 50*time.Millisecond, slog.Default()) - got := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - - if got.Status != model.TaskStatusCompleted { - t.Errorf("expected task completed, got %s", got.Status) - } - if walletCalled.Load() { - t.Fatal("wallet should not be queried before upload funding preparation") - } - if upload, err := env.repos.Uploads.FindLatestUploadBySourceVersion(context.Background(), versionID); err != nil || upload == nil { - t.Fatalf("expected upload attempt, upload=%v err=%v", upload, err) - } -} - -// writeBlockedCommitError is the SDK's refusal of a data set whose PDP payment -// ended. The SDK raises it before contacting the provider, so the commit is -// known not to have been submitted and the attempt can be released. -func writeBlockedCommitError(dataSetID uint64) error { - return fmt.Errorf("storage.DataSetContext.SubmitCommit: %w", &storage.DataSetPDPPaymentTerminatedError{ - DataSetID: sdktypes.NewBigInt(dataSetID), PDPEndEpoch: 3778900, - }) -} - -func assertCommitReleasedForFailover(t *testing.T, env *testWorkerEnv, bindingID, copyID int64) { - t.Helper() - ctx := context.Background() - binding, err := env.repos.Uploads.GetDataSetBindingByID(ctx, bindingID) - if err != nil || binding == nil || binding.Status != model.StorageDataSetStatusDraining { - t.Fatalf("binding after write-blocked commit = %#v err=%v, want draining", binding, err) - } - copyRow, err := env.repos.Uploads.GetUploadCopyByID(ctx, copyID) - if err != nil || copyRow == nil || copyRow.Status != model.StorageUploadCopyStatusPieceReady || - copyRow.CommitAttemptID != nil || copyRow.CommitReadyAt != nil || copyRow.CommitExtraDataHex != nil || - copyRow.CommitTransactionID != nil { - t.Fatalf("copy after write-blocked commit = %#v err=%v, want a released piece ready to move", copyRow, err) - } -} - -func TestUploader_WriteBlockedIngressCommitDrainsDataSet(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadyIngressCommitTask(t, env, 1, 2) - fixture.primary.submitPreflightErr = writeBlockedCommitError(1001) - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, - cache.EvictionPolicyAfterUpload, 1, 1, 10*time.Millisecond, slog.Default()) - runWorkerUntilTaskStatus(t, env, uploader, fixture.task.ID, model.TaskStatusWaiting, 5*time.Second) - - gotTask, err := env.repos.Tasks.GetByID(ctx, fixture.task.ID) - if err != nil || gotTask == nil || gotTask.RetryCount != 0 || - gotTask.WaitReason == nil || *gotTask.WaitReason != model.TaskWaitReasonDependency { - t.Fatalf("ingress commit task = %#v err=%v, want dependency wait without retry", gotTask, err) - } - assertCommitReleasedForFailover(t, env, fixture.binding.ID, fixture.copyRow.ID) -} - -func TestUploader_WriteBlockedPeerCommitDrainsDataSet(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - pieceCID := testCID(t) - if err := env.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: fixture.upload.ID, - CopyIndex: 1, - PieceCID: pieceCID.String(), - RetrievalURL: fmt.Sprintf("https://provider-202.example/piece/%s", pieceCID.String()), - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, 1) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - stage := "peer_commit" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "object", RefID: fixture.objID, RefVersionID: fixture.versionID, - IdempotencyKey: fmt.Sprintf("upload:%s:peer_commit:%d:1", fixture.versionID, fixture.upload.ID), - Payload: map[string]interface{}{"upload_id": fixture.upload.ID, "copy_index": 1, "transfer_method": string(model.StorageCopyTransferMethodPeerPull)}, - Status: model.TaskStatusQueued, MaxRetries: 2, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create peer commit task: %v", err) - } - peerCtx := newFakeUploadContext(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(302), pieceCID) - peerDataSetID := sdktypes.NewBigInt(2002) - peerCtx.boundDataSet = &peerDataSetID - peerCtx.submitPreflightErr = writeBlockedCommitError(2002) - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextDataSetIDEqual(opts, peerDataSetID) { - return peerCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, - cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - // A peer copy on a write-blocked data set has nothing left to wait for: the - // replica is moved by the data set's own replacement, so the task completes - // rather than parking on the dependency the ingress path parks on. - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("peer commit task = %#v, want completed without retry", gotTask) - } - assertCommitReleasedForFailover(t, env, fixture.peer.ID, copyRow.ID) -} - -func TestUploader_WriteBlockedReplicaRepairCommitDrainsDataSet(t *testing.T) { - env := newTestWorkerEnv(t) - ctx := context.Background() - fixture := seedReadableUploadWithPendingPeer(t, env) - pieceCID := testCID(t) - if err := env.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: fixture.upload.ID, - CopyIndex: 1, - PieceCID: pieceCID.String(), - RetrievalURL: fmt.Sprintf("https://provider-202.example/piece/%s", pieceCID.String()), - }); err != nil { - t.Fatalf("MarkUploadCopyPieceReady: %v", err) - } - copyRow, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, 1) - if err != nil || copyRow == nil { - t.Fatalf("GetUploadCopy: copy=%#v err=%v", copyRow, err) - } - stage := "repair_replica" - task := &model.Task{ - Type: model.TaskTypeUpload, Stage: &stage, RefType: "bucket", RefID: fixture.upload.BucketID, - RefVersionID: fixture.versionID, IdempotencyKey: fmt.Sprintf("upload:repair-data-set:%d", fixture.peer.ID), - Payload: map[string]interface{}{"storage_data_set_id": fixture.peer.ID, "storage_upload_copy_id": copyRow.ID}, - Status: model.TaskStatusQueued, MaxRetries: 2, ScheduledAt: time.Now(), - } - if err := env.repos.Tasks.Create(ctx, task); err != nil { - t.Fatalf("Create repair task: %v", err) - } - peerCtx := newFakeUploadContext(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(302), pieceCID) - peerDataSetID := sdktypes.NewBigInt(2002) - peerCtx.boundDataSet = &peerDataSetID - peerCtx.submitPreflightErr = writeBlockedCommitError(2002) - env.storage.OpenTargetFunc = func(_ context.Context, opts *testutil.OpenTargetOptions) (synapse.StorageTarget, error) { - if createContextDataSetIDEqual(opts, peerDataSetID) { - return peerCtx, nil - } - return nil, fmt.Errorf("unexpected OpenTarget opts: %#v", opts) - } - - uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, - cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) - // A draining data set has nothing left for in-place repair to do, so the task - // completes instead of waiting on the dependency the other paths wait on. - gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) - if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { - t.Fatalf("repair task = %#v, want completed without retry", gotTask) - } - assertCommitReleasedForFailover(t, env, fixture.peer.ID, copyRow.ID) -} diff --git a/internal/worker/wallet_operation_runner.go b/internal/worker/wallet_operation_runner.go deleted file mode 100644 index d7e6f24..0000000 --- a/internal/worker/wallet_operation_runner.go +++ /dev/null @@ -1,383 +0,0 @@ -package worker - -import ( - "context" - "errors" - "fmt" - "log/slog" - "math/big" - "time" - - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/common" - ethtypes "github.com/ethereum/go-ethereum/core/types" - "github.com/strahe/synaps3/internal/admin" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/synapse" - "github.com/strahe/synapse-go/payments" -) - -const walletOperationUpdatedTopic = "wallet_operation_updated" -const ( - walletOperationMarkSubmittedAttempts = 3 - walletOperationMarkSubmittedTimeout = 5 * time.Second - walletOperationBroadcastTimeout = 2 * time.Minute - walletOperationReceiptLookupTimeout = 15 * time.Second - walletOperationSubmittedScanLimit = 20 -) - -type WalletReceiptChecker interface { - TransactionReceipt(ctx context.Context, txHash common.Hash) (*ethtypes.Receipt, error) -} - -type WalletOperationRunner struct { - repos *repository.Repositories - operator synapse.WalletOperator - receipts WalletReceiptChecker - publisher admin.EventPublisher - pollInterval time.Duration - leaseTTL time.Duration - broadcastTTL time.Duration - receiptTTL time.Duration - logger *slog.Logger - *livenessTracker -} - -type walletOperationBroadcastResult struct { - TxHash string - AlreadyComplete bool -} - -type WalletOperationRunnerOption func(*WalletOperationRunner) - -func WithWalletOperationEventPublisher(publisher admin.EventPublisher) WalletOperationRunnerOption { - return func(r *WalletOperationRunner) { - r.publisher = publisher - } -} - -func WithWalletOperationTimeouts(broadcast, receipt time.Duration) WalletOperationRunnerOption { - return func(r *WalletOperationRunner) { - if broadcast > 0 { - r.broadcastTTL = broadcast - } - if receipt > 0 { - r.receiptTTL = receipt - } - } -} - -func NewWalletOperationRunner( - repos *repository.Repositories, - operator synapse.WalletOperator, - receipts WalletReceiptChecker, - pollInterval time.Duration, - logger *slog.Logger, - opts ...WalletOperationRunnerOption, -) *WalletOperationRunner { - if pollInterval <= 0 { - pollInterval = 5 * time.Second - } - if logger == nil { - logger = slog.Default() - } - r := &WalletOperationRunner{ - repos: repos, - operator: operator, - receipts: receipts, - pollInterval: pollInterval, - leaseTTL: 5 * time.Minute, - broadcastTTL: walletOperationBroadcastTimeout, - receiptTTL: walletOperationReceiptLookupTimeout, - logger: logger, - livenessTracker: newLivenessTracker(pollInterval), - } - for _, opt := range opts { - opt(r) - } - return r -} - -func (r *WalletOperationRunner) Name() string { return "wallet_operations" } - -func (r *WalletOperationRunner) Healthy() bool { return r.healthy() } - -func (r *WalletOperationRunner) Run(ctx context.Context) error { - ticker := time.NewTicker(r.pollInterval) - defer ticker.Stop() - - for { - if ctx.Err() != nil { - return ctx.Err() - } - r.runOnce(ctx) - select { - case <-ctx.Done(): - return ctx.Err() - case <-ticker.C: - } - } -} - -func (r *WalletOperationRunner) runOnce(ctx context.Context) { - if r == nil || r.repos == nil || r.repos.WalletOperations == nil { - return - } - r.recordTick() - r.markExpiredRunningUnknown(ctx) - r.confirmSubmitted(ctx) - r.processPending(ctx) -} - -func (r *WalletOperationRunner) markExpiredRunningUnknown(ctx context.Context) { - ops, err := r.repos.WalletOperations.MarkExpiredRunningUnknown(ctx) - if err != nil { - r.logger.Error("marking expired wallet operations unknown", "error", err) - return - } - for i := range ops { - r.publish(&ops[i]) - } -} - -func (r *WalletOperationRunner) confirmSubmitted(ctx context.Context) { - if r.receipts == nil { - return - } - ops, err := r.repos.WalletOperations.ListSubmitted(ctx, walletOperationSubmittedScanLimit) - if err != nil { - r.logger.Error("listing submitted wallet operations", "error", err) - return - } - if len(ops) == 0 { - return - } - r.recordWorkStarted() - defer r.recordWorkFinished() - for i := range ops { - r.confirmOperation(ctx, &ops[i]) - } -} - -func (r *WalletOperationRunner) processPending(ctx context.Context) { - if r.operator == nil { - return - } - op, err := r.repos.WalletOperations.ClaimPending(ctx, r.leaseTTL) - if err != nil { - r.logger.Error("claiming wallet operation", "error", err) - return - } - if op == nil { - return - } - r.recordWorkStarted() - defer r.recordWorkFinished() - r.publish(op) - r.processClaimed(ctx, op) -} - -func (r *WalletOperationRunner) processClaimed(ctx context.Context, op *model.WalletOperation) { - amount, ok := new(big.Int).SetString(op.Amount, 10) - if !ok || !validClaimedWalletOperationAmount(op.Type, amount) { - r.markFailedAndPublish(ctx, op.ID, "invalid wallet operation amount") - return - } - - broadcastCtx, cancel := context.WithTimeout(ctx, r.broadcastTTL) - result, err := r.broadcast(broadcastCtx, op.Type, amount) - cancel() - if err != nil { - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - r.logger.Warn("wallet operation broadcast did not finish before context ended", "id", op.ID, "error", err) - return - } - r.markFailedAndPublish(ctx, op.ID, walletOperationError(err)) - return - } - if result.AlreadyComplete { - if err := r.repos.WalletOperations.MarkConfirmedWithoutTransaction(context.WithoutCancel(ctx), op.ID); err != nil { - r.logger.Error("marking wallet operation confirmed without transaction failed", "id", op.ID, "error", err) - return - } - r.publish(r.mustGet(ctx, op.ID)) - return - } - if result.TxHash == "" { - r.markFailedAndPublish(ctx, op.ID, "wallet operation broadcast did not return a transaction hash") - return - } - if !r.markSubmittedAfterBroadcast(context.WithoutCancel(ctx), op.ID, result.TxHash) { - return - } - submitted := r.mustGet(ctx, op.ID) - r.publish(submitted) - r.confirmOperation(ctx, submitted) -} - -func (r *WalletOperationRunner) markSubmittedAfterBroadcast(ctx context.Context, id int64, txHash string) bool { - ctx, cancel := context.WithTimeout(ctx, walletOperationMarkSubmittedTimeout) - defer cancel() - - var lastErr error - for attempt := 1; attempt <= walletOperationMarkSubmittedAttempts; attempt++ { - err := r.repos.WalletOperations.MarkSubmitted(ctx, id, txHash) - if err == nil { - return true - } - lastErr = err - r.logger.Warn("marking wallet operation submitted after broadcast failed", "id", id, "tx_hash", txHash, "attempt", attempt, "error", err) - if attempt == walletOperationMarkSubmittedAttempts || ctx.Err() != nil { - break - } - } - r.logger.Error("wallet operation broadcast transaction hash was not persisted", "id", id, "tx_hash", txHash, "error", lastErr) - return false -} - -func validClaimedWalletOperationAmount(opType model.WalletOperationType, amount *big.Int) bool { - switch opType { - case model.WalletOperationTypeApprove: - return amount.Sign() == 0 - case model.WalletOperationTypeFund, model.WalletOperationTypeWithdraw: - return amount.Sign() > 0 - default: - return false - } -} - -func (r *WalletOperationRunner) broadcast(ctx context.Context, opType model.WalletOperationType, amount *big.Int) (walletOperationBroadcastResult, error) { - switch opType { - case model.WalletOperationTypeFund: - txHash, err := r.operator.FundUSDFC(ctx, amount) - return walletOperationBroadcastResult{TxHash: txHash}, err - case model.WalletOperationTypeWithdraw: - txHash, err := r.operator.WithdrawUSDFC(ctx, amount) - return walletOperationBroadcastResult{TxHash: txHash}, err - case model.WalletOperationTypeApprove: - txHash, err := r.operator.ApproveFWSS(ctx) - if errors.Is(err, payments.ErrNothingToFund) { - return walletOperationBroadcastResult{AlreadyComplete: true}, nil - } - return walletOperationBroadcastResult{TxHash: txHash}, err - default: - return walletOperationBroadcastResult{}, fmt.Errorf("unsupported wallet operation type %q", opType) - } -} - -func (r *WalletOperationRunner) confirmOperation(ctx context.Context, op *model.WalletOperation) { - if op == nil || op.TxHash == nil || *op.TxHash == "" || r.receipts == nil { - return - } - receiptCtx, cancel := context.WithTimeout(ctx, r.receiptTTL) - defer cancel() - - receipt, err := r.receipts.TransactionReceipt(receiptCtx, common.HexToHash(*op.TxHash)) - if err != nil { - if errors.Is(err, ethereum.NotFound) { - return - } - r.logger.Warn("wallet operation receipt lookup failed", "id", op.ID, "tx_hash", *op.TxHash, "error", err) - return - } - if receipt == nil { - return - } - if receipt.Status == ethtypes.ReceiptStatusSuccessful { - if err := r.repos.WalletOperations.MarkConfirmed(ctx, op.ID); err != nil { - r.logger.Error("marking wallet operation confirmed", "id", op.ID, "error", err) - return - } - r.publish(r.mustGet(ctx, op.ID)) - return - } - r.markFailedAndPublish(ctx, op.ID, revertedReceiptMessage(op, receipt)) -} - -func (r *WalletOperationRunner) markFailedAndPublish(ctx context.Context, id int64, message string) { - if err := r.repos.WalletOperations.MarkFailed(ctx, id, message); err != nil { - r.logger.Error("marking wallet operation failed", "id", id, "error", err) - return - } - r.publish(r.mustGet(ctx, id)) -} - -func (r *WalletOperationRunner) mustGet(ctx context.Context, id int64) *model.WalletOperation { - op, err := r.repos.WalletOperations.GetByID(ctx, id) - if err != nil { - r.logger.Error("loading wallet operation", "id", id, "error", err) - return nil - } - return op -} - -func (r *WalletOperationRunner) publish(op *model.WalletOperation) { - if r == nil || r.publisher == nil || op == nil { - return - } - r.publisher.Publish(walletOperationUpdatedTopic, map[string]any{ - "operation": walletOperationEventFromModel(op), - }) -} - -type walletOperationEventPayload struct { - ID int64 `json:"id"` - Type string `json:"type"` - ClientRequestID string `json:"client_request_id"` - Amount string `json:"amount"` - Status string `json:"status"` - TxHash *string `json:"tx_hash,omitempty"` - LastError *string `json:"last_error,omitempty"` - LeaseUntil *string `json:"lease_until,omitempty"` - StartedAt *string `json:"started_at,omitempty"` - SubmittedAt *string `json:"submitted_at,omitempty"` - CompletedAt *string `json:"completed_at,omitempty"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` -} - -func walletOperationEventFromModel(op *model.WalletOperation) walletOperationEventPayload { - return walletOperationEventPayload{ - ID: op.ID, - Type: string(op.Type), - ClientRequestID: op.ClientRequestID, - Amount: op.Amount, - Status: string(op.Status), - TxHash: op.TxHash, - LastError: op.LastError, - LeaseUntil: operationTimeString(op.LeaseUntil), - StartedAt: operationTimeString(op.StartedAt), - SubmittedAt: operationTimeString(op.SubmittedAt), - CompletedAt: operationTimeString(op.CompletedAt), - CreatedAt: op.CreatedAt.UTC().Format(time.RFC3339Nano), - UpdatedAt: op.UpdatedAt.UTC().Format(time.RFC3339Nano), - } -} - -func operationTimeString(v *time.Time) *string { - if v == nil { - return nil - } - out := v.UTC().Format(time.RFC3339Nano) - return &out -} - -func walletOperationError(err error) string { - if err == nil { - return "" - } - return err.Error() -} - -func revertedReceiptMessage(op *model.WalletOperation, receipt *ethtypes.Receipt) string { - txHash := "" - if op != nil && op.TxHash != nil { - txHash = *op.TxHash - } - blockNumber := "unknown" - if receipt != nil && receipt.BlockNumber != nil { - blockNumber = receipt.BlockNumber.String() - } - return fmt.Sprintf("transaction reverted: status=%d tx_hash=%s block_number=%s", receipt.Status, txHash, blockNumber) -} diff --git a/internal/worker/wallet_operation_runner_test.go b/internal/worker/wallet_operation_runner_test.go deleted file mode 100644 index f26812b..0000000 --- a/internal/worker/wallet_operation_runner_test.go +++ /dev/null @@ -1,751 +0,0 @@ -package worker - -import ( - "context" - "errors" - "math/big" - "strings" - "sync" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - ethtypes "github.com/ethereum/go-ethereum/core/types" - "github.com/strahe/synaps3/internal/db/repository" - "github.com/strahe/synaps3/internal/model" - "github.com/strahe/synaps3/internal/testutil" - "github.com/strahe/synapse-go/payments" -) - -func TestWalletOperationRunner_SubmitsAndConfirmsPendingOperation(t *testing.T) { - tests := []struct { - name string - opType model.WalletOperationType - amount string - }{ - {name: "fund", opType: model.WalletOperationTypeFund, amount: "100"}, - {name: "withdraw", opType: model.WalletOperationTypeWithdraw, amount: "50"}, - {name: "approve", opType: model.WalletOperationTypeApprove, amount: "0"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - op, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: tt.opType, - ClientRequestID: string(tt.opType) + "-1", - Amount: tt.amount, - }) - if err != nil { - t.Fatalf("CreateOrGet: %v", err) - } - - txHash := common.HexToHash("0xabc") - operator := &fakeWalletOperator{fundHash: txHash.Hex(), withdrawHash: txHash.Hex(), approveHash: txHash.Hex()} - receipts := &fakeWalletReceiptChecker{receipts: map[common.Hash]*ethtypes.Receipt{ - txHash: {Status: ethtypes.ReceiptStatusSuccessful}, - }} - publisher := &fakeWalletEventPublisher{} - runner := NewWalletOperationRunner(repos, operator, receipts, time.Millisecond, nil, WithWalletOperationEventPublisher(publisher)) - - runner.runOnce(ctx) - - got, err := repos.WalletOperations.GetByID(ctx, op.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.Status != model.WalletOperationStatusConfirmed { - t.Fatalf("status = %q, want confirmed", got.Status) - } - if got.TxHash == nil || *got.TxHash != txHash.Hex() { - t.Fatalf("tx_hash = %v, want %s", got.TxHash, txHash.Hex()) - } - switch tt.opType { - case model.WalletOperationTypeFund: - if operator.fundAmount == nil || operator.fundAmount.String() != tt.amount { - t.Fatalf("fund amount = %v, want %s", operator.fundAmount, tt.amount) - } - if operator.withdrawAmount != nil { - t.Fatalf("withdraw amount = %s, want no withdraw", operator.withdrawAmount) - } - if operator.approveCalled { - t.Fatal("ApproveFWSS was called, want no approve") - } - case model.WalletOperationTypeWithdraw: - if operator.withdrawAmount == nil || operator.withdrawAmount.String() != tt.amount { - t.Fatalf("withdraw amount = %v, want %s", operator.withdrawAmount, tt.amount) - } - if operator.fundAmount != nil { - t.Fatalf("fund amount = %s, want no fund", operator.fundAmount) - } - if operator.approveCalled { - t.Fatal("ApproveFWSS was called, want no approve") - } - case model.WalletOperationTypeApprove: - if !operator.approveCalled { - t.Fatal("ApproveFWSS was not called") - } - if operator.fundAmount != nil || operator.withdrawAmount != nil { - t.Fatalf("fund=%v withdraw=%v, want no fund or withdraw", operator.fundAmount, operator.withdrawAmount) - } - default: - t.Fatalf("unexpected operation type %q", tt.opType) - } - if !publisher.hasStatus(model.WalletOperationStatusSubmitted) || !publisher.hasStatus(model.WalletOperationStatusConfirmed) { - t.Fatalf("published statuses = %v, want submitted and confirmed", publisher.statuses()) - } - }) - } -} - -func TestWalletOperationRunner_ConfirmsAlreadyApprovedWithoutTransaction(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - op, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeApprove, - ClientRequestID: "approve-already", - Amount: "0", - }) - if err != nil { - t.Fatalf("CreateOrGet: %v", err) - } - - operator := &fakeWalletOperator{approveErr: payments.ErrNothingToFund} - publisher := &fakeWalletEventPublisher{} - runner := NewWalletOperationRunner(repos, operator, nil, time.Millisecond, nil, WithWalletOperationEventPublisher(publisher)) - - runner.runOnce(ctx) - - got, err := repos.WalletOperations.GetByID(ctx, op.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.Status != model.WalletOperationStatusConfirmed { - t.Fatalf("status = %q, want confirmed", got.Status) - } - if got.TxHash != nil { - t.Fatalf("tx_hash = %v, want nil", *got.TxHash) - } - if got.SubmittedAt != nil { - t.Fatalf("submitted_at = %v, want nil", got.SubmittedAt) - } - if !operator.approveCalled { - t.Fatal("ApproveFWSS was not called") - } - if !publisher.hasStatus(model.WalletOperationStatusConfirmed) { - t.Fatalf("published statuses = %v, want confirmed", publisher.statuses()) - } -} - -func TestWalletOperationRunner_RetriesSubmittingBroadcastTxHash(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - repos.WalletOperations = &flakyMarkSubmittedWalletOperationRepo{ - WalletOperationRepository: repos.WalletOperations, - failuresRemaining: 1, - } - ctx := context.Background() - op, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "fund-retry-submitted", - Amount: "100", - }) - if err != nil { - t.Fatalf("CreateOrGet: %v", err) - } - - txHash := common.HexToHash("0xabc") - operator := &fakeWalletOperator{fundHash: txHash.Hex()} - receipts := &fakeWalletReceiptChecker{receipts: map[common.Hash]*ethtypes.Receipt{ - txHash: {Status: ethtypes.ReceiptStatusSuccessful}, - }} - runner := NewWalletOperationRunner(repos, operator, receipts, time.Millisecond, nil) - - runner.runOnce(ctx) - - got, err := repos.WalletOperations.GetByID(ctx, op.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.Status != model.WalletOperationStatusConfirmed { - t.Fatalf("status = %q, want confirmed", got.Status) - } - if got.TxHash == nil || *got.TxHash != txHash.Hex() { - t.Fatalf("tx_hash = %v, want %s", got.TxHash, txHash.Hex()) - } - flaky := repos.WalletOperations.(*flakyMarkSubmittedWalletOperationRepo) - if flaky.markSubmittedCalls != 2 { - t.Fatalf("MarkSubmitted calls = %d, want 2", flaky.markSubmittedCalls) - } -} - -func TestWalletOperationRunner_PersistsBroadcastTxHashAfterContextCancellation(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx, cancel := context.WithCancel(context.Background()) - op, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "fund-cancel-after-broadcast", - Amount: "100", - }) - if err != nil { - t.Fatalf("CreateOrGet: %v", err) - } - - txHash := common.HexToHash("0xabc") - operator := &fakeWalletOperator{ - fundHash: txHash.Hex(), - onFund: func(context.Context) { - cancel() - }, - } - runner := NewWalletOperationRunner(repos, operator, nil, time.Millisecond, nil) - - runner.runOnce(ctx) - - got, err := repos.WalletOperations.GetByID(context.Background(), op.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.Status != model.WalletOperationStatusSubmitted { - t.Fatalf("status = %q, want submitted", got.Status) - } - if got.TxHash == nil || *got.TxHash != txHash.Hex() { - t.Fatalf("tx_hash = %v, want %s", got.TxHash, txHash.Hex()) - } -} - -func TestWalletOperationRunner_WaitsForSubmittedOperationBeforeBroadcastingNext(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - submitted := seedSubmittedWalletOperation(t, repos, model.WalletOperationTypeFund, "fund-1", "100", common.HexToHash("0xabc").Hex()) - if _, err := db.NewUpdate(). - Model(submitted). - Set("submitted_at = ?", time.Now().Add(-3*time.Hour)). - WherePK(). - Exec(ctx); err != nil { - t.Fatalf("age submitted operation: %v", err) - } - pending, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "fund-2", - Amount: "200", - }) - if err != nil { - t.Fatalf("CreateOrGet pending: %v", err) - } - - operator := &fakeWalletOperator{fundHash: common.HexToHash("0xdef").Hex()} - receipts := &fakeWalletReceiptChecker{receipts: map[common.Hash]*ethtypes.Receipt{ - common.HexToHash(*submitted.TxHash): nil, - }} - runner := NewWalletOperationRunner(repos, operator, receipts, time.Millisecond, nil) - - runner.runOnce(ctx) - - got, err := repos.WalletOperations.GetByID(ctx, pending.ID) - if err != nil { - t.Fatalf("GetByID pending: %v", err) - } - if got.Status != model.WalletOperationStatusPending { - t.Fatalf("pending status = %q, want pending", got.Status) - } - if operator.fundAmount != nil { - t.Fatalf("fund amount = %s, want no broadcast", operator.fundAmount) - } -} - -func TestWalletOperationRunner_ReceiptRevertFailsSubmittedOperation(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - op := seedSubmittedWalletOperation(t, repos, model.WalletOperationTypeWithdraw, "withdraw-1", "50", common.HexToHash("0xdef").Hex()) - - receipts := &fakeWalletReceiptChecker{receipts: map[common.Hash]*ethtypes.Receipt{ - common.HexToHash(*op.TxHash): {Status: ethtypes.ReceiptStatusFailed}, - }} - runner := NewWalletOperationRunner(repos, &fakeWalletOperator{}, receipts, time.Millisecond, nil) - - runner.runOnce(ctx) - - got, err := repos.WalletOperations.GetByID(ctx, op.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.Status != model.WalletOperationStatusFailed { - t.Fatalf("status = %q, want failed", got.Status) - } - if got.LastError == nil || *got.LastError == "" { - t.Fatal("last_error is empty, want revert reason") - } - if !strings.Contains(*got.LastError, "transaction reverted") || !strings.Contains(*got.LastError, "status=0") || !strings.Contains(*got.LastError, "tx_hash=") { - t.Fatalf("last_error = %q, want detailed revert context", *got.LastError) - } -} - -func TestWalletOperationRunner_ReceiptLookupFailureKeepsSubmittedOperationInFlight(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - op := seedSubmittedWalletOperation(t, repos, model.WalletOperationTypeFund, "fund-rpc-error", "50", common.HexToHash("0x456").Hex()) - pending, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "fund-after-rpc-error", - Amount: "60", - }) - if err != nil { - t.Fatalf("CreateOrGet pending: %v", err) - } - - operator := &fakeWalletOperator{fundHash: common.HexToHash("0x789").Hex()} - receipts := &fakeWalletReceiptChecker{err: errors.New("rpc timeout while reading receipt")} - runner := NewWalletOperationRunner(repos, operator, receipts, time.Millisecond, nil) - - runner.runOnce(ctx) - - got, err := repos.WalletOperations.GetByID(ctx, op.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.Status != model.WalletOperationStatusSubmitted { - t.Fatalf("status = %q, want submitted", got.Status) - } - gotPending, err := repos.WalletOperations.GetByID(ctx, pending.ID) - if err != nil { - t.Fatalf("GetByID pending: %v", err) - } - if gotPending.Status != model.WalletOperationStatusPending { - t.Fatalf("pending status = %q, want pending", gotPending.Status) - } - if operator.fundAmount != nil { - t.Fatalf("fund amount = %s, want no broadcast", operator.fundAmount) - } -} - -func TestWalletOperationRunner_ReceiptLookupTimeoutDoesNotBlockRunner(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - op := seedSubmittedWalletOperation(t, repos, model.WalletOperationTypeFund, "fund-receipt-timeout", "50", common.HexToHash("0x456").Hex()) - - receipts := &fakeWalletReceiptChecker{block: true} - runner := NewWalletOperationRunner(repos, &fakeWalletOperator{}, receipts, time.Millisecond, nil, WithWalletOperationTimeouts(0, time.Millisecond)) - - started := time.Now() - runner.runOnce(ctx) - if time.Since(started) > time.Second { - t.Fatal("runOnce did not return after receipt lookup timeout") - } - - got, err := repos.WalletOperations.GetByID(ctx, op.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.Status != model.WalletOperationStatusSubmitted { - t.Fatalf("status = %q, want submitted", got.Status) - } -} - -func TestWalletOperationRunner_BroadcastTimeoutDoesNotFailOperation(t *testing.T) { - for _, tc := range []struct { - name string - opType model.WalletOperationType - amount string - operator *fakeWalletOperator - }{ - {name: "fund", opType: model.WalletOperationTypeFund, amount: "100", operator: &fakeWalletOperator{blockFund: true}}, - {name: "withdraw", opType: model.WalletOperationTypeWithdraw, amount: "100", operator: &fakeWalletOperator{blockWithdraw: true}}, - {name: "approve", opType: model.WalletOperationTypeApprove, amount: "0", operator: &fakeWalletOperator{blockApprove: true}}, - } { - t.Run(tc.name, func(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - op, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: tc.opType, - ClientRequestID: string(tc.opType) + "-broadcast-timeout", - Amount: tc.amount, - }) - if err != nil { - t.Fatalf("CreateOrGet: %v", err) - } - - runner := NewWalletOperationRunner(repos, tc.operator, nil, time.Millisecond, nil, WithWalletOperationTimeouts(time.Millisecond, 0)) - - started := time.Now() - runner.runOnce(ctx) - if time.Since(started) > time.Second { - t.Fatal("runOnce did not return after broadcast timeout") - } - - got, err := repos.WalletOperations.GetByID(ctx, op.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.Status != model.WalletOperationStatusRunning { - t.Fatalf("status = %q, want running until lease expiry", got.Status) - } - if got.TxHash != nil { - t.Fatalf("tx_hash = %v, want nil", got.TxHash) - } - }) - } -} - -func TestWalletOperationRunner_RemainsHealthyWhileBroadcasting(t *testing.T) { - for _, tc := range []struct { - name string - opType model.WalletOperationType - amount string - operator func(started, release chan struct{}) *fakeWalletOperator - }{ - { - name: "fund", - opType: model.WalletOperationTypeFund, - amount: "100", - operator: func(started, release chan struct{}) *fakeWalletOperator { - return &fakeWalletOperator{ - fundHash: common.HexToHash("0x123").Hex(), - onFund: func(context.Context) { - close(started) - <-release - }, - } - }, - }, - { - name: "withdraw", - opType: model.WalletOperationTypeWithdraw, - amount: "100", - operator: func(started, release chan struct{}) *fakeWalletOperator { - return &fakeWalletOperator{ - withdrawHash: common.HexToHash("0x123").Hex(), - onWithdraw: func(context.Context) { - close(started) - <-release - }, - } - }, - }, - { - name: "approve", - opType: model.WalletOperationTypeApprove, - amount: "0", - operator: func(started, release chan struct{}) *fakeWalletOperator { - return &fakeWalletOperator{ - approveHash: common.HexToHash("0x123").Hex(), - onApprove: func(context.Context) { - close(started) - <-release - }, - } - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - if _, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: tc.opType, - ClientRequestID: string(tc.opType) + "-slow-broadcast", - Amount: tc.amount, - }); err != nil { - t.Fatalf("CreateOrGet: %v", err) - } - - started := make(chan struct{}) - release := make(chan struct{}) - var releaseOnce sync.Once - defer releaseOnce.Do(func() { close(release) }) - - operator := tc.operator(started, release) - runner := NewWalletOperationRunner(repos, operator, nil, time.Nanosecond, nil) - - done := make(chan struct{}) - go func() { - runner.runOnce(ctx) - close(done) - }() - select { - case <-started: - case <-time.After(time.Second): - t.Fatal("wallet broadcast did not start") - } - - if !runner.Healthy() { - t.Fatal("runner is unhealthy during an active wallet broadcast") - } - - releaseOnce.Do(func() { close(release) }) - select { - case <-done: - case <-time.After(time.Second): - t.Fatal("runOnce did not finish after releasing broadcast") - } - }) - } -} - -func TestWalletOperationRunner_RecoversSubmittedAndMarksExpiredRunningUnknown(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - submitted := seedSubmittedWalletOperation(t, repos, model.WalletOperationTypeFund, "submitted-1", "10", common.HexToHash("0x123").Hex()) - expired, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: model.WalletOperationTypeFund, - ClientRequestID: "expired-1", - Amount: "20", - }) - if err != nil { - t.Fatalf("CreateOrGet expired: %v", err) - } - now := time.Now() - if _, err := db.NewUpdate(). - Model(expired). - Set("status = ?", model.WalletOperationStatusRunning). - Set("started_at = ?", now.Add(-2*time.Second)). - Set("lease_until = ?", now.Add(-time.Second)). - Set("updated_at = ?", now). - WherePK(). - Exec(ctx); err != nil { - t.Fatalf("seed expired running operation: %v", err) - } - - receipts := &fakeWalletReceiptChecker{receipts: map[common.Hash]*ethtypes.Receipt{ - common.HexToHash(*submitted.TxHash): {Status: ethtypes.ReceiptStatusSuccessful}, - }} - publisher := &fakeWalletEventPublisher{} - runner := NewWalletOperationRunner(repos, &fakeWalletOperator{}, receipts, time.Millisecond, nil, WithWalletOperationEventPublisher(publisher)) - - runner.runOnce(ctx) - - gotSubmitted, err := repos.WalletOperations.GetByID(ctx, submitted.ID) - if err != nil { - t.Fatalf("GetByID submitted: %v", err) - } - if gotSubmitted.Status != model.WalletOperationStatusConfirmed { - t.Fatalf("submitted status = %q, want confirmed", gotSubmitted.Status) - } - gotExpired, err := repos.WalletOperations.GetByID(ctx, expired.ID) - if err != nil { - t.Fatalf("GetByID expired: %v", err) - } - if gotExpired.Status != model.WalletOperationStatusUnknown { - t.Fatalf("expired status = %q, want unknown", gotExpired.Status) - } - if !publisher.hasStatus(model.WalletOperationStatusUnknown) { - t.Fatalf("published statuses = %v, want unknown", publisher.statuses()) - } -} - -func seedSubmittedWalletOperation(t *testing.T, repos *repository.Repositories, opType model.WalletOperationType, requestID, amount, txHash string) *model.WalletOperation { - t.Helper() - ctx := context.Background() - op, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: opType, - ClientRequestID: requestID, - Amount: amount, - }) - if err != nil { - t.Fatalf("CreateOrGet: %v", err) - } - claimed, err := repos.WalletOperations.ClaimPending(ctx, time.Minute) - if err != nil { - t.Fatalf("ClaimPending: %v", err) - } - if claimed.ID != op.ID { - t.Fatalf("claimed ID = %d, want %d", claimed.ID, op.ID) - } - if err := repos.WalletOperations.MarkSubmitted(ctx, op.ID, txHash); err != nil { - t.Fatalf("MarkSubmitted: %v", err) - } - got, err := repos.WalletOperations.GetByID(ctx, op.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - return got -} - -type fakeWalletOperator struct { - fundHash string - withdrawHash string - approveHash string - fundAmount *big.Int - withdrawAmount *big.Int - approveCalled bool - onFund func(context.Context) - onWithdraw func(context.Context) - onApprove func(context.Context) - blockFund bool - blockWithdraw bool - blockApprove bool - fundErr error - withdrawErr error - approveErr error -} - -func (f *fakeWalletOperator) FundUSDFC(ctx context.Context, amount *big.Int) (string, error) { - f.fundAmount = new(big.Int).Set(amount) - if f.onFund != nil { - f.onFund(ctx) - } - if f.blockFund { - <-ctx.Done() - return "", ctx.Err() - } - if f.fundErr != nil { - return "", f.fundErr - } - return f.fundHash, nil -} - -func (f *fakeWalletOperator) WithdrawUSDFC(ctx context.Context, amount *big.Int) (string, error) { - f.withdrawAmount = new(big.Int).Set(amount) - if f.onWithdraw != nil { - f.onWithdraw(ctx) - } - if f.blockWithdraw { - <-ctx.Done() - return "", ctx.Err() - } - if f.withdrawErr != nil { - return "", f.withdrawErr - } - return f.withdrawHash, nil -} - -func (f *fakeWalletOperator) ApproveFWSS(ctx context.Context) (string, error) { - f.approveCalled = true - if f.onApprove != nil { - f.onApprove(ctx) - } - if f.blockApprove { - <-ctx.Done() - return "", ctx.Err() - } - if f.approveErr != nil { - return "", f.approveErr - } - return f.approveHash, nil -} - -type fakeWalletReceiptChecker struct { - receipts map[common.Hash]*ethtypes.Receipt - err error - block bool -} - -func (f *fakeWalletReceiptChecker) TransactionReceipt(ctx context.Context, txHash common.Hash) (*ethtypes.Receipt, error) { - if f.block { - <-ctx.Done() - return nil, ctx.Err() - } - if f.err != nil { - return nil, f.err - } - return f.receipts[txHash], nil -} - -type flakyMarkSubmittedWalletOperationRepo struct { - repository.WalletOperationRepository - failuresRemaining int - markSubmittedCalls int -} - -func (f *flakyMarkSubmittedWalletOperationRepo) MarkSubmitted(ctx context.Context, id int64, txHash string) error { - f.markSubmittedCalls++ - if f.failuresRemaining > 0 { - f.failuresRemaining-- - return errors.New("temporary database error") - } - return f.WalletOperationRepository.MarkSubmitted(ctx, id, txHash) -} - -type fakeWalletEventPublisher struct { - mu sync.Mutex - events []map[string]any -} - -func (f *fakeWalletEventPublisher) Publish(topic string, payload map[string]any) { - f.mu.Lock() - defer f.mu.Unlock() - if topic != walletOperationUpdatedTopic { - return - } - f.events = append(f.events, payload) -} - -func (f *fakeWalletEventPublisher) hasStatus(status model.WalletOperationStatus) bool { - for _, got := range f.statuses() { - if got == status { - return true - } - } - return false -} - -func (f *fakeWalletEventPublisher) statuses() []model.WalletOperationStatus { - f.mu.Lock() - defer f.mu.Unlock() - statuses := make([]model.WalletOperationStatus, 0, len(f.events)) - for _, event := range f.events { - op, ok := event["operation"].(walletOperationEventPayload) - if ok { - statuses = append(statuses, model.WalletOperationStatus(op.Status)) - } - } - return statuses -} - -func TestWalletOperationRunner_BroadcastErrorFailsOperation(t *testing.T) { - for _, tc := range []struct { - name string - opType model.WalletOperationType - amount string - err error - }{ - {name: "fund", opType: model.WalletOperationTypeFund, amount: "100", err: errors.New("fund rpc node offline")}, - {name: "withdraw", opType: model.WalletOperationTypeWithdraw, amount: "100", err: errors.New("withdraw rpc node offline")}, - {name: "approve", opType: model.WalletOperationTypeApprove, amount: "0", err: errors.New("approve rpc node offline")}, - } { - t.Run(tc.name, func(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - ctx := context.Background() - op, _, err := repos.WalletOperations.CreateOrGet(ctx, repository.CreateWalletOperationInput{ - Type: tc.opType, - ClientRequestID: string(tc.opType) + "-broadcast-error", - Amount: tc.amount, - }) - if err != nil { - t.Fatalf("CreateOrGet: %v", err) - } - - operator := &fakeWalletOperator{fundErr: tc.err, withdrawErr: tc.err, approveErr: tc.err} - runner := NewWalletOperationRunner(repos, operator, nil, time.Millisecond, nil) - - runner.runOnce(ctx) - - got, err := repos.WalletOperations.GetByID(ctx, op.ID) - if err != nil { - t.Fatalf("GetByID: %v", err) - } - if got.Status != model.WalletOperationStatusFailed { - t.Fatalf("status = %q, want failed", got.Status) - } - if got.LastError == nil { - t.Fatalf("last_error is nil, want %q", tc.err.Error()) - } - if !strings.Contains(*got.LastError, tc.err.Error()) { - t.Fatalf("last_error = %q, want %q", *got.LastError, tc.err.Error()) - } - }) - } -} diff --git a/tests/integration/calibration/calibration_test.go b/tests/integration/calibration/calibration_test.go index ebfa0e2..e422f97 100644 --- a/tests/integration/calibration/calibration_test.go +++ b/tests/integration/calibration/calibration_test.go @@ -251,9 +251,15 @@ func (r *calibrationRuntime) PrepareConfig() error { cfg.Filecoin.Observability.Concurrency = 4 cfg.Cache.MaxSizeGB = 1 cfg.Cache.EvictionPolicy = "after_upload" - cfg.Worker.Upload = config.WorkerPoolConfig{Concurrency: 1, PollInterval: 5 * time.Second, MaxRetries: 3} - cfg.Worker.Evictor = config.WorkerPoolConfig{Concurrency: 1, PollInterval: 5 * time.Second, MaxRetries: 3} - cfg.Worker.StorageCleanup = config.WorkerPoolConfig{Concurrency: 1, PollInterval: 30 * time.Second, MaxRetries: 3} + cfg.Worker.Tasks = config.TaskWorkerConfig{ + Concurrency: 4, + PollInterval: 5 * time.Second, + LeaseDuration: 5 * time.Minute, + MaxRetries: 3, + Retention: 7 * 24 * time.Hour, + ProviderMutationConcurrency: 4, + DestructiveMutationConcurrency: 2, + } cfg.Logging.Level = "warn" cfg.Logging.Format = "text" cfg.Logging.S3Access.Enabled = false @@ -724,11 +730,11 @@ func waitForStoredObject(t *testing.T, admin *e2e.AdminClient, bucket, key strin return storedObject{Snapshot: raw}, false, nil } item := list.Objects[0] - if item.Status == "warning" || item.State == "failed" || item.UploadStatus == "failed" || item.UploadStatus == "rejected" { + if item.Status == "warning" || item.State == "failed" { return storedObject{VersionID: item.CurrentVersionID, Snapshot: raw}, false, fmt.Errorf("object entered failed state: %s", e2e.DiagnosticValue(item)) } resolveUploadDependency(t, ctx, admin, item.CurrentVersionID, actions, taskProgress) - return storedObject{VersionID: item.CurrentVersionID, Snapshot: raw}, item.CurrentVersionID != "" && item.UploadStatus == "complete" && item.Location.Filecoin, nil + return storedObject{VersionID: item.CurrentVersionID, Snapshot: raw}, item.CurrentVersionID != "" && item.State == "stored" && item.Location.Filecoin, nil }, e2e.WithPollInterval(5*time.Second)) } @@ -738,7 +744,7 @@ func resolveUploadDependency(t *testing.T, ctx context.Context, admin *e2e.Admin return } var tasks e2e.TaskListResponse - raw, err := admin.GetJSON(ctx, "/api/v1/tasks?type=upload&limit=100", &tasks) + raw, err := admin.GetJSON(ctx, "/api/v1/tasks?type=upload_plan&limit=100", &tasks) if err != nil { t.Fatalf("GET upload tasks: %v; body=%s", err, e2e.Redact(raw)) } @@ -746,7 +752,7 @@ func resolveUploadDependency(t *testing.T, ctx context.Context, admin *e2e.Admin progress.Changed(t, "upload tasks", uploadTaskSummary(tasks, versionID)) } for _, task := range tasks.Tasks { - if task.RefVersionID != versionID || task.Status != "waiting" || task.WaitReason == nil || *task.WaitReason != "dependency" { + if !taskHasSubject(task, "object_version", versionID) || task.Status != "pending" || task.WaitReason == nil || *task.WaitReason != "funding" { continue } message := nullableString(task.StatusMessage) @@ -817,7 +823,7 @@ func waitForCommittedCopies(t *testing.T, admin *e2e.AdminClient, bucket, versio return provenance, false, err } progress.Changed(t, "provenance", provenanceSummary(provenance)) - if provenance.UploadStatus == "failed" || provenance.UploadStatus == "rejected" { + if provenance.State == "failed" { return provenance, false, fmt.Errorf("upload failed: %s", e2e.DiagnosticValue(provenance)) } if provenance.RequestedCopies != integrationCopies || provenance.SuccessCopies != integrationCopies || len(provenance.Copies) != integrationCopies { @@ -871,7 +877,7 @@ func waitForCompletedUploadTasks(t *testing.T, admin *e2e.AdminClient, versionID lastSummary := "none" e2e.Eventually(t, t.Context(), uploadTaskTimeout, "all upload tasks completed without failures", func(ctx context.Context) (string, bool, error) { var tasks e2e.TaskListResponse - _, err := admin.GetJSON(ctx, "/api/v1/tasks?type=upload&limit=100", &tasks) + _, err := admin.GetJSON(ctx, "/api/v1/tasks?type=upload_plan&limit=100", &tasks) if err != nil { return lastSummary, false, err } @@ -879,13 +885,13 @@ func waitForCompletedUploadTasks(t *testing.T, admin *e2e.AdminClient, versionID progress.Changed(t, "upload tasks", lastSummary) seen, active, failed := 0, 0, 0 for _, task := range tasks.Tasks { - if task.RefVersionID != versionID { + if !taskHasSubject(task, "object_version", versionID) { continue } seen++ switch task.Status { case "completed": - case "failed", "exhausted", "cancelled": + case "failed", "cancelled": failed++ default: active++ @@ -1112,12 +1118,11 @@ func objectListSummary(list e2e.ObjectListResponse) string { } item := list.Objects[0] return fmt.Sprintf( - "count=%d version=%s state=%s status=%s upload=%s cache=%t filecoin=%t", + "count=%d version=%s state=%s status=%s cache=%t filecoin=%t", len(list.Objects), item.CurrentVersionID, item.State, item.Status, - item.UploadStatus, item.Location.Cache, item.Location.Filecoin, ) @@ -1125,9 +1130,9 @@ func objectListSummary(list e2e.ObjectListResponse) string { func provenanceSummary(provenance e2e.ProvenanceResponse) string { lines := []string{fmt.Sprintf( - "status=%s upload=%s requested=%d success=%d", + "state=%s status=%s requested=%d success=%d", + provenance.State, provenance.Status, - provenance.UploadStatus, provenance.RequestedCopies, provenance.SuccessCopies, )} @@ -1148,17 +1153,16 @@ func provenanceSummary(provenance e2e.ProvenanceResponse) string { func uploadTaskSummary(tasks e2e.TaskListResponse, versionID string) string { lines := make([]string, 0, len(tasks.Tasks)) for _, task := range tasks.Tasks { - if task.RefVersionID != versionID { + if !taskHasSubject(task, "object_version", versionID) { continue } lines = append(lines, fmt.Sprintf( - "- id=%d type=%s stage=%s status=%s retry=%d claimed=%s wait=%s message=%s error=%s", + "- id=%d type=%s status=%s retry=%d available=%s wait=%s message=%s error=%s", task.ID, task.Type, - nullableString(task.Stage), task.Status, task.RetryCount, - nullableString(task.ClaimedAt), + task.AvailableAt, nullableString(task.WaitReason), shortLogValue(nullableString(task.StatusMessage)), shortLogValue(nullableString(task.LastError)), @@ -1170,6 +1174,11 @@ func uploadTaskSummary(tasks e2e.TaskListResponse, versionID string) string { return strings.Join(lines, "\n") } +func taskHasSubject(task e2e.TaskItem, subjectType, subjectKey string) bool { + return task.SubjectType != nil && task.SubjectKey != nil && + *task.SubjectType == subjectType && *task.SubjectKey == subjectKey +} + func observabilitySnapshotSummary(providers e2e.ProviderObservationPage, dataSets e2e.DataSetObservationPage) string { lines := []string{fmt.Sprintf( "providers total=%d available=%d degraded=%d unavailable=%d unknown=%d", diff --git a/tests/system/golden_test.go b/tests/system/golden_test.go index c777d7d..d7ae736 100644 --- a/tests/system/golden_test.go +++ b/tests/system/golden_test.go @@ -77,7 +77,7 @@ func TestSystemGoldenPath(t *testing.T) { return struct { VersionID string Snapshot string - }{VersionID: item.CurrentVersionID, Snapshot: raw}, item.UploadStatus == "complete" && item.Location.Filecoin, nil + }{VersionID: item.CurrentVersionID, Snapshot: raw}, item.State == "stored" && item.Location.Filecoin, nil }) provenancePath := "/api/v1/buckets/" + bucket + "/objects/provenance?version_id=" + url.QueryEscape(object.VersionID) @@ -97,19 +97,21 @@ func TestSystemGoldenPath(t *testing.T) { } return value, true, nil }) - if provenance.UploadStatus != "complete" { - t.Fatalf("provenance upload status = %q, want complete", provenance.UploadStatus) + if provenance.State != "stored" { + t.Fatalf("provenance state = %q, want stored", provenance.State) } e2e.Eventually(t, t.Context(), 5*time.Second, "completed upload tasks", func(ctx context.Context) (string, bool, error) { var tasks e2e.TaskListResponse - raw, err := admin.GetJSON(ctx, "/api/v1/tasks?type=upload&limit=100", &tasks) + raw, err := admin.GetJSON(ctx, "/api/v1/tasks?type=upload_plan&limit=100", &tasks) if err != nil { return raw, false, err } + // Ingest is planned for the content, not for one version of it, and this + // bucket holds exactly one object, so every plan task here is its plan. completed, failed := 0, 0 for _, task := range tasks.Tasks { - if task.RefVersionID != object.VersionID { + if task.SubjectType == nil || *task.SubjectType != "storage_content" { continue } switch task.Status { diff --git a/tests/testutil/e2e/admin.go b/tests/testutil/e2e/admin.go index 0e695a2..e5297c5 100644 --- a/tests/testutil/e2e/admin.go +++ b/tests/testutil/e2e/admin.go @@ -57,7 +57,6 @@ type ObjectListItem struct { CurrentVersionID string `json:"current_version_id"` State string `json:"state"` Status string `json:"status"` - UploadStatus string `json:"upload_status"` Location ObjectLocation `json:"location"` } @@ -67,8 +66,8 @@ type ObjectLocation struct { } type ProvenanceResponse struct { + State string `json:"state"` Status string `json:"status"` - UploadStatus string `json:"upload_status"` RequestedCopies int `json:"requested_copies"` SuccessCopies int `json:"success_copies"` Copies []ProvenanceCopy `json:"copies"` @@ -83,20 +82,24 @@ type ProvenanceCopy struct { } type TaskListResponse struct { - Tasks []TaskItem `json:"tasks"` + Tasks []TaskItem `json:"tasks"` + NextCursor *int64 `json:"next_cursor,omitempty"` } type TaskItem struct { - ID int64 `json:"id"` - Type string `json:"type"` - Stage *string `json:"stage,omitempty"` - Status string `json:"status"` - RefVersionID string `json:"ref_version_id"` - RetryCount int `json:"retry_count"` - LastError *string `json:"last_error,omitempty"` - StatusMessage *string `json:"status_message,omitempty"` - WaitReason *string `json:"wait_reason,omitempty"` - ClaimedAt *string `json:"claimed_at,omitempty"` + ID int64 `json:"id"` + Type string `json:"type"` + Operation string `json:"operation"` + Status string `json:"status"` + PresentationStatus string `json:"presentation_status"` + SubjectType *string `json:"subject_type,omitempty"` + SubjectKey *string `json:"subject_key,omitempty"` + RetryCount int `json:"retry_count"` + RetryLimit *int `json:"retry_limit,omitempty"` + LastError *string `json:"last_error,omitempty"` + StatusMessage *string `json:"status_message,omitempty"` + WaitReason *string `json:"wait_reason,omitempty"` + AvailableAt string `json:"available_at"` } type ReadinessResult struct { diff --git a/tests/testutil/e2e/eventually.go b/tests/testutil/e2e/eventually.go index d0dd27f..6f2cdeb 100644 --- a/tests/testutil/e2e/eventually.go +++ b/tests/testutil/e2e/eventually.go @@ -42,7 +42,10 @@ func Eventually[T any]( var lastErr error for { value, ready, err := poll(waitCtx) - last, lastErr = value, err + if err == nil { + last = value + } + lastErr = err if err == nil && ready { return value } diff --git a/ui/src/api/client.ts b/ui/src/api/client.ts index bc23d1a..bf028fc 100644 --- a/ui/src/api/client.ts +++ b/ui/src/api/client.ts @@ -107,8 +107,8 @@ export interface OverviewData { } tasks: { by_status: Record - attention: { failed: number; exhausted: number } - active_pipeline: Array<{ pipeline: string; by_status: Record; total: number }> + attention: { failed: number } + active_pipeline: Array<{ operation: string; by_status: Record; total: number }> } cache: { used_bytes: number; max_bytes: number } workers: Record @@ -189,15 +189,15 @@ export interface OverviewFilecoinStorageHealth { partial_errors: Record } +export type BucketStatus = 'provisioning' | 'ready' + export interface BucketItem { id: number name: string owner_access_key: string | null - default_copies: number | null - effective_copies: number - minimum_durable_copies: number | null - effective_minimum_durable_copies: number - status: string + default_copies: number + minimum_durable_copies: number + status: BucketStatus object_count: number total_size_bytes: number storage_health: BucketStorageHealthSummary @@ -363,10 +363,8 @@ export interface BucketMutationResponse { id: number name: string owner_access_key: string | null - default_copies: number | null - effective_copies: number - minimum_durable_copies: number | null - effective_minimum_durable_copies: number + default_copies: number + minimum_durable_copies: number status: string } @@ -376,16 +374,7 @@ export interface ObjectLocation { } export type ObjectStatus = 'uploading' | 'syncing' | 'success' | 'warning' | 'unavailable' -export type ObjectState = 'cached' | 'uploading' | 'committing' | 'replicating' | 'stored' | 'cache_evicted' | 'failed' -export type ObjectUploadStatus = - | 'running' - | 'ingress_ready' - | 'readable' - | 'complete' - | 'failed' - | 'rejected' - | 'superseded' - +export type ObjectState = 'cached' | 'uploading' | 'committing' | 'replicating' | 'stored' | 'failed' export interface UploadTransferProgress { scope: 'ingress_store' attempt: number @@ -405,7 +394,6 @@ export interface ObjectItem { size: number state: ObjectState status: ObjectStatus - upload_status?: ObjectUploadStatus progress?: UploadTransferProgress location: ObjectLocation content_type: string @@ -477,7 +465,7 @@ export interface RestoreObjectVersionResponse { export interface PermanentDeleteObjectResponse { key: string version_id: string - cache_cleanup_status: string + cache_release: string storage_cleanup_task_id?: number } @@ -497,7 +485,6 @@ export interface ObjectVersionItem { state: ObjectState status: ObjectStatus is_delete_marker: boolean - upload_status?: ObjectUploadStatus progress?: UploadTransferProgress location: ObjectLocation content_type: string @@ -556,9 +543,7 @@ export interface ObjectStatusDetail { version_id: string state: ObjectState status: ObjectStatus - upload_status?: ObjectUploadStatus progress?: UploadTransferProgress - failed_at_state?: string message?: string updated_at: string } @@ -610,184 +595,54 @@ export interface ObjectProvenanceCopy { attention_at?: string } -export interface ObjectProvenanceFailure { - attempt_index: number - provider_id?: string - provider_identity?: ProviderIdentity - transfer_method: string - stage?: string - error?: string -} - export interface ObjectProvenance { version_id: string state: ObjectState status: ObjectStatus - upload_status?: ObjectUploadStatus progress?: UploadTransferProgress piece_cid?: string requested_copies: number success_copies: number copy_health: CopyHealthSummary copies: ObjectProvenanceCopy[] - failures: ObjectProvenanceFailure[] updated_at: string } export interface TaskItem { id: number type: string - stage?: string - upload_id?: number - copy_index?: number - ref_type: string - ref_id: number - bucket_name?: string - ref_version_id: string - status: string - progress?: TaskProgress + operation: string + status: 'pending' | 'running' | 'completed' | 'failed' | 'cancelled' + presentation_status: + | 'queued' + | 'scheduled' + | 'waiting' + | 'running' + | 'completed' + | 'failed' + | 'cancelled' + | 'dismissed' + subject_type?: string + subject_key?: string retry_count: number - max_retries: number + retry_limit?: number + retryable: boolean + acknowledgeable: boolean last_error?: string status_message?: string wait_reason?: string - scheduled_at: string - claimed_at?: string - completed_at?: string + failure_reason?: string + available_at: string + started_at?: string + finished_at?: string + acknowledged_at?: string + created_at: string + updated_at: string } export interface TaskListResponse { tasks: TaskItem[] - total: number - limit: number - offset: number -} - -export type TaskDiagnosticCurrentState = - | 'not_applicable' - | 'preparing' - | 'transferring' - | 'waiting_for_chain' - | 'confirmed' - | 'rejected' - | 'mismatch' - | 'unavailable' - | 'unknown' - -export type TaskDiagnosticNextAction = - | 'none' - | 'wait' - | 'retry_task' - | 'check_wallet_funds' - | 'check_wallet_approval' - | 'inspect_provider' - | 'inspect_task' - -export type TaskDiagnosticOperation = 'none' | 'prepare_upload' | 'transfer_piece' | 'create_data_set' | 'add_pieces' - -export interface TaskDiagnosticTaskFacts { - id?: number - type: string - stage?: string - status: string - retry_count?: number - max_retries?: number - last_error?: string - status_message?: string - wait_reason?: string - scheduled_at?: string -} - -export interface TaskDiagnosticUploadFacts { - id?: number - status?: string - requested_copies?: number - error_message?: string - accept_error?: string -} - -export interface TaskDiagnosticCopyFacts { - upload_id?: number - copy_index?: number - status?: string - provider_id?: string - storage_data_set_id?: number - chain_data_set_id?: string - piece_id?: string - transfer_method?: string - commit_transaction_id?: string - last_error?: string -} - -export interface TaskDiagnosticDataSetFacts { - id?: number - status?: string - provider_id?: string - copy_index?: number - chain_data_set_id?: string - client_data_set_id?: string - create_transaction_id?: string - create_status_url?: string - last_error?: string -} - -export interface TaskDiagnosticProviderFacts { - provider_id?: string - status?: ObservabilityStatus - reason_codes?: string[] - service_url?: string - health_status?: string - last_error?: string -} - -export interface TaskDiagnosticTransactionFacts { - kind: TaskDiagnosticOperation - status_url?: string - service_url?: string - data_set_id?: string - transaction_id?: string - piece_count?: number -} - -export type TaskDiagnosticLiveState = - | 'skipped' - | 'pending' - | 'confirmed' - | 'rejected' - | 'mismatch' - | 'unavailable' - | 'unknown' - -export interface TaskDiagnosticLiveCheck { - state: TaskDiagnosticLiveState - status_url?: string - tx_status?: string - data_set_id?: string - data_set_created?: boolean - pieces_added?: boolean - piece_count?: number - confirmed_piece_ids?: string[] - error?: string -} - -export interface TaskDiagnosticEvidence { - task: TaskDiagnosticTaskFacts - upload?: TaskDiagnosticUploadFacts - copy?: TaskDiagnosticCopyFacts - data_set?: TaskDiagnosticDataSetFacts - provider?: TaskDiagnosticProviderFacts - transaction?: TaskDiagnosticTransactionFacts - live_check?: TaskDiagnosticLiveCheck - operation: TaskDiagnosticOperation -} - -export interface TaskDiagnostic { - checked_at: string - current_state: TaskDiagnosticCurrentState - signal: ObservabilitySignal - reason_codes: string[] - next_action: TaskDiagnosticNextAction - evidence: TaskDiagnosticEvidence + next_cursor?: number } export interface TaskStatusCount { @@ -796,55 +651,6 @@ export interface TaskStatusCount { count: number } -export interface TaskRefObjectDetail { - bucket_name: string - key: string - version_id: string - size: number - state: ObjectState - status: ObjectStatus - upload_status?: ObjectUploadStatus - progress?: UploadTransferProgress - location: ObjectLocation - content_type: string - updated_at: string -} - -export interface TaskStorageCleanupCopyDetail { - copy_index: number - provider_id?: string - data_set_id?: string - client_data_set_id?: string - piece_id?: string - piece_cid: string - status: string - delete_tx_hash?: string - last_error?: string -} - -export interface TaskStorageCleanupDeletedVersionDetail { - bucket_name: string - key: string - version_id: string - size: number - deleted_at: string -} - -export interface TaskStorageCleanupDetail { - upload_id: number - deleted_versions: TaskStorageCleanupDeletedVersionDetail[] - copies: TaskStorageCleanupCopyDetail[] -} - -export interface TaskRefDetail { - ref_type: string - ref_id: number - ref_version_id: string - bucket_name?: string - object: TaskRefObjectDetail | null - storage_cleanup?: TaskStorageCleanupDetail -} - export interface PaymentAccountData { funds: string | null available_funds: string | null @@ -920,7 +726,6 @@ export interface WalletOperation { status: WalletOperationStatus tx_hash?: string last_error?: string - lease_until?: string started_at?: string submitted_at?: string completed_at?: string @@ -1029,16 +834,17 @@ export interface SettingsCacheConfig { } export interface SettingsWorkerConfig { - upload: SettingsWorkerPoolConfig - provider_replacement: SettingsWorkerPoolConfig - evictor: SettingsWorkerPoolConfig - storage_cleanup: SettingsWorkerPoolConfig + tasks: SettingsTaskWorkerConfig } -export interface SettingsWorkerPoolConfig { +export interface SettingsTaskWorkerConfig { concurrency: number poll_interval: string + lease_duration: string max_retries: number + retention: string + provider_mutation_concurrency: number + destructive_mutation_concurrency: number } export interface SettingsLoggingConfig { @@ -1106,10 +912,7 @@ export type SettingsUpdatePayload = Partial<{ } cache: Partial worker: Partial<{ - upload: Partial - provider_replacement: Partial - evictor: Partial - storage_cleanup: Partial + tasks: Partial }> logging: Partial> & { s3_access?: Partial @@ -1330,23 +1133,18 @@ export const api = { if (versionId) params.push(`version_id=${encodeURIComponent(versionId)}`) return `${BASE}/buckets/${encodeURIComponent(name)}/objects/download?${params.join('&')}` }, - getTasks: (params: { type?: string; stage?: string; status?: string; limit?: number; offset?: number }) => { + getTasks: (params: { type?: string; status?: string; limit?: number; cursor?: number }) => { const sp = new URLSearchParams() if (params.type) sp.set('type', params.type) - if (params.stage) sp.set('stage', params.stage) if (params.status) sp.set('status', params.status) if (params.limit) sp.set('limit', params.limit.toString()) - if (params.offset) sp.set('offset', params.offset.toString()) + if (params.cursor) sp.set('cursor', params.cursor.toString()) const qs = sp.toString() return fetchJSON(`/tasks${qs ? `?${qs}` : ''}`) }, getTaskStats: () => fetchJSON('/tasks/stats'), - getTaskRefDetail: (id: number) => fetchJSON(`/tasks/${id}/ref-detail`), - getTaskDiagnostic: (id: number, options?: APIRequestOptions) => - fetchJSON(`/tasks/${id}/diagnostic`, options), - refreshTaskDiagnostic: (id: number, options?: APIRequestOptions) => - fetchJSON(`/tasks/${id}/diagnostic/refresh`, { method: 'POST', ...options }), retryTask: (id: number) => fetchJSON(`/tasks/${id}/retry`, { method: 'POST' }), + acknowledgeTask: (id: number) => fetchJSON(`/tasks/${id}/acknowledge`, { method: 'POST' }), getSystemInfo: () => fetchJSON('/system/info'), getWorkers: () => fetchJSON<{ workers: Record }>('/workers'), getCacheStats: () => fetchJSON<{ used_bytes: number; max_bytes: number }>('/cache/stats'), diff --git a/ui/src/components/app/StatusBadge.tsx b/ui/src/components/app/StatusBadge.tsx index 95355ec..644693c 100644 --- a/ui/src/components/app/StatusBadge.tsx +++ b/ui/src/components/app/StatusBadge.tsx @@ -32,9 +32,9 @@ export function StatusBadge({ export function bucketStatusTone(status: string): StatusTone { switch (status) { - case 'active': + case 'ready': return 'success' - case 'creating': + case 'provisioning': case 'deleting': return 'warning' case 'create_failed': @@ -56,7 +56,6 @@ export function taskStatusTone(status: string): StatusTone { case 'waiting': return 'info' case 'failed': - case 'exhausted': return 'danger' default: return 'neutral' @@ -78,8 +77,6 @@ export function objectStateTone(state: string): StatusTone { case 'cached': case 'uploaded': return 'info' - case 'cache_evicted': - return 'neutral' default: return 'neutral' } diff --git a/ui/src/hooks/queries.ts b/ui/src/hooks/queries.ts index 5bdbb58..d3fd500 100644 --- a/ui/src/hooks/queries.ts +++ b/ui/src/hooks/queries.ts @@ -318,23 +318,14 @@ export function useRestoreBucketObjectVersion() { }) } -export function useTasks(taskType: string, stage: string, status: string, limit: number, offset: number) { +export function useTasks(taskType: string, status: string, limit: number, cursor?: number) { return useQuery({ - queryKey: ['tasks', taskType, stage, status, limit, offset], - queryFn: () => api.getTasks({ type: taskType, stage, status, limit, offset }), + queryKey: ['tasks', taskType, status, limit, cursor], + queryFn: () => api.getTasks({ type: taskType, status, limit, cursor }), refetchInterval: 10_000, }) } -export function useTaskRefDetail(taskId: number, enabled = true) { - return useQuery({ - queryKey: ['taskRefDetail', taskId], - queryFn: () => api.getTaskRefDetail(taskId), - enabled: Boolean(taskId && enabled), - staleTime: 60_000, - }) -} - export function useTaskStats() { return useQuery({ queryKey: ['taskStats'], diff --git a/ui/src/lib/bucket-copy-policy.ts b/ui/src/lib/bucket-copy-policy.ts index 98080cb..d1cd21c 100644 --- a/ui/src/lib/bucket-copy-policy.ts +++ b/ui/src/lib/bucket-copy-policy.ts @@ -1,66 +1,63 @@ import type { BucketItem } from '@/api/client' -type BucketCopyPolicy = Pick< - BucketItem, - 'default_copies' | 'effective_copies' | 'minimum_durable_copies' | 'effective_minimum_durable_copies' -> +type BucketCopyPolicy = Pick -export const inheritedCopyPolicyValue = 'inherit' -export const strictMinimumDurableCopiesValue = 'strict' export const copyPolicyOptions = Array.from({ length: 8 }, (_, index) => index + 1) export function bucketCopyPolicyValue(bucket: Pick) { - return bucket.default_copies == null ? inheritedCopyPolicyValue : bucket.default_copies.toString() + return bucket.default_copies.toString() } -export function bucketCopyPolicyLabel(bucket: BucketCopyPolicy) { - const copies = copyCountLabel(bucket.effective_copies) - return bucket.default_copies == null ? `Inherits global default (${copies})` : `Override (${copies})` -} - -export function bucketCopyPolicyInheritOptionLabel(bucket: BucketCopyPolicy, runtimeDefaultCopies?: number) { - const copies = bucket.default_copies == null ? bucket.effective_copies : runtimeDefaultCopies - if (copies == null) return 'Inherit current runtime default' - return `Inherit current runtime default (${copyCountLabel(copies)})` +export function bucketCopyPolicyLabel(bucket: Pick) { + return copyCountLabel(bucket.default_copies) } export function replicaTargetChoiceNote() { return 'Applies to new uploads. Existing objects keep the replica target they started with.' } +// Lowering the target would leave the replicas above it stored and billed with +// nothing to retire them, so the server refuses it. Offer those counts as +// visibly unavailable rather than hiding them, so the choice reads as +// temporarily closed instead of missing. +export function replicaTargetLocked(copies: number, currentCopies: number) { + return copies < currentCopies +} + +export function replicaTargetLockNote() { + return 'Lowering the replica target is not supported yet.' +} + export function bucketCopyPolicySavedMessage() { return 'Saved. New uploads use this replica target. Cache can be released after the selected count.' } export function minimumDurableCopiesChoiceNote() { - return 'Keeps cache until every replica of that upload is stored, including later Replicas increases.' + return 'Keeps cache until every replica of that upload is stored.' } export function minimumDurableCopiesFixedCountNote() { return 'Keeps this count if Replicas later increases.' } -export function minimumDurableCopiesValue(bucket: Pick) { - if (bucket.minimum_durable_copies == null) return strictMinimumDurableCopiesValue - if (bucket.minimum_durable_copies > bucket.effective_copies) return bucket.effective_copies.toString() - return bucket.minimum_durable_copies.toString() +export function minimumDurableCopiesValue(bucket: BucketCopyPolicy) { + return Math.min(bucket.minimum_durable_copies, bucket.default_copies).toString() } export function minimumDurableCopiesLabel(bucket: BucketCopyPolicy) { - if (bucket.minimum_durable_copies == null) return 'All replicas (strict)' - return `${bucket.effective_minimum_durable_copies} of ${bucket.effective_copies} ${bucket.effective_copies === 1 ? 'replica' : 'replicas'}` + const target = bucket.default_copies + return `${Math.min(bucket.minimum_durable_copies, target)} of ${target} ${target === 1 ? 'replica' : 'replicas'}` } export function minimumDurableCopiesOptionLabel(copies: number, targetCopies?: number | null) { const count = `${copies} ${copies === 1 ? 'replica' : 'replicas'}` if (targetCopies != null && copies === targetCopies) { - return `${count} (fixed count)` + return `${count} (all replicas)` } return count } -export function selectedTargetCopies(copyPolicy: string, runtimeDefaultCopies?: number) { - if (copyPolicy === inheritedCopyPolicyValue) return runtimeDefaultCopies ?? null +export function selectedTargetCopies(copyPolicy: string) { const copies = Number(copyPolicy) return Number.isInteger(copies) && copies >= 1 && copies <= 8 ? copies : null } @@ -71,22 +68,18 @@ export function minimumDurableCopiesOptions(targetCopies: number | null) { } export function clampMinimumDurableCopiesValue(value: string, targetCopies: number | null) { - if (value === strictMinimumDurableCopiesValue) return value - if (targetCopies == null) return strictMinimumDurableCopiesValue + if (targetCopies == null) return value const copies = Number(value) - if (!Number.isInteger(copies) || copies < 1) return strictMinimumDurableCopiesValue + if (!Number.isInteger(copies) || copies < 1) return targetCopies.toString() if (copies > targetCopies) return targetCopies.toString() return value } export function persistMinimumDurableCopies( selected: string, - storedMinimum: number | null, + storedMinimum: number, targetCopies: number | null -): number | null | undefined { - if (selected === strictMinimumDurableCopiesValue) { - return storedMinimum == null ? undefined : null - } +): number | undefined { const selectedNumber = Number(selected) if (!Number.isInteger(selectedNumber) || selectedNumber < 1 || selectedNumber > 8) { return undefined @@ -101,7 +94,6 @@ export function minimumDurableCopiesWarning() { } export function showsMinimumDurableCopiesWarning(value: string, targetCopies: number | null) { - if (value === strictMinimumDurableCopiesValue) return false const copies = Number(value) return Number.isInteger(copies) && targetCopies != null && copies < targetCopies } diff --git a/ui/src/lib/overview.ts b/ui/src/lib/overview.ts index f561bf8..7f3d3f8 100644 --- a/ui/src/lib/overview.ts +++ b/ui/src/lib/overview.ts @@ -11,16 +11,14 @@ export interface AttentionDisplayRow { value: number tone: AttentionTone target: 'buckets' | 'tasks' - taskStatus?: 'failed' | 'exhausted' + taskStatus?: 'failed' } export interface PipelineDisplayRow { key: string label: string total: number - queued: number - scheduled: number - waiting: number + pending: number running: number } @@ -71,24 +69,29 @@ const filecoinStorageHealthLevelStyles: Record = { - uploader: 'Upload', - evictor: 'Cache Evictor', - storage_cleanup: 'Replica Cleanup', - wallet_operations: 'Wallet Operations', + tasks: 'Task Engine', } -const pipelineOrder = ['prepare', 'upload', 'commit', 'sync', 'evict', 'cleanup'] as const -type PipelineKey = (typeof pipelineOrder)[number] - -const pipelineLabels: Record = { - prepare: 'Prepare', - upload: 'Upload', - commit: 'Commit', - sync: 'Sync', - evict: 'Evict', - cleanup: 'Cleanup', +const taskOperationLabels: Record = { + bucket_provision: 'Prepare bucket storage', + upload_plan: 'Prepare upload', + storage_dataset_ensure: 'Prepare storage', + storage_transfer_plan: 'Plan transfer', + storage_store: 'Store content', + storage_pull: 'Transfer content', + storage_commit_coordinate: 'Prepare confirmation', + storage_commit: 'Confirm storage', + provider_replacement_coordinate: 'Replace provider', + cache_capacity_reconcile: 'Manage local cache capacity', + cache_evict: 'Remove cached copy', + cache_reconcile_durability: 'Review durability', + storage_cleanup: 'Remove remote copy', + storage_dataset_retire: 'Retire service', + wallet_operation: 'Wallet request', + observability_refresh: 'Refresh health', + task_gc: 'Remove expired task records', } export function workerHealthRows(workers: Record) { @@ -130,31 +133,19 @@ export function attentionDisplayRows(attention: { target: 'tasks' as const, taskStatus: 'failed' as const, }, - { - key: 'exhausted_tasks', - label: 'Retry limit reached', - value: attention.tasks.exhausted, - tone: 'danger' as const, - target: 'tasks' as const, - taskStatus: 'exhausted' as const, - }, ].filter((row) => row.value > 0) } export function overviewPipelineRows(activePipeline: OverviewData['tasks']['active_pipeline']): PipelineDisplayRow[] { - const byPipeline = new Map(activePipeline.map((row) => [row.pipeline, row])) - return pipelineOrder.map((key) => { - const row = byPipeline.get(key) - return { - key, - label: pipelineLabels[key], - total: row?.total ?? 0, - queued: row?.by_status.queued ?? 0, - scheduled: row?.by_status.scheduled ?? 0, - waiting: row?.by_status.waiting ?? 0, - running: row?.by_status.running ?? 0, - } - }) + return activePipeline + .map((row) => ({ + key: row.operation, + label: taskOperationLabels[row.operation] ?? 'Background operation', + total: row.total, + pending: row.by_status.pending ?? 0, + running: row.by_status.running ?? 0, + })) + .sort((left, right) => left.label.localeCompare(right.label)) } export function filecoinStorageHealthLevelLabel(level: FilecoinStorageHealthLevel) { diff --git a/ui/src/lib/provider-identity-events.ts b/ui/src/lib/provider-identity-events.ts index 14b50e0..fe4717a 100644 --- a/ui/src/lib/provider-identity-events.ts +++ b/ui/src/lib/provider-identity-events.ts @@ -51,10 +51,5 @@ function patchObjectProvenance(data: ObjectProvenance | undefined, providerID: s changed = true return { ...copy, provider_identity: identity } }) - const failures = data.failures.map((failure) => { - if (failure.provider_id !== providerID) return failure - changed = true - return { ...failure, provider_identity: identity } - }) - return changed ? { ...data, copies, failures } : data + return changed ? { ...data, copies } : data } diff --git a/ui/src/lib/provider-replacement.ts b/ui/src/lib/provider-replacement.ts index 724476d..183df45 100644 --- a/ui/src/lib/provider-replacement.ts +++ b/ui/src/lib/provider-replacement.ts @@ -144,24 +144,6 @@ const replacementErrorMessages: Record = { replacement_task_running: 'Replacement work is still running. Try again shortly.', } -/** - * Provider replacement work resumes from the Data Sets view, never from the task - * queue, so the Tasks page must not offer a Retry the API will refuse. - */ -export function taskRetryableFromQueue(task: { stage?: string | null }) { - return ( - task.stage !== 'replace_provider' && task.stage !== 'retire_data_set' && task.stage !== 'retire_abandoned_target' - ) -} - -export function taskHasDataSetsRecovery(task: { stage?: string | null; status: string; bucket_name?: string | null }) { - return ( - Boolean(task.bucket_name) && - ['replace_provider', 'retire_data_set', 'retire_abandoned_target'].includes(task.stage ?? '') && - ['completed', 'failed', 'exhausted', 'cancelled'].includes(task.status) - ) -} - /** * What the operator should do next, in their terms. The recorded error is * developer diagnostics and belongs behind a detail view, not on the card. diff --git a/ui/src/lib/risk-confirmation.ts b/ui/src/lib/risk-confirmation.ts index da19609..b1b8ffd 100644 --- a/ui/src/lib/risk-confirmation.ts +++ b/ui/src/lib/risk-confirmation.ts @@ -147,59 +147,65 @@ export function collectSettingsRiskChanges( 'Changes when capacity-based cache eviction stops.' ) - addWorkerRiskChanges(changes, initial, next, envManaged, metadata, 'upload') - addWorkerRiskChanges(changes, initial, next, envManaged, metadata, 'provider_replacement') - addWorkerRiskChanges(changes, initial, next, envManaged, metadata, 'evictor') - - return changes -} - -function addWorkerRiskChanges( - changes: SettingsRiskChange[], - initial: SettingsEditableConfig, - next: SettingsEditableConfig, - envManaged: Record, - metadata: Record, - pool: 'upload' | 'provider_replacement' | 'evictor' -) { - const prefix = `worker.${pool}` - const initialPool = initial.worker[pool] - const nextPool = next.worker[pool] - - if (nextPool.concurrency > initialPool.concurrency) { - pushRiskChange( - changes, - metadata, - envManaged, - `${prefix}.concurrency`, - initialPool.concurrency, - nextPool.concurrency, + if (next.worker.tasks.concurrency > initial.worker.tasks.concurrency) { + addChanged( + 'worker.tasks.concurrency', + initial.worker.tasks.concurrency, + next.worker.tasks.concurrency, 'medium', 'Increases concurrent background work.' ) } - pushRiskChange( - changes, - metadata, - envManaged, - `${prefix}.poll_interval`, - initialPool.poll_interval, - nextPool.poll_interval, + addChanged( + 'worker.tasks.poll_interval', + initial.worker.tasks.poll_interval, + next.worker.tasks.poll_interval, + 'medium', + 'Changes how often ready background work is checked.' + ) + addChanged( + 'worker.tasks.lease_duration', + initial.worker.tasks.lease_duration, + next.worker.tasks.lease_duration, + 'medium', + 'Changes how quickly interrupted work can be recovered.' + ) + if (next.worker.tasks.max_retries > initial.worker.tasks.max_retries) { + addChanged( + 'worker.tasks.max_retries', + initial.worker.tasks.max_retries, + next.worker.tasks.max_retries, + 'medium', + 'Increases automatic recovery attempts for failed background work.' + ) + } + addChanged( + 'worker.tasks.retention', + initial.worker.tasks.retention, + next.worker.tasks.retention, 'medium', - 'Changes how often background work is polled.' + 'Changes how long finished background operations remain visible.' ) - if (nextPool.max_retries > initialPool.max_retries) { - pushRiskChange( - changes, - metadata, - envManaged, - `${prefix}.max_retries`, - initialPool.max_retries, - nextPool.max_retries, + if (next.worker.tasks.provider_mutation_concurrency > initial.worker.tasks.provider_mutation_concurrency) { + addChanged( + 'worker.tasks.provider_mutation_concurrency', + initial.worker.tasks.provider_mutation_concurrency, + next.worker.tasks.provider_mutation_concurrency, 'medium', - 'Increases retry attempts for failed background work.' + 'Increases concurrent requests that change remote storage.' ) } + if (next.worker.tasks.destructive_mutation_concurrency > initial.worker.tasks.destructive_mutation_concurrency) { + addChanged( + 'worker.tasks.destructive_mutation_concurrency', + initial.worker.tasks.destructive_mutation_concurrency, + next.worker.tasks.destructive_mutation_concurrency, + 'medium', + 'Increases concurrent remote cleanup and retirement requests.' + ) + } + + return changes } function pushRiskChange( diff --git a/ui/src/lib/settings-payload.ts b/ui/src/lib/settings-payload.ts index 46f8b61..1b3903e 100644 --- a/ui/src/lib/settings-payload.ts +++ b/ui/src/lib/settings-payload.ts @@ -44,29 +44,17 @@ export function buildSettingsPayload( if (include('cache.lru_low_watermark_percent')) payload.cache.lru_low_watermark_percent = form.cache.lru_low_watermark_percent - const upload: NonNullable['upload']> = {} - const providerReplacement: NonNullable['provider_replacement']> = {} - const evictor: NonNullable['evictor']> = {} - const storageCleanup: NonNullable['storage_cleanup']> = {} - if (include('worker.upload.concurrency')) upload.concurrency = form.worker.upload.concurrency - if (include('worker.upload.poll_interval')) upload.poll_interval = form.worker.upload.poll_interval - if (include('worker.upload.max_retries')) upload.max_retries = form.worker.upload.max_retries - if (include('worker.provider_replacement.concurrency')) - providerReplacement.concurrency = form.worker.provider_replacement.concurrency - if (include('worker.provider_replacement.poll_interval')) - providerReplacement.poll_interval = form.worker.provider_replacement.poll_interval - if (include('worker.provider_replacement.max_retries')) - providerReplacement.max_retries = form.worker.provider_replacement.max_retries - if (include('worker.evictor.concurrency')) evictor.concurrency = form.worker.evictor.concurrency - if (include('worker.evictor.poll_interval')) evictor.poll_interval = form.worker.evictor.poll_interval - if (include('worker.evictor.max_retries')) evictor.max_retries = form.worker.evictor.max_retries - if (include('worker.storage_cleanup.concurrency')) - storageCleanup.concurrency = form.worker.storage_cleanup.concurrency - if (include('worker.storage_cleanup.poll_interval')) - storageCleanup.poll_interval = form.worker.storage_cleanup.poll_interval - if (include('worker.storage_cleanup.max_retries')) - storageCleanup.max_retries = form.worker.storage_cleanup.max_retries - payload.worker = { upload, provider_replacement: providerReplacement, evictor, storage_cleanup: storageCleanup } + const tasks: NonNullable['tasks']> = {} + if (include('worker.tasks.concurrency')) tasks.concurrency = form.worker.tasks.concurrency + if (include('worker.tasks.poll_interval')) tasks.poll_interval = form.worker.tasks.poll_interval + if (include('worker.tasks.lease_duration')) tasks.lease_duration = form.worker.tasks.lease_duration + if (include('worker.tasks.max_retries')) tasks.max_retries = form.worker.tasks.max_retries + if (include('worker.tasks.retention')) tasks.retention = form.worker.tasks.retention + if (include('worker.tasks.provider_mutation_concurrency')) + tasks.provider_mutation_concurrency = form.worker.tasks.provider_mutation_concurrency + if (include('worker.tasks.destructive_mutation_concurrency')) + tasks.destructive_mutation_concurrency = form.worker.tasks.destructive_mutation_concurrency + payload.worker = { tasks } payload.logging = {} if (include('logging.level')) payload.logging.level = form.logging.level diff --git a/ui/src/lib/storage-status-labels.ts b/ui/src/lib/storage-status-labels.ts index cb26a1c..a08f56d 100644 --- a/ui/src/lib/storage-status-labels.ts +++ b/ui/src/lib/storage-status-labels.ts @@ -1,82 +1,13 @@ -import type { ObjectState, ObjectStatus, ObjectUploadStatus } from '@/api/client' +import type { ObjectState, ObjectStatus } from '@/api/client' import type { StatusTone } from '@/components/app/StatusBadge' import { titleCaseEnum } from './utils.ts' -export const taskStageOptions = [ - 'all', - 'prepare_upload', - 'ensure_dataset', - 'ingress_store', - 'ingress_commit', - 'peer_pull', - 'peer_commit', - 'repair_replica', - 'replace_provider', - 'reconcile_bucket_durability', -] as const - -export type TaskStageOption = (typeof taskStageOptions)[number] - -const taskStageLabels: Record | '', string> = { - prepare_upload: 'Prepare Filecoin storage', - ensure_dataset: 'Prepare replica target', - ingress_store: 'Upload source replica', - ingress_commit: 'Register source replica on-chain', - peer_pull: 'Sync peer replica', - peer_commit: 'Register peer replica on-chain', - repair_replica: 'Resume replica upload', - replace_provider: 'Replace provider', - reconcile_bucket_durability: 'Apply cache policy', - '': 'Upload', -} - -// The retirement stages run on the storage cleanup worker, so they are labelled -// but never offered in the Upload stage filter, which would return nothing. -const replacementCleanupStageLabels: Record = { - retire_data_set: 'Retire replaced provider', - retire_abandoned_target: 'Retire unused provider', -} - -export function replacementCleanupStageLabel(stage?: string | null) { - return stage ? replacementCleanupStageLabels[stage] : undefined -} - -export function taskTypeLabel(type?: string) { - switch (type) { - case 'all': - return 'All' - case 'upload': - return 'Upload' - case 'evict_cache': - return 'Evict Cache' - case 'storage_cleanup': - return 'Replica Cleanup' - default: - return titleCaseEnum(type) - } -} - -export function taskOperationOptionLabel(stage: TaskStageOption) { - if (stage === 'all') return 'All' - return taskStageLabels[stage] -} - -export function taskOperationLabel(task: { type?: string; stage?: string }) { - const stage = task.stage ?? '' - return taskOperationBaseLabel(task.type, stage) -} - export function taskReplicaLabel(task: { copy_index?: number; copyIndex?: number }) { const copyIndex = task.copy_index ?? task.copyIndex if (typeof copyIndex !== 'number') return '—' return replicaLabel(copyIndex) } -export function taskHasByteTransfer(task: { type?: string; stage?: string }) { - if (task.type !== 'upload') return false - return task.stage === 'ingress_store' || task.stage === '' -} - export function storageCleanupStatusLabel(copies: Array<{ status?: string }>) { if (copies.length === 0) return 'No remote replicas to delete' if (copies.some((copy) => copy.status === 'failed' || copy.status === 'unsupported')) return 'Needs attention' @@ -118,25 +49,6 @@ export function storageCleanupCopyStatusTone(status?: string): StatusTone { } } -function taskOperationBaseLabel(type: string | undefined, stage: string) { - // Retiring a replaced provider ends a paid service; it is not the ordinary - // replica deletion the storage_cleanup type otherwise means. - const replacementLabel = replacementCleanupStageLabel(stage) - if (replacementLabel) return replacementLabel - const stageLabel = taskStageLabels[stage as keyof typeof taskStageLabels] - if (stageLabel && stage !== '') return stageLabel - switch (type) { - case 'evict_cache': - return 'Evict local cache' - case 'storage_cleanup': - return 'Delete remote replicas' - case 'upload': - return 'Upload object' - default: - return titleCaseEnum(type) || 'Run task' - } -} - export function replicaLabel(copyIndex: number) { return `Replica ${copyIndex + 1}` } @@ -152,25 +64,6 @@ export function transferMethodLabel(method?: string) { } } -export function uploadStatusLabel(uploadStatus: ObjectUploadStatus, progressPercent: number | null = null) { - switch (uploadStatus) { - case 'running': - return progressPercent === null ? 'Preparing Filecoin storage' : `Uploading to Filecoin ${progressPercent}%` - case 'ingress_ready': - return 'Registering storage record on-chain' - case 'readable': - return 'Available, syncing replicas' - case 'complete': - return 'Stored on Filecoin · On-chain verified' - case 'failed': - return 'Needs attention' - case 'rejected': - return 'Upload rejected' - case 'superseded': - return 'Replaced by newer version' - } -} - export function objectStatusLabel(status: ObjectStatus) { switch (status) { case 'success': @@ -189,26 +82,21 @@ export function objectStatusLabel(status: ObjectStatus) { export function objectStateLabel( state: ObjectState | undefined, status: ObjectStatus, - uploadStatus?: ObjectUploadStatus, progressPercent: number | null = null ) { - if (uploadStatus === 'readable' && (state === 'stored' || state === 'cache_evicted')) { - return 'Available, syncing remaining replicas' - } - if (uploadStatus) return uploadStatusLabel(uploadStatus, progressPercent) switch (state) { case 'cached': return 'Stored in cache' case 'uploading': - return 'Uploading' + return progressPercent === null ? 'Uploading' : `Uploading to Filecoin ${progressPercent}%` case 'committing': return 'Registering storage record on-chain' case 'replicating': return 'Syncing replicas' case 'stored': - return 'Stored' - case 'cache_evicted': - return 'Stored remotely' + // Distinct from a single replica's "Stored" so the object summary and the + // per-copy rows stay tellable apart. + return 'Stored on Filecoin' case 'failed': return 'Needs attention' default: diff --git a/ui/src/lib/storage-topology.ts b/ui/src/lib/storage-topology.ts index 176bee1..520b6ed 100644 --- a/ui/src/lib/storage-topology.ts +++ b/ui/src/lib/storage-topology.ts @@ -195,7 +195,6 @@ export function localStatusTone(status: string): StorageTopologyTone { case 'pending': return 'warning' case 'failed': - case 'unavailable': return 'danger' default: return 'neutral' diff --git a/ui/src/lib/task-diagnostics.ts b/ui/src/lib/task-diagnostics.ts deleted file mode 100644 index 270f69d..0000000 --- a/ui/src/lib/task-diagnostics.ts +++ /dev/null @@ -1,421 +0,0 @@ -import type { TaskDiagnostic, TaskDiagnosticLiveState } from '@/api/client' -import type { StatusTone } from '@/components/app/StatusBadge' - -export interface TaskDiagnosticFactRow { - label: string - value: string - monospace?: boolean - detail?: boolean - detailValue?: string - displayMaxLength?: number -} - -export interface TaskDiagnosticViewModel { - title: string - primaryFacts: TaskDiagnosticFactRow[] - detailFacts: TaskDiagnosticFactRow[] -} - -export const taskDiagnosticSheetContentClassName = - 'data-[side=right]:w-[min(36rem,calc(100vw-2rem))] data-[side=right]:max-w-[calc(100vw-2rem)] data-[side=right]:sm:max-w-xl' - -const stateLabels: Record = { - not_applicable: 'Not applicable', - preparing: 'Preparing', - transferring: 'Transferring', - waiting_for_chain: 'Waiting for confirmation', - confirmed: 'Confirmed', - rejected: 'Rejected', - mismatch: 'Mismatch', - unavailable: 'Unavailable', - unknown: 'Unknown', -} - -const reasonLabels: Record = { - task_chain_pending: 'storage confirmation pending', - task_chain_confirmed: 'storage confirmation recorded', - task_transaction_rejected: 'provider transaction rejected', - task_piece_status_mismatch: 'piece confirmation mismatch', - task_diagnostic_unavailable: 'diagnostic evidence unavailable', - task_rpc_unavailable: 'provider or RPC unavailable', - task_insufficient_funds: 'payment wallet funds missing', - task_missing_approval: 'wallet approval missing', - task_missing_evidence: 'task evidence missing', - task_unknown_status: 'task status unknown', - task_not_applicable: 'task type not applicable', - copy_pending: 'copy still pending', -} - -export function buildTaskDiagnosticViewModel(diagnostic: TaskDiagnostic): TaskDiagnosticViewModel { - return { - title: diagnosticTitle(diagnostic), - primaryFacts: primaryFacts(diagnostic), - detailFacts: detailFacts(diagnostic), - } -} - -export function taskDiagnosticStateLabel(state: string) { - return stateLabels[state] ?? humanize(state) -} - -export function taskDiagnosticStateTone(state: string): StatusTone { - switch (state) { - case 'confirmed': - case 'not_applicable': - return 'success' - case 'rejected': - case 'mismatch': - case 'unavailable': - return 'danger' - case 'waiting_for_chain': - case 'preparing': - case 'transferring': - return 'info' - case 'unknown': - return 'warning' - default: - return 'neutral' - } -} - -export function taskDiagnosticReasonLabel(reason: string) { - return reasonLabels[reason] ?? reason.replace(/^task_/, '').replace(/_/g, ' ') -} - -export function shouldRefreshTaskDiagnostic(diagnostic: TaskDiagnostic) { - const transaction = diagnostic.evidence.transaction - switch (diagnostic.evidence.operation) { - case 'create_data_set': - return transaction?.kind === 'create_data_set' && Boolean(transaction.status_url) - case 'add_pieces': - return ( - transaction?.kind === 'add_pieces' && - Boolean( - transaction.status_url || (transaction.service_url && transaction.data_set_id && transaction.transaction_id) - ) - ) - default: - return false - } -} - -function diagnosticTitle(diagnostic: TaskDiagnostic) { - const operation = diagnostic.evidence.operation - switch (diagnostic.current_state) { - case 'not_applicable': - return 'No upload diagnosis available' - case 'preparing': - return 'Preparing upload work' - case 'transferring': - return 'Sending data to provider' - case 'waiting_for_chain': - if (operation === 'create_data_set') return 'Waiting for data set confirmation' - if (operation === 'add_pieces') return 'Waiting for storage confirmation' - return 'Waiting for provider confirmation' - case 'confirmed': - if (operation === 'create_data_set') return 'Data set confirmation received' - if (operation === 'add_pieces') return 'Storage confirmation received' - return 'Task evidence is confirmed' - case 'rejected': - return 'Storage confirmation was rejected' - case 'mismatch': - return 'Provider confirmation does not match the submitted work' - case 'unavailable': - return 'Storage status check is unavailable' - case 'unknown': - return 'Diagnosis needs manual inspection' - default: - return taskDiagnosticStateLabel(diagnostic.current_state) - } -} - -function primaryFacts(diagnostic: TaskDiagnostic): TaskDiagnosticFactRow[] { - const { evidence } = diagnostic - const rows: TaskDiagnosticFactRow[] = [] - const providerID = evidence.copy?.provider_id ?? evidence.provider?.provider_id ?? evidence.data_set?.provider_id - const providerStatus = evidence.provider?.status - - if (providerID || providerStatus) { - rows.push({ - label: 'Provider', - value: providerPrimaryValue(providerID, providerStatus), - }) - } - - const dataSetValue = storageTargetValue(diagnostic) - if (dataSetValue) rows.push({ label: 'Storage target', value: dataSetValue }) - - const submittedWorkValue = submittedWork(diagnostic) - if (submittedWorkValue) rows.push({ label: 'Pieces', value: submittedWorkValue }) - - const confirmationValue = confirmationFact(diagnostic) - if (confirmationValue) rows.push({ label: 'Confirmation', value: confirmationValue }) - - const dataSetTx = evidence.data_set?.create_transaction_id - if (dataSetTx && evidence.operation === 'create_data_set') { - rows.push(transactionFact('Data set setup transaction', dataSetTx)) - } - - const addPiecesTx = evidence.copy?.commit_transaction_id - if (addPiecesTx) { - rows.push(transactionFact('Storage update transaction', addPiecesTx)) - } - - addPrimaryIssue(rows, 'Upload issue', evidence.upload?.error_message || evidence.upload?.accept_error) - addPrimaryIssue(rows, 'Copy issue', evidence.copy?.last_error) - addPrimaryIssue(rows, 'Data set issue', evidence.data_set?.last_error) - addPrimaryIssue(rows, 'Provider issue', evidence.provider?.last_error) - addPrimaryIssue(rows, 'Task issue', evidence.task.last_error || evidence.task.status_message) - - return rows -} - -function detailFacts(diagnostic: TaskDiagnostic): TaskDiagnosticFactRow[] { - const { evidence } = diagnostic - const rows: TaskDiagnosticFactRow[] = [] - - addFact(rows, 'Task ID', formatID(evidence.task.id)) - addFact(rows, 'Task type', evidence.task.type) - addFact(rows, 'Task stage', evidence.task.stage) - addFact(rows, 'Task status', evidence.task.status) - addFact(rows, 'Retries', retryLabel(evidence.task.retry_count, evidence.task.max_retries)) - addFact(rows, 'Wait reason', evidence.task.wait_reason) - addFact(rows, 'Operation', humanize(evidence.operation)) - - if (evidence.upload) { - addFact(rows, 'Upload ID', formatID(evidence.upload.id)) - addFact(rows, 'Upload status', evidence.upload.status) - addFact(rows, 'Requested copies', numberValue(evidence.upload.requested_copies)) - addFact(rows, 'Upload error', evidence.upload.error_message || evidence.upload.accept_error, { detail: true }) - } - - if (evidence.copy) { - addFact(rows, 'Copy replica', replicaLabel(evidence.copy.copy_index)) - addFact(rows, 'Copy status', evidence.copy.status) - addFact(rows, 'Copy provider ID', evidence.copy.provider_id, { monospace: true }) - addFact(rows, 'Storage data set record', formatID(evidence.copy.storage_data_set_id)) - addFact(rows, 'Chain data set ID', evidence.copy.chain_data_set_id, { monospace: true }) - addFact(rows, 'Piece ID', evidence.copy.piece_id, { monospace: true }) - addFact(rows, 'Transfer method', evidence.copy.transfer_method) - addFact(rows, 'Add-pieces transaction', evidence.copy.commit_transaction_id, { monospace: true, detail: true }) - addFact(rows, 'Copy error', evidence.copy.last_error, { detail: true }) - } - - if (evidence.data_set) { - addFact(rows, 'Data set record', formatID(evidence.data_set.id)) - addFact(rows, 'Data set status', evidence.data_set.status) - addFact(rows, 'Data set provider ID', evidence.data_set.provider_id, { monospace: true }) - addFact(rows, 'Data set copy', replicaLabel(evidence.data_set.copy_index)) - addFact(rows, 'Data set chain ID', evidence.data_set.chain_data_set_id, { monospace: true }) - addFact(rows, 'Client data set ID', evidence.data_set.client_data_set_id, { - monospace: true, - detail: true, - displayMaxLength: 18, - }) - addFact(rows, 'Data-set creation transaction', evidence.data_set.create_transaction_id, { - monospace: true, - detail: true, - }) - addFact(rows, 'Data-set status URL', evidence.data_set.create_status_url, { monospace: true, detail: true }) - addFact(rows, 'Data set error', evidence.data_set.last_error, { detail: true }) - } - - if (evidence.provider) { - addFact(rows, 'Provider ID', evidence.provider.provider_id, { monospace: true }) - addFact(rows, 'Provider status', evidence.provider.status) - addFact(rows, 'Provider health status', evidence.provider.health_status) - addFact(rows, 'Provider service URL', evidence.provider.service_url, { monospace: true, detail: true }) - addFact(rows, 'Provider reason codes', evidence.provider.reason_codes?.map(taskDiagnosticReasonLabel).join(', ')) - addFact(rows, 'Provider error', evidence.provider.last_error, { detail: true }) - } - - if (evidence.transaction) { - addFact(rows, 'Transaction kind', humanize(evidence.transaction.kind)) - addFact(rows, 'Transaction status URL', evidence.transaction.status_url, { monospace: true, detail: true }) - addFact(rows, 'Transaction service URL', evidence.transaction.service_url, { monospace: true, detail: true }) - addFact(rows, 'Transaction data set ID', evidence.transaction.data_set_id, { monospace: true }) - addFact(rows, 'Transaction ID', evidence.transaction.transaction_id, { monospace: true, detail: true }) - addFact(rows, 'Transaction piece count', numberValue(evidence.transaction.piece_count)) - } - - if (evidence.live_check) { - addFact(rows, 'Latest status check', evidence.live_check.state) - addFact(rows, 'Recorded transaction status', evidence.live_check.tx_status) - addFact(rows, 'Status check data set ID', evidence.live_check.data_set_id, { monospace: true }) - addFact(rows, 'Data set created', booleanStatus(evidence.live_check.data_set_created, evidence.live_check.state)) - addFact( - rows, - 'Provider reported pieces added', - booleanStatus(evidence.live_check.pieces_added, evidence.live_check.state) - ) - addFact(rows, 'Piece count', recordedNumberValue(evidence.live_check.piece_count)) - addFact(rows, 'Confirmed piece IDs', listValue(evidence.live_check.confirmed_piece_ids), { - monospace: true, - detail: true, - }) - addFact(rows, 'Status check error', evidence.live_check.error, { detail: true }) - } - - addFact(rows, 'Recorded reasons', diagnostic.reason_codes.map(taskDiagnosticReasonLabel).join(', ')) - addFact(rows, 'Diagnostic checked at', diagnostic.checked_at) - - return rows -} - -function providerPrimaryValue(providerID?: string, status?: string) { - const label = providerID ? `Provider #${providerID}` : 'Provider' - switch (status) { - case 'available': - return `${label} is reachable` - case 'degraded': - return `${label} is degraded` - case 'unavailable': - return `${label} is unavailable` - case 'unknown': - return `${label} status is unknown` - default: - return `${label} selected` - } -} - -function storageTargetValue(diagnostic: TaskDiagnostic) { - const { evidence } = diagnostic - const chainDataSetID = evidence.copy?.chain_data_set_id ?? evidence.data_set?.chain_data_set_id - const dataSetStatus = evidence.data_set?.status - if (chainDataSetID) { - return dataSetStatus - ? `Data set #${chainDataSetID} is ${humanize(dataSetStatus)}` - : `Data set #${chainDataSetID} selected` - } - if (evidence.data_set?.id !== undefined) { - return dataSetStatus - ? `Data set record #${evidence.data_set.id} is ${humanize(dataSetStatus)}` - : `Data set record #${evidence.data_set.id} selected` - } - return '' -} - -function submittedWork(diagnostic: TaskDiagnostic) { - const { evidence } = diagnostic - const pieceCount = evidence.transaction?.piece_count - if (evidence.operation === 'add_pieces') { - if (pieceCount !== undefined) return `${pieceCount} ${plural(pieceCount, 'piece')} submitted` - if (evidence.copy?.piece_id) return '1 piece submitted' - return 'Piece submission recorded' - } - if (evidence.operation === 'create_data_set') return 'Data set creation submitted' - if (evidence.operation === 'transfer_piece') return 'Piece transfer in progress' - if (evidence.operation === 'prepare_upload') return 'Upload work is being prepared' - return '' -} - -function confirmationFact(diagnostic: TaskDiagnostic) { - const live = diagnostic.evidence.live_check - const work = diagnostic.evidence.operation === 'create_data_set' ? 'data set setup' : 'storage update' - if (live?.state) return liveStateConfirmation(live.state, work) - - switch (diagnostic.current_state) { - case 'waiting_for_chain': - return `Provider has not confirmed this ${work} yet` - case 'confirmed': - return `Provider confirmed this ${work}` - case 'rejected': - return `Provider rejected this ${work}` - case 'mismatch': - return 'Provider report differs from submitted pieces' - case 'unavailable': - return 'Latest storage status check is unavailable' - case 'unknown': - return 'Latest storage status is unknown' - default: - return '' - } -} - -function liveStateConfirmation(state: TaskDiagnosticLiveState, work: string) { - switch (state) { - case 'pending': - return `Provider has not confirmed this ${work} yet` - case 'confirmed': - return `Provider confirmed this ${work}` - case 'rejected': - return `Provider rejected this ${work}` - case 'mismatch': - return 'Provider report differs from submitted pieces' - case 'unavailable': - return 'Latest storage status check is unavailable' - case 'unknown': - return 'Latest storage status is unknown' - case 'skipped': - return 'Latest storage status check was not run' - default: - return humanize(state) - } -} - -function transactionFact(label: string, transactionID: string): TaskDiagnosticFactRow { - return { - label, - value: transactionID, - monospace: true, - detail: true, - } -} - -function addPrimaryIssue(rows: TaskDiagnosticFactRow[], label: string, value?: string) { - if (!value) return - rows.push({ label, value: 'Recorded error available', detail: true, detailValue: value }) -} - -function addFact( - rows: TaskDiagnosticFactRow[], - label: string, - value: string | number | boolean | null | undefined, - options: Pick = {} -) { - if (value === undefined || value === null || value === '') return - rows.push({ label, value: String(value), ...options }) -} - -function retryLabel(retryCount?: number, maxRetries?: number) { - if (retryCount === undefined && maxRetries === undefined) return '' - return `${retryCount ?? 0} of ${maxRetries ?? 0}` -} - -function numberValue(value?: number) { - return value === undefined || value === 0 ? '' : String(value) -} - -function recordedNumberValue(value?: number) { - return value === undefined ? '' : String(value) -} - -function listValue(values?: string[]) { - if (values === undefined) return '' - if (values.length === 0) return 'none' - return values.join(', ') -} - -function booleanStatus(value: boolean | undefined, liveState: TaskDiagnosticLiveState) { - if (value === true) return 'true' - if (value === false) return liveState === 'pending' ? 'pending' : 'false' - if (liveState === 'pending') return 'pending' - return '' -} - -function formatID(value?: string | number) { - return value === undefined || value === null ? '' : String(value) -} - -function replicaLabel(index?: number) { - if (index === undefined) return '' - return `replica ${index + 1}` -} - -function plural(count: number, singular: string) { - return count === 1 ? singular : `${singular}s` -} - -function humanize(value: string) { - return value.replace(/_/g, ' ') -} diff --git a/ui/src/lib/upload-progress-events.ts b/ui/src/lib/upload-progress-events.ts index 3a6a685..2199231 100644 --- a/ui/src/lib/upload-progress-events.ts +++ b/ui/src/lib/upload-progress-events.ts @@ -4,8 +4,6 @@ import type { ObjectProvenance, ObjectStatusDetail, ObjectVersionListResponse, - TaskListResponse, - TaskRefDetail, UploadTransferProgress, } from '@/api/client' @@ -59,6 +57,11 @@ export function applyUploadStateChangedEventData(queryClient: QueryClient, raw: queryClient.invalidateQueries({ queryKey: ['objectStatusDetail', payload.bucket_name, payload.version_id] }) queryClient.invalidateQueries({ queryKey: ['objectProvenance', payload.bucket_name, payload.version_id] }) } + } else { + queryClient.invalidateQueries({ queryKey: ['objects'] }) + queryClient.invalidateQueries({ queryKey: ['objectVersions'] }) + queryClient.invalidateQueries({ queryKey: ['objectStatusDetail'] }) + queryClient.invalidateQueries({ queryKey: ['objectProvenance'] }) } queryClient.invalidateQueries({ queryKey: ['tasks'] }) queryClient.invalidateQueries({ queryKey: ['taskRefDetail'] }) @@ -105,30 +108,6 @@ export function applyUploadProgressUpdate(queryClient: QueryClient, payload: Upl const next = mergeProgress(data.progress, progress) return next === data.progress ? data : { ...data, progress: next } }) - - queryClient.setQueriesData({ queryKey: ['tasks'] }, (data) => { - if (!data?.tasks.length) return data - let changed = false - const tasks = data.tasks.map((task) => { - const matches = - (typeof payload.task_id === 'number' && task.id === payload.task_id) || - (typeof payload.upload_id === 'number' && task.upload_id === payload.upload_id) || - task.ref_version_id === payload.version_id - if (!matches) return task - if (task.progress?.scope === 'provider_replacement') return task - const next = mergeProgress(task.progress, progress) - if (next === task.progress) return task - changed = true - return { ...task, progress: next } - }) - return changed ? { ...data, tasks } : data - }) - - queryClient.setQueriesData({ queryKey: ['taskRefDetail'] }, (data) => { - if (!data?.object || data.object.version_id !== payload.version_id) return data - const next = mergeProgress(data.object.progress, progress) - return next === data.object.progress ? data : { ...data, object: { ...data.object, progress: next } } - }) } function mergeProgress(current: UploadTransferProgress | undefined, next: UploadTransferProgress) { diff --git a/ui/src/routes/buckets.$name.tsx b/ui/src/routes/buckets.$name.tsx index 4800b8c..7ab7d41 100644 --- a/ui/src/routes/buckets.$name.tsx +++ b/ui/src/routes/buckets.$name.tsx @@ -34,11 +34,9 @@ import { type ObjectItem, type ObjectProvenance, type ObjectProvenanceCopy, - type ObjectProvenanceFailure, type ObjectState, type ObjectStatus, type ObjectUploadClientProgress, - type ObjectUploadStatus, type ObjectVersionItem, objectVersionAlreadyCurrentCode, type ProviderReplacement, @@ -115,19 +113,16 @@ import { useRestoreBucketObjectVersion, useRetryProviderReplacement, useS3Users, - useSettings, useStartProviderReplacement, useUpdateBucketCopyPolicy, useUpdateBucketOwner, } from '@/hooks/queries' import { - bucketCopyPolicyInheritOptionLabel, bucketCopyPolicyLabel, bucketCopyPolicySavedMessage, bucketCopyPolicyValue, clampMinimumDurableCopiesValue, copyPolicyOptions, - inheritedCopyPolicyValue, minimumDurableCopiesChoiceNote, minimumDurableCopiesFixedCountNote, minimumDurableCopiesLabel, @@ -137,9 +132,10 @@ import { minimumDurableCopiesWarning, persistMinimumDurableCopies, replicaTargetChoiceNote, + replicaTargetLocked, + replicaTargetLockNote, selectedTargetCopies, showsMinimumDurableCopiesWarning, - strictMinimumDurableCopiesValue, } from '@/lib/bucket-copy-policy' import { type BucketRouteSearch, normalizeBucketRouteSearch } from '@/lib/bucket-route-search' import { @@ -181,12 +177,10 @@ import { storageConfirmationListCommand, storageConfirmationReleaseWarning, } from '@/lib/storage-confirmation-attention' -import { objectStateLabel, replicaLabel, transferMethodLabel, uploadStatusLabel } from '@/lib/storage-status-labels' +import { objectStateLabel, replicaLabel, transferMethodLabel } from '@/lib/storage-status-labels' import { bucketStorageDataSetTopologyLinkModel } from '@/lib/storage-topology' import { cn, formatBytes, formatNumber, timeAgo } from '@/lib/utils' -type ProvenanceFailureDialogState = { title: string; text: string } - const objectBrowserSkeletonRows = ['row-1', 'row-2', 'row-3', 'row-4', 'row-5', 'row-6', 'row-7', 'row-8'] export const Route = createFileRoute('/buckets/$name')({ @@ -433,7 +427,6 @@ function ObjectVersionsDialog({ versionID={version.version_id} state={version.state} status={version.status} - uploadStatus={version.upload_status} progress={version.progress} compact /> @@ -716,82 +709,60 @@ function ObjectProvenanceDialog({ }) { const provenance = useObjectProvenance(bucketName, versionID, open) const data = provenance.data - const [failureDialog, setFailureDialog] = useState(null) - - const handleOpenChange = (next: boolean) => { - if (!next) setFailureDialog(null) - onOpenChange(next) - } return ( - <> - - - - Storage provenance - - Storage provenance for selected object version. - -
- - -
-
+ + + + Storage provenance + + Storage provenance for selected object version. + +
+ + +
+
- {provenance.isLoading ? ( -
- -
- ) : provenance.error ? ( -
Failed to load provenance
- ) : data ? ( -
- - - { - if (!failure.error) return - setFailureDialog({ title: 'Failed Attempt Error', text: failure.error }) - }} - /> -
- ) : null} -
-
- setFailureDialog(null)} - /> - + {provenance.isLoading ? ( +
+ +
+ ) : provenance.error ? ( +
Failed to load provenance
+ ) : data ? ( +
+ + +
+ ) : null} +
+
) } function ProvenanceSummary({ data }: { data: ObjectProvenance }) { - const progressPercent = data.upload_status === 'running' ? uploadProgressPercent(data.progress) : null - const uploadLabel = data.upload_status ? uploadStatusLabel(data.upload_status, progressPercent) : 'No upload recorded' + const progressPercent = uploadProgressPercent(data.progress) return (
- void -}) { - return ( -
-
Failed attempts
- - - - Provider - Transfer - Stage - Error - - - - {failures.map((failure) => ( - - - - - {transferMethodLabel(failure.transfer_method)} - {failure.stage ?? '—'} - - - - - ))} - {failures.length === 0 && ( - - - No failed attempts recorded - - - )} - -
-
- ) -} - function OptionalCopyableValue({ label, value, @@ -986,28 +912,6 @@ function OptionalCopyableValue({ return } -function ProvenanceFailureErrorCell({ - failure, - onOpenError, -}: { - failure: ObjectProvenanceFailure - onOpenError: (failure: ObjectProvenanceFailure) => void -}) { - if (!failure.error) { - return — - } - return ( - - ) -} - function LocationBadges({ location }: { location: { cache: boolean; filecoin: boolean } }) { if (!location.cache && !location.filecoin) { return None @@ -1026,7 +930,6 @@ function ObjectStatusIcon({ versionID, state, status, - uploadStatus, progress, compact = false, }: { @@ -1034,22 +937,20 @@ function ObjectStatusIcon({ versionID: string state?: ObjectState status: ObjectStatus - uploadStatus?: ObjectUploadStatus progress?: UploadTransferProgress compact?: boolean }) { const [detailEnabled, setDetailEnabled] = useState(false) - const visualStatus = objectVisualStatus(status, uploadStatus) - const detail = useObjectStatusDetail(bucketName, versionID, visualStatus === 'warning' && detailEnabled) - const progressPercent = uploadStatus === 'running' ? uploadProgressPercent(progress) : null - const displayLabel = objectStateLabel(state, status, uploadStatus, progressPercent) + const detail = useObjectStatusDetail(bucketName, versionID, status === 'warning' && detailEnabled) + const progressPercent = uploadProgressPercent(progress) + const displayLabel = objectStateLabel(state, status, progressPercent) const progressDetail = progressPercent === null || !progress ? null : `${formatBytes(progress.uploaded_bytes)} of ${formatBytes(progress.total_bytes)} uploaded` const loadDetail = () => { - if (visualStatus === 'warning') setDetailEnabled(true) + if (status === 'warning') setDetailEnabled(true) } return ( @@ -1067,21 +968,21 @@ function ObjectStatusIcon({ onFocus={loadDetail} onClick={loadDetail} > - {objectStatusIcon(visualStatus, compact, progressPercent)} + {objectStatusIcon(status, compact, progressPercent)}
{displayLabel} {progressDetail && {progressDetail}} - {visualStatus === 'warning' && ( + {status === 'warning' && ( {detail.isLoading ? 'Loading issue details' : detail.error ? 'Failed to load issue details' : detail.data?.message - ? `${failureStageLabel(detail.data.failed_at_state)}: ${detail.data.message}` + ? detail.data.message : 'No issue details recorded'} )} @@ -1113,16 +1014,6 @@ function objectStatusIcon(status: ObjectStatus, compact = false, progressPercent } } -function objectVisualStatus(status: ObjectStatus, uploadStatus?: ObjectUploadStatus): ObjectStatus { - switch (uploadStatus) { - case 'failed': - case 'rejected': - return 'warning' - default: - return status - } -} - function copyStatusTone(copy: ObjectProvenanceCopy): StatusTone { if (copy.attention_code) return 'warning' switch (copy.status) { @@ -1180,25 +1071,6 @@ function CopyAttentionDetails({ reasonCode, attentionAt }: { reasonCode: string; ) } -function failureStageLabel(state?: string) { - switch (state) { - case 'uploading': - return 'Failed while uploading' - case 'committing': - return 'Failed while registering storage record' - case 'replicating': - return 'Failed while syncing replicas' - case 'stored': - return 'Failed after storage' - case 'cached': - return 'Failed while cached' - case 'cache_evicted': - return 'Failed after cache removal' - default: - return 'Failure' - } -} - function ObjectBrowserPage() { const { name } = Route.useParams() const search = Route.useSearch() @@ -1408,7 +1280,7 @@ function ObjectBrowserPage() { variant="outline" size="sm" onClick={() => setUploadOpen(true)} - disabled={bucket.data?.status !== 'active'} + disabled={bucket.data?.status !== 'ready'} > Upload @@ -2560,17 +2432,13 @@ function BucketDetailsSettings({ onChangeOwner: () => void }) { const updateCopyPolicy = useUpdateBucketCopyPolicy() - const { data: settings } = useSettings() const currentCopyPolicy = bucketCopyPolicyValue(bucket) const currentMinimumDurableCopies = minimumDurableCopiesValue(bucket) const [copyPolicy, setCopyPolicy] = useState(currentCopyPolicy) const [minimumDurableCopies, setMinimumDurableCopies] = useState(currentMinimumDurableCopies) const [copyPolicyError, setCopyPolicyError] = useState(null) const [copyPolicyNotice, setCopyPolicyNotice] = useState(null) - const runtimeDefaultCopies = settings?.runtime_filecoin_default_copies - const inheritedTargetCopies = - currentCopyPolicy === inheritedCopyPolicyValue ? bucket.effective_copies : runtimeDefaultCopies - const targetCopies = selectedTargetCopies(copyPolicy, inheritedTargetCopies) + const targetCopies = selectedTargetCopies(copyPolicy) const minimumOptions = minimumDurableCopiesOptions(targetCopies) useEffect(() => { @@ -2594,19 +2462,15 @@ function BucketDetailsSettings({ nextMinimumDurableCopies !== undefined) && copyPolicyNotice == null const handleCopyPolicyChange = (next: string) => { - const nextTarget = selectedTargetCopies(next, inheritedTargetCopies) + const nextTarget = selectedTargetCopies(next) setCopyPolicy(next) - setMinimumDurableCopies((current) => { - if ( - bucket.minimum_durable_copies != null && - nextTarget != null && - bucket.minimum_durable_copies > bucket.effective_copies && - bucket.minimum_durable_copies <= nextTarget - ) { - return bucket.minimum_durable_copies.toString() - } - return clampMinimumDurableCopiesValue(current, nextTarget) - }) + // "All replicas" follows the target: a minimum that matched the old target + // keeps matching the new one instead of quietly becoming a fixed count. + setMinimumDurableCopies((current) => + Number(current) === Number(currentCopyPolicy) && nextTarget != null + ? nextTarget.toString() + : clampMinimumDurableCopiesValue(current, nextTarget) + ) setCopyPolicyError(null) setCopyPolicyNotice(null) } @@ -2621,12 +2485,7 @@ function BucketDetailsSettings({ updateCopyPolicy.mutate( { name: bucket.name, - defaultCopies: - copyPolicy === currentCopyPolicy - ? undefined - : copyPolicy === inheritedCopyPolicyValue - ? null - : Number(copyPolicy), + defaultCopies: copyPolicy === currentCopyPolicy ? undefined : Number(copyPolicy), minimumDurableCopies: nextMinimumDurableCopies, }, { @@ -2674,11 +2533,12 @@ function BucketDetailsSettings({ - - {bucketCopyPolicyInheritOptionLabel(bucket, runtimeDefaultCopies)} - {copyPolicyOptions.map((copies) => ( - + {copies} {copies === 1 ? 'copy' : 'copies'} ))} @@ -2686,6 +2546,7 @@ function BucketDetailsSettings({ {replicaTargetChoiceNote()} + {bucket.default_copies > 1 && {replicaTargetLockNote()}} Release cache after @@ -2699,7 +2560,6 @@ function BucketDetailsSettings({ - All replicas (strict) {minimumOptions.map((copies) => ( {minimumDurableCopiesOptionLabel(copies, targetCopies)} @@ -2708,7 +2568,7 @@ function BucketDetailsSettings({ - {minimumDurableCopies === strictMinimumDurableCopiesValue ? ( + {targetCopies != null && Number(minimumDurableCopies) === targetCopies ? ( {minimumDurableCopiesChoiceNote()} ) : ( {minimumDurableCopiesFixedCountNote()} @@ -2849,7 +2709,6 @@ function ObjectBrowserTable({ versionID={object.current_version_id} state={object.state} status={object.status} - uploadStatus={object.upload_status} progress={object.progress} compact /> diff --git a/ui/src/routes/buckets.index.tsx b/ui/src/routes/buckets.index.tsx index ba2a5f0..f53847a 100644 --- a/ui/src/routes/buckets.index.tsx +++ b/ui/src/routes/buckets.index.tsx @@ -29,7 +29,6 @@ import { bucketCopyPolicyLabel, clampMinimumDurableCopiesValue, copyPolicyOptions, - inheritedCopyPolicyValue, minimumDurableCopiesChoiceNote, minimumDurableCopiesFixedCountNote, minimumDurableCopiesLabel, @@ -39,7 +38,6 @@ import { replicaTargetChoiceNote, selectedTargetCopies, showsMinimumDurableCopiesWarning, - strictMinimumDurableCopiesValue, } from '@/lib/bucket-copy-policy' import { bucketStorageHealthLabel, @@ -57,22 +55,26 @@ function CreateBucketDialog() { const [open, setOpen] = useState(false) const [bucketName, setBucketName] = useState('') const [ownerAccessKey, setOwnerAccessKey] = useState('') - const [copyPolicy, setCopyPolicy] = useState(inheritedCopyPolicyValue) - const [minimumDurableCopies, setMinimumDurableCopies] = useState(strictMinimumDurableCopiesValue) + const [copyPolicyOverride, setCopyPolicyOverride] = useState(null) + const [minimumDurableCopiesOverride, setMinimumDurableCopiesOverride] = useState(null) const [error, setError] = useState(null) const { data: users = [], isLoading: usersLoading, error: usersError } = useS3Users() const { data: settings } = useSettings() const createBucket = useCreateBucket() const navigate = useNavigate() const runtimeDefaultCopies = settings?.runtime_filecoin_default_copies - const targetCopies = selectedTargetCopies(copyPolicy, runtimeDefaultCopies) + // A bucket stores its own policy, so the dialog only pre-selects the + // configured target instead of offering an "inherit" value to store. + const copyPolicy = copyPolicyOverride ?? runtimeDefaultCopies?.toString() ?? '' + const targetCopies = selectedTargetCopies(copyPolicy) + const minimumDurableCopies = minimumDurableCopiesOverride ?? targetCopies?.toString() ?? '' const minimumOptions = minimumDurableCopiesOptions(targetCopies) const reset = () => { setBucketName('') setOwnerAccessKey('') - setCopyPolicy(inheritedCopyPolicyValue) - setMinimumDurableCopies(strictMinimumDurableCopiesValue) + setCopyPolicyOverride(null) + setMinimumDurableCopiesOverride(null) setError(null) createBucket.reset() } @@ -95,8 +97,8 @@ function CreateBucketDialog() { } setError(null) - const defaultCopies = copyPolicy === inheritedCopyPolicyValue ? null : Number(copyPolicy) - const minimumCopies = minimumDurableCopies === strictMinimumDurableCopiesValue ? null : Number(minimumDurableCopies) + const defaultCopies = targetCopies + const minimumCopies = selectedTargetCopies(minimumDurableCopies) createBucket.mutate( { name, ownerAccessKey, defaultCopies, minimumDurableCopies: minimumCopies }, { @@ -116,9 +118,11 @@ function CreateBucketDialog() { const ownerError = error === 'Bucket owner is required' ? error : null const formError = error && !bucketNameError && !ownerError ? error : null const handleCopyPolicyChange = (next: string) => { - const nextTarget = selectedTargetCopies(next, runtimeDefaultCopies) - setCopyPolicy(next) - setMinimumDurableCopies((current) => clampMinimumDurableCopiesValue(current, nextTarget)) + const nextTarget = selectedTargetCopies(next) + setCopyPolicyOverride(next) + setMinimumDurableCopiesOverride((current) => + current == null ? null : clampMinimumDurableCopiesValue(current, nextTarget) + ) } return ( @@ -173,11 +177,6 @@ function CreateBucketDialog() { - - {runtimeDefaultCopies == null - ? 'Inherit current runtime default' - : `Inherit current runtime default (${runtimeDefaultCopies} ${runtimeDefaultCopies === 1 ? 'copy' : 'copies'})`} - {copyPolicyOptions.map((copies) => ( {copies} {copies === 1 ? 'copy' : 'copies'} @@ -186,10 +185,9 @@ function CreateBucketDialog() { - {copyPolicy === inheritedCopyPolicyValue && runtimeDefaultCopies == null ? ( + {targetCopies == null ? ( - The current runtime default is unavailable. Choose a replica count to configure an explicit - cache-release threshold. + The current runtime default is unavailable. Choose a replica count for this bucket. ) : ( {replicaTargetChoiceNote()} @@ -199,7 +197,7 @@ function CreateBucketDialog() { Release cache after - {minimumDurableCopies === strictMinimumDurableCopiesValue ? ( + {targetCopies != null && Number(minimumDurableCopies) === targetCopies ? ( {minimumDurableCopiesChoiceNote()} ) : ( {minimumDurableCopiesFixedCountNote()} diff --git a/ui/src/routes/index.tsx b/ui/src/routes/index.tsx index 9e9401f..2aeb67c 100644 --- a/ui/src/routes/index.tsx +++ b/ui/src/routes/index.tsx @@ -41,7 +41,7 @@ function OverviewPage() { const attentionRows = attentionDisplayRows({ objects: data.objects.attention ?? { needs_attention: 0, unavailable: 0 }, - tasks: data.tasks.attention ?? { failed: 0, exhausted: 0 }, + tasks: data.tasks.attention ?? { failed: 0 }, }) const pipelineRows = overviewPipelineRows(data.tasks.active_pipeline ?? []) const hasActiveTasks = pipelineRows.some((row) => row.total > 0) @@ -82,9 +82,7 @@ function OverviewPage() {
No attention needed
-
- Object failures 0 · Unavailable 0 · Failed tasks 0 · Retry limit reached 0 -
+
Object failures 0 · Unavailable 0 · Failed tasks 0
)} @@ -297,7 +295,7 @@ function AttentionLinkRow({ row }: { row: ReturnType @@ -327,12 +325,8 @@ function PipelineTooltip({ active, payload, label }: PipelineTooltipProps) {
{label}
- Queued - {formatNumber(row.queued)} - Scheduled - {formatNumber(row.scheduled)} - Waiting - {formatNumber(row.waiting)} + Pending + {formatNumber(row.pending)} Running {formatNumber(row.running)}
diff --git a/ui/src/routes/settings.tsx b/ui/src/routes/settings.tsx index b82fa5e..ed4e7a0 100644 --- a/ui/src/routes/settings.tsx +++ b/ui/src/routes/settings.tsx @@ -98,18 +98,13 @@ const tabFields = { 'cache.lru_low_watermark_percent', ], workers: [ - 'worker.upload.concurrency', - 'worker.upload.poll_interval', - 'worker.upload.max_retries', - 'worker.provider_replacement.concurrency', - 'worker.provider_replacement.poll_interval', - 'worker.provider_replacement.max_retries', - 'worker.evictor.concurrency', - 'worker.evictor.poll_interval', - 'worker.evictor.max_retries', - 'worker.storage_cleanup.concurrency', - 'worker.storage_cleanup.poll_interval', - 'worker.storage_cleanup.max_retries', + 'worker.tasks.concurrency', + 'worker.tasks.poll_interval', + 'worker.tasks.lease_duration', + 'worker.tasks.max_retries', + 'worker.tasks.retention', + 'worker.tasks.provider_mutation_concurrency', + 'worker.tasks.destructive_mutation_concurrency', ], logging: ['logging.level', 'logging.format', 'logging.s3_access.enabled', 'logging.s3_access.level'], runtime: ['database.driver', 'database.dsn', 'database.max_open_conns', 'database.max_idle_conns', 'admin.addr'], @@ -576,40 +571,12 @@ function SettingsPage() { -
- setForm({ ...form, worker: { ...form.worker, upload: value } })} - /> - setForm({ ...form, worker: { ...form.worker, provider_replacement: value } })} - /> - setForm({ ...form, worker: { ...form.worker, evictor: value } })} - /> - setForm({ ...form, worker: { ...form.worker, storage_cleanup: value } })} - /> -
+ setForm({ ...form, worker: { tasks: value } })} + />
@@ -935,27 +902,23 @@ function CheckboxField({ ) } -function WorkerSection({ - title, - prefix, +function TaskWorkerSection({ value, data, errors, onChange, }: { - title: string - prefix: 'worker.upload' | 'worker.provider_replacement' | 'worker.evictor' | 'worker.storage_cleanup' - value: SettingsEditableConfig['worker']['upload'] + value: SettingsEditableConfig['worker']['tasks'] data: SettingsData errors: Record - onChange: (value: SettingsEditableConfig['worker']['upload']) => void + onChange: (value: SettingsEditableConfig['worker']['tasks']) => void }) { return ( -
-
+
+
onChange({ ...value, poll_interval: next })} /> + onChange({ ...value, lease_duration: next })} + /> onChange({ ...value, max_retries: next })} /> + onChange({ ...value, retention: next })} + /> + onChange({ ...value, provider_mutation_concurrency: next })} + /> + onChange({ ...value, destructive_mutation_concurrency: next })} + />
) diff --git a/ui/src/routes/tasks.tsx b/ui/src/routes/tasks.tsx index 51dc838..f9b046c 100644 --- a/ui/src/routes/tasks.tsx +++ b/ui/src/routes/tasks.tsx @@ -1,79 +1,55 @@ import { useMutation, useQueryClient } from '@tanstack/react-query' -import { createFileRoute, Link } from '@tanstack/react-router' -import { - ChevronDown, - ExternalLink, - ListTodo, - Loader2, - RefreshCw, - RotateCcw, - Stethoscope, - TriangleAlert, -} from 'lucide-react' -import { type ReactNode, useEffect, useState } from 'react' -import { api, type TaskDiagnostic, type TaskItem, type TaskStorageCleanupDetail } from '@/api/client' +import { createFileRoute, useNavigate } from '@tanstack/react-router' +import { ChevronLeft, ChevronRight, ListTodo, Loader2, RefreshCw, RotateCcw, X } from 'lucide-react' +import { useEffect, useRef, useState } from 'react' + +import { api, type TaskItem } from '@/api/client' import { CopyableValue } from '@/components/app/CopyableValue' -import { DangerActionAlertDialog } from '@/components/app/DangerActionAlertDialog' -import { DetailTextDialog } from '@/components/app/DetailTextDialog' import { PageErrorState } from '@/components/app/PageErrorState' import { PageHeader } from '@/components/app/PageHeader' -import { ProviderReplacementProgress } from '@/components/app/ProviderReplacementProgress' -import { ReviewDetails } from '@/components/app/ReviewDetails' import { StatusBadge, taskStatusTone } from '@/components/app/StatusBadge' -import { UploadProgressBar } from '@/components/app/UploadProgress' import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert' import { Button } from '@/components/ui/button' -import { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible' import { Empty, EmptyDescription, EmptyHeader, EmptyMedia, EmptyTitle } from '@/components/ui/empty' import { Label } from '@/components/ui/label' import { Pagination, PaginationContent, PaginationItem } from '@/components/ui/pagination' -import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover' -import { ScrollArea } from '@/components/ui/scroll-area' import { Select, SelectContent, SelectGroup, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' -import { Separator } from '@/components/ui/separator' -import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from '@/components/ui/sheet' import { Skeleton } from '@/components/ui/skeleton' import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table' -import { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs' -import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' -import { useTaskRefDetail, useTasks } from '@/hooks/queries' -import { taskHasDataSetsRecovery, taskRetryableFromQueue } from '@/lib/provider-replacement' -import { - replacementCleanupStageLabel, - storageCleanupCopyStatusLabel, - storageCleanupCopyStatusTone, - storageCleanupStatusLabel, - taskHasByteTransfer, - taskOperationLabel, - taskOperationOptionLabel, - taskReplicaLabel, - taskStageOptions, - taskTypeLabel, -} from '@/lib/storage-status-labels' -import { - buildTaskDiagnosticViewModel, - shouldRefreshTaskDiagnostic, - type TaskDiagnosticFactRow, - taskDiagnosticSheetContentClassName, - taskDiagnosticStateLabel, - taskDiagnosticStateTone, -} from '@/lib/task-diagnostics' -import { cn, formatBytes, timeAgo } from '@/lib/utils' +import { useTasks } from '@/hooks/queries' +import { timeAgo } from '@/lib/utils' + +const PAGE_SIZE = 20 + +const taskOperations = [ + { value: 'all', label: 'All operations' }, + { value: 'bucket_provision', label: 'Prepare bucket storage' }, + { value: 'upload_plan', label: 'Prepare upload' }, + { value: 'storage_dataset_ensure', label: 'Prepare storage' }, + { value: 'storage_transfer_plan', label: 'Plan storage transfer' }, + { value: 'storage_store', label: 'Store content' }, + { value: 'storage_pull', label: 'Transfer stored content' }, + { value: 'storage_commit_coordinate', label: 'Prepare storage confirmation' }, + { value: 'storage_commit', label: 'Confirm storage' }, + { value: 'provider_replacement_coordinate', label: 'Replace storage provider' }, + { value: 'cache_evict', label: 'Remove local cached copy' }, + { value: 'cache_reconcile_durability', label: 'Review cache durability' }, + { value: 'storage_cleanup', label: 'Remove remote storage copy' }, + { value: 'storage_dataset_retire', label: 'Retire storage service' }, + { value: 'wallet_operation', label: 'Process wallet request' }, +] as const -const taskTypeTabs = ['all', 'upload', 'evict_cache', 'storage_cleanup'] as const -const statusOptions = [ - 'all', - 'queued', - 'scheduled', - 'waiting', - 'running', - 'completed', - 'failed', - 'exhausted', - 'cancelled', +const taskStatuses = [ + { value: 'all', label: 'All statuses' }, + { value: 'pending', label: 'Pending' }, + { value: 'running', label: 'Running' }, + { value: 'completed', label: 'Completed' }, + { value: 'failed', label: 'Failed' }, + { value: 'dismissed', label: 'Dismissed' }, + { value: 'cancelled', label: 'Cancelled' }, ] as const -const statusLabels: Record = { - all: 'All', + +const presentationLabels: Record = { queued: 'Queued', scheduled: 'Scheduled', waiting: 'Waiting', @@ -81,878 +57,143 @@ const statusLabels: Record = { completed: 'Completed', failed: 'Failed', cancelled: 'Cancelled', - exhausted: 'Exhausted', + dismissed: 'Dismissed', } -const PAGE_SIZE = 20 - -type TaskTypeTab = (typeof taskTypeTabs)[number] -type TaskStatusFilter = Exclude<(typeof statusOptions)[number], 'all'> +type TaskOperationFilter = (typeof taskOperations)[number]['value'] +type TaskStatusFilter = (typeof taskStatuses)[number]['value'] type TasksSearch = { - type?: TaskTypeTab - status?: TaskStatusFilter -} -type TaskDetailDialogState = { title: string; text: string } -type TaskTableProps = { - tasks: TaskItem[] - retryPending: boolean - onRetry: (task: TaskItem) => void - onOpenDiagnostic: (task: TaskItem) => void - onOpenDetail: (dialog: TaskDetailDialogState) => void + type?: Exclude + status?: Exclude } -const taskTypeSearchValues = new Set(taskTypeTabs) -const taskStatusSearchValues = new Set(statusOptions.filter((option) => option !== 'all')) +const taskOperationValues = new Set( + taskOperations.map((option) => option.value).filter((value) => value !== 'all') +) +const taskStatusValues = new Set(taskStatuses.map((option) => option.value).filter((value) => value !== 'all')) export const Route = createFileRoute('/tasks')({ - validateSearch: (search: Record): TasksSearch => { - const type = typeof search.type === 'string' && taskTypeSearchValues.has(search.type) ? search.type : undefined - const status = - typeof search.status === 'string' && taskStatusSearchValues.has(search.status) ? search.status : undefined - return { - type: type as TaskTypeTab | undefined, - status: status as TaskStatusFilter | undefined, - } - }, + validateSearch: (search: Record): TasksSearch => ({ + type: + typeof search.type === 'string' && taskOperationValues.has(search.type) + ? (search.type as TasksSearch['type']) + : undefined, + status: + typeof search.status === 'string' && taskStatusValues.has(search.status) + ? (search.status as TasksSearch['status']) + : undefined, + }), component: TasksPage, }) -function taskDetailText(task: TaskItem) { - return task.status_message || task.last_error || '' -} - -function taskDetailTitle(task: TaskItem) { - return task.last_error ? 'Error Details' : 'Status Details' -} - -function TaskRefCell({ task }: { task: TaskItem }) { - const [detailsOpen, setDetailsOpen] = useState(false) - const detail = useTaskRefDetail(task.id, detailsOpen) - // Replacement cleanup ends a storage service; calling it a deleted object - // would send the operator looking in the wrong place. - const replacementCleanup = replacementCleanupStageLabel(task.stage) - const refLabel = replacementCleanup - ? task.bucket_name - ? `${replacementCleanup} · ${task.bucket_name}` - : replacementCleanup - : task.type === 'storage_cleanup' - ? 'Deleted object' - : `${task.ref_type}:${task.ref_id}` - - return ( - - - - - - - - - ) -} - -function TaskRefPopoverContent({ detail, enabled }: { detail: ReturnType; enabled: boolean }) { - if (detail.error) { - return Details unavailable - } - if (!detail.data) { - return {enabled ? 'Loading details' : ''} - } - if (!detail.data.object && !detail.data.storage_cleanup) { - if (detail.data.bucket_name) { - return ( -
- Bucket - {detail.data.bucket_name} -
- ) - } - return Details unavailable - } - - if (detail.data.storage_cleanup) { - return - } - - if (!detail.data.object) { - return Details unavailable - } - - return -} - -function TaskStorageCleanupContent({ cleanup }: { cleanup: TaskStorageCleanupDetail }) { - const versions = cleanup.deleted_versions ?? [] - if (versions.length === 0) { - return Details unavailable - } - - const totalSize = versions.reduce((sum, version) => sum + version.size, 0) - const firstDeletedAt = versions.find((version) => version.deleted_at)?.deleted_at - const copies = cleanup.copies ?? [] - - return ( -
-
- Bucket - - {storageCleanupValueLabel( - versions.map((version) => version.bucket_name), - 'buckets' - )} - - Key - - {storageCleanupValueLabel( - versions.map((version) => version.key), - 'objects' - )} - - Version - {storageCleanupVersionLabel(versions)} - Status - {storageCleanupStatusLabel(copies)} - Size - {formatBytes(totalSize)} - Deleted - {firstDeletedAt ? timeAgo(firstDeletedAt) : '—'} -
- {copies.length > 0 && ( -
- Replica cleanup -
- {copies.map((copy) => ( -
-
- - {taskReplicaLabel({ copy_index: copy.copy_index })} - - - {storageCleanupCopyStatusLabel(copy.status)} - -
- {copy.delete_tx_hash && ( - - )} - {copy.last_error && ( - {copy.last_error} - )} -
- ))} -
-
- )} -
- ) -} - -function TaskObjectRefContent({ - object, -}: { - object: NonNullable['data']>['object'] -}) { - if (!object) { - return Details unavailable - } - - return ( -
- Bucket - {object.bucket_name} - Key - {object.key} - Version - {object.version_id} - State - {object.state} - {object.upload_status && ( - <> - Upload - {object.upload_status} - - )} - Size - {formatBytes(object.size)} - Location - {taskObjectLocationLabel(object.location)} - Updated - {timeAgo(object.updated_at)} -
- ) -} - -function storageCleanupValueLabel(values: string[], pluralName: string) { - const uniqueValues = uniqueNonEmpty(values) - if (uniqueValues.length === 0) return '—' - if (uniqueValues.length === 1) return uniqueValues[0] - return `${uniqueValues[0]} +${uniqueValues.length - 1} ${pluralName}` -} - -function storageCleanupVersionLabel(versions: TaskStorageCleanupDetail['deleted_versions']) { - const first = versions[0] - if (!first) return '—' - if (versions.length === 1) return first.version_id - return `${first.version_id} +${versions.length - 1} versions` -} - -function uniqueNonEmpty(values: Array) { - return Array.from(new Set(values.filter((value): value is string => Boolean(value)))) -} - -function taskObjectLocationLabel(location: { cache: boolean; filecoin: boolean }) { - if (location.cache && location.filecoin) return 'Cache + Filecoin' - if (location.cache) return 'Cache' - if (location.filecoin) return 'Filecoin' - return 'None' -} - -function TaskDetailsCell({ - task, - onOpenDetail, -}: { - task: TaskItem - onOpenDetail: (dialog: TaskDetailDialogState) => void -}) { - if (task.progress?.scope === 'provider_replacement') { - return ( -
- - {task.last_error && ( - - )} -
- ) - } - const detailText = taskDetailText(task) - if (!detailText) { - return — - } - return ( - - ) -} - -function TaskActionsCell({ - task, - retryPending, - onRetry, - onOpenDiagnostic, -}: { - task: TaskItem - retryPending: boolean - onRetry: (task: TaskItem) => void - onOpenDiagnostic: (task: TaskItem) => void -}) { - const showDiagnostic = task.type === 'upload' - const showRetry = task.status === 'exhausted' && taskRetryableFromQueue(task) - const showDataSets = taskHasDataSetsRecovery(task) - if (!showDiagnostic && !showRetry && !showDataSets) return — - - return ( -
- {showDiagnostic && ( - - - - - Diagnostics - - )} - {showRetry && ( - - - - - Retry - - )} - {showDataSets && task.bucket_name && ( - - - - - Open Data Sets - - )} -
- ) -} - -function TaskCommonCells({ - task, - retryPending, - onRetry, - onOpenDiagnostic, - onOpenDetail, -}: { - task: TaskItem - retryPending: boolean - onRetry: (task: TaskItem) => void - onOpenDiagnostic: (task: TaskItem) => void - onOpenDetail: (dialog: TaskDetailDialogState) => void -}) { - return ( - <> - - - - - {task.status} - - - {task.retry_count}/{task.max_retries} - - - - - {timeAgo(task.scheduled_at)} - - - - - ) -} - -function AllTasksTable({ tasks, retryPending, onRetry, onOpenDiagnostic, onOpenDetail }: TaskTableProps) { - return ( - - - - - ID - Type - Operation - Object - Status - Retries - Details - Scheduled - Actions - - - - {tasks.length > 0 ? ( - tasks.map((task) => ( - - {task.id} - - {taskTypeLabel(task.type)} - - {taskOperationLabel(task)} - - - )) - ) : ( - - )} - -
-
- ) -} - -function UploadTasksTable({ tasks, retryPending, onRetry, onOpenDiagnostic, onOpenDetail }: TaskTableProps) { - return ( - - - - - ID - Operation - Replica - Object - Status - Retries - Details - Scheduled - Actions - - - - {tasks.length > 0 ? ( - tasks.map((task) => ( - - {task.id} - -
- {taskOperationLabel(task)} - {taskHasByteTransfer(task) && task.progress?.scope === 'ingress_store' && ( - - )} -
-
- - {taskReplicaLabel(task)} - - -
- )) - ) : ( - - )} -
-
-
- ) -} - -function EvictCacheTasksTable({ tasks, retryPending, onRetry, onOpenDiagnostic, onOpenDetail }: TaskTableProps) { - return ( - - - - - ID - Object - Status - Retries - Details - Scheduled - Actions - - - - {tasks.length > 0 ? ( - tasks.map((task) => ( - - {task.id} - - - )) - ) : ( - - )} - -
-
- ) -} - -function TaskTableFrame({ children }: { children: ReactNode }) { - return
{children}
-} - -function TaskEmptyRow({ colSpan }: { colSpan: number }) { - return ( - - - - - - - - No tasks found - There are no tasks matching your current filters. - - - - - ) -} - -function TaskDiagnosticSheet({ - task, - diagnostic, - loading, - refreshing, - error, - onClose, -}: { - task: TaskItem | null - diagnostic: TaskDiagnostic | null - loading: boolean - refreshing: boolean - error: string | null - onClose: () => void -}) { - const checking = refreshing - const [detailsOpen, setDetailsOpen] = useState(true) - const view = diagnostic ? buildTaskDiagnosticViewModel(diagnostic) : null - - return ( - { - if (!open) { - setDetailsOpen(true) - onClose() - } - }} - > - - - Upload diagnostics - {task ? `Task #${task.id}` : 'Upload task diagnostics'} - - -
- {loading && !diagnostic ? ( - - ) : diagnostic && view ? ( - <> -
-
-

Current status

- - {taskDiagnosticStateLabel(diagnostic.current_state)} - -
-
-

{view.title}

-
- {checking && ( -
- - Checking latest storage status -
- )} -
- - - -
-

Evidence

- -
- - - - -
-

Recorded details

- - - -
- - - -
- - ) : null} - - {error && ( - - - Diagnostics unavailable - {error} - - )} -
-
-
-
- ) -} - -function TaskDiagnosticSkeleton() { - return ( -
-
-
- - -
- -
- -
- -
- - - - - - -
-
-
- ) -} - -function TaskDiagnosticFactList({ rows }: { rows: TaskDiagnosticFactRow[] }) { - return ( -
- {rows.map((row) => ( - - ))} -
- ) -} - -function TaskDiagnosticFactRowView({ row }: { row: TaskDiagnosticFactRow }) { - return ( - <> -
{row.label}
-
- {row.detail ? ( - - ) : ( - {row.value} - )} -
- - ) -} - -function isAbortError(err: unknown) { - return typeof err === 'object' && err !== null && 'name' in err && err.name === 'AbortError' -} - function TasksPage() { const search = Route.useSearch() - const [status, setStatus] = useState(search.status ?? '') - const [taskType, setTaskType] = useState(search.type ?? 'all') - const [stage, setStage] = useState('') - const [offset, setOffset] = useState(0) - const queryTaskType = taskType === 'all' ? '' : taskType - const { data, isLoading, error } = useTasks(queryTaskType, stage, status, PAGE_SIZE, offset) - const qc = useQueryClient() - - const [retryTarget, setRetryTarget] = useState(null) - const [detailDialog, setDetailDialog] = useState<{ title: string; text: string } | null>(null) - const [diagnosticTarget, setDiagnosticTarget] = useState(null) - const [diagnostic, setDiagnostic] = useState(null) - const [diagnosticLoading, setDiagnosticLoading] = useState(false) - const [diagnosticRefreshing, setDiagnosticRefreshing] = useState(false) - const [diagnosticError, setDiagnosticError] = useState(null) - const retryMutation = useMutation({ - mutationFn: (taskId: number) => api.retryTask(taskId), - onSuccess: () => setRetryTarget(null), - onSettled: () => { - qc.invalidateQueries({ queryKey: ['tasks'] }) - }, - }) - const retryPending = retryMutation.isPending - - const totalPages = data ? Math.ceil(data.total / PAGE_SIZE) : 0 - const currentPage = Math.floor(offset / PAGE_SIZE) + 1 + const navigate = useNavigate() + const queryClient = useQueryClient() + const [cursor, setCursor] = useState() + const [cursorHistory, setCursorHistory] = useState>([]) + const taskType = search.type ?? '' + const status = search.status ?? '' + const filterKey = `${taskType}:${status}` + const previousFilterKey = useRef(filterKey) + const tasks = useTasks(taskType, status, PAGE_SIZE, cursor) useEffect(() => { - setTaskType(search.type ?? 'all') - setStatus(search.status ?? '') - setStage('') - setOffset(0) - }, [search.type, search.status]) - - useEffect(() => { - if (!diagnosticTarget) { - setDiagnostic(null) - setDiagnosticLoading(false) - setDiagnosticRefreshing(false) - setDiagnosticError(null) - return - } - let active = true - const controller = new AbortController() - setDiagnostic(null) - setDiagnosticError(null) - setDiagnosticLoading(true) - setDiagnosticRefreshing(false) - - api - .getTaskDiagnostic(diagnosticTarget.id, { signal: controller.signal }) - .then((initial) => { - if (!active) return null - setDiagnostic(initial) - setDiagnosticLoading(false) - if (!shouldRefreshTaskDiagnostic(initial)) return null - setDiagnosticRefreshing(true) - return api.refreshTaskDiagnostic(diagnosticTarget.id, { signal: controller.signal }) - }) - .then((refreshed) => { - if (active && refreshed) setDiagnostic(refreshed) - }) - .catch((err: unknown) => { - if (isAbortError(err)) return - if (active) setDiagnosticError(err instanceof Error ? err.message : 'Failed to load task diagnostics') - }) - .finally(() => { - if (!active) return - setDiagnosticLoading(false) - setDiagnosticRefreshing(false) - }) + if (previousFilterKey.current === filterKey) return + previousFilterKey.current = filterKey + setCursor(undefined) + setCursorHistory([]) + }, [filterKey]) + + const refreshTasks = () => { + queryClient.invalidateQueries({ queryKey: ['tasks'] }) + queryClient.invalidateQueries({ queryKey: ['taskStats'] }) + } + const retry = useMutation({ mutationFn: api.retryTask, onSuccess: refreshTasks }) + const acknowledge = useMutation({ mutationFn: api.acknowledgeTask, onSuccess: refreshTasks }) + const actionError = retry.error ?? acknowledge.error + + const setFilters = (nextType: TaskOperationFilter, nextStatus: TaskStatusFilter) => { + retry.reset() + acknowledge.reset() + navigate({ + to: '/tasks', + search: { + type: nextType === 'all' ? undefined : nextType, + status: nextStatus === 'all' ? undefined : nextStatus, + }, + replace: true, + }) + } - return () => { - active = false - controller.abort() - } - }, [diagnosticTarget]) + const nextPage = () => { + if (!tasks.data?.next_cursor) return + setCursorHistory((current) => [...current, cursor]) + setCursor(tasks.data.next_cursor) + } - function openRetryDialog(task: TaskItem) { - if (retryPending) return - retryMutation.reset() - setRetryTarget(task) + const previousPage = () => { + const previous = cursorHistory[cursorHistory.length - 1] + setCursorHistory((current) => current.slice(0, -1)) + setCursor(previous) } - function closeDiagnostic() { - setDiagnosticTarget(null) + if (tasks.error) { + return } return ( -
+
qc.invalidateQueries({ queryKey: ['tasks'] })}> - Refresh + } /> -
- { - setTaskType(value as TaskTypeTab) - setStage('') - setOffset(0) - }} - className="min-w-0" - > - - {taskTypeTabs.map((tab) => ( - - {taskTypeLabel(tab)} - - ))} - - - -
- {taskType === 'upload' && ( - <> - - - - )} - +
+
+ +
+
+ +