diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d8520e..ad9a630 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: cache: true - name: Test PostgreSQL persistence contracts - run: go test ./internal/db/migrations ./internal/db/repository -count=1 + run: go test ./internal/db/migrations ./internal/db/repository ./internal/task -count=1 check: runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }} diff --git a/cmd/synaps3/admin.go b/cmd/synaps3/admin.go index 08c0682..ddd9c00 100644 --- a/cmd/synaps3/admin.go +++ b/cmd/synaps3/admin.go @@ -446,28 +446,45 @@ func adminTaskCommand() *cli.Command { }, { Name: "acknowledge", - Usage: "dismiss a failed task", - ArgsUsage: "", + Usage: "dismiss a failed task, or a backlog of failed tasks", + ArgsUsage: "[id]", + Flags: []cli.Flag{ + &cli.StringFlag{Name: "type", Usage: "dismiss only failures of this operation type"}, + &cli.StringFlag{Name: "before", Usage: "dismiss only failures recorded before this RFC 3339 time (default: now)"}, + &cli.BoolFlag{Name: "yes", Usage: "confirm dismissing every matching failed task"}, + }, Action: func(ctx context.Context, cmd *cli.Command) error { - taskID, err := requireSingleArg(cmd, "task id") - if err != nil { - return err + if cmd.Args().Len() > 0 { + if cmd.String("type") != "" || cmd.String("before") != "" || cmd.Bool("yes") { + return errors.New("pass either a task id or the bulk flags, not both") + } + return acknowledgeSingleTask(ctx, cmd) } - if _, err := strconv.ParseInt(taskID, 10, 64); err != nil { - return fmt.Errorf("invalid task id %q", taskID) + if !cmd.Bool("yes") { + return errors.New("dismissing a backlog requires --yes") + } + payload := map[string]string{} + if value := cmd.String("type"); value != "" { + payload["type"] = value + } + if value := cmd.String("before"); value != "" { + if _, err := time.Parse(time.RFC3339, value); err != nil { + return fmt.Errorf("invalid --before time %q", value) + } + payload["failed_before"] = value } client, opts, err := newAdminClientFromCommand(ctx, cmd) if err != nil { return err } - var resp map[string]string - if err := client.postJSON(ctx, "/api/v1/tasks/"+url.PathEscape(taskID)+"/acknowledge", nil, &resp, false); err != nil { + var resp adminTaskAcknowledgeResult + if err := client.postJSON(ctx, "/api/v1/tasks/acknowledge", payload, &resp, false); err != nil { return err } if opts.JSON { return writeAdminJSON(cmd.Root().Writer, resp) } - _, err = fmt.Fprintf(cmd.Root().Writer, "Task %s %s\n", taskID, resp["status"]) + _, err = fmt.Fprintf(cmd.Root().Writer, "Dismissed %d failed tasks\n", resp.Acknowledged) return err }, }, @@ -1210,6 +1227,33 @@ func validateAdminRole(role string, allowEmpty bool) error { } } +type adminTaskAcknowledgeResult struct { + Acknowledged int `json:"acknowledged"` +} + +func acknowledgeSingleTask(ctx context.Context, cmd *cli.Command) error { + taskID, err := requireSingleArg(cmd, "task id") + if err != nil { + return err + } + if _, err := strconv.ParseInt(taskID, 10, 64); err != nil { + return fmt.Errorf("invalid task id %q", taskID) + } + client, opts, err := newAdminClientFromCommand(ctx, cmd) + if err != nil { + return err + } + var resp map[string]string + if err := client.postJSON(ctx, "/api/v1/tasks/"+url.PathEscape(taskID)+"/acknowledge", nil, &resp, false); err != nil { + return err + } + if opts.JSON { + return writeAdminJSON(cmd.Root().Writer, resp) + } + _, err = fmt.Fprintf(cmd.Root().Writer, "Task %s %s\n", taskID, resp["status"]) + return err +} + func requireSingleArg(cmd *cli.Command, label string) (string, error) { if cmd.Args().Len() != 1 { return "", fmt.Errorf("expected one %s argument", label) diff --git a/cmd/synaps3/main.go b/cmd/synaps3/main.go index c85ed0f..9930b0c 100644 --- a/cmd/synaps3/main.go +++ b/cmd/synaps3/main.go @@ -21,6 +21,8 @@ import ( "github.com/strahe/synaps3/internal/provider" "github.com/strahe/synaps3/internal/synapse" sdk "github.com/strahe/synapse-go" + sdkstorage "github.com/strahe/synapse-go/storage" + sdktypes "github.com/strahe/synapse-go/types" "github.com/uptrace/bun" "github.com/urfave/cli/v3" ) @@ -316,7 +318,15 @@ func runServe(ctx context.Context, src config.Source) error { } defer func() { _ = client.Close() }() resolvedAddresses := client.ResolvedAddresses() - storageClient := synapse.AdaptStorageService(client.Storage()) + // What this node signs for. Destructive storage requests compare it against + // what a recorded request used, so a changed wallet or network stops them + // instead of acting on a data set ID that means something else there. + storageIdentity := sdkstorage.ContextIdentity{ + Payer: client.Address(), + ChainID: sdktypes.ChainID(client.Chain().ChainID()), + RecordKeeper: resolvedAddresses.FWSS, + } + storageClient := synapse.AdaptStorageService(client.Storage(), client.WarmStorage(), storageIdentity) walletQuerier := synapse.NewWalletQuerier(client.Payments(), client.Address(), client.Chain(), resolvedAddresses) walletOperator := synapse.NewWalletOperator(client.Payments(), resolvedAddresses.USDFC) filecoinReadiness := synapse.NewReadinessChecker( diff --git a/docs/en/concepts/filecoin-storage-flow.md b/docs/en/concepts/filecoin-storage-flow.md index a91c52f..e2cd958 100644 --- a/docs/en/concepts/filecoin-storage-flow.md +++ b/docs/en/concepts/filecoin-storage-flow.md @@ -68,7 +68,7 @@ One confirmation covers the whole move. SynapS3 creates the new storage service, While replacements run, the Data Sets card shows every move that still needs progress or operator attention, including parallel moves on other replicas. Discovery uses an indeterminate progress bar because the final count is not known yet. Once discovery finishes, progress uses the processed share of the final total. Progress counts unique stored content, so content shared by several versions is copied once, and separates content transferred from content deleted before it needed to move. -Some steps wait rather than fail. The dashboard distinguishes creating the new service, waiting for it to become writable, an unreachable provider, wallet funds, and missing readable content. Most waits resume on their own. If an object has no other replica and no local cache, replacement waits until a source is available; this does not count toward the retry limit. Temporary copy failures resume after a restart and do not stop other content or another replacement. Use **Retry replacement** from the Data Sets list when the dashboard shows that content needs attention, or when shutting down the old provider needs a payment settled first. A target already in use cannot be retried; choose another provider. +Some steps wait rather than fail. The dashboard distinguishes creating the new service, waiting for it to become writable, an unreachable provider, wallet funds, and missing readable content. Most waits resume on their own. If an object has no other replica and no local cache, replacement waits until a source is available; this does not count toward the retry limit. Temporary copy failures resume after a restart and do not stop other content or another replacement. If a copy stops with a result that has to be checked, the replacement waits until you retry that copy's task on the Tasks page. If the result of shutting down the old provider is unclear, SynapS3 checks the chain and tries again on its own. Use **Retry replacement** from the Data Sets list when the dashboard shows that content needs attention, or when shutting down the old provider needs a payment settled first. A target already in use cannot be retried; choose another provider. A replica cannot be replaced again while an earlier replacement is still setting up its new storage service; wait for it to finish, or retry that setup on the Tasks page. ## What Users See diff --git a/docs/en/configuration/model.md b/docs/en/configuration/model.md index 7aef8a9..b03754e 100644 --- a/docs/en/configuration/model.md +++ b/docs/en/configuration/model.md @@ -61,6 +61,8 @@ The Admin endpoint has separate exposure controls. Keep `admin.addr` on loopback SQLite is the default and recommended database for SynapS3 single-node deployments. PostgreSQL remains available when a deployment already operates an external PostgreSQL service or needs an external metadata database. Keep its DSN in protected configuration or secret storage. +`database.max_open_conns` sizes the connection pool. SQLite still writes through one connection at a time; the other connections serve reads, and a write that finds the database busy waits up to five seconds (`busy_timeout`) before it fails with `SQLITE_BUSY`. + ## Main Sections | Section | Purpose | @@ -86,7 +88,7 @@ SQLite is the default and recommended database for SynapS3 single-node deploymen | `filecoin.network` | `calibration` | | `filecoin.default_copies` | `3` | | `database.driver` | `sqlite` | -| `database.max_open_conns` | `4` | +| `database.max_open_conns` | `32` | | `database.max_idle_conns` | `2` | | `cache.max_size_gb` | `100` | | `cache.eviction_policy` | `lru` | @@ -105,7 +107,7 @@ SQLite is the default and recommended database for SynapS3 single-node deploymen | `admin.auth.username` | `admin` | | `admin.auth.session_ttl` | `12h` | -`worker.tasks.concurrency` limits all background operations. Remote storage creation, Store, Pull, and commit submission additionally share `provider_mutation_concurrency`; remote cleanup and service retirement share `destructive_mutation_concurrency`. Status and confirmation checks do not consume either mutation limit. Wallet mutations are serialized. Task settings require a SynapS3 restart, and existing tasks retain the retry limit recorded when they were created. +`worker.tasks.concurrency` limits all background operations. Remote storage creation, Store, Pull, and commit submission additionally share `provider_mutation_concurrency`; remote cleanup and service retirement share `destructive_mutation_concurrency`. Status and confirmation checks do not consume either mutation limit. Wallet mutations are serialized. An operation that finds its limit full steps aside and tries again shortly instead of holding a `concurrency` slot, so other background work keeps running. Task settings require a SynapS3 restart, and existing tasks retain the retry limit recorded when they were created. ## Admin Session Lifetime diff --git a/docs/en/getting-started/s3-clients.md b/docs/en/getting-started/s3-clients.md index c04e278..bce5c11 100644 --- a/docs/en/getting-started/s3-clients.md +++ b/docs/en/getting-started/s3-clients.md @@ -135,6 +135,7 @@ The HTTP endpoints in these examples are for local evaluation. For production, u | `AccessDenied` | Confirm the access key and secret key came from `synaps3 admin s3-user create`. | | Client tries virtual-hosted buckets | Enable path-style addressing or equivalent client setting. | | The first upload after creating a bucket returns `SlowDown` | The storage provider may still be preparing the bucket. Wait briefly, retry the upload, then confirm it with the read command. | +| Uploading data right after deleting its last version with `versionId` returns `SlowDown` | SynapS3 is still cleaning up the earlier copy of the same bytes. Wait a few minutes, then retry the upload. | | Upload succeeds but Filecoin storage is pending | Check the dashboard task view or `synaps3 admin task list --status pending`. | | Object size is rejected | Keep the object between `127` and `1,065,353,216` bytes. | | Remote host cannot reach the admin dashboard | Keep admin on loopback and use `ssh -L 9090:127.0.0.1:9090 user@server`. | diff --git a/docs/en/operations/troubleshooting.md b/docs/en/operations/troubleshooting.md index 9ed72ec..7aa4f67 100644 --- a/docs/en/operations/troubleshooting.md +++ b/docs/en/operations/troubleshooting.md @@ -132,7 +132,7 @@ Retry only after RPC connectivity, storage provider availability, wallet funds, synaps3 admin task retry 42 ``` -The API decides whether each failed task can be retried safely. Provider replacement work is recovered from **Details** → **Storage** → **Data Sets**. A wallet operation can be recovered from Tasks only when no broadcast started; an uncertain broadcast remains non-retryable. An uncertain Store offers **Check again**, which observes the provider without uploading again. Use **Dismiss** or `synaps3 admin task acknowledge ` only after reviewing the failure; acknowledged tasks remain available for the configured retention period before cleanup. +The API decides whether each failed task can be retried safely. Provider replacement work is recovered from **Details** → **Storage** → **Data Sets**. A wallet operation can be recovered from Tasks only when no broadcast started; an uncertain broadcast remains non-retryable. An uncertain Store offers **Check again**, which observes the provider without uploading again. Use **Dismiss** or `synaps3 admin task acknowledge ` only after reviewing the failure; acknowledged tasks remain available for the configured retention period before cleanup. When failures have piled up, **Dismiss all** on the Tasks page clears the ones the current Operation filter selects, and `synaps3 admin task acknowledge --type --yes` does the same from the CLI; failures recorded after you confirm stay in the list. ## Provider or RPC Issues diff --git a/docs/en/reference/admin-api.md b/docs/en/reference/admin-api.md index 9967fba..eca668d 100644 --- a/docs/en/reference/admin-api.md +++ b/docs/en/reference/admin-api.md @@ -284,11 +284,33 @@ If the attempt changed after it was inspected, the API returns `409 Conflict`. I | `GET` | `/api/v1/tasks/stats` | Count tasks by status. | | `POST` | `/api/v1/tasks/{id}/retry` | Recover a failed task when `retryable` is true. | | `POST` | `/api/v1/tasks/{id}/acknowledge` | Dismiss a failed task when `acknowledgeable` is true. Acknowledgement starts its retention period, after which it may be cleaned up. | +| `GET` | `/api/v1/tasks/acknowledge/preview` | Count what a bulk dismissal would cover. Accepts the same optional `type`. Returns `count` and the `as_of` cutoff it counted at. | +| `POST` | `/api/v1/tasks/acknowledge` | Dismiss a backlog of failed tasks at once. Returns `acknowledged` with the number dismissed. | `status` is `pending`, `running`, `completed`, `failed`, or `cancelled`. `presentation_status` renders pending work as `queued`, `scheduled`, or `waiting`, and acknowledged failures as `dismissed`. Responses also include `operation`, optional subject identity, and server-computed `retryable` and `acknowledgeable` flags. The `status` filter also accepts `dismissed`. `status=failed` returns only unacknowledged failures, while `status=dismissed` returns acknowledged failures. `/api/v1/tasks/stats` reports those groups separately as `failed` and `dismissed`. +Bulk dismissal takes a JSON body with an optional `type` and an optional RFC 3339 `failed_before`, which defaults to the moment the request is handled: + +```json +{ + "type": "storage_store", + "failed_before": "2026-09-12T08:30:00Z" +} +``` + +Failures recorded after that moment stay visible, so a backlog can be cleared without hiding a failure nobody has reviewed. An unknown `type` or an unparsable `failed_before` returns `400 Bad Request`, as does a body with unknown fields, trailing content, or more than 4 KiB. + +To show a number before dismissing, count first and then confirm with the cutoff that count was taken at: + +```bash +curl -s "$ADMIN/api/v1/tasks/acknowledge/preview?type=storage_store" +# {"count":12,"as_of":"2026-09-12T08:30:00.123456789Z"} +``` + +Passing that `as_of` back as `failed_before` dismisses exactly what was counted. + `/api/v1/overview` groups `tasks.by_status` by `status`, so its `failed` count includes acknowledged failures. Use `tasks.attention.failed` for unacknowledged failures or `/api/v1/tasks/stats` for counts split between `failed` and `dismissed`. Pagination is newest-first. When `next_cursor` is present, pass it as `cursor` to fetch the next page. Provider replacement recovery remains in the Data Sets API. Wallet operations are retryable only before a broadcast starts. Retrying an uncertain Store checks the provider and does not upload the bytes again. diff --git a/docs/en/reference/cli-api.md b/docs/en/reference/cli-api.md index b762d46..c19521b 100644 --- a/docs/en/reference/cli-api.md +++ b/docs/en/reference/cli-api.md @@ -84,6 +84,7 @@ synaps3 admin task stats synaps3 admin task list --status failed --limit 100 synaps3 admin task retry 42 synaps3 admin task acknowledge 42 +synaps3 admin task acknowledge --type storage_store --yes synaps3 admin storage-confirmation list synaps3 admin storage-confirmation release 42 --attempt-id current-attempt-id --yes ``` @@ -100,7 +101,7 @@ Admin global flags must appear after `admin` and before the subcommand: Task listing supports `--type`, `--status`, `--limit`, and ID-based `--cursor`. Valid status filters are `pending`, `running`, `completed`, `failed`, `cancelled`, and `dismissed`. Pending work is presented as queued, scheduled, or waiting; `failed` returns unacknowledged failures and `dismissed` returns acknowledged failures. -`synaps3 admin task retry` recovers only tasks whose response says they are retryable. Provider replacement recovery remains under **Details** → **Storage** → **Data Sets**. A wallet operation can be retried only when its broadcast never started; an operation with an uncertain broadcast remains non-retryable. For an uncertain Store, the dashboard labels Retry as **Check again**: this checks the provider without uploading again. Use `synaps3 admin task acknowledge ` to dismiss a failed task after reviewing its outcome; acknowledgement starts its retention period, after which it may be cleaned up. +`synaps3 admin task retry` recovers only tasks whose response says they are retryable. Provider replacement recovery remains under **Details** → **Storage** → **Data Sets**. A wallet operation can be retried only when its broadcast never started; an operation with an uncertain broadcast remains non-retryable. For an uncertain Store, the dashboard labels Retry as **Check again**: this checks the provider without uploading again. Use `synaps3 admin task acknowledge ` to dismiss a failed task after reviewing its outcome; acknowledgement starts its retention period, after which it may be cleaned up. To clear a backlog, run it without an ID and confirm with `--yes`: `--type` limits it to one operation, and `--before` sets an RFC 3339 cutoff that defaults to now, so failures recorded later stay visible. `synaps3 admin storage-confirmation list` shows storage confirmations that need review. Verify the piece CID, provider, current attempt ID, attempted time, and any available transaction evidence before running `storage-confirmation release --attempt-id --yes`; release only if you accept that the provider may already store the piece and resubmission may create duplicate paid storage. A stale attempt ID is refused. diff --git a/docs/en/reference/s3-compatibility.md b/docs/en/reference/s3-compatibility.md index 331f9ef..4aa229e 100644 --- a/docs/en/reference/s3-compatibility.md +++ b/docs/en/reference/s3-compatibility.md @@ -59,7 +59,7 @@ SynapS3 mainly supports path-style S3 access for writing bucket and object data ## Versioning Behavior -Buckets behave as versioning-enabled. A delete without `versionId` creates a delete marker. A delete with `versionId` either deletes that data version or delete marker, or returns an error; the gateway does not keep the request and run it later. Deleting a delete marker is not blocked by Filecoin storage. A data-version delete returns `400 InvalidRequest`, or an entry-specific `InvalidRequest` from `DeleteObjects`, while storage work is active or a submitted Filecoin transaction is awaiting confirmation. For an otherwise eligible data version, stopped storage work that has not submitted a transaction does not block deletion. After a data version is deleted, unreferenced remote storage is queued for background cleanup; storage shared with another version is preserved. Version listing returns object versions and delete markers. +Buckets behave as versioning-enabled. A delete without `versionId` creates a delete marker. A delete with `versionId` either deletes that data version or delete marker, or returns an error; the gateway does not keep the request and run it later. Deleting a delete marker is not blocked by Filecoin storage. A data-version delete returns `400 InvalidRequest`, or an entry-specific `InvalidRequest` from `DeleteObjects`, while storage work is active or a submitted Filecoin transaction is awaiting confirmation. For an otherwise eligible data version, stopped storage work that has not submitted a transaction does not block deletion. After a data version is deleted, unreferenced remote storage is queued for background cleanup; storage shared with another version is preserved. Cleanup asks the storage provider to remove the copy, which not every provider supports: a copy the provider will not or cannot remove may remain with it after the version is gone from the gateway. Until that cleanup finishes, usually within minutes, uploading the same bytes to the bucket returns `503 SlowDown`; a later retry stores them as new data. Version listing returns object versions and delete markers. ## What Is Intentionally Out of Scope diff --git a/docs/zh/concepts/filecoin-storage-flow.md b/docs/zh/concepts/filecoin-storage-flow.md index 59b410d..f7986fe 100644 --- a/docs/zh/concepts/filecoin-storage-flow.md +++ b/docs/zh/concepts/filecoin-storage-flow.md @@ -68,7 +68,7 @@ synaps3 admin task retry 42 替换进行期间，Data Sets 卡片会显示仍需推进或需要操作的所有迁移，包括其他副本上的并行迁移。发现内容时还不知道最终总数，因此使用不确定进度条；发现完成后，按最终总数中的已处理比例显示确定进度。进度按唯一存储内容计数，因此被多个版本共享的内容只复制一次，并分别统计已迁移内容与复制前已删除、无需再迁移的内容。 -有些步骤是等待而不是失败。仪表盘会区分创建新服务、等待服务可写、存储提供方不可达、钱包资金和缺少可读内容。多数等待会自行继续。如果某个对象既没有其他副本、也没有本地缓存，替换会等待到来源可用，并且这段等待不计入重试次数。暂时的复制失败会在重启后继续，也不会阻断其他内容或另一条替换。当仪表盘提示内容需要处理，或关闭旧存储提供方前需要结清欠费时，在 Data Sets 列表中使用 **Retry replacement**。目标已被占用时不能重试，请改选存储提供方。 +有些步骤是等待而不是失败。仪表盘会区分创建新服务、等待服务可写、存储提供方不可达、钱包资金和缺少可读内容。多数等待会自行继续。如果某个对象既没有其他副本、也没有本地缓存，替换会等待到来源可用，并且这段等待不计入重试次数。暂时的复制失败会在重启后继续，也不会阻断其他内容或另一条替换。如果某个副本的复制停在需要核实的结果上，替换会等待你在 Tasks 页面重试这个复制任务。关闭旧存储提供方的结果不明确时，SynapS3 会查询链上状态并自动重试。当仪表盘提示内容需要处理，或关闭旧存储提供方前需要结清欠费时，在 Data Sets 列表中使用 **Retry replacement**。目标已被占用时不能重试，请改选存储提供方。如果之前的替换仍在创建新的存储服务，这个副本暂时不能再次替换；请等待它完成，或在 Tasks 页面重试该创建任务。 ## 用户能看到什么 diff --git a/docs/zh/configuration/model.md b/docs/zh/configuration/model.md index 7b3fe73..ad47943 100644 --- a/docs/zh/configuration/model.md +++ b/docs/zh/configuration/model.md @@ -61,6 +61,8 @@ Admin 端点有独立的暴露范围控制。让 `admin.addr` 保持回环地址 SQLite 是 SynapS3 单机部署的默认且推荐数据库。已有 PostgreSQL 运维体系或需要外置元数据数据库时，可以使用 PostgreSQL；其 DSN 必须保存在受保护的配置或密钥存储中。 +`database.max_open_conns` 决定连接池大小。SQLite 同一时刻仍只有一个连接能写入，其余连接用于读取；写入遇到数据库忙时最多等待 5 秒（`busy_timeout`），超时则以 `SQLITE_BUSY` 失败。 + ## 主要配置段 | 配置段 | 用途 | @@ -86,7 +88,7 @@ SQLite 是 SynapS3 单机部署的默认且推荐数据库。已有 PostgreSQL | `filecoin.network` | `calibration` | | `filecoin.default_copies` | `3` | | `database.driver` | `sqlite` | -| `database.max_open_conns` | `4` | +| `database.max_open_conns` | `32` | | `database.max_idle_conns` | `2` | | `cache.max_size_gb` | `100` | | `cache.eviction_policy` | `lru` | @@ -105,7 +107,7 @@ SQLite 是 SynapS3 单机部署的默认且推荐数据库。已有 PostgreSQL | `admin.auth.username` | `admin` | | `admin.auth.session_ttl` | `12h` | -`worker.tasks.concurrency` 限制全部后台操作。创建远端存储、Store、Pull 和提交存储承诺共同受 `provider_mutation_concurrency` 限制；远端清理与服务退休共同受 `destructive_mutation_concurrency` 限制。状态和确认查询不占用这些变更并发额度。钱包变更始终串行执行。任务设置修改后必须重启 SynapS3，已经创建的任务保留创建时记录的重试上限。 +`worker.tasks.concurrency` 限制全部后台操作。创建远端存储、Store、Pull 和提交存储承诺共同受 `provider_mutation_concurrency` 限制；远端清理与服务退休共同受 `destructive_mutation_concurrency` 限制。状态和确认查询不占用这些变更并发额度。钱包变更始终串行执行。操作遇到对应额度已满时会先让出、稍后自动再试，不占用 `concurrency` 名额，其他后台任务照常运行。任务设置修改后必须重启 SynapS3，已经创建的任务保留创建时记录的重试上限。 ## Admin 会话时长 diff --git a/docs/zh/getting-started/s3-clients.md b/docs/zh/getting-started/s3-clients.md index c4a894d..a283a88 100644 --- a/docs/zh/getting-started/s3-clients.md +++ b/docs/zh/getting-started/s3-clients.md @@ -135,6 +135,7 @@ alias 会把凭据保存在 `~/.mc/config.json`。无回显提示可以避免 se | `AccessDenied` | 确认 access key 和 secret key 来自 `synaps3 admin s3-user create`。 | | 客户端使用 virtual-hosted 存储桶访问 | 开启 path-style addressing 或客户端中的等价设置。 | | 创建存储桶后的首次上传返回 `SlowDown` | 存储提供方可能仍在准备该存储桶。短暂等待后重试上传，再使用读取命令确认结果。 | +| 用 `versionId` 删除某份数据的最后一个版本后，立即上传相同内容返回 `SlowDown` | SynapS3 仍在清理这份数据的旧副本。等待几分钟后再重试上传。 | | 上传成功但 Filecoin 存储仍在等待 | 查看仪表盘任务页，或运行 `synaps3 admin task list --status pending`。 | | 对象大小被拒绝 | 确保对象大小在 `127` 到 `1,065,353,216` 字节之间。 | | 远程主机无法访问 Admin 仪表盘 | 保持 Admin 监听本机回环地址，并使用 `ssh -L 9090:127.0.0.1:9090 user@server`。 | diff --git a/docs/zh/operations/troubleshooting.md b/docs/zh/operations/troubleshooting.md index 1635512..bb6ab4b 100644 --- a/docs/zh/operations/troubleshooting.md +++ b/docs/zh/operations/troubleshooting.md @@ -132,7 +132,7 @@ synaps3 admin task list --status failed --limit 100 synaps3 admin task retry 42 ``` -API 会判断每个失败任务能否安全重试。存储提供方替换从 **Details** → **Storage** → **Data Sets** 恢复。只有尚未发出广播的钱包操作可以从 Tasks 恢复；广播结果不确定时仍不可重试。Store 结果不确定时会提供 **Check again**，它只观察存储提供方，不会再次上传。只有在核对失败结果后才使用 **Dismiss** 或 `synaps3 admin task acknowledge `；确认后的任务会继续保留配置的时长，再由后台清理。 +API 会判断每个失败任务能否安全重试。存储提供方替换从 **Details** → **Storage** → **Data Sets** 恢复。只有尚未发出广播的钱包操作可以从 Tasks 恢复；广播结果不确定时仍不可重试。Store 结果不确定时会提供 **Check again**，它只观察存储提供方，不会再次上传。只有在核对失败结果后才使用 **Dismiss** 或 `synaps3 admin task acknowledge `；确认后的任务会继续保留配置的时长，再由后台清理。失败任务积压时，任务页的 **Dismiss all** 会处理当前操作类型筛选下的失败任务，命令行对应 `synaps3 admin task acknowledge --type <操作> --yes`；在你确认之后才记录的失败仍会留在列表里。 ## 存储提供方或 RPC 问题 diff --git a/docs/zh/reference/admin-api.md b/docs/zh/reference/admin-api.md index f06e273..1fb8e27 100644 --- a/docs/zh/reference/admin-api.md +++ b/docs/zh/reference/admin-api.md @@ -284,11 +284,33 @@ Admin 响应包含 `Content-Security-Policy`、`X-Content-Type-Options: nosniff` | `GET` | `/api/v1/tasks/stats` | 按状态统计任务。 | | `POST` | `/api/v1/tasks/{id}/retry` | 当 `retryable` 为 true 时恢复失败任务。 | | `POST` | `/api/v1/tasks/{id}/acknowledge` | 当 `acknowledgeable` 为 true 时把失败任务标记为已处理。确认后开始计算保留期，到期后可能被清理。 | +| `GET` | `/api/v1/tasks/acknowledge/preview` | 统计批量处理会覆盖多少条失败任务，同样接受可选的 `type`，返回 `count` 和统计时刻 `as_of`。 | +| `POST` | `/api/v1/tasks/acknowledge` | 一次性处理积压的失败任务，返回 `acknowledged` 表示处理了多少条。 | `status` 为 `pending`、`running`、`completed`、`failed` 或 `cancelled`。`presentation_status` 会把 pending 工作显示为 `queued`、`scheduled` 或 `waiting`，并把已确认的失败任务显示为 `dismissed`。响应还包含 `operation`、可选的 subject 身份，以及服务端计算的 `retryable` 和 `acknowledgeable`。 `status` 过滤还接受 `dismissed`。`status=failed` 只返回尚未确认的失败，`status=dismissed` 返回已确认的失败；`/api/v1/tasks/stats` 也分别以 `failed` 和 `dismissed` 统计两组任务。 +批量处理接受 JSON 请求体，其中 `type` 和 RFC 3339 格式的 `failed_before` 都是可选的，`failed_before` 默认为服务端处理请求的时刻： + +```json +{ + "type": "storage_store", + "failed_before": "2026-09-12T08:30:00Z" +} +``` + +该时刻之后记录的失败仍然可见，因此清理积压不会掩盖还没有人看过的失败。`type` 未知、`failed_before` 无法解析，或请求体包含未知字段、多余内容、超过 4 KiB 时返回 `400 Bad Request`。 + +需要先看到数量再处理时，先统计，再用统计时刻确认： + +```bash +curl -s "$ADMIN/api/v1/tasks/acknowledge/preview?type=storage_store" +# {"count":12,"as_of":"2026-09-12T08:30:00.123456789Z"} +``` + +把返回的 `as_of` 作为 `failed_before` 回传，处理的就正好是统计到的那些。 + `/api/v1/overview` 的 `tasks.by_status` 按 `status` 聚合，因此其中的 `failed` 会包含已确认的失败。需要尚未确认的失败数时使用 `tasks.attention.failed`；需要分别统计 `failed` 和 `dismissed` 时使用 `/api/v1/tasks/stats`。 分页按任务 ID 从新到旧。响应存在 `next_cursor` 时，把它作为下一次请求的 `cursor`。存储提供方替换仍通过 Data Sets API 恢复。钱包操作只有在广播开始前才可重试。重试结果不确定的 Store 只会查询存储提供方，不会重新上传字节。 diff --git a/docs/zh/reference/cli-api.md b/docs/zh/reference/cli-api.md index c4a1126..3956c0b 100644 --- a/docs/zh/reference/cli-api.md +++ b/docs/zh/reference/cli-api.md @@ -84,6 +84,7 @@ synaps3 admin task stats synaps3 admin task list --status failed --limit 100 synaps3 admin task retry 42 synaps3 admin task acknowledge 42 +synaps3 admin task acknowledge --type storage_store --yes synaps3 admin storage-confirmation list synaps3 admin storage-confirmation release 42 --attempt-id current-attempt-id --yes ``` @@ -100,7 +101,7 @@ Admin 全局 flags 必须放在 `admin` 之后、子命令之前： 列出后台任务时支持 `--type`、`--status`、`--limit` 和基于任务 ID 的 `--cursor`。有效的状态过滤值为 `pending`、`running`、`completed`、`failed`、`cancelled` 和 `dismissed`。pending 工作会显示为 queued、scheduled 或 waiting；`failed` 返回尚未确认的失败，`dismissed` 返回已确认的失败。 -`synaps3 admin task retry` 只恢复响应中标记为可重试的失败任务。存储提供方替换仍在 **Details** → **Storage** → **Data Sets** 中恢复。只有尚未发出广播的钱包操作可以重试；广播结果不确定时仍不可重试。Store 结果不确定时，dashboard 会把 Retry 显示为 **Check again**：该操作只查询存储提供方，不会重新上传。核对失败结果后，可用 `synaps3 admin task acknowledge ` 将任务标记为已处理；确认后开始计算保留期，到期后可能被清理。 +`synaps3 admin task retry` 只恢复响应中标记为可重试的失败任务。存储提供方替换仍在 **Details** → **Storage** → **Data Sets** 中恢复。只有尚未发出广播的钱包操作可以重试；广播结果不确定时仍不可重试。Store 结果不确定时，dashboard 会把 Retry 显示为 **Check again**：该操作只查询存储提供方，不会重新上传。核对失败结果后，可用 `synaps3 admin task acknowledge ` 将任务标记为已处理；确认后开始计算保留期，到期后可能被清理。需要清理积压时，不带 ID 运行并用 `--yes` 确认：`--type` 限定某一种操作，`--before` 指定 RFC 3339 截止时刻（默认为当前时间），该时刻之后记录的失败仍然可见。 `synaps3 admin storage-confirmation list` 会显示需要核对的存储确认。核对存储提供方、transaction 和当前 attempt 后，使用 `storage-confirmation release --attempt-id --yes` 表示确认存储提供方可能已经接受该 piece，并允许正常恢复流程再次提交。过期的 attempt ID 会被拒绝。 diff --git a/docs/zh/reference/s3-compatibility.md b/docs/zh/reference/s3-compatibility.md index cbe9928..cc8d70a 100644 --- a/docs/zh/reference/s3-compatibility.md +++ b/docs/zh/reference/s3-compatibility.md @@ -59,7 +59,7 @@ SynapS3 主要支持 path-style S3 访问，负责把存储桶和对象数据写 ## 版本控制行为 -存储桶按 versioning-enabled 处理。不带 `versionId` 的删除会创建 delete marker。带 `versionId` 的请求要么删除对应的数据版本或 delete marker，要么返回错误；系统不会保留请求并在之后自动执行。删除 delete marker 不受 Filecoin 存储进度阻塞。当存储工作仍在进行，或已提交的 Filecoin 交易仍在等待确认时，删除数据版本会收到 `400 InvalidRequest`，`DeleteObjects` 则为对应条目返回 `InvalidRequest`。对于其他条件均符合永久删除要求的数据版本，已停止且尚未提交交易的存储工作不会阻止删除。数据版本删除后，不再被任何版本引用的远端存储会进入后台清理；其他版本仍在使用的共享存储会保留。Version listing 会返回对象版本和 delete markers。 +存储桶按 versioning-enabled 处理。不带 `versionId` 的删除会创建 delete marker。带 `versionId` 的请求要么删除对应的数据版本或 delete marker，要么返回错误；系统不会保留请求并在之后自动执行。删除 delete marker 不受 Filecoin 存储进度阻塞。当存储工作仍在进行，或已提交的 Filecoin 交易仍在等待确认时，删除数据版本会收到 `400 InvalidRequest`，`DeleteObjects` 则为对应条目返回 `InvalidRequest`。对于其他条件均符合永久删除要求的数据版本，已停止且尚未提交交易的存储工作不会阻止删除。数据版本删除后，不再被任何版本引用的远端存储会进入后台清理；其他版本仍在使用的共享存储会保留。清理会请求存储提供方删除副本，但并非所有存储提供方都支持：提供方不支持或无法删除的副本，在该版本从网关消失后仍可能留在提供方处。清理完成前（通常几分钟内），向该存储桶上传相同内容会收到 `503 SlowDown`；稍后重试即可，届时会作为新数据存储。Version listing 会返回对象版本和 delete markers。 ## 有意不支持 diff --git a/go.mod b/go.mod index 3f17b4f..4aea488 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/knadh/koanf/v2 v2.3.4 github.com/multiformats/go-multihash v0.2.3 github.com/prometheus/client_golang v1.23.2 - github.com/strahe/synapse-go v0.5.1 + github.com/strahe/synapse-go v0.6.0-beta.1 github.com/uptrace/bun v1.2.18 github.com/uptrace/bun/dialect/pgdialect v1.2.18 github.com/uptrace/bun/dialect/sqlitedialect v1.2.18 diff --git a/go.sum b/go.sum index ae71d72..0f30fc2 100644 --- a/go.sum +++ b/go.sum @@ -388,8 +388,8 @@ github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/strahe/synapse-go v0.5.1 h1:l7FaCuzxbrurkhxP4SlAH1DvqIL8ucf7u2q+pizkrA8= -github.com/strahe/synapse-go v0.5.1/go.mod h1:bOw0zW2mWUeB2MNlplO9lOJq0c9teZNicMiEx0vjR6c= +github.com/strahe/synapse-go v0.6.0-beta.1 h1:+s3c6K1pRKmW8LaNUFMATzYR1K4qajFsdY0qkLO47xo= +github.com/strahe/synapse-go v0.6.0-beta.1/go.mod h1:bOw0zW2mWUeB2MNlplO9lOJq0c9teZNicMiEx0vjR6c= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= diff --git a/internal/admin/api_buckets_test.go b/internal/admin/api_buckets_test.go index b122485..7911e08 100644 --- a/internal/admin/api_buckets_test.go +++ b/internal/admin/api_buckets_test.go @@ -30,6 +30,7 @@ import ( "github.com/strahe/synaps3/internal/objectreader" "github.com/strahe/synaps3/internal/observability" "github.com/strahe/synaps3/internal/s3iam" + "github.com/strahe/synaps3/internal/storagepipeline" taskengine "github.com/strahe/synaps3/internal/task" "github.com/strahe/synaps3/internal/testutil" idtypes "github.com/strahe/synaps3/internal/types" @@ -663,7 +664,10 @@ func markAdminFailedUpload(t *testing.T, db *bun.DB, repos *repository.Repositor } // Ingest failure is reported to operators from the content, so the copy // error is recorded there too, the way the pipeline does it. - if err := repos.Contents.RecordContentFailure(ctx, upload.ID, message); err != nil { + if _, err := db.NewUpdate().Model((*model.StorageContent)(nil)). + Set("error_message = ?", message). + Where("id = ?", upload.ID). + Exec(ctx); err != nil { t.Fatalf("record content failure: %v", err) } return upload @@ -3046,9 +3050,15 @@ func TestAPIBucketDeletedObjectPermanentDeleteReportsActiveStorageWork(t *testin t.Fatalf("Buckets.Create: %v", err) } _, versionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/file.txt", 7, "etag-file", "checksum-file", "text/plain", model.ObjectStateUploading) + version, err := repos.Objects.GetVersionByID(ctx, versionID) + if err != nil || version == nil || version.ContentID == nil { + t.Fatalf("GetVersionByID = %#v, err=%v", version, err) + } + // Ingest is scheduled against the content the version names. if _, _, err := srv.taskService.Enqueue(ctx, taskengine.EnqueueRequest{ - Type: model.TaskTypeUploadPlan, IdempotencyKey: "upload:" + versionID, - Input: map[string]any{"version_id": versionID}, SubjectType: "object_version", SubjectKey: versionID, + Type: model.TaskTypeUploadPlan, IdempotencyKey: storagepipeline.UploadPlanKey(*version.ContentID), + Input: map[string]any{"content_id": *version.ContentID}, + SubjectType: "storage_content", SubjectKey: strconv.FormatInt(*version.ContentID, 10), }); err != nil { t.Fatalf("Enqueue upload task: %v", err) } @@ -4305,9 +4315,15 @@ func TestAPIBucketObjectPermanentDeleteReportsActiveStorageWork(t *testing.T) { t.Fatalf("Buckets.Create: %v", err) } _, versionID := seedAdminObjectVersion(t, srv.db, repos, bucket, "folder/file.txt", 8, "etag-current", "checksum-current", "text/plain", model.ObjectStateCached) + version, err := repos.Objects.GetVersionByID(ctx, versionID) + if err != nil || version == nil || version.ContentID == nil { + t.Fatalf("GetVersionByID = %#v, err=%v", version, err) + } + // Ingest is scheduled against the content the version names. if _, _, err := srv.taskService.Enqueue(ctx, taskengine.EnqueueRequest{ - Type: model.TaskTypeUploadPlan, IdempotencyKey: "upload:" + versionID, - Input: map[string]any{"version_id": versionID}, SubjectType: "object_version", SubjectKey: versionID, + Type: model.TaskTypeUploadPlan, IdempotencyKey: storagepipeline.UploadPlanKey(*version.ContentID), + Input: map[string]any{"content_id": *version.ContentID}, + SubjectType: "storage_content", SubjectKey: strconv.FormatInt(*version.ContentID, 10), }); err != nil { t.Fatalf("Enqueue upload task: %v", err) } diff --git a/internal/admin/api_overview_test.go b/internal/admin/api_overview_test.go index 23c8d8c..ee9a810 100644 --- a/internal/admin/api_overview_test.go +++ b/internal/admin/api_overview_test.go @@ -215,7 +215,7 @@ func TestAPIOverviewIncludesAttentionAndActivePipeline(t *testing.T) { if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, failed); err != nil { t.Fatalf("seed failed object: %v", err) } - overviewSeedFailedCopy(t, repos, bucket.ID, *failed.ContentID) + overviewSeedFailedCopy(t, db, repos, bucket.ID, *failed.ContentID) // A version is unavailable when nothing can serve it: no cached bytes and // no readable committed copy. @@ -321,7 +321,7 @@ func TestAPIOverviewIncludesAttentionAndActivePipeline(t *testing.T) { // overviewSeedFailedCopy binds one copy for a content and fails it, which is // how a content's ingest failure is now expressed. -func overviewSeedFailedCopy(t *testing.T, repos *repository.Repositories, bucketID, contentID int64) { +func overviewSeedFailedCopy(t *testing.T, db *bun.DB, repos *repository.Repositories, bucketID, contentID int64) { t.Helper() ctx := context.Background() binding, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ @@ -336,7 +336,10 @@ func overviewSeedFailedCopy(t *testing.T, repos *repository.Repositories, bucket }}); err != nil { t.Fatalf("seed failed copy: %v", err) } - if err := repos.Contents.RecordContentFailure(ctx, contentID, "ingest failed"); err != nil { + if _, err := db.NewUpdate().Model((*model.StorageContent)(nil)). + Set("error_message = ?", "ingest failed"). + Where("id = ?", contentID). + Exec(ctx); err != nil { t.Fatalf("record content failure: %v", err) } } diff --git a/internal/admin/api_replacement.go b/internal/admin/api_replacement.go index 1c119a4..5f11955 100644 --- a/internal/admin/api_replacement.go +++ b/internal/admin/api_replacement.go @@ -370,6 +370,11 @@ func (s *Server) writeReplacementError(w http.ResponseWriter, err error, bucketN "error": "this replica is already being replaced", "code": code, }) + case errors.Is(err, storagereplacement.ErrTargetCreating): + writeJSON(w, http.StatusConflict, map[string]string{ + "error": "the earlier replacement of this replica is still setting up its storage service", + "code": code, + }) case errors.Is(err, storagereplacement.ErrTargetInUse): writeJSON(w, http.StatusConflict, map[string]string{ "error": "that provider already stores a replica of this bucket", diff --git a/internal/admin/api_tasks.go b/internal/admin/api_tasks.go index b22e3cd..c43a556 100644 --- a/internal/admin/api_tasks.go +++ b/internal/admin/api_tasks.go @@ -1,6 +1,9 @@ package admin import ( + "encoding/json" + "errors" + "io" "net/http" "strconv" "time" @@ -240,6 +243,100 @@ type taskStatsItem struct { Count int64 `json:"count"` } +type taskAcknowledgeRequest struct { + Type string `json:"type,omitempty"` + FailedBefore string `json:"failed_before,omitempty"` +} + +type taskAcknowledgeResponse struct { + Acknowledged int `json:"acknowledged"` +} + +type taskAcknowledgePreviewResponse struct { + Count int `json:"count"` + AsOf string `json:"as_of"` +} + +// taskAcknowledgeMaxBodyBytes bounds the bulk dismissal body. It carries an +// operation name and a timestamp and nothing else. +const taskAcknowledgeMaxBodyBytes = 4096 + +// handleAPITaskAcknowledgePreview reports how many failures a bulk dismissal +// would cover, and the cutoff it counted them at. Confirming with that same +// cutoff dismisses exactly what was counted: failures recorded in between stay +// visible instead of being swept up by a number the operator never saw. +func (s *Server) handleAPITaskAcknowledgePreview(w http.ResponseWriter, r *http.Request) { + if s.taskService == nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "task service unavailable"}) + return + } + filter := repository.TaskAcknowledgeFilter{FailedBefore: time.Now().UTC()} + if taskType := r.URL.Query().Get("type"); taskType != "" { + if !validTaskType(model.TaskType(taskType)) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "unknown task type"}) + return + } + filter.Type = model.TaskType(taskType) + } + count, err := s.taskService.CountAcknowledgeable(r.Context(), filter) + if err != nil { + s.logger.Error("api: failed to count dismissable tasks", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) + return + } + writeJSON(w, http.StatusOK, taskAcknowledgePreviewResponse{ + Count: count, AsOf: filter.FailedBefore.Format(time.RFC3339Nano), + }) +} + +// handleAPITaskAcknowledgeMatching dismisses a backlog of failures in one call. +// It takes the same selection the Tasks page offers: an optional operation and +// the moment the operator decided, so failures recorded later stay visible. +func (s *Server) handleAPITaskAcknowledgeMatching(w http.ResponseWriter, r *http.Request) { + if s.taskService == nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "task service unavailable"}) + return + } + // A body that does not decode cleanly is refused rather than partially + // applied: a mistyped field would otherwise widen the dismissal to everything + // instead of the selection the operator confirmed. An empty body, however it + // is framed, still selects every failure before now. + var request taskAcknowledgeRequest + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, taskAcknowledgeMaxBodyBytes)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&request); err != nil && !errors.Is(err, io.EOF) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) + return + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) + return + } + filter := repository.TaskAcknowledgeFilter{FailedBefore: time.Now()} + if request.Type != "" { + if !validTaskType(model.TaskType(request.Type)) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "unknown task type"}) + return + } + filter.Type = model.TaskType(request.Type) + } + if request.FailedBefore != "" { + failedBefore, err := time.Parse(time.RFC3339, request.FailedBefore) + if err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "failed_before must be an RFC 3339 time"}) + return + } + filter.FailedBefore = failedBefore + } + acknowledged, err := s.taskService.AcknowledgeMatching(r.Context(), filter) + if err != nil { + s.logger.Error("api: failed to acknowledge tasks", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) + return + } + writeJSON(w, http.StatusOK, taskAcknowledgeResponse{Acknowledged: acknowledged}) +} + func (s *Server) handleAPITaskStats(w http.ResponseWriter, r *http.Request) { counts, err := s.repos.Tasks.CountByPresentationStatus(r.Context()) if err != nil { diff --git a/internal/admin/api_tasks_test.go b/internal/admin/api_tasks_test.go index 1e3c961..9d72df1 100644 --- a/internal/admin/api_tasks_test.go +++ b/internal/admin/api_tasks_test.go @@ -3,6 +3,7 @@ package admin import ( "context" "encoding/json" + "io" "net/http" "net/http/httptest" "strconv" @@ -15,6 +16,7 @@ import ( "github.com/strahe/synaps3/internal/model" taskengine "github.com/strahe/synaps3/internal/task" "github.com/strahe/synaps3/internal/testutil" + "github.com/uptrace/bun" ) type adminTaskHandler struct { @@ -346,18 +348,193 @@ func TestAPITaskFlagsComeFromRegistry(t *testing.T) { type adminTaskFixture struct { t *testing.T + db *bun.DB repos *repository.Repositories service *taskengine.Service server *Server } +// TestAPITaskBulkAcknowledgeDismissesTheSelectedBacklog checks that a bulk +// dismissal covers the selected operation only, and reports how many it +// dismissed. +func TestAPITaskBulkAcknowledgeDismissesTheSelectedBacklog(t *testing.T) { + fixture := newAdminTaskFixture(t) + now := time.Now() + fail := func(taskType model.TaskType, key string) *model.Task { + t.Helper() + row := fixture.enqueue(t, taskType, key, now, "storage_copy", key) + fixture.transition(t, row.ID, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, + FailureReason: new("store_not_started"), LastError: new("provider unavailable"), + }) + return row + } + stored := fail(model.TaskTypeStorageStore, "bulk-store") + evicted := fail(model.TaskTypeCacheEvict, "bulk-evict") + + rr := fixture.request(http.MethodPost, "/api/v1/tasks/acknowledge", strings.NewReader(`{"type":"storage_store"}`)) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusOK, rr.Body.String()) + } + var body struct { + Acknowledged int `json:"acknowledged"` + } + decodeJSON(t, rr, &body) + if body.Acknowledged != 1 { + t.Fatalf("acknowledged = %d, want 1", body.Acknowledged) + } + dismissed, err := fixture.repos.Tasks.GetByID(t.Context(), stored.ID) + if err != nil || dismissed == nil || dismissed.AcknowledgedAt == nil || dismissed.RetentionUntil == nil { + t.Fatalf("dismissed task = %#v, err=%v", dismissed, err) + } + untouched, err := fixture.repos.Tasks.GetByID(t.Context(), evicted.ID) + if err != nil || untouched == nil || untouched.AcknowledgedAt != nil { + t.Fatalf("task of another operation = %#v, err=%v", untouched, err) + } + + rr = fixture.request(http.MethodPost, "/api/v1/tasks/acknowledge", strings.NewReader(`{"type":"not-an-operation"}`)) + if rr.Code != http.StatusBadRequest { + t.Fatalf("unknown type status = %d, want %d", rr.Code, http.StatusBadRequest) + } + rr = fixture.request(http.MethodPost, "/api/v1/tasks/acknowledge", strings.NewReader(`{"failed_before":"yesterday"}`)) + if rr.Code != http.StatusBadRequest { + t.Fatalf("invalid cutoff status = %d, want %d", rr.Code, http.StatusBadRequest) + } +} + +// The number an operator confirms has to be the number that is dismissed. The +// preview counts and reports the cutoff it counted at; confirming with that +// cutoff leaves anything that failed in between visible. +func TestAPITaskBulkAcknowledgePreviewFreezesWhatIsDismissed(t *testing.T) { + fixture := newAdminTaskFixture(t) + now := time.Now() + fail := func(taskType model.TaskType, key string) *model.Task { + t.Helper() + row := fixture.enqueue(t, taskType, key, now, "storage_copy", key) + fixture.transition(t, row.ID, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, + FailureReason: new("store_not_started"), LastError: new("provider unavailable"), + }) + return row + } + seen := fail(model.TaskTypeStorageStore, "preview-seen") + fail(model.TaskTypeCacheEvict, "preview-other-operation") + + rr := fixture.request(http.MethodGet, "/api/v1/tasks/acknowledge/preview?type=storage_store", nil) + if rr.Code != http.StatusOK { + t.Fatalf("preview status = %d, want %d, body=%s", rr.Code, http.StatusOK, rr.Body.String()) + } + var preview struct { + Count int `json:"count"` + AsOf string `json:"as_of"` + } + decodeJSON(t, rr, &preview) + if preview.Count != 1 || preview.AsOf == "" { + t.Fatalf("preview = %#v, want one dismissable failure and a cutoff", preview) + } + + // A failure recorded after the operator looked must survive the dismissal. + later := fail(model.TaskTypeStorageStore, "preview-unseen") + asOf, err := time.Parse(time.RFC3339Nano, preview.AsOf) + if err != nil { + t.Fatalf("parse preview cutoff: %v", err) + } + if _, err := fixture.db.NewUpdate().Model((*model.Task)(nil)). + Set("finished_at = ?", asOf.Add(time.Minute)).Where("id = ?", later.ID).Exec(t.Context()); err != nil { + t.Fatalf("record a later failure: %v", err) + } + + rr = fixture.request(http.MethodPost, "/api/v1/tasks/acknowledge", + strings.NewReader(`{"type":"storage_store","failed_before":"`+preview.AsOf+`"}`)) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusOK, rr.Body.String()) + } + var body struct { + Acknowledged int `json:"acknowledged"` + } + decodeJSON(t, rr, &body) + if body.Acknowledged != preview.Count { + t.Fatalf("acknowledged = %d, want the previewed %d", body.Acknowledged, preview.Count) + } + dismissed, err := fixture.repos.Tasks.GetByID(t.Context(), seen.ID) + if err != nil || dismissed == nil || dismissed.AcknowledgedAt == nil { + t.Fatalf("previewed failure = %#v, err=%v, want it dismissed", dismissed, err) + } + kept, err := fixture.repos.Tasks.GetByID(t.Context(), later.ID) + if err != nil || kept == nil || kept.AcknowledgedAt != nil { + t.Fatalf("failure recorded after the preview = %#v, err=%v, want it still visible", kept, err) + } + + rr = fixture.request(http.MethodGet, "/api/v1/tasks/acknowledge/preview?type=not-an-operation", nil) + if rr.Code != http.StatusBadRequest { + t.Fatalf("unknown preview type status = %d, want %d", rr.Code, http.StatusBadRequest) + } +} + +// A body that does not decode cleanly is refused rather than partially applied: +// a mistyped field would otherwise dismiss every operation instead of the one +// the operator selected. +func TestAPITaskBulkAcknowledgeRefusesUnusableRequests(t *testing.T) { + fixture := newAdminTaskFixture(t) + now := time.Now() + row := fixture.enqueue(t, model.TaskTypeStorageStore, "strict-decode", now, "storage_copy", "strict-decode") + fixture.transition(t, row.ID, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, + FailureReason: new("store_not_started"), LastError: new("provider unavailable"), + }) + + bodies := map[string]string{ + "misspelled field": `{"typ":"storage_store"}`, + "wrong value type": `{"type":123}`, + "trailing content": `{"type":"storage_store"}{"type":"cache_evict"}`, + "oversized body": `{"type":"` + strings.Repeat("x", taskAcknowledgeMaxBodyBytes) + `"}`, + } + for name, body := range bodies { + t.Run(name, func(t *testing.T) { + rr := fixture.request(http.MethodPost, "/api/v1/tasks/acknowledge", strings.NewReader(body)) + if rr.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusBadRequest, rr.Body.String()) + } + }) + } + kept, err := fixture.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || kept == nil || kept.AcknowledgedAt != nil { + t.Fatalf("failure = %#v, err=%v, want it untouched by every refused request", kept, err) + } +} + +// Scripts and proxies may stream a request without declaring its length. An +// empty body still selects every failure recorded before now. +func TestAPITaskBulkAcknowledgeAcceptsAnEmptyBodyOfUnknownLength(t *testing.T) { + fixture := newAdminTaskFixture(t) + row := fixture.enqueue(t, model.TaskTypeStorageStore, "streamed-empty", time.Now(), "storage_copy", "streamed-empty") + fixture.transition(t, row.ID, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, + FailureReason: new("store_not_started"), LastError: new("provider unavailable"), + }) + + // httptest reports a reader it cannot measure as ContentLength -1, which is + // how a chunked request arrives. + rr := fixture.request(http.MethodPost, "/api/v1/tasks/acknowledge", io.MultiReader()) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rr.Code, http.StatusOK, rr.Body.String()) + } + var body struct { + Acknowledged int `json:"acknowledged"` + } + decodeJSON(t, rr, &body) + if body.Acknowledged != 1 { + t.Fatalf("acknowledged = %d, want 1", body.Acknowledged) + } +} + func newAdminTaskFixture(t *testing.T) *adminTaskFixture { t.Helper() db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) service := newAdminTestTaskService(t, repos) server := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()).WithTaskService(service) - return &adminTaskFixture{t: t, repos: repos, service: service, server: server} + return &adminTaskFixture{t: t, db: db, repos: repos, service: service, server: server} } func (f *adminTaskFixture) enqueue(t *testing.T, taskType model.TaskType, key string, availableAt time.Time, subjectType, subjectKey string) *model.Task { @@ -386,21 +563,20 @@ func (f *adminTaskFixture) transition(t *testing.T, id int64, transition reposit } } -func (f *adminTaskFixture) request(method, path string, body *strings.Reader) *httptest.ResponseRecorder { +func (f *adminTaskFixture) request(method, path string, body io.Reader) *httptest.ResponseRecorder { f.t.Helper() mux := http.NewServeMux() mux.HandleFunc("GET /api/v1/tasks", f.server.handleAPITasks) mux.HandleFunc("GET /api/v1/tasks/stats", f.server.handleAPITaskStats) mux.HandleFunc("POST /api/v1/tasks/{id}/retry", f.server.handleAPITaskRetry) mux.HandleFunc("POST /api/v1/tasks/{id}/acknowledge", f.server.handleAPITaskAcknowledge) - var requestBody *strings.Reader - if body != nil { - requestBody = body - } else { - requestBody = strings.NewReader("") + mux.HandleFunc("GET /api/v1/tasks/acknowledge/preview", f.server.handleAPITaskAcknowledgePreview) + mux.HandleFunc("POST /api/v1/tasks/acknowledge", f.server.handleAPITaskAcknowledgeMatching) + if body == nil { + body = strings.NewReader("") } rr := httptest.NewRecorder() - mux.ServeHTTP(rr, httptest.NewRequest(method, path, requestBody)) + mux.ServeHTTP(rr, httptest.NewRequest(method, path, body)) return rr } diff --git a/internal/admin/server.go b/internal/admin/server.go index b1823f4..8a16ced 100644 --- a/internal/admin/server.go +++ b/internal/admin/server.go @@ -271,6 +271,8 @@ func (s *Server) Serve(ctx context.Context, listener net.Listener) error { mux.HandleFunc("GET /api/v1/tasks/stats", s.handleAPITaskStats) mux.HandleFunc("POST /api/v1/tasks/{id}/retry", s.handleAPITaskRetry) mux.HandleFunc("POST /api/v1/tasks/{id}/acknowledge", s.handleAPITaskAcknowledge) + mux.HandleFunc("GET /api/v1/tasks/acknowledge/preview", s.handleAPITaskAcknowledgePreview) + mux.HandleFunc("POST /api/v1/tasks/acknowledge", s.handleAPITaskAcknowledgeMatching) mux.HandleFunc("GET /api/v1/system/info", s.handleAPISystemInfo) mux.HandleFunc("GET /api/v1/workers", s.handleAPIWorkers) mux.HandleFunc("GET /api/v1/cache/stats", s.handleAPICacheStats) diff --git a/internal/app/runtime.go b/internal/app/runtime.go index 4ebc668..719ef8b 100644 --- a/internal/app/runtime.go +++ b/internal/app/runtime.go @@ -302,9 +302,6 @@ func publishUploadTaskSettlement(events admin.EventPublisher, taskRow *model.Tas } if taskRow.SubjectKey != nil { payload["subject_key"] = *taskRow.SubjectKey - if taskRow.SubjectType != nil && *taskRow.SubjectType == "object_version" { - payload["version_id"] = *taskRow.SubjectKey - } } events.Publish("upload_state_changed", payload) } diff --git a/internal/backend/multipart.go b/internal/backend/multipart.go index ac2d5e5..70451a8 100644 --- a/internal/backend/multipart.go +++ b/internal/backend/multipart.go @@ -345,7 +345,7 @@ func (b *SynapseBackend) CompleteMultipartUpload(ctx context.Context, input *s3. if cacheCommitted { b.releaseContentCacheIfUnreferenced(ctx, bucketName, content.ID, "orphaned content cache file after multipart complete tx failure") } - return s3response.CompleteMultipartUploadResult{}, "", err + return s3response.CompleteMultipartUploadResult{}, "", contentWriteError(err) } completed = true diff --git a/internal/backend/object.go b/internal/backend/object.go index 80ffa4d..153da0f 100644 --- a/internal/backend/object.go +++ b/internal/backend/object.go @@ -122,7 +122,7 @@ func (b *SynapseBackend) PutObject(ctx context.Context, input s3response.PutObje if cacheCommitted { b.releaseContentCacheIfUnreferenced(ctx, bucketName, content.ID, "orphaned content cache file after put tx failure") } - return s3response.PutObjectOutput{}, err + return s3response.PutObjectOutput{}, contentWriteError(err) } b.logger.Info("object stored", "bucket", bucketName, "key", keyName, "size", cacheInfo.Size, "versionID", versionID) @@ -791,7 +791,7 @@ func (b *SynapseBackend) copyObjectVersion(ctx context.Context, input copyObject if cacheCommitted { b.releaseContentCacheIfUnreferenced(ctx, input.DestinationBucket.Name, content.ID, "orphaned content cache file after copy tx failure") } - return copyObjectVersionResult{}, err + return copyObjectVersionResult{}, contentWriteError(err) } return copyObjectVersionResult{ @@ -1030,6 +1030,18 @@ func (b *SynapseBackend) requireWritableBucket(ctx context.Context, name string) return bucket, nil } +// contentWriteError asks the client to retry a write whose bytes are still +// being removed after their last version was deleted; the retry then stores +// them as new content. +func contentWriteError(err error) error { + if !errors.Is(err, repository.ErrContentCleanupInProgress) { + return err + } + apiErr := s3err.GetAPIError(s3err.ErrSlowDown) + apiErr.Description = "The same data is still being removed after an earlier delete. Please retry shortly." + return apiErr +} + func stringOrDefault(s *string, def string) string { if s != nil && *s != "" { return *s diff --git a/internal/backend/object_test.go b/internal/backend/object_test.go index 5352bf1..f7152a2 100644 --- a/internal/backend/object_test.go +++ b/internal/backend/object_test.go @@ -994,8 +994,12 @@ func TestPutObjectIdenticalStoredContentQueuesAfterUploadEviction(t *testing.T) t.Fatal("expected evict task for reused stored content") } // Eviction frees one cache file, and that file belongs to the content, so - // the task names the content rather than either version of it. - task := &page.Tasks[0] + // the task names the content rather than either version of it. A listing + // leaves inputs out, so read the task itself. + task, err := tb.repos.Tasks.GetByID(ctx, page.Tasks[0].ID) + if err != nil || task == nil { + t.Fatalf("load evict task = %#v, err=%v", task, err) + } contentID := *firstObj.ContentID if task.SubjectKey == nil || *task.SubjectKey != strconv.FormatInt(contentID, 10) { t.Fatalf("evict task content = %v, want %d", task.SubjectKey, contentID) @@ -1088,6 +1092,69 @@ func TestPutObjectIdenticalStoredContentReusesChainStorage(t *testing.T) { } } +// TestPutObjectDuringContentCleanupAsksClientToRetry checks that writing bytes +// whose last version was just deleted asks the client to retry without leaving +// a version or cache file behind, and that once cleanup finishes the same bytes +// are stored as new content. +func TestPutObjectDuringContentCleanupAsksClientToRetry(t *testing.T) { + tb := newTestBackend(t) + ctx := context.Background() + bucket := seedActiveBucket(t, tb, "cleanup-reupload-bucket") + first := putValidTestObjectOutput(t, tb, bucket.Name, "file.txt", "cleanup data") + version, err := tb.repos.Objects.GetVersionByID(ctx, first.VersionID) + if err != nil || version == nil || version.ContentID == nil { + t.Fatalf("first version = %#v, err=%v", version, err) + } + contentID := *version.ContentID + // The upload fails before anything is stored remotely, so the delete is allowed. + plan, err := tb.repos.Tasks.GetByIdentity(ctx, model.TaskTypeUploadPlan, storagepipeline.UploadPlanKey(contentID)) + if err != nil || plan == nil { + t.Fatalf("upload plan = %#v, err=%v", plan, err) + } + claimed, err := tb.repos.Tasks.ClaimNext(ctx, time.Minute) + if err != nil || claimed == nil || claimed.ID != plan.ID { + t.Fatalf("claimed upload plan = %#v, err=%v", claimed, err) + } + if err := tb.repos.Tasks.Settle(ctx, claimed.ID, claimed.ClaimGeneration, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, + FailureReason: new("upload_failed"), LastError: new("upload failed"), + }); err != nil { + t.Fatalf("fail upload plan: %v", err) + } + if _, err := tb.backend.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucket.Name), Key: aws.String("file.txt"), VersionId: aws.String(first.VersionID), + }); err != nil { + t.Fatalf("DeleteObject(version): %v", err) + } + content, err := tb.repos.Contents.GetByID(ctx, contentID) + if err != nil || content == nil || content.CleanupTaskID == nil { + t.Fatalf("content after last delete = %#v, err=%v", content, err) + } + + contentType := "text/plain" + _, err = tb.backend.PutObject(ctx, s3response.PutObjectInput{ + Bucket: &bucket.Name, Key: new("again.txt"), Body: strings.NewReader(validTestObjectBody("cleanup data")), ContentType: &contentType, + }) + requireAPIErrorCode(t, err, s3err.GetAPIError(s3err.ErrSlowDown)) + if current, err := tb.repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "again.txt"); current != nil || (err != nil && !errors.Is(err, repository.ErrNotFound)) { + t.Fatalf("version after refused put = %#v, err=%v", current, err) + } + if tb.cache.Exists(ctx, bucket.Name, model.ContentCacheKey(contentID)) { + t.Fatal("refused put left a cache file under the content being removed") + } + + if err := tb.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { + return txRepos.StorageCleanup.FinalizeContent(ctx, contentID, content.CleanupGeneration, *content.CleanupTaskID) + }); err != nil { + t.Fatalf("FinalizeContent: %v", err) + } + again := putValidTestObjectOutput(t, tb, bucket.Name, "again.txt", "cleanup data") + stored, err := tb.repos.Objects.GetVersionByID(ctx, again.VersionID) + if err != nil || stored == nil || stored.ContentID == nil || *stored.ContentID == contentID { + t.Fatalf("version after cleanup = %#v, err=%v, want new content", stored, err) + } +} + func TestPutObjectIdenticalReplicatingContentReusesPrimaryCommittedUpload(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() diff --git a/internal/config/config.go b/internal/config/config.go index f0a9e1b..4f6a48a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,6 +3,7 @@ package config import ( "errors" "fmt" + "maps" "net" "net/netip" "net/url" @@ -142,9 +143,7 @@ func DefaultFilecoinRPCURL(network string) (string, bool) { // DefaultFilecoinRPCURLs returns a copy of the built-in Filecoin RPC URL map. func DefaultFilecoinRPCURLs() map[string]string { out := make(map[string]string, len(defaultFilecoinRPCURLs)) - for network, rpcURL := range defaultFilecoinRPCURLs { - out[network] = rpcURL - } + maps.Copy(out, defaultFilecoinRPCURLs) return out } @@ -178,7 +177,7 @@ func defaultConfig() *Config { }, Database: DatabaseConfig{ Driver: "sqlite", - MaxOpenConns: 4, + MaxOpenConns: 32, MaxIdleConns: 2, }, Cache: CacheConfig{ @@ -296,7 +295,7 @@ func loadWithOptions(path string, includeEnv, applyRuntimeDefaults bool) (*Confi if includeEnv { // Overlay environment variables: SYNAPS3_SERVER_PORT → server.port - if err := k.Load(env.ProviderWithValue("SYNAPS3_", ".", func(s, value string) (string, interface{}) { + if err := k.Load(env.ProviderWithValue("SYNAPS3_", ".", func(s, value string) (string, any) { if field, ok := EnvFieldForName(s); ok { if field == "admin.trusted_proxies" { return field, splitEnvList(value) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index f1690d6..55b4815 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -907,8 +907,8 @@ func TestDefaultConfig_DatabasePool(t *testing.T) { if err != nil { t.Fatalf("DefaultConfig() failed: %v", err) } - if cfg.Database.MaxOpenConns != 4 { - t.Errorf("Database.MaxOpenConns = %d, want 4", cfg.Database.MaxOpenConns) + if cfg.Database.MaxOpenConns != 32 { + t.Errorf("Database.MaxOpenConns = %d, want 32", cfg.Database.MaxOpenConns) } if cfg.Database.MaxIdleConns != 2 { t.Errorf("Database.MaxIdleConns = %d, want 2", cfg.Database.MaxIdleConns) diff --git a/internal/config/persistence_test.go b/internal/config/persistence_test.go index cf663a9..9180959 100644 --- a/internal/config/persistence_test.go +++ b/internal/config/persistence_test.go @@ -253,7 +253,7 @@ func TestInitAppDataDir_WritesCommentedReferenceConfig(t *testing.T) { "[database]", "driver = \"sqlite\"", "dsn = ", - "# max_open_conns = 4", + "# max_open_conns = 32", "[cache]", "dir = ", "# max_size_gb = 100", @@ -423,7 +423,7 @@ func TestSaveForSettingsGeneratedTOMLCommentsAndPreservesAbsentManualFields(t *t for _, want := range []string{ "# Database connection string.", "# dsn = \"\"", - "# max_open_conns = 4", + "# max_open_conns = 32", "driver = \"sqlite\"", "dir = \"/var/lib/synaps3/cache\"", } { @@ -456,7 +456,7 @@ func assertConfigContains(t *testing.T, text, want string) { func assertConfigLacksEnabledLine(t *testing.T, text, want string) { t.Helper() - for _, line := range strings.Split(text, "\n") { + for line := range strings.SplitSeq(text, "\n") { if line == want { t.Fatalf("generated config contains enabled line %q:\n%s", want, text) } @@ -465,7 +465,7 @@ func assertConfigLacksEnabledLine(t *testing.T, text, want string) { func assertConfigLacksEnabledPrefix(t *testing.T, text, prefix string) { t.Helper() - for _, line := range strings.Split(text, "\n") { + for line := range strings.SplitSeq(text, "\n") { if strings.HasPrefix(line, prefix) { t.Fatalf("generated config contains enabled line with prefix %q:\n%s", prefix, text) } diff --git a/internal/db/db_test.go b/internal/db/db_test.go index cee1d7e..3f1f248 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -457,12 +457,14 @@ func TestRunMigrations_StorageProvenanceConstraints(t *testing.T) { mustReject(t, db, "expected committed copy without piece identity to fail", `UPDATE storage_copies SET status = 'committed' WHERE content_id = 1 AND copy_index = 1`) mustExec(t, db, `INSERT INTO storage_commit_attempts (attempt_id, content_id, storage_data_set_id, status, extra_data_hex, transaction_id, confirmed_transaction_id, attempted_at, resolved_at, created_at, updated_at) VALUES ('attempt-2', 1, 2, 'confirmed', 'abcd', 'tx-2', 'tx-2', current_timestamp, current_timestamp, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) mustExec(t, db, `UPDATE storage_copies SET status = 'committed', piece_id = '0', retrieval_url = 'https://provider.example/zero-piece', confirmed_attempt_id = 'attempt-2', confirmed_attempt_status = 'confirmed' WHERE content_id = 1 AND copy_index = 1`) - mustExec(t, db, `INSERT INTO storage_cleanup_copies (content_id, bucket_id, copy_index, provider_id, storage_data_set_id, piece_id, piece_cid, created_at, updated_at) VALUES (1, 1, 0, '101', 1, '2001', 'bafk2bzacefake', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - mustReject(t, db, "expected duplicate physical cleanup identity to fail", `INSERT INTO storage_cleanup_copies (content_id, bucket_id, copy_index, provider_id, storage_data_set_id, piece_id, piece_cid, created_at, updated_at) VALUES (1, 1, 0, '101', 1, '2001', 'bafk2bzaceduplicate', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - // Cleanup evidence requires both the data set and the piece it removed; - // each NOT NULL is the invariant under test, so supply every other column. - mustRejectRequiredColumn(t, db, "expected cleanup evidence without storage_data_set_id to fail", `INSERT INTO storage_cleanup_copies (content_id, bucket_id, copy_index, provider_id, piece_id, piece_cid, created_at, updated_at) VALUES (1, 1, 0, '101', '2002', 'bafk2bzacemissingdataset', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - mustRejectRequiredColumn(t, db, "expected cleanup evidence without piece_id to fail", `INSERT INTO storage_cleanup_copies (content_id, bucket_id, copy_index, provider_id, storage_data_set_id, piece_cid, created_at, updated_at) VALUES (1, 1, 0, '101', 1, 'bafk2bzacemissingpiece', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO storage_cleanup_copies (content_id, bucket_id, copy_index, provider_id, storage_data_set_id, piece_id, piece_cid, checksum, created_at, updated_at) VALUES (1, 1, 0, '101', 1, '2001', 'bafk2bzacefake', printf('%064x', 1), CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustReject(t, db, "expected duplicate physical cleanup identity to fail", `INSERT INTO storage_cleanup_copies (content_id, bucket_id, copy_index, provider_id, storage_data_set_id, piece_id, piece_cid, checksum, created_at, updated_at) VALUES (1, 1, 0, '101', 1, '2001', 'bafk2bzaceduplicate', printf('%064x', 1), CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + // Cleanup evidence requires the data set, the piece it removed, and the + // bytes it names; each NOT NULL is the invariant under test, so supply + // every other column. + mustRejectRequiredColumn(t, db, "expected cleanup evidence without storage_data_set_id to fail", `INSERT INTO storage_cleanup_copies (content_id, bucket_id, copy_index, provider_id, piece_id, piece_cid, checksum, created_at, updated_at) VALUES (1, 1, 0, '101', '2002', 'bafk2bzacemissingdataset', printf('%064x', 1), CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustRejectRequiredColumn(t, db, "expected cleanup evidence without piece_id to fail", `INSERT INTO storage_cleanup_copies (content_id, bucket_id, copy_index, provider_id, storage_data_set_id, piece_cid, checksum, created_at, updated_at) VALUES (1, 1, 0, '101', 1, 'bafk2bzacemissingpiece', printf('%064x', 1), CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustRejectRequiredColumn(t, db, "expected cleanup evidence without checksum to fail", `INSERT INTO storage_cleanup_copies (content_id, bucket_id, copy_index, provider_id, storage_data_set_id, piece_id, piece_cid, created_at, updated_at) VALUES (1, 1, 0, '101', 1, '2003', 'bafk2bzacemissingchecksum', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) } func TestRunMigrations_TaskAndMultipartConstraints(t *testing.T) { diff --git a/internal/db/migrations/2026090101_initial_schema.go b/internal/db/migrations/2026090101_initial_schema.go index 316c0f7..e6f196a 100644 --- a/internal/db/migrations/2026090101_initial_schema.go +++ b/internal/db/migrations/2026090101_initial_schema.go @@ -8,6 +8,7 @@ import ( "time" "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" ) func init() { @@ -149,3 +150,973 @@ func createTaskSchema(ctx context.Context, db bun.IDB) error { initialIndexSpec{name: "idx_tasks_subject", table: "tasks", columns: []string{"subject_type", "subject_key", "id"}}, ) } + +type s3Account2026090101 struct { + bun.BaseModel `bun:"table:s3_accounts"` + + AccessKey string `bun:"type:text,pk"` + SecretKey string `bun:"type:text,notnull"` + Role string `bun:"type:text,notnull"` + IsRoot bool `bun:",notnull,default:false"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type bucket2026090101 struct { + bun.BaseModel `bun:"table:buckets"` + + ID int64 `bun:",pk,autoincrement,identity"` + Name string `bun:"type:text,notnull,unique"` + ACL []byte + OwnerAccessKey *string `bun:"type:text"` + DefaultCopies int `bun:"type:integer,notnull"` + MinimumDurableCopies int `bun:"type:integer,notnull"` + DurabilityGeneration int64 `bun:",notnull,default:0"` + DurabilityTaskID *int64 + Status string `bun:"type:text,notnull,default:'provisioning'"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +// bucketReplicaSlot2026090101 gives a replica slot a row of its own. copy_index +// used to be a bare integer repeated across five tables and bounded only by a +// range check; as a table it becomes a foreign key target, and a slot keeps its +// history because retired generations still reference it. +type bucketReplicaSlot2026090101 struct { + bun.BaseModel `bun:"table:bucket_replica_slots"` + + ID int64 `bun:",pk,autoincrement,identity"` + BucketID int64 `bun:",notnull"` + CopyIndex int `bun:"type:integer,notnull"` + Status string `bun:"type:text,notnull,default:'active'"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type object2026090101 struct { + bun.BaseModel `bun:"table:objects"` + + ID int64 `bun:",pk,autoincrement,identity"` + BucketID int64 `bun:",notnull"` + Key string `bun:"type:text,notnull"` + // CurrentVersionID points at the version S3 reads serve. One column can hold + // one value, so "at most one current version" is structural here rather than + // a partial unique index over every version of the object. + CurrentVersionID *string `bun:"type:text"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type multipartUpload2026090101 struct { + bun.BaseModel `bun:"table:multipart_uploads"` + + UploadID string `bun:"type:text,pk"` + BucketID int64 `bun:",notnull"` + Key string `bun:"type:text,notnull"` + ContentType string `bun:"type:text,notnull,default:'application/octet-stream'"` + Metadata json.RawMessage `bun:"type:jsonb,notnull,default:'{}'"` + Status string `bun:"type:text,notnull,default:'initiated'"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type multipartPart2026090101 struct { + bun.BaseModel `bun:"table:multipart_parts"` + + ID int64 `bun:",pk,autoincrement,identity"` + UploadID string `bun:"type:text,notnull"` + PartNumber int `bun:"type:integer,notnull"` + Size int64 `bun:",notnull"` + ETag string `bun:"e_tag,type:text,notnull"` + Checksum *string `bun:"type:text"` + CreatedAt time.Time `bun:",notnull"` +} + +func createCoreRootSchema(ctx context.Context, db bun.IDB) error { + tables := []initialTableSpec{ + { + name: "s3_accounts", + model: (*s3Account2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_s3_accounts_identity CHECK (access_key <> '' AND secret_key <> '')", + "CONSTRAINT chk_s3_accounts_role CHECK (role IN ('admin', 'user', 'userplus'))", + }, + }, + { + name: "buckets", + model: (*bucket2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_buckets_identity CHECK (name <> '' AND (owner_access_key IS NULL OR owner_access_key <> ''))", + "CONSTRAINT chk_buckets_status CHECK (status IN ('provisioning', 'ready'))", + // The durability policy is materialised from configuration when + // the bucket is created, so "how many replicas does this bucket + // want" never depends on reading config at query time. + "CONSTRAINT chk_buckets_default_copies CHECK (default_copies BETWEEN 1 AND 8)", + "CONSTRAINT chk_buckets_minimum_durable_copies CHECK (minimum_durable_copies BETWEEN 1 AND 8)", + "CONSTRAINT chk_buckets_explicit_copy_policy CHECK (minimum_durable_copies <= default_copies)", + "CONSTRAINT chk_buckets_durability_generation CHECK (durability_generation >= 0)", + }, + foreignKeys: []string{ + "(owner_access_key) REFERENCES s3_accounts (access_key) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(durability_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }, + { + name: "bucket_replica_slots", + model: (*bucketReplicaSlot2026090101)(nil), + constraints: []string{ + "CONSTRAINT uq_bucket_replica_slots_identity UNIQUE (bucket_id, copy_index)", + "CONSTRAINT chk_bucket_replica_slots_copy_index CHECK (copy_index BETWEEN 0 AND 7)", + // A shrunk slot is decommissioned, never deleted: retired data + // set generations still point at it. + "CONSTRAINT chk_bucket_replica_slots_status CHECK (status IN ('active', 'decommissioned'))", + }, + foreignKeys: []string{ + "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }, + { + name: "objects", + model: (*object2026090101)(nil), + constraints: []string{ + "CONSTRAINT uq_objects_identity UNIQUE (id, bucket_id, key)", + "CONSTRAINT chk_objects_identity CHECK (key <> '')", + }, + foreignKeys: []string{ + "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + forwardForeignKeys: []initialForwardForeignKey{objectCurrentVersionForeignKey2026090101()}, + }, + { + name: "multipart_uploads", + model: (*multipartUpload2026090101)(nil), + jsonColumns: initialJSONColumns("multipart_uploads"), + constraints: []string{ + "CONSTRAINT uq_multipart_uploads_identity UNIQUE (upload_id, bucket_id, key)", + "CONSTRAINT chk_multipart_uploads_identity CHECK (key <> '' AND upload_id <> '')", + "CONSTRAINT chk_multipart_uploads_status CHECK (status IN ('initiated', 'completing', 'completed', 'aborted'))", + }, + foreignKeys: []string{ + "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }, + { + name: "multipart_parts", + model: (*multipartPart2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_multipart_parts_identity CHECK (upload_id <> '' AND e_tag <> '' AND (checksum IS NULL OR checksum <> ''))", + "CONSTRAINT chk_multipart_parts_part_number CHECK (part_number BETWEEN 1 AND 10000)", + "CONSTRAINT chk_multipart_parts_size CHECK (size >= 0)", + }, + foreignKeys: []string{ + "(upload_id) REFERENCES multipart_uploads (upload_id) ON UPDATE RESTRICT ON DELETE CASCADE", + }, + }, + } + for _, table := range tables { + if err := createInitialTable(ctx, db, table); err != nil { + return err + } + } + indexes := []initialIndexSpec{ + {name: "idx_objects_bucket_key", table: "objects", columns: []string{"bucket_id", "key"}, unique: true}, + {name: "idx_objects_current_version", table: "objects", columns: []string{"current_version_id"}, where: "current_version_id IS NOT NULL"}, + {name: "idx_s3_accounts_single_root", table: "s3_accounts", columns: []string{"is_root"}, where: "is_root = TRUE", unique: true}, + {name: "idx_buckets_owner_access_key", table: "buckets", columns: []string{"owner_access_key"}}, + {name: "idx_buckets_durability_task", table: "buckets", columns: []string{"durability_task_id"}, where: "durability_task_id IS NOT NULL", unique: true}, + {name: "idx_multipart_parts_upload_part", table: "multipart_parts", columns: []string{"upload_id", "part_number"}, unique: true}, + } + if db.Dialect().Name() != dialect.PG { + indexes = append(indexes, initialIndexSpec{name: "idx_multipart_uploads_bucket_status_key_upload", table: "multipart_uploads", columns: []string{"bucket_id", "status", "key", "upload_id"}}) + } + return createInitialIndexes(ctx, db, indexes...) +} + +// objectVersion2026090101 carries S3 naming semantics only. Pipeline state is +// a function of the copy rows and stays a query; cache residency belongs to +// object_cache; the bytes themselves are storage_contents. A NULL content_id +// is exactly a delete marker. +type objectVersion2026090101 struct { + bun.BaseModel `bun:"table:object_versions"` + + VersionID string `bun:"type:text,pk"` + ObjectID int64 `bun:",notnull"` + BucketID int64 `bun:",notnull"` + Key string `bun:"type:text,notnull"` + ContentID *int64 + Size int64 `bun:",notnull"` + ETag string `bun:"e_tag,type:text,notnull"` + ContentType string `bun:"type:text,notnull,default:'application/octet-stream'"` + Metadata json.RawMessage `bun:"type:jsonb,notnull,default:'{}'"` + MultipartUploadID *string `bun:"type:text"` + IsDeleteMarker bool `bun:",notnull,default:false"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +// objectCache2026090101 records local cache residency per content, not per +// version, so identical bytes written under several keys share one file. The +// cache key is derived from content_id and therefore not stored. +type objectCache2026090101 struct { + bun.BaseModel `bun:"table:object_cache"` + + ContentID int64 `bun:",pk"` + InCache bool `bun:",notnull"` + CacheAccessedAt *time.Time + CachePresenceGeneration int64 `bun:",notnull,default:0"` + CacheOperationGeneration int64 `bun:",notnull,default:0"` + CacheActiveTaskID *int64 + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +// objectDeletion2026090101 is an append-only tombstone. Cache cleanup is not +// tracked here: with content-keyed residency the trigger is the content's +// reference count reaching zero, not the removal of one version. +type objectDeletion2026090101 struct { + bun.BaseModel `bun:"table:object_deletions"` + + ID int64 `bun:",pk,autoincrement,identity"` + BucketID int64 `bun:",notnull"` + ObjectID int64 `bun:",notnull"` + Key string `bun:"type:text,notnull"` + VersionID string `bun:"type:text,notnull,unique"` + ContentID *int64 + Size int64 `bun:",notnull"` + DeletedAt time.Time `bun:",notnull"` +} + +func createObjectLifecycleSchema(ctx context.Context, db bun.IDB) error { + if err := createInitialTable(ctx, db, initialTableSpec{ + name: "object_versions", + model: (*objectVersion2026090101)(nil), + jsonColumns: initialJSONColumns("object_versions"), + constraints: []string{ + "CONSTRAINT chk_object_versions_identity CHECK (version_id <> '' AND key <> '' AND (multipart_upload_id IS NULL OR multipart_upload_id <> ''))", + "CONSTRAINT chk_object_versions_size CHECK (size >= 0)", + // Candidate key for the objects.current_version_id pointer, which + // names both the version and the object it must belong to. + "CONSTRAINT uq_object_versions_object_identity UNIQUE (version_id, object_id)", + // A delete marker is exactly a version with no content. + "CONSTRAINT chk_object_versions_delete_marker_shape CHECK ((is_delete_marker = TRUE AND content_id IS NULL AND size = 0 AND e_tag = '' AND content_type = '') OR (is_delete_marker = FALSE AND content_id IS NOT NULL AND e_tag <> ''))", + "CONSTRAINT fk_object_versions_object FOREIGN KEY (object_id, bucket_id, key) REFERENCES objects (id, bucket_id, key) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + foreignKeys: []string{ + "(multipart_upload_id, bucket_id, key) REFERENCES multipart_uploads (upload_id, bucket_id, key) ON UPDATE RESTRICT ON DELETE RESTRICT", + // size repeats the content size so listings need no join; the + // composite key makes that repetition unable to drift. + "(content_id, bucket_id, size) REFERENCES storage_contents (id, bucket_id, content_size) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }); err != nil { + return err + } + if err := createInitialTable(ctx, db, initialTableSpec{ + name: "object_cache", + model: (*objectCache2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_object_cache_generation CHECK (cache_presence_generation >= 0 AND cache_operation_generation >= 0)", + }, + foreignKeys: []string{ + "(content_id) REFERENCES storage_contents (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(cache_active_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }); err != nil { + return err + } + if err := createInitialTable(ctx, db, initialTableSpec{ + name: "object_deletions", + model: (*objectDeletion2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_object_deletions_identity CHECK (key <> '' AND version_id <> '')", + "CONSTRAINT chk_object_deletions_size CHECK (size >= 0)", + }, + // The tombstone keeps content_id as a value: the content row is deleted + // once its cleanup finishes. + foreignKeys: []string{ + "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }); err != nil { + return err + } + indexes := []initialIndexSpec{ + {name: "idx_object_versions_object_created", table: "object_versions", columns: []string{"object_id", "created_at DESC", "version_id DESC"}}, + // Content reference count and foreign key coverage in one index. + {name: "idx_object_versions_content", table: "object_versions", columns: []string{"content_id"}}, + {name: "idx_object_versions_multipart_upload", table: "object_versions", columns: []string{"multipart_upload_id"}}, + {name: "idx_object_cache_lru", table: "object_cache", columns: []string{"cache_accessed_at", "content_id"}, where: "in_cache = TRUE"}, + {name: "idx_object_cache_active_task", table: "object_cache", columns: []string{"cache_active_task_id"}, where: "cache_active_task_id IS NOT NULL", unique: true}, + {name: "idx_object_deletions_bucket_key_deleted", table: "object_deletions", columns: []string{"bucket_id", "key", "deleted_at"}}, + {name: "idx_object_deletions_content", table: "object_deletions", columns: []string{"content_id"}}, + {name: "idx_object_deletions_bucket_deleted", table: "object_deletions", columns: []string{"bucket_id", "deleted_at DESC", "id DESC"}}, + } + if db.Dialect().Name() != dialect.PG { + indexes = append(indexes, initialIndexSpec{name: "idx_object_versions_bucket_key_created", table: "object_versions", columns: []string{"bucket_id", "key", "created_at DESC", "version_id DESC"}}) + } + if err := createInitialIndexes(ctx, db, indexes...); err != nil { + return err + } + // objects was created before object_versions existed, so PostgreSQL takes + // the pointer constraint here. + return addForwardForeignKey(ctx, db, "objects", objectCurrentVersionForeignKey2026090101()) +} + +// objectCurrentVersionForeignKey2026090101 keeps an object from pointing at a +// version of some other object, and keeps the pointed-at version from being +// deleted before the pointer moves. +func objectCurrentVersionForeignKey2026090101() initialForwardForeignKey { + return initialForwardForeignKey{ + name: "fk_objects_current_version", + definition: "(current_version_id, id) REFERENCES object_versions (version_id, object_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + } +} + +func createPostgresPrefixIndexes(ctx context.Context, db bun.IDB) error { + if db.Dialect().Name() != dialect.PG { + return nil + } + return createInitialIndexes(ctx, db, + // Listing current objects orders by a C-collated key, so the objects + // unique index needs a C-collated companion to drive it. + initialIndexSpec{name: "idx_objects_bucket_key_c", table: "objects", columns: []string{"bucket_id", `(key COLLATE "C")`}}, + initialIndexSpec{name: "idx_object_versions_bucket_key_created", table: "object_versions", columns: []string{"bucket_id", `(key COLLATE "C")`, "created_at DESC", "version_id DESC"}}, + initialIndexSpec{name: "idx_multipart_uploads_bucket_status_key_upload", table: "multipart_uploads", columns: []string{"bucket_id", "status", `(key COLLATE "C")`, "upload_id"}}, + ) +} + +// storageContent2026090101 is the identity of one bucket-scoped byte payload. +// Durability, pipeline state and ingress progress are deliberately absent: the +// first two are functions of the copy rows and stay queries, while progress +// belongs to the concrete ingress transfer that produced it. +type storageContent2026090101 struct { + bun.BaseModel `bun:"table:storage_contents"` + + ID int64 `bun:",pk,autoincrement,identity"` + BucketID int64 `bun:",notnull"` + Checksum string `bun:"type:text,notnull"` + ContentSize int64 `bun:",notnull"` + PieceCID *string `bun:"type:text"` + RequestedCopies int `bun:"type:integer,notnull"` + ErrorMessage *string `bun:"type:text"` + AcceptedAt *time.Time + CleanupGeneration int64 `bun:",notnull,default:0"` + CleanupTaskID *int64 + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type storageDataSet2026090101 struct { + bun.BaseModel `bun:"table:storage_data_sets"` + + ID int64 `bun:",pk,autoincrement,identity"` + BucketID int64 `bun:",notnull"` + ProviderID string `bun:"type:text,notnull"` + CopyIndex int `bun:"type:integer,notnull"` + Generation int64 `bun:",notnull,default:1"` + IsCurrent bool `bun:",notnull"` + DataSetID *string `bun:"type:text"` + ClientDataSetID *string `bun:"type:text"` + Status string `bun:"type:text,notnull,default:'pending'"` + CreateTransactionID *string `bun:"type:text"` + CreateStatusURL *string `bun:"type:text"` + CreatedByContentID *int64 + LastUsedContentID *int64 + LastError *string `bun:"type:text"` + EnsureTaskID *int64 + RetirementGeneration int64 `bun:",notnull,default:0"` + RetirementTaskID *int64 + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type storageCopy2026090101 struct { + bun.BaseModel `bun:"table:storage_copies"` + + ID int64 `bun:",pk,autoincrement,identity"` + ContentID int64 `bun:",notnull"` + BucketID int64 `bun:",notnull"` + // ContentSize repeats storage_contents.content_size so the ingress bound + // stays a local CHECK. A composite foreign key keeps the copy from + // drifting away from the content it transfers. + ContentSize int64 `bun:",notnull"` + StorageDataSetID int64 `bun:",notnull"` + CopyIndex int `bun:"type:integer,notnull"` + ProviderID string `bun:"type:text,notnull"` + PieceID *string `bun:"type:text"` + TransferMethod string `bun:"type:text,notnull"` + Status string `bun:"type:text,notnull,default:'pending'"` + RetrievalURL *string `bun:"type:text"` + CommitExtraDataHex *string `bun:"type:text"` + CommitReadyAt *time.Time + // The confirmed commit attempt this copy projects. Its status is repeated so + // a composite foreign key can require the referenced attempt to be + // confirmed, which is what keeps the projection from drifting. + ConfirmedAttemptID *string `bun:"type:text"` + ConfirmedAttemptStatus *string `bun:"type:text"` + IngressBytesTransferred int64 `bun:",notnull,default:0"` + IngressStoreAttempt int `bun:"type:integer,notnull,default:0"` + ProgressUpdatedAt *time.Time + WorkGeneration int64 `bun:",notnull,default:0"` + ActiveTaskID *int64 + LastError *string `bun:"type:text"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type storageCommitAttempt2026090101 struct { + bun.BaseModel `bun:"table:storage_commit_attempts"` + + AttemptID string `bun:"type:text,pk"` + ContentID int64 `bun:",notnull"` + StorageDataSetID int64 `bun:",notnull"` + Status string `bun:"type:text,notnull,default:'reserved'"` + ExtraDataHex *string `bun:"type:text"` + TransactionID *string `bun:"type:text"` + SubmissionJSON *string `bun:"type:text"` + ConfirmedTransactionID *string `bun:"type:text"` + AttentionCode *string `bun:"type:text"` + AttentionAt *time.Time + ReleaseReason *string `bun:"type:text"` + LastError *string `bun:"type:text"` + AttemptedAt *time.Time + ResolvedAt *time.Time + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +// storagePullAttempt2026090101 is the ledger of provider-side copy requests. +// Every source column is NOT NULL, which replaces the all-or-nothing check that +// used to guard five nullable columns on the copy. +type storagePullAttempt2026090101 struct { + bun.BaseModel `bun:"table:storage_pull_attempts"` + + AttemptID string `bun:"type:text,pk"` + ContentID int64 `bun:",notnull"` + StorageDataSetID int64 `bun:",notnull"` + Status string `bun:"type:text,notnull"` + SourceProviderID string `bun:"type:text,notnull"` + SourceDataSetID string `bun:"type:text,notnull"` + SourcePieceID string `bun:"type:text,notnull"` + SourcePieceCID string `bun:"type:text,notnull"` + SourceRetrievalURL string `bun:"type:text,notnull"` + LastError *string `bun:"type:text"` + AttemptedAt time.Time `bun:",notnull"` + ResolvedAt *time.Time + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type storageReplacement2026090101 struct { + bun.BaseModel `bun:"table:storage_replacements"` + + ID int64 `bun:",pk,autoincrement,identity"` + BucketID int64 `bun:",notnull"` + CopyIndex int `bun:"type:integer,notnull"` + SourceDataSetID int64 `bun:",notnull"` + TargetDataSetID int64 `bun:",notnull"` + SelectionMode string `bun:"type:text,notnull"` + RequestedProviderID *string `bun:"type:text"` + ClientRequestID string `bun:"type:text,notnull"` + Status string `bun:"type:text,notnull"` + WaitReason *string `bun:"type:text"` + FailureReason *string `bun:"type:text"` + LastError *string `bun:"type:text"` + ItemsTotal int `bun:"type:integer,notnull,default:0"` + ItemsCopied int `bun:"type:integer,notnull,default:0"` + SeedCursorContentID int64 `bun:",notnull,default:0"` + SeedingComplete bool `bun:",notnull,default:false"` + TaskGeneration int64 `bun:",notnull,default:1"` + TaskID *int64 + SupersededByID *int64 + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +// storageDataSetTermination2026090101 records one data set's end of term. A +// replacement terminates the source it replaces and, once superseded, the +// target it abandoned; a third kind of termination adds a row here instead of +// another repeated column group on storage_replacements. +type storageDataSetTermination2026090101 struct { + bun.BaseModel `bun:"table:storage_data_set_terminations"` + + ID int64 `bun:",pk,autoincrement,identity"` + ReplacementID int64 `bun:",notnull"` + Role string `bun:"type:text,notnull"` + // Exactly one of the data set columns is set, chosen by role. Each carries a + // composite foreign key back to the matching column on the replacement, so a + // row cannot name a data set the replacement never held in that role. + SourceDataSetID *int64 + AbandonedTargetDataSetID *int64 + TxHash *string `bun:"type:text"` + Epoch int64 `bun:",notnull"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +func storageDataSetTerminationTable2026090101() initialTableSpec { + return initialTableSpec{ + name: "storage_data_set_terminations", + model: (*storageDataSetTermination2026090101)(nil), + constraints: []string{ + "CONSTRAINT uq_storage_data_set_terminations_role UNIQUE (replacement_id, role)", + "CONSTRAINT chk_storage_data_set_terminations_role CHECK (role IN ('source', 'abandoned_target'))", + "CONSTRAINT chk_storage_data_set_terminations_subject CHECK ((role = 'source') = (source_data_set_id IS NOT NULL) AND (role = 'abandoned_target') = (abandoned_target_data_set_id IS NOT NULL))", + "CONSTRAINT chk_storage_data_set_terminations_epoch CHECK (epoch >= 0)", + "CONSTRAINT chk_storage_data_set_terminations_tx_hash CHECK (tx_hash IS NULL OR tx_hash <> '')", + }, + foreignKeys: []string{ + "(replacement_id) REFERENCES storage_replacements (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(replacement_id, source_data_set_id) REFERENCES storage_replacements (id, source_data_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(replacement_id, abandoned_target_data_set_id) REFERENCES storage_replacements (id, target_data_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + } +} + +type storageReplacementItem2026090101 struct { + bun.BaseModel `bun:"table:storage_replacement_items"` + + ID int64 `bun:",pk,autoincrement,identity"` + ReplacementID int64 `bun:",notnull"` + ContentID int64 `bun:",notnull"` + // TargetDataSetID is written when the item is seeded, together with the + // pending copy it names. A nullable column would make the composite foreign + // key below skip validation entirely whenever it was unset. + TargetDataSetID int64 `bun:",notnull"` + Status string `bun:"type:text,notnull,default:'pending'"` + LastError *string `bun:"type:text"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type storageCleanupCopy2026090101 struct { + bun.BaseModel `bun:"table:storage_cleanup_copies"` + + ID int64 `bun:",pk,autoincrement,identity"` + ContentID int64 `bun:",notnull"` + BucketID int64 `bun:",notnull"` + CopyIndex int `bun:"type:integer,notnull"` + ProviderID string `bun:"type:text,notnull"` + StorageDataSetID int64 `bun:",notnull"` + DataSetID *string `bun:"type:text"` + ClientDataSetID *string `bun:"type:text"` + PieceID string `bun:"type:text,notnull"` + PieceCID string `bun:"type:text,notnull"` + Checksum string `bun:"type:text,notnull"` + RetrievalURL *string `bun:"type:text"` + Status string `bun:"type:text,notnull,default:'pending'"` + DeleteTxHash *string `bun:"type:text"` + LastError *string `bun:"type:text"` + ScheduledAt *time.Time + RemovedAt *time.Time + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +func createStorageSchema(ctx context.Context, db bun.IDB) error { + checksumCheck := "length(checksum) = 64 AND checksum NOT GLOB '*[^0-9a-f]*'" + if db.Dialect().Name() == dialect.PG { + checksumCheck = "checksum ~ '^[0-9a-f]{64}$'" + } + tables := []initialTableSpec{ + { + name: "storage_contents", + model: (*storageContent2026090101)(nil), + constraints: []string{ + // Content dedup is a unique-key lookup, not an index scan. + "CONSTRAINT uq_storage_contents_bytes UNIQUE (bucket_id, checksum, content_size)", + // Candidate keys for the composite foreign keys that pin + // denormalized identity on object_versions and storage_copies. + "CONSTRAINT uq_storage_contents_id_bucket UNIQUE (id, bucket_id)", + "CONSTRAINT uq_storage_contents_addr UNIQUE (id, bucket_id, content_size)", + "CONSTRAINT chk_storage_contents_identity CHECK ((" + checksumCheck + ") AND (piece_cid IS NULL OR piece_cid <> ''))", + "CONSTRAINT chk_storage_contents_content_size CHECK (content_size >= 0)", + "CONSTRAINT chk_storage_contents_requested_copies CHECK (requested_copies BETWEEN 1 AND 8)", + "CONSTRAINT chk_storage_contents_cleanup_generation CHECK (cleanup_generation >= 0)", + }, + foreignKeys: []string{ + "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(cleanup_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }, + { + name: "storage_data_sets", + model: (*storageDataSet2026090101)(nil), + constraints: []string{ + "CONSTRAINT uq_storage_data_sets_id_bucket_slot UNIQUE (id, bucket_id, copy_index)", + "CONSTRAINT uq_storage_data_sets_identity UNIQUE (id, bucket_id, copy_index, provider_id)", + "CONSTRAINT chk_storage_data_sets_identity CHECK (provider_id <> '' AND (data_set_id IS NULL OR data_set_id <> '') AND (client_data_set_id IS NULL OR client_data_set_id <> '') AND (create_transaction_id IS NULL OR create_transaction_id <> '') AND (create_status_url IS NULL OR create_status_url <> ''))", + // The replica slot is a row, so the index is a foreign key rather than a range check. + "CONSTRAINT fk_storage_data_sets_replica_slot FOREIGN KEY (bucket_id, copy_index) REFERENCES bucket_replica_slots (bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", + "CONSTRAINT chk_storage_data_sets_generation CHECK (generation >= 1 AND retirement_generation >= 0)", + "CONSTRAINT chk_storage_data_sets_status CHECK (status IN ('pending', 'creating', 'ready', 'failed', 'draining', 'retired'))", + "CONSTRAINT chk_storage_data_sets_ready_identity CHECK (status <> 'ready' OR data_set_id IS NOT NULL)", + "CONSTRAINT chk_storage_data_sets_current_shape CHECK (status NOT IN ('failed', 'draining', 'retired') OR is_current = FALSE)", + }, + foreignKeys: []string{ + "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(created_by_content_id, bucket_id) REFERENCES storage_contents (id, bucket_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(last_used_content_id, bucket_id) REFERENCES storage_contents (id, bucket_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(ensure_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(retirement_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }, + { + name: "storage_copies", + model: (*storageCopy2026090101)(nil), + constraints: []string{ + "CONSTRAINT uq_storage_copies_content_data_set UNIQUE (content_id, storage_data_set_id)", + // The replica slot is a row, so the index is a foreign key rather than a range check. + "CONSTRAINT fk_storage_copies_replica_slot FOREIGN KEY (bucket_id, copy_index) REFERENCES bucket_replica_slots (bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", + "CONSTRAINT chk_storage_copies_work_generation CHECK (work_generation >= 0)", + "CONSTRAINT chk_storage_copies_status CHECK (status IN ('pending', 'piece_ready', 'committing', 'committed', 'failed'))", + // The projected status is pinned to a literal so the composite + // foreign key below can only ever reach a confirmed attempt. + "CONSTRAINT chk_storage_copies_confirmed_attempt_status CHECK (confirmed_attempt_status IS NULL OR confirmed_attempt_status = 'confirmed')", + // Committed and "has confirmed evidence" are the same fact. The + // foreign key alone would still allow a committed copy with no + // evidence at all, so both directions are stated here. + "CONSTRAINT chk_storage_copies_committed_evidence CHECK ((status = 'committed') = (confirmed_attempt_id IS NOT NULL))", + "CONSTRAINT chk_storage_copies_confirmed_attempt_shape CHECK ((confirmed_attempt_id IS NULL AND confirmed_attempt_status IS NULL) OR (confirmed_attempt_id IS NOT NULL AND confirmed_attempt_id <> '' AND confirmed_attempt_status IS NOT NULL))", + "CONSTRAINT chk_storage_copies_transfer_method CHECK (transfer_method IN ('ingress', 'peer_pull'))", + "CONSTRAINT chk_storage_copies_optional_identity CHECK (provider_id <> '' AND (piece_id IS NULL OR piece_id <> '') AND (retrieval_url IS NULL OR retrieval_url <> '') AND (commit_extra_data_hex IS NULL OR commit_extra_data_hex <> ''))", + "CONSTRAINT chk_storage_copies_committed_shape CHECK (status <> 'committed' OR (piece_id IS NOT NULL AND piece_id <> '' AND retrieval_url IS NOT NULL AND retrieval_url <> ''))", + "CONSTRAINT chk_storage_copies_commit_ready CHECK (commit_ready_at IS NULL OR status IN ('piece_ready', 'committing', 'committed'))", + "CONSTRAINT chk_storage_copies_content_size CHECK (content_size >= 0)", + // Ingress progress belongs to the transfer that produces it, so + // only an ingress copy may carry it. + "CONSTRAINT chk_storage_copies_ingress_progress CHECK (transfer_method = 'ingress' OR (ingress_bytes_transferred = 0 AND ingress_store_attempt = 0 AND progress_updated_at IS NULL))", + "CONSTRAINT chk_storage_copies_ingress_bytes CHECK (ingress_bytes_transferred >= 0 AND ingress_bytes_transferred <= content_size)", + "CONSTRAINT chk_storage_copies_ingress_attempt CHECK (ingress_store_attempt >= 0)", + }, + foreignKeys: []string{ + "(content_id, bucket_id, content_size) REFERENCES storage_contents (id, bucket_id, content_size) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(storage_data_set_id, bucket_id, copy_index, provider_id) REFERENCES storage_data_sets (id, bucket_id, copy_index, provider_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(active_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + forwardForeignKeys: []initialForwardForeignKey{storageCopyConfirmedAttemptForeignKey2026090101()}, + }, + storageCommitAttemptTable2026090101(), + storagePullAttemptTable2026090101(), + storageReplacementTable2026090101(), + storageDataSetTerminationTable2026090101(), + storageReplacementItemTable2026090101(), + storageCleanupCopyTable2026090101(), + } + for _, table := range tables { + if err := createInitialTable(ctx, db, table); err != nil { + return err + } + } + if err := createInitialIndexes(ctx, db, storageIndexes2026090101()...); err != nil { + return err + } + // storage_copies was created before storage_commit_attempts existed, so + // PostgreSQL takes the projection constraint here. + return addForwardForeignKey(ctx, db, "storage_copies", storageCopyConfirmedAttemptForeignKey2026090101()) +} + +// storageCopyConfirmedAttemptForeignKey2026090101 welds the copy's committed +// state to the ledger row that proves it. storage_copies is created before +// storage_commit_attempts, so the constraint is forward-declared. +func storageCopyConfirmedAttemptForeignKey2026090101() initialForwardForeignKey { + return initialForwardForeignKey{ + name: "fk_storage_copies_confirmed_attempt", + definition: "(confirmed_attempt_id, confirmed_attempt_status) REFERENCES storage_commit_attempts (attempt_id, status) ON UPDATE RESTRICT ON DELETE RESTRICT", + } +} + +func storageCommitAttemptTable2026090101() initialTableSpec { + return initialTableSpec{ + name: "storage_commit_attempts", + jsonColumns: initialJSONColumns("storage_commit_attempts"), + model: (*storageCommitAttempt2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_storage_commit_attempts_identity CHECK (attempt_id <> '' AND (extra_data_hex IS NULL OR extra_data_hex <> '') AND (transaction_id IS NULL OR transaction_id <> '') AND (submission_json IS NULL OR submission_json <> '') AND (confirmed_transaction_id IS NULL OR confirmed_transaction_id <> '') AND (attention_code IS NULL OR attention_code <> '') AND (release_reason IS NULL OR release_reason <> ''))", + "CONSTRAINT chk_storage_commit_attempts_status CHECK (status IN ('reserved', 'attempted', 'confirmed', 'released', 'rejected'))", + // Candidate key for the copy's confirmed-attempt projection. + "CONSTRAINT uq_storage_commit_attempts_status UNIQUE (attempt_id, status)", + "CONSTRAINT chk_storage_commit_attempts_resolution CHECK ((status IN ('reserved', 'attempted') AND resolved_at IS NULL) OR (status IN ('confirmed', 'released', 'rejected') AND resolved_at IS NOT NULL))", + `CONSTRAINT chk_storage_commit_attempts_evidence_shape CHECK ( + (status = 'reserved' AND attempted_at IS NULL AND extra_data_hex IS NULL AND transaction_id IS NULL AND submission_json IS NULL AND confirmed_transaction_id IS NULL AND attention_code IS NULL AND attention_at IS NULL AND last_error IS NULL) + OR (status = 'attempted' AND attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL AND confirmed_transaction_id IS NULL AND last_error IS NULL) + OR (status = 'confirmed' AND attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL AND transaction_id IS NOT NULL AND confirmed_transaction_id IS NOT NULL AND last_error IS NULL) + OR (status = 'released' AND confirmed_transaction_id IS NULL AND last_error IS NULL AND ((attempted_at IS NULL AND extra_data_hex IS NULL AND transaction_id IS NULL AND submission_json IS NULL AND attention_code IS NULL AND attention_at IS NULL) OR (attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL))) + OR (status = 'rejected' AND attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL AND confirmed_transaction_id IS NULL AND last_error IS NOT NULL AND last_error <> '') + )`, + "CONSTRAINT chk_storage_commit_attempts_submission CHECK (submission_json IS NULL OR transaction_id IS NOT NULL)", + "CONSTRAINT chk_storage_commit_attempts_attention CHECK ((attention_code IS NULL AND attention_at IS NULL) OR (attention_code IS NOT NULL AND attention_at IS NOT NULL AND attempted_at IS NOT NULL))", + "CONSTRAINT chk_storage_commit_attempts_release CHECK ((status = 'released' AND release_reason IS NOT NULL) OR (status <> 'released' AND release_reason IS NULL))", + }, + // The ledger outlives the copy it was made for: finishing a content's + // cleanup deletes the copy, and (content_id, storage_data_set_id) stays + // here as a value. + } +} + +func storagePullAttemptTable2026090101() initialTableSpec { + return initialTableSpec{ + name: "storage_pull_attempts", + model: (*storagePullAttempt2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_storage_pull_attempts_identity CHECK (attempt_id <> '' AND source_provider_id <> '' AND source_data_set_id <> '' AND source_piece_id <> '' AND source_piece_cid <> '' AND source_retrieval_url <> '')", + "CONSTRAINT chk_storage_pull_attempts_status CHECK (status IN ('attempted', 'abandoned'))", + // An error only makes sense on a request nobody will observe again; + // reopening a copy carries no error string. + "CONSTRAINT chk_storage_pull_attempts_error CHECK (last_error IS NULL OR (status = 'abandoned' AND last_error <> ''))", + "CONSTRAINT chk_storage_pull_attempts_abandoned CHECK (status <> 'abandoned' OR resolved_at IS NOT NULL)", + }, + // Like commit attempts, pull attempts keep the copy identity as values + // once cleanup has deleted the copy. + } +} + +func storageReplacementTable2026090101() initialTableSpec { + return initialTableSpec{ + name: "storage_replacements", + model: (*storageReplacement2026090101)(nil), + constraints: []string{ + "CONSTRAINT uq_storage_replacements_id_source UNIQUE (id, source_data_set_id)", + "CONSTRAINT uq_storage_replacements_id_target UNIQUE (id, target_data_set_id)", + "CONSTRAINT chk_storage_replacements_identity CHECK (client_request_id <> '' AND (requested_provider_id IS NULL OR requested_provider_id <> ''))", + // The replica slot is a row, so the index is a foreign key rather than a range check. + "CONSTRAINT fk_storage_replacements_replica_slot FOREIGN KEY (bucket_id, copy_index) REFERENCES bucket_replica_slots (bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", + "CONSTRAINT chk_storage_replacements_selection_mode CHECK (selection_mode IN ('automatic', 'manual'))", + "CONSTRAINT chk_storage_replacements_status CHECK (status IN ('preparing_target', 'migrating', 'waiting', 'retiring', 'cleanup_attention', 'failed', 'completed', 'superseded'))", + "CONSTRAINT chk_storage_replacements_wait_reason CHECK (wait_reason IS NULL OR wait_reason <> '')", + "CONSTRAINT chk_storage_replacements_failure_reason CHECK (failure_reason IS NULL OR failure_reason <> '')", + "CONSTRAINT chk_storage_replacements_client_request_id CHECK (length(client_request_id) BETWEEN 1 AND 128)", + "CONSTRAINT chk_storage_replacements_distinct_data_sets CHECK (source_data_set_id <> target_data_set_id)", + "CONSTRAINT chk_storage_replacements_items CHECK (items_total >= 0 AND items_copied >= 0 AND items_copied <= items_total)", + "CONSTRAINT chk_storage_replacements_generation CHECK (task_generation >= 1)", + }, + foreignKeys: []string{ + "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(source_data_set_id, bucket_id, copy_index) REFERENCES storage_data_sets (id, bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(target_data_set_id, bucket_id, copy_index) REFERENCES storage_data_sets (id, bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(superseded_by_id) REFERENCES storage_replacements (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + } +} + +func storageReplacementItemTable2026090101() initialTableSpec { + return initialTableSpec{ + name: "storage_replacement_items", + model: (*storageReplacementItem2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_storage_replacement_items_status CHECK (status IN ('pending', 'copied', 'cancelled', 'attention'))", + "CONSTRAINT uq_storage_replacement_items_content UNIQUE (replacement_id, content_id)", + }, + foreignKeys: []string{ + "(replacement_id) REFERENCES storage_replacements (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + "(replacement_id, target_data_set_id) REFERENCES storage_replacements (id, target_data_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + // content_id stays a value once cleanup deletes the content and its + // copies; cleanup waits while an item that blocks retirement names it. + } +} + +func storageCleanupCopyTable2026090101() initialTableSpec { + return initialTableSpec{ + name: "storage_cleanup_copies", + model: (*storageCleanupCopy2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_storage_cleanup_copies_identity CHECK (provider_id <> '' AND piece_id <> '' AND piece_cid <> '' AND checksum <> '' AND (data_set_id IS NULL OR data_set_id <> '') AND (client_data_set_id IS NULL OR client_data_set_id <> '') AND (delete_tx_hash IS NULL OR delete_tx_hash <> ''))", + // The replica slot is a row, so the index is a foreign key rather than a range check. + "CONSTRAINT fk_storage_cleanup_copies_replica_slot FOREIGN KEY (bucket_id, copy_index) REFERENCES bucket_replica_slots (bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", + "CONSTRAINT chk_storage_cleanup_copies_status CHECK (status IN ('pending', 'delete_scheduled', 'removed', 'failed', 'unsupported'))", + "CONSTRAINT chk_storage_cleanup_copies_delete_scheduled CHECK (status <> 'delete_scheduled' OR (delete_tx_hash IS NOT NULL AND scheduled_at IS NOT NULL))", + "CONSTRAINT uq_storage_cleanup_copies_physical UNIQUE (content_id, storage_data_set_id, piece_id)", + }, + // The content a cleanup removed is deleted once the cleanup finishes, so + // its identity and checksum stay here as values rather than a reference. + foreignKeys: []string{ + "(storage_data_set_id, bucket_id, copy_index, provider_id) REFERENCES storage_data_sets (id, bucket_id, copy_index, provider_id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + } +} + +func storageIndexes2026090101() []initialIndexSpec { + return []initialIndexSpec{ + {name: "idx_storage_contents_bucket_id", table: "storage_contents", columns: []string{"bucket_id", "id"}}, + {name: "idx_storage_contents_cleanup_task", table: "storage_contents", columns: []string{"cleanup_task_id"}, where: "cleanup_task_id IS NOT NULL", unique: true}, + {name: "idx_storage_data_sets_provider_data_set", table: "storage_data_sets", columns: []string{"provider_id", "data_set_id"}, where: "data_set_id IS NOT NULL", unique: true}, + {name: "idx_storage_data_sets_bucket_copy_current", table: "storage_data_sets", columns: []string{"bucket_id", "copy_index"}, where: "is_current = TRUE", unique: true}, + {name: "idx_storage_data_sets_bucket_copy_generation", table: "storage_data_sets", columns: []string{"bucket_id", "copy_index", "generation"}, unique: true}, + {name: "idx_storage_data_sets_bucket_provider_active", table: "storage_data_sets", columns: []string{"bucket_id", "provider_id"}, where: "status <> 'retired'", unique: true}, + {name: "idx_storage_data_sets_created_by_content", table: "storage_data_sets", columns: []string{"created_by_content_id"}}, + {name: "idx_storage_data_sets_last_used_content", table: "storage_data_sets", columns: []string{"last_used_content_id"}}, + {name: "idx_storage_data_sets_ensure_task", table: "storage_data_sets", columns: []string{"ensure_task_id"}, where: "ensure_task_id IS NOT NULL", unique: true}, + {name: "idx_storage_data_sets_retirement_task", table: "storage_data_sets", columns: []string{"retirement_task_id"}, where: "retirement_task_id IS NOT NULL", unique: true}, + {name: "idx_storage_copies_content_slot", table: "storage_copies", columns: []string{"content_id", "copy_index"}}, + {name: "idx_storage_copies_data_set_identity", table: "storage_copies", columns: []string{"storage_data_set_id", "bucket_id", "copy_index", "provider_id"}}, + {name: "idx_storage_copies_content_transfer_method_index", table: "storage_copies", columns: []string{"content_id", "transfer_method", "copy_index"}}, + {name: "idx_storage_copies_ingress_content", table: "storage_copies", columns: []string{"content_id"}, where: "transfer_method = 'ingress'", unique: true}, + {name: "idx_storage_copies_status_data_set_content", table: "storage_copies", columns: []string{"status", "storage_data_set_id", "content_id"}}, + {name: "idx_storage_copies_status_piece_identity_content", table: "storage_copies", columns: []string{"status", "provider_id", "piece_id", "content_id"}}, + {name: "idx_storage_copies_commit_ready", table: "storage_copies", columns: []string{"storage_data_set_id", "commit_ready_at", "id"}, where: "status = 'piece_ready' AND commit_ready_at IS NOT NULL"}, + {name: "idx_storage_copies_active_task", table: "storage_copies", columns: []string{"active_task_id"}, where: "active_task_id IS NOT NULL", unique: true}, + {name: "idx_storage_commit_attempts_unresolved_copy", table: "storage_commit_attempts", columns: []string{"content_id", "storage_data_set_id"}, where: "resolved_at IS NULL", unique: true}, + {name: "idx_storage_commit_attempts_unresolved_data_set", table: "storage_commit_attempts", columns: []string{"storage_data_set_id", "created_at", "attempt_id"}, where: "resolved_at IS NULL"}, + {name: "idx_storage_commit_attempts_copy_history", table: "storage_commit_attempts", columns: []string{"content_id", "storage_data_set_id", "created_at DESC", "attempt_id"}}, + {name: "idx_storage_cleanup_copies_replica_slot", table: "storage_cleanup_copies", columns: []string{"bucket_id", "copy_index"}}, + {name: "idx_storage_copies_confirmed_attempt", table: "storage_copies", columns: []string{"confirmed_attempt_id", "confirmed_attempt_status"}, where: "confirmed_attempt_id IS NOT NULL"}, + {name: "idx_storage_copies_replica_slot", table: "storage_copies", columns: []string{"bucket_id", "copy_index"}}, + // One unresolved attempt per copy: a second source can only be tried + // after the first is abandoned. + {name: "idx_storage_pull_attempts_unresolved_copy", table: "storage_pull_attempts", columns: []string{"content_id", "storage_data_set_id"}, where: "resolved_at IS NULL", unique: true}, + {name: "idx_storage_pull_attempts_copy_history", table: "storage_pull_attempts", columns: []string{"content_id", "storage_data_set_id", "created_at DESC", "attempt_id"}}, + {name: "idx_storage_replacements_active_source", table: "storage_replacements", columns: []string{"source_data_set_id"}, where: "status NOT IN ('completed', 'superseded')", unique: true}, + {name: "idx_storage_replacements_active_target", table: "storage_replacements", columns: []string{"target_data_set_id"}, where: "status NOT IN ('completed', 'superseded')", unique: true}, + {name: "idx_storage_replacements_active_bucket_slot", table: "storage_replacements", columns: []string{"bucket_id", "copy_index"}, where: "status NOT IN ('completed', 'superseded')", unique: true}, + {name: "idx_storage_replacements_source_identity", table: "storage_replacements", columns: []string{"source_data_set_id", "bucket_id", "copy_index"}}, + {name: "idx_storage_replacements_target_identity", table: "storage_replacements", columns: []string{"target_data_set_id", "bucket_id", "copy_index"}}, + {name: "idx_storage_replacements_superseded_by", table: "storage_replacements", columns: []string{"superseded_by_id"}, where: "superseded_by_id IS NOT NULL"}, + {name: "idx_storage_replacements_bucket_slot", table: "storage_replacements", columns: []string{"bucket_id", "copy_index", "id"}}, + {name: "idx_storage_replacements_bucket_request", table: "storage_replacements", columns: []string{"bucket_id", "client_request_id"}, unique: true}, + {name: "idx_storage_replacements_task", table: "storage_replacements", columns: []string{"task_id"}, where: "task_id IS NOT NULL", unique: true}, + {name: "idx_storage_replacement_items_state", table: "storage_replacement_items", columns: []string{"replacement_id", "status", "id"}}, + {name: "idx_storage_replacement_items_content_id", table: "storage_replacement_items", columns: []string{"content_id"}}, + {name: "idx_storage_replacement_items_target_copy", table: "storage_replacement_items", columns: []string{"content_id", "target_data_set_id"}}, + {name: "idx_storage_cleanup_copies_content_status", table: "storage_cleanup_copies", columns: []string{"content_id", "status", "id"}}, + {name: "idx_storage_cleanup_copies_data_set_identity", table: "storage_cleanup_copies", columns: []string{"storage_data_set_id", "bucket_id", "copy_index", "provider_id"}}, + {name: "idx_storage_cleanup_copies_status_scheduled", table: "storage_cleanup_copies", columns: []string{"status", "scheduled_at", "id"}}, + } +} + +type walletOperation2026090101 struct { + bun.BaseModel `bun:"table:wallet_operations"` + + ID int64 `bun:",pk,autoincrement,identity"` + Type string `bun:"type:text,notnull"` + ClientRequestID string `bun:"type:text,notnull"` + Amount string `bun:"type:text,notnull"` + Status string `bun:"type:text,notnull,default:'pending'"` + TxHash *string `bun:"type:text"` + LastError *string `bun:"type:text"` + BroadcastAttemptedAt *time.Time + TaskID *int64 + StartedAt *time.Time + SubmittedAt *time.Time + CompletedAt *time.Time + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +func createWalletSchema(ctx context.Context, db bun.IDB) error { + amountCheck := `((type = 'approve' AND amount = '0') OR (type IN ('fund', 'withdraw') AND amount GLOB '[1-9]*' AND amount NOT GLOB '*[^0-9]*'))` + if db.Dialect().Name() == dialect.PG { + amountCheck = `((type = 'approve' AND amount = '0') OR (type IN ('fund', 'withdraw') AND amount ~ '^[1-9][0-9]*$'))` + } + if err := createInitialTable(ctx, db, initialTableSpec{ + name: "wallet_operations", + model: (*walletOperation2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_wallet_operations_identity CHECK (client_request_id <> '' AND amount <> '' AND (tx_hash IS NULL OR tx_hash <> ''))", + "CONSTRAINT chk_wallet_operations_type CHECK (type IN ('fund', 'withdraw', 'approve'))", + "CONSTRAINT chk_wallet_operations_status CHECK (status IN ('pending', 'submitted', 'confirmed', 'failed', 'unknown'))", + "CONSTRAINT chk_wallet_operations_submitted_shape CHECK (status <> 'submitted' OR (tx_hash IS NOT NULL AND submitted_at IS NOT NULL))", + "CONSTRAINT chk_wallet_operations_amount CHECK (" + amountCheck + ")", + // A uint256 in base 10 is at most 78 digits. The column stays text + // because the value is returned and compared verbatim. + "CONSTRAINT chk_wallet_operations_amount_length CHECK (length(amount) BETWEEN 1 AND 78)", + }, + foreignKeys: []string{ + "(task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", + }, + }); err != nil { + return err + } + return createInitialIndexes(ctx, db, + initialIndexSpec{name: "idx_wallet_operations_request", table: "wallet_operations", columns: []string{"type", "client_request_id"}, unique: true}, + initialIndexSpec{name: "idx_wallet_operations_status_created", table: "wallet_operations", columns: []string{"status", "created_at", "id"}}, + initialIndexSpec{name: "idx_wallet_operations_recent", table: "wallet_operations", columns: []string{"created_at DESC", "id DESC"}}, + initialIndexSpec{name: "idx_wallet_operations_task", table: "wallet_operations", columns: []string{"task_id"}, where: "task_id IS NOT NULL", unique: true}, + ) +} + +type observabilityCollectionState2026090101 struct { + bun.BaseModel `bun:"table:observability_collection_states"` + + CollectionType string `bun:"type:text,pk"` + LastCheckedAt time.Time `bun:",notnull"` + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} + +type observabilityProviderState2026090101 struct { + bun.BaseModel `bun:"table:observability_provider_states"` + + ProviderID string `bun:"type:text,pk"` + Status string `bun:"type:text,notnull"` + ReasonCodes json.RawMessage `bun:"type:jsonb,notnull"` + Active *bool + HasPDP *bool + ServiceURL *string `bun:"type:text"` + HealthStatus *string `bun:"type:text"` + LastCheckedAt time.Time `bun:",notnull"` + LastError *string `bun:"type:text"` + Evidence json.RawMessage `bun:"evidence_json,type:jsonb,notnull"` +} + +type observabilityDataSetState2026090101 struct { + bun.BaseModel `bun:"table:observability_data_set_states"` + + LocalDataSetID int64 `bun:",pk"` + BucketID int64 `bun:",notnull"` + CopyIndex int `bun:"type:integer,notnull"` + ProviderID string `bun:"type:text,notnull"` + ChainDataSetID *string `bun:"type:text"` + ClientDataSetID *string `bun:"type:text"` + // The bucket's name and the data set's own status are a join away and can + // disagree with the authorities that own them, so neither is copied here. + Status string `bun:"type:text,notnull"` + ReasonCodes json.RawMessage `bun:"type:jsonb,notnull"` + ActivePieceCount *int64 + LastCheckedAt time.Time `bun:",notnull"` + LastError *string `bun:"type:text"` + Evidence json.RawMessage `bun:"evidence_json,type:jsonb,notnull"` +} + +func createObservabilitySchema(ctx context.Context, db bun.IDB) error { + tables := []initialTableSpec{ + { + name: "observability_collection_states", + model: (*observabilityCollectionState2026090101)(nil), + constraints: []string{ + "CONSTRAINT chk_observability_collection_type CHECK (collection_type IN ('providers', 'data_sets'))", + }, + }, + { + name: "observability_provider_states", + model: (*observabilityProviderState2026090101)(nil), + jsonColumns: initialJSONColumns("observability_provider_states"), + constraints: []string{ + "CONSTRAINT chk_observability_provider_identity CHECK (provider_id <> '')", + "CONSTRAINT chk_observability_provider_status CHECK (status IN ('available', 'degraded', 'unavailable', 'unknown'))", + }, + }, + { + name: "observability_data_set_states", + model: (*observabilityDataSetState2026090101)(nil), + jsonColumns: initialJSONColumns("observability_data_set_states"), + constraints: []string{ + "CONSTRAINT chk_observability_data_set_identity CHECK (provider_id <> '' AND (chain_data_set_id IS NULL OR chain_data_set_id <> '') AND (client_data_set_id IS NULL OR client_data_set_id <> ''))", + "CONSTRAINT fk_observability_data_set_replica_slot FOREIGN KEY (bucket_id, copy_index) REFERENCES bucket_replica_slots (bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", + "CONSTRAINT chk_observability_data_set_status CHECK (status IN ('available', 'degraded', 'unavailable', 'unknown'))", + }, + foreignKeys: []string{ + "(local_data_set_id, bucket_id, copy_index, provider_id) REFERENCES storage_data_sets (id, bucket_id, copy_index, provider_id) ON UPDATE RESTRICT ON DELETE CASCADE", + }, + }, + } + for _, table := range tables { + if err := createInitialTable(ctx, db, table); err != nil { + return err + } + } + return createInitialIndexes(ctx, db, + initialIndexSpec{name: "idx_observability_provider_states_status", table: "observability_provider_states", columns: []string{"status", "last_checked_at"}}, + initialIndexSpec{name: "idx_observability_data_set_states_bucket_status", table: "observability_data_set_states", columns: []string{"bucket_id", "status", "last_checked_at"}}, + initialIndexSpec{name: "idx_observability_data_set_states_provider_status", table: "observability_data_set_states", columns: []string{"provider_id", "status", "last_checked_at"}}, + ) +} diff --git a/internal/db/migrations/migrations.go b/internal/db/migrations/migrations.go index 32c63a9..ba3c37a 100644 --- a/internal/db/migrations/migrations.go +++ b/internal/db/migrations/migrations.go @@ -166,6 +166,7 @@ func initialSchemaPostStateComplete(ctx context.Context, db bun.IDB) (bool, erro {"storage_commit_attempts", "attempt_id"}, {"storage_replacement_items", "target_data_set_id"}, {"storage_cleanup_copies", "bucket_id"}, + {"storage_cleanup_copies", "checksum"}, {"object_versions", "content_id"}, {"object_cache", "content_id"}, } { diff --git a/internal/db/migrations/schema_behavior_test.go b/internal/db/migrations/schema_behavior_test.go index 0bcf222..5391282 100644 --- a/internal/db/migrations/schema_behavior_test.go +++ b/internal/db/migrations/schema_behavior_test.go @@ -74,8 +74,8 @@ func TestBaselineConstraintsRejectInvalidWrites(t *testing.T) { dataSetID := insertBaselineTestDataSet(t, db, bucketID, "cleanup-provider", 2, 1, false) var cleanupID int64 if err := db.QueryRow(`INSERT INTO storage_cleanup_copies - (content_id, bucket_id, copy_index, provider_id, storage_data_set_id, piece_id, piece_cid, created_at, updated_at) - VALUES (?, ?, 2, 'cleanup-provider', ?, 'piece-1', 'piece-cid-1', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + (content_id, bucket_id, copy_index, provider_id, storage_data_set_id, piece_id, piece_cid, checksum, created_at, updated_at) + VALUES (?, ?, 2, 'cleanup-provider', ?, 'piece-1', 'piece-cid-1', 'checksum-1', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) RETURNING id`, contentID, bucketID, dataSetID).Scan(&cleanupID); err != nil { t.Fatalf("insert cleanup copy: %v", err) } @@ -197,9 +197,6 @@ func TestBaselineStorageIdentityAndLedgerConstraints(t *testing.T) { SET status = 'attempted', attempted_at = current_timestamp, extra_data_hex = 'abcd', submission_json = '{}' WHERE attempt_id = 'attempt-2'`) - mustRejectStatement(t, db, `INSERT INTO storage_commit_attempts - (attempt_id, content_id, storage_data_set_id, created_at, updated_at) - VALUES ('cross-copy', ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, contentA2, source) target := insertBaselineTestDataSet(t, db, bucketA, "202", 0, 2, false) var replacementID int64 @@ -230,9 +227,6 @@ func TestBaselineStorageIdentityAndLedgerConstraints(t *testing.T) { (bucket_id, copy_index, source_data_set_id, target_data_set_id, selection_mode, client_request_id, status, created_at, updated_at) VALUES (?, 0, ?, ?, 'manual', 'replacement-3', 'preparing_target', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, bucketA, source, target) - mustRejectStatement(t, db, `INSERT INTO storage_replacement_items - (replacement_id, content_id, target_data_set_id, created_at, updated_at) - VALUES (?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, replacementID, contentA, target) mustRejectStatement(t, db, `INSERT INTO storage_copies (content_id, bucket_id, content_size, storage_data_set_id, copy_index, provider_id, transfer_method, created_at, updated_at) VALUES (?, ?, 1, ?, 0, '202', 'ingress', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, contentA, bucketA, target) diff --git a/internal/db/migrations/schema_core.go b/internal/db/migrations/schema_core.go deleted file mode 100644 index 024465e..0000000 --- a/internal/db/migrations/schema_core.go +++ /dev/null @@ -1,341 +0,0 @@ -package migrations - -import ( - "context" - "encoding/json" - "time" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" -) - -type s3Account2026090101 struct { - bun.BaseModel `bun:"table:s3_accounts"` - - AccessKey string `bun:"type:text,pk"` - SecretKey string `bun:"type:text,notnull"` - Role string `bun:"type:text,notnull"` - IsRoot bool `bun:",notnull,default:false"` - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -type bucket2026090101 struct { - bun.BaseModel `bun:"table:buckets"` - - ID int64 `bun:",pk,autoincrement,identity"` - Name string `bun:"type:text,notnull,unique"` - ACL []byte - OwnerAccessKey *string `bun:"type:text"` - DefaultCopies int `bun:"type:integer,notnull"` - MinimumDurableCopies int `bun:"type:integer,notnull"` - DurabilityGeneration int64 `bun:",notnull,default:0"` - DurabilityTaskID *int64 - Status string `bun:"type:text,notnull,default:'provisioning'"` - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -// bucketReplicaSlot2026090101 gives a replica slot a row of its own. copy_index -// used to be a bare integer repeated across five tables and bounded only by a -// range check; as a table it becomes a foreign key target, and a slot keeps its -// history because retired generations still reference it. -type bucketReplicaSlot2026090101 struct { - bun.BaseModel `bun:"table:bucket_replica_slots"` - - ID int64 `bun:",pk,autoincrement,identity"` - BucketID int64 `bun:",notnull"` - CopyIndex int `bun:"type:integer,notnull"` - Status string `bun:"type:text,notnull,default:'active'"` - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -type object2026090101 struct { - bun.BaseModel `bun:"table:objects"` - - ID int64 `bun:",pk,autoincrement,identity"` - BucketID int64 `bun:",notnull"` - Key string `bun:"type:text,notnull"` - // CurrentVersionID points at the version S3 reads serve. One column can hold - // one value, so "at most one current version" is structural here rather than - // a partial unique index over every version of the object. - CurrentVersionID *string `bun:"type:text"` - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -type multipartUpload2026090101 struct { - bun.BaseModel `bun:"table:multipart_uploads"` - - UploadID string `bun:"type:text,pk"` - BucketID int64 `bun:",notnull"` - Key string `bun:"type:text,notnull"` - ContentType string `bun:"type:text,notnull,default:'application/octet-stream'"` - Metadata json.RawMessage `bun:"type:jsonb,notnull,default:'{}'"` - Status string `bun:"type:text,notnull,default:'initiated'"` - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -type multipartPart2026090101 struct { - bun.BaseModel `bun:"table:multipart_parts"` - - ID int64 `bun:",pk,autoincrement,identity"` - UploadID string `bun:"type:text,notnull"` - PartNumber int `bun:"type:integer,notnull"` - Size int64 `bun:",notnull"` - ETag string `bun:"e_tag,type:text,notnull"` - Checksum *string `bun:"type:text"` - CreatedAt time.Time `bun:",notnull"` -} - -func createCoreRootSchema(ctx context.Context, db bun.IDB) error { - tables := []initialTableSpec{ - { - name: "s3_accounts", - model: (*s3Account2026090101)(nil), - constraints: []string{ - "CONSTRAINT chk_s3_accounts_identity CHECK (access_key <> '' AND secret_key <> '')", - "CONSTRAINT chk_s3_accounts_role CHECK (role IN ('admin', 'user', 'userplus'))", - }, - }, - { - name: "buckets", - model: (*bucket2026090101)(nil), - constraints: []string{ - "CONSTRAINT chk_buckets_identity CHECK (name <> '' AND (owner_access_key IS NULL OR owner_access_key <> ''))", - "CONSTRAINT chk_buckets_status CHECK (status IN ('provisioning', 'ready'))", - // The durability policy is materialised from configuration when - // the bucket is created, so "how many replicas does this bucket - // want" never depends on reading config at query time. - "CONSTRAINT chk_buckets_default_copies CHECK (default_copies BETWEEN 1 AND 8)", - "CONSTRAINT chk_buckets_minimum_durable_copies CHECK (minimum_durable_copies BETWEEN 1 AND 8)", - "CONSTRAINT chk_buckets_explicit_copy_policy CHECK (minimum_durable_copies <= default_copies)", - "CONSTRAINT chk_buckets_durability_generation CHECK (durability_generation >= 0)", - }, - foreignKeys: []string{ - "(owner_access_key) REFERENCES s3_accounts (access_key) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(durability_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - }, - { - name: "bucket_replica_slots", - model: (*bucketReplicaSlot2026090101)(nil), - constraints: []string{ - "CONSTRAINT uq_bucket_replica_slots_identity UNIQUE (bucket_id, copy_index)", - "CONSTRAINT chk_bucket_replica_slots_copy_index CHECK (copy_index BETWEEN 0 AND 7)", - // A shrunk slot is decommissioned, never deleted: retired data - // set generations still point at it. - "CONSTRAINT chk_bucket_replica_slots_status CHECK (status IN ('active', 'decommissioned'))", - }, - foreignKeys: []string{ - "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - }, - { - name: "objects", - model: (*object2026090101)(nil), - constraints: []string{ - "CONSTRAINT uq_objects_identity UNIQUE (id, bucket_id, key)", - "CONSTRAINT chk_objects_identity CHECK (key <> '')", - }, - foreignKeys: []string{ - "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - forwardForeignKeys: []initialForwardForeignKey{objectCurrentVersionForeignKey2026090101()}, - }, - { - name: "multipart_uploads", - model: (*multipartUpload2026090101)(nil), - jsonColumns: initialJSONColumns("multipart_uploads"), - constraints: []string{ - "CONSTRAINT uq_multipart_uploads_identity UNIQUE (upload_id, bucket_id, key)", - "CONSTRAINT chk_multipart_uploads_identity CHECK (key <> '' AND upload_id <> '')", - "CONSTRAINT chk_multipart_uploads_status CHECK (status IN ('initiated', 'completing', 'completed', 'aborted'))", - }, - foreignKeys: []string{ - "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - }, - { - name: "multipart_parts", - model: (*multipartPart2026090101)(nil), - constraints: []string{ - "CONSTRAINT chk_multipart_parts_identity CHECK (upload_id <> '' AND e_tag <> '' AND (checksum IS NULL OR checksum <> ''))", - "CONSTRAINT chk_multipart_parts_part_number CHECK (part_number BETWEEN 1 AND 10000)", - "CONSTRAINT chk_multipart_parts_size CHECK (size >= 0)", - }, - foreignKeys: []string{ - "(upload_id) REFERENCES multipart_uploads (upload_id) ON UPDATE RESTRICT ON DELETE CASCADE", - }, - }, - } - for _, table := range tables { - if err := createInitialTable(ctx, db, table); err != nil { - return err - } - } - indexes := []initialIndexSpec{ - {name: "idx_objects_bucket_key", table: "objects", columns: []string{"bucket_id", "key"}, unique: true}, - {name: "idx_objects_current_version", table: "objects", columns: []string{"current_version_id"}, where: "current_version_id IS NOT NULL"}, - {name: "idx_s3_accounts_single_root", table: "s3_accounts", columns: []string{"is_root"}, where: "is_root = TRUE", unique: true}, - {name: "idx_buckets_owner_access_key", table: "buckets", columns: []string{"owner_access_key"}}, - {name: "idx_buckets_durability_task", table: "buckets", columns: []string{"durability_task_id"}, where: "durability_task_id IS NOT NULL", unique: true}, - {name: "idx_multipart_parts_upload_part", table: "multipart_parts", columns: []string{"upload_id", "part_number"}, unique: true}, - } - if db.Dialect().Name() != dialect.PG { - indexes = append(indexes, initialIndexSpec{name: "idx_multipart_uploads_bucket_status_key_upload", table: "multipart_uploads", columns: []string{"bucket_id", "status", "key", "upload_id"}}) - } - return createInitialIndexes(ctx, db, indexes...) -} - -// objectVersion2026090101 carries S3 naming semantics only. Pipeline state is -// a function of the copy rows and stays a query; cache residency belongs to -// object_cache; the bytes themselves are storage_contents. A NULL content_id -// is exactly a delete marker. -type objectVersion2026090101 struct { - bun.BaseModel `bun:"table:object_versions"` - - VersionID string `bun:"type:text,pk"` - ObjectID int64 `bun:",notnull"` - BucketID int64 `bun:",notnull"` - Key string `bun:"type:text,notnull"` - ContentID *int64 - Size int64 `bun:",notnull"` - ETag string `bun:"e_tag,type:text,notnull"` - ContentType string `bun:"type:text,notnull,default:'application/octet-stream'"` - Metadata json.RawMessage `bun:"type:jsonb,notnull,default:'{}'"` - MultipartUploadID *string `bun:"type:text"` - IsDeleteMarker bool `bun:",notnull,default:false"` - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -// objectCache2026090101 records local cache residency per content, not per -// version, so identical bytes written under several keys share one file. The -// cache key is derived from content_id and therefore not stored. -type objectCache2026090101 struct { - bun.BaseModel `bun:"table:object_cache"` - - ContentID int64 `bun:",pk"` - InCache bool `bun:",notnull"` - CacheAccessedAt *time.Time - CachePresenceGeneration int64 `bun:",notnull,default:0"` - CacheOperationGeneration int64 `bun:",notnull,default:0"` - CacheActiveTaskID *int64 - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -// objectDeletion2026090101 is an append-only tombstone. Cache cleanup is not -// tracked here: with content-keyed residency the trigger is the content's -// reference count reaching zero, not the removal of one version. -type objectDeletion2026090101 struct { - bun.BaseModel `bun:"table:object_deletions"` - - ID int64 `bun:",pk,autoincrement,identity"` - BucketID int64 `bun:",notnull"` - ObjectID int64 `bun:",notnull"` - Key string `bun:"type:text,notnull"` - VersionID string `bun:"type:text,notnull,unique"` - ContentID *int64 - Size int64 `bun:",notnull"` - DeletedAt time.Time `bun:",notnull"` -} - -func createObjectLifecycleSchema(ctx context.Context, db bun.IDB) error { - if err := createInitialTable(ctx, db, initialTableSpec{ - name: "object_versions", - model: (*objectVersion2026090101)(nil), - jsonColumns: initialJSONColumns("object_versions"), - constraints: []string{ - "CONSTRAINT chk_object_versions_identity CHECK (version_id <> '' AND key <> '' AND (multipart_upload_id IS NULL OR multipart_upload_id <> ''))", - "CONSTRAINT chk_object_versions_size CHECK (size >= 0)", - // Candidate key for the objects.current_version_id pointer, which - // names both the version and the object it must belong to. - "CONSTRAINT uq_object_versions_object_identity UNIQUE (version_id, object_id)", - // A delete marker is exactly a version with no content. - "CONSTRAINT chk_object_versions_delete_marker_shape CHECK ((is_delete_marker = TRUE AND content_id IS NULL AND size = 0 AND e_tag = '' AND content_type = '') OR (is_delete_marker = FALSE AND content_id IS NOT NULL AND e_tag <> ''))", - "CONSTRAINT fk_object_versions_object FOREIGN KEY (object_id, bucket_id, key) REFERENCES objects (id, bucket_id, key) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - foreignKeys: []string{ - "(multipart_upload_id, bucket_id, key) REFERENCES multipart_uploads (upload_id, bucket_id, key) ON UPDATE RESTRICT ON DELETE RESTRICT", - // size repeats the content size so listings need no join; the - // composite key makes that repetition unable to drift. - "(content_id, bucket_id, size) REFERENCES storage_contents (id, bucket_id, content_size) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - }); err != nil { - return err - } - if err := createInitialTable(ctx, db, initialTableSpec{ - name: "object_cache", - model: (*objectCache2026090101)(nil), - constraints: []string{ - "CONSTRAINT chk_object_cache_generation CHECK (cache_presence_generation >= 0 AND cache_operation_generation >= 0)", - }, - foreignKeys: []string{ - "(content_id) REFERENCES storage_contents (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(cache_active_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - }); err != nil { - return err - } - if err := createInitialTable(ctx, db, initialTableSpec{ - name: "object_deletions", - model: (*objectDeletion2026090101)(nil), - constraints: []string{ - "CONSTRAINT chk_object_deletions_identity CHECK (key <> '' AND version_id <> '')", - "CONSTRAINT chk_object_deletions_size CHECK (size >= 0)", - }, - foreignKeys: []string{ - "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(content_id, bucket_id) REFERENCES storage_contents (id, bucket_id) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - }); err != nil { - return err - } - indexes := []initialIndexSpec{ - {name: "idx_object_versions_object_created", table: "object_versions", columns: []string{"object_id", "created_at DESC", "version_id DESC"}}, - // Content reference count and foreign key coverage in one index. - {name: "idx_object_versions_content", table: "object_versions", columns: []string{"content_id"}}, - {name: "idx_object_versions_multipart_upload", table: "object_versions", columns: []string{"multipart_upload_id"}}, - {name: "idx_object_cache_lru", table: "object_cache", columns: []string{"cache_accessed_at", "content_id"}, where: "in_cache = TRUE"}, - {name: "idx_object_cache_active_task", table: "object_cache", columns: []string{"cache_active_task_id"}, where: "cache_active_task_id IS NOT NULL", unique: true}, - {name: "idx_object_deletions_bucket_key_deleted", table: "object_deletions", columns: []string{"bucket_id", "key", "deleted_at"}}, - {name: "idx_object_deletions_content", table: "object_deletions", columns: []string{"content_id"}}, - {name: "idx_object_deletions_bucket_deleted", table: "object_deletions", columns: []string{"bucket_id", "deleted_at DESC", "id DESC"}}, - } - if db.Dialect().Name() != dialect.PG { - indexes = append(indexes, initialIndexSpec{name: "idx_object_versions_bucket_key_created", table: "object_versions", columns: []string{"bucket_id", "key", "created_at DESC", "version_id DESC"}}) - } - if err := createInitialIndexes(ctx, db, indexes...); err != nil { - return err - } - // objects was created before object_versions existed, so PostgreSQL takes - // the pointer constraint here. - return addForwardForeignKey(ctx, db, "objects", objectCurrentVersionForeignKey2026090101()) -} - -// objectCurrentVersionForeignKey2026090101 keeps an object from pointing at a -// version of some other object, and keeps the pointed-at version from being -// deleted before the pointer moves. -func objectCurrentVersionForeignKey2026090101() initialForwardForeignKey { - return initialForwardForeignKey{ - name: "fk_objects_current_version", - definition: "(current_version_id, id) REFERENCES object_versions (version_id, object_id) ON UPDATE RESTRICT ON DELETE RESTRICT", - } -} - -func createPostgresPrefixIndexes(ctx context.Context, db bun.IDB) error { - if db.Dialect().Name() != dialect.PG { - return nil - } - return createInitialIndexes(ctx, db, - // Listing current objects orders by a C-collated key, so the objects - // unique index needs a C-collated companion to drive it. - initialIndexSpec{name: "idx_objects_bucket_key_c", table: "objects", columns: []string{"bucket_id", `(key COLLATE "C")`}}, - initialIndexSpec{name: "idx_object_versions_bucket_key_created", table: "object_versions", columns: []string{"bucket_id", `(key COLLATE "C")`, "created_at DESC", "version_id DESC"}}, - initialIndexSpec{name: "idx_multipart_uploads_bucket_status_key_upload", table: "multipart_uploads", columns: []string{"bucket_id", "status", `(key COLLATE "C")`, "upload_id"}}, - ) -} diff --git a/internal/db/migrations/schema_integrity_test.go b/internal/db/migrations/schema_integrity_test.go index 7f04786..075439e 100644 --- a/internal/db/migrations/schema_integrity_test.go +++ b/internal/db/migrations/schema_integrity_test.go @@ -26,7 +26,7 @@ import ( ) const ( - initialPortableSchemaFingerprint = "82c448b3d6eb4bc627cc6ae913091c21c8b440903dd09a85032a653515a80323" + initialPortableSchemaFingerprint = "21723d7d64f621047bd6fd1879851008fdc18c3fd0b173168e063a0ae3d0c7d6" ) func TestMigrationRegistryStartsWithUniqueOrderedBaseline(t *testing.T) { @@ -162,6 +162,7 @@ func TestInitialSchemaContractSQLite(t *testing.T) { {"storage_data_sets", "ensure_task_id"}, {"storage_replacement_items", "target_data_set_id"}, {"storage_cleanup_copies", "bucket_id"}, + {"storage_cleanup_copies", "checksum"}, {"object_versions", "content_id"}, {"object_cache", "cache_active_task_id"}, {"wallet_operations", "broadcast_attempted_at"}, diff --git a/internal/db/migrations/schema_query_plan_test.go b/internal/db/migrations/schema_query_plan_test.go index 0da3335..bfdac87 100644 --- a/internal/db/migrations/schema_query_plan_test.go +++ b/internal/db/migrations/schema_query_plan_test.go @@ -15,6 +15,7 @@ func TestBaselineRepresentativeQueriesUseSupportingIndexes(t *testing.T) { t.Fatalf("create initial schema: %v", err) } seedObjectPlanBacklog(t, db) + seedStorageCommitPlanBacklog(t, db) seedTaskPlanBacklog(t, db) seedWalletPlanBacklog(t, db) @@ -226,6 +227,97 @@ func seedObjectPlanBacklog(t *testing.T, db *bun.DB) { } } +func seedStorageCommitPlanBacklog(t *testing.T, db *bun.DB) { + t.Helper() + if _, err := db.ExecContext(t.Context(), `INSERT INTO storage_data_sets ( + id, bucket_id, provider_id, copy_index, generation, is_current, + data_set_id, status, created_at, updated_at) + VALUES (1, 1, 'query-plan-provider', 0, 1, TRUE, + 'query-plan-data-set', 'ready', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`); err != nil { + t.Fatalf("seed query-plan storage data set: %v", err) + } + + // A representative history backlog makes the unresolved partial index + // materially cheaper than the broader copy-history index. + var insertCopies, insertHistory, insertUnresolved string + if db.Dialect().Name() == dialect.PG { + insertCopies = `INSERT INTO storage_copies ( + content_id, bucket_id, content_size, storage_data_set_id, copy_index, + provider_id, piece_id, transfer_method, status, commit_ready_at, + created_at, updated_at) + SELECT value, 1, 1, 1, 0, + 'query-plan-provider', 'piece-' || value, 'ingress', 'piece_ready', + '2026-01-01 00:00:00', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM generate_series(1, 512) AS series(value)` + insertHistory = `INSERT INTO storage_commit_attempts ( + attempt_id, content_id, storage_data_set_id, status, release_reason, + resolved_at, created_at, updated_at) + SELECT 'history-' || content_id || '-' || generation, content_id, 1, + 'released', 'query_plan_history', CURRENT_TIMESTAMP, + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM generate_series(1, 512) AS contents(content_id) + CROSS JOIN generate_series(1, 8) AS generations(generation)` + insertUnresolved = `INSERT INTO storage_commit_attempts ( + attempt_id, content_id, storage_data_set_id, status, created_at, updated_at) + SELECT 'unresolved-' || content_id, content_id, 1, 'reserved', + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM generate_series(8, 512, 8) AS contents(content_id)` + } else { + insertCopies = `WITH RECURSIVE contents(content_id) AS ( + SELECT 1 UNION ALL SELECT content_id + 1 FROM contents WHERE content_id < 512 + ) + INSERT INTO storage_copies ( + content_id, bucket_id, content_size, storage_data_set_id, copy_index, + provider_id, piece_id, transfer_method, status, commit_ready_at, + created_at, updated_at) + SELECT content_id, 1, 1, 1, 0, + 'query-plan-provider', 'piece-' || content_id, 'ingress', 'piece_ready', + '2026-01-01 00:00:00', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM contents` + insertHistory = `WITH RECURSIVE + contents(content_id) AS ( + SELECT 1 UNION ALL SELECT content_id + 1 FROM contents WHERE content_id < 512 + ), + generations(generation) AS ( + SELECT 1 UNION ALL SELECT generation + 1 FROM generations WHERE generation < 8 + ) + INSERT INTO storage_commit_attempts ( + attempt_id, content_id, storage_data_set_id, status, release_reason, + resolved_at, created_at, updated_at) + SELECT 'history-' || content_id || '-' || generation, content_id, 1, + 'released', 'query_plan_history', CURRENT_TIMESTAMP, + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM contents CROSS JOIN generations` + insertUnresolved = `WITH RECURSIVE contents(content_id) AS ( + SELECT 8 UNION ALL SELECT content_id + 8 FROM contents WHERE content_id < 512 + ) + INSERT INTO storage_commit_attempts ( + attempt_id, content_id, storage_data_set_id, status, created_at, updated_at) + SELECT 'unresolved-' || content_id, content_id, 1, 'reserved', + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM contents` + } + + statements := []struct { + name string + statement string + }{ + {"storage copies", insertCopies}, + {"storage commit history", insertHistory}, + {"unresolved commit attempts", insertUnresolved}, + } + for _, item := range statements { + if _, err := db.ExecContext(t.Context(), item.statement); err != nil { + t.Fatalf("seed query-plan %s: %v", item.name, err) + } + } + for _, table := range []string{"storage_copies", "storage_commit_attempts"} { + if _, err := db.ExecContext(t.Context(), "ANALYZE "+table); err != nil { + t.Fatalf("analyze query-plan table %s: %v", table, err) + } + } +} + func seedTaskPlanBacklog(t *testing.T, db *bun.DB) { t.Helper() var statements []string diff --git a/internal/db/migrations/schema_storage.go b/internal/db/migrations/schema_storage.go deleted file mode 100644 index 2444308..0000000 --- a/internal/db/migrations/schema_storage.go +++ /dev/null @@ -1,508 +0,0 @@ -package migrations - -import ( - "context" - "time" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" -) - -// storageContent2026090101 is the identity of one bucket-scoped byte payload. -// Durability, pipeline state and ingress progress are deliberately absent: the -// first two are functions of the copy rows and stay queries, while progress -// belongs to the concrete ingress transfer that produced it. -type storageContent2026090101 struct { - bun.BaseModel `bun:"table:storage_contents"` - - ID int64 `bun:",pk,autoincrement,identity"` - BucketID int64 `bun:",notnull"` - Checksum string `bun:"type:text,notnull"` - ContentSize int64 `bun:",notnull"` - PieceCID *string `bun:"type:text"` - RequestedCopies int `bun:"type:integer,notnull"` - ErrorMessage *string `bun:"type:text"` - AcceptedAt *time.Time - CleanupGeneration int64 `bun:",notnull,default:0"` - CleanupTaskID *int64 - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -type storageDataSet2026090101 struct { - bun.BaseModel `bun:"table:storage_data_sets"` - - ID int64 `bun:",pk,autoincrement,identity"` - BucketID int64 `bun:",notnull"` - ProviderID string `bun:"type:text,notnull"` - CopyIndex int `bun:"type:integer,notnull"` - Generation int64 `bun:",notnull,default:1"` - IsCurrent bool `bun:",notnull"` - DataSetID *string `bun:"type:text"` - ClientDataSetID *string `bun:"type:text"` - Status string `bun:"type:text,notnull,default:'pending'"` - CreateTransactionID *string `bun:"type:text"` - CreateStatusURL *string `bun:"type:text"` - CreatedByContentID *int64 - LastUsedContentID *int64 - LastError *string `bun:"type:text"` - EnsureTaskID *int64 - RetirementGeneration int64 `bun:",notnull,default:0"` - RetirementTaskID *int64 - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -type storageCopy2026090101 struct { - bun.BaseModel `bun:"table:storage_copies"` - - ID int64 `bun:",pk,autoincrement,identity"` - ContentID int64 `bun:",notnull"` - BucketID int64 `bun:",notnull"` - // ContentSize repeats storage_contents.content_size so the ingress bound - // stays a local CHECK. A composite foreign key keeps the copy from - // drifting away from the content it transfers. - ContentSize int64 `bun:",notnull"` - StorageDataSetID int64 `bun:",notnull"` - CopyIndex int `bun:"type:integer,notnull"` - ProviderID string `bun:"type:text,notnull"` - PieceID *string `bun:"type:text"` - TransferMethod string `bun:"type:text,notnull"` - Status string `bun:"type:text,notnull,default:'pending'"` - RetrievalURL *string `bun:"type:text"` - CommitExtraDataHex *string `bun:"type:text"` - CommitReadyAt *time.Time - // The confirmed commit attempt this copy projects. Its status is repeated so - // a composite foreign key can require the referenced attempt to be - // confirmed, which is what keeps the projection from drifting. - ConfirmedAttemptID *string `bun:"type:text"` - ConfirmedAttemptStatus *string `bun:"type:text"` - IngressBytesTransferred int64 `bun:",notnull,default:0"` - IngressStoreAttempt int `bun:"type:integer,notnull,default:0"` - ProgressUpdatedAt *time.Time - WorkGeneration int64 `bun:",notnull,default:0"` - ActiveTaskID *int64 - LastError *string `bun:"type:text"` - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -type storageCommitAttempt2026090101 struct { - bun.BaseModel `bun:"table:storage_commit_attempts"` - - AttemptID string `bun:"type:text,pk"` - ContentID int64 `bun:",notnull"` - StorageDataSetID int64 `bun:",notnull"` - Status string `bun:"type:text,notnull,default:'reserved'"` - ExtraDataHex *string `bun:"type:text"` - TransactionID *string `bun:"type:text"` - SubmissionJSON *string `bun:"type:text"` - ConfirmedTransactionID *string `bun:"type:text"` - AttentionCode *string `bun:"type:text"` - AttentionAt *time.Time - ReleaseReason *string `bun:"type:text"` - LastError *string `bun:"type:text"` - AttemptedAt *time.Time - ResolvedAt *time.Time - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -// storagePullAttempt2026090101 is the ledger of provider-side copy requests. -// Every source column is NOT NULL, which replaces the all-or-nothing check that -// used to guard five nullable columns on the copy. -type storagePullAttempt2026090101 struct { - bun.BaseModel `bun:"table:storage_pull_attempts"` - - AttemptID string `bun:"type:text,pk"` - ContentID int64 `bun:",notnull"` - StorageDataSetID int64 `bun:",notnull"` - Status string `bun:"type:text,notnull"` - SourceProviderID string `bun:"type:text,notnull"` - SourceDataSetID string `bun:"type:text,notnull"` - SourcePieceID string `bun:"type:text,notnull"` - SourcePieceCID string `bun:"type:text,notnull"` - SourceRetrievalURL string `bun:"type:text,notnull"` - LastError *string `bun:"type:text"` - AttemptedAt time.Time `bun:",notnull"` - ResolvedAt *time.Time - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -type storageReplacement2026090101 struct { - bun.BaseModel `bun:"table:storage_replacements"` - - ID int64 `bun:",pk,autoincrement,identity"` - BucketID int64 `bun:",notnull"` - CopyIndex int `bun:"type:integer,notnull"` - SourceDataSetID int64 `bun:",notnull"` - TargetDataSetID int64 `bun:",notnull"` - SelectionMode string `bun:"type:text,notnull"` - RequestedProviderID *string `bun:"type:text"` - ClientRequestID string `bun:"type:text,notnull"` - Status string `bun:"type:text,notnull"` - WaitReason *string `bun:"type:text"` - FailureReason *string `bun:"type:text"` - LastError *string `bun:"type:text"` - ItemsTotal int `bun:"type:integer,notnull,default:0"` - ItemsCopied int `bun:"type:integer,notnull,default:0"` - SeedCursorContentID int64 `bun:",notnull,default:0"` - SeedingComplete bool `bun:",notnull,default:false"` - TaskGeneration int64 `bun:",notnull,default:1"` - TaskID *int64 - SupersededByID *int64 - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -// storageDataSetTermination2026090101 records one data set's end of term. A -// replacement terminates the source it replaces and, once superseded, the -// target it abandoned; a third kind of termination adds a row here instead of -// another repeated column group on storage_replacements. -type storageDataSetTermination2026090101 struct { - bun.BaseModel `bun:"table:storage_data_set_terminations"` - - ID int64 `bun:",pk,autoincrement,identity"` - ReplacementID int64 `bun:",notnull"` - Role string `bun:"type:text,notnull"` - // Exactly one of the data set columns is set, chosen by role. Each carries a - // composite foreign key back to the matching column on the replacement, so a - // row cannot name a data set the replacement never held in that role. - SourceDataSetID *int64 - AbandonedTargetDataSetID *int64 - TxHash *string `bun:"type:text"` - Epoch int64 `bun:",notnull"` - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -func storageDataSetTerminationTable2026090101() initialTableSpec { - return initialTableSpec{ - name: "storage_data_set_terminations", - model: (*storageDataSetTermination2026090101)(nil), - constraints: []string{ - "CONSTRAINT uq_storage_data_set_terminations_role UNIQUE (replacement_id, role)", - "CONSTRAINT chk_storage_data_set_terminations_role CHECK (role IN ('source', 'abandoned_target'))", - "CONSTRAINT chk_storage_data_set_terminations_subject CHECK ((role = 'source') = (source_data_set_id IS NOT NULL) AND (role = 'abandoned_target') = (abandoned_target_data_set_id IS NOT NULL))", - "CONSTRAINT chk_storage_data_set_terminations_epoch CHECK (epoch >= 0)", - "CONSTRAINT chk_storage_data_set_terminations_tx_hash CHECK (tx_hash IS NULL OR tx_hash <> '')", - }, - foreignKeys: []string{ - "(replacement_id) REFERENCES storage_replacements (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(replacement_id, source_data_set_id) REFERENCES storage_replacements (id, source_data_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(replacement_id, abandoned_target_data_set_id) REFERENCES storage_replacements (id, target_data_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - } -} - -type storageReplacementItem2026090101 struct { - bun.BaseModel `bun:"table:storage_replacement_items"` - - ID int64 `bun:",pk,autoincrement,identity"` - ReplacementID int64 `bun:",notnull"` - ContentID int64 `bun:",notnull"` - // TargetDataSetID is written when the item is seeded, together with the - // pending copy it names. A nullable column would make the composite foreign - // key below skip validation entirely whenever it was unset. - TargetDataSetID int64 `bun:",notnull"` - Status string `bun:"type:text,notnull,default:'pending'"` - LastError *string `bun:"type:text"` - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -type storageCleanupCopy2026090101 struct { - bun.BaseModel `bun:"table:storage_cleanup_copies"` - - ID int64 `bun:",pk,autoincrement,identity"` - ContentID int64 `bun:",notnull"` - BucketID int64 `bun:",notnull"` - CopyIndex int `bun:"type:integer,notnull"` - ProviderID string `bun:"type:text,notnull"` - StorageDataSetID int64 `bun:",notnull"` - DataSetID *string `bun:"type:text"` - ClientDataSetID *string `bun:"type:text"` - PieceID string `bun:"type:text,notnull"` - PieceCID string `bun:"type:text,notnull"` - RetrievalURL *string `bun:"type:text"` - Status string `bun:"type:text,notnull,default:'pending'"` - DeleteTxHash *string `bun:"type:text"` - LastError *string `bun:"type:text"` - ScheduledAt *time.Time - RemovedAt *time.Time - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -func createStorageSchema(ctx context.Context, db bun.IDB) error { - checksumCheck := "length(checksum) = 64 AND checksum NOT GLOB '*[^0-9a-f]*'" - if db.Dialect().Name() == dialect.PG { - checksumCheck = "checksum ~ '^[0-9a-f]{64}$'" - } - tables := []initialTableSpec{ - { - name: "storage_contents", - model: (*storageContent2026090101)(nil), - constraints: []string{ - // Content dedup is a unique-key lookup, not an index scan. - "CONSTRAINT uq_storage_contents_bytes UNIQUE (bucket_id, checksum, content_size)", - // Candidate keys for the composite foreign keys that pin - // denormalized identity on object_versions and storage_copies. - "CONSTRAINT uq_storage_contents_id_bucket UNIQUE (id, bucket_id)", - "CONSTRAINT uq_storage_contents_addr UNIQUE (id, bucket_id, content_size)", - "CONSTRAINT chk_storage_contents_identity CHECK ((" + checksumCheck + ") AND (piece_cid IS NULL OR piece_cid <> ''))", - "CONSTRAINT chk_storage_contents_content_size CHECK (content_size >= 0)", - "CONSTRAINT chk_storage_contents_requested_copies CHECK (requested_copies BETWEEN 1 AND 8)", - "CONSTRAINT chk_storage_contents_cleanup_generation CHECK (cleanup_generation >= 0)", - }, - foreignKeys: []string{ - "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(cleanup_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - }, - { - name: "storage_data_sets", - model: (*storageDataSet2026090101)(nil), - constraints: []string{ - "CONSTRAINT uq_storage_data_sets_id_bucket_slot UNIQUE (id, bucket_id, copy_index)", - "CONSTRAINT uq_storage_data_sets_identity UNIQUE (id, bucket_id, copy_index, provider_id)", - "CONSTRAINT chk_storage_data_sets_identity CHECK (provider_id <> '' AND (data_set_id IS NULL OR data_set_id <> '') AND (client_data_set_id IS NULL OR client_data_set_id <> '') AND (create_transaction_id IS NULL OR create_transaction_id <> '') AND (create_status_url IS NULL OR create_status_url <> ''))", - // The replica slot is a row, so the index is a foreign key rather than a range check. - "CONSTRAINT fk_storage_data_sets_replica_slot FOREIGN KEY (bucket_id, copy_index) REFERENCES bucket_replica_slots (bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", - "CONSTRAINT chk_storage_data_sets_generation CHECK (generation >= 1 AND retirement_generation >= 0)", - "CONSTRAINT chk_storage_data_sets_status CHECK (status IN ('pending', 'creating', 'ready', 'failed', 'draining', 'retired'))", - "CONSTRAINT chk_storage_data_sets_ready_identity CHECK (status <> 'ready' OR data_set_id IS NOT NULL)", - "CONSTRAINT chk_storage_data_sets_current_shape CHECK (status NOT IN ('failed', 'draining', 'retired') OR is_current = FALSE)", - }, - foreignKeys: []string{ - "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(created_by_content_id, bucket_id) REFERENCES storage_contents (id, bucket_id) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(last_used_content_id, bucket_id) REFERENCES storage_contents (id, bucket_id) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(ensure_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(retirement_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - }, - { - name: "storage_copies", - model: (*storageCopy2026090101)(nil), - constraints: []string{ - "CONSTRAINT uq_storage_copies_content_data_set UNIQUE (content_id, storage_data_set_id)", - // The replica slot is a row, so the index is a foreign key rather than a range check. - "CONSTRAINT fk_storage_copies_replica_slot FOREIGN KEY (bucket_id, copy_index) REFERENCES bucket_replica_slots (bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", - "CONSTRAINT chk_storage_copies_work_generation CHECK (work_generation >= 0)", - "CONSTRAINT chk_storage_copies_status CHECK (status IN ('pending', 'piece_ready', 'committing', 'committed', 'failed'))", - // The projected status is pinned to a literal so the composite - // foreign key below can only ever reach a confirmed attempt. - "CONSTRAINT chk_storage_copies_confirmed_attempt_status CHECK (confirmed_attempt_status IS NULL OR confirmed_attempt_status = 'confirmed')", - // Committed and "has confirmed evidence" are the same fact. The - // foreign key alone would still allow a committed copy with no - // evidence at all, so both directions are stated here. - "CONSTRAINT chk_storage_copies_committed_evidence CHECK ((status = 'committed') = (confirmed_attempt_id IS NOT NULL))", - "CONSTRAINT chk_storage_copies_confirmed_attempt_shape CHECK ((confirmed_attempt_id IS NULL AND confirmed_attempt_status IS NULL) OR (confirmed_attempt_id IS NOT NULL AND confirmed_attempt_id <> '' AND confirmed_attempt_status IS NOT NULL))", - "CONSTRAINT chk_storage_copies_transfer_method CHECK (transfer_method IN ('ingress', 'peer_pull'))", - "CONSTRAINT chk_storage_copies_optional_identity CHECK (provider_id <> '' AND (piece_id IS NULL OR piece_id <> '') AND (retrieval_url IS NULL OR retrieval_url <> '') AND (commit_extra_data_hex IS NULL OR commit_extra_data_hex <> ''))", - "CONSTRAINT chk_storage_copies_committed_shape CHECK (status <> 'committed' OR (piece_id IS NOT NULL AND piece_id <> '' AND retrieval_url IS NOT NULL AND retrieval_url <> ''))", - "CONSTRAINT chk_storage_copies_commit_ready CHECK (commit_ready_at IS NULL OR status IN ('piece_ready', 'committing', 'committed'))", - "CONSTRAINT chk_storage_copies_content_size CHECK (content_size >= 0)", - // Ingress progress belongs to the transfer that produces it, so - // only an ingress copy may carry it. - "CONSTRAINT chk_storage_copies_ingress_progress CHECK (transfer_method = 'ingress' OR (ingress_bytes_transferred = 0 AND ingress_store_attempt = 0 AND progress_updated_at IS NULL))", - "CONSTRAINT chk_storage_copies_ingress_bytes CHECK (ingress_bytes_transferred >= 0 AND ingress_bytes_transferred <= content_size)", - "CONSTRAINT chk_storage_copies_ingress_attempt CHECK (ingress_store_attempt >= 0)", - }, - foreignKeys: []string{ - "(content_id, bucket_id, content_size) REFERENCES storage_contents (id, bucket_id, content_size) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(storage_data_set_id, bucket_id, copy_index, provider_id) REFERENCES storage_data_sets (id, bucket_id, copy_index, provider_id) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(active_task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - forwardForeignKeys: []initialForwardForeignKey{storageCopyConfirmedAttemptForeignKey2026090101()}, - }, - storageCommitAttemptTable2026090101(), - storagePullAttemptTable2026090101(), - storageReplacementTable2026090101(), - storageDataSetTerminationTable2026090101(), - storageReplacementItemTable2026090101(), - storageCleanupCopyTable2026090101(), - } - for _, table := range tables { - if err := createInitialTable(ctx, db, table); err != nil { - return err - } - } - if err := createInitialIndexes(ctx, db, storageIndexes2026090101()...); err != nil { - return err - } - // storage_copies was created before storage_commit_attempts existed, so - // PostgreSQL takes the projection constraint here. - return addForwardForeignKey(ctx, db, "storage_copies", storageCopyConfirmedAttemptForeignKey2026090101()) -} - -// storageCopyConfirmedAttemptForeignKey2026090101 welds the copy's committed -// state to the ledger row that proves it. storage_copies is created before -// storage_commit_attempts, so the constraint is forward-declared. -func storageCopyConfirmedAttemptForeignKey2026090101() initialForwardForeignKey { - return initialForwardForeignKey{ - name: "fk_storage_copies_confirmed_attempt", - definition: "(confirmed_attempt_id, confirmed_attempt_status) REFERENCES storage_commit_attempts (attempt_id, status) ON UPDATE RESTRICT ON DELETE RESTRICT", - } -} - -func storageCommitAttemptTable2026090101() initialTableSpec { - return initialTableSpec{ - name: "storage_commit_attempts", - jsonColumns: initialJSONColumns("storage_commit_attempts"), - model: (*storageCommitAttempt2026090101)(nil), - constraints: []string{ - "CONSTRAINT chk_storage_commit_attempts_identity CHECK (attempt_id <> '' AND (extra_data_hex IS NULL OR extra_data_hex <> '') AND (transaction_id IS NULL OR transaction_id <> '') AND (submission_json IS NULL OR submission_json <> '') AND (confirmed_transaction_id IS NULL OR confirmed_transaction_id <> '') AND (attention_code IS NULL OR attention_code <> '') AND (release_reason IS NULL OR release_reason <> ''))", - "CONSTRAINT chk_storage_commit_attempts_status CHECK (status IN ('reserved', 'attempted', 'confirmed', 'released', 'rejected'))", - // Candidate key for the copy's confirmed-attempt projection. - "CONSTRAINT uq_storage_commit_attempts_status UNIQUE (attempt_id, status)", - "CONSTRAINT chk_storage_commit_attempts_resolution CHECK ((status IN ('reserved', 'attempted') AND resolved_at IS NULL) OR (status IN ('confirmed', 'released', 'rejected') AND resolved_at IS NOT NULL))", - `CONSTRAINT chk_storage_commit_attempts_evidence_shape CHECK ( - (status = 'reserved' AND attempted_at IS NULL AND extra_data_hex IS NULL AND transaction_id IS NULL AND submission_json IS NULL AND confirmed_transaction_id IS NULL AND attention_code IS NULL AND attention_at IS NULL AND last_error IS NULL) - OR (status = 'attempted' AND attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL AND confirmed_transaction_id IS NULL AND last_error IS NULL) - OR (status = 'confirmed' AND attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL AND transaction_id IS NOT NULL AND confirmed_transaction_id IS NOT NULL AND last_error IS NULL) - OR (status = 'released' AND confirmed_transaction_id IS NULL AND last_error IS NULL AND ((attempted_at IS NULL AND extra_data_hex IS NULL AND transaction_id IS NULL AND submission_json IS NULL AND attention_code IS NULL AND attention_at IS NULL) OR (attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL))) - OR (status = 'rejected' AND attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL AND confirmed_transaction_id IS NULL AND last_error IS NOT NULL AND last_error <> '') - )`, - "CONSTRAINT chk_storage_commit_attempts_submission CHECK (submission_json IS NULL OR transaction_id IS NOT NULL)", - "CONSTRAINT chk_storage_commit_attempts_attention CHECK ((attention_code IS NULL AND attention_at IS NULL) OR (attention_code IS NOT NULL AND attention_at IS NOT NULL AND attempted_at IS NOT NULL))", - "CONSTRAINT chk_storage_commit_attempts_release CHECK ((status = 'released' AND release_reason IS NOT NULL) OR (status <> 'released' AND release_reason IS NULL))", - }, - foreignKeys: []string{ - "(content_id, storage_data_set_id) REFERENCES storage_copies (content_id, storage_data_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - } -} - -func storagePullAttemptTable2026090101() initialTableSpec { - return initialTableSpec{ - name: "storage_pull_attempts", - model: (*storagePullAttempt2026090101)(nil), - constraints: []string{ - "CONSTRAINT chk_storage_pull_attempts_identity CHECK (attempt_id <> '' AND source_provider_id <> '' AND source_data_set_id <> '' AND source_piece_id <> '' AND source_piece_cid <> '' AND source_retrieval_url <> '')", - "CONSTRAINT chk_storage_pull_attempts_status CHECK (status IN ('attempted', 'abandoned'))", - // An error only makes sense on a request nobody will observe again; - // reopening a copy carries no error string. - "CONSTRAINT chk_storage_pull_attempts_error CHECK (last_error IS NULL OR (status = 'abandoned' AND last_error <> ''))", - "CONSTRAINT chk_storage_pull_attempts_abandoned CHECK (status <> 'abandoned' OR resolved_at IS NOT NULL)", - }, - foreignKeys: []string{ - "(content_id, storage_data_set_id) REFERENCES storage_copies (content_id, storage_data_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - } -} - -func storageReplacementTable2026090101() initialTableSpec { - return initialTableSpec{ - name: "storage_replacements", - model: (*storageReplacement2026090101)(nil), - constraints: []string{ - "CONSTRAINT uq_storage_replacements_id_source UNIQUE (id, source_data_set_id)", - "CONSTRAINT uq_storage_replacements_id_target UNIQUE (id, target_data_set_id)", - "CONSTRAINT chk_storage_replacements_identity CHECK (client_request_id <> '' AND (requested_provider_id IS NULL OR requested_provider_id <> ''))", - // The replica slot is a row, so the index is a foreign key rather than a range check. - "CONSTRAINT fk_storage_replacements_replica_slot FOREIGN KEY (bucket_id, copy_index) REFERENCES bucket_replica_slots (bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", - "CONSTRAINT chk_storage_replacements_selection_mode CHECK (selection_mode IN ('automatic', 'manual'))", - "CONSTRAINT chk_storage_replacements_status CHECK (status IN ('preparing_target', 'migrating', 'waiting', 'retiring', 'cleanup_attention', 'failed', 'completed', 'superseded'))", - "CONSTRAINT chk_storage_replacements_wait_reason CHECK (wait_reason IS NULL OR wait_reason <> '')", - "CONSTRAINT chk_storage_replacements_failure_reason CHECK (failure_reason IS NULL OR failure_reason <> '')", - "CONSTRAINT chk_storage_replacements_client_request_id CHECK (length(client_request_id) BETWEEN 1 AND 128)", - "CONSTRAINT chk_storage_replacements_distinct_data_sets CHECK (source_data_set_id <> target_data_set_id)", - "CONSTRAINT chk_storage_replacements_items CHECK (items_total >= 0 AND items_copied >= 0 AND items_copied <= items_total)", - "CONSTRAINT chk_storage_replacements_generation CHECK (task_generation >= 1)", - }, - foreignKeys: []string{ - "(bucket_id) REFERENCES buckets (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(source_data_set_id, bucket_id, copy_index) REFERENCES storage_data_sets (id, bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(target_data_set_id, bucket_id, copy_index) REFERENCES storage_data_sets (id, bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(superseded_by_id) REFERENCES storage_replacements (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - } -} - -func storageReplacementItemTable2026090101() initialTableSpec { - return initialTableSpec{ - name: "storage_replacement_items", - model: (*storageReplacementItem2026090101)(nil), - constraints: []string{ - "CONSTRAINT chk_storage_replacement_items_status CHECK (status IN ('pending', 'copied', 'cancelled', 'attention'))", - "CONSTRAINT uq_storage_replacement_items_content UNIQUE (replacement_id, content_id)", - }, - foreignKeys: []string{ - "(replacement_id) REFERENCES storage_replacements (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(content_id) REFERENCES storage_contents (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(replacement_id, target_data_set_id) REFERENCES storage_replacements (id, target_data_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(content_id, target_data_set_id) REFERENCES storage_copies (content_id, storage_data_set_id) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - } -} - -func storageCleanupCopyTable2026090101() initialTableSpec { - return initialTableSpec{ - name: "storage_cleanup_copies", - model: (*storageCleanupCopy2026090101)(nil), - constraints: []string{ - "CONSTRAINT chk_storage_cleanup_copies_identity CHECK (provider_id <> '' AND piece_id <> '' AND piece_cid <> '' AND (data_set_id IS NULL OR data_set_id <> '') AND (client_data_set_id IS NULL OR client_data_set_id <> '') AND (delete_tx_hash IS NULL OR delete_tx_hash <> ''))", - // The replica slot is a row, so the index is a foreign key rather than a range check. - "CONSTRAINT fk_storage_cleanup_copies_replica_slot FOREIGN KEY (bucket_id, copy_index) REFERENCES bucket_replica_slots (bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", - "CONSTRAINT chk_storage_cleanup_copies_status CHECK (status IN ('pending', 'delete_scheduled', 'removed', 'failed', 'unsupported'))", - "CONSTRAINT chk_storage_cleanup_copies_delete_scheduled CHECK (status <> 'delete_scheduled' OR (delete_tx_hash IS NOT NULL AND scheduled_at IS NOT NULL))", - "CONSTRAINT uq_storage_cleanup_copies_physical UNIQUE (content_id, storage_data_set_id, piece_id)", - }, - foreignKeys: []string{ - "(content_id, bucket_id) REFERENCES storage_contents (id, bucket_id) ON UPDATE RESTRICT ON DELETE RESTRICT", - "(storage_data_set_id, bucket_id, copy_index, provider_id) REFERENCES storage_data_sets (id, bucket_id, copy_index, provider_id) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - } -} - -func storageIndexes2026090101() []initialIndexSpec { - return []initialIndexSpec{ - {name: "idx_storage_contents_bucket_id", table: "storage_contents", columns: []string{"bucket_id", "id"}}, - {name: "idx_storage_contents_cleanup_task", table: "storage_contents", columns: []string{"cleanup_task_id"}, where: "cleanup_task_id IS NOT NULL", unique: true}, - {name: "idx_storage_data_sets_provider_data_set", table: "storage_data_sets", columns: []string{"provider_id", "data_set_id"}, where: "data_set_id IS NOT NULL", unique: true}, - {name: "idx_storage_data_sets_bucket_copy_current", table: "storage_data_sets", columns: []string{"bucket_id", "copy_index"}, where: "is_current = TRUE", unique: true}, - {name: "idx_storage_data_sets_bucket_copy_generation", table: "storage_data_sets", columns: []string{"bucket_id", "copy_index", "generation"}, unique: true}, - {name: "idx_storage_data_sets_bucket_provider_active", table: "storage_data_sets", columns: []string{"bucket_id", "provider_id"}, where: "status <> 'retired'", unique: true}, - {name: "idx_storage_data_sets_created_by_content", table: "storage_data_sets", columns: []string{"created_by_content_id"}}, - {name: "idx_storage_data_sets_last_used_content", table: "storage_data_sets", columns: []string{"last_used_content_id"}}, - {name: "idx_storage_data_sets_ensure_task", table: "storage_data_sets", columns: []string{"ensure_task_id"}, where: "ensure_task_id IS NOT NULL", unique: true}, - {name: "idx_storage_data_sets_retirement_task", table: "storage_data_sets", columns: []string{"retirement_task_id"}, where: "retirement_task_id IS NOT NULL", unique: true}, - {name: "idx_storage_copies_content_slot", table: "storage_copies", columns: []string{"content_id", "copy_index"}}, - {name: "idx_storage_copies_data_set_identity", table: "storage_copies", columns: []string{"storage_data_set_id", "bucket_id", "copy_index", "provider_id"}}, - {name: "idx_storage_copies_content_transfer_method_index", table: "storage_copies", columns: []string{"content_id", "transfer_method", "copy_index"}}, - {name: "idx_storage_copies_ingress_content", table: "storage_copies", columns: []string{"content_id"}, where: "transfer_method = 'ingress'", unique: true}, - {name: "idx_storage_copies_status_data_set_content", table: "storage_copies", columns: []string{"status", "storage_data_set_id", "content_id"}}, - {name: "idx_storage_copies_status_piece_identity_content", table: "storage_copies", columns: []string{"status", "provider_id", "piece_id", "content_id"}}, - {name: "idx_storage_copies_commit_ready", table: "storage_copies", columns: []string{"storage_data_set_id", "commit_ready_at", "id"}, where: "status = 'piece_ready' AND commit_ready_at IS NOT NULL"}, - {name: "idx_storage_copies_active_task", table: "storage_copies", columns: []string{"active_task_id"}, where: "active_task_id IS NOT NULL", unique: true}, - {name: "idx_storage_commit_attempts_unresolved_copy", table: "storage_commit_attempts", columns: []string{"content_id", "storage_data_set_id"}, where: "resolved_at IS NULL", unique: true}, - {name: "idx_storage_commit_attempts_unresolved_data_set", table: "storage_commit_attempts", columns: []string{"storage_data_set_id", "created_at", "attempt_id"}, where: "resolved_at IS NULL"}, - {name: "idx_storage_commit_attempts_copy_history", table: "storage_commit_attempts", columns: []string{"content_id", "storage_data_set_id", "created_at DESC", "attempt_id"}}, - {name: "idx_storage_cleanup_copies_replica_slot", table: "storage_cleanup_copies", columns: []string{"bucket_id", "copy_index"}}, - {name: "idx_storage_copies_confirmed_attempt", table: "storage_copies", columns: []string{"confirmed_attempt_id", "confirmed_attempt_status"}, where: "confirmed_attempt_id IS NOT NULL"}, - {name: "idx_storage_copies_replica_slot", table: "storage_copies", columns: []string{"bucket_id", "copy_index"}}, - // One unresolved attempt per copy: a second source can only be tried - // after the first is abandoned. - {name: "idx_storage_pull_attempts_unresolved_copy", table: "storage_pull_attempts", columns: []string{"content_id", "storage_data_set_id"}, where: "resolved_at IS NULL", unique: true}, - {name: "idx_storage_pull_attempts_copy_history", table: "storage_pull_attempts", columns: []string{"content_id", "storage_data_set_id", "created_at DESC", "attempt_id"}}, - {name: "idx_storage_replacements_active_source", table: "storage_replacements", columns: []string{"source_data_set_id"}, where: "status NOT IN ('completed', 'superseded')", unique: true}, - {name: "idx_storage_replacements_active_target", table: "storage_replacements", columns: []string{"target_data_set_id"}, where: "status NOT IN ('completed', 'superseded')", unique: true}, - {name: "idx_storage_replacements_active_bucket_slot", table: "storage_replacements", columns: []string{"bucket_id", "copy_index"}, where: "status NOT IN ('completed', 'superseded')", unique: true}, - {name: "idx_storage_replacements_source_identity", table: "storage_replacements", columns: []string{"source_data_set_id", "bucket_id", "copy_index"}}, - {name: "idx_storage_replacements_target_identity", table: "storage_replacements", columns: []string{"target_data_set_id", "bucket_id", "copy_index"}}, - {name: "idx_storage_replacements_superseded_by", table: "storage_replacements", columns: []string{"superseded_by_id"}, where: "superseded_by_id IS NOT NULL"}, - {name: "idx_storage_replacements_bucket_slot", table: "storage_replacements", columns: []string{"bucket_id", "copy_index", "id"}}, - {name: "idx_storage_replacements_bucket_request", table: "storage_replacements", columns: []string{"bucket_id", "client_request_id"}, unique: true}, - {name: "idx_storage_replacements_task", table: "storage_replacements", columns: []string{"task_id"}, where: "task_id IS NOT NULL", unique: true}, - {name: "idx_storage_replacement_items_state", table: "storage_replacement_items", columns: []string{"replacement_id", "status", "id"}}, - {name: "idx_storage_replacement_items_content_id", table: "storage_replacement_items", columns: []string{"content_id"}}, - {name: "idx_storage_replacement_items_target_copy", table: "storage_replacement_items", columns: []string{"content_id", "target_data_set_id"}}, - {name: "idx_storage_cleanup_copies_content_status", table: "storage_cleanup_copies", columns: []string{"content_id", "status", "id"}}, - {name: "idx_storage_cleanup_copies_data_set_identity", table: "storage_cleanup_copies", columns: []string{"storage_data_set_id", "bucket_id", "copy_index", "provider_id"}}, - {name: "idx_storage_cleanup_copies_status_scheduled", table: "storage_cleanup_copies", columns: []string{"status", "scheduled_at", "id"}}, - } -} diff --git a/internal/db/migrations/schema_wallet_observability.go b/internal/db/migrations/schema_wallet_observability.go deleted file mode 100644 index 25116ab..0000000 --- a/internal/db/migrations/schema_wallet_observability.go +++ /dev/null @@ -1,148 +0,0 @@ -package migrations - -import ( - "context" - "encoding/json" - "time" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" -) - -type walletOperation2026090101 struct { - bun.BaseModel `bun:"table:wallet_operations"` - - ID int64 `bun:",pk,autoincrement,identity"` - Type string `bun:"type:text,notnull"` - ClientRequestID string `bun:"type:text,notnull"` - Amount string `bun:"type:text,notnull"` - Status string `bun:"type:text,notnull,default:'pending'"` - TxHash *string `bun:"type:text"` - LastError *string `bun:"type:text"` - BroadcastAttemptedAt *time.Time - TaskID *int64 - StartedAt *time.Time - SubmittedAt *time.Time - CompletedAt *time.Time - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -func createWalletSchema(ctx context.Context, db bun.IDB) error { - amountCheck := `((type = 'approve' AND amount = '0') OR (type IN ('fund', 'withdraw') AND amount GLOB '[1-9]*' AND amount NOT GLOB '*[^0-9]*'))` - if db.Dialect().Name() == dialect.PG { - amountCheck = `((type = 'approve' AND amount = '0') OR (type IN ('fund', 'withdraw') AND amount ~ '^[1-9][0-9]*$'))` - } - if err := createInitialTable(ctx, db, initialTableSpec{ - name: "wallet_operations", - model: (*walletOperation2026090101)(nil), - constraints: []string{ - "CONSTRAINT chk_wallet_operations_identity CHECK (client_request_id <> '' AND amount <> '' AND (tx_hash IS NULL OR tx_hash <> ''))", - "CONSTRAINT chk_wallet_operations_type CHECK (type IN ('fund', 'withdraw', 'approve'))", - "CONSTRAINT chk_wallet_operations_status CHECK (status IN ('pending', 'submitted', 'confirmed', 'failed', 'unknown'))", - "CONSTRAINT chk_wallet_operations_submitted_shape CHECK (status <> 'submitted' OR (tx_hash IS NOT NULL AND submitted_at IS NOT NULL))", - "CONSTRAINT chk_wallet_operations_amount CHECK (" + amountCheck + ")", - // A uint256 in base 10 is at most 78 digits. The column stays text - // because the value is returned and compared verbatim. - "CONSTRAINT chk_wallet_operations_amount_length CHECK (length(amount) BETWEEN 1 AND 78)", - }, - foreignKeys: []string{ - "(task_id) REFERENCES tasks (id) ON UPDATE RESTRICT ON DELETE RESTRICT", - }, - }); err != nil { - return err - } - return createInitialIndexes(ctx, db, - initialIndexSpec{name: "idx_wallet_operations_request", table: "wallet_operations", columns: []string{"type", "client_request_id"}, unique: true}, - initialIndexSpec{name: "idx_wallet_operations_status_created", table: "wallet_operations", columns: []string{"status", "created_at", "id"}}, - initialIndexSpec{name: "idx_wallet_operations_recent", table: "wallet_operations", columns: []string{"created_at DESC", "id DESC"}}, - initialIndexSpec{name: "idx_wallet_operations_task", table: "wallet_operations", columns: []string{"task_id"}, where: "task_id IS NOT NULL", unique: true}, - ) -} - -type observabilityCollectionState2026090101 struct { - bun.BaseModel `bun:"table:observability_collection_states"` - - CollectionType string `bun:"type:text,pk"` - LastCheckedAt time.Time `bun:",notnull"` - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} - -type observabilityProviderState2026090101 struct { - bun.BaseModel `bun:"table:observability_provider_states"` - - ProviderID string `bun:"type:text,pk"` - Status string `bun:"type:text,notnull"` - ReasonCodes json.RawMessage `bun:"type:jsonb,notnull"` - Active *bool - HasPDP *bool - ServiceURL *string `bun:"type:text"` - HealthStatus *string `bun:"type:text"` - LastCheckedAt time.Time `bun:",notnull"` - LastError *string `bun:"type:text"` - Evidence json.RawMessage `bun:"evidence_json,type:jsonb,notnull"` -} - -type observabilityDataSetState2026090101 struct { - bun.BaseModel `bun:"table:observability_data_set_states"` - - LocalDataSetID int64 `bun:",pk"` - BucketID int64 `bun:",notnull"` - CopyIndex int `bun:"type:integer,notnull"` - ProviderID string `bun:"type:text,notnull"` - ChainDataSetID *string `bun:"type:text"` - ClientDataSetID *string `bun:"type:text"` - // The bucket's name and the data set's own status are a join away and can - // disagree with the authorities that own them, so neither is copied here. - Status string `bun:"type:text,notnull"` - ReasonCodes json.RawMessage `bun:"type:jsonb,notnull"` - ActivePieceCount *int64 - LastCheckedAt time.Time `bun:",notnull"` - LastError *string `bun:"type:text"` - Evidence json.RawMessage `bun:"evidence_json,type:jsonb,notnull"` -} - -func createObservabilitySchema(ctx context.Context, db bun.IDB) error { - tables := []initialTableSpec{ - { - name: "observability_collection_states", - model: (*observabilityCollectionState2026090101)(nil), - constraints: []string{ - "CONSTRAINT chk_observability_collection_type CHECK (collection_type IN ('providers', 'data_sets'))", - }, - }, - { - name: "observability_provider_states", - model: (*observabilityProviderState2026090101)(nil), - jsonColumns: initialJSONColumns("observability_provider_states"), - constraints: []string{ - "CONSTRAINT chk_observability_provider_identity CHECK (provider_id <> '')", - "CONSTRAINT chk_observability_provider_status CHECK (status IN ('available', 'degraded', 'unavailable', 'unknown'))", - }, - }, - { - name: "observability_data_set_states", - model: (*observabilityDataSetState2026090101)(nil), - jsonColumns: initialJSONColumns("observability_data_set_states"), - constraints: []string{ - "CONSTRAINT chk_observability_data_set_identity CHECK (provider_id <> '' AND (chain_data_set_id IS NULL OR chain_data_set_id <> '') AND (client_data_set_id IS NULL OR client_data_set_id <> ''))", - "CONSTRAINT fk_observability_data_set_replica_slot FOREIGN KEY (bucket_id, copy_index) REFERENCES bucket_replica_slots (bucket_id, copy_index) ON UPDATE RESTRICT ON DELETE RESTRICT", - "CONSTRAINT chk_observability_data_set_status CHECK (status IN ('available', 'degraded', 'unavailable', 'unknown'))", - }, - foreignKeys: []string{ - "(local_data_set_id, bucket_id, copy_index, provider_id) REFERENCES storage_data_sets (id, bucket_id, copy_index, provider_id) ON UPDATE RESTRICT ON DELETE CASCADE", - }, - }, - } - for _, table := range tables { - if err := createInitialTable(ctx, db, table); err != nil { - return err - } - } - return createInitialIndexes(ctx, db, - initialIndexSpec{name: "idx_observability_provider_states_status", table: "observability_provider_states", columns: []string{"status", "last_checked_at"}}, - initialIndexSpec{name: "idx_observability_data_set_states_bucket_status", table: "observability_data_set_states", columns: []string{"bucket_id", "status", "last_checked_at"}}, - initialIndexSpec{name: "idx_observability_data_set_states_provider_status", table: "observability_data_set_states", columns: []string{"provider_id", "status", "last_checked_at"}}, - ) -} diff --git a/internal/db/repository/errors.go b/internal/db/repository/errors.go index 466399e..d8d8317 100644 --- a/internal/db/repository/errors.go +++ b/internal/db/repository/errors.go @@ -30,6 +30,15 @@ var ErrReplicaTargetLowered = fmt.Errorf("lowering the replica target is not sup // with ErrConflict for existing callers. var ErrPermanentDeleteStorageBusy = fmt.Errorf("permanent delete blocked by storage work: %w", ErrConflict) +// ErrContentCleanupInProgress reports that a write names content whose cleanup +// has started or already deleted it. The bytes have to be written again once +// the cleanup finishes, which then creates new content. +var ErrContentCleanupInProgress = errors.New("storage content is being cleaned up") + +// ErrContentCleanupNotReady reports that a finished cleanup cannot delete its +// content yet because other work still needs the rows. +var ErrContentCleanupNotReady = errors.New("storage content cleanup is waiting for other work") + // ErrTaskLeaseLost reports that a claim generation is stale or its lease can // no longer be proven valid. var ErrTaskLeaseLost = errors.New("task lease lost") diff --git a/internal/db/repository/interfaces.go b/internal/db/repository/interfaces.go index 301b183..f5f0518 100644 --- a/internal/db/repository/interfaces.go +++ b/internal/db/repository/interfaces.go @@ -2,6 +2,7 @@ package repository import ( "context" + "encoding/json" "time" "github.com/strahe/synaps3/internal/model" @@ -333,7 +334,9 @@ type StorageCleanupRepository interface { MarkCopyUnsupported(ctx context.Context, id int64, message string) error UploadHasObjectReferences(ctx context.Context, contentID int64) (bool, error) CleanupHasObjectReferences(ctx context.Context, contentID int64) (bool, error) - CompleteTask(ctx context.Context, contentID, generation, taskID int64) error + // FinalizeContent deletes a cleaned-up content's current-state rows: its + // cache record, its copies, and the content row. Ledgers keep theirs. + FinalizeContent(ctx context.Context, contentID, generation, taskID int64) error } type EnsureDataSetBindingInput struct { @@ -441,13 +444,6 @@ type MarkUploadCopyCommittedInput struct { CommitConfirmedTransactionID string } -// IncompleteReadableUpload identifies one durable upload that still needs -// work to reach its frozen target copy count. -type IncompleteReadableUpload struct { - Upload model.StorageContent - Version model.ObjectVersion -} - type BindReadableUploadInput struct { ContentID int64 BucketID int64 @@ -471,7 +467,6 @@ type StorageContentRepository interface { EnsureContent(ctx context.Context, input EnsureContentInput) (*model.StorageContent, error) GetByID(ctx context.Context, contentID int64) (*model.StorageContent, error) GetByIDs(ctx context.Context, contentIDs []int64) (map[int64]model.StorageContent, error) - RecordContentFailure(ctx context.Context, contentID int64, message string) error GetIngressCopy(ctx context.Context, contentID int64) (*model.StorageCopy, error) // ContentPipelineState derives pipeline position from the copy rows. ContentPipelineState(ctx context.Context, contentID int64) (model.ObjectState, error) @@ -490,6 +485,9 @@ type StorageContentRepository interface { GetDataSetBindingByCopyIndex(ctx context.Context, bucketID int64, copyIndex int) (*model.StorageDataSet, error) EnsureDataSetBinding(ctx context.Context, input EnsureDataSetBindingInput) (*model.StorageDataSet, error) MarkDataSetCreating(ctx context.Context, input MarkDataSetCreatingInput) error + // RecordDataSetClientID ties a generation to the client data set ID of its + // create request before the request is sent. The ID never changes later. + RecordDataSetClientID(ctx context.Context, id int64, clientDataSetID types.OnChainID) error MarkDataSetReady(ctx context.Context, input MarkDataSetReadyInput) error BackfillClientDataSetID(ctx context.Context, input BackfillClientDataSetIDInput) error MarkDataSetDraining(ctx context.Context, id int64, lastError string) error @@ -519,7 +517,6 @@ type StorageContentRepository interface { // GetUploadCopyForDataSet addresses one concrete data set generation. GetUploadCopyForDataSet(ctx context.Context, contentID, storageDataSetID int64) (*model.StorageCopy, error) NextFinalizableCopyForDataSet(ctx context.Context, storageDataSetID int64) (*model.StorageCopy, error) - ListIncompleteReadableUploads(ctx context.Context, afterID int64, limit int) ([]IncompleteReadableUpload, error) MarkUploadCopyPieceReady(ctx context.Context, input MarkUploadCopyPieceReadyInput) error ReopenFailedUploadCopy(ctx context.Context, copyID int64) error ReserveCommitAttempt(ctx context.Context, input storagecommit.ReserveInput) (storagecommit.ReserveResult, error) @@ -579,7 +576,6 @@ type StorageReplacementRepository interface { // identity remains reserved until retry, supersession, or completion. HasInProgressForDataSet(ctx context.Context, dataSetID int64) (bool, error) ListActive(ctx context.Context, afterID int64, limit int) ([]storagereplacement.Replacement, error) - ListSupersededCleanupCandidates(ctx context.Context, afterID int64, limit int) ([]storagereplacement.Replacement, error) // Activate makes the target the write target and marks the source draining // in one transaction. It touches a fixed number of rows regardless of how @@ -610,11 +606,10 @@ type StorageReplacementRepository interface { // refuses premature completion even when called outside the worker. CompleteRetirement(ctx context.Context, replacementID int64, observedEpoch int64) error - // CountAbandonedTargetSoleCopies and RetireAbandonedTarget clean up a target - // a later confirmation replaced. They retire the opposite generation from - // CompleteRetirement and never change the replacement record. + // CountAbandonedTargetSoleCopies counts the copies that only the target a + // later confirmation replaced still holds. CompleteAbandonedTargetTermination + // refuses to retire that target while any remain. CountAbandonedTargetSoleCopies(ctx context.Context, targetDataSetID int64) (int, error) - RetireAbandonedTarget(ctx context.Context, replacementID int64) error } // AuthorizeReplacementInput is one operator confirmation. @@ -681,7 +676,7 @@ type TaskRepository interface { GetByIdentity(ctx context.Context, taskType model.TaskType, idempotencyKey string) (*model.Task, error) ClaimNext(ctx context.Context, leaseDuration time.Duration) (*model.Task, error) RenewLease(ctx context.Context, id, generation int64, leaseDuration time.Duration) (time.Time, error) - WriteCheckpoint(ctx context.Context, id, generation int64, checkpoint []byte) error + WriteCheckpoint(ctx context.Context, id, generation int64, checkpoint json.RawMessage) error ValidateClaim(ctx context.Context, id, generation int64) error Settle(ctx context.Context, id, generation int64, transition TaskTransition) error ShortenLease(ctx context.Context, id, generation int64, duration time.Duration) error @@ -690,14 +685,18 @@ type TaskRepository interface { RetryFailed(ctx context.Context, id int64) error ReactivateTerminal(ctx context.Context, id int64) error AcknowledgeFailed(ctx context.Context, id int64, retention time.Duration) error + // AcknowledgeFailedMatching dismisses every unacknowledged failure the filter + // covers and reports how many it dismissed. + AcknowledgeFailedMatching(ctx context.Context, filter TaskAcknowledgeFilter, retention time.Duration) (int, error) + // CountFailedMatching reports how many failures the same filter covers, so a + // bulk dismissal can be previewed before it is confirmed. + CountFailedMatching(ctx context.Context, filter TaskAcknowledgeFilter) (int, error) DeleteRetained(ctx context.Context, now time.Time, limit int) (int, error) List(ctx context.Context, filter TaskListFilter) (TaskPage, error) CountByStatus(ctx context.Context) ([]TaskStatusCount, error) CountByPresentationStatus(ctx context.Context) ([]TaskStatusCount, error) CountUnacknowledgedFailed(ctx context.Context) (int64, error) CountOverviewActivePipeline(ctx context.Context) ([]TaskPipelineCount, error) - CountActiveObjectTasksByBucket(ctx context.Context, bucketID int64) (int64, error) - CountActiveBucketTasksByBucketID(ctx context.Context, bucketID int64) (int64, error) } type TaskTransition struct { @@ -712,6 +711,13 @@ type TaskTransition struct { RetentionUntil *time.Time } +// TaskAcknowledgeFilter selects the failures one bulk dismissal covers. The +// cutoff is what the operator saw: failures recorded after it stay visible. +type TaskAcknowledgeFilter struct { + Type model.TaskType + FailedBefore time.Time +} + type TaskListFilter struct { Type model.TaskType Status model.TaskStatus diff --git a/internal/db/repository/object_repo.go b/internal/db/repository/object_repo.go index 5ee21df..92c9642 100644 --- a/internal/db/repository/object_repo.go +++ b/internal/db/repository/object_repo.go @@ -459,6 +459,15 @@ func (r *BunObjectRepo) ReleaseContentCacheIfUnreferenced( released := false err := r.runMaybeTx(ctx, func(db bun.IDB) error { contents, err := lockStorageContentsByID(ctx, db, []int64{contentID}) + if errors.Is(err, ErrNotFound) { + // A finished cleanup deleted the content. Its ID is never reused, so a + // file still under its key is an orphan that nothing can name. + if err := release(); err != nil { + return fmt.Errorf("deleting orphaned content cache file: %w", err) + } + released = true + return nil + } if err != nil { return fmt.Errorf("locking content for cache release: %w", err) } @@ -1109,24 +1118,11 @@ func prepareObjectVersionsForPermanentDelete( } } - relatedVersionIDs := append([]string(nil), deletingVersionIDs...) - var boundVersionIDs []string - if len(contentIDs) > 0 { - if err := db.NewSelect(). - Model((*model.ObjectVersion)(nil)). - Column("version_id"). - Where("content_id IN (?)", bun.List(contentIDs)). - Scan(ctx, &boundVersionIDs); err != nil { - return fmt.Errorf("loading storage upload references for permanent delete: %w", err) - } - } - for _, versionID := range boundVersionIDs { - relatedVersionIDs = appendUniqueString(relatedVersionIDs, versionID) + if len(contentIDs) == 0 { + return nil } - sort.Strings(relatedVersionIDs) - // Ingest is scheduled against the content, so a version cannot be removed - // while work is in flight for either the version or the bytes it names. + // while work is in flight for the bytes it names. contentSubjectKeys := make([]string, 0, len(contentIDs)) for _, contentID := range contentIDs { contentSubjectKeys = append(contentSubjectKeys, strconv.FormatInt(contentID, 10)) @@ -1135,16 +1131,8 @@ func prepareObjectVersionsForPermanentDelete( taskQuery := db.NewSelect(). Model(&relatedTasks). Column("id", "status"). + Where("subject_type = ? AND subject_key IN (?)", "storage_content", bun.List(contentSubjectKeys)). OrderExpr("id ASC") - if len(contentSubjectKeys) > 0 { - taskQuery = taskQuery.Where( - "(subject_type = ? AND subject_key IN (?)) OR (subject_type = ? AND subject_key IN (?))", - "object_version", bun.List(relatedVersionIDs), - "storage_content", bun.List(contentSubjectKeys), - ) - } else { - taskQuery = taskQuery.Where("subject_type = ? AND subject_key IN (?)", "object_version", bun.List(relatedVersionIDs)) - } if db.Dialect().Name() == dialect.PG { taskQuery = taskQuery.For("UPDATE") } @@ -1156,9 +1144,6 @@ func prepareObjectVersionsForPermanentDelete( return ErrPermanentDeleteStorageBusy } } - if len(contentIDs) == 0 { - return nil - } copiesByContentID := make(map[int64][]model.StorageCopy) for _, copyRow := range copies { @@ -1225,6 +1210,9 @@ func prepareObjectVersionsForPermanentDelete( if rows == 0 { return fmt.Errorf("cancelling storage copy %d for permanent delete: %w", copyRow.ID, ErrPermanentDeleteStorageBusy) } + if err := wakeCommitFIFOHead(ctx, db, copyRow.StorageDataSetID); err != nil { + return err + } } if _, err := db.NewUpdate(). Model((*model.StorageContent)(nil)). @@ -1330,28 +1318,37 @@ func storageUploadCopyHasAttemptedCommit(copyRow model.StorageCopy) bool { copyRow.CommitTransactionID != nil && *copyRow.CommitTransactionID != "" } +// reserveStorageCleanupForDeletedVersions starts the cleanup of content once the +// deletion removes its last live version; while another version still names the +// bytes nothing is reserved. It is reserved even without a committed copy, +// because the cleanup is also what finally deletes the content's rows. func reserveStorageCleanupForDeletedVersions(ctx context.Context, db bun.IDB, contentID int64, deletedVersionIDs []string) (*StorageCleanupReservation, error) { if contentID == 0 || len(deletedVersionIDs) == 0 { return nil, fmt.Errorf("preparing storage cleanup: %w", ErrInvalidInput) } - copies, err := storageCleanupCopySnapshots(ctx, db, contentID) + live, err := selectLiveObjectVersionForStorageContent(ctx, db, &model.StorageContent{ID: contentID}, deletedVersionIDs) if err != nil { return nil, err } - if len(copies) == 0 { + if live != nil { return nil, nil } - now := time.Now() - for i := range copies { - copies[i].CreatedAt = now - copies[i].UpdatedAt = now - } - _, err = db.NewInsert(). - Model(&copies). - On("CONFLICT (content_id, storage_data_set_id, piece_id) DO NOTHING"). - Exec(ctx) + copies, err := storageCleanupCopySnapshots(ctx, db, contentID) if err != nil { - return nil, fmt.Errorf("persisting storage cleanup snapshots: %w", err) + return nil, err + } + now := time.Now() + if len(copies) > 0 { + for i := range copies { + copies[i].CreatedAt = now + copies[i].UpdatedAt = now + } + if _, err := db.NewInsert(). + Model(&copies). + On("CONFLICT (content_id, storage_data_set_id, piece_id) DO NOTHING"). + Exec(ctx); err != nil { + return nil, fmt.Errorf("persisting storage cleanup snapshots: %w", err) + } } reservation := new(StorageCleanupReservation) err = db.NewRaw(`UPDATE storage_contents @@ -1365,16 +1362,6 @@ func reserveStorageCleanupForDeletedVersions(ctx context.Context, db bun.IDB, co return reservation, nil } -func appendUniqueString(values []string, value string) []string { - if value == "" { - return values - } - if slices.Contains(values, value) { - return values - } - return append(values, value) -} - func storageCleanupCopySnapshots(ctx context.Context, db bun.IDB, contentID int64) ([]model.StorageCleanupCopy, error) { var copies []model.StorageCleanupCopy err := db.NewRaw(`SELECT @@ -1387,6 +1374,7 @@ func storageCleanupCopySnapshots(ctx context.Context, db bun.IDB, contentID int6 storage_data_set.client_data_set_id, storage_copy.piece_id, storage_content.piece_cid, + storage_content.checksum, storage_copy.retrieval_url, ? AS status FROM storage_copies AS storage_copy diff --git a/internal/db/repository/object_repo_test.go b/internal/db/repository/object_repo_test.go index e6387bd..5dc8f3d 100644 --- a/internal/db/repository/object_repo_test.go +++ b/internal/db/repository/object_repo_test.go @@ -111,7 +111,10 @@ func TestObjectRepo_CountOverviewAttention(t *testing.T) { if _, err := createVersion(t, repos, failedContent); err != nil { t.Fatalf("seed failed-content version: %v", err) } - if err := repos.Contents.RecordContentFailure(ctx, *failedContent.ContentID, "provider failed"); err != nil { + if _, err := db.NewUpdate().Model((*model.StorageContent)(nil)). + Set("error_message = ?", "provider failed"). + Where("id = ?", *failedContent.ContentID). + Exec(ctx); err != nil { t.Fatalf("record content failure: %v", err) } diff --git a/internal/db/repository/storage_cleanup_repo.go b/internal/db/repository/storage_cleanup_repo.go index 2e685c2..dce576a 100644 --- a/internal/db/repository/storage_cleanup_repo.go +++ b/internal/db/repository/storage_cleanup_repo.go @@ -8,6 +8,8 @@ import ( "time" "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagecommit" + "github.com/strahe/synaps3/internal/storagereplacement" "github.com/uptrace/bun" ) @@ -218,23 +220,103 @@ func (r *BunStorageCleanupRepo) CleanupHasObjectReferences(ctx context.Context, return row.Count > 0, nil } -func (r *BunStorageCleanupRepo) CompleteTask(ctx context.Context, contentID, generation, taskID int64) error { - result, err := r.db.NewUpdate(). - Model((*model.StorageContent)(nil)). - Set("cleanup_task_id = NULL"). - Set("updated_at = ?", time.Now()). - Where("id = ? AND cleanup_generation = ? AND cleanup_task_id = ?", contentID, generation, taskID). - Exec(ctx) +// FinalizeContent deletes the current-state rows of content whose remote +// cleanup finished: its cache record, its copies, and the content row. Commit, +// pull, replacement, cleanup, and deletion ledgers keep their rows and name the +// content by value. It returns ErrContentCleanupNotReady while anything could +// still need those rows, and must run in the caller's transaction. +func (r *BunStorageCleanupRepo) FinalizeContent(ctx context.Context, contentID, generation, taskID int64) error { + if contentID < 1 || generation < 1 || taskID < 1 { + return ErrInvalidInput + } + contents, err := lockStorageContentsByID(ctx, r.db, []int64{contentID}) if err != nil { - return fmt.Errorf("completing storage cleanup task: %w", err) + return fmt.Errorf("finalizing storage cleanup: %w", err) } - rows, _ := result.RowsAffected() - if rows != 1 { - return ErrConflict + content := contents[contentID] + if content.CleanupGeneration != generation || content.CleanupTaskID == nil || *content.CleanupTaskID != taskID { + return fmt.Errorf("finalizing storage cleanup: %w", ErrConflict) + } + ready, err := contentCleanupReady(ctx, r.db, contentID) + if err != nil { + return err + } + if !ready { + return ErrContentCleanupNotReady + } + now := time.Now() + for _, column := range []string{"created_by_content_id", "last_used_content_id"} { + if _, err := r.db.NewUpdate(). + Model((*model.StorageDataSet)(nil)). + Set(column+" = NULL"). + Set("updated_at = ?", now). + Where(column+" = ?", contentID). + Exec(ctx); err != nil { + return fmt.Errorf("finalizing storage cleanup: clearing data set %s: %w", column, err) + } + } + for _, rows := range []struct { + model any + column string + }{ + {(*model.ObjectCache)(nil), "content_id"}, + {(*model.StorageCopy)(nil), "content_id"}, + {(*model.StorageContent)(nil), "id"}, + } { + if _, err := r.db.NewDelete().Model(rows.model).Where(rows.column+" = ?", contentID).Exec(ctx); err != nil { + return fmt.Errorf("finalizing storage cleanup: %w", err) + } } return nil } +// contentCleanupReady reports whether nothing can still need a cleaned-up +// content's rows: no live version names it, its bytes are out of the cache with +// no cache task running, no commit attempt or copy task is still open, and no +// replacement item that blocks retirement names it. +func contentCleanupReady(ctx context.Context, db bun.IDB, contentID int64) (bool, error) { + unreferenced, err := contentIsUnreferenced(ctx, db, contentID) + if err != nil || !unreferenced { + return false, err + } + for _, check := range []struct { + what string + query *bun.SelectQuery + }{ + {"cache residency", db.NewSelect().Model((*model.ObjectCache)(nil)). + Where("content_id = ?", contentID). + Where(`in_cache = ? OR EXISTS ( + SELECT 1 FROM tasks AS cache_task + WHERE cache_task.id = object_cache.cache_active_task_id + AND cache_task.status IN (?, ?) + )`, true, model.TaskStatusPending, model.TaskStatusRunning)}, + {"open commit attempts", db.NewSelect().Model((*storagecommit.Attempt)(nil)). + Where("content_id = ? AND resolved_at IS NULL", contentID)}, + {"copy tasks", db.NewSelect().Model((*model.StorageCopy)(nil)). + Where("content_id = ?", contentID). + Where(`EXISTS ( + SELECT 1 FROM tasks AS copy_task + WHERE copy_task.id = storage_copy.active_task_id + AND copy_task.status IN (?, ?) + )`, model.TaskStatusPending, model.TaskStatusRunning)}, + // Pending and attention items are the ones ItemStatus.Blocking counts. + {"replacement items", db.NewSelect().Model((*storagereplacement.Item)(nil)). + Where("content_id = ?", contentID). + Where("status IN (?)", bun.List([]storagereplacement.ItemStatus{ + storagereplacement.ItemStatusPending, storagereplacement.ItemStatusAttention, + }))}, + } { + count, err := check.query.Count(ctx) + if err != nil { + return false, fmt.Errorf("checking %s before finalizing storage cleanup: %w", check.what, err) + } + if count > 0 { + return false, nil + } + } + return true, nil +} + func storageCleanupCopyUpdateResult(res sql.Result, err error, op string) error { if err != nil { return fmt.Errorf("%s: %w", op, err) diff --git a/internal/db/repository/storage_cleanup_repo_test.go b/internal/db/repository/storage_cleanup_repo_test.go index d2293f4..c2b2103 100644 --- a/internal/db/repository/storage_cleanup_repo_test.go +++ b/internal/db/repository/storage_cleanup_repo_test.go @@ -2,13 +2,157 @@ package repository_test import ( "errors" + "fmt" "strconv" "testing" + "time" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagereplacement" ) +// TestStorageCleanupBlocksReuseUntilContentIsFinalized walks content through +// its last delete: cleanup starts only once no live version remains, new +// versions cannot name the content until it is finalized, and finalizing waits +// for cached bytes and blocking replacement items before it deletes the +// current-state rows and leaves the ledgers. +func TestStorageCleanupBlocksReuseUntilContentIsFinalized(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := t.Context() + bucket := seedBucket(t, db, "cleanup-finalize") + contentID := seedContent(t, repos, bucket.ID, "cleanup-finalize", 10) + source, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, CreatedByContentID: contentID, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + versions := make([]*model.ObjectVersion, 2) + for i := range versions { + versions[i] = newObjectVersion(bucket.ID, fmt.Sprintf("file-%d.txt", i), model.NewVersionID(), 10) + versions[i].ContentID = &contentID + if _, err := createVersion(t, repos, versions[i]); err != nil { + t.Fatalf("create version %d: %v", i, err) + } + } + deleteVersion := func(version *model.ObjectVersion) *repository.StorageCleanupReservation { + t.Helper() + result, err := repos.Objects.DeleteObjectVersionPermanently(ctx, repository.DeleteObjectVersionInput{ + BucketID: bucket.ID, Key: version.Key, VersionID: version.VersionID, + }) + if err != nil { + t.Fatalf("DeleteObjectVersionPermanently(%s): %v", version.Key, err) + } + return result.StorageCleanup + } + if cleanup := deleteVersion(versions[0]); cleanup != nil { + t.Fatalf("cleanup reserved while another version is live: %#v", cleanup) + } + // Nothing was ever stored remotely, and the last delete still starts cleanup. + cleanup := deleteVersion(versions[1]) + if cleanup == nil || cleanup.ContentID != contentID || cleanup.TaskID != nil { + t.Fatalf("last delete cleanup = %#v", cleanup) + } + taskRow, created, err := repos.Tasks.Enqueue(ctx, &model.Task{ + Type: model.TaskTypeStorageCleanup, IdempotencyKey: "cleanup-finalize", InputVersion: 1, + Input: []byte(`{}`), InputHash: "cleanup-finalize", Status: model.TaskStatusPending, + ResumeMode: model.TaskResumeModeExecute, AvailableAt: time.Now(), + }) + if err != nil || !created { + t.Fatalf("Enqueue = %#v, created=%v, err=%v", taskRow, created, err) + } + if err := repos.StorageCleanup.BindTask(ctx, contentID, cleanup.Generation, taskRow.ID); err != nil { + t.Fatalf("BindTask: %v", err) + } + reuse := func() error { + version := newObjectVersion(bucket.ID, "reuse.txt", model.NewVersionID(), 10) + version.ContentID = &contentID + _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version) + return err + } + if err := reuse(); !errors.Is(err, repository.ErrContentCleanupInProgress) { + t.Fatalf("reuse during cleanup = %v, want ErrContentCleanupInProgress", err) + } + + replacement, _, err := repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, "202"), ClientRequestID: "cleanup-finalize", + }) + if err != nil { + t.Fatalf("Authorize: %v", err) + } + item := &storagereplacement.Item{ + ReplacementID: replacement.ID, ContentID: contentID, TargetDataSetID: replacement.TargetDataSetID, + Status: storagereplacement.ItemStatusAttention, + } + if _, err := db.NewInsert().Model(item).Exec(ctx); err != nil { + t.Fatalf("insert replacement item: %v", err) + } + now := time.Now() + if _, err := db.NewInsert().Model(&model.ObjectCache{ContentID: contentID, InCache: true, CreatedAt: now, UpdatedAt: now}). + On("CONFLICT (content_id) DO UPDATE").Set("in_cache = EXCLUDED.in_cache").Exec(ctx); err != nil { + t.Fatalf("mark content cached: %v", err) + } + finalize := func(taskID int64) error { + return repos.WithTx(ctx, func(txRepos *repository.Repositories) error { + return txRepos.StorageCleanup.FinalizeContent(ctx, contentID, cleanup.Generation, taskID) + }) + } + if err := finalize(taskRow.ID); !errors.Is(err, repository.ErrContentCleanupNotReady) { + t.Fatalf("finalize with cached bytes = %v, want ErrContentCleanupNotReady", err) + } + if released, err := repos.Objects.ReleaseContentCacheIfUnreferenced(ctx, contentID, func() error { return nil }); err != nil || !released { + t.Fatalf("release cache = %t, %v", released, err) + } + if err := finalize(taskRow.ID); !errors.Is(err, repository.ErrContentCleanupNotReady) { + t.Fatalf("finalize with a replacement item in attention = %v, want ErrContentCleanupNotReady", err) + } + if _, err := db.NewUpdate().Model(item).Set("status = ?", storagereplacement.ItemStatusCancelled).WherePK().Exec(ctx); err != nil { + t.Fatalf("cancel replacement item: %v", err) + } + if err := finalize(taskRow.ID + 1); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("finalize by another task = %v, want ErrConflict", err) + } + if err := finalize(taskRow.ID); err != nil { + t.Fatalf("FinalizeContent: %v", err) + } + + for _, rows := range []struct { + table, column string + want int + }{ + {"storage_contents", "id", 0}, + {"storage_copies", "content_id", 0}, + {"object_cache", "content_id", 0}, + {"storage_data_sets", "created_by_content_id", 0}, + {"object_deletions", "content_id", 2}, + {"storage_replacement_items", "content_id", 1}, + } { + var count int + if err := db.NewRaw("SELECT count(*) FROM "+rows.table+" WHERE "+rows.column+" = ?", contentID).Scan(ctx, &count); err != nil { + t.Fatalf("count %s: %v", rows.table, err) + } + if count != rows.want { + t.Fatalf("%s rows naming the content = %d, want %d", rows.table, count, rows.want) + } + } + // A write that resolved the content before it was finalized is refused the + // same way, and a cache file left under its key is an orphan. + if err := reuse(); !errors.Is(err, repository.ErrContentCleanupInProgress) { + t.Fatalf("reuse after finalizing = %v, want ErrContentCleanupInProgress", err) + } + releases := 0 + released, err := repos.Objects.ReleaseContentCacheIfUnreferenced(ctx, contentID, func() error { + releases++ + return nil + }) + if err != nil || !released || releases != 1 { + t.Fatalf("orphan cache release = %t, %v, calls=%d", released, err, releases) + } +} + func TestStorageCleanupCopyTransitionsAreGuardedAndIdempotent(t *testing.T) { db := testDB(t) repos := repository.NewRepositories(db) @@ -26,7 +170,7 @@ func TestStorageCleanupCopyTransitionsAreGuardedAndIdempotent(t *testing.T) { row := &model.StorageCleanupCopy{ ContentID: contentID, BucketID: bucket.ID, CopyIndex: 0, ProviderID: providerID, StorageDataSetID: binding.ID, PieceID: onChainID(t, strconv.FormatInt(piece, 10)), PieceCID: "piece-cid", - Status: model.StorageCleanupCopyStatusPending, + Checksum: "cleanup-transitions-checksum", Status: model.StorageCleanupCopyStatusPending, } if _, err := db.NewInsert().Model(row).Exec(t.Context()); err != nil { t.Fatalf("insert cleanup copy: %v", err) diff --git a/internal/db/repository/storage_commit_repo.go b/internal/db/repository/storage_commit_repo.go index 277ef8a..d9fa9fb 100644 --- a/internal/db/repository/storage_commit_repo.go +++ b/internal/db/repository/storage_commit_repo.go @@ -76,22 +76,7 @@ func (r *BunStorageContentRepo) ReserveCommitAttempt( return nil } var headID int64 - err = db.NewSelect(). - Model((*model.StorageCopy)(nil)). - Column("id"). - Where("storage_data_set_id = ?", input.Copy.StorageDataSetID). - Where("status = ?", model.StorageCopyStatusPieceReady). - Where("commit_ready_at IS NOT NULL"). - Where(`NOT EXISTS ( - SELECT 1 FROM storage_commit_attempts AS unresolved_attempt - WHERE unresolved_attempt.content_id = storage_copy.content_id - AND unresolved_attempt.storage_data_set_id = storage_copy.storage_data_set_id - AND unresolved_attempt.resolved_at IS NULL - )`). - OrderExpr("commit_ready_at ASC"). - OrderExpr("id ASC"). - Limit(1). - Scan(ctx, &headID) + err = commitFIFOHead(db, input.Copy.StorageDataSetID).Column("id").Scan(ctx, &headID) if err != nil { return fmt.Errorf("selecting next FIFO storage commit: %w", err) } @@ -118,6 +103,11 @@ func (r *BunStorageContentRepo) ReserveCommitAttempt( if err != nil { return err } + // Capacity may remain after this reservation; the next copy in line + // must not sleep through it. + if err := wakeCommitFIFOHead(ctx, db, input.Copy.StorageDataSetID); err != nil { + return err + } out.State = storagecommit.ReservationAcquired out.Copy = *copyRow return nil @@ -305,7 +295,10 @@ func (r *BunStorageContentRepo) ResetCommitAttempt(ctx context.Context, input st if rows, _ := res.RowsAffected(); rows != 1 { return ErrConflict } - return projectResolvedCommitAttempt(ctx, db, copyID, now, true, true, nullableString(input.LastError)) + if err := projectResolvedCommitAttempt(ctx, db, copyID, now, true, true, nullableString(input.LastError)); err != nil { + return err + } + return wakeCommitFIFOHead(ctx, db, input.Copy.StorageDataSetID) }) } @@ -341,7 +334,10 @@ func (r *BunStorageContentRepo) ReleaseCommitAttempt(ctx context.Context, input if rows, _ := res.RowsAffected(); rows != 1 { return ErrConflict } - return projectResolvedCommitAttempt(ctx, db, copyID, now, input.ClearReadyAt, input.ClearExtraData, nil) + if err := projectResolvedCommitAttempt(ctx, db, copyID, now, input.ClearReadyAt, input.ClearExtraData, nil); err != nil { + return err + } + return wakeCommitFIFOHead(ctx, db, input.Copy.StorageDataSetID) }) } @@ -377,7 +373,7 @@ func (r *BunStorageContentRepo) ReleaseCommitReservation( if rows, _ := res.RowsAffected(); rows != 1 { return ErrConflict } - return nil + return wakeCommitFIFOHead(ctx, db, input.Copy.StorageDataSetID) }) } @@ -399,6 +395,53 @@ func countActiveCommitAttemptsForDataSet(ctx context.Context, db bun.IDB, storag return count, nil } +// commitFIFOHead selects the only copy of a data set allowed to reserve the +// next commit attempt: the oldest ready copy without an unresolved attempt. +func commitFIFOHead(db bun.IDB, storageDataSetID int64) *bun.SelectQuery { + return db.NewSelect(). + Model((*model.StorageCopy)(nil)). + Where("storage_data_set_id = ?", storageDataSetID). + Where("status = ?", model.StorageCopyStatusPieceReady). + Where("commit_ready_at IS NOT NULL"). + Where(`NOT EXISTS ( + SELECT 1 FROM storage_commit_attempts AS unresolved_attempt + WHERE unresolved_attempt.content_id = storage_copy.content_id + AND unresolved_attempt.storage_data_set_id = storage_copy.storage_data_set_id + AND unresolved_attempt.resolved_at IS NULL + )`). + OrderExpr("commit_ready_at ASC"). + OrderExpr("id ASC"). + Limit(1) +} + +// wakeCommitFIFOHead makes the queue head's task runnable when its data set has +// commit capacity, so waiting copies advance without polling. Callers invoke it +// after any change that frees capacity or moves the head. +func wakeCommitFIFOHead(ctx context.Context, db bun.IDB, storageDataSetID int64) error { + active, err := countActiveCommitAttemptsForDataSet(ctx, db, storageDataSetID) + if err != nil { + return err + } + if active >= storagecommit.MaxActiveAttemptsPerDataSet { + return nil + } + var taskID sql.NullInt64 + err = commitFIFOHead(db, storageDataSetID).Column("active_task_id").Scan(ctx, &taskID) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return fmt.Errorf("selecting storage commit queue head: %w", err) + } + if !taskID.Valid { + return nil + } + if _, err := (&BunTaskRepo{db: db}).WakePending(ctx, []int64{taskID.Int64}); err != nil { + return fmt.Errorf("waking storage commit queue head: %w", err) + } + return nil +} + func countCommitAttentionAttemptsForDataSet(ctx context.Context, db bun.IDB, storageDataSetID int64) (int, error) { count, err := db.NewSelect(). Model((*storagecommit.Attempt)(nil)). @@ -509,6 +552,9 @@ func (r *BunStorageContentRepo) ReleaseCommitAttention(ctx context.Context, inpu if err := projectResolvedCommitAttempt(ctx, db, copyID, now, true, true, nil); err != nil { return fmt.Errorf("releasing storage confirmation attention: %w", err) } + if err := wakeCommitFIFOHead(ctx, db, initial.StorageDataSetID); err != nil { + return err + } return resumeCommitTaskAfterAttentionRelease(ctx, db, initial.ActiveTaskID) }) } diff --git a/internal/db/repository/storage_content_reference.go b/internal/db/repository/storage_content_reference.go index eea4f28..3bb14d0 100644 --- a/internal/db/repository/storage_content_reference.go +++ b/internal/db/repository/storage_content_reference.go @@ -3,6 +3,7 @@ package repository import ( "context" "database/sql" + "errors" "fmt" "slices" "time" @@ -113,12 +114,23 @@ func prepareNewObjectVersionStorageReference(ctx context.Context, db bun.IDB, ve return nil } contents, err := lockStorageContentsByID(ctx, db, []int64{*version.ContentID}) + if errors.Is(err, ErrNotFound) { + // Content rows are only deleted when their cleanup finishes, so the bytes + // have to be written again, which creates new content. + return fmt.Errorf("storage content %d: %w", *version.ContentID, ErrContentCleanupInProgress) + } if err != nil { return err } - if contents[*version.ContentID] == nil { + content := contents[*version.ContentID] + if content == nil { return fmt.Errorf("storage content %d: %w", *version.ContentID, ErrNotFound) } + // Cleanup starts only when the last version is gone, and a new version + // cannot bring content back while it is being removed. + if content.CleanupTaskID != nil { + return fmt.Errorf("storage content %d: %w", *version.ContentID, ErrContentCleanupInProgress) + } // A data version is only created after its bytes are durably in the local // cache, so the content is resident. Residency is per content: versions that // share bytes share this row. diff --git a/internal/db/repository/storage_content_repo.go b/internal/db/repository/storage_content_repo.go index 014586f..883a830 100644 --- a/internal/db/repository/storage_content_repo.go +++ b/internal/db/repository/storage_content_repo.go @@ -168,26 +168,6 @@ func (r *BunStorageContentRepo) ContentPipelineState(ctx context.Context, conten return model.ObjectState(state), nil } -// RecordContentFailure stores the message explaining why this content could not -// be placed. Whether it counts as failed is derived from its copies; only the -// human-readable reason is persisted. -func (r *BunStorageContentRepo) RecordContentFailure(ctx context.Context, contentID int64, message string) error { - if contentID < 1 { - return fmt.Errorf("recording content failure: %w", ErrInvalidInput) - } - _, err := r.db.NewUpdate(). - Model((*model.StorageContent)(nil)). - Set("error_message = ?", message). - Set("updated_at = ?", time.Now()). - Where("id = ?", contentID). - Where("accepted_at IS NULL"). - Exec(ctx) - if err != nil { - return fmt.Errorf("recording content failure: %w", err) - } - return nil -} - func (r *BunStorageContentRepo) GetIngressCopy(ctx context.Context, contentID int64) (*model.StorageCopy, error) { copyRow, err := r.ingressCopy(ctx, contentID) if errors.Is(err, ErrNotFound) { @@ -730,6 +710,23 @@ func (r *BunStorageContentRepo) MarkDataSetCreating(ctx context.Context, input M return nil } +func (r *BunStorageContentRepo) RecordDataSetClientID(ctx context.Context, id int64, clientDataSetID types.OnChainID) error { + if id <= 0 || clientDataSetID.IsZero() { + return fmt.Errorf("recording storage client data set ID: %w", ErrInvalidInput) + } + res, err := r.db.NewUpdate(). + Model((*model.StorageDataSet)(nil)). + Set("client_data_set_id = ?", clientDataSetID). + Set("updated_at = ?", time.Now()). + Where("id = ?", id). + Where("(client_data_set_id IS NULL OR client_data_set_id = ?)", clientDataSetID). + Exec(ctx) + if err != nil { + return fmt.Errorf("recording storage client data set ID: %w", err) + } + return requireDataSetStatusUpdate(ctx, r.db, id, res, "recording storage client data set ID") +} + func (r *BunStorageContentRepo) MarkDataSetReady(ctx context.Context, input MarkDataSetReadyInput) error { return r.runMaybeTx(ctx, func(db bun.IDB) error { return markDataSetReady(ctx, db, input.ID, input.ContentID, input.DataSetID, input.ClientDataSetID) @@ -1094,58 +1091,6 @@ func (r *BunStorageContentRepo) NextFinalizableCopyForDataSet(ctx context.Contex return copyRow, nil } -func (r *BunStorageContentRepo) ListIncompleteReadableUploads( - ctx context.Context, - afterID int64, - limit int, -) ([]IncompleteReadableUpload, error) { - var uploads []model.StorageContent - q := r.db.NewSelect(). - Model(&uploads). - Where("accepted_at IS NULL"). - Where("id > ?", afterID). - Where(`EXISTS ( - SELECT 1 FROM object_versions AS live_version - WHERE live_version.is_delete_marker = ? - AND live_version.state = ? - AND `+objectVersionReferencesStorageContentSQL("live_version", "storage_content")+` - )`, false, model.ObjectStateStored). - OrderExpr("id ASC") - if limit > 0 { - q = q.Limit(limit) - } - if err := q.Scan(ctx); err != nil { - return nil, fmt.Errorf("listing incomplete readable storage uploads: %w", err) - } - - items := make([]IncompleteReadableUpload, 0, len(uploads)) - for i := range uploads { - version := new(model.ObjectVersion) - err := r.db.NewSelect(). - Model(version). - Where("is_delete_marker = ?", false). - Where("state = ?", model.ObjectStateStored). - Where("content_id = ?", uploads[i].ID). - OrderExpr("in_cache DESC"). - OrderExpr("CASE WHEN EXISTS (SELECT 1 FROM objects AS pointer WHERE pointer.id = object_version.object_id AND pointer.current_version_id = object_version.version_id) THEN 0 ELSE 1 END ASC"). - OrderExpr("created_at DESC"). - OrderExpr("version_id DESC"). - Limit(1). - Scan(ctx) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - continue - } - return nil, fmt.Errorf("selecting durable version for incomplete readable upload %d: %w", uploads[i].ID, err) - } - items = append(items, IncompleteReadableUpload{ - Upload: uploads[i], - Version: *version, - }) - } - return items, nil -} - func (r *BunStorageContentRepo) MarkUploadCopyPieceReady(ctx context.Context, input MarkUploadCopyPieceReadyInput) error { return r.runMaybeTx(ctx, func(db bun.IDB) error { if err := lockStorageContentForCopyMutation(ctx, db, input.ContentID); err != nil { @@ -1430,7 +1375,7 @@ func (r *BunStorageContentRepo) MarkUploadCopyCommitted(ctx context.Context, inp if err := updateUploadReadable(ctx, db, input.ContentID, input.PieceCID, now); err != nil { return err } - return nil + return wakeCommitFIFOHead(ctx, db, initial.StorageDataSetID) }) } @@ -1543,6 +1488,14 @@ func (r *BunStorageContentRepo) MarkUploadCopyFailed(ctx context.Context, input } return nil } + var storageDataSetID int64 + if err := db.NewSelect().Model((*model.StorageCopy)(nil)).Column("storage_data_set_id"). + Where("id = ?", copyID).Scan(ctx, &storageDataSetID); err != nil { + return fmt.Errorf("loading failed storage copy data set: %w", err) + } + if err := wakeCommitFIFOHead(ctx, db, storageDataSetID); err != nil { + return err + } readableCount, err := countReadableReplicaSlots(ctx, db, contentID) if err != nil { return err @@ -1559,6 +1512,9 @@ func (r *BunStorageContentRepo) MarkUploadCopyFailed(ctx context.Context, input if err != nil { return fmt.Errorf("counting viable storage upload copies: %w", err) } + // Content needs attention only when no copy is left that holds it or may + // still hold it; a failed copy next to readable, submitted, or pending + // ones is not a problem with the content. if readableCount == 0 && submittedCount == 0 && viableCount == 0 { _, err = db.NewUpdate(). Model((*model.StorageContent)(nil)). @@ -1570,16 +1526,6 @@ func (r *BunStorageContentRepo) MarkUploadCopyFailed(ctx context.Context, input if err != nil { return fmt.Errorf("marking storage upload failed: %w", err) } - } else if readableCount > 0 { - _, err = db.NewUpdate(). - Model((*model.StorageContent)(nil)). - Set("error_message = ?", lastError). - Set("updated_at = ?", now). - Where("id = ?", contentID). - Exec(ctx) - if err != nil { - return fmt.Errorf("marking storage upload readable after copy failure: %w", err) - } } return nil }) @@ -1884,7 +1830,7 @@ func markDataSetReady(ctx context.Context, db bun.IDB, id int64, contentID int64 if dataSetID.IsZero() { return fmt.Errorf("dataSetID is required: %w", ErrInvalidInput) } - res, err := db.NewUpdate(). + query := db.NewUpdate(). Model((*model.StorageDataSet)(nil)). Set("status = ?", model.StorageDataSetStatusReady). Set("data_set_id = ?", dataSetID). @@ -1898,8 +1844,13 @@ func markDataSetReady(ctx context.Context, db bun.IDB, id int64, contentID int64 WHERE other.id <> ? AND other.provider_id = (SELECT provider_id FROM storage_data_sets WHERE id = ?) AND other.data_set_id = ? - )`, id, id, dataSetID). - Exec(ctx) + )`, id, id, dataSetID) + if clientDataSetID != nil { + // A recorded ID names the request this generation sent, so a result + // carrying a different one is never written over it. + query = query.Where("(client_data_set_id IS NULL OR client_data_set_id = '' OR client_data_set_id = ?)", clientDataSetID) + } + res, err := query.Exec(ctx) if err != nil { return fmt.Errorf("marking storage data set ready: %w", err) } diff --git a/internal/db/repository/storage_domain_helpers_test.go b/internal/db/repository/storage_domain_helpers_test.go index 4a8bc00..36dec20 100644 --- a/internal/db/repository/storage_domain_helpers_test.go +++ b/internal/db/repository/storage_domain_helpers_test.go @@ -152,6 +152,137 @@ func TestAuthorizeReplacementPreservesCurrentSource(t *testing.T) { } } +// An earlier replacement whose target may still be created on chain cannot be +// abandoned: superseding it would leave that service untracked. Once the +// target's creation fence is released, a new request proceeds as before. +func TestAuthorizeReplacementWaitsForEarlierTargetCreation(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + bucket := seedBucket(t, db, "replacement-target-creating") + source, err := repos.Contents.EnsureDataSetBinding(t.Context(), repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + first, _, err := repos.Replacements.Authorize(t.Context(), repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, "202"), ClientRequestID: "target-creating-first", + }) + if err != nil { + t.Fatalf("Authorize(first): %v", err) + } + ensureTask, _, err := repos.Tasks.Enqueue(t.Context(), &model.Task{ + Type: model.TaskTypeStorageDataSetEnsure, IdempotencyKey: "target-creating-ensure", + InputVersion: 1, Input: []byte(`{}`), InputHash: "target-creating-ensure", + }) + if err != nil { + t.Fatalf("enqueue target ensure: %v", err) + } + if err := repos.Contents.BindDataSetEnsureTask(t.Context(), first.TargetDataSetID, ensureTask.ID); err != nil { + t.Fatalf("bind target ensure: %v", err) + } + successor := repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, "303"), ClientRequestID: "target-creating-successor", + } + if _, _, err := repos.Replacements.Authorize(t.Context(), successor); !errors.Is(err, storagereplacement.ErrTargetCreating) { + t.Fatalf("Authorize(successor) error = %v, want ErrTargetCreating", err) + } + if kept, err := repos.Replacements.GetByID(t.Context(), first.ID); err != nil || kept.Status == storagereplacement.StatusSuperseded { + t.Fatalf("first replacement = %#v err=%v, want it still in charge of its target", kept, err) + } + + if err := repos.Contents.CompleteDataSetEnsureTask(t.Context(), first.TargetDataSetID, ensureTask.ID); err != nil { + t.Fatalf("release target ensure: %v", err) + } + if _, created, err := repos.Replacements.Authorize(t.Context(), successor); err != nil || !created { + t.Fatalf("Authorize(successor after creation) created=%v err=%v", created, err) + } + if superseded, err := repos.Replacements.GetByID(t.Context(), first.ID); err != nil || superseded.Status != storagereplacement.StatusSuperseded { + t.Fatalf("first replacement = %#v err=%v, want it superseded", superseded, err) + } +} + +// TestAuthorizeReplacementStopsWaitingOnACreationThatNeverSent checks that a +// creation fence left behind by a dead task holds the replica slot only while +// the target row shows a request that may have reached the chain. +func TestAuthorizeReplacementStopsWaitingOnACreationThatNeverSent(t *testing.T) { + for _, tt := range []struct { + name string + bucket string + recordID bool + blocked bool + }{ + {name: "nothing was sent", bucket: "replacement-dead-fence-unsent", recordID: false, blocked: false}, + {name: "a request may have gone out", bucket: "replacement-dead-fence-sent", recordID: true, blocked: true}, + } { + t.Run(tt.name, func(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := t.Context() + bucket := seedBucket(t, db, tt.bucket) + source, err := repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + first, _, err := repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, "202"), ClientRequestID: "dead-fence-first", + }) + if err != nil { + t.Fatalf("Authorize(first): %v", err) + } + ensureTask, _, err := repos.Tasks.Enqueue(ctx, &model.Task{ + Type: model.TaskTypeStorageDataSetEnsure, IdempotencyKey: "dead-fence-ensure", + InputVersion: 1, Input: []byte(`{}`), InputHash: "dead-fence-ensure", + }) + if err != nil { + t.Fatalf("enqueue target ensure: %v", err) + } + if err := repos.Contents.BindDataSetEnsureTask(ctx, first.TargetDataSetID, ensureTask.ID); err != nil { + t.Fatalf("bind target ensure: %v", err) + } + if tt.recordID { + if err := repos.Contents.RecordDataSetClientID(ctx, first.TargetDataSetID, onChainID(t, "909")); err != nil { + t.Fatalf("record client data set ID: %v", err) + } + } + // The creation task ended without releasing its fence. + if _, err := db.NewRaw(`UPDATE tasks SET status = ?, finished_at = ? WHERE id = ?`, + model.TaskStatusFailed, time.Now(), ensureTask.ID).Exec(ctx); err != nil { + t.Fatalf("fail target ensure: %v", err) + } + + successor := repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, "303"), ClientRequestID: "dead-fence-successor", + } + _, created, err := repos.Replacements.Authorize(ctx, successor) + if tt.blocked { + if !errors.Is(err, storagereplacement.ErrTargetCreating) { + t.Fatalf("Authorize(successor) error = %v, want ErrTargetCreating", err) + } + return + } + if err != nil || !created { + t.Fatalf("Authorize(successor) created=%v err=%v, want the slot released", created, err) + } + // The earlier target is given up together with its fence, so retrying + // the dead creation task finds nothing left to create. + abandoned, err := repos.Contents.GetDataSetBindingByID(ctx, first.TargetDataSetID) + if err != nil || abandoned == nil || abandoned.Status != model.StorageDataSetStatusRetired || abandoned.EnsureTaskID != nil { + t.Fatalf("earlier target = %#v err=%v, want it retired with its creation fence released", abandoned, err) + } + if _, err := repos.Contents.AuthorizeDataSetEnsureTask(ctx, first.TargetDataSetID, ensureTask.ID); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("authorizing the dead creation task error = %v, want ErrConflict", err) + } + }) + } +} + func TestAttachReplacementTargetCopyRejectsMismatchedUpload(t *testing.T) { db := testDB(t) repos := repository.NewRepositories(db) @@ -358,12 +489,6 @@ func TestStorageContentBindingRejectsCrossBucketIdentity(t *testing.T) { WHERE version_id = ?`, content.ID, version.VersionID).Exec(t.Context()); err == nil { t.Fatal("direct cross-bucket object version binding succeeded") } - if _, err := db.NewRaw(`INSERT INTO object_deletions - (bucket_id, object_id, key, version_id, content_id, size, deleted_at) - VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)`, - targetBucket.ID, 1, "cross-bucket.txt", model.NewVersionID(), content.ID, 11).Exec(t.Context()); err == nil { - t.Fatal("direct cross-bucket object deletion binding succeeded") - } } func startCopyHealthUpload( @@ -461,6 +586,68 @@ func bindStorageHealthVersion( } } +// A failed copy flags its content only once no copy is left that holds it or +// may still hold it. +func TestCopyFailureFlagsContentOnlyWhenNoCopyCanServeIt(t *testing.T) { + tests := []struct { + name string + other string + wantFlag bool + }{ + {name: "a readable copy remains", other: "readable"}, + {name: "another copy is still pending", other: "pending"}, + {name: "the last copy fails", other: "failed", wantFlag: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + bucket := seedBucket(t, db, "copy-failure-"+tt.other) + upload := startCopyHealthUpload(t, repos, bucket.ID, model.NewVersionID(), 11, "copy-failure-"+tt.other, 2) + if tt.other == "readable" { + commitStorageHealthCopy(t, db, repos, bucket.ID, upload.ID, 0, "101", "1001", "3001", "https://provider-101.example/piece") + } else { + ensureCopyHealthBinding(t, repos, bucket.ID, upload.ID, 0, "101") + } + failing := ensureCopyHealthBinding(t, repos, bucket.ID, upload.ID, 1, "102") + bindings := []repository.UploadCopyBindingInput{{ + StorageDataSetID: failing.ID, CopyIndex: 1, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "102"), + }} + if tt.other != "readable" { + other, err := repos.Contents.GetDataSetBindingByCopyIndex(t.Context(), bucket.ID, 0) + if err != nil || other == nil { + t.Fatalf("GetDataSetBindingByCopyIndex = %#v, err=%v", other, err) + } + bindings = append(bindings, repository.UploadCopyBindingInput{ + StorageDataSetID: other.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101"), + }) + } + if err := repos.Contents.CreateUploadCopiesForBindings(t.Context(), upload.ID, bindings); err != nil { + t.Fatalf("CreateUploadCopiesForBindings: %v", err) + } + if tt.other == "failed" { + if err := repos.Contents.MarkUploadCopyFailed(t.Context(), repository.MarkUploadCopyFailedInput{ + ContentID: upload.ID, CopyIndex: 0, LastError: "first provider rejected the transfer", + }); err != nil { + t.Fatalf("MarkUploadCopyFailed(0): %v", err) + } + } + if err := repos.Contents.MarkUploadCopyFailed(t.Context(), repository.MarkUploadCopyFailedInput{ + ContentID: upload.ID, CopyIndex: 1, LastError: "second provider rejected the transfer", + }); err != nil { + t.Fatalf("MarkUploadCopyFailed(1): %v", err) + } + stored, err := repos.Contents.GetByID(t.Context(), upload.ID) + if err != nil || stored == nil { + t.Fatalf("GetByID = %#v, err=%v", stored, err) + } + if flagged := stored.ErrorMessage != nil; flagged != tt.wantFlag { + t.Fatalf("content error = %v, want flagged=%t", stored.ErrorMessage, tt.wantFlag) + } + }) + } +} + // A termination is a ledger row, so every path that asks "has the end of term // been recorded?" has to read that row. The retirement gate blocks forever if it // reads the replacement alone. diff --git a/internal/db/repository/storage_replacement_gate.go b/internal/db/repository/storage_replacement_gate.go index d8d3e31..3ebb20c 100644 --- a/internal/db/repository/storage_replacement_gate.go +++ b/internal/db/repository/storage_replacement_gate.go @@ -209,24 +209,11 @@ func (r *BunStorageReplacementRepo) CountAbandonedTargetSoleCopies(ctx context.C return count, nil } -// RetireAbandonedTarget marks an abandoned generation retired. It never touches -// the replacement record, which stays superseded, and it refuses a generation -// that still owns its slot. -func (r *BunStorageReplacementRepo) RetireAbandonedTarget(ctx context.Context, replacementID int64) error { - return r.retireAbandonedTarget(ctx, replacementID, false) -} - // CompleteAbandonedTargetTermination retires the superseded target once its end -// of term has been recorded, refusing to retire one that has none. +// of term has been recorded, refusing to retire one that has none. It never +// touches the replacement record, which stays superseded, and it refuses a +// generation that still owns its slot. func (r *BunStorageReplacementRepo) CompleteAbandonedTargetTermination(ctx context.Context, replacementID int64) error { - return r.retireAbandonedTarget(ctx, replacementID, true) -} - -func (r *BunStorageReplacementRepo) retireAbandonedTarget( - ctx context.Context, - replacementID int64, - requireRecordedTermination bool, -) error { return runMaybeTx(ctx, r.db, func(db bun.IDB) error { row, err := lockReplacementByID(ctx, db, replacementID) if err != nil { @@ -251,8 +238,7 @@ func (r *BunStorageReplacementRepo) retireAbandonedTarget( return fmt.Errorf("retiring abandoned target of replacement %d holds %d sole copies: %w", replacementID, sole, storagereplacement.ErrPrematureComplete) } - if requireRecordedTermination && - (row.Status != storagereplacement.StatusSuperseded || row.AbandonedTerminationEpoch == nil) { + if row.Status != storagereplacement.StatusSuperseded || row.AbandonedTerminationEpoch == nil { return fmt.Errorf("completing abandoned target termination: %w", ErrConflict) } res, err := db.NewUpdate(). diff --git a/internal/db/repository/storage_replacement_items.go b/internal/db/repository/storage_replacement_items.go index 50ea410..cc88e0e 100644 --- a/internal/db/repository/storage_replacement_items.go +++ b/internal/db/repository/storage_replacement_items.go @@ -411,7 +411,10 @@ func clearUnattemptedReplacementReservation( AND unresolved_attempt.resolved_at IS NULL )`). Exec(ctx) - return err + if err != nil { + return err + } + return wakeCommitFIFOHead(ctx, db, targetDataSetID) } func sourceCopyState(ctx context.Context, db bun.IDB, contentID, sourceDataSetID int64) (bool, bool, error) { diff --git a/internal/db/repository/storage_replacement_repo.go b/internal/db/repository/storage_replacement_repo.go index b0bcd05..07ff12c 100644 --- a/internal/db/repository/storage_replacement_repo.go +++ b/internal/db/repository/storage_replacement_repo.go @@ -100,6 +100,43 @@ func (r *BunStorageReplacementRepo) Authorize(ctx context.Context, input Authori if inUse > 0 { return fmt.Errorf("authorizing provider replacement: %w", storagereplacement.ErrTargetInUse) } + // Superseding an earlier replacement abandons its target. While that + // target's creation fence is held the storage service may still be + // created on chain with nothing left to track or retire it, so the new + // request waits until the creation finishes or is proven rejected. + creating, err := db.NewSelect(). + Model((*storagereplacement.Replacement)(nil)). + Join("JOIN storage_data_sets AS target_data_set ON target_data_set.id = storage_replacement.target_data_set_id"). + Where("storage_replacement.source_data_set_id = ?", source.ID). + Where("storage_replacement.status NOT IN (?, ?)", storagereplacement.StatusCompleted, storagereplacement.StatusSuperseded). + Where("target_data_set.ensure_task_id IS NOT NULL"). + // A fence only holds the slot while something can still come of it: + // the creation task is live, or the row remembers a request that may + // have reached the chain. A dead task over a generation that never + // asked for a data set holds nothing worth protecting. + Where(`( + EXISTS ( + SELECT 1 FROM tasks AS ensure_task + WHERE ensure_task.id = target_data_set.ensure_task_id + AND ensure_task.status IN (?, ?) + ) + OR (target_data_set.client_data_set_id IS NOT NULL AND target_data_set.client_data_set_id <> '') + OR (target_data_set.create_transaction_id IS NOT NULL AND target_data_set.create_transaction_id <> '') + )`, model.TaskStatusPending, model.TaskStatusRunning). + Count(ctx) + if err != nil { + return fmt.Errorf("checking earlier replacement targets: %w", err) + } + if creating > 0 { + return fmt.Errorf("authorizing provider replacement: %w", storagereplacement.ErrTargetCreating) + } + // A fence still held past that check was left by a creation task that + // died before sending anything. Its target is given up here, fence and + // all: left in place, retrying that task would create a storage service + // for a replacement nothing is completing anymore. + if err := abandonUnsentReplacementTargets(ctx, db, source.ID); err != nil { + return err + } now := time.Now() // The single-active-replacement index rejects a second live row for one @@ -166,6 +203,44 @@ func (r *BunStorageReplacementRepo) Authorize(ctx context.Context, input Authori return result, created, nil } +// abandonUnsentReplacementTargets gives up the targets of a source's earlier +// unfinished replacements whose creation fence outlived its task. Authorize +// calls it only after refusing while any of them might still be created, so +// every target left here never sent a request and holds no storage service. +func abandonUnsentReplacementTargets(ctx context.Context, db bun.IDB, sourceDataSetID int64) error { + var targets []struct { + ID int64 `bun:"id"` + EnsureTaskID int64 `bun:"ensure_task_id"` + } + if err := db.NewSelect(). + Model((*storagereplacement.Replacement)(nil)). + Join("JOIN storage_data_sets AS target_data_set ON target_data_set.id = storage_replacement.target_data_set_id"). + ColumnExpr("target_data_set.id, target_data_set.ensure_task_id"). + Where("storage_replacement.source_data_set_id = ?", sourceDataSetID). + Where("storage_replacement.status NOT IN (?, ?)", storagereplacement.StatusCompleted, storagereplacement.StatusSuperseded). + Where("target_data_set.ensure_task_id IS NOT NULL"). + Scan(ctx, &targets); err != nil { + return fmt.Errorf("selecting unsent replacement targets: %w", err) + } + contents := &BunStorageContentRepo{db: db} + for _, target := range targets { + if err := contents.MarkDataSetFailed(ctx, target.ID, "Replaced before its storage service was created"); err != nil { + return err + } + retired, err := contents.RetireRejectedDataSet(ctx, target.ID) + if err != nil { + return err + } + if !retired { + return fmt.Errorf("retiring unsent replacement target %d: %w", target.ID, ErrConflict) + } + if err := contents.CompleteDataSetEnsureTask(ctx, target.ID, target.EnsureTaskID); err != nil { + return err + } + } + return nil +} + func replacementRequestMatches(row *storagereplacement.Replacement, input AuthorizeReplacementInput) bool { if row.BucketID != input.BucketID || row.SourceDataSetID != input.SourceDataSetID || row.SelectionMode != input.SelectionMode { return false @@ -467,9 +542,12 @@ func (r *BunStorageReplacementRepo) MarkFailed( // MarkCleanupAttention is committed in the same transaction that stops the // coordinator task, so automatic retry can never resume suppressed cleanup. +// Marking a replacement that already waits for attention only refreshes its +// error, so a retried task that stops again for the same reason can still +// settle. func (r *BunStorageReplacementRepo) MarkCleanupAttention(ctx context.Context, replacementID int64, lastError string) error { return r.transition(ctx, replacementID, - []storagereplacement.Status{storagereplacement.StatusRetiring, storagereplacement.StatusWaiting}, + []storagereplacement.Status{storagereplacement.StatusRetiring, storagereplacement.StatusWaiting, storagereplacement.StatusCleanupAttention}, storagereplacement.StatusCleanupAttention, func(q *bun.UpdateQuery) *bun.UpdateQuery { return q.Set("last_error = ?", lastError).Set("wait_reason = NULL") @@ -700,28 +778,6 @@ func (r *BunStorageReplacementRepo) ListActive(ctx context.Context, afterID int6 return rows, nil } -// A superseded replacement leaves behind a target generation that holds partly -// migrated data and no coverage obligation. It still needs its own retirement -// so the abandoned service does not keep costing money. -func (r *BunStorageReplacementRepo) ListSupersededCleanupCandidates(ctx context.Context, afterID int64, limit int) ([]storagereplacement.Replacement, error) { - var rows []storagereplacement.Replacement - if err := withReplacementTerminations(r.db.NewSelect().Model(&rows)). - Where("storage_replacement.id > ?", afterID). - Where("storage_replacement.status = ?", storagereplacement.StatusSuperseded). - Where(`EXISTS ( - SELECT 1 FROM storage_data_sets AS abandoned_target - WHERE abandoned_target.id = storage_replacement.target_data_set_id - AND abandoned_target.is_current = ? - AND abandoned_target.status <> ? - )`, false, model.StorageDataSetStatusRetired). - OrderExpr("storage_replacement.id ASC"). - Limit(limit). - Scan(ctx); err != nil { - return nil, fmt.Errorf("listing superseded replacement cleanup candidates: %w", err) - } - return rows, nil -} - func (r *BunStorageReplacementRepo) transition( ctx context.Context, replacementID int64, diff --git a/internal/db/repository/task_repo.go b/internal/db/repository/task_repo.go index a799ed6..9f4f06d 100644 --- a/internal/db/repository/task_repo.go +++ b/internal/db/repository/task_repo.go @@ -3,6 +3,7 @@ package repository import ( "context" "database/sql" + "encoding/json" "errors" "fmt" "time" @@ -195,7 +196,14 @@ func (r *BunTaskRepo) ClaimNext(ctx context.Context, leaseDuration time.Duration return nil, fmt.Errorf("lease duration must be positive: %w", ErrInvalidInput) } if r.db.Dialect().Name() != dialect.PG { - return r.claimNextSQLite(ctx, r.db, leaseDuration) + // The claim and its payload read commit together, as on PostgreSQL. + var claimed *model.Task + err := runMaybeTx(ctx, r.db, func(db bun.IDB) error { + var err error + claimed, err = r.claimNextSQLite(ctx, db, leaseDuration) + return err + }) + return claimed, err } db, ok := r.db.(*bun.DB) if !ok { @@ -270,7 +278,7 @@ func (r *BunTaskRepo) RenewLease(ctx context.Context, id, generation int64, leas return until, nil } -func (r *BunTaskRepo) WriteCheckpoint(ctx context.Context, id, generation int64, checkpoint []byte) error { +func (r *BunTaskRepo) WriteCheckpoint(ctx context.Context, id, generation int64, checkpoint json.RawMessage) error { if len(checkpoint) == 0 { return fmt.Errorf("checkpoint is required: %w", ErrInvalidInput) } @@ -290,13 +298,19 @@ func (r *BunTaskRepo) WriteCheckpoint(ctx context.Context, id, generation int64, if rows, _ := result.RowsAffected(); rows != 1 { return ErrTaskLeaseLost } - if _, err := db.NewUpdate(). + // checkpoint must stay a json.RawMessage: bun renders a plain []byte as + // a bytea/blob literal, which PostgreSQL jsonb rejects. + result, err = db.NewUpdate(). Model((*model.TaskPayload)(nil)). Set("checkpoint_json = ?", checkpoint). Where("task_id = ?", id). - Exec(ctx); err != nil { + Exec(ctx) + if err != nil { return fmt.Errorf("writing task %d checkpoint: %w", id, err) } + if rows, _ := result.RowsAffected(); rows != 1 { + return fmt.Errorf("writing task %d checkpoint: payload row not found: %w", id, ErrNotFound) + } return nil }) } @@ -382,10 +396,12 @@ func (r *BunTaskRepo) ShortenLease(ctx context.Context, id, generation int64, du return fmt.Errorf("lease duration must be positive: %w", ErrInvalidInput) } now := time.Now() + shortened := now.Add(duration) result, err := r.db.NewUpdate(). Model((*model.Task)(nil)). Set("resume_mode = ?", model.TaskResumeModeRecover). - Set("lease_until = ?", now.Add(duration)). + // Shortening never extends a lease that already expires sooner. + Set("lease_until = CASE WHEN lease_until < ? THEN lease_until ELSE ? END", shortened, shortened). Set("updated_at = ?", now). Where("id = ? AND status = ?", id, model.TaskStatusRunning). Where("claim_generation = ?", generation). @@ -548,6 +564,61 @@ func (r *BunTaskRepo) AcknowledgeFailed(ctx context.Context, id int64, retention return nil } +// acknowledgeFailedMatching selects the failures one bulk dismissal covers. The +// preview and the dismissal itself share it, so the number an operator confirms +// is the number that is dismissed. +func acknowledgeFailedMatching(filter TaskAcknowledgeFilter) func(bun.QueryBuilder) bun.QueryBuilder { + return func(query bun.QueryBuilder) bun.QueryBuilder { + query = query. + Where("status = ? AND acknowledged_at IS NULL", model.TaskStatusFailed). + Where("finished_at IS NOT NULL AND finished_at <= ?", filter.FailedBefore) + if filter.Type != "" { + query = query.Where("type = ?", filter.Type) + } + return query + } +} + +func (r *BunTaskRepo) CountFailedMatching(ctx context.Context, filter TaskAcknowledgeFilter) (int, error) { + if filter.FailedBefore.IsZero() { + return 0, fmt.Errorf("counting failed tasks: %w", ErrInvalidInput) + } + count, err := r.db.NewSelect(). + Model((*model.Task)(nil)). + ApplyQueryBuilder(acknowledgeFailedMatching(filter)). + Count(ctx) + if err != nil { + return 0, fmt.Errorf("counting failed tasks: %w", err) + } + return count, nil +} + +func (r *BunTaskRepo) AcknowledgeFailedMatching( + ctx context.Context, + filter TaskAcknowledgeFilter, + retention time.Duration, +) (int, error) { + if retention <= 0 { + return 0, fmt.Errorf("retention must be positive: %w", ErrInvalidInput) + } + if filter.FailedBefore.IsZero() { + return 0, fmt.Errorf("acknowledging failed tasks: %w", ErrInvalidInput) + } + now := time.Now() + result, err := r.db.NewUpdate(). + Model((*model.Task)(nil)). + Set("acknowledged_at = ?", now). + Set("retention_until = ?", now.Add(retention)). + Set("updated_at = ?", now). + ApplyQueryBuilder(acknowledgeFailedMatching(filter)). + Exec(ctx) + if err != nil { + return 0, fmt.Errorf("acknowledging failed tasks: %w", err) + } + rows, _ := result.RowsAffected() + return int(rows), nil +} + func (r *BunTaskRepo) DeleteRetained(ctx context.Context, now time.Time, limit int) (int, error) { if now.IsZero() || limit < 1 { return 0, fmt.Errorf("deleting retained tasks: %w", ErrInvalidInput) @@ -594,7 +665,12 @@ func (r *BunTaskRepo) List(ctx context.Context, filter TaskListFilter) (TaskPage limit = 50 } var tasks []model.Task - query := withTaskPayload(r.db.NewSelect().Model(&tasks)). + // A listing never decodes inputs, so it leaves them out; manual-retry checks + // read the checkpoint, which stays. + query := r.db.NewSelect().Model(&tasks). + ColumnExpr("task.*"). + ColumnExpr("task_payload.checkpoint_json AS checkpoint"). + Join("JOIN task_payloads AS task_payload ON task_payload.task_id = task.id"). OrderExpr("task.id DESC"). Limit(limit + 1) if filter.Type != "" { @@ -686,32 +762,6 @@ func (r *BunTaskRepo) CountOverviewActivePipeline(ctx context.Context) ([]TaskPi return counts, nil } -func (r *BunTaskRepo) CountActiveObjectTasksByBucket(ctx context.Context, bucketID int64) (int64, error) { - var count int64 - err := r.db.NewRaw(`SELECT COUNT(*) - FROM tasks AS t - JOIN object_versions AS ov ON ov.version_id = t.subject_key - WHERE t.subject_type = 'object_version' - AND t.status IN ('pending', 'running') - AND ov.bucket_id = ?`, bucketID).Scan(ctx, &count) - if err != nil { - return 0, fmt.Errorf("counting active object tasks by bucket: %w", err) - } - return count, nil -} - -func (r *BunTaskRepo) CountActiveBucketTasksByBucketID(ctx context.Context, bucketID int64) (int64, error) { - count, err := r.db.NewSelect(). - Model((*model.Task)(nil)). - Where("subject_type = 'bucket' AND subject_key = ?", fmt.Sprint(bucketID)). - Where("status IN (?, ?)", model.TaskStatusPending, model.TaskStatusRunning). - Count(ctx) - if err != nil { - return 0, fmt.Errorf("counting active bucket tasks: %w", err) - } - return int64(count), nil -} - func nullableText(value string) any { if value == "" { return nil diff --git a/internal/db/repository/task_repo_test.go b/internal/db/repository/task_repo_test.go index 64e3cd5..9ddbf20 100644 --- a/internal/db/repository/task_repo_test.go +++ b/internal/db/repository/task_repo_test.go @@ -2,6 +2,7 @@ package repository_test import ( "context" + "errors" "strings" "sync" "testing" @@ -98,3 +99,169 @@ func TestTaskGCDoesNotDeleteTaskRecoveredAfterSelection(t *testing.T) { t.Fatalf("recovered task = %#v err=%v", stored, err) } } + +func enqueueAndClaimTask(t *testing.T, repos *repository.Repositories, key string, lease time.Duration) *model.Task { + t.Helper() + if _, created, err := repos.Tasks.Enqueue(t.Context(), &model.Task{ + Type: "repository_test", IdempotencyKey: key, InputVersion: 1, + Input: []byte(`{}`), InputHash: "test", Status: model.TaskStatusPending, + ResumeMode: model.TaskResumeModeExecute, AvailableAt: time.Now(), + }); err != nil || !created { + t.Fatalf("enqueue task %s: created=%v err=%v", key, created, err) + } + claimed, err := repos.Tasks.ClaimNext(t.Context(), lease) + if err != nil || claimed == nil { + t.Fatalf("claim task %s = %#v err=%v", key, claimed, err) + } + return claimed +} + +func TestWriteCheckpointStoresJSONText(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + claimed := enqueueAndClaimTask(t, repos, "checkpoint-text", time.Minute) + if err := repos.Tasks.WriteCheckpoint(t.Context(), claimed.ID, claimed.ClaimGeneration, []byte(`{"attempted":true}`)); err != nil { + t.Fatalf("write checkpoint: %v", err) + } + // A BLOB would still pass json_valid(); the schema promises TEXT JSON. + var storageClass string + if err := db.NewRaw(`SELECT typeof(checkpoint_json) FROM task_payloads WHERE task_id = ?`, claimed.ID).Scan(t.Context(), &storageClass); err != nil || storageClass != "text" { + t.Fatalf("checkpoint storage class = %q, err=%v", storageClass, err) + } + stored, err := repos.Tasks.GetByID(t.Context(), claimed.ID) + if err != nil { + t.Fatalf("load task: %v", err) + } + if string(stored.Checkpoint) != `{"attempted":true}` || stored.ResumeMode != model.TaskResumeModeRecover { + t.Fatalf("stored checkpoint = %s, resume mode = %s", stored.Checkpoint, stored.ResumeMode) + } +} + +// TestListKeepsCheckpointForRetryChecks checks that a task listing carries the +// checkpoint that manual-retry decisions read, and leaves the input out. +func TestListKeepsCheckpointForRetryChecks(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + claimed := enqueueAndClaimTask(t, repos, "list-checkpoint", time.Minute) + if err := repos.Tasks.WriteCheckpoint(t.Context(), claimed.ID, claimed.ClaimGeneration, []byte(`{"attempted":true}`)); err != nil { + t.Fatalf("write checkpoint: %v", err) + } + page, err := repos.Tasks.List(t.Context(), repository.TaskListFilter{Limit: 10}) + if err != nil || len(page.Tasks) != 1 { + t.Fatalf("List = %#v, err=%v", page, err) + } + if listed := page.Tasks[0]; string(listed.Checkpoint) != `{"attempted":true}` || len(listed.Input) != 0 { + t.Fatalf("listed checkpoint = %s, input = %s", listed.Checkpoint, listed.Input) + } +} + +// TestAcknowledgeFailedMatchingDismissesTheSelectedBacklog checks that one bulk +// dismissal covers unacknowledged failures of the selected operation that failed +// before the cutoff, and leaves everything else visible. +func TestAcknowledgeFailedMatchingDismissesTheSelectedBacklog(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := t.Context() + seedFailed := func(key string, taskType model.TaskType, failedAt time.Time) int64 { + t.Helper() + row, created, err := repos.Tasks.Enqueue(ctx, &model.Task{ + Type: taskType, IdempotencyKey: key, InputVersion: 1, + Input: []byte(`{}`), InputHash: key, Status: model.TaskStatusPending, + ResumeMode: model.TaskResumeModeExecute, AvailableAt: time.Now(), + }) + if err != nil || !created { + t.Fatalf("enqueue %s: created=%v err=%v", key, created, err) + } + claimed, err := repos.Tasks.ClaimNext(ctx, time.Minute) + if err != nil || claimed == nil || claimed.ID != row.ID { + t.Fatalf("claim %s = %#v err=%v", key, claimed, err) + } + if err := repos.Tasks.Settle(ctx, claimed.ID, claimed.ClaimGeneration, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, + FailureReason: new("store_not_started"), LastError: new("provider unavailable"), + }); err != nil { + t.Fatalf("settle %s: %v", key, err) + } + if _, err := db.NewRaw(`UPDATE tasks SET finished_at = ? WHERE id = ?`, failedAt, row.ID).Exec(ctx); err != nil { + t.Fatalf("stamp %s failure time: %v", key, err) + } + return row.ID + } + cutoff := time.Now().Add(-time.Minute) + matching := seedFailed("bulk-matching", model.TaskTypeStorageStore, cutoff.Add(-time.Minute)) + otherType := seedFailed("bulk-other-type", model.TaskTypeCacheEvict, cutoff.Add(-time.Minute)) + afterCutoff := seedFailed("bulk-after-cutoff", model.TaskTypeStorageStore, cutoff.Add(time.Minute)) + alreadyDismissed := seedFailed("bulk-already-dismissed", model.TaskTypeStorageStore, cutoff.Add(-time.Minute)) + if err := repos.Tasks.AcknowledgeFailed(ctx, alreadyDismissed, time.Hour); err != nil { + t.Fatalf("AcknowledgeFailed: %v", err) + } + + count, err := repos.Tasks.AcknowledgeFailedMatching(ctx, repository.TaskAcknowledgeFilter{ + Type: model.TaskTypeStorageStore, FailedBefore: cutoff, + }, time.Hour) + if err != nil || count != 1 { + t.Fatalf("AcknowledgeFailedMatching = %d, err=%v, want 1", count, err) + } + for _, check := range []struct { + id int64 + dismissed bool + what string + }{ + {matching, true, "matching failure"}, + {afterCutoff, false, "failure after the cutoff"}, + {otherType, false, "failure of another operation"}, + } { + stored, err := repos.Tasks.GetByID(ctx, check.id) + if err != nil || stored == nil { + t.Fatalf("load %s: %#v err=%v", check.what, stored, err) + } + if (stored.AcknowledgedAt != nil) != check.dismissed { + t.Fatalf("%s dismissed = %v, want %v", check.what, stored.AcknowledgedAt != nil, check.dismissed) + } + if check.dismissed && stored.RetentionUntil == nil { + t.Fatalf("%s was dismissed without a retention deadline", check.what) + } + } +} + +func TestWriteCheckpointRequiresPayloadRow(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + claimed := enqueueAndClaimTask(t, repos, "checkpoint-missing-payload", time.Minute) + if _, err := db.NewRaw(`DELETE FROM task_payloads WHERE task_id = ?`, claimed.ID).Exec(t.Context()); err != nil { + t.Fatalf("delete payload: %v", err) + } + err := repos.Tasks.WriteCheckpoint(t.Context(), claimed.ID, claimed.ClaimGeneration, []byte(`{"attempted":true}`)) + if !errors.Is(err, repository.ErrNotFound) { + t.Fatalf("checkpoint without payload err = %v, want ErrNotFound", err) + } + var resumeMode string + if err := db.NewRaw(`SELECT resume_mode FROM tasks WHERE id = ?`, claimed.ID).Scan(t.Context(), &resumeMode); err != nil || resumeMode != string(model.TaskResumeModeExecute) { + t.Fatalf("resume mode after rejected checkpoint = %q, err=%v", resumeMode, err) + } +} + +func TestShortenLeaseNeverExtendsLease(t *testing.T) { + repos := repository.NewRepositories(testDB(t)) + claimed := enqueueAndClaimTask(t, repos, "shorten-lease", 2*time.Second) + if err := repos.Tasks.ShortenLease(t.Context(), claimed.ID, claimed.ClaimGeneration, time.Hour); err != nil { + t.Fatalf("shorten lease: %v", err) + } + kept, err := repos.Tasks.GetByID(t.Context(), claimed.ID) + if err != nil { + t.Fatalf("load task: %v", err) + } + if kept.LeaseUntil == nil || kept.LeaseUntil.After(*claimed.LeaseUntil) { + t.Fatalf("lease after longer request = %v, want at most %v", kept.LeaseUntil, claimed.LeaseUntil) + } + if err := repos.Tasks.ShortenLease(t.Context(), claimed.ID, claimed.ClaimGeneration, 100*time.Millisecond); err != nil { + t.Fatalf("shorten lease: %v", err) + } + shortened, err := repos.Tasks.GetByID(t.Context(), claimed.ID) + if err != nil { + t.Fatalf("load task: %v", err) + } + if shortened.LeaseUntil == nil || !shortened.LeaseUntil.Before(*kept.LeaseUntil) { + t.Fatalf("lease after shorter request = %v, want before %v", shortened.LeaseUntil, kept.LeaseUntil) + } +} diff --git a/internal/model/deletion.go b/internal/model/deletion.go index d120cda..43a3969 100644 --- a/internal/model/deletion.go +++ b/internal/model/deletion.go @@ -49,6 +49,7 @@ type StorageCleanupCopy struct { ClientDataSetID *types.OnChainID `bun:"type:text"` PieceID types.OnChainID `bun:"type:text,notnull"` PieceCID string `bun:"type:text,notnull"` + Checksum string `bun:"type:text,notnull"` RetrievalURL *string `bun:"type:text,nullzero"` Status StorageCleanupCopyStatus `bun:"type:text,notnull,default:'pending'"` DeleteTxHash *string `bun:"type:text,nullzero"` diff --git a/internal/observability/adapters.go b/internal/observability/adapters.go index 7fd21a8..4ee97e9 100644 --- a/internal/observability/adapters.go +++ b/internal/observability/adapters.go @@ -52,7 +52,7 @@ func (s *StorageDataSetScanner) ScanWalletDataSets(ctx context.Context) ([]Chain } out := make([]ChainDataSet, 0, len(dataSets)) for _, dataSet := range dataSets { - if dataSet == nil || dataSet.DataSetInfo == nil { + if dataSet == nil || dataSet.DataSetID.IsZero() { continue } hasActivePieces := dataSet.HasActivePieces diff --git a/internal/observability/adapters_test.go b/internal/observability/adapters_test.go index 255e375..5f73e32 100644 --- a/internal/observability/adapters_test.go +++ b/internal/observability/adapters_test.go @@ -15,7 +15,7 @@ func TestStorageDataSetScannerMapsDataSetDetails(t *testing.T) { finder := &dataSetDetailsFinder{dataSets: []*storage.DataSetDetails{ nil, { - DataSetInfo: &warmstorage.DataSetInfo{ + DataSetInfo: warmstorage.DataSetInfo{ DataSetID: sdktypes.NewBigInt(1001), ClientDataSetID: sdktypes.NewBigInt(9001), ProviderID: sdktypes.NewBigInt(101), @@ -26,7 +26,7 @@ func TestStorageDataSetScannerMapsDataSetDetails(t *testing.T) { Metadata: map[string]string{"source": "synaps3", "bucket": "photos"}, }, { - DataSetInfo: &warmstorage.DataSetInfo{ + DataSetInfo: warmstorage.DataSetInfo{ DataSetID: sdktypes.NewBigInt(1002), ClientDataSetID: sdktypes.NewBigInt(9002), ProviderID: sdktypes.NewBigInt(102), @@ -37,7 +37,7 @@ func TestStorageDataSetScannerMapsDataSetDetails(t *testing.T) { Metadata: map[string]string{"source": "synaps3", "bucket": "empty"}, }, { - DataSetInfo: &warmstorage.DataSetInfo{ + DataSetInfo: warmstorage.DataSetInfo{ DataSetID: sdktypes.NewBigInt(1003), ClientDataSetID: sdktypes.NewBigInt(9003), ProviderID: sdktypes.NewBigInt(103), diff --git a/internal/storagecommit/advancer_test.go b/internal/storagecommit/advancer_test.go index 3f408d9..c19cebc 100644 --- a/internal/storagecommit/advancer_test.go +++ b/internal/storagecommit/advancer_test.go @@ -1068,6 +1068,102 @@ func (f commitStatusCheckerFunc) GetAddPiecesStatus( return f(ctx, input) } +func TestCommitCapacityWakesOnlyTheQueueHead(t *testing.T) { + db := testutil.NewTestDB(t) + repos := repository.NewRepositories(db) + _, copies, pieceCID := seedAdvancerCopies(t, db, 7) + taskIDs := make([]int64, len(copies)) + for i, copyRow := range copies { + task, _, err := repos.Tasks.Enqueue(t.Context(), &model.Task{ + Type: model.TaskTypeStorageCommit, IdempotencyKey: fmt.Sprintf("queue-%d", copyRow.ID), + InputVersion: 1, Input: []byte(`{}`), InputHash: "queue", AvailableAt: time.Now().Add(time.Hour), + }) + if err != nil { + t.Fatalf("enqueue commit task %d: %v", i, err) + } + taskIDs[i] = task.ID + if _, err := db.NewUpdate().Model((*model.StorageCopy)(nil)). + Set("active_task_id = ?", task.ID).Where("id = ?", copyRow.ID).Exec(t.Context()); err != nil { + t.Fatalf("bind commit task %d: %v", i, err) + } + } + reserve := func(i int) storagecommit.ReservationState { + t.Helper() + result, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ + Copy: advancerCopyIdentity(copies[i]), AttemptID: fmt.Sprintf("queue-attempt-%d", i), + }) + if err != nil { + t.Fatalf("reserve copy %d: %v", i, err) + } + return result.State + } + release := func(i int) { + t.Helper() + if err := repos.Contents.ReleaseCommitAttempt(t.Context(), storagecommit.ReleaseInput{ + Copy: advancerCopyIdentity(copies[i]), AttemptID: fmt.Sprintf("queue-attempt-%d", i), + Reason: storagecommit.ReleaseBeforeSubmitCanceled, ClearReadyAt: true, ClearExtraData: true, + }); err != nil { + t.Fatalf("release copy %d: %v", i, err) + } + } + runnable := func(step string, want ...int) { + t.Helper() + var got []int + for i, id := range taskIDs { + task, err := repos.Tasks.GetByID(t.Context(), id) + if err != nil { + t.Fatalf("load commit task %d: %v", i, err) + } + if !task.AvailableAt.After(time.Now()) { + got = append(got, i) + } + } + if fmt.Sprint(got) != fmt.Sprint(want) { + t.Fatalf("%s: runnable queue tasks = %v, want %v", step, got, want) + } + } + + for i := range 7 { + want := storagecommit.ReservationAcquired + if i >= storagecommit.MaxActiveAttemptsPerDataSet { + want = storagecommit.ReservationWaiting + } + if state := reserve(i); state != want { + t.Fatalf("reserve copy %d = %s, want %s", i, state, want) + } + } + runnable("while every slot is taken") + + if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ + Copy: advancerCopyIdentity(copies[0]), AttemptID: "queue-attempt-0", ExtraDataHex: "abcd", + }); err != nil { + t.Fatalf("mark copy 0 attempted: %v", err) + } + pieceID := idtypes.OnChainIDFromSDK(sdktypes.NewBigInt(5001)) + if err := repos.Contents.MarkUploadCopyCommitted(t.Context(), repository.MarkUploadCopyCommittedInput{ + StorageCopyID: copies[0].ID, ContentID: copies[0].ContentID, CopyIndex: 0, + PieceCID: pieceCID.String(), PieceID: &pieceID, RetrievalURL: "https://provider.example/piece", + CommitExtraDataHex: "abcd", CommitTransactionID: "0x01", CommitAttemptID: "queue-attempt-0", + CommitConfirmedTransactionID: "0x01", + }); err != nil { + t.Fatalf("confirm copy 0: %v", err) + } + runnable("after a confirmation frees one slot", 4) + + if state := reserve(4); state != storagecommit.ReservationAcquired { + t.Fatalf("woken head reservation = %s", state) + } + runnable("after the head takes the last slot", 4) + + release(1) + release(2) + runnable("after two releases", 4, 5) + if state := reserve(5); state != storagecommit.ReservationAcquired { + t.Fatalf("second head reservation = %s", state) + } + runnable("after the head reserves with a slot left", 4, 5, 6) +} + func seedAdvancerCopies(t *testing.T, db *bun.DB, count int) (*model.StorageDataSet, []model.StorageCopy, cid.Cid) { t.Helper() pieceCID := advancerTestCID(t) diff --git a/internal/storagereplacement/codes.go b/internal/storagereplacement/codes.go index 8604cac..3192011 100644 --- a/internal/storagereplacement/codes.go +++ b/internal/storagereplacement/codes.go @@ -6,6 +6,7 @@ import "errors" // on these, so treat them as part of the public contract. const ( CodeActive = "replacement_active" + CodeTargetCreating = "replacement_target_creating" CodeSuperseded = "replacement_superseded" CodeNotRetryable = "replacement_not_retryable" CodeTaskRunning = "replacement_task_running" @@ -23,6 +24,8 @@ func Code(err error) string { switch { case errors.Is(err, ErrActiveReplacement): return CodeActive + case errors.Is(err, ErrTargetCreating): + return CodeTargetCreating case errors.Is(err, ErrSuperseded): return CodeSuperseded case errors.Is(err, ErrNotRetryable): diff --git a/internal/storagereplacement/errors.go b/internal/storagereplacement/errors.go index 018b1eb..557dd0f 100644 --- a/internal/storagereplacement/errors.go +++ b/internal/storagereplacement/errors.go @@ -7,6 +7,11 @@ var ( // that has not reached a terminal state. ErrActiveReplacement = errors.New("data set already has an active replacement") + // ErrTargetCreating means an earlier replacement of this replica may still + // get its target storage service created on chain, so superseding it would + // leave that service untracked. + ErrTargetCreating = errors.New("an earlier replacement target is still being created") + // ErrSuperseded means a later confirmation took ownership of this work. ErrSuperseded = errors.New("replacement has been superseded") diff --git a/internal/synapse/interfaces.go b/internal/synapse/interfaces.go index f6bedb8..3b72895 100644 --- a/internal/synapse/interfaces.go +++ b/internal/synapse/interfaces.go @@ -28,6 +28,12 @@ type ProviderTarget interface { StorageTarget CreateDataSet(context.Context, *storage.CreateDataSetOptions) (*storage.CreateDataSetResult, error) WaitForDataSetCreated(context.Context, storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) + // ContextIdentity is the payer, chain, and record keeper the target signs + // for. Client data set IDs are only unique within it. + ContextIdentity() storage.ContextIdentity + // FindDataSetByClientDataSetID reads the chain for the data set a create + // request with this ID produced. Not found only means none is visible yet. + FindDataSetByClientDataSetID(context.Context, sdktypes.BigInt) (storage.DataSetRef, bool, error) } // DataSetTarget is an immutable existing data set used for piece operations. @@ -80,6 +86,15 @@ type ParkedPieceChecker interface { // have both passed. type ServiceTerminator interface { TerminateService(ctx context.Context, dataSetID sdktypes.BigInt) (*TerminationResult, error) + // VerifyServicePayer reads the data set's record and fails with + // ErrServicePaidByAnother when this wallet does not pay for it; any other + // error means the record could not be read. It sends nothing, so a caller can + // check before committing to a termination. + VerifyServicePayer(ctx context.Context, dataSetID sdktypes.BigInt) error + // ContextIdentity reports the payer, chain, and record keeper this + // terminator signs for, so a caller can tell that a recorded request was + // made under another one. + ContextIdentity() storage.ContextIdentity } // TerminationResult records what the chain agreed to. EndEpoch is the epoch at diff --git a/internal/synapse/storage_client.go b/internal/synapse/storage_client.go index 2dd448f..9b9365c 100644 --- a/internal/synapse/storage_client.go +++ b/internal/synapse/storage_client.go @@ -10,6 +10,7 @@ import ( "github.com/ipfs/go-cid" "github.com/strahe/synapse-go/storage" sdktypes "github.com/strahe/synapse-go/types" + "github.com/strahe/synapse-go/warmstorage" ) // StorageServiceAdapter adapts synapse-go's concrete immutable storage @@ -17,10 +18,26 @@ import ( type StorageServiceAdapter struct { service *storage.Service terminator storageServiceTerminator + dataSets dataSetStateReader + identity storage.ContextIdentity } -func AdaptStorageService(service *storage.Service) *StorageServiceAdapter { - return &StorageServiceAdapter{service: service, terminator: service} +// AdaptStorageService wraps the SDK storage service. dataSets reads the FWSS +// service state that termination checks before every request, and identity is +// what this service signs for: destructive requests refuse a record that +// belongs to another payer. +func AdaptStorageService(service *storage.Service, dataSets *warmstorage.Service, identity storage.ContextIdentity) *StorageServiceAdapter { + adapter := &StorageServiceAdapter{service: service, terminator: service, identity: identity} + if dataSets != nil { + adapter.dataSets = dataSets + } + return adapter +} + +// ContextIdentity reports the payer, chain, and record keeper this service +// signs for. +func (s *StorageServiceAdapter) ContextIdentity() storage.ContextIdentity { + return s.identity } func (s *StorageServiceAdapter) Download(ctx context.Context, pieceCID cid.Cid, opts *storage.DownloadOptions) (io.ReadCloser, error) { @@ -121,7 +138,7 @@ func (s *StorageServiceAdapter) FindMatchingDataSet( } var best *storage.DataSetDetails for _, dataSet := range dataSets { - if dataSet == nil || dataSet.DataSetInfo == nil || dataSet.DataSetID.IsZero() || + if dataSet == nil || dataSet.DataSetID.IsZero() || !dataSet.ProviderID.Equal(providerID) || dataSet.PDPEndEpoch != 0 || !dataSet.IsLive || !dataSet.IsManaged || !maps.Equal(dataSet.Metadata, wanted) { continue @@ -209,6 +226,19 @@ func (c *providerTargetAdapter) WaitForDataSetCreated(ctx context.Context, submi return result, NormalizeProviderOperationError(ctx, err) } +func (c *providerTargetAdapter) ContextIdentity() storage.ContextIdentity { + return c.provider.ContextIdentity() +} + +// FindDataSetByClientDataSetID reads the chain rather than the provider, so its +// errors are not classified as provider failures. +func (c *providerTargetAdapter) FindDataSetByClientDataSetID( + ctx context.Context, + clientDataSetID sdktypes.BigInt, +) (storage.DataSetRef, bool, error) { + return c.provider.FindDataSetByClientDataSetID(ctx, clientDataSetID) +} + type dataSetTargetAdapter struct { storageTargetAdapter dataSet *storage.DataSetContext diff --git a/internal/synapse/storage_client_test.go b/internal/synapse/storage_client_test.go index a91794f..c42635c 100644 --- a/internal/synapse/storage_client_test.go +++ b/internal/synapse/storage_client_test.go @@ -45,7 +45,7 @@ func TestStorageServiceAdapterSelectUploadTargetsPreservesPartialSelection(t *te t.Fatalf("storage.New: %v", err) } - targets, err := AdaptStorageService(service).SelectUploadTargets(t.Context(), storage.SelectUploadContextsOptions{Copies: 2}) + targets, err := AdaptStorageService(service, nil, storage.ContextIdentity{}).SelectUploadTargets(t.Context(), storage.SelectUploadContextsOptions{Copies: 2}) if !IsNoProviderCandidates(err) || len(targets) != 1 { t.Fatalf("targets = %#v, error = %T %v; want one usable target and NoProviderCandidatesError", targets, err, err) } @@ -78,7 +78,7 @@ func TestStorageServiceAdapterFindMatchingDataSetUsesExactMetadataAndStablePrefe t.Fatalf("storage.New: %v", err) } - ref, err := AdaptStorageService(service).FindMatchingDataSet(t.Context(), providerID, map[string]string{ + ref, err := AdaptStorageService(service, nil, storage.ContextIdentity{}).FindMatchingDataSet(t.Context(), providerID, map[string]string{ "source": "caller-value-must-not-override-synaps3", "bucket": "photos", "withCDN": "caller-value-must-not-enable-cdn", @@ -116,7 +116,7 @@ func TestStorageServiceAdapterFindMatchingDataSetRequiresEmptyMetadataKey(t *tes t.Fatalf("storage.New: %v", err) } - ref, err := AdaptStorageService(service).FindMatchingDataSet( + ref, err := AdaptStorageService(service, nil, storage.ContextIdentity{}).FindMatchingDataSet( t.Context(), providerID, map[string]string{"bucket": "photos"}, @@ -157,7 +157,7 @@ func TestStorageServiceAdapterPrepareUploadReturnsCostsWithoutFunding(t *testing if err != nil { t.Fatalf("storage.New: %v", err) } - adapter := AdaptStorageService(service) + adapter := AdaptStorageService(service, nil, storage.ContextIdentity{}) target := newProviderTargetAdapter(providerContext) got, err := adapter.PrepareUpload(t.Context(), 4096, []StorageTarget{target}) @@ -202,7 +202,7 @@ func (f *staticDataSetFinder) FindDataSets(_ context.Context, payer common.Addre func dataSetDetails(dataSetID, clientDataSetID, providerID uint64, live, managed, active bool, endEpoch sdktypes.Epoch, metadata map[string]string) *storage.DataSetDetails { return &storage.DataSetDetails{ - DataSetInfo: &warmstorage.DataSetInfo{ + DataSetInfo: warmstorage.DataSetInfo{ DataSetID: sdktypes.NewBigInt(dataSetID), ClientDataSetID: sdktypes.NewBigInt(clientDataSetID), ProviderID: sdktypes.NewBigInt(providerID), diff --git a/internal/synapse/terminate.go b/internal/synapse/terminate.go index 3289f1e..6b6aef5 100644 --- a/internal/synapse/terminate.go +++ b/internal/synapse/terminate.go @@ -7,9 +7,11 @@ import ( "math/big" "time" + "github.com/ethereum/go-ethereum/common" "github.com/strahe/synapse-go/pdp" "github.com/strahe/synapse-go/storage" sdktypes "github.com/strahe/synapse-go/types" + "github.com/strahe/synapse-go/warmstorage" ) const providerTerminationWaitTimeout = 30 * time.Second @@ -18,6 +20,11 @@ type storageServiceTerminator interface { TerminateService(context.Context, sdktypes.BigInt, *storage.TerminateServiceOptions) (*storage.TerminateServiceResult, error) } +// dataSetStateReader reads a data set's FWSS service state from the chain. +type dataSetStateReader interface { + GetDataSet(context.Context, sdktypes.BigInt) (*warmstorage.DataSetInfo, error) +} + // TerminationBlockedError means the service cannot be terminated until an // operator resolves something the gateway must not decide on its own, such as // settling outstanding payment debt. It is never retried automatically. @@ -51,29 +58,50 @@ func IsTerminationBlocked(err error) bool { return errors.As(err, &blocked) } +// ErrServicePaidByAnother means the chain records the data set as paid for by a +// wallet other than the one signing. Under this configuration its number names +// someone else's service, so it is never terminated; an operator has to put the +// original wallet or network back. +var ErrServicePaidByAnother = errors.New("storage service is paid for by another wallet") + // TerminateService ends the storage service for one data set. It is the // destructive boundary of provider replacement and must only be called after // the retirement safety gate passes. // -// Termination is relayed through the provider first. A provider-side failure -// falls back to the direct FWSS transaction path; a pending relay does not, -// because the provider has already accepted it. An already-terminated service -// reports its recorded end epoch rather than failing. +// The chain is read before every request: a service that already ended reports +// its recorded end epoch and nothing is sent, so calling this again after an +// unobserved outcome cannot end a service twice. Termination is relayed through +// the provider first. Payment debt and a pending relay are returned as they are; +// any other relay failure falls back to the direct FWSS transaction only while +// the chain still shows the service running. func (s *StorageServiceAdapter) TerminateService(ctx context.Context, dataSetID sdktypes.BigInt) (*TerminationResult, error) { - if s == nil || s.terminator == nil { + if s == nil || s.terminator == nil || s.dataSets == nil { return nil, errors.New("storage service terminator is not configured") } + if ended, err := s.recordedTermination(ctx, dataSetID); err != nil || ended != nil { + return ended, err + } res, err := s.terminator.TerminateService(ctx, dataSetID, &storage.TerminateServiceOptions{ ProviderWaitTimeout: providerTerminationWaitTimeout, }) if err != nil { providerErr := normalizeTerminationError(ctx, err) - if terminationPending(err) || ctx.Err() != nil { + if IsTerminationBlocked(providerErr) || terminationPending(err) || ctx.Err() != nil { return nil, providerErr } + ended, readErr := s.recordedTermination(ctx, dataSetID) + if readErr != nil { + return nil, fmt.Errorf("provider termination failed (%v) and the service state is unknown: %w", providerErr, readErr) + } + if ended != nil { + return ended, nil + } res, err = s.terminator.TerminateService(ctx, dataSetID, &storage.TerminateServiceOptions{SkipProvider: true}) if err != nil { directErr := NormalizeProviderOperationError(ctx, err) + if ended, readErr := s.recordedTermination(ctx, dataSetID); readErr == nil && ended != nil { + return ended, nil + } return nil, fmt.Errorf("direct termination failed after provider relay error (%v): %w", providerErr, directErr) } } @@ -87,6 +115,38 @@ func (s *StorageServiceAdapter) TerminateService(ctx context.Context, dataSetID return out, nil } +// VerifyServicePayer refuses a data set this wallet does not pay for. It only +// reads the chain, so retirement runs it before committing to a request. +func (s *StorageServiceAdapter) VerifyServicePayer(ctx context.Context, dataSetID sdktypes.BigInt) error { + if s == nil || s.dataSets == nil { + return errors.New("storage service terminator is not configured") + } + _, err := s.recordedTermination(ctx, dataSetID) + return err +} + +// recordedTermination returns the recorded end of a service the chain already +// shows terminated, or nil while the service is still running. +func (s *StorageServiceAdapter) recordedTermination(ctx context.Context, dataSetID sdktypes.BigInt) (*TerminationResult, error) { + info, err := s.dataSets.GetDataSet(ctx, dataSetID) + if err != nil { + return nil, fmt.Errorf("reading storage service state: %w", err) + } + if info == nil { + return nil, errors.New("reading storage service state: no data set record") + } + // The direct path carries no payer check of its own, so a record that + // belongs to someone else is refused here rather than terminated. + if s.identity.Payer != (common.Address{}) && info.Payer != s.identity.Payer { + return nil, fmt.Errorf("reading storage service state: data set %s is paid for by %s, not %s: %w", + dataSetID.String(), info.Payer.Hex(), s.identity.Payer.Hex(), ErrServicePaidByAnother) + } + if info.PDPEndEpoch == 0 { + return nil, nil + } + return &TerminationResult{EndEpoch: int64(info.PDPEndEpoch)}, nil +} + func terminationPending(err error) bool { var pending *pdp.TerminateServicePendingError return errors.As(err, &pending) @@ -96,14 +156,12 @@ func normalizeTerminationError(ctx context.Context, err error) error { if err == nil { return nil } - var debt *storage.TerminateServiceDebtError - if errors.As(err, &debt) { + if debt, ok := errors.AsType[*storage.TerminateServiceDebtError](err); ok { return &TerminationBlockedError{Reason: "payment_debt", Shortfall: debt.Shortfall, Err: err} } // The provider accepted the request but has not published it yet. That // resolves on its own, so it is a dependency wait rather than a failure. - var pending *pdp.TerminateServicePendingError - if errors.As(err, &pending) { + if _, ok := errors.AsType[*pdp.TerminateServicePendingError](err); ok { return &ProviderUnavailableError{Cause: err} } return NormalizeProviderOperationError(ctx, err) diff --git a/internal/synapse/terminate_test.go b/internal/synapse/terminate_test.go index 2df4a76..3a77281 100644 --- a/internal/synapse/terminate_test.go +++ b/internal/synapse/terminate_test.go @@ -9,9 +9,11 @@ import ( "strings" "testing" + "github.com/ethereum/go-ethereum/common" "github.com/strahe/synapse-go/pdp" "github.com/strahe/synapse-go/storage" sdktypes "github.com/strahe/synapse-go/types" + "github.com/strahe/synapse-go/warmstorage" ) type terminatorCall struct { @@ -41,15 +43,35 @@ func (s *stubStorageServiceTerminator) TerminateService( return result, nil } +// stubDataSetStateReader reports successive PDP end epochs, repeating the last. +// Zero means the service is still running. +type stubDataSetStateReader struct { + endEpochs []int64 + payer common.Address + err error + reads int +} + +func (s *stubDataSetStateReader) GetDataSet(context.Context, sdktypes.BigInt) (*warmstorage.DataSetInfo, error) { + s.reads++ + if s.err != nil { + return nil, s.err + } + var epoch int64 + if len(s.endEpochs) > 0 { + epoch = s.endEpochs[min(s.reads, len(s.endEpochs))-1] + } + return &warmstorage.DataSetInfo{PDPEndEpoch: sdktypes.Epoch(epoch), Payer: s.payer}, nil +} + +func runningService() *stubDataSetStateReader { return &stubDataSetStateReader{} } + func TestStorageServiceAdapterTerminationFallsBackToDirect(t *testing.T) { stub := &stubStorageServiceTerminator{ results: []*storage.TerminateServiceResult{nil, {EndEpoch: 84}}, - errors: []error{ - &storage.TerminateServiceDebtError{Shortfall: big.NewInt(1)}, - nil, - }, + errors: []error{errors.New("provider relay failed"), nil}, } - adapter := &StorageServiceAdapter{terminator: stub} + adapter := &StorageServiceAdapter{terminator: stub, dataSets: runningService()} got, err := adapter.TerminateService(context.Background(), sdktypes.NewBigInt(42)) if err != nil { t.Fatalf("TerminateService: %v", err) @@ -70,7 +92,7 @@ func TestStorageServiceAdapterTerminationFallsBackAfterProviderSubTimeout(t *tes results: []*storage.TerminateServiceResult{nil, {EndEpoch: 91}}, errors: []error{context.DeadlineExceeded, nil}, } - adapter := &StorageServiceAdapter{terminator: stub} + adapter := &StorageServiceAdapter{terminator: stub, dataSets: runningService()} got, err := adapter.TerminateService(context.Background(), sdktypes.NewBigInt(42)) if err != nil { t.Fatalf("TerminateService: %v", err) @@ -85,7 +107,7 @@ func TestStorageServiceAdapterTerminationFallsBackAfterProviderSubTimeout(t *tes func TestStorageServiceAdapterTerminationDoesNotDuplicatePendingRelay(t *testing.T) { stub := &stubStorageServiceTerminator{errors: []error{&pdp.TerminateServicePendingError{Message: "queued"}}} - adapter := &StorageServiceAdapter{terminator: stub} + adapter := &StorageServiceAdapter{terminator: stub, dataSets: runningService()} _, err := adapter.TerminateService(context.Background(), sdktypes.NewBigInt(42)) if !IsProviderUnavailable(err) { t.Fatalf("error = %T %v, want dependency wait", err, err) @@ -99,7 +121,7 @@ func TestStorageServiceAdapterTerminationHonorsCallerCancellation(t *testing.T) ctx, cancel := context.WithCancel(context.Background()) cancel() stub := &stubStorageServiceTerminator{errors: []error{context.Canceled}} - adapter := &StorageServiceAdapter{terminator: stub} + adapter := &StorageServiceAdapter{terminator: stub, dataSets: runningService()} _, err := adapter.TerminateService(ctx, sdktypes.NewBigInt(42)) if !errors.Is(err, context.Canceled) { t.Fatalf("error = %v, want context cancellation", err) @@ -114,13 +136,114 @@ func TestStorageServiceAdapterTerminationPreservesDirectFailure(t *testing.T) { stub := &stubStorageServiceTerminator{ errors: []error{errors.New("provider unavailable"), directErr}, } - adapter := &StorageServiceAdapter{terminator: stub} + adapter := &StorageServiceAdapter{terminator: stub, dataSets: runningService()} _, err := adapter.TerminateService(context.Background(), sdktypes.NewBigInt(42)) if !errors.Is(err, directErr) || !strings.Contains(err.Error(), "provider unavailable") { t.Fatalf("error = %v, want both provider and direct evidence", err) } } +// Payment debt needs the operator. Falling back to the direct path would hide +// it behind whatever that path reports. +func TestStorageServiceAdapterTerminationReportsPaymentDebtWithoutFallback(t *testing.T) { + stub := &stubStorageServiceTerminator{errors: []error{&storage.TerminateServiceDebtError{Shortfall: big.NewInt(1)}}} + adapter := &StorageServiceAdapter{terminator: stub, dataSets: runningService()} + _, err := adapter.TerminateService(context.Background(), sdktypes.NewBigInt(42)) + if !IsTerminationBlocked(err) { + t.Fatalf("error = %T %v, want a blocked termination", err, err) + } + if len(stub.calls) != 1 { + t.Fatalf("termination requests = %d, want no direct fallback", len(stub.calls)) + } +} + +// A service the chain already shows terminated is never sent another request, +// which is what makes repeating termination after an unobserved outcome safe. +func TestStorageServiceAdapterTerminationReadsTheChainBeforeSending(t *testing.T) { + tests := []struct { + name string + endEpochs []int64 + errs []error + wantEpoch int64 + wantCalls int + }{ + {name: "already terminated", endEpochs: []int64{77}, wantEpoch: 77}, + {name: "relay landed despite its error", endEpochs: []int64{0, 77}, errs: []error{errors.New("provider relay failed")}, wantEpoch: 77, wantCalls: 1}, + { + name: "direct path failed after the service ended", endEpochs: []int64{0, 0, 88}, + errs: []error{errors.New("provider relay failed"), errors.New("already terminated")}, wantEpoch: 88, wantCalls: 2, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stub := &stubStorageServiceTerminator{errors: tt.errs} + adapter := &StorageServiceAdapter{terminator: stub, dataSets: &stubDataSetStateReader{endEpochs: tt.endEpochs}} + got, err := adapter.TerminateService(context.Background(), sdktypes.NewBigInt(42)) + if err != nil || got == nil || got.EndEpoch != tt.wantEpoch { + t.Fatalf("result = %#v err=%v, want recorded end epoch %d", got, err, tt.wantEpoch) + } + if len(stub.calls) != tt.wantCalls { + t.Fatalf("termination requests = %d, want %d", len(stub.calls), tt.wantCalls) + } + }) + } +} + +func TestStorageServiceAdapterTerminationRefusesToSendWhileStateIsUnknown(t *testing.T) { + stub := &stubStorageServiceTerminator{} + adapter := &StorageServiceAdapter{terminator: stub, dataSets: &stubDataSetStateReader{err: errors.New("rpc unavailable")}} + if _, err := adapter.TerminateService(context.Background(), sdktypes.NewBigInt(42)); err == nil { + t.Fatal("termination succeeded without knowing the service state") + } + if len(stub.calls) != 0 { + t.Fatalf("termination requests = %d, want none while the service state is unknown", len(stub.calls)) + } +} + +// A data set ID is only a number, and the record it names may be paid for by +// another wallet. The direct FWSS path carries no payer check of its own, so +// the adapter refuses a record it does not pay for rather than ending it. +func TestStorageServiceAdapterTerminationRefusesADataSetItDoesNotPayFor(t *testing.T) { + t.Parallel() + + ours := common.HexToAddress("0x00000000000000000000000000000000000000a1") + theirs := common.HexToAddress("0x00000000000000000000000000000000000000c3") + tests := []struct { + name string + payer common.Address + wantCalls int + }{ + {name: "someone else's data set", payer: theirs}, + {name: "our own data set", payer: ours, wantCalls: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + stub := &stubStorageServiceTerminator{results: []*storage.TerminateServiceResult{{EndEpoch: 84}}} + adapter := &StorageServiceAdapter{ + terminator: stub, dataSets: &stubDataSetStateReader{payer: tt.payer}, + identity: storage.ContextIdentity{Payer: ours}, + } + refused := tt.wantCalls == 0 + err := adapter.VerifyServicePayer(context.Background(), sdktypes.NewBigInt(42)) + if refused != errors.Is(err, ErrServicePaidByAnother) || (!refused && err != nil) { + t.Fatalf("VerifyServicePayer err = %v, want refused=%v", err, refused) + } + _, err = adapter.TerminateService(context.Background(), sdktypes.NewBigInt(42)) + if refused && (!errors.Is(err, ErrServicePaidByAnother) || !strings.Contains(err.Error(), theirs.Hex())) { + t.Fatalf("err = %v, want a refusal naming the paying wallet", err) + } + if !refused && err != nil { + t.Fatalf("terminating our own data set: %v", err) + } + if len(stub.calls) != tt.wantCalls { + t.Fatalf("termination requests = %d, want %d", len(stub.calls), tt.wantCalls) + } + }) + } +} + // Outstanding debt is a decision for the operator, not something to retry, so // it must never look like a transient provider problem. func TestNormalizeTerminationErrorReportsPaymentDebtAsBlocked(t *testing.T) { diff --git a/internal/systemtest/filecoin.go b/internal/systemtest/filecoin.go index 4ad478e..4f85a8b 100644 --- a/internal/systemtest/filecoin.go +++ b/internal/systemtest/filecoin.go @@ -334,6 +334,9 @@ func (c *memoryProviderTarget) CreateDataSet(ctx context.Context, opts *storage. } clientIDValue, _ := id.Uint64() clientID := sdktypes.NewBigInt(clientIDValue + 500000) + if opts != nil && opts.ClientDataSetID != nil { + clientID = opts.ClientDataSetID.Copy() + } txID := "create-" + id.String() m.submissions[txID] = id.Copy() dataSet := &memoryDataSet{ @@ -359,6 +362,29 @@ func (c *memoryProviderTarget) CreateDataSet(ctx context.Context, opts *storage. return &storage.CreateDataSetResult{TransactionID: txID, DataSet: ref}, nil } +func (c *memoryProviderTarget) ContextIdentity() storage.ContextIdentity { + return storage.ContextIdentity{ + Payer: common.HexToAddress("0x00000000000000000000000000000000000000a1"), + ChainID: 314159, + RecordKeeper: common.HexToAddress("0x00000000000000000000000000000000000000b2"), + } +} + +func (c *memoryProviderTarget) FindDataSetByClientDataSetID(ctx context.Context, clientID sdktypes.BigInt) (storage.DataSetRef, bool, error) { + if err := ctx.Err(); err != nil { + return storage.DataSetRef{}, false, err + } + c.filecoin.mu.RLock() + defer c.filecoin.mu.RUnlock() + for _, dataSet := range c.filecoin.dataSets { + if dataSet.provider.Equal(c.provider) && dataSet.clientID.Equal(clientID) { + ref, err := storage.NewDataSetRef(c.provider, dataSet.id, dataSet.clientID) + return ref, err == nil, err + } + } + return storage.DataSetRef{}, false, nil +} + func (c *memoryProviderTarget) WaitForDataSetCreated(ctx context.Context, submission storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) { if err := ctx.Err(); err != nil { return nil, err @@ -615,6 +641,22 @@ func copyBigIntPtr(value sdktypes.BigInt) *sdktypes.BigInt { } // GetWalletInfo returns a complete, funded wallet snapshot. +// ContextIdentity reports what this fake signs for, matching the identity its +// provider contexts carry. +func (m *MemoryFilecoin) ContextIdentity() storage.ContextIdentity { + return storage.ContextIdentity{ + Payer: common.HexToAddress("0x00000000000000000000000000000000000000a1"), + ChainID: 314159, + RecordKeeper: common.HexToAddress("0x00000000000000000000000000000000000000b2"), + } +} + +// VerifyServicePayer accepts every data set: the fake only holds data sets its +// own wallet created. +func (m *MemoryFilecoin) VerifyServicePayer(context.Context, sdktypes.BigInt) error { + return nil +} + // TerminateService ends a data set's service. The fake reports an end of term // the chain has already reached, because a system test exercises the operator // flow rather than chain timing; use terminationDelay to make retirement wait. diff --git a/internal/task/contract.go b/internal/task/contract.go index f8f028b..2c5b7f7 100644 --- a/internal/task/contract.go +++ b/internal/task/contract.go @@ -21,6 +21,7 @@ var ( ErrInvalidResult = errors.New("invalid task result") ErrRegistryFrozen = errors.New("task registry is frozen") ErrEffectForbidden = errors.New("external effects are forbidden during recovery") + ErrResourceBusy = errors.New("task resource is busy") ErrCodecPanic = errors.New("task input codec panicked") ErrInvalidCanonical = errors.New("task input codec returned invalid canonical JSON") ) @@ -136,6 +137,7 @@ type Result struct { kind ResultKind delay time.Duration retryBackoff bool + resourceWait bool resumeMode model.TaskResumeMode waitReason string failureReason string @@ -155,6 +157,16 @@ func Suspend(mode model.TaskResumeMode, delay time.Duration, reason, message str } } +// ResourceWait yields a task that found its resource gate full. It resumes in +// execute mode after a backoff that grows with the task's consecutive waits, +// and it consumes no retry budget. +func ResourceWait(message string) Result { + return Result{ + kind: resultSuspend, resumeMode: model.TaskResumeModeExecute, resourceWait: true, + waitReason: "resource", message: message, + } +} + func Retry(err error, failureReason string, delay time.Duration, settlement Settlement) Result { return Result{ kind: resultRetry, resumeMode: model.TaskResumeModeRecover, delay: delay, @@ -233,6 +245,10 @@ func (e Execution) WriteCheckpointWith(ctx context.Context, value any, settlemen return e.checkpoint(ctx, value, settlement) } +// WithResource runs fn while holding one slot of resource. It never waits for a +// slot: a full gate returns ErrResourceBusy, which handlers turn into +// ResourceWait. A nested call for a resource the context already holds reuses +// that slot. func (e Execution) WithResource(ctx context.Context, resource Resource, fn func(context.Context) error) error { if e.Mode() != model.TaskResumeModeExecute { return ErrEffectForbidden diff --git a/internal/task/engine.go b/internal/task/engine.go index 91086de..34b4032 100644 --- a/internal/task/engine.go +++ b/internal/task/engine.go @@ -30,9 +30,11 @@ type EngineConfig struct { } const ( - retryBaseDelay = 10 * time.Second - retryMaximumDelay = 5 * time.Minute - retryJitterFraction = 0.20 + retryBaseDelay = 10 * time.Second + retryMaximumDelay = 5 * time.Minute + resourceWaitBaseDelay = 2 * time.Second + resourceWaitMaximumDelay = time.Minute + backoffJitterFraction = 0.20 ) // Engine is the only task claimant and lease owner. @@ -48,7 +50,12 @@ type Engine struct { settlementRetryDelays []time.Duration renewalRetryDelays []time.Duration retryDelay func(int) time.Duration - lastTick atomic.Int64 + resourceWaitDelay func(int) time.Duration + resourceWaitMu sync.Mutex + // resourceWaits counts each task's consecutive resource waits. It is kept + // in memory only; a restart merely restarts the backoff. + resourceWaits map[int64]int + lastTick atomic.Int64 } type recoveryRequest struct { @@ -80,6 +87,8 @@ func NewEngine(config EngineConfig, repos *repository.Repositories, registry *Re settlementRetryDelays: []time.Duration{0, time.Second, 2 * time.Second, 4 * time.Second}, renewalRetryDelays: []time.Duration{0, time.Second, 2 * time.Second, 4 * time.Second}, retryDelay: defaultRetryDelay, + resourceWaitDelay: defaultResourceWaitDelay, + resourceWaits: make(map[int64]int), }, nil } @@ -357,6 +366,7 @@ func (e *Engine) commitResult(ctx context.Context, claimed *model.Task, result R }) cancel() if lastErr == nil { + e.recordResourceWait(claimed.ID, result.resourceWait) e.notifyTaskSettled(claimed, transition) return nil } @@ -427,9 +437,13 @@ func (e *Engine) transitionFor(claimed *model.Task, result Result) repository.Ta transition.Status = model.TaskStatusCompleted transition.RetentionUntil = &retention case resultSuspend: + delay := result.delay + if result.resourceWait { + delay = e.resourceWaitDelay(e.consecutiveResourceWaits(claimed.ID)) + } transition.Status = model.TaskStatusPending transition.ResumeMode = result.resumeMode - transition.AvailableAt = now.Add(result.delay) + transition.AvailableAt = now.Add(delay) case resultRetry: if claimed.RetryLimit != nil && claimed.RetryCount >= *claimed.RetryLimit { transition.Status = model.TaskStatusFailed @@ -453,24 +467,46 @@ func (e *Engine) transitionFor(claimed *model.Task, result Result) repository.Ta } func defaultRetryDelay(retryCount int) time.Duration { - return retryDelayWithJitter(retryCount, rand.Float64()) + return backoffDelay(retryCount, retryBaseDelay, retryMaximumDelay, rand.Float64()) +} + +func defaultResourceWaitDelay(waits int) time.Duration { + return backoffDelay(waits, resourceWaitBaseDelay, resourceWaitMaximumDelay, rand.Float64()) } -func retryDelayWithJitter(retryCount int, jitterUnit float64) time.Duration { - if retryCount < 0 { - retryCount = 0 +func backoffDelay(attempt int, base, maximum time.Duration, jitterUnit float64) time.Duration { + if attempt < 0 { + attempt = 0 } - delay := retryBaseDelay - for range retryCount { - if delay >= retryMaximumDelay/2 { - delay = retryMaximumDelay + delay := base + for range attempt { + if delay >= maximum/2 { + delay = maximum break } delay *= 2 } jitterUnit = min(max(jitterUnit, 0), 1) - jitter := 1 + retryJitterFraction*(2*jitterUnit-1) - return min(time.Duration(float64(delay)*jitter), retryMaximumDelay) + jitter := 1 + backoffJitterFraction*(2*jitterUnit-1) + return min(time.Duration(float64(delay)*jitter), maximum) +} + +func (e *Engine) consecutiveResourceWaits(id int64) int { + e.resourceWaitMu.Lock() + defer e.resourceWaitMu.Unlock() + return e.resourceWaits[id] +} + +// recordResourceWait extends a task's wait streak after a settled resource +// wait; any other settled result ends the streak. +func (e *Engine) recordResourceWait(id int64, waited bool) { + e.resourceWaitMu.Lock() + defer e.resourceWaitMu.Unlock() + if waited { + e.resourceWaits[id]++ + return + } + delete(e.resourceWaits, id) } func (e *Engine) renewLease( @@ -614,18 +650,29 @@ func (e *Engine) nextRecovery() (recoveryRequest, bool) { return recoveryRequest{}, false } +// heldResource marks a context whose claim already holds a slot of a gate. +type heldResource struct{ resource Resource } + +// withResource never waits for a slot: a full gate returns ErrResourceBusy so +// the worker is free to claim other tasks instead of idling behind the gate. func (e *Engine) withResource(ctx context.Context, resource Resource, fn func(context.Context) error) error { gate, ok := e.gates[resource] if !ok || fn == nil { return fmt.Errorf("unknown task resource %q", resource) } + if err := ctx.Err(); err != nil { + return err + } + if ctx.Value(heldResource{resource}) != nil { + return fn(ctx) + } select { case gate <- struct{}{}: - defer func() { <-gate }() - return fn(ctx) - case <-ctx.Done(): - return ctx.Err() + default: + return ErrResourceBusy } + defer func() { <-gate }() + return fn(context.WithValue(ctx, heldResource{resource}, true)) } func sleepContext(ctx context.Context, delay time.Duration) bool { diff --git a/internal/task/service.go b/internal/task/service.go index c18eddf..725c9f6 100644 --- a/internal/task/service.go +++ b/internal/task/service.go @@ -198,6 +198,18 @@ func (s *Service) Acknowledge(ctx context.Context, id int64) error { return s.repos.Tasks.AcknowledgeFailed(ctx, id, s.retention) } +// AcknowledgeMatching dismisses a backlog of failures in one step and reports +// how many it dismissed. +func (s *Service) AcknowledgeMatching(ctx context.Context, filter repository.TaskAcknowledgeFilter) (int, error) { + return s.repos.Tasks.AcknowledgeFailedMatching(ctx, filter, s.retention) +} + +// CountAcknowledgeable reports how many failures AcknowledgeMatching would +// dismiss under the same filter. +func (s *Service) CountAcknowledgeable(ctx context.Context, filter repository.TaskAcknowledgeFilter) (int, error) { + return s.repos.Tasks.CountFailedMatching(ctx, filter) +} + func (s *Service) WakeInTransaction(ctx context.Context, txRepos *repository.Repositories, ids []int64) (int, error) { if txRepos == nil || txRepos.Tasks == nil { return 0, errors.New("transaction task repository is required") diff --git a/internal/task/task_test.go b/internal/task/task_test.go index ac00b43..ee37ef9 100644 --- a/internal/task/task_test.go +++ b/internal/task/task_test.go @@ -74,7 +74,11 @@ type taskHarness struct { func newTaskHarness(t *testing.T, handler Handler, config *EngineConfig) taskHarness { t.Helper() - db := testutil.NewTestFileDB(t) + return newTaskHarnessWithDB(t, testutil.NewTestFileDB(t), handler, config) +} + +func newTaskHarnessWithDB(t *testing.T, db *bun.DB, handler Handler, config *EngineConfig) taskHarness { + t.Helper() repos := repository.NewRepositories(db) registry := NewRegistry() if err := registry.Register(handler); err != nil { @@ -323,28 +327,72 @@ func TestRetryBackoffUsesPersistedRetryCount(t *testing.T) { } } -func TestRetryDelayIsExponentialJitteredAndCapped(t *testing.T) { +func TestBackoffDelayIsExponentialJitteredAndCapped(t *testing.T) { tests := []struct { - name string - retryCount int - jitter float64 - want time.Duration + name string + attempt int + base, maximum time.Duration + jitter float64 + want time.Duration }{ - {name: "first low jitter", retryCount: 0, jitter: 0, want: 8 * time.Second}, - {name: "first midpoint", retryCount: 0, jitter: 0.5, want: 10 * time.Second}, - {name: "third midpoint", retryCount: 2, jitter: 0.5, want: 40 * time.Second}, - {name: "negative count", retryCount: -1, jitter: 0.5, want: 10 * time.Second}, - {name: "hard cap", retryCount: 20, jitter: 1, want: 5 * time.Minute}, + {name: "first retry low jitter", attempt: 0, base: retryBaseDelay, maximum: retryMaximumDelay, jitter: 0, want: 8 * time.Second}, + {name: "first retry midpoint", attempt: 0, base: retryBaseDelay, maximum: retryMaximumDelay, jitter: 0.5, want: 10 * time.Second}, + {name: "third retry midpoint", attempt: 2, base: retryBaseDelay, maximum: retryMaximumDelay, jitter: 0.5, want: 40 * time.Second}, + {name: "negative count", attempt: -1, base: retryBaseDelay, maximum: retryMaximumDelay, jitter: 0.5, want: 10 * time.Second}, + {name: "retry hard cap", attempt: 20, base: retryBaseDelay, maximum: retryMaximumDelay, jitter: 1, want: 5 * time.Minute}, + {name: "first resource wait", attempt: 0, base: resourceWaitBaseDelay, maximum: resourceWaitMaximumDelay, jitter: 0.5, want: 2 * time.Second}, + {name: "resource wait hard cap", attempt: 20, base: resourceWaitBaseDelay, maximum: resourceWaitMaximumDelay, jitter: 1, want: time.Minute}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := retryDelayWithJitter(tt.retryCount, tt.jitter); got != tt.want { + if got := backoffDelay(tt.attempt, tt.base, tt.maximum, tt.jitter); got != tt.want { t.Fatalf("delay = %s, want %s", got, tt.want) } }) } } +func TestResourceWaitBacksOffWithoutConsumingRetries(t *testing.T) { + noRetries := 0 + var executions atomic.Int64 + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&noRetries, true), + execute: func(context.Context, Execution) Result { + switch executions.Add(1) { + case 3: + return Suspend(model.TaskResumeModeExecute, 0, "dependency", "Waiting for something else", nil) + case 5: + return Complete("admitted", nil) + default: + return ResourceWait("Waiting for capacity") + } + }, + }, nil) + var streaks []int + harness.engine.resourceWaitDelay = func(consecutive int) time.Duration { + streaks = append(streaks, consecutive) + return 0 + } + row := enqueueTestTask(t, harness, "resource-wait", "resource-wait") + + harness.engine.executeClaim(t.Context(), claimTestTask(t, harness)) + waiting, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || waiting.Status != model.TaskStatusPending || waiting.ResumeMode != model.TaskResumeModeExecute || + waiting.WaitReason == nil || *waiting.WaitReason != "resource" || waiting.RetryCount != 0 || len(waiting.Checkpoint) != 0 { + t.Fatalf("waiting task = %#v, err=%v", waiting, err) + } + for range 4 { + harness.engine.executeClaim(t.Context(), claimTestTask(t, harness)) + } + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil || stored.Status != model.TaskStatusCompleted || stored.RetryCount != 0 { + t.Fatalf("task after resource waits = %#v, err=%v", stored, err) + } + if fmt.Sprint(streaks) != "[0 1 0]" { + t.Fatalf("resource wait streaks = %v, want [0 1 0]: consecutive waits back off and any other outcome resets", streaks) + } +} + func TestServiceRetryForcesRecoverAndAcknowledgeStartsRetention(t *testing.T) { limit := 5 harness := newTaskHarness(t, scriptedHandler{ @@ -810,6 +858,53 @@ func TestCheckpointedEffectRollsBackEvidenceBeforeEffect(t *testing.T) { } } +func TestCheckpointedEffectCommitsCheckpointBeforeEffect(t *testing.T) { + for name, openDB := range map[string]func(*testing.T) *bun.DB{ + "sqlite": testutil.NewTestFileDB, + "postgres": testutil.NewTestPostgresDB, + } { + t.Run(name, func(t *testing.T) { + limit := 5 + var repos *repository.Repositories + var duringEffect json.RawMessage + harness := newTaskHarnessWithDB(t, openDB(t), scriptedHandler{ + definition: testDefinition(&limit, true), + execute: func(ctx context.Context, execution Execution) Result { + attempted, err := execution.WithCheckpointedEffect(ctx, ResourceProviderMutation, map[string]string{"attempt": "one"}, nil, func(ctx context.Context) error { + stored, err := repos.Tasks.GetByID(ctx, execution.ID()) + if err != nil { + return err + } + duringEffect = stored.Checkpoint + return nil + }) + if !attempted || err != nil { + return Fail(fmt.Errorf("checkpointed effect = attempted:%v err:%v", attempted, err), "unexpected_effect_result", nil) + } + return Complete("effect finished", nil) + }, + }, nil) + repos = harness.repos + row := enqueueTestTask(t, harness, "checkpoint-commit", "checkpoint-commit") + harness.engine.executeClaim(t.Context(), claimTestTask(t, harness)) + + stored, err := harness.repos.Tasks.GetByID(t.Context(), row.ID) + if err != nil { + t.Fatalf("load task: %v", err) + } + if stored.Status != model.TaskStatusCompleted { + t.Fatalf("task after effect = %#v", stored) + } + for label, raw := range map[string]json.RawMessage{"during effect": duringEffect, "after settlement": stored.Checkpoint} { + var checkpoint map[string]string + if err := json.Unmarshal(raw, &checkpoint); err != nil || checkpoint["attempt"] != "one" { + t.Fatalf("checkpoint %s = %s, err=%v", label, raw, err) + } + } + }) + } +} + func TestExternalEffectRevalidatesClaimAfterResourceAdmission(t *testing.T) { limit := 5 var called atomic.Bool @@ -1204,57 +1299,153 @@ func TestClaimNextConcurrentClaimsAreUnique(t *testing.T) { } } -func TestProviderResourceGateReachesAndEnforcesConfiguredLimit(t *testing.T) { +func TestResourceGateYieldsWhenFullAndReusesHeldSlot(t *testing.T) { limit := 5 harness := newTaskHarness(t, scriptedHandler{definition: testDefinition(&limit, true)}, nil) - const calls = 12 - var running, maximum atomic.Int64 - entered := make(chan struct{}, calls) + capacity := harness.engine.config.ProviderMutationConcurrency + entered := make(chan struct{}, capacity) + nest := make(chan struct{}) + nested := make(chan error, capacity) release := make(chan struct{}) - errorsCh := make(chan error, calls) - var workers sync.WaitGroup - for range calls { - workers.Go(func() { - err := harness.engine.withResource(t.Context(), ResourceProviderMutation, func(context.Context) error { - current := running.Add(1) - for { - observed := maximum.Load() - if current <= observed || maximum.CompareAndSwap(observed, current) { - break - } - } + held := make(chan error, capacity) + var holders sync.WaitGroup + for range capacity { + holders.Go(func() { + held <- harness.engine.withResource(t.Context(), ResourceProviderMutation, func(ctx context.Context) error { entered <- struct{}{} + <-nest + nested <- harness.engine.withResource(ctx, ResourceProviderMutation, func(context.Context) error { return nil }) <-release - running.Add(-1) return nil }) - errorsCh <- err }) } - for range harness.engine.config.ProviderMutationConcurrency { + for range capacity { select { case <-entered: case <-time.After(time.Second): - t.Fatal("provider mutation gate did not reach configured concurrency") + t.Fatal("provider mutation gate did not admit its configured capacity") } } - if maximum.Load() != int64(harness.engine.config.ProviderMutationConcurrency) { - t.Fatalf("provider mutation concurrency = %d", maximum.Load()) + called := false + err := harness.engine.withResource(t.Context(), ResourceProviderMutation, func(context.Context) error { + called = true + return nil + }) + if !errors.Is(err, ErrResourceBusy) || called { + t.Fatalf("full gate = called:%v err:%v, want immediate ErrResourceBusy", called, err) } - select { - case <-entered: - t.Fatal("provider mutation gate exceeded configured concurrency") - case <-time.After(20 * time.Millisecond): + close(nest) + for range capacity { + if err := <-nested; err != nil { + t.Fatalf("nested use of a held slot on a full gate: %v", err) + } } close(release) - workers.Wait() - close(errorsCh) - for err := range errorsCh { + holders.Wait() + close(held) + for err := range held { if err != nil { - t.Fatalf("provider resource call: %v", err) + t.Fatalf("slot holder: %v", err) } } - if maximum.Load() != 4 { - t.Fatalf("provider mutation maximum = %d, want 4", maximum.Load()) + if err := harness.engine.withResource(t.Context(), ResourceProviderMutation, func(context.Context) error { return nil }); err != nil { + t.Fatalf("gate after release: %v", err) + } +} + +func TestResourceWaitFreesWorkersForOtherTasks(t *testing.T) { + noRetries := 0 + holding := make(chan string, 3) + release := make(chan struct{}) + harness := newTaskHarness(t, scriptedHandler{ + definition: testDefinition(&noRetries, true), + execute: func(ctx context.Context, execution Execution) Result { + input, err := DecodeInput[testInput](execution) + if err != nil { + return Fail(err, "invalid_input", nil) + } + if input.Value == "plain" { + return Complete("plain work finished", nil) + } + attempted, err := execution.WithCheckpointedEffect(ctx, ResourceProviderMutation, input, nil, func(ctx context.Context) error { + holding <- input.Value + select { + case <-release: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }) + if !attempted && errors.Is(err, ErrResourceBusy) { + return ResourceWait("Waiting for capacity") + } + if err != nil { + return Fail(err, "unexpected_effect_error", nil) + } + return Complete("gated work finished", nil) + }, + }, &EngineConfig{ + Concurrency: 2, PollInterval: 10 * time.Millisecond, LeaseDuration: 5 * time.Second, Retention: time.Hour, + ProviderMutationConcurrency: 1, DestructiveMutationConcurrency: 1, + }) + harness.engine.resourceWaitDelay = func(int) time.Duration { return 50 * time.Millisecond } + gated := map[string]*model.Task{} + for _, key := range []string{"gated-1", "gated-2", "gated-3"} { + gated[key] = enqueueTestTask(t, harness, key, key) + } + ctx, cancel := context.WithCancel(t.Context()) + stopped := make(chan struct{}) + go func() { + _ = harness.engine.Run(ctx) + close(stopped) + }() + defer func() { + cancel() + <-stopped + }() + waitFor := func(id int64, predicate func(*model.Task) bool) *model.Task { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for { + stored, err := harness.repos.Tasks.GetByID(t.Context(), id) + if err != nil { + t.Fatalf("load task %d: %v", id, err) + } + if predicate(stored) { + return stored + } + if time.Now().After(deadline) { + t.Fatalf("task %d did not reach the expected state: %#v", id, stored) + } + time.Sleep(5 * time.Millisecond) + } + } + + var holder string + select { + case holder = <-holding: + case <-time.After(3 * time.Second): + t.Fatal("no gated task acquired the provider slot") + } + // With both workers saturated by gated work, a blocking gate would starve + // this task until the slot holder finished. + plain := enqueueTestTask(t, harness, "plain", "plain") + waitFor(plain.ID, func(task *model.Task) bool { return task.Status == model.TaskStatusCompleted }) + for key, row := range gated { + if key == holder { + continue + } + waitFor(row.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeExecute && + task.WaitReason != nil && *task.WaitReason == "resource" && task.RetryCount == 0 && len(task.Checkpoint) == 0 + }) + } + close(release) + for _, row := range gated { + stored := waitFor(row.ID, func(task *model.Task) bool { return task.Status == model.TaskStatusCompleted }) + if stored.RetryCount != 0 { + t.Fatalf("gated task consumed retries while waiting: %#v", stored) + } } } diff --git a/internal/testutil/db.go b/internal/testutil/db.go index 436c477..8420b99 100644 --- a/internal/testutil/db.go +++ b/internal/testutil/db.go @@ -8,15 +8,19 @@ import ( "database/sql" "encoding/hex" "fmt" + "os" "path/filepath" "sync/atomic" "testing" "time" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/stdlib" "github.com/strahe/synaps3/internal/db/migrations" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect/pgdialect" "github.com/uptrace/bun/dialect/sqlitedialect" _ "modernc.org/sqlite" @@ -80,6 +84,49 @@ func newTestSQLiteDB(t *testing.T, dsn string) *bun.DB { return db } +// NewTestPostgresDB creates an isolated PostgreSQL schema with all migrations +// applied, reading SYNAPS3_POSTGRES_TEST_DSN. The test is skipped when the DSN +// is unset, and the schema is dropped when the test completes. +func NewTestPostgresDB(t *testing.T) *bun.DB { + t.Helper() + + dsn := os.Getenv("SYNAPS3_POSTGRES_TEST_DSN") + if dsn == "" { + t.Skip("SYNAPS3_POSTGRES_TEST_DSN is not set") + } + config, err := pgx.ParseConfig(dsn) + if err != nil { + t.Fatalf("parse PostgreSQL DSN: %v", err) + } + adminDB, err := sql.Open("pgx", dsn) + if err != nil { + t.Fatalf("open PostgreSQL admin connection: %v", err) + } + digest := sha256.Sum256(fmt.Appendf(nil, "%s-%d", t.Name(), time.Now().UnixNano())) + schema := "test_" + hex.EncodeToString(digest[:16]) + if _, err := adminDB.Exec("CREATE SCHEMA " + schema); err != nil { + _ = adminDB.Close() + t.Fatalf("create PostgreSQL schema: %v", err) + } + config.RuntimeParams["search_path"] = schema + db := bun.NewDB(stdlib.OpenDB(*config), pgdialect.New()) + t.Cleanup(func() { + _ = db.Close() + _, _ = adminDB.Exec("DROP SCHEMA " + schema + " CASCADE") + _ = adminDB.Close() + }) + + ctx := context.Background() + migrator := migrations.NewMigrator(db) + if err := migrator.Init(ctx); err != nil { + t.Fatalf("init migrator: %v", err) + } + if _, err := migrator.Migrate(ctx); err != nil { + t.Fatalf("running migrations: %v", err) + } + return db +} + // NewTestRepos creates a Repositories instance backed by an in-memory SQLite DB. func NewTestRepos(t *testing.T) *repository.Repositories { t.Helper() diff --git a/internal/testutil/mocks.go b/internal/testutil/mocks.go index d740f56..d9e4da8 100644 --- a/internal/testutil/mocks.go +++ b/internal/testutil/mocks.go @@ -5,6 +5,7 @@ import ( "errors" "io" + "github.com/ethereum/go-ethereum/common" "github.com/ipfs/go-cid" "github.com/strahe/synaps3/internal/cache" "github.com/strahe/synaps3/internal/synapse" @@ -187,12 +188,15 @@ type MockStorageTarget struct { WithCDNValue bool CreateDataSetFunc func(context.Context, *storage.CreateDataSetOptions) (*storage.CreateDataSetResult, error) WaitDataSetFunc func(context.Context, storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) - StoreFunc func(context.Context, io.Reader, *storage.StoreOptions) (*storage.StoreResult, error) - PresignForCommitFunc func(context.Context, []storage.PieceInput) ([]byte, error) - PullFunc func(context.Context, storage.PullRequest) (*storage.PullResult, error) - SubmitCommitFunc func(context.Context, storage.CommitRequest) (*storage.CommitSubmission, error) - GetCommitStatusFunc func(context.Context, storage.CommitSubmission) (*storage.CommitStatus, error) - PieceStatusFunc func(context.Context, cid.Cid) (*storage.PieceStatus, error) + // ContextIdentityValue overrides DefaultContextIdentity. + ContextIdentityValue storage.ContextIdentity + FindDataSetByClientIDFunc func(context.Context, sdktypes.BigInt) (storage.DataSetRef, bool, error) + StoreFunc func(context.Context, io.Reader, *storage.StoreOptions) (*storage.StoreResult, error) + PresignForCommitFunc func(context.Context, []storage.PieceInput) ([]byte, error) + PullFunc func(context.Context, storage.PullRequest) (*storage.PullResult, error) + SubmitCommitFunc func(context.Context, storage.CommitRequest) (*storage.CommitSubmission, error) + GetCommitStatusFunc func(context.Context, storage.CommitSubmission) (*storage.CommitStatus, error) + PieceStatusFunc func(context.Context, cid.Cid) (*storage.PieceStatus, error) } func NewMockProviderTarget(providerID sdktypes.BigInt, opts storage.NewProviderContextOptions) *MockStorageTarget { @@ -258,6 +262,28 @@ func (m *MockStorageTarget) WaitForDataSetCreated(ctx context.Context, submissio return nil, errors.New("MockStorageTarget.WaitForDataSetCreated not configured") } +// DefaultContextIdentity is the signing identity mock targets report unless a +// test sets ContextIdentityValue. +var DefaultContextIdentity = storage.ContextIdentity{ + Payer: common.HexToAddress("0x00000000000000000000000000000000000000a1"), + ChainID: 314159, + RecordKeeper: common.HexToAddress("0x00000000000000000000000000000000000000b2"), +} + +func (m *MockStorageTarget) ContextIdentity() storage.ContextIdentity { + if m.ContextIdentityValue != (storage.ContextIdentity{}) { + return m.ContextIdentityValue + } + return DefaultContextIdentity +} + +func (m *MockStorageTarget) FindDataSetByClientDataSetID(ctx context.Context, clientDataSetID sdktypes.BigInt) (storage.DataSetRef, bool, error) { + if m.FindDataSetByClientIDFunc != nil { + return m.FindDataSetByClientIDFunc(ctx, clientDataSetID) + } + return storage.DataSetRef{}, false, errors.New("MockStorageTarget.FindDataSetByClientDataSetID not configured") +} + func (m *MockStorageTarget) Store(ctx context.Context, reader io.Reader, opts *storage.StoreOptions) (*storage.StoreResult, error) { if m.StoreFunc != nil { return m.StoreFunc(ctx, reader, opts) @@ -417,6 +443,28 @@ func (m *MockCache) DeleteUpload(ctx context.Context, uploadID string) error { // synapse.ServiceTerminator. type MockServiceTerminator struct { TerminateServiceFunc func(ctx context.Context, dataSetID sdktypes.BigInt) (*synapse.TerminationResult, error) + // VerifyServicePayerFunc defaults to accepting every data set. + VerifyServicePayerFunc func(ctx context.Context, dataSetID sdktypes.BigInt) error + // ContextIdentityValue overrides DefaultContextIdentity. + ContextIdentityValue storage.ContextIdentity +} + +// VerifyServicePayer calls VerifyServicePayerFunc, accepting every data set when +// no test sets it. +func (m *MockServiceTerminator) VerifyServicePayer(ctx context.Context, dataSetID sdktypes.BigInt) error { + if m.VerifyServicePayerFunc != nil { + return m.VerifyServicePayerFunc(ctx, dataSetID) + } + return nil +} + +// ContextIdentity returns ContextIdentityValue, or DefaultContextIdentity when +// no test sets it. +func (m *MockServiceTerminator) ContextIdentity() storage.ContextIdentity { + if m.ContextIdentityValue != (storage.ContextIdentity{}) { + return m.ContextIdentityValue + } + return DefaultContextIdentity } func (m *MockServiceTerminator) TerminateService(ctx context.Context, dataSetID sdktypes.BigInt) (*synapse.TerminationResult, error) { diff --git a/internal/worker/bucket_task_handlers.go b/internal/worker/bucket_task_handlers.go index 9569434..cfb3566 100644 --- a/internal/worker/bucket_task_handlers.go +++ b/internal/worker/bucket_task_handlers.go @@ -19,7 +19,9 @@ func (h *TaskHandlers) bucketProvisionHandler() taskengine.Handler { Codec: taskengine.StrictJSONCodec(func(input *bucketlifecycle.ProvisionInput) error { return bucketlifecycle.ValidateProvisionInput(*input) }), - RetryLimit: h.retryLimit(), AllowRetry: true, + // Provisioning re-reads bucket state on every wake and can wait a long + // time for providers, so transient errors must not exhaust it. + RetryLimit: nil, AllowRetry: true, } run := func(ctx context.Context, execution taskengine.Execution) taskengine.Result { input, err := taskengine.DecodeInput[bucketlifecycle.ProvisionInput](execution) diff --git a/internal/worker/core_task_handlers.go b/internal/worker/core_task_handlers.go index dd4449d..3afa6f0 100644 --- a/internal/worker/core_task_handlers.go +++ b/internal/worker/core_task_handlers.go @@ -17,6 +17,7 @@ import ( "github.com/strahe/synaps3/internal/cacheeviction" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/objectdeletion" "github.com/strahe/synaps3/internal/storagecleanup" "github.com/strahe/synaps3/internal/synapse" "github.com/strahe/synaps3/internal/systemtask" @@ -37,6 +38,8 @@ const ( type cleanupCheckpoint struct { CopyID int64 `json:"copy_id"` AttemptedAt time.Time `json:"attempted_at"` + // Finalized records that the content's rows were deleted. + Finalized bool `json:"finalized,omitempty"` } type cacheCapacityCheckpoint struct { @@ -501,6 +504,15 @@ func (h *TaskHandlers) runStorageCleanup(ctx context.Context, execution taskengi if h.deps.Storage == nil { return taskengine.Fail(errors.New("storage cleanup client is unavailable"), "dependency_unavailable", nil) } + checkpoint, hasCheckpoint, err := taskengine.DecodeCheckpoint[cleanupCheckpoint](execution) + if err != nil { + return taskengine.Fail(err, "invalid_checkpoint", nil) + } + // Finalizing deleted the content's rows, so there is nothing left to + // authorize against. + if checkpoint.Finalized { + return taskengine.Complete("Stored data removed", nil) + } copies, err := h.deps.Repositories.StorageCleanup.AuthorizeTask(ctx, input.ContentID, input.Generation, execution.ID()) if err != nil { if errors.Is(err, repository.ErrConflict) || errors.Is(err, repository.ErrNotFound) { @@ -518,18 +530,12 @@ func (h *TaskHandlers) runStorageCleanup(ctx context.Context, execution taskengi if hasReferences { return taskengine.Suspend(model.TaskResumeModeRecover, dependencyWait, "references", "Waiting for stored data references", nil) } - checkpoint, hasCheckpoint, err := taskengine.DecodeCheckpoint[cleanupCheckpoint](execution) - if err != nil { - return taskengine.Fail(err, "invalid_checkpoint", nil) - } - hasUnsupported := false for i := range copies { copyRow := copies[i] switch copyRow.Status { - case model.StorageCleanupCopyStatusRemoved: - continue - case model.StorageCleanupCopyStatusUnsupported: - hasUnsupported = true + // A deletion the provider cannot perform stays recorded as unsupported + // and does not keep the content from being finalized. + case model.StorageCleanupCopyStatusRemoved, model.StorageCleanupCopyStatusUnsupported: continue } if copyRow.DataSetID == nil || copyRow.DataSetID.IsZero() || copyRow.ProviderID.IsZero() || copyRow.PieceID.IsZero() || copyRow.PieceCID == "" { @@ -537,7 +543,6 @@ func (h *TaskHandlers) runStorageCleanup(ctx context.Context, execution taskengi if err := h.deps.Repositories.StorageCleanup.MarkCopyUnsupported(ctx, copyRow.ID, message); err != nil { return retryTask(err, "cleanup_evidence_failed") } - hasUnsupported = true continue } pieceCID, err := cid.Parse(copyRow.PieceCID) @@ -546,7 +551,6 @@ func (h *TaskHandlers) runStorageCleanup(ctx context.Context, execution taskengi if markErr := h.deps.Repositories.StorageCleanup.MarkCopyUnsupported(ctx, copyRow.ID, message); markErr != nil { return retryTask(markErr, "cleanup_evidence_failed") } - hasUnsupported = true continue } providerID := copyRow.ProviderID.SDK() @@ -591,6 +595,9 @@ func (h *TaskHandlers) runStorageCleanup(ctx context.Context, execution taskengi }) if err != nil { if !attempted { + if errors.Is(err, taskengine.ErrResourceBusy) { + return taskengine.ResourceWait("Waiting for other removal operations to finish") + } return retryTask(err, "cleanup_not_started") } return taskengine.Suspend(model.TaskResumeModeRecover, externalPollInterval, "provider_confirmation", "Checking remote cleanup", nil) @@ -600,12 +607,40 @@ func (h *TaskHandlers) runStorageCleanup(ctx context.Context, execution taskengi } return taskengine.Suspend(model.TaskResumeModeRecover, externalPollInterval, "provider_confirmation", "Waiting for remote cleanup", nil) } - if hasUnsupported { - return taskengine.Fail(errors.New("one or more remote replica deletions are unsupported"), "cleanup_unsupported", nil) + return h.finishStorageCleanup(ctx, execution, input) +} + +// finishStorageCleanup releases the content's cached bytes and then deletes its +// current-state rows, so writing the same bytes again starts new content. The +// ledgers keep their rows, including any remote deletion left unsupported. +func (h *TaskHandlers) finishStorageCleanup(ctx context.Context, execution taskengine.Execution, input storagecleanup.Input) taskengine.Result { + if h.deps.Cache == nil || h.deps.CacheGate == nil || h.deps.CacheTracker == nil { + return taskengine.Fail(errors.New("cache dependencies are unavailable"), "dependency_unavailable", nil) + } + content, err := h.deps.Repositories.Contents.GetByID(ctx, input.ContentID) + if err != nil || content == nil { + return retryTask(errors.Join(err, repository.ErrNotFound), "cleanup_content_load_failed") + } + bucket, err := h.deps.Repositories.Buckets.GetByID(ctx, content.BucketID) + if err != nil || bucket == nil { + return retryTask(errors.Join(err, repository.ErrNotFound), "cleanup_bucket_load_failed") } - return taskengine.Complete("Remote replicas removed", func(ctx context.Context, repos *repository.Repositories) error { - return repos.StorageCleanup.CompleteTask(ctx, input.ContentID, input.Generation, execution.ID()) + if _, err := objectdeletion.ReleaseContentCache( + ctx, h.deps.Cache, h.deps.CacheGate, h.deps.CacheTracker, h.deps.Repositories.Objects, bucket.Name, input.ContentID, + ); err != nil { + return retryTask(err, "cleanup_cache_release_failed") + } + finalized := cleanupCheckpoint{AttemptedAt: time.Now().UTC(), Finalized: true} + err = execution.WriteCheckpointWith(ctx, finalized, func(ctx context.Context, repos *repository.Repositories) error { + return repos.StorageCleanup.FinalizeContent(ctx, input.ContentID, input.Generation, execution.ID()) }) + if errors.Is(err, repository.ErrContentCleanupNotReady) { + return taskengine.Suspend(model.TaskResumeModeRecover, dependencyWait, "references", "Waiting for other work on the stored data to finish", nil) + } + if err != nil { + return retryTask(err, "cleanup_finalize_failed") + } + return taskengine.Complete("Stored data removed", nil) } func (h *TaskHandlers) walletHandler() taskengine.Handler { @@ -666,6 +701,9 @@ func (h *TaskHandlers) executeWalletOperation(ctx context.Context, execution tas }) if err != nil { if !attempted { + if errors.Is(err, taskengine.ErrResourceBusy) { + return taskengine.ResourceWait("Waiting for another wallet operation to finish") + } return retryTask(err, "wallet_broadcast_not_started") } message := fmt.Sprintf("wallet broadcast outcome is unknown: %v", err) diff --git a/internal/worker/replacement_task_handlers.go b/internal/worker/replacement_task_handlers.go index 1afaf48..d8476b0 100644 --- a/internal/worker/replacement_task_handlers.go +++ b/internal/worker/replacement_task_handlers.go @@ -11,15 +11,25 @@ import ( "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/storagereplacement" + "github.com/strahe/synaps3/internal/synapse" taskengine "github.com/strahe/synaps3/internal/task" + "github.com/strahe/synapse-go/storage" ) const replacementSeedBatchSize = 100 type retirementCheckpoint struct { + // AttemptedAt is when the latest termination request was sent. AttemptedAt time.Time `json:"attempted_at"` TerminationEpoch *int64 `json:"termination_epoch,omitempty"` TransactionHash string `json:"transaction_hash,omitempty"` + // Sends counts termination requests; it paces the ones that follow an + // unobserved outcome. + Sends int `json:"sends,omitempty"` + // Identity is what the first request signed for. A numeric data set ID means + // something else on another chain, so recovery compares this before it reads + // the chain or sends again. + Identity *storage.ContextIdentity `json:"identity,omitempty"` } func (h *TaskHandlers) replacementCoordinateHandler() taskengine.Handler { @@ -28,7 +38,9 @@ func (h *TaskHandlers) replacementCoordinateHandler() taskengine.Handler { Codec: taskengine.StrictJSONCodec(func(input *storagereplacement.CoordinateInput) error { return storagereplacement.ValidateCoordinateInput(*input) }), - RetryLimit: h.retryLimit(), AllowRetry: false, + // The coordinator re-reads the replacement ledger on every wake and a + // replacement can run for days, so transient errors must not exhaust it. + RetryLimit: nil, AllowRetry: false, } run := func(ctx context.Context, execution taskengine.Execution) taskengine.Result { input, err := taskengine.DecodeInput[storagereplacement.CoordinateInput](execution) @@ -173,6 +185,12 @@ func (h *TaskHandlers) coordinateReplacementItem( return h.retryReplacement(execution, replacement.ID, err, "replacement_copy_task_load_failed") } if copyTask != nil && copyTask.Status == model.TaskStatusFailed { + // The copy task still holds the copy and an operator can retry it, for + // example after an unknown transfer outcome, so the replacement waits + // for that retry instead of failing as a whole. + if h.taskService != nil && h.taskService.Retryable(copyTask) { + return taskengine.Suspend(model.TaskResumeModeRecover, storageDependencyWait, "copy_work", "Waiting for a failed migration task to be retried", nil) + } message := "stored content migration failed" if copyTask.LastError != nil { message = *copyTask.LastError @@ -337,6 +355,15 @@ func (h *TaskHandlers) runDataSetRetirement(ctx context.Context, execution taske if err != nil { return taskengine.Fail(err, "invalid_checkpoint", nil) } + if hasCheckpoint && checkpoint.Identity != nil && h.deps.Terminator != nil && + *checkpoint.Identity != h.deps.Terminator.ContextIdentity() { + // The wallet or network moved after a request went out. The same numeric + // data set ID names a different service on another chain, and an end + // epoch read there would say nothing about ours, so this reads nothing + // and sends nothing until the original configuration is back. + return stopRetirement(abandoned, row.ID, + errors.New("the wallet or network changed after storage service retirement began"), "termination_identity_changed") + } if terminationEpoch == nil && checkpoint.TerminationEpoch != nil { if *checkpoint.TerminationEpoch < 0 { return taskengine.Fail(errors.New("storage service retirement checkpoint has an invalid epoch"), "invalid_checkpoint", nil) @@ -346,14 +373,12 @@ func (h *TaskHandlers) runDataSetRetirement(ctx context.Context, execution taske )) } if terminationEpoch == nil { - if hasCheckpoint { - err := errors.New("storage service termination outcome could not be recovered") - if abandoned { - return taskengine.Fail(err, "termination_outcome_unknown", nil) - } - return taskengine.Fail(err, "termination_outcome_unknown", func(ctx context.Context, repos *repository.Repositories) error { - return repos.Replacements.MarkCleanupAttention(ctx, row.ID, err.Error()) - }) + if hasCheckpoint && !mayTerminate { + // A request may already have been sent. Execute reads the chain before + // sending another one, so resuming there cannot end the service twice; + // the delay gives the earlier request time to land. + wait := max(time.Until(checkpoint.AttemptedAt.Add(unobservedOutcomeDelay(checkpoint.Sends))), 0) + return taskengine.Suspend(model.TaskResumeModeExecute, wait, "provider_confirmation", "Checking storage service retirement", nil) } if !mayTerminate { return taskengine.Suspend(model.TaskResumeModeExecute, 0, "safe_to_execute", "Storage service is ready to retire", nil) @@ -361,7 +386,23 @@ func (h *TaskHandlers) runDataSetRetirement(ctx context.Context, execution taske if h.deps.Terminator == nil { return taskengine.Fail(errors.New("storage service terminator is unavailable"), "dependency_unavailable", nil) } - checkpoint = retirementCheckpoint{AttemptedAt: time.Now().UTC()} + identity := h.deps.Terminator.ContextIdentity() + if !contextIdentityComplete(identity) { + return taskengine.Fail(errors.New("storage signing identity is incomplete"), "dependency_unavailable", nil) + } + // Ownership is read before anything is recorded. The checkpoint below + // binds the identity a request goes out under; a refusal after it would + // bind an identity nothing was sent for, and an operator who put the right + // wallet back could never retry past it. + if err := h.deps.Terminator.VerifyServicePayer(ctx, dataSet.DataSetID.SDK()); err != nil { + if errors.Is(err, synapse.ErrServicePaidByAnother) { + return stopRetirement(abandoned, row.ID, err, "termination_payer_mismatch") + } + return taskengine.Suspend(model.TaskResumeModeExecute, storageDependencyWait, "provider_confirmation", "Checking storage service retirement", nil) + } + checkpoint = retirementCheckpoint{ + AttemptedAt: time.Now().UTC(), Sends: checkpoint.Sends + 1, Identity: &identity, + } var terminationEpochValue int64 var txHash string attempted, err := execution.WithCheckpointedEffect(ctx, taskengine.ResourceDestructiveMutation, checkpoint, nil, func(ctx context.Context) error { @@ -373,18 +414,28 @@ func (h *TaskHandlers) runDataSetRetirement(ctx context.Context, execution taske return terminateErr }) if err != nil && !attempted { + if errors.Is(err, taskengine.ErrResourceBusy) { + return taskengine.ResourceWait("Waiting for other removal operations to finish") + } return retryTask(err, "termination_not_started") } - if err != nil || terminationEpochValue < 0 { - if err == nil { - err = errors.New("storage service termination returned an invalid epoch") - } - if abandoned { - return taskengine.Fail(err, "termination_outcome_unknown", nil) - } - return taskengine.Fail(err, "termination_outcome_unknown", func(ctx context.Context, repos *repository.Repositories) error { - return repos.Replacements.MarkCleanupAttention(ctx, row.ID, err.Error()) - }) + if synapse.IsTerminationBlocked(err) { + // Settling payment debt is the operator's decision; it is never + // retried automatically. + return stopRetirement(abandoned, row.ID, err, "termination_blocked") + } + if errors.Is(err, synapse.ErrServicePaidByAnother) { + return stopRetirement(abandoned, row.ID, err, "termination_payer_mismatch") + } + if err != nil { + // The outcome is unknown, or the provider is still publishing it. The + // next request reads the chain first and only goes out while the + // service is still running there. + return taskengine.Suspend(model.TaskResumeModeExecute, unobservedOutcomeDelay(checkpoint.Sends), "provider_confirmation", "Checking storage service retirement", nil) + } + if terminationEpochValue < 0 { + return stopRetirement(abandoned, row.ID, + errors.New("storage service termination returned an invalid epoch"), "termination_outcome_unknown") } checkpoint.TerminationEpoch = &terminationEpochValue checkpoint.TransactionHash = txHash @@ -409,8 +460,16 @@ func (h *TaskHandlers) runDataSetRetirement(ctx context.Context, execution taske if err := repos.Replacements.CompleteAbandonedTargetTermination(ctx, row.ID); err != nil { return err } - } else if err := repos.Replacements.CompleteRetirement(ctx, row.ID, observedEpoch); err != nil { - return err + } else { + // A task retried after it stopped for attention finishes with the + // replacement still marked for it. The retirement it was stopped on is + // done, so the replacement returns to retiring and completes. + if err := repos.Replacements.BeginRetirement(ctx, row.ID); err != nil && !errors.Is(err, repository.ErrConflict) { + return err + } + if err := repos.Replacements.CompleteRetirement(ctx, row.ID, observedEpoch); err != nil { + return err + } } return repos.Contents.CompleteDataSetRetirementTask(ctx, input.DataSetID, input.Generation, execution.ID()) }) @@ -451,6 +510,18 @@ func (h *TaskHandlers) retryRetirement( }) } +// stopRetirement fails a retirement that needs an operator. A replacement still +// in progress is marked for cleanup attention; an abandoned target belongs to +// one that already ended, so only the task stops. +func stopRetirement(abandoned bool, replacementID int64, err error, reason string) taskengine.Result { + if abandoned { + return taskengine.Fail(err, reason, nil) + } + return taskengine.Fail(err, reason, func(ctx context.Context, repos *repository.Repositories) error { + return repos.Replacements.MarkCleanupAttention(ctx, replacementID, err.Error()) + }) +} + func containsString(values []string, wanted string) bool { return slices.Contains(values, wanted) } diff --git a/internal/worker/storage_task_handlers.go b/internal/worker/storage_task_handlers.go index fe08b8f..7ab7bc2 100644 --- a/internal/worker/storage_task_handlers.go +++ b/internal/worker/storage_task_handlers.go @@ -6,12 +6,14 @@ import ( "encoding/hex" "errors" "fmt" + "math/big" "os" "sort" "strconv" "strings" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ipfs/go-cid" "github.com/strahe/synaps3/internal/bucketlifecycle" "github.com/strahe/synaps3/internal/cache" @@ -32,15 +34,30 @@ import ( const ( storageDependencyWait = time.Minute storagePollInterval = 5 * time.Second - dataSetAttentionAfter = 15 * time.Minute storeAttentionAfter = 30 * time.Minute + // A request whose outcome was never observed is checked on chain, and sent + // again with the same identity, one minute after the first request, + // doubling with each further request up to 30 minutes. + unobservedOutcomeBaseDelay = time.Minute + unobservedOutcomeMaxDelay = 30 * time.Minute + // Resolving a commit attempt wakes the head of its data set's queue; this + // delay only covers a missed wake. + commitCapacityBackstop = 5 * time.Minute ) type dataSetCreationCheckpoint struct { + // AttemptedAt is when the latest create request was sent. AttemptedAt time.Time `json:"attempted_at"` TransactionID string `json:"transaction_id,omitempty"` StatusURL string `json:"status_url,omitempty"` ClientDataSetID string `json:"client_data_set_id,omitempty"` + // Identity is the payer, chain, and record keeper the request was signed + // for; a client data set ID is only unique within it. + Identity *storage.ContextIdentity `json:"identity,omitempty"` + // Sends counts create requests carrying ClientDataSetID. A request is only + // repeated after one whose outcome was never observed, so more than one + // send means a rejection no longer proves that no data set exists. + Sends int `json:"sends,omitempty"` } type storeCheckpoint struct { @@ -320,6 +337,23 @@ func (h *TaskHandlers) dataSetEnsureHandler() taskengine.Handler { return storagepipeline.ValidateDataSetInput(*input) }), RetryLimit: h.retryLimit(), AllowRetry: true, + // These outcomes already ended the generation: the chain refused the + // creation, the chain resolved its ID to a record that is not ours, or + // the row proved nothing was ever sent. It is retired and its fence + // released, so a retry has nothing left to do. Everything else stays + // retryable, because an operator who restores the wallet, the network, + // or a missing dependency can finish a creation that was only interrupted. + CanManualRetry: func(task *model.Task) bool { + if task == nil || task.FailureReason == nil { + return true + } + switch *task.FailureReason { + case "dataset_creation_rejected", "dataset_correlation_conflict", "dataset_creation_unsent": + return false + default: + return true + } + }, } return taskHandler{ definition: definition, @@ -337,9 +371,6 @@ func (h *TaskHandlers) runDataSetEnsure(ctx context.Context, execution taskengin if err != nil { return decodeFailure(string(model.TaskTypeStorageDataSetEnsure), err) } - if h.deps.Storage == nil { - return taskengine.Fail(errors.New("storage client is unavailable"), "dependency_unavailable", nil) - } binding, err := h.deps.Repositories.Contents.AuthorizeDataSetEnsureTask(ctx, input.DataSetID, execution.ID()) if errors.Is(err, repository.ErrConflict) || errors.Is(err, repository.ErrNotFound) { return taskengine.Cancel("Storage service setup was superseded", nil) @@ -347,6 +378,9 @@ func (h *TaskHandlers) runDataSetEnsure(ctx context.Context, execution taskengin if err != nil { return retryTask(err, "dataset_authorization_failed") } + if h.deps.Storage == nil { + return failDataSetEnsure(binding, execution.ID(), errors.New("storage client is unavailable"), "dependency_unavailable") + } if binding.Status == model.StorageDataSetStatusReady && binding.DataSetID != nil && !binding.DataSetID.IsZero() { return taskengine.Complete("Storage service is ready", func(ctx context.Context, repos *repository.Repositories) error { return h.finishDataSetEnsure(ctx, repos, binding, execution.ID()) @@ -365,11 +399,53 @@ func (h *TaskHandlers) runDataSetEnsure(ctx context.Context, execution taskengin if err != nil { return taskengine.Suspend(model.TaskResumeModeRecover, storageDependencyWait, "provider", "Waiting for storage provider", nil) } + // A request that already went out is resolved by the ID it used, before any + // metadata search: every generation this bucket ever had at this provider + // shares that metadata, so a match proves nothing about which one is ours. + checkpoint, hasCheckpoint, err := taskengine.DecodeCheckpoint[dataSetCreationCheckpoint](execution) + if err != nil { + return h.recoverUnnamedCreation(ctx, execution, binding, provider, err) + } + if hasCheckpoint { + // Whether or not its submission was recorded, a request is resolved only + // under the identity it was signed for: a submission's status says + // nothing about who pays for the data set it created. + clientDataSetID, reason, err := checkpoint.requestIdentity(provider) + if err != nil { + if reason == "invalid_checkpoint" { + return h.recoverUnnamedCreation(ctx, execution, binding, provider, err) + } + // The wallet or network moved. That proves only that this context + // cannot look the ID up, wait on it, or resend it safely — the data + // set the original identity asked for may exist and be billing — so + // the generation, its copies, and its fence are all kept for an + // operator who restores the configuration and retries. + return taskengine.Fail(err, reason, nil) + } + if checkpoint.TransactionID != "" { + return h.waitDataSetCreation(ctx, execution, binding, provider, checkpoint, clientDataSetID) + } + // The request went out without its submission being recorded, so it may + // or may not have created the data set. Only the chain can tell. + if !mayCreate { + return h.findRequestedDataSet(ctx, execution, binding, provider, checkpoint, clientDataSetID) + } + return h.sendDataSetCreation(ctx, execution, binding, provider, checkpoint, clientDataSetID) + } + if dataSetCreationSent(binding) { + // The row remembers a request this task no longer can: resolve it by that + // ID rather than adopting a data set by metadata or minting a new one. + return h.recoverUnnamedCreation(ctx, execution, binding, provider, + errors.New("storage service creation checkpoint is missing")) + } + + // Nothing was ever sent for this generation, so a data set the provider + // already holds for the bucket may be adopted. matching, err := h.deps.Storage.FindMatchingDataSet(ctx, binding.ProviderID.SDK(), map[string]string{"bucket": bucket.Name}, provider.CDNEnabled()) if err == nil && matching != nil { dataSetID, clientDataSetID, identityErr := dataSetRefIDs(binding, *matching) if identityErr != nil { - return taskengine.Fail(identityErr, "dataset_identity_mismatch", nil) + return failDataSetEnsure(binding, execution.ID(), identityErr, "dataset_identity_mismatch") } return h.completeDataSetEnsure(binding, execution.ID(), dataSetID, clientDataSetID) } @@ -379,118 +455,279 @@ func (h *TaskHandlers) runDataSetEnsure(ctx context.Context, execution taskengin } return retryTask(err, "dataset_discovery_failed") } - - checkpoint, hasCheckpoint, err := taskengine.DecodeCheckpoint[dataSetCreationCheckpoint](execution) - if err != nil { - return taskengine.Fail(err, "invalid_checkpoint", nil) - } - if !hasCheckpoint && binding.CreateTransactionID != nil && binding.CreateStatusURL != nil && binding.ClientDataSetID != nil { - checkpoint = dataSetCreationCheckpoint{ - AttemptedAt: time.Now().UTC(), TransactionID: *binding.CreateTransactionID, - StatusURL: *binding.CreateStatusURL, ClientDataSetID: binding.ClientDataSetID.String(), - } - hasCheckpoint = true - } - if hasCheckpoint && checkpoint.TransactionID != "" { - return h.waitDataSetCreation(ctx, execution, binding, provider, checkpoint) - } - if hasCheckpoint { - if time.Since(checkpoint.AttemptedAt) >= dataSetAttentionAfter { - err := errors.New("storage service creation outcome could not be recovered") - return taskengine.Fail(err, "dataset_creation_unknown", dataSetFailureSettlement(binding.ID, execution.ID(), err.Error(), false)) - } - return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Checking storage service creation", nil) - } if !mayCreate { return taskengine.Suspend(model.TaskResumeModeExecute, 0, "safe_to_execute", "Storage service is ready to create", nil) } + identity := provider.ContextIdentity() + if !contextIdentityComplete(identity) { + return failDataSetEnsure(binding, execution.ID(), errors.New("storage signing identity is incomplete"), "dependency_unavailable") + } + clientDataSetID, err := newClientDataSetID() + if err != nil { + return retryTask(err, "dataset_creation_not_started") + } + checkpoint = dataSetCreationCheckpoint{ClientDataSetID: clientDataSetID.String(), Identity: &identity} + return h.sendDataSetCreation(ctx, execution, binding, provider, checkpoint, clientDataSetID) +} - checkpoint = dataSetCreationCheckpoint{AttemptedAt: time.Now().UTC()} +// sendDataSetCreation records the generation's client data set ID, then sends a +// create request carrying it. Sending the same ID again is safe: the contract +// registers at most one data set per client data set ID and payer, so a request +// that already landed cannot create a second one. +func (h *TaskHandlers) sendDataSetCreation( + ctx context.Context, + execution taskengine.Execution, + binding *model.StorageDataSet, + provider synapse.ProviderTarget, + checkpoint dataSetCreationCheckpoint, + clientDataSetID sdktypes.BigInt, +) taskengine.Result { + checkpoint.AttemptedAt = time.Now().UTC() + checkpoint.Sends++ + recordedID := idtypes.OnChainIDFromSDK(clientDataSetID) var ( created *storage.CreateDataSetResult evidenceErr error submission storage.CreateDataSetSubmission ) createCtx, cancelCreate := context.WithCancel(ctx) - attempted, createErr := execution.WithCheckpointedEffect(ctx, taskengine.ResourceProviderMutation, checkpoint, nil, func(context.Context) error { - var err error - created, err = provider.CreateDataSet(createCtx, &storage.CreateDataSetOptions{OnSubmitted: func(sub storage.CreateDataSetSubmission) { - submission = sub - checkpoint.TransactionID = sub.TransactionID - checkpoint.StatusURL = sub.StatusURL - if sub.ClientDataSetID != nil { - checkpoint.ClientDataSetID = sub.ClientDataSetID.String() - } - evidenceErr = execution.WriteCheckpointWith(ctx, checkpoint, func(ctx context.Context, repos *repository.Repositories) error { - return repos.Contents.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ - ID: binding.ID, ContentID: derefInt64(binding.CreatedByContentID), TransactionID: sub.TransactionID, - StatusURL: sub.StatusURL, ClientDataSetID: onChainIDPtr(sub.ClientDataSetID), - }) + attempted, createErr := execution.WithCheckpointedEffect(ctx, taskengine.ResourceProviderMutation, checkpoint, + func(ctx context.Context, repos *repository.Repositories) error { + return repos.Contents.RecordDataSetClientID(ctx, binding.ID, recordedID) + }, + func(context.Context) error { + var err error + created, err = provider.CreateDataSet(createCtx, &storage.CreateDataSetOptions{ + ClientDataSetID: &clientDataSetID, + OnSubmitted: func(sub storage.CreateDataSetSubmission) { + submission = sub + checkpoint.TransactionID = sub.TransactionID + checkpoint.StatusURL = sub.StatusURL + evidenceErr = execution.WriteCheckpointWith(ctx, checkpoint, func(ctx context.Context, repos *repository.Repositories) error { + return repos.Contents.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ + ID: binding.ID, ContentID: derefInt64(binding.CreatedByContentID), TransactionID: sub.TransactionID, + StatusURL: sub.StatusURL, ClientDataSetID: &recordedID, + }) + }) + cancelCreate() + }, }) - cancelCreate() - }}) - return err - }) + return err + }) cancelCreate() if createErr != nil && !attempted { + if errors.Is(createErr, taskengine.ErrResourceBusy) { + return taskengine.ResourceWait("Waiting for other storage operations to finish") + } return retryTask(createErr, "dataset_creation_not_started") } if evidenceErr != nil { - clientDataSetID := onChainIDPtr(submission.ClientDataSetID) return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Recording storage service creation", func(ctx context.Context, repos *repository.Repositories) error { if _, err := repos.Contents.AuthorizeDataSetEnsureTask(ctx, binding.ID, execution.ID()); err != nil { return err } return repos.Contents.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ ID: binding.ID, ContentID: derefInt64(binding.CreatedByContentID), TransactionID: submission.TransactionID, - StatusURL: submission.StatusURL, ClientDataSetID: clientDataSetID, + StatusURL: submission.StatusURL, ClientDataSetID: &recordedID, }) }) } if created != nil { - dataSetID, clientDataSetID, identityErr := dataSetResultIDs(binding, created) + dataSetID, createdClientID, identityErr := dataSetResultIDs(binding, created) if identityErr != nil { return taskengine.Fail(identityErr, "dataset_identity_mismatch", nil) } - return h.completeDataSetEnsure(binding, execution.ID(), dataSetID, clientDataSetID) + return h.completeDataSetEnsure(binding, execution.ID(), dataSetID, createdClientID) } - return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Checking storage service creation", nil) + if submission.TransactionID != "" { + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Checking storage service creation", nil) + } + // The request may have reached the provider without its submission reaching + // us, so the chain is checked before anything is sent again. + return taskengine.Suspend(model.TaskResumeModeRecover, unobservedOutcomeDelay(checkpoint.Sends), "provider_confirmation", "Checking storage service creation", nil) } -func (h *TaskHandlers) waitDataSetCreation( +// findRequestedDataSet reads the chain for the data set a request with an +// unobserved outcome may have created. It never sends: once the latest request +// has had time to land and nothing is visible, execute sends the same ID again. +func (h *TaskHandlers) findRequestedDataSet( ctx context.Context, execution taskengine.Execution, binding *model.StorageDataSet, provider synapse.ProviderTarget, checkpoint dataSetCreationCheckpoint, + clientDataSetID sdktypes.BigInt, ) taskengine.Result { - clientID, err := idtypes.ParseOnChainID("clientDataSetID", checkpoint.ClientDataSetID) + ref, found, err := provider.FindDataSetByClientDataSetID(ctx, clientDataSetID) + if errors.Is(err, storage.ErrDataSetCorrelationConflict) { + // The ID is consumed on chain but does not resolve to this request's + // data set, so it is never sent again. + return taskengine.Fail(err, "dataset_correlation_conflict", dataSetFailureSettlement(binding.ID, execution.ID(), err.Error(), true)) + } if err != nil { - return taskengine.Fail(err, "invalid_checkpoint", nil) + return taskengine.Suspend(model.TaskResumeModeRecover, storageDependencyWait, "provider_confirmation", "Checking storage service creation", nil) + } + if found { + dataSetID, foundClientID, identityErr := dataSetRefIDs(binding, ref) + if identityErr != nil { + return taskengine.Fail(identityErr, "dataset_identity_mismatch", nil) + } + return h.completeDataSetEnsure(binding, execution.ID(), dataSetID, foundClientID) + } + if wait := time.Until(checkpoint.AttemptedAt.Add(unobservedOutcomeDelay(checkpoint.Sends))); wait > 0 { + return taskengine.Suspend(model.TaskResumeModeRecover, wait, "provider_confirmation", "Checking storage service creation", nil) + } + return taskengine.Suspend(model.TaskResumeModeExecute, 0, "safe_to_execute", "Retrying storage service creation", nil) +} + +// requestIdentity returns the client data set ID a checkpointed request used, +// once the rebuilt context is confirmed to sign for the same payer, chain, and +// record keeper. Under another identity the ID would be looked up, or sent +// again, where the contract cannot tell that it was already used. +func (c dataSetCreationCheckpoint) requestIdentity(provider synapse.ProviderTarget) (sdktypes.BigInt, string, error) { + clientID, err := idtypes.ParseOnChainID("clientDataSetID", c.ClientDataSetID) + if err != nil || clientID.IsZero() || c.Identity == nil { + return sdktypes.BigInt{}, "invalid_checkpoint", errors.New("storage service creation checkpoint has no request identity") + } + if provider.ContextIdentity() != *c.Identity { + return sdktypes.BigInt{}, "dataset_identity_changed", errors.New("the wallet or network changed after storage service creation began") + } + return clientID.SDK(), "", nil +} + +// newClientDataSetID draws a random nonzero uint256. It is never derived from +// local row IDs, which start over in a fresh database while IDs consumed on +// chain stay consumed. +func newClientDataSetID() (sdktypes.BigInt, error) { + var raw [32]byte + for { + if _, err := rand.Read(raw[:]); err != nil { + return sdktypes.BigInt{}, fmt.Errorf("creating client data set ID: %w", err) + } + if value := new(big.Int).SetBytes(raw[:]); value.Sign() != 0 { + return sdktypes.BigIntFromBig(value) + } } - clientSDK := clientID.SDK() +} + +func contextIdentityComplete(identity storage.ContextIdentity) bool { + return identity.Payer != (common.Address{}) && identity.ChainID.IsValid() && identity.RecordKeeper != (common.Address{}) +} + +// unobservedOutcomeDelay spaces the chain checks and resends that follow +// requests whose outcome was never observed. +func unobservedOutcomeDelay(sends int) time.Duration { + delay := unobservedOutcomeBaseDelay + for i := 1; i < sends && delay < unobservedOutcomeMaxDelay; i++ { + delay *= 2 + } + return min(delay, unobservedOutcomeMaxDelay) +} + +func (h *TaskHandlers) waitDataSetCreation( + ctx context.Context, + execution taskengine.Execution, + binding *model.StorageDataSet, + provider synapse.ProviderTarget, + checkpoint dataSetCreationCheckpoint, + clientDataSetID sdktypes.BigInt, +) taskengine.Result { result, err := provider.WaitForDataSetCreated(ctx, storage.CreateDataSetSubmission{ ProviderID: binding.ProviderID.SDK(), TransactionID: checkpoint.TransactionID, - StatusURL: checkpoint.StatusURL, ClientDataSetID: &clientSDK, + StatusURL: checkpoint.StatusURL, ClientDataSetID: &clientDataSetID, }) if err != nil { if errors.Is(err, synapse.ErrProviderTransactionRejected) { + if checkpoint.Sends > 1 { + // An earlier request with this ID had no observed outcome, so the + // rejection may only mean that request created the data set first. + // The dead submission is dropped and the ID is looked up instead. + checkpoint.TransactionID, checkpoint.StatusURL = "", "" + if err := execution.WriteCheckpoint(ctx, checkpoint); err != nil { + return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Waiting for storage service", nil) + } + return taskengine.Suspend(model.TaskResumeModeRecover, 0, "provider_confirmation", "Checking storage service creation", nil) + } return taskengine.Fail(err, "dataset_creation_rejected", dataSetFailureSettlement(binding.ID, execution.ID(), err.Error(), true)) } return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Waiting for storage service", nil) } - dataSetID, clientDataSetID, err := dataSetResultIDs(binding, result) + dataSetID, createdClientID, err := dataSetResultIDs(binding, result) if err != nil { return taskengine.Fail(err, "dataset_identity_mismatch", nil) } - return h.completeDataSetEnsure(binding, execution.ID(), dataSetID, clientDataSetID) + return h.completeDataSetEnsure(binding, execution.ID(), dataSetID, createdClientID) +} + +// dataSetCreationSent reports whether a create request may already have gone out +// for this generation. The client data set ID and the submission are both +// written before the provider call, so a row carrying neither proves nothing was +// sent — and a generation that never asked for a data set can be given up +// without risking one that exists and is billing. +func dataSetCreationSent(binding *model.StorageDataSet) bool { + if binding == nil { + return true + } + if binding.ClientDataSetID != nil && !binding.ClientDataSetID.IsZero() { + return true + } + return binding.CreateTransactionID != nil && *binding.CreateTransactionID != "" +} + +// failDataSetEnsure fails a creation that cannot go on. When the row proves no +// request was ever sent, nothing can exist: the generation is given up under +// dataset_creation_unsent, which offers no retry because a retry would have +// nothing to do. Otherwise the task fails with reason and keeps the generation, +// its copies, and its fence, because a data set may exist and may be billing. +func failDataSetEnsure(binding *model.StorageDataSet, taskID int64, err error, reason string) taskengine.Result { + if dataSetCreationSent(binding) { + return taskengine.Fail(err, reason, nil) + } + return taskengine.Fail(err, "dataset_creation_unsent", dataSetFailureSettlement(binding.ID, taskID, err.Error(), true)) +} + +// recoverUnnamedCreation resolves a creation whose checkpoint can no longer name +// its request. The client data set ID reaches the row before the provider call, +// so the row can still name it, and the lookup that follows only reads. Without +// that ID nothing was ever sent and the generation is given up instead. +func (h *TaskHandlers) recoverUnnamedCreation( + ctx context.Context, + execution taskengine.Execution, + binding *model.StorageDataSet, + provider synapse.ProviderTarget, + cause error, +) taskengine.Result { + if binding.ClientDataSetID == nil || binding.ClientDataSetID.IsZero() { + return failDataSetEnsure(binding, execution.ID(), cause, "invalid_checkpoint") + } + ref, found, err := provider.FindDataSetByClientDataSetID(ctx, binding.ClientDataSetID.SDK()) + if errors.Is(err, storage.ErrDataSetCorrelationConflict) { + return taskengine.Fail(err, "dataset_correlation_conflict", + dataSetFailureSettlement(binding.ID, execution.ID(), err.Error(), true)) + } + if err != nil { + return taskengine.Suspend(model.TaskResumeModeRecover, storageDependencyWait, "provider_confirmation", "Checking storage service creation", nil) + } + if found { + dataSetID, foundClientID, identityErr := dataSetRefIDs(binding, ref) + if identityErr != nil { + return taskengine.Fail(identityErr, "dataset_identity_mismatch", nil) + } + return h.completeDataSetEnsure(binding, execution.ID(), dataSetID, foundClientID) + } + // The chain holds nothing under that ID, and nothing here can prove which + // identity signed the request, so it is never sent again from this task. + return taskengine.Fail(cause, "invalid_checkpoint", nil) } // dataSetFailureSettlement records that a generation could not be created. -// rejected says the chain refused the creation, which is proof no data set -// exists; an unknown outcome is not, and leaves the row in place so an operator -// still has the provider and the transaction to work from. -func dataSetFailureSettlement(dataSetID, taskID int64, message string, rejected bool) taskengine.Settlement { +// terminal says this generation will never hold a data set of ours: the chain +// refused the creation, the ID its request used does not resolve to one, or the +// row proves no request was ever sent. It then fails the copies, retires the +// generation, and releases its creation fence. Anything short of that proof +// leaves the row in place so an operator still has the provider, the ID, and any +// transaction to work from. +func dataSetFailureSettlement(dataSetID, taskID int64, message string, terminal bool) taskengine.Settlement { return func(ctx context.Context, repos *repository.Repositories) error { if _, err := repos.Contents.AuthorizeDataSetEnsureTask(ctx, dataSetID, taskID); err != nil { return err @@ -498,18 +735,17 @@ func dataSetFailureSettlement(dataSetID, taskID int64, message string, rejected if err := repos.Contents.MarkDataSetFailed(ctx, dataSetID, message); err != nil { return err } - if !rejected { + if !terminal { // The outcome is unknown, so the generation may still exist on the - // provider. Retrying this task rediscovers it through - // FindMatchingDataSet and continues the copies bound to it, and - // continuation skips copies that already failed — so they are left - // alone rather than terminated on a guess. + // provider. Retrying this task looks the recorded ID up again and + // continues the copies bound to it, and continuation skips copies that + // already failed — so they are left alone rather than terminated on a + // guess. return nil } - // The chain refused the creation, so nothing will ever serve these - // copies: the data set they name will not exist, and continuation would - // have nothing to rediscover. Failing them keeps their objects from - // sitting at "uploading" forever. + // Nothing will ever serve these copies: the data set they name will not + // exist, and continuation would have nothing to rediscover. Failing them + // keeps their objects from sitting at "uploading" forever. if err := failDataSetCopies(ctx, repos, dataSetID, message); err != nil { return err } @@ -520,7 +756,10 @@ func dataSetFailureSettlement(dataSetID, taskID int64, message string, rejected if _, err := repos.Contents.RetireRejectedDataSet(ctx, dataSetID); err != nil { return err } - return nil + // Nothing will be created for the retired generation, so its creation + // fence is released: a new replacement of the slot is refused while an + // earlier target still holds one. + return repos.Contents.CompleteDataSetEnsureTask(ctx, dataSetID, taskID) } } @@ -615,7 +854,7 @@ func (h *TaskHandlers) transferPlanHandler() taskengine.Handler { return h.completeCopyTask(input, execution.ID(), "Storage copy is complete") } if copyRow.Status == model.StorageCopyStatusPieceReady || copyRow.Status == model.StorageCopyStatusCommitting { - return h.advanceCopyTask(input, execution.ID(), model.TaskTypeStorageCommitCoordinate, "Storage copy is ready to register") + return h.advanceCopyTask(input, execution.ID(), model.TaskTypeStorageCommit, "Storage copy is ready to register") } binding, err := h.deps.Repositories.Contents.GetDataSetBindingByID(ctx, copyRow.StorageDataSetID) if err != nil { @@ -699,7 +938,7 @@ func (h *TaskHandlers) runStore(ctx context.Context, execution taskengine.Execut return h.completeCopyTask(input, execution.ID(), "Storage copy is complete") } if copyRow.Status == model.StorageCopyStatusPieceReady || copyRow.Status == model.StorageCopyStatusCommitting { - return h.advanceCopyTask(input, execution.ID(), model.TaskTypeStorageCommitCoordinate, "Storage copy is ready to register") + return h.advanceCopyTask(input, execution.ID(), model.TaskTypeStorageCommit, "Storage copy is ready to register") } checkpoint, hasCheckpoint, err := taskengine.DecodeCheckpoint[storeCheckpoint](execution) if err != nil { @@ -721,6 +960,34 @@ func (h *TaskHandlers) runStore(ctx context.Context, execution taskengine.Execut cacheKey := model.ContentCacheKey(content.ID) releaseCache := h.deps.CacheGate.HoldRead(cacheKey) defer releaseCache() + // One provider slot spans identity calculation and the transfer, so a task + // that has to wait for a slot never hashes the bytes first. + var outcome taskengine.Result + err = execution.WithResource(ctx, taskengine.ResourceProviderMutation, func(ctx context.Context) error { + outcome = h.storeWithProviderSlot(ctx, execution, input, copyRow, target, content, bucket, cacheKey) + return nil + }) + if errors.Is(err, taskengine.ErrResourceBusy) { + return taskengine.ResourceWait("Waiting for other storage operations to finish") + } + if err != nil { + return h.retryStoreNotStarted(execution, err) + } + return outcome +} + +// storeWithProviderSlot runs while the caller holds a provider mutation slot and +// the cache read gate for cacheKey. +func (h *TaskHandlers) storeWithProviderSlot( + ctx context.Context, + execution taskengine.Execution, + input storagepipeline.CopyGenerationInput, + copyRow *model.StorageCopy, + target synapse.DataSetTarget, + content *model.StorageContent, + bucket *model.Bucket, + cacheKey string, +) taskengine.Result { calculateReader, _, err := h.deps.Cache.Get(ctx, bucket.Name, cacheKey) if err != nil { if os.IsNotExist(err) { @@ -748,7 +1015,7 @@ func (h *TaskHandlers) runStore(ctx context.Context, execution taskengine.Execut return h.retryCopyTask(execution, input, copyRow, err, "cache_open_failed") } defer func() { _ = storeReader.Close() }() - checkpoint = storeCheckpoint{ + checkpoint := storeCheckpoint{ AttemptedAt: time.Now().UTC(), IntendedPieceCID: pieceInfo.CIDv2.String(), ProviderServiceURL: target.ServiceURL(), } @@ -858,7 +1125,7 @@ func (h *TaskHandlers) runPull(ctx context.Context, execution taskengine.Executi return h.completeCopyTask(input, execution.ID(), "Storage copy is complete") } if copyRow.Status == model.StorageCopyStatusPieceReady || copyRow.Status == model.StorageCopyStatusCommitting { - return h.advanceCopyTask(input, execution.ID(), model.TaskTypeStorageCommitCoordinate, "Storage copy is ready to register") + return h.advanceCopyTask(input, execution.ID(), model.TaskTypeStorageCommit, "Storage copy is ready to register") } checkpoint, hasCheckpoint, err := taskengine.DecodeCheckpoint[pullCheckpoint](execution) if err != nil { @@ -936,6 +1203,9 @@ func (h *TaskHandlers) runPull(ctx context.Context, execution taskengine.Executi }) return pullErr }) + if errors.Is(err, taskengine.ErrResourceBusy) { + return taskengine.ResourceWait("Waiting for other storage operations to finish") + } if err != nil { switch synapse.ClassifyPullError(err) { case synapse.PullErrorRetryable: @@ -949,6 +1219,8 @@ func (h *TaskHandlers) runPull(ctx context.Context, execution taskengine.Executi return h.finishPieceTransferWithExtra(execution, input, copyRow, target, pieceCID, checkpoint.CommitExtraDataHex, checkpoint.AttemptID) } +// commitCoordinateHandler finishes relay tasks that an earlier build enqueued +// between transfer and commit; nothing enqueues this type any more. func (h *TaskHandlers) commitCoordinateHandler() taskengine.Handler { definition := copyDefinition(model.TaskTypeStorageCommitCoordinate, h.retryLimit()) run := func(ctx context.Context, execution taskengine.Execution) taskengine.Result { @@ -988,6 +1260,9 @@ func (h *TaskHandlers) runCommit(ctx context.Context, execution taskengine.Execu if copyRow.Status == model.StorageCopyStatusCommitted { return h.completeCopyTask(input, execution.ID(), "Storage copy is complete") } + if copyRow.Status != model.StorageCopyStatusPieceReady && copyRow.Status != model.StorageCopyStatusCommitting { + return h.failCopyTask(execution, input, copyRow, errors.New("storage copy has no transferable piece"), "piece_not_ready") + } binding, err := h.deps.Repositories.Contents.GetDataSetBindingByID(ctx, copyRow.StorageDataSetID) if err != nil || binding == nil { if err == nil { @@ -1046,6 +1321,9 @@ func (h *TaskHandlers) runCommit(ctx context.Context, execution taskengine.Execu } else { advanced, err = advance(ctx) } + if errors.Is(err, taskengine.ErrResourceBusy) { + return taskengine.ResourceWait("Waiting for other storage operations to finish") + } if err != nil { if advanced.State == storagecommit.AdvancePending || advanced.State == storagecommit.AdvanceSubmitted { return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Checking storage registration", nil) @@ -1054,7 +1332,7 @@ func (h *TaskHandlers) runCommit(ctx context.Context, execution taskengine.Execu } switch advanced.State { case storagecommit.AdvanceWaitingCapacity: - return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "capacity", "Waiting to register storage", nil) + return taskengine.Suspend(model.TaskResumeModeRecover, commitCapacityBackstop, "capacity", "Waiting to register storage", nil) case storagecommit.AdvanceSubmitted, storagecommit.AdvancePending: return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Waiting for storage registration", nil) case storagecommit.AdvanceRejected: @@ -1337,7 +1615,7 @@ func (h *TaskHandlers) finishPieceTransferWithExtra( }); err != nil { return err } - return h.enqueueSuccessorCopyTask(ctx, repos, input, execution.ID(), model.TaskTypeStorageCommitCoordinate) + return h.enqueueSuccessorCopyTask(ctx, repos, input, execution.ID(), model.TaskTypeStorageCommit) }) } @@ -1434,16 +1712,9 @@ func (h *TaskHandlers) settleCopyFailure( }); err != nil { return err } - if err := repos.Contents.CompleteCopyTask(ctx, input.CopyID, input.Generation, execution.ID()); err != nil { - return err - } - upload, err := repos.Contents.GetByID(ctx, copyRow.ContentID) - if err != nil || upload == nil { - return errors.Join(err, repository.ErrNotFound) - } - // The content row is shared by every version of these bytes, so recording - // the failure once covers all of them; there are no followers to fan out to. - return repos.Contents.RecordContentFailure(ctx, upload.ID, message) + // The content itself is flagged by MarkUploadCopyFailed, and only once no + // copy is left that holds or may still hold it. + return repos.Contents.CompleteCopyTask(ctx, input.CopyID, input.Generation, execution.ID()) } func commitAttentionFailureReason(code storagecommit.AttentionCode) string { @@ -1512,7 +1783,15 @@ func dataSetRefIDs(binding *model.StorageDataSet, ref storage.DataSetRef) (idtyp if dataSetID.IsZero() { return idtypes.OnChainID{}, idtypes.OnChainID{}, errors.New("data set resolved a zero identity") } - return dataSetID, idtypes.OnChainIDFromSDK(ref.ClientDataSetID()), nil + clientDataSetID := idtypes.OnChainIDFromSDK(ref.ClientDataSetID()) + // Once a request goes out, this generation owns the ID it used. A data set + // resolved under any other ID belongs to a different generation, however + // well its metadata matches. + if binding.ClientDataSetID != nil && !binding.ClientDataSetID.IsZero() && !clientDataSetID.Equal(*binding.ClientDataSetID) { + return idtypes.OnChainID{}, idtypes.OnChainID{}, fmt.Errorf( + "data set resolved client ID %s, want %s", clientDataSetID.String(), binding.ClientDataSetID.String()) + } + return dataSetID, clientDataSetID, nil } func dataSetResultIDs(binding *model.StorageDataSet, result *storage.CreateDataSetResult) (idtypes.OnChainID, idtypes.OnChainID, error) { @@ -1522,14 +1801,6 @@ func dataSetResultIDs(binding *model.StorageDataSet, result *storage.CreateDataS return dataSetRefIDs(binding, result.DataSet) } -func onChainIDPtr(value *sdktypes.BigInt) *idtypes.OnChainID { - if value == nil { - return nil - } - converted := idtypes.OnChainIDFromSDK(*value) - return &converted -} - func (h *TaskHandlers) enqueueAfterUploadEvictions(ctx context.Context, repos *repository.Repositories, refs []repository.ObjectVersionRef) error { if h.deps.EvictionPolicy != cache.EvictionPolicyAfterUpload || h.taskService == nil { return nil diff --git a/internal/worker/task_handlers_test.go b/internal/worker/task_handlers_test.go index 37af314..fed4864 100644 --- a/internal/worker/task_handlers_test.go +++ b/internal/worker/task_handlers_test.go @@ -76,6 +76,7 @@ type handlerRuntimeOptions struct { lowPercent int concurrency int maxRetries *int + leaseDuration time.Duration register func(*worker.TaskHandlers, *taskengine.Registry) error } @@ -127,8 +128,12 @@ func newHandlerTestRuntime(t *testing.T, options handlerRuntimeOptions) handlerT } handlers.SetTaskService(service) concurrency := max(options.concurrency, 1) + leaseDuration := options.leaseDuration + if leaseDuration == 0 { + leaseDuration = 300 * time.Millisecond + } engine, err := taskengine.NewEngine(taskengine.EngineConfig{ - Concurrency: concurrency, PollInterval: 5 * time.Millisecond, LeaseDuration: 300 * time.Millisecond, + Concurrency: concurrency, PollInterval: 5 * time.Millisecond, LeaseDuration: leaseDuration, Retention: time.Hour, ProviderMutationConcurrency: 4, DestructiveMutationConcurrency: 2, }, repos, registry, slog.Default()) if err != nil { @@ -164,9 +169,12 @@ type testReceiptChecker struct { } type testServiceTerminator struct { - calls atomic.Int64 - result *synapse.TerminationResult - err error + calls atomic.Int64 + result *synapse.TerminationResult + err error + identity storage.ContextIdentity + // notPaidFor makes the chain report the data set as another wallet's. + notPaidFor atomic.Bool } func (t *testServiceTerminator) TerminateService(context.Context, sdktypes.BigInt) (*synapse.TerminationResult, error) { @@ -174,6 +182,20 @@ func (t *testServiceTerminator) TerminateService(context.Context, sdktypes.BigIn return t.result, t.err } +func (t *testServiceTerminator) VerifyServicePayer(context.Context, sdktypes.BigInt) error { + if t.notPaidFor.Load() { + return fmt.Errorf("data set is paid for by another wallet: %w", synapse.ErrServicePaidByAnother) + } + return nil +} + +func (t *testServiceTerminator) ContextIdentity() storage.ContextIdentity { + if t.identity != (storage.ContextIdentity{}) { + return t.identity + } + return testutil.DefaultContextIdentity +} + type testEpochReader struct { epoch int64 err error @@ -580,20 +602,147 @@ func TestStorageCleanupContinuesPastUnsupportedCopy(t *testing.T) { } runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{storage: storageClient}) ctx := t.Context() + content, taskRow := seedStorageCleanup(t, runtime, model.StorageCleanupCopyStatusUnsupported, model.StorageCleanupCopyStatusPending) + limited := &limitedClaimRepository{TaskRepository: runtime.repos.Tasks, maximum: 1} + runtime.repos.Tasks = limited + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return limited.claims.Load() == 1 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover + }) + if deleteCalls.Load() != 1 { + t.Fatalf("cleanup delete calls = %d, want 1", deleteCalls.Load()) + } + copies, err := runtime.repos.StorageCleanup.AuthorizeTask(ctx, content.ID, 1, taskRow.ID) + if err != nil || len(copies) != 2 || copies[1].Status != model.StorageCleanupCopyStatusDeleteScheduled { + t.Fatalf("cleanup copies = %#v, err=%v", copies, err) + } +} + +// TestStorageCleanupFinalizesContent checks that cleanup finishes past a +// deletion the provider cannot perform: it releases the cached bytes, deletes +// the content's current-state rows, and keeps the cleanup ledger. +func TestStorageCleanupFinalizesContent(t *testing.T) { + var cacheDeletes atomic.Int64 + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: removedPieceStorageClient(), + cache: &testutil.MockCache{DeleteFunc: func(context.Context, string, string) error { + cacheDeletes.Add(1) + return nil + }}, + }) + ctx := t.Context() + content, taskRow := seedStorageCleanup(t, runtime, model.StorageCleanupCopyStatusUnsupported, model.StorageCleanupCopyStatusPending) + now := time.Now() + if _, err := runtime.db.NewInsert().Model(&model.ObjectCache{ContentID: content.ID, InCache: true, CreatedAt: now, UpdatedAt: now}).Exec(ctx); err != nil { + t.Fatalf("create cache record: %v", err) + } + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusCompleted + }) + // An uncertain lease makes the engine run the handler again, and releasing + // the cached bytes is idempotent, so the count is a lower bound. + if cacheDeletes.Load() < 1 { + t.Fatal("cleanup finished without releasing the cached bytes") + } + for _, rows := range []struct{ table, column string }{ + {"storage_contents", "id"}, {"object_cache", "content_id"}, {"storage_data_sets", "created_by_content_id"}, + } { + var count int + if err := runtime.db.NewRaw("SELECT count(*) FROM "+rows.table+" WHERE "+rows.column+" = ?", content.ID).Scan(ctx, &count); err != nil || count != 0 { + t.Fatalf("%s rows naming finalized content = %d, err=%v", rows.table, count, err) + } + } + var statuses []model.StorageCleanupCopyStatus + if err := runtime.db.NewSelect().Model((*model.StorageCleanupCopy)(nil)).Column("status"). + Where("content_id = ?", content.ID).Order("copy_index").Scan(ctx, &statuses); err != nil { + t.Fatalf("load cleanup ledger: %v", err) + } + if len(statuses) != 2 || statuses[0] != model.StorageCleanupCopyStatusUnsupported || statuses[1] != model.StorageCleanupCopyStatusRemoved { + t.Fatalf("cleanup ledger statuses = %v", statuses) + } +} + +// TestStorageCleanupRetriesCacheRelease checks that a failed cache release +// keeps the content bound to its cleanup, and a later run finishes it. +func TestStorageCleanupRetriesCacheRelease(t *testing.T) { + var cacheDeletes atomic.Int64 + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: removedPieceStorageClient(), + cache: &testutil.MockCache{DeleteFunc: func(context.Context, string, string) error { + if cacheDeletes.Add(1) == 1 { + return errors.New("cache volume is busy") + } + return nil + }}, + }) + ctx := t.Context() + content, taskRow := seedStorageCleanup(t, runtime, model.StorageCleanupCopyStatusPending) + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusPending && task.RetryCount == 1 + }) + stored, err := runtime.repos.Contents.GetByID(ctx, content.ID) + if err != nil || stored == nil || stored.CleanupTaskID == nil || *stored.CleanupTaskID != taskRow.ID { + t.Fatalf("content after failed cache release = %#v, err=%v", stored, err) + } + if _, err := runtime.repos.Tasks.WakePending(ctx, []int64{taskRow.ID}); err != nil { + t.Fatalf("WakePending: %v", err) + } + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusCompleted + }) + if stored, err := runtime.repos.Contents.GetByID(ctx, content.ID); stored != nil { + t.Fatalf("content after cleanup = %#v, err=%v", stored, err) + } + if cacheDeletes.Load() < 2 { + t.Fatalf("cache deletes = %d, want the failed release and at least one retry", cacheDeletes.Load()) + } +} + +// removedPieceStorageClient reports every cleanup piece as already gone. +func removedPieceStorageClient() *testutil.MockStorageClient { + cleanupContext := testCleanupContext{ + pieceStatus: func(context.Context, cid.Cid) (*storage.PieceStatus, error) { + return &storage.PieceStatus{Exists: false}, nil + }, + deletePiece: func(context.Context, sdktypes.BigInt) (*sdktypes.WriteResult, error) { + return nil, errors.New("unexpected remote deletion") + }, + } + return &testutil.MockStorageClient{ + OpenCleanupContextFunc: func(context.Context, sdktypes.BigInt, storage.NewDataSetContextOptions) (synapse.CleanupContext, error) { + return cleanupContext, nil + }, + } +} + +// seedStorageCleanup stores content whose versions are gone, with one cleanup +// copy per status on its own provider, and binds a cleanup task to it. +func seedStorageCleanup(t *testing.T, runtime handlerTestRuntime, statuses ...model.StorageCleanupCopyStatus) (*model.StorageContent, *model.Task) { + t.Helper() + ctx := t.Context() sequence := storedObjectSequence.Add(1) - bucket := &model.Bucket{Name: fmt.Sprintf("cleanup-task-%d", sequence), Status: model.BucketStatusActive, DefaultCopies: 2, MinimumDurableCopies: 2} + copies := len(statuses) + bucket := &model.Bucket{Name: fmt.Sprintf("cleanup-task-%d", sequence), Status: model.BucketStatusActive, DefaultCopies: copies, MinimumDurableCopies: copies} if err := runtime.repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("create cleanup bucket: %v", err) } content := &model.StorageContent{ BucketID: bucket.ID, ContentSize: 1, Checksum: testutil.StorageChecksum(fmt.Sprintf("cleanup-%d", sequence)), - RequestedCopies: 2, CleanupGeneration: 1, + RequestedCopies: copies, CleanupGeneration: 1, } if _, err := runtime.db.NewInsert().Model(content).Exec(ctx); err != nil { t.Fatalf("create cleanup content: %v", err) } - for copyIndex := range 2 { + for copyIndex, status := range statuses { providerID := testOnChainID(t, 9000+sequence*10+int64(copyIndex)) dataSetID := testOnChainID(t, 10000+sequence*10+int64(copyIndex)) clientDataSetID := testOnChainID(t, 11000+sequence*10+int64(copyIndex)) @@ -608,15 +757,12 @@ func TestStorageCleanupContinuesPastUnsupportedCopy(t *testing.T) { }); err != nil { t.Fatalf("mark cleanup binding %d ready: %v", copyIndex, err) } - status := model.StorageCleanupCopyStatusPending - if copyIndex == 0 { - status = model.StorageCleanupCopyStatusUnsupported - } row := &model.StorageCleanupCopy{ ContentID: content.ID, BucketID: bucket.ID, CopyIndex: copyIndex, ProviderID: providerID, StorageDataSetID: binding.ID, DataSetID: &dataSetID, ClientDataSetID: &clientDataSetID, PieceID: testOnChainID(t, 12000+sequence*10+int64(copyIndex)), - PieceCID: testPieceCID(t, fmt.Sprintf("cleanup-piece-%d-%d", sequence, copyIndex)).String(), Status: status, + PieceCID: testPieceCID(t, fmt.Sprintf("cleanup-piece-%d-%d", sequence, copyIndex)).String(), + Checksum: content.Checksum, Status: status, } if _, err := runtime.db.NewInsert().Model(row).Exec(ctx); err != nil { t.Fatalf("create cleanup copy %d: %v", copyIndex, err) @@ -632,21 +778,7 @@ func TestStorageCleanupContinuesPastUnsupportedCopy(t *testing.T) { if err := runtime.repos.StorageCleanup.BindTask(ctx, content.ID, 1, taskRow.ID); err != nil { t.Fatalf("bind cleanup task: %v", err) } - limited := &limitedClaimRepository{TaskRepository: runtime.repos.Tasks, maximum: 1} - runtime.repos.Tasks = limited - cancel, done := runHandlerEngine(t, runtime) - defer stopHandlerEngine(t, cancel, done) - - waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { - return limited.claims.Load() == 1 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover - }) - if deleteCalls.Load() != 1 { - t.Fatalf("cleanup delete calls = %d, want 1", deleteCalls.Load()) - } - copies, err := runtime.repos.StorageCleanup.AuthorizeTask(ctx, content.ID, 1, taskRow.ID) - if err != nil || len(copies) != 2 || copies[1].Status != model.StorageCleanupCopyStatusDeleteScheduled { - t.Fatalf("cleanup copies = %#v, err=%v", copies, err) - } + return content, taskRow } func TestStorageCleanupAdmissionFailureDoesNotScheduleDeletion(t *testing.T) { @@ -696,7 +828,7 @@ func TestStorageCleanupAdmissionFailureDoesNotScheduleDeletion(t *testing.T) { ContentID: content.ID, BucketID: bucket.ID, CopyIndex: 0, ProviderID: providerID, StorageDataSetID: binding.ID, DataSetID: &dataSetID, ClientDataSetID: &clientDataSetID, PieceID: testOnChainID(t, 29104), PieceCID: testPieceCID(t, "cleanup-admission-piece").String(), - Status: model.StorageCleanupCopyStatusPending, + Checksum: content.Checksum, Status: model.StorageCleanupCopyStatusPending, } if _, err := runtime.db.NewInsert().Model(cleanupCopy).Exec(t.Context()); err != nil { t.Fatalf("create cleanup admission copy: %v", err) @@ -1596,15 +1728,20 @@ func TestTaskGCLeavesStorageEvidenceIntact(t *testing.T) { } func TestStoreTasksReachAndRespectProviderMutationLimit(t *testing.T) { - var running, maximum atomic.Int64 + var running, maximum, cacheOpens atomic.Int64 entered := make(chan struct{}, 6) release := make(chan struct{}) cacheStore := &testutil.MockCache{GetFunc: func(context.Context, string, string) (io.ReadCloser, *cache.ObjectInfo, error) { + cacheOpens.Add(1) return io.NopCloser(strings.NewReader(strings.Repeat("s", 128))), &cache.ObjectInfo{Size: 128}, nil }} storageClient := &testutil.MockStorageClient{} runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ cache: cacheStore, storage: storageClient, policy: cache.EvictionPolicyNone, concurrency: 8, + // The holders wait on the provider for as long as the test keeps them. A + // lease they cannot lose keeps a slow race run from claiming them again + // mid-wait, which is not what this test is about. + leaseDuration: 5 * time.Second, register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { return handlers.RegisterStorage(registry) }, @@ -1659,14 +1796,52 @@ func TestStoreTasksReachAndRespectProviderMutationLimit(t *testing.T) { t.Fatal("store tasks exceeded provider mutation concurrency") case <-time.After(50 * time.Millisecond): } + // Tasks beyond the limit give their workers back instead of blocking on the + // gate, and they neither hash the bytes, checkpoint, nor spend retries. + yielded := make(map[int64]bool) + deadline := time.Now().Add(3 * time.Second) + for len(yielded) != 2 && time.Now().Before(deadline) { + clear(yielded) + for _, taskRow := range tasks { + stored, err := runtime.repos.Tasks.GetByID(t.Context(), taskRow.ID) + if err != nil { + close(release) + t.Fatalf("load store task: %v", err) + } + if stored.Status == model.TaskStatusPending && stored.ResumeMode == model.TaskResumeModeExecute && + stored.WaitReason != nil && *stored.WaitReason == "resource" && stored.RetryCount == 0 && len(stored.Checkpoint) == 0 { + yielded[stored.ID] = true + } + } + time.Sleep(5 * time.Millisecond) + } + if len(yielded) != 2 || cacheOpens.Load() != 8 { + close(release) + t.Fatalf("store tasks beyond the limit = yielded:%d cache opens:%d, want 2 yielded and 8 opens", len(yielded), cacheOpens.Load()) + } close(release) for _, taskRow := range tasks { + if yielded[taskRow.ID] { + continue + } waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { return task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover }) } - if maximum.Load() != 4 { - t.Fatalf("store provider mutation maximum = %d, want 4", maximum.Load()) + for id := range yielded { + if _, err := runtime.db.NewRaw(`UPDATE tasks SET available_at = ? WHERE id = ?`, time.Now().Add(-time.Second), id).Exec(t.Context()); err != nil { + t.Fatalf("wake yielded store task: %v", err) + } + waitForTask(t, runtime.repos, id, func(task *model.Task) bool { + return task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover + }) + } + // Every task read its bytes twice after taking a slot, once to identify them + // and once to send them. A store whose lease became uncertain while it waited + // on the provider is claimed again and may read them again, so the total is + // only a floor. + if maximum.Load() != 4 || cacheOpens.Load() < 12 { + t.Fatalf("store provider mutation = maximum:%d cache opens:%d, want 4 and at least 12", maximum.Load(), cacheOpens.Load()) } } @@ -1718,8 +1893,8 @@ func TestStoreAdmissionFailureDoesNotStartUploadOrProgress(t *testing.T) { if err != nil || copyRow.Status != model.StorageCopyStatusPending || copyRow.ActiveTaskID == nil || *copyRow.ActiveTaskID != taskRow.ID || copyRow.IngressStoreAttempt != 0 { t.Fatalf("copy after store admission failure = %#v, err=%v", copyRow, err) } - if storeCalls.Load() != 0 || cacheOpens.Load() != 2 { - t.Fatalf("store admission calls = store:%d cache:%d, want 0/2", storeCalls.Load(), cacheOpens.Load()) + if storeCalls.Load() != 0 || cacheOpens.Load() != 0 { + t.Fatalf("store admission calls = store:%d cache:%d, want 0/0", storeCalls.Load(), cacheOpens.Load()) } if err := runtime.service.Retry(t.Context(), taskRow.ID); err != nil { t.Fatalf("retry store admission task: %v", err) @@ -1727,7 +1902,7 @@ func TestStoreAdmissionFailureDoesNotStartUploadOrProgress(t *testing.T) { waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { return limited.claims.Load() == 2 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeExecute }) - if storeCalls.Load() != 0 || cacheOpens.Load() != 2 { + if storeCalls.Load() != 0 || cacheOpens.Load() != 0 { t.Fatalf("store recovery calls = store:%d cache:%d, want no new calls", storeCalls.Load(), cacheOpens.Load()) } } @@ -2029,6 +2204,15 @@ func TestPullRecoverObservesThenRepeatsIdenticalRequestInExecute(t *testing.T) { if pullCalls.Load() != 2 { t.Fatalf("pull calls = %d, want 2", pullCalls.Load()) } + // A finished transfer schedules registration directly. + copyRow, err := runtime.repos.Contents.GetUploadCopyByID(t.Context(), pipeline.target.ID) + if err != nil || copyRow == nil || copyRow.ActiveTaskID == nil { + t.Fatalf("target copy after pull = %#v, err=%v", copyRow, err) + } + next, err := runtime.repos.Tasks.GetByID(t.Context(), *copyRow.ActiveTaskID) + if err != nil || next == nil || next.Type != model.TaskTypeStorageCommit { + t.Fatalf("task after pull = %#v, err=%v, want storage_commit", next, err) + } observedMu.Lock() defer observedMu.Unlock() if len(observed) != 2 || observed[0] != observed[1] { @@ -2072,6 +2256,10 @@ func TestPullProviderFailureAtomicallyAbandonsAttemptAndCopy(t *testing.T) { if err != nil || copyRow.Status != model.StorageCopyStatusFailed || copyRow.ActiveTaskID != nil { t.Fatalf("failed pull copy = %#v, err=%v", copyRow, err) } + // The source still holds a readable copy, so the content itself is fine. + if content, err := runtime.repos.Contents.GetByID(t.Context(), pipeline.upload.ID); err != nil || content.ErrorMessage != nil { + t.Fatalf("content after a failed copy next to a readable one = %#v, err=%v, want it not flagged", content, err) + } var attempts []storagepull.Attempt if err := runtime.db.NewSelect().Model(&attempts).Where("content_id = ?", pipeline.target.ContentID).Scan(t.Context()); err != nil { t.Fatalf("load pull attempts: %v", err) @@ -2687,6 +2875,163 @@ func TestReplacementCoordinatorRetiresAfterCancelledItemsAreProcessed(t *testing } } +// A migration task that stopped but can still be retried keeps the replacement +// waiting for that retry; only a task that cannot be retried fails it. +func TestReplacementWaitsForARetryableFailedMigrationTask(t *testing.T) { + tests := []struct { + name string + reason string + checkpoint string + wantFailed bool + }{ + {name: "retryable", reason: "store_outcome_unknown", checkpoint: `{"attempted_at":"2026-09-11T00:00:00Z"}`}, + {name: "not retryable", reason: "store_result_invalid", wantFailed: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + if err := handlers.RegisterStorage(registry); err != nil { + return err + } + return handlers.RegisterReplacement(registry) + }, + }) + ctx := t.Context() + sequence := storedObjectSequence.Add(1) + id := func(offset int64) idtypes.OnChainID { return testOnChainID(t, 36000+sequence*10+offset) } + bucket := &model.Bucket{ + Name: fmt.Sprintf("replacement-copy-retry-%d", sequence), Status: model.BucketStatusActive, + DefaultCopies: 1, MinimumDurableCopies: 1, + } + if err := runtime.repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("create bucket: %v", err) + } + content, err := runtime.repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ + BucketID: bucket.ID, ContentSize: 11, Checksum: testutil.StorageChecksum(bucket.Name), RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("ensure content: %v", err) + } + version := &model.ObjectVersion{ + VersionID: model.NewVersionID(), BucketID: bucket.ID, Key: "migrating.bin", Size: content.ContentSize, + ETag: bucket.Name, ContentType: "application/octet-stream", ContentID: &content.ID, + } + if _, err := runtime.repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + t.Fatalf("create object version: %v", err) + } + source, err := runtime.repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: id(1), CopyIndex: 0, CreatedByContentID: content.ID, + }) + if err != nil { + t.Fatalf("create source: %v", err) + } + sourceClientID := id(3) + if err := runtime.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: source.ID, ContentID: content.ID, DataSetID: id(2), ClientDataSetID: &sourceClientID, + }); err != nil { + t.Fatalf("mark source ready: %v", err) + } + if err := runtime.repos.Contents.CreateUploadCopiesForBindings(ctx, content.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: source.ID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: id(1), + }}); err != nil { + t.Fatalf("create source copy: %v", err) + } + pieceID := id(4) + testutil.CommitStorageCopy(t, runtime.db, runtime.repos, repository.MarkUploadCopyCommittedInput{ + ContentID: content.ID, CopyIndex: 0, PieceCID: testPieceCID(t, bucket.Name).String(), + PieceID: &pieceID, RetrievalURL: "https://source.example/piece", + }) + replacement, _, err := runtime.repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: id(5), ClientRequestID: bucket.Name, + }) + if err != nil { + t.Fatalf("authorize replacement: %v", err) + } + targetClientID := id(7) + if err := runtime.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: replacement.TargetDataSetID, ContentID: content.ID, DataSetID: id(6), ClientDataSetID: &targetClientID, + }); err != nil { + t.Fatalf("mark target ready: %v", err) + } + if err := runtime.repos.Replacements.Activate(ctx, replacement.ID); err != nil { + t.Fatalf("activate replacement: %v", err) + } + if err := runtime.repos.Contents.CreateUploadCopiesForBindings(ctx, content.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: replacement.TargetDataSetID, CopyIndex: 0, TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: id(5), + }}); err != nil { + t.Fatalf("create target copy: %v", err) + } + targetCopy, err := runtime.repos.Contents.GetUploadCopyForDataSet(ctx, content.ID, replacement.TargetDataSetID) + if err != nil || targetCopy == nil { + t.Fatalf("load target copy = %#v, err=%v", targetCopy, err) + } + item := &storagereplacement.Item{ + ReplacementID: replacement.ID, ContentID: content.ID, TargetDataSetID: replacement.TargetDataSetID, + Status: storagereplacement.ItemStatusPending, + } + if _, err := runtime.db.NewInsert().Model(item).Exec(ctx); err != nil { + t.Fatalf("insert replacement item: %v", err) + } + if _, err := runtime.db.NewUpdate().Model((*storagereplacement.Replacement)(nil)). + Set("seeding_complete = ?", true).Set("items_total = ?", 1). + Where("id = ?", replacement.ID).Exec(ctx); err != nil { + t.Fatalf("complete replacement seeding: %v", err) + } + // The migration task for the target copy has stopped. + storeTask := bindCopyTask(t, runtime, targetCopy, model.TaskTypeStorageStore) + claimed, err := runtime.repos.Tasks.ClaimNext(ctx, time.Minute) + if err != nil || claimed == nil || claimed.ID != storeTask.ID { + t.Fatalf("claim migration task = %#v, err=%v", claimed, err) + } + if tt.checkpoint != "" { + if err := runtime.repos.Tasks.WriteCheckpoint(ctx, claimed.ID, claimed.ClaimGeneration, json.RawMessage(tt.checkpoint)); err != nil { + t.Fatalf("write migration checkpoint: %v", err) + } + } + reason, message := tt.reason, "migration stopped" + if err := runtime.repos.Tasks.Settle(ctx, claimed.ID, claimed.ClaimGeneration, repository.TaskTransition{ + Status: model.TaskStatusFailed, ResumeMode: model.TaskResumeModeRecover, FailureReason: &reason, LastError: &message, + }); err != nil { + t.Fatalf("fail migration task: %v", err) + } + coordinateTask, _, err := runtime.service.EnqueueTx(ctx, taskengine.EnqueueRequest{ + Type: model.TaskTypeProviderReplacementCoordinate, + IdempotencyKey: storagereplacement.CoordinateTaskKey(replacement.ID, replacement.TaskGeneration), + Input: storagereplacement.CoordinateInput{ReplacementID: replacement.ID, Generation: replacement.TaskGeneration}, + SubjectType: "storage_replacement", SubjectKey: fmt.Sprint(replacement.ID), + }, func(ctx context.Context, repos *repository.Repositories, taskRow *model.Task, _ bool) error { + return repos.Replacements.BindTask(ctx, replacement.ID, replacement.TaskGeneration, taskRow.ID) + }) + if err != nil { + t.Fatalf("enqueue replacement coordinator: %v", err) + } + + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + if tt.wantFailed { + failed := waitForTask(t, runtime.repos, coordinateTask.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + stored, err := runtime.repos.Replacements.GetByID(ctx, replacement.ID) + if failed.FailureReason == nil || *failed.FailureReason != "replacement_copy_failed" || err != nil || stored.Status != storagereplacement.StatusFailed { + t.Fatalf("coordinator = %#v, replacement = %#v err=%v, want the replacement failed", failed, stored, err) + } + return + } + waitForTask(t, runtime.repos, coordinateTask.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusPending && task.StatusMessage != nil && + *task.StatusMessage == "Waiting for a failed migration task to be retried" + }) + stored, err := runtime.repos.Replacements.GetByID(ctx, replacement.ID) + if err != nil || stored.Status == storagereplacement.StatusFailed { + t.Fatalf("replacement = %#v err=%v, want it waiting for the migration retry", stored, err) + } + }) + } +} + func TestRetirementAdmissionFailureDoesNotEnterCleanupAttention(t *testing.T) { noRetries := 0 terminator := &testServiceTerminator{err: errors.New("unexpected termination")} @@ -2786,6 +3131,339 @@ func TestRetirementAdmissionFailureDoesNotEnterCleanupAttention(t *testing.T) { } } +type sequencedTerminator struct { + calls atomic.Int64 + respond func(call int64) (*synapse.TerminationResult, error) +} + +func (s *sequencedTerminator) TerminateService(context.Context, sdktypes.BigInt) (*synapse.TerminationResult, error) { + return s.respond(s.calls.Add(1)) +} + +func (s *sequencedTerminator) ContextIdentity() storage.ContextIdentity { + return testutil.DefaultContextIdentity +} + +func (s *sequencedTerminator) VerifyServicePayer(context.Context, sdktypes.BigInt) error { + return nil +} + +// seedAbandonedTargetRetirement leaves the ready target of a superseded +// replacement with a bound retirement task. +func seedAbandonedTargetRetirement(t *testing.T, runtime handlerTestRuntime, sequence int64) (*storagereplacement.Replacement, *model.Task) { + t.Helper() + ctx := t.Context() + bucket := &model.Bucket{Name: fmt.Sprintf("abandoned-target-%d", sequence), Status: model.BucketStatusActive, DefaultCopies: 2, MinimumDurableCopies: 2} + if err := runtime.repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("create bucket: %v", err) + } + id := func(offset int64) idtypes.OnChainID { return testOnChainID(t, 35000+sequence*10+offset) } + source, err := runtime.repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: id(1), CopyIndex: 0, + }) + if err != nil { + t.Fatalf("create source: %v", err) + } + sourceClientID := id(2) + if err := runtime.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: source.ID, DataSetID: id(3), ClientDataSetID: &sourceClientID, + }); err != nil { + t.Fatalf("mark source ready: %v", err) + } + first, _, err := runtime.repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: id(4), ClientRequestID: fmt.Sprintf("abandoned-first-%d", sequence), + }) + if err != nil { + t.Fatalf("authorize replacement: %v", err) + } + targetClientID := id(5) + if err := runtime.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: first.TargetDataSetID, DataSetID: id(6), ClientDataSetID: &targetClientID, + }); err != nil { + t.Fatalf("mark target ready: %v", err) + } + if _, _, err := runtime.repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: id(7), ClientRequestID: fmt.Sprintf("abandoned-successor-%d", sequence), + }); err != nil { + t.Fatalf("authorize successor: %v", err) + } + generation, err := runtime.repos.Contents.NextDataSetRetirementGeneration(ctx, first.TargetDataSetID) + if err != nil { + t.Fatalf("next retirement generation: %v", err) + } + taskRow, _, err := runtime.service.Enqueue(ctx, taskengine.EnqueueRequest{ + Type: model.TaskTypeStorageDataSetRetire, IdempotencyKey: storagereplacement.RetireTaskKey(first.TargetDataSetID, generation), + Input: storagereplacement.RetireInput{ReplacementID: first.ID, DataSetID: first.TargetDataSetID, Generation: generation}, + SubjectType: "storage_data_set", SubjectKey: fmt.Sprint(first.TargetDataSetID), + }) + if err != nil { + t.Fatalf("enqueue retirement: %v", err) + } + if err := runtime.repos.Contents.BindDataSetRetirementTask(ctx, first.TargetDataSetID, generation, taskRow.ID); err != nil { + t.Fatalf("bind retirement: %v", err) + } + return first, taskRow +} + +// A termination whose outcome was not observed is not given up on. The next +// request goes out only after the adapter has read the chain, so it cannot end +// the service twice. +func TestRetirementWithUnobservedTerminationTriesAgain(t *testing.T) { + terminator := &sequencedTerminator{respond: func(call int64) (*synapse.TerminationResult, error) { + if call == 1 { + return nil, errors.New("provider relay failed after submitting") + } + return &synapse.TerminationResult{TxHash: "0xretired", EndEpoch: 84}, nil + }} + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + terminator: terminator, epochs: testEpochReader{epoch: 90}, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterReplacement(registry) + }, + }) + replacement, taskRow := seedAbandonedTargetRetirement(t, runtime, storedObjectSequence.Add(1)) + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return terminator.calls.Load() == 1 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeExecute + }) + wakeTask(t, runtime, taskRow.ID) + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return terminator.calls.Load() == 2 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover + }) + wakeTask(t, runtime, taskRow.ID) + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusCompleted + }) + stored, err := runtime.repos.Replacements.GetByID(t.Context(), replacement.ID) + if err != nil || stored.AbandonedTerminationEpoch == nil || *stored.AbandonedTerminationEpoch != 84 || terminator.calls.Load() != 2 { + t.Fatalf("replacement = %#v err=%v after %d requests, want the returned end epoch recorded", stored, err, terminator.calls.Load()) + } +} + +// Payment debt is the operator's to settle, so it stops the task instead of +// being retried, and a retry stays available for after it is settled. +func TestRetirementBlockedByPaymentDebtWaitsForTheOperator(t *testing.T) { + terminator := &sequencedTerminator{respond: func(int64) (*synapse.TerminationResult, error) { + return nil, &synapse.TerminationBlockedError{Reason: "payment_debt", Shortfall: big.NewInt(42)} + }} + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + terminator: terminator, epochs: testEpochReader{epoch: 90}, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterReplacement(registry) + }, + }) + _, taskRow := seedAbandonedTargetRetirement(t, runtime, storedObjectSequence.Add(1)) + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + failed := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + if failed.FailureReason == nil || *failed.FailureReason != "termination_blocked" || !runtime.service.Retryable(failed) || terminator.calls.Load() != 1 { + t.Fatalf("blocked retirement = %#v after %d requests, want a retryable termination_blocked failure", failed, terminator.calls.Load()) + } +} + +// seedSourceRetirement leaves a replaced source with a bound retirement task, +// its slot already handed to the target and the safety gate clear. +func seedSourceRetirement(t *testing.T, runtime handlerTestRuntime, sequence int64) (*storagereplacement.Replacement, *model.Task) { + t.Helper() + ctx := t.Context() + bucket := &model.Bucket{Name: fmt.Sprintf("replaced-source-%d", sequence), Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} + if err := runtime.repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("create bucket: %v", err) + } + id := func(offset int64) idtypes.OnChainID { return testOnChainID(t, 36000+sequence*10+offset) } + source, err := runtime.repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: id(1), CopyIndex: 0, + }) + if err != nil { + t.Fatalf("create source: %v", err) + } + sourceClientID := id(2) + if err := runtime.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: source.ID, DataSetID: id(3), ClientDataSetID: &sourceClientID, + }); err != nil { + t.Fatalf("mark source ready: %v", err) + } + replacement, _, err := runtime.repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: id(4), ClientRequestID: fmt.Sprintf("replaced-source-%d", sequence), + }) + if err != nil { + t.Fatalf("authorize replacement: %v", err) + } + targetClientID := id(5) + if err := runtime.repos.Contents.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: replacement.TargetDataSetID, DataSetID: id(6), ClientDataSetID: &targetClientID, + }); err != nil { + t.Fatalf("mark target ready: %v", err) + } + if err := runtime.repos.Replacements.Activate(ctx, replacement.ID); err != nil { + t.Fatalf("activate replacement: %v", err) + } + if err := runtime.repos.Replacements.BeginRetirement(ctx, replacement.ID); err != nil { + t.Fatalf("begin retirement: %v", err) + } + generation, err := runtime.repos.Contents.NextDataSetRetirementGeneration(ctx, source.ID) + if err != nil { + t.Fatalf("next retirement generation: %v", err) + } + taskRow, _, err := runtime.service.Enqueue(ctx, taskengine.EnqueueRequest{ + Type: model.TaskTypeStorageDataSetRetire, IdempotencyKey: storagereplacement.RetireTaskKey(source.ID, generation), + Input: storagereplacement.RetireInput{ReplacementID: replacement.ID, DataSetID: source.ID, Generation: generation}, + SubjectType: "storage_data_set", SubjectKey: fmt.Sprint(source.ID), + }) + if err != nil { + t.Fatalf("enqueue retirement: %v", err) + } + if err := runtime.repos.Contents.BindDataSetRetirementTask(ctx, source.ID, generation, taskRow.ID); err != nil { + t.Fatalf("bind retirement: %v", err) + } + return replacement, taskRow +} + +// A data set ID is a number on one chain, paid for by one wallet. If either +// moved after a termination request went out, the recorded end epoch read here +// would describe someone else's service, so the task reads nothing, sends +// nothing and stops for an operator. +func TestRetirementStopsWhenTheSigningIdentityChanged(t *testing.T) { + movedIdentity := testutil.DefaultContextIdentity + movedIdentity.ChainID = testutil.DefaultContextIdentity.ChainID + 1 + tests := []struct { + name string + seed func(*testing.T, handlerTestRuntime, int64) (*storagereplacement.Replacement, *model.Task) + wantStatus storagereplacement.Status + }{ + // The replacement is still live, so it must show up as needing attention. + {name: "replaced source", seed: seedSourceRetirement, wantStatus: storagereplacement.StatusCleanupAttention}, + // A superseded replacement is already finished; only the task stops. + {name: "abandoned target", seed: seedAbandonedTargetRetirement, wantStatus: storagereplacement.StatusSuperseded}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + terminator := &testServiceTerminator{identity: movedIdentity, result: &synapse.TerminationResult{TxHash: "0xretire", EndEpoch: 84}} + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + terminator: terminator, epochs: testEpochReader{epoch: 90}, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterReplacement(registry) + }, + }) + ctx := t.Context() + replacement, taskRow := tt.seed(t, runtime, storedObjectSequence.Add(1)) + // One request went out under the original identity, and its outcome + // was never observed. + checkpoint, err := json.Marshal(map[string]any{ + "attempted_at": time.Now().UTC().Add(-time.Hour), + "identity": testutil.DefaultContextIdentity, "sends": 1, + }) + if err != nil { + t.Fatalf("encode checkpoint: %v", err) + } + if _, err := runtime.db.NewRaw(`UPDATE task_payloads SET checkpoint_json = ? WHERE task_id = ?`, string(checkpoint), taskRow.ID).Exec(ctx); err != nil { + t.Fatalf("write checkpoint: %v", err) + } + + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + failed := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + if failed.FailureReason == nil || *failed.FailureReason != "termination_identity_changed" || terminator.calls.Load() != 0 { + t.Fatalf("failure = %v after %d termination requests, want termination_identity_changed and no request", + failed.FailureReason, terminator.calls.Load()) + } + stored, err := runtime.repos.Replacements.GetByID(ctx, replacement.ID) + if err != nil || stored == nil || stored.Status != tt.wantStatus { + t.Fatalf("replacement = %#v err=%v, want status %s", stored, err, tt.wantStatus) + } + }) + } +} + +// A data set another wallet pays for is never terminated. Ownership is read +// before anything is recorded, so once the original wallet is back a retry goes +// ahead instead of stopping on the identity of a request that was never sent. +func TestRetirementStopsOnADataSetAnotherWalletPaysFor(t *testing.T) { + tests := []struct { + name string + seed func(*testing.T, handlerTestRuntime, int64) (*storagereplacement.Replacement, *model.Task) + wantStatus storagereplacement.Status + }{ + {name: "replaced source", seed: seedSourceRetirement, wantStatus: storagereplacement.StatusCleanupAttention}, + {name: "abandoned target", seed: seedAbandonedTargetRetirement, wantStatus: storagereplacement.StatusSuperseded}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + terminator := &testServiceTerminator{result: &synapse.TerminationResult{TxHash: "0xretire", EndEpoch: 84}} + terminator.notPaidFor.Store(true) + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + terminator: terminator, epochs: testEpochReader{epoch: 90}, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterReplacement(registry) + }, + }) + ctx := t.Context() + replacement, taskRow := tt.seed(t, runtime, storedObjectSequence.Add(1)) + + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + failed := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + if failed.FailureReason == nil || *failed.FailureReason != "termination_payer_mismatch" || + terminator.calls.Load() != 0 || len(failed.Checkpoint) != 0 { + t.Fatalf("failure = %v after %d termination requests with checkpoint %s, want termination_payer_mismatch with nothing sent or recorded", + failed.FailureReason, terminator.calls.Load(), failed.Checkpoint) + } + stored, err := runtime.repos.Replacements.GetByID(ctx, replacement.ID) + if err != nil || stored == nil || stored.Status != tt.wantStatus { + t.Fatalf("replacement = %#v err=%v, want status %s", stored, err, tt.wantStatus) + } + + // Retrying before the wallet is fixed stops the same way, instead of + // leaving the task running against a replacement that already waits + // for an operator. + if err := runtime.service.Retry(ctx, taskRow.ID); err != nil { + t.Fatalf("retry retirement: %v", err) + } + refused := waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + if refused.FailureReason == nil || *refused.FailureReason != "termination_payer_mismatch" || terminator.calls.Load() != 0 { + t.Fatalf("second failure = %v after %d termination requests, want termination_payer_mismatch and none", + refused.FailureReason, terminator.calls.Load()) + } + + // The original wallet is back, and the retirement runs to completion. + terminator.notPaidFor.Store(false) + if err := runtime.service.Retry(ctx, taskRow.ID); err != nil { + t.Fatalf("retry retirement: %v", err) + } + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return terminator.calls.Load() == 1 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover + }) + wakeTask(t, runtime, taskRow.ID) + waitForTask(t, runtime.repos, taskRow.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusCompleted + }) + wantFinished := storagereplacement.StatusCompleted + if tt.wantStatus == storagereplacement.StatusSuperseded { + wantFinished = storagereplacement.StatusSuperseded + } + finished, err := runtime.repos.Replacements.GetByID(ctx, replacement.ID) + if err != nil || finished == nil || finished.Status != wantFinished || terminator.calls.Load() != 1 { + t.Fatalf("replacement = %#v err=%v after %d termination requests, want status %s after one", + finished, err, terminator.calls.Load(), wantFinished) + } + }) + } +} + func TestRetirementRecoveryPersistsReturnedEpochWithoutRepeatingTermination(t *testing.T) { terminator := &testServiceTerminator{result: &synapse.TerminationResult{TxHash: "0xretire", EndEpoch: 84}} runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ @@ -3115,7 +3793,8 @@ func TestDataSetEnsureReleasesTheProviderWhenCreationIsRejected(t *testing.T) { if err != nil { t.Fatalf("EnsureDataSetBinding: %v", err) } - // The submission was recorded, so recovery resumes by waiting on it. + // The only request sent for this ID was submitted, and the task resumes by + // waiting on that submission. clientDataSetID := testOnChainID(t, 27000+sequence) if err := runtime.repos.Contents.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ ID: binding.ID, TransactionID: "0xrejected", StatusURL: "https://provider.example/status", @@ -3133,16 +3812,30 @@ func TestDataSetEnsureReleasesTheProviderWhenCreationIsRejected(t *testing.T) { if err != nil { t.Fatalf("enqueue data set ensure: %v", err) } + checkpoint, err := json.Marshal(map[string]any{ + "attempted_at": time.Now().UTC(), "client_data_set_id": clientDataSetID.String(), + "identity": testutil.DefaultContextIdentity, "sends": 1, + "transaction_id": "0xrejected", "status_url": "https://provider.example/status", + }) + if err != nil { + t.Fatalf("encode checkpoint: %v", err) + } + if _, err := runtime.db.NewRaw(`UPDATE task_payloads SET checkpoint_json = ? WHERE task_id = ?`, string(checkpoint), ensureTask.ID).Exec(ctx); err != nil { + t.Fatalf("write checkpoint: %v", err) + } cancel, done := runHandlerEngine(t, runtime) defer stopHandlerEngine(t, cancel, done) - waitForTask(t, runtime.repos, ensureTask.ID, func(task *model.Task) bool { + failed := waitForTask(t, runtime.repos, ensureTask.ID, func(task *model.Task) bool { return task.Status == model.TaskStatusFailed }) + if runtime.service.Retryable(failed) { + t.Fatal("a rejected creation offers a retry that could only fail again") + } ended, err := runtime.repos.Contents.GetDataSetBindingByID(ctx, binding.ID) - if err != nil || ended == nil || ended.Status != model.StorageDataSetStatusRetired { - t.Fatalf("rejected generation = %#v err=%v, want it retired rather than deleted", ended, err) + if err != nil || ended == nil || ended.Status != model.StorageDataSetStatusRetired || ended.EnsureTaskID != nil { + t.Fatalf("rejected generation = %#v err=%v, want it retired rather than deleted, with its creation fence released", ended, err) } // The provider is what retirement buys back. reused, err := runtime.repos.Contents.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ @@ -3153,43 +3846,29 @@ func TestDataSetEnsureReleasesTheProviderWhenCreationIsRejected(t *testing.T) { } } -// An unknown creation outcome must not terminate the copies bound to the -// generation. Retrying the ensure task rediscovers a data set the provider did -// create but never reported, and continuation only picks up copies that are -// still in transfer states — so failing them here would destroy the recovery -// that keeping the row is for. -func TestDataSetEnsureKeepsCopiesWhenTheCreationOutcomeIsUnknown(t *testing.T) { - sequence := storedObjectSequence.Add(1) - providerID := testOnChainID(t, 28000+sequence) - storageClient := &testutil.MockStorageClient{ - OpenProviderTargetFunc: func(context.Context, sdktypes.BigInt, storage.NewProviderContextOptions) (synapse.ProviderTarget, error) { - return &testutil.MockStorageTarget{ProviderIDValue: providerID.SDK()}, nil - }, - } - runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ - storage: storageClient, policy: cache.EvictionPolicyNone, - register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { - return handlers.RegisterStorage(registry) - }, - }) +type dataSetEnsureFixture struct { + binding *model.StorageDataSet + ensureTask *model.Task +} + +// seedDataSetEnsure prepares a pending generation with one bound copy and its +// ensure task, the state a first upload to a new provider leaves behind. +func seedDataSetEnsure(t *testing.T, runtime handlerTestRuntime, providerID idtypes.OnChainID, name string) dataSetEnsureFixture { + t.Helper() ctx := t.Context() - bucket := &model.Bucket{ - Name: fmt.Sprintf("unknown-%d", sequence), Status: model.BucketStatusActive, - DefaultCopies: 1, MinimumDurableCopies: 1, - } + bucket := &model.Bucket{Name: name, Status: model.BucketStatusActive, DefaultCopies: 1, MinimumDurableCopies: 1} if err := runtime.repos.Buckets.Create(ctx, bucket); err != nil { t.Fatalf("create bucket: %v", err) } content, err := runtime.repos.Contents.EnsureContent(ctx, repository.EnsureContentInput{ - BucketID: bucket.ID, ContentSize: 11, - Checksum: testutil.StorageChecksum(fmt.Sprintf("unknown-checksum-%d", sequence)), RequestedCopies: 1, + BucketID: bucket.ID, ContentSize: 11, Checksum: testutil.StorageChecksum(name), RequestedCopies: 1, }) if err != nil { t.Fatalf("ensure content: %v", err) } version := &model.ObjectVersion{ - VersionID: model.NewVersionID(), BucketID: bucket.ID, Key: "unknown.bin", ContentID: &content.ID, - Size: 11, ETag: fmt.Sprintf("unknown-etag-%d", sequence), ContentType: "application/octet-stream", + VersionID: model.NewVersionID(), BucketID: bucket.ID, Key: name + ".bin", ContentID: &content.ID, + Size: 11, ETag: name, ContentType: "application/octet-stream", } if _, err := runtime.repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { t.Fatalf("create object version: %v", err) @@ -3216,33 +3895,457 @@ func TestDataSetEnsureKeepsCopiesWhenTheCreationOutcomeIsUnknown(t *testing.T) { if err != nil { t.Fatalf("enqueue data set ensure: %v", err) } - // A creation was attempted long enough ago that the outcome is given up on, - // and no transaction was ever learned. - attempted := time.Now().UTC().Add(-30 * time.Minute).Format(time.RFC3339Nano) - if _, err := runtime.db.NewUpdate(). - Model((*model.TaskPayload)(nil)). - Set("checkpoint_json = ?", fmt.Sprintf(`{"attempted_at":%q}`, attempted)). - Where("task_id = ?", ensureTask.ID). - Exec(ctx); err != nil { - t.Fatalf("write attempted checkpoint: %v", err) + return dataSetEnsureFixture{binding: binding, ensureTask: ensureTask} +} + +func wakeTask(t *testing.T, runtime handlerTestRuntime, taskID int64) { + t.Helper() + if _, err := runtime.db.NewRaw(`UPDATE tasks SET available_at = ? WHERE id = ?`, time.Now().Add(-time.Second), taskID).Exec(t.Context()); err != nil { + t.Fatalf("wake task %d: %v", taskID, err) + } +} + +// A create request whose outcome was lost is never given up on: the chain is +// checked for its client data set ID, the same ID is sent again while nothing +// is visible, and a later rejection is not taken as proof that the first +// request failed. +func TestDataSetCreationWithUnobservedOutcomeResendsTheSameID(t *testing.T) { + sequence := storedObjectSequence.Add(1) + providerID := testOnChainID(t, 31000+sequence) + createdID := sdktypes.NewBigInt(uint64(32000 + sequence)) + var ( + mu sync.Mutex + sentIDs []string + landed atomic.Bool + ) + sends := func() int { + mu.Lock() + defer mu.Unlock() + return len(sentIDs) + } + target := &testutil.MockStorageTarget{ + ProviderIDValue: providerID.SDK(), + CreateDataSetFunc: func(ctx context.Context, opts *storage.CreateDataSetOptions) (*storage.CreateDataSetResult, error) { + mu.Lock() + sentIDs = append(sentIDs, opts.ClientDataSetID.String()) + first := len(sentIDs) == 1 + mu.Unlock() + if first { + return nil, errors.New("connection reset by provider") + } + opts.OnSubmitted(storage.CreateDataSetSubmission{ + ProviderID: providerID.SDK(), TransactionID: "0x" + strings.Repeat("ab", 32), + StatusURL: "https://provider.example/pdp/data-sets/created/2", ClientDataSetID: opts.ClientDataSetID, + }) + <-ctx.Done() + return nil, ctx.Err() + }, + // The resent request loses to the first one, which did land after all. + WaitDataSetFunc: func(context.Context, storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) { + landed.Store(true) + return nil, pdp.ErrTxRejected + }, + FindDataSetByClientIDFunc: func(_ context.Context, clientID sdktypes.BigInt) (storage.DataSetRef, bool, error) { + if !landed.Load() { + return storage.DataSetRef{}, false, nil + } + ref, err := storage.NewDataSetRef(providerID.SDK(), createdID, clientID) + return ref, err == nil, err + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: &testutil.MockStorageClient{ + OpenProviderTargetFunc: func(context.Context, sdktypes.BigInt, storage.NewProviderContextOptions) (synapse.ProviderTarget, error) { + return target, nil + }, + }, + policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + fixture := seedDataSetEnsure(t, runtime, providerID, fmt.Sprintf("unobserved-%d", sequence)) + ctx := t.Context() + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + + waitForTask(t, runtime.repos, fixture.ensureTask.ID, func(task *model.Task) bool { + return sends() == 1 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover + }) + recorded, err := runtime.repos.Contents.GetDataSetBindingByID(ctx, fixture.binding.ID) + if err != nil || recorded.ClientDataSetID == nil || recorded.ClientDataSetID.String() != sentIDs[0] { + t.Fatalf("generation after the first request = %#v err=%v, want its client data set ID recorded", recorded, err) + } + + // Nothing is visible once the request has had time to land. + stored, err := runtime.repos.Tasks.GetByID(ctx, fixture.ensureTask.ID) + if err != nil { + t.Fatalf("load ensure task: %v", err) + } + var checkpoint map[string]any + if err := json.Unmarshal(stored.Checkpoint, &checkpoint); err != nil { + t.Fatalf("decode checkpoint: %v", err) + } + checkpoint["attempted_at"] = time.Now().UTC().Add(-time.Hour) + aged, err := json.Marshal(checkpoint) + if err != nil { + t.Fatalf("encode checkpoint: %v", err) + } + if _, err := runtime.db.NewRaw(`UPDATE task_payloads SET checkpoint_json = ? WHERE task_id = ?`, string(aged), fixture.ensureTask.ID).Exec(ctx); err != nil { + t.Fatalf("age checkpoint: %v", err) + } + wakeTask(t, runtime, fixture.ensureTask.ID) + waitForTask(t, runtime.repos, fixture.ensureTask.ID, func(task *model.Task) bool { + return sends() == 2 && task.Status == model.TaskStatusPending && task.ResumeMode == model.TaskResumeModeRecover + }) + + wakeTask(t, runtime, fixture.ensureTask.ID) + waitForTask(t, runtime.repos, fixture.ensureTask.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusCompleted + }) + if sends() != 2 || sentIDs[0] != sentIDs[1] { + t.Fatalf("create requests = %v, want the same client data set ID sent twice", sentIDs) + } + ready, err := runtime.repos.Contents.GetDataSetBindingByID(ctx, fixture.binding.ID) + if err != nil || ready.Status != model.StorageDataSetStatusReady || ready.DataSetID == nil || + ready.DataSetID.String() != createdID.String() || ready.ClientDataSetID.String() != sentIDs[0] { + t.Fatalf("generation = %#v err=%v, want the data set the first request created", ready, err) + } +} + +// A request is only looked up, waited on, or resent under the identity it was +// signed for, and an ID the chain resolves to some other data set is never used +// again. Both stop the task; only the conflict, which the chain settled, gives +// the generation up. +func TestDataSetCreationRecoveryStopsOnChangedIdentityOrConflict(t *testing.T) { + otherIdentity := testutil.DefaultContextIdentity + otherIdentity.Payer = common.HexToAddress("0x00000000000000000000000000000000000000c3") + tests := []struct { + name string + identity storage.ContextIdentity + submission bool + lookup func(context.Context, sdktypes.BigInt) (storage.DataSetRef, bool, error) + reason string + status model.StorageDataSetStatus + fenceHeld bool + retryable bool + incompleteCopies int + }{ + { + // Only this context cannot look the ID up. The data set the original + // identity asked for may exist and be billing, so the generation, its + // copies and its fence are all kept for an operator who restores the + // configuration and retries. + name: "changed identity", identity: otherIdentity, reason: "dataset_identity_changed", + status: model.StorageDataSetStatusPending, fenceHeld: true, retryable: true, incompleteCopies: 1, + }, + { + // A recorded submission is no exception. Its status would report the + // data set the original wallet's request created, which the wallet + // signing now does not pay for. + name: "changed identity after the submission was recorded", identity: otherIdentity, submission: true, + reason: "dataset_identity_changed", status: model.StorageDataSetStatusCreating, + fenceHeld: true, retryable: true, incompleteCopies: 1, + }, + { + // The chain resolved the ID to a record that is not ours, and said so + // on two reads. Nothing will ever serve these copies. + name: "correlation conflict", reason: "dataset_correlation_conflict", + status: model.StorageDataSetStatusRetired, fenceHeld: false, retryable: false, incompleteCopies: 0, + lookup: func(context.Context, sdktypes.BigInt) (storage.DataSetRef, bool, error) { + return storage.DataSetRef{}, false, fmt.Errorf("lookup: %w", storage.ErrDataSetCorrelationConflict) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sequence := storedObjectSequence.Add(1) + providerID := testOnChainID(t, 33000+sequence) + clientID := testOnChainID(t, 34000+sequence) + createdID := testOnChainID(t, 38000+sequence) + var sends, waits atomic.Int64 + target := &testutil.MockStorageTarget{ + ProviderIDValue: providerID.SDK(), ContextIdentityValue: tt.identity, FindDataSetByClientIDFunc: tt.lookup, + CreateDataSetFunc: func(context.Context, *storage.CreateDataSetOptions) (*storage.CreateDataSetResult, error) { + sends.Add(1) + return nil, errors.New("unexpected create request") + }, + WaitDataSetFunc: func(context.Context, storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) { + waits.Add(1) + // What the provider would report: the data set the original + // wallet's request created. + ref, err := storage.NewDataSetRef(providerID.SDK(), createdID.SDK(), clientID.SDK()) + if err != nil { + return nil, err + } + return &storage.CreateDataSetResult{DataSet: ref}, nil + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: &testutil.MockStorageClient{ + OpenProviderTargetFunc: func(context.Context, sdktypes.BigInt, storage.NewProviderContextOptions) (synapse.ProviderTarget, error) { + return target, nil + }, + }, + policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + fixture := seedDataSetEnsure(t, runtime, providerID, fmt.Sprintf("recovery-stop-%d", sequence)) + ctx := t.Context() + // One request was sent long ago under the original identity. + if err := runtime.repos.Contents.RecordDataSetClientID(ctx, fixture.binding.ID, clientID); err != nil { + t.Fatalf("record client data set ID: %v", err) + } + recorded := map[string]any{ + "attempted_at": time.Now().UTC().Add(-time.Hour), "client_data_set_id": clientID.String(), + "identity": testutil.DefaultContextIdentity, "sends": 1, + } + if tt.submission { + const transactionID, statusURL = "0xsubmitted", "https://provider.example/status" + if err := runtime.repos.Contents.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ + ID: fixture.binding.ID, TransactionID: transactionID, StatusURL: statusURL, ClientDataSetID: &clientID, + }); err != nil { + t.Fatalf("mark creating: %v", err) + } + recorded["transaction_id"], recorded["status_url"] = transactionID, statusURL + } + checkpoint, err := json.Marshal(recorded) + if err != nil { + t.Fatalf("encode checkpoint: %v", err) + } + if _, err := runtime.db.NewRaw(`UPDATE task_payloads SET checkpoint_json = ? WHERE task_id = ?`, string(checkpoint), fixture.ensureTask.ID).Exec(ctx); err != nil { + t.Fatalf("write checkpoint: %v", err) + } + if _, err := runtime.db.NewRaw(`UPDATE tasks SET resume_mode = ? WHERE id = ?`, model.TaskResumeModeRecover, fixture.ensureTask.ID).Exec(ctx); err != nil { + t.Fatalf("resume in recovery: %v", err) + } + + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + failed := waitForTask(t, runtime.repos, fixture.ensureTask.ID, func(task *model.Task) bool { + return task.Status == model.TaskStatusFailed + }) + if failed.FailureReason == nil || *failed.FailureReason != tt.reason || sends.Load() != 0 || waits.Load() != 0 { + t.Fatalf("failure = %v after %d create requests and %d waits, want %s with neither", + failed.FailureReason, sends.Load(), waits.Load(), tt.reason) + } + if runtime.service.Retryable(failed) != tt.retryable { + t.Fatalf("retryable = %v, want %v", runtime.service.Retryable(failed), tt.retryable) + } + kept, err := runtime.repos.Contents.GetDataSetBindingByID(ctx, fixture.binding.ID) + if err != nil || kept.Status != tt.status || (kept.EnsureTaskID != nil) != tt.fenceHeld { + t.Fatalf("generation = %#v err=%v, want status %s with fence held=%v", kept, err, tt.status, tt.fenceHeld) + } + incomplete, err := runtime.repos.Contents.ListIncompleteCopiesForDataSet(ctx, fixture.binding.ID) + if err != nil || len(incomplete) != tt.incompleteCopies { + t.Fatalf("incomplete copies = %#v err=%v, want %d", incomplete, err, tt.incompleteCopies) + } + }) + } +} + +// TestDataSetCreationRecoversFromAnUnusableCheckpointThroughTheRow checks that a +// checkpoint that can no longer name its request is resolved through the ID the +// row recorded before the provider call, and that nothing is sent when the chain +// holds nothing under it. +func TestDataSetCreationRecoversFromAnUnusableCheckpointThroughTheRow(t *testing.T) { + for _, tt := range []struct { + name string + found bool + status model.StorageDataSetStatus + fenceHeld bool + }{ + {name: "chain holds the data set", found: true, status: model.StorageDataSetStatusReady, fenceHeld: false}, + {name: "chain holds nothing", found: false, status: model.StorageDataSetStatusPending, fenceHeld: true}, + } { + t.Run(tt.name, func(t *testing.T) { + sequence := storedObjectSequence.Add(1) + providerID := testOnChainID(t, 41000+sequence) + clientID := testOnChainID(t, 42000+sequence) + createdID := testOnChainID(t, 43000+sequence) + var sends, lookups atomic.Int64 + target := &testutil.MockStorageTarget{ + ProviderIDValue: providerID.SDK(), + FindDataSetByClientIDFunc: func(_ context.Context, asked sdktypes.BigInt) (storage.DataSetRef, bool, error) { + lookups.Add(1) + if asked.String() != clientID.SDK().String() { + t.Errorf("looked up %s, want the recorded %s", asked.String(), clientID.String()) + } + if !tt.found { + return storage.DataSetRef{}, false, nil + } + ref, err := storage.NewDataSetRef(providerID.SDK(), createdID.SDK(), asked) + if err != nil { + t.Errorf("new data set ref: %v", err) + return storage.DataSetRef{}, false, err + } + return ref, true, nil + }, + CreateDataSetFunc: func(context.Context, *storage.CreateDataSetOptions) (*storage.CreateDataSetResult, error) { + sends.Add(1) + return nil, errors.New("unexpected create request") + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: &testutil.MockStorageClient{ + OpenProviderTargetFunc: func(context.Context, sdktypes.BigInt, storage.NewProviderContextOptions) (synapse.ProviderTarget, error) { + return target, nil + }, + }, + policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + ctx := t.Context() + fixture := seedDataSetEnsure(t, runtime, providerID, fmt.Sprintf("unnamed-creation-%d", sequence)) + if err := runtime.repos.Contents.RecordDataSetClientID(ctx, fixture.binding.ID, clientID); err != nil { + t.Fatalf("record client data set ID: %v", err) + } + // A checkpoint this build cannot decode: the ID arrives as a number + // where the record expects text. + if _, err := runtime.db.NewRaw(`UPDATE task_payloads SET checkpoint_json = ? WHERE task_id = ?`, + `{"client_data_set_id": 12}`, fixture.ensureTask.ID).Exec(ctx); err != nil { + t.Fatalf("write checkpoint: %v", err) + } + if _, err := runtime.db.NewRaw(`UPDATE tasks SET resume_mode = ? WHERE id = ?`, model.TaskResumeModeRecover, fixture.ensureTask.ID).Exec(ctx); err != nil { + t.Fatalf("resume in recovery: %v", err) + } + + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + settled := waitForTask(t, runtime.repos, fixture.ensureTask.ID, func(task *model.Task) bool { + if tt.found { + return task.Status == model.TaskStatusCompleted + } + return task.Status == model.TaskStatusFailed + }) + if !tt.found && (settled.FailureReason == nil || *settled.FailureReason != "invalid_checkpoint") { + t.Fatalf("failure = %v, want invalid_checkpoint", settled.FailureReason) + } + if sends.Load() != 0 || lookups.Load() == 0 { + t.Fatalf("sends = %d lookups = %d, want the recorded ID looked up and nothing sent", sends.Load(), lookups.Load()) + } + kept, err := runtime.repos.Contents.GetDataSetBindingByID(ctx, fixture.binding.ID) + if err != nil || kept.Status != tt.status || (kept.EnsureTaskID != nil) != tt.fenceHeld { + t.Fatalf("generation = %#v err=%v, want status %s with fence held=%v", kept, err, tt.status, tt.fenceHeld) + } + }) } +} + +// TestDataSetEnsureGivesUpAGenerationThatNeverSent checks that a task that +// cannot run gives up its generation — and its creation fence — only because the +// row proves no request ever went out. +func TestDataSetEnsureGivesUpAGenerationThatNeverSent(t *testing.T) { + sequence := storedObjectSequence.Add(1) + providerID := testOnChainID(t, 44000+sequence) + target := &testutil.MockStorageTarget{ + ProviderIDValue: providerID.SDK(), + // An identity missing its chain and record keeper cannot sign, so the + // task fails before anything is sent. + ContextIdentityValue: storage.ContextIdentity{Payer: common.HexToAddress("0x00000000000000000000000000000000000000d4")}, + CreateDataSetFunc: func(context.Context, *storage.CreateDataSetOptions) (*storage.CreateDataSetResult, error) { + t.Error("a creation was sent without a complete signing identity") + return nil, errors.New("unexpected create request") + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: &testutil.MockStorageClient{ + OpenProviderTargetFunc: func(context.Context, sdktypes.BigInt, storage.NewProviderContextOptions) (synapse.ProviderTarget, error) { + return target, nil + }, + }, + policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + ctx := t.Context() + fixture := seedDataSetEnsure(t, runtime, providerID, fmt.Sprintf("never-sent-%d", sequence)) cancel, done := runHandlerEngine(t, runtime) defer stopHandlerEngine(t, cancel, done) - failed := waitForTask(t, runtime.repos, ensureTask.ID, func(task *model.Task) bool { + failed := waitForTask(t, runtime.repos, fixture.ensureTask.ID, func(task *model.Task) bool { return task.Status == model.TaskStatusFailed }) - if failed.FailureReason == nil || *failed.FailureReason != "dataset_creation_unknown" { - t.Fatalf("failure reason = %v, want dataset_creation_unknown", failed.FailureReason) + if failed.FailureReason == nil || *failed.FailureReason != "dataset_creation_unsent" || runtime.service.Retryable(failed) { + t.Fatalf("failure = %v retryable=%v, want dataset_creation_unsent with no retry offered", failed.FailureReason, runtime.service.Retryable(failed)) + } + kept, err := runtime.repos.Contents.GetDataSetBindingByID(ctx, fixture.binding.ID) + if err != nil || kept.Status != model.StorageDataSetStatusRetired || kept.EnsureTaskID != nil { + t.Fatalf("generation = %#v err=%v, want it retired with its creation fence released", kept, err) + } +} + +// TestDataSetCreationLooksUpARejectedResendInsteadOfGivingUp checks that a +// rejection arriving after an unobserved outcome proves nothing: the request +// before it may have created the data set, so the submission is dropped, the ID +// is looked up, and the generation keeps its fence. +func TestDataSetCreationLooksUpARejectedResendInsteadOfGivingUp(t *testing.T) { + sequence := storedObjectSequence.Add(1) + providerID := testOnChainID(t, 46000+sequence) + clientID := testOnChainID(t, 47000+sequence) + var lookups atomic.Int64 + target := &testutil.MockStorageTarget{ + ProviderIDValue: providerID.SDK(), + WaitDataSetFunc: func(context.Context, storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) { + return nil, fmt.Errorf("submission: %w", synapse.ErrProviderTransactionRejected) + }, + FindDataSetByClientIDFunc: func(context.Context, sdktypes.BigInt) (storage.DataSetRef, bool, error) { + lookups.Add(1) + return storage.DataSetRef{}, false, nil + }, + } + runtime := newHandlerTestRuntime(t, handlerRuntimeOptions{ + storage: &testutil.MockStorageClient{ + OpenProviderTargetFunc: func(context.Context, sdktypes.BigInt, storage.NewProviderContextOptions) (synapse.ProviderTarget, error) { + return target, nil + }, + }, + policy: cache.EvictionPolicyNone, + register: func(handlers *worker.TaskHandlers, registry *taskengine.Registry) error { + return handlers.RegisterStorage(registry) + }, + }) + ctx := t.Context() + fixture := seedDataSetEnsure(t, runtime, providerID, fmt.Sprintf("dropped-submission-%d", sequence)) + if err := runtime.repos.Contents.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ + ID: fixture.binding.ID, TransactionID: "0xdead", StatusURL: "https://provider.example/status", + ClientDataSetID: &clientID, + }); err != nil { + t.Fatalf("mark creating: %v", err) + } + // An earlier request with this ID had no observed outcome, so the rejection + // only means that one may have created the data set first. + checkpoint, err := json.Marshal(map[string]any{ + "attempted_at": time.Now().UTC().Add(-time.Hour), "client_data_set_id": clientID.String(), + "identity": testutil.DefaultContextIdentity, "sends": 2, + "transaction_id": "0xdead", "status_url": "https://provider.example/status", + }) + if err != nil { + t.Fatalf("encode checkpoint: %v", err) + } + if _, err := runtime.db.NewRaw(`UPDATE task_payloads SET checkpoint_json = ? WHERE task_id = ?`, string(checkpoint), fixture.ensureTask.ID).Exec(ctx); err != nil { + t.Fatalf("write checkpoint: %v", err) + } + if _, err := runtime.db.NewRaw(`UPDATE tasks SET resume_mode = ? WHERE id = ?`, model.TaskResumeModeRecover, fixture.ensureTask.ID).Exec(ctx); err != nil { + t.Fatalf("resume in recovery: %v", err) } - kept, err := runtime.repos.Contents.GetDataSetBindingByID(ctx, binding.ID) - if err != nil || kept == nil || kept.Status != model.StorageDataSetStatusFailed { - t.Fatalf("unknown generation = %#v err=%v, want it kept as failed", kept, err) + cancel, done := runHandlerEngine(t, runtime) + defer stopHandlerEngine(t, cancel, done) + waitForTask(t, runtime.repos, fixture.ensureTask.ID, func(*model.Task) bool { + return lookups.Load() > 0 + }) + kept, err := runtime.repos.Contents.GetDataSetBindingByID(ctx, fixture.binding.ID) + if err != nil || kept.Status != model.StorageDataSetStatusCreating || kept.EnsureTaskID == nil { + t.Fatalf("generation = %#v err=%v, want it kept with its creation fence", kept, err) } - // The copy stays in a transfer state, which is what continuation picks up. - incomplete, err := runtime.repos.Contents.ListIncompleteCopiesForDataSet(ctx, binding.ID) - if err != nil || len(incomplete) != 1 { - t.Fatalf("incomplete copies = %#v err=%v, want the bound copy still recoverable", incomplete, err) + stored, err := runtime.repos.Tasks.GetByID(ctx, fixture.ensureTask.ID) + if err != nil || stored == nil || strings.Contains(string(stored.Checkpoint), `"transaction_id"`) { + t.Fatalf("task = %#v err=%v, want the rejected submission dropped from its checkpoint", stored, err) } } diff --git a/ui/src/api/client.ts b/ui/src/api/client.ts index bf028fc..f6073fb 100644 --- a/ui/src/api/client.ts +++ b/ui/src/api/client.ts @@ -1145,6 +1145,15 @@ export const api = { getTaskStats: () => fetchJSON('/tasks/stats'), retryTask: (id: number) => fetchJSON(`/tasks/${id}/retry`, { method: 'POST' }), acknowledgeTask: (id: number) => fetchJSON(`/tasks/${id}/acknowledge`, { method: 'POST' }), + previewAcknowledgeTasks: (payload: { type?: string }) => { + const qs = payload.type ? `?${new URLSearchParams({ type: payload.type }).toString()}` : '' + return fetchJSON<{ count: number; as_of: string }>(`/tasks/acknowledge/preview${qs}`) + }, + acknowledgeTasks: (payload: { type?: string; failed_before: string }) => + fetchJSON<{ acknowledged: number }>('/tasks/acknowledge', { + method: 'POST', + body: JSON.stringify(payload), + }), getSystemInfo: () => fetchJSON('/system/info'), getWorkers: () => fetchJSON<{ workers: Record }>('/workers'), getCacheStats: () => fetchJSON<{ used_bytes: number; max_bytes: number }>('/cache/stats'), diff --git a/ui/src/lib/provider-replacement.ts b/ui/src/lib/provider-replacement.ts index 183df45..c100ccc 100644 --- a/ui/src/lib/provider-replacement.ts +++ b/ui/src/lib/provider-replacement.ts @@ -132,6 +132,8 @@ export function dataSetGenerationLabel(dataSet: StorageDataSetSummary) { const replacementErrorMessages: Record = { replacement_active: 'This replica is already being replaced. Wait for it to finish or retry it below.', + replacement_target_creating: + 'The earlier replacement of this replica is still setting up its new storage service. Wait for it to finish, or retry that setup from Tasks if it stopped.', replacement_target_in_use: 'That provider already stores a replica of this bucket. Choose a different one.', replacement_target_invalid: 'Choose a provider other than the one being replaced.', replacement_no_eligible_provider: diff --git a/ui/src/routes/tasks.tsx b/ui/src/routes/tasks.tsx index f9b046c..95995d7 100644 --- a/ui/src/routes/tasks.tsx +++ b/ui/src/routes/tasks.tsx @@ -5,6 +5,7 @@ import { useEffect, useRef, useState } from 'react' import { api, type TaskItem } from '@/api/client' import { CopyableValue } from '@/components/app/CopyableValue' +import { DangerActionAlertDialog } from '@/components/app/DangerActionAlertDialog' import { PageErrorState } from '@/components/app/PageErrorState' import { PageHeader } from '@/components/app/PageHeader' import { StatusBadge, taskStatusTone } from '@/components/app/StatusBadge' @@ -97,6 +98,10 @@ function TasksPage() { const filterKey = `${taskType}:${status}` const previousFilterKey = useRef(filterKey) const tasks = useTasks(taskType, status, PAGE_SIZE, cursor) + // What the operator is asked to confirm: the server's own count, the moment + // it counted, and the operation it counted for. Confirming sends all three + // back, so nothing that failed while the dialog was open is swept up. + const [dismissAllScope, setDismissAllScope] = useState(null) useEffect(() => { if (previousFilterKey.current === filterKey) return @@ -111,11 +116,24 @@ function TasksPage() { } const retry = useMutation({ mutationFn: api.retryTask, onSuccess: refreshTasks }) const acknowledge = useMutation({ mutationFn: api.acknowledgeTask, onSuccess: refreshTasks }) - const actionError = retry.error ?? acknowledge.error + const previewDismissAll = useMutation({ + mutationFn: api.previewAcknowledgeTasks, + onSuccess: (preview, variables) => setDismissAllScope({ ...preview, type: variables.type }), + }) + const dismissAll = useMutation({ + mutationFn: api.acknowledgeTasks, + onSuccess: () => { + setDismissAllScope(null) + refreshTasks() + }, + }) + const actionError = retry.error ?? acknowledge.error ?? previewDismissAll.error const setFilters = (nextType: TaskOperationFilter, nextStatus: TaskStatusFilter) => { retry.reset() acknowledge.reset() + previewDismissAll.reset() + dismissAll.reset() navigate({ to: '/tasks', search: { @@ -148,10 +166,26 @@ function TasksPage() { title="Tasks" description="Review background operations and recover work that needs attention." actions={ - +
+ {status === 'failed' && ( + + )} + +
} /> @@ -256,10 +290,39 @@ function TasksPage() { )} + + { + if (open) return + dismissAll.reset() + setDismissAllScope(null) + }} + title="Dismiss failed tasks" + description={dismissAllDescription(dismissAllScope?.count ?? 0, dismissAllScope?.type)} + confirmLabel={dismissAllScope?.count === 1 ? 'Dismiss 1 task' : `Dismiss ${dismissAllScope?.count ?? 0} tasks`} + pending={dismissAll.isPending} + confirmDisabled={dismissAllScope?.count === 0} + error={dismissAll.error ? errorMessage(dismissAll.error) : null} + onConfirm={() => { + if (!dismissAllScope) return + dismissAll.mutate({ type: dismissAllScope.type, failed_before: dismissAllScope.as_of }) + }} + /> ) } +type DismissAllScope = { count: number; as_of: string; type?: string } + +function dismissAllDescription(count: number, operation?: string) { + const label = operation ? (taskOperations.find((option) => option.value === operation)?.label ?? operation) : '' + const scope = label ? ` for ${label}` : '' + if (count === 0) return `No failed tasks${scope} are left to dismiss.` + const tasks = count === 1 ? '1 failed task' : `${count} failed tasks` + return `${tasks}${scope} will move to Dismissed and be removed after the retention period. Tasks that fail after you confirm stay in the list, and nothing is retried.` +} + function TaskTable({ tasks, retryingID, diff --git a/ui/test/api-client.test.ts b/ui/test/api-client.test.ts index 0ad2910..d9a9059 100644 --- a/ui/test/api-client.test.ts +++ b/ui/test/api-client.test.ts @@ -56,6 +56,33 @@ function installFakeXMLHttpRequest() { } } +test('bulk task dismissal posts the operation filter and the cutoff', async () => { + const originalFetch = globalThis.fetch + const calls: Array<{ url: string; method?: string; body?: BodyInit | null }> = [] + globalThis.fetch = (async (input, init) => { + calls.push({ url: String(input), method: init?.method, body: init?.body }) + return new Response(JSON.stringify({ acknowledged: 3 }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + }) as typeof fetch + + try { + const result = await api.acknowledgeTasks({ type: 'storage_store', failed_before: '2026-09-12T08:30:00Z' }) + assert.equal(result.acknowledged, 3) + } finally { + globalThis.fetch = originalFetch + } + + assert.equal(calls.length, 1) + assert.equal(calls[0]?.url, '/api/v1/tasks/acknowledge') + assert.equal(calls[0]?.method, 'POST') + assert.deepEqual(JSON.parse(String(calls[0]?.body)), { + type: 'storage_store', + failed_before: '2026-09-12T08:30:00Z', + }) +}) + test('admin login sends remember mode and refreshes the stored csrf token', async () => { const originalFetch = globalThis.fetch const calls: Array<{