From 1bd2ae122c4c6c3f96a05f35b2db76ce2355308b Mon Sep 17 00:00:00 2001 From: Lee <7932644+strahe@users.noreply.github.com> Date: Sun, 20 Sep 2026 13:46:32 +0800 Subject: [PATCH 1/5] chore(synapse): upgrade SDK to v0.6.0-beta.2 --- go.mod | 8 +- go.sum | 16 ++-- internal/config/metadata.go | 2 +- internal/storagecommit/submission_test.go | 21 +++++ internal/synapse/client.go | 1 + internal/synapse/interfaces.go | 3 +- internal/synapse/readiness.go | 9 +- internal/synapse/readiness_test.go | 38 +++++---- internal/synapse/storage_client.go | 7 +- internal/synapse/storage_client_test.go | 22 +++-- internal/systemtest/filecoin.go | 5 +- internal/testutil/mocks.go | 7 +- internal/worker/storage_task_handlers.go | 3 +- .../calibration/calibration_test.go | 84 +++++++++++++------ ui/src/lib/risk-confirmation.ts | 2 +- ui/test/risk-confirmation.test.ts | 2 +- 16 files changed, 151 insertions(+), 79 deletions(-) diff --git a/go.mod b/go.mod index 4aea488..a328d7b 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/knadh/koanf/v2 v2.3.4 github.com/multiformats/go-multihash v0.2.3 github.com/prometheus/client_golang v1.23.2 - github.com/strahe/synapse-go v0.6.0-beta.1 + github.com/strahe/synapse-go v0.6.0-beta.2 github.com/uptrace/bun v1.2.18 github.com/uptrace/bun/dialect/pgdialect v1.2.18 github.com/uptrace/bun/dialect/sqlitedialect v1.2.18 @@ -74,9 +74,9 @@ require ( github.com/whyrusleeping/cbor-gen v0.3.1 // indirect github.com/zeebo/xxh3 v1.1.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel v1.40.0 // indirect - go.opentelemetry.io/otel/metric v1.40.0 // indirect - go.opentelemetry.io/otel/trace v1.40.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect diff --git a/go.sum b/go.sum index 0f30fc2..496ae67 100644 --- a/go.sum +++ b/go.sum @@ -388,8 +388,8 @@ github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/strahe/synapse-go v0.6.0-beta.1 h1:+s3c6K1pRKmW8LaNUFMATzYR1K4qajFsdY0qkLO47xo= -github.com/strahe/synapse-go v0.6.0-beta.1/go.mod h1:bOw0zW2mWUeB2MNlplO9lOJq0c9teZNicMiEx0vjR6c= +github.com/strahe/synapse-go v0.6.0-beta.2 h1:EjA7wVfWJIZoWuaBOpcW7+RfufAZMNRsNIBNUQDupvw= +github.com/strahe/synapse-go v0.6.0-beta.2/go.mod h1:B6t1aDkuqfVtcJuOau1n5erDsyiAUAEs1jlHjnmBvsA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= @@ -461,14 +461,14 @@ gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 h1:qwDnMxjkyLmAF gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02/go.mod h1:JTnUj0mpYiAsuZLmKjTx/ex3AtMowcCgnE7YNyCEP0I= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= -go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= -go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= -go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= -go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= -go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= diff --git a/internal/config/metadata.go b/internal/config/metadata.go index fdc9ddf..4b86648 100644 --- a/internal/config/metadata.go +++ b/internal/config/metadata.go @@ -84,7 +84,7 @@ var fieldMetadataByPath = map[string]FieldMetadata{ }, "filecoin.allow_private_networks": { Label: "Allow Private Networks", - Description: "Allows private-network provider URLs for retrieval and diagnostics; enable only in trusted environments.", + Description: "Allows private-network URLs used for storage provider operations, retrieval, and diagnostics; enable only for trusted infrastructure.", Env: "SYNAPS3_FILECOIN_ALLOW_PRIVATE_NETWORKS", Editable: true, }, diff --git a/internal/storagecommit/submission_test.go b/internal/storagecommit/submission_test.go index 3ff92fa..e1536df 100644 --- a/internal/storagecommit/submission_test.go +++ b/internal/storagecommit/submission_test.go @@ -27,6 +27,27 @@ func TestSubmissionEnvelopeRoundTrip(t *testing.T) { } } +func TestSubmissionEnvelopeDecodesBeta1Fixture(t *testing.T) { + const beta1Envelope = `{"version":1,"submission":{"kind":"add-pieces","transactionId":"0x0000000000000000000000000000000000000000000000000000000000000011","statusUrl":"https://provider.example/commit/0x0000000000000000000000000000000000000000000000000000000000000011","providerId":"1","identity":{"payer":"0x0000000000000000000000000000000000001234","chainId":314159,"recordKeeper":"0x0000000000000000000000000000000000005678"},"dataSet":{"providerId":"1","dataSetId":"42","clientDataSetId":"7"},"clientDataSetId":null,"pieceCids":[{"/":"bafkzcibcaaces3nobte6ezpp4wqan2age2s5yxcatzotcvobhgcmv5wi2xh5mbi"}]}}` + + decoded, err := DecodeSubmission(beta1Envelope) + if err != nil { + t.Fatalf("DecodeSubmission beta.1 fixture: %v", err) + } + if decoded.Kind != storage.CommitKindAddPieces || decoded.TransactionID != "0x0000000000000000000000000000000000000000000000000000000000000011" { + t.Fatalf("decoded beta.1 submission = %#v", decoded) + } + if decoded.ProviderID.String() != "1" || decoded.Identity.ChainID != sdktypes.ChainID(314159) || decoded.Identity.Payer != common.HexToAddress("0x1234") { + t.Fatalf("decoded beta.1 identity = provider %s, chain %d", decoded.ProviderID.String(), decoded.Identity.ChainID) + } + if decoded.DataSet == nil || decoded.DataSet.ProviderID().String() != "1" || decoded.DataSet.DataSetID().String() != "42" || decoded.DataSet.ClientDataSetID().String() != "7" { + t.Fatalf("decoded beta.1 data set = %#v", decoded.DataSet) + } + if decoded.ClientDataSetID != nil || len(decoded.PieceCIDs) != 1 || decoded.PieceCIDs[0].String() != "bafkzcibcaaces3nobte6ezpp4wqan2age2s5yxcatzotcvobhgcmv5wi2xh5mbi" { + t.Fatalf("decoded beta.1 recovery fields = %#v", decoded) + } +} + func TestSubmissionEnvelopeRejectsUnsupportedOrMalformedData(t *testing.T) { encoded, err := EncodeSubmission(testCommitSubmission(t)) if err != nil { diff --git a/internal/synapse/client.go b/internal/synapse/client.go index bf50f96..4aed063 100644 --- a/internal/synapse/client.go +++ b/internal/synapse/client.go @@ -26,6 +26,7 @@ func NewClient(ctx context.Context, cfg ClientConfig) (*sdk.Client, error) { sdk.WithRPCURL(cfg.RPCURL), sdk.WithSource(dataSetSource), sdk.WithCDN(cfg.WithCDN), + sdk.WithoutUploadBatching(), sdk.WithAllowPrivateNetworks(cfg.AllowPrivateNetworks), } if cfg.Logger != nil { diff --git a/internal/synapse/interfaces.go b/internal/synapse/interfaces.go index 3b72895..5834398 100644 --- a/internal/synapse/interfaces.go +++ b/internal/synapse/interfaces.go @@ -7,6 +7,7 @@ import ( "time" "github.com/ipfs/go-cid" + sdkcosts "github.com/strahe/synapse-go/costs" "github.com/strahe/synapse-go/storage" sdktypes "github.com/strahe/synapse-go/types" ) @@ -57,7 +58,7 @@ type CleanupContext interface { // staged provider operations. type StorageClient interface { Download(ctx context.Context, pieceCID cid.Cid, opts *storage.DownloadOptions) (io.ReadCloser, error) - PrepareUpload(ctx context.Context, dataSize uint64, targets []StorageTarget) (*storage.MultiContextCosts, error) + PrepareUpload(ctx context.Context, dataSize uint64, targets []StorageTarget) (*sdkcosts.MultiContextCosts, error) SelectUploadTargets(ctx context.Context, opts storage.SelectUploadContextsOptions) ([]StorageTarget, error) OpenProviderTarget(ctx context.Context, providerID sdktypes.BigInt, opts storage.NewProviderContextOptions) (ProviderTarget, error) OpenDataSetTarget(ctx context.Context, dataSetID sdktypes.BigInt, opts storage.NewDataSetContextOptions) (DataSetTarget, error) diff --git a/internal/synapse/readiness.go b/internal/synapse/readiness.go index cc9d96b..8994c75 100644 --- a/internal/synapse/readiness.go +++ b/internal/synapse/readiness.go @@ -15,6 +15,7 @@ import ( "github.com/strahe/synaps3/internal/objectlimits" sdk "github.com/strahe/synapse-go" "github.com/strahe/synapse-go/chain" + sdkcosts "github.com/strahe/synapse-go/costs" "github.com/strahe/synapse-go/payments" "github.com/strahe/synapse-go/spregistry" "github.com/strahe/synapse-go/storage" @@ -76,11 +77,11 @@ type readinessStorage interface { GetStorageInfo(ctx context.Context, opts *storage.GetStorageInfoOptions) (*storage.StorageInfo, error) CalculateMultiContextCosts( ctx context.Context, - dataSizeBytes uint64, + pieceSizes []uint64, refs []storage.ContextCostRef, opts storage.MultiCostOptions, payer common.Address, - ) (*storage.MultiContextCosts, error) + ) (*sdkcosts.MultiContextCosts, error) } type ReadinessClient interface { @@ -195,7 +196,7 @@ func (c *ReadinessChecker) checkClient(ctx context.Context, cfg ReadinessConfig, result.warning( "private_networks", "Private network provider URLs are allowed.", - "Use this only for trusted private infrastructure that serves retrieval and diagnostic URLs.", + "Use this only with trusted private infrastructure for storage, retrieval, and diagnostics.", ) } @@ -341,7 +342,7 @@ func (c *ReadinessChecker) checkStorage( costs, err := storageSvc.CalculateMultiContextCosts( ctx, - readinessCostEstimateDataSize(), + []uint64{readinessCostEstimateDataSize()}, refs, storage.MultiCostOptions{EnableCDN: cfg.WithCDN}, client.Address(), diff --git a/internal/synapse/readiness_test.go b/internal/synapse/readiness_test.go index 3889408..cd90f58 100644 --- a/internal/synapse/readiness_test.go +++ b/internal/synapse/readiness_test.go @@ -14,6 +14,7 @@ import ( "github.com/strahe/synaps3/internal/objectlimits" sdk "github.com/strahe/synapse-go" "github.com/strahe/synapse-go/chain" + sdkcosts "github.com/strahe/synapse-go/costs" "github.com/strahe/synapse-go/payments" "github.com/strahe/synapse-go/spregistry" "github.com/strahe/synapse-go/storage" @@ -67,8 +68,13 @@ func TestReadinessCheckerUsesApprovedProviderInventoryAndCostEstimate(t *testing if len(client.storage.costRefs) != cfg.DefaultCopies { t.Fatalf("cost refs = %d, want %d", len(client.storage.costRefs), cfg.DefaultCopies) } - if client.storage.costDataSize != uint64(objectlimits.MinFOCUploadSize) { - t.Fatalf("cost data size = %d, want %d", client.storage.costDataSize, objectlimits.MinFOCUploadSize) + if len(client.storage.costPieceSizes) != 1 || client.storage.costPieceSizes[0] != uint64(objectlimits.MinFOCUploadSize) { + t.Fatalf("cost piece sizes = %v, want [%d]", client.storage.costPieceSizes, objectlimits.MinFOCUploadSize) + } + for i, ref := range client.storage.costRefs { + if ref.DataSetID != nil { + t.Fatalf("cost ref %d data set ID = %v, want a new data set", i, ref.DataSetID) + } } if client.storage.costPayer != client.address { t.Fatalf("cost payer = %s, want %s", client.storage.costPayer.Hex(), client.address.Hex()) @@ -225,8 +231,8 @@ func TestReadinessCheckerWarnsWhenPrivateNetworksAreAllowed(t *testing.T) { if check.Message != "Private network provider URLs are allowed." { t.Fatalf("private_networks message = %q, want provider URL warning", check.Message) } - if check.Action != "Use this only for trusted private infrastructure that serves retrieval and diagnostic URLs." { - t.Fatalf("private_networks action = %q, want retrieval and diagnostic URL warning", check.Action) + if check.Action != "Use this only with trusted private infrastructure for storage, retrieval, and diagnostics." { + t.Fatalf("private_networks action = %q, want storage, retrieval, and diagnostics warning", check.Action) } } @@ -353,8 +359,8 @@ func readinessAccountWithFundedUntil(funds int64, fundedUntil *big.Int) *payment } } -func readinessCosts(deposit int64, needsApproval, ready bool) *storage.MultiContextCosts { - return &storage.MultiContextCosts{ +func readinessCosts(deposit int64, needsApproval, ready bool) *sdkcosts.MultiContextCosts { + return &sdkcosts.MultiContextCosts{ RatePerEpoch: big.NewInt(1), RatePerMonth: big.NewInt(100), DepositNeeded: big.NewInt(deposit), @@ -503,13 +509,13 @@ func (f *fakeReadinessPayments) AccountInfo(context.Context, common.Address, com } type fakeReadinessStorage struct { - info *storage.StorageInfo - infoErr error - costs *storage.MultiContextCosts - costErr error - costRefs []storage.ContextCostRef - costPayer common.Address - costDataSize uint64 + info *storage.StorageInfo + infoErr error + costs *sdkcosts.MultiContextCosts + costErr error + costRefs []storage.ContextCostRef + costPayer common.Address + costPieceSizes []uint64 } func (f *fakeReadinessStorage) GetStorageInfo(context.Context, *storage.GetStorageInfoOptions) (*storage.StorageInfo, error) { @@ -518,13 +524,13 @@ func (f *fakeReadinessStorage) GetStorageInfo(context.Context, *storage.GetStora func (f *fakeReadinessStorage) CalculateMultiContextCosts( _ context.Context, - dataSize uint64, + pieceSizes []uint64, refs []storage.ContextCostRef, _ storage.MultiCostOptions, payer common.Address, -) (*storage.MultiContextCosts, error) { +) (*sdkcosts.MultiContextCosts, error) { f.costRefs = append([]storage.ContextCostRef(nil), refs...) f.costPayer = payer - f.costDataSize = dataSize + f.costPieceSizes = append([]uint64(nil), pieceSizes...) return f.costs, f.costErr } diff --git a/internal/synapse/storage_client.go b/internal/synapse/storage_client.go index 9b9365c..52e58cd 100644 --- a/internal/synapse/storage_client.go +++ b/internal/synapse/storage_client.go @@ -8,6 +8,7 @@ import ( "maps" "github.com/ipfs/go-cid" + sdkcosts "github.com/strahe/synapse-go/costs" "github.com/strahe/synapse-go/storage" sdktypes "github.com/strahe/synapse-go/types" "github.com/strahe/synapse-go/warmstorage" @@ -44,7 +45,7 @@ func (s *StorageServiceAdapter) Download(ctx context.Context, pieceCID cid.Cid, return s.service.Download(ctx, pieceCID, opts) } -func (s *StorageServiceAdapter) PrepareUpload(ctx context.Context, dataSize uint64, targets []StorageTarget) (*storage.MultiContextCosts, error) { +func (s *StorageServiceAdapter) PrepareUpload(ctx context.Context, dataSize uint64, targets []StorageTarget) (*sdkcosts.MultiContextCosts, error) { contexts := make([]storage.StorageContext, 0, len(targets)) for i, target := range targets { adapter, ok := target.(interface{ sdkStorageContext() storage.StorageContext }) @@ -54,8 +55,8 @@ func (s *StorageServiceAdapter) PrepareUpload(ctx context.Context, dataSize uint contexts = append(contexts, adapter.sdkStorageContext()) } prepared, err := s.service.Prepare(ctx, &storage.PrepareOptions{ - DataSize: dataSize, - Contexts: contexts, + PieceSizes: []uint64{dataSize}, + Contexts: contexts, }) if err != nil { return nil, normalizeResolutionOperationError(err) diff --git a/internal/synapse/storage_client_test.go b/internal/synapse/storage_client_test.go index c42635c..eb899a1 100644 --- a/internal/synapse/storage_client_test.go +++ b/internal/synapse/storage_client_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + sdkcosts "github.com/strahe/synapse-go/costs" "github.com/strahe/synapse-go/payments" "github.com/strahe/synapse-go/storage" sdktypes "github.com/strahe/synapse-go/types" @@ -144,7 +145,7 @@ func TestStorageServiceAdapterPrepareUploadReturnsCostsWithoutFunding(t *testing if err != nil { t.Fatalf("NewProviderContext: %v", err) } - want := &storage.MultiContextCosts{DepositNeeded: big.NewInt(123), NeedsFWSSMaxApproval: true, Ready: false} + want := &sdkcosts.MultiContextCosts{DepositNeeded: big.NewInt(123), NeedsFWSSMaxApproval: true, Ready: false} costs := &staticCostCalculator{costs: want} funder := &recordingPaymentsFunder{} service, err := storage.New(storage.Options{ @@ -164,8 +165,11 @@ func TestStorageServiceAdapterPrepareUploadReturnsCostsWithoutFunding(t *testing if err != nil { t.Fatalf("PrepareUpload: %v", err) } - if got != want || costs.dataSize.Cmp(big.NewInt(4096)) != 0 || len(costs.refs) != 1 { - t.Fatalf("costs = %#v, calculator size = %v refs = %#v", got, costs.dataSize, costs.refs) + if got != want || len(costs.pieceSizes) != 1 || costs.pieceSizes[0] != 4096 || len(costs.refs) != 1 { + t.Fatalf("costs = %#v, calculator piece sizes = %v refs = %#v", got, costs.pieceSizes, costs.refs) + } + if !costs.refs[0].IsNewDataSet { + t.Fatalf("calculator ref = %#v, want a new data set", costs.refs[0]) } if funder.calls != 0 { t.Fatalf("funding calls = %d, want none", funder.calls) @@ -216,14 +220,14 @@ func dataSetDetails(dataSetID, clientDataSetID, providerID uint64, live, managed } type staticCostCalculator struct { - costs *storage.MultiContextCosts - dataSize *big.Int - refs []storage.ContextCostRef + costs *sdkcosts.MultiContextCosts + pieceSizes []uint64 + refs []sdkcosts.MultiContextRef } -func (c *staticCostCalculator) CalculateMultiContextCosts(_ context.Context, _ common.Address, dataSize *big.Int, refs []storage.ContextCostRef, _ storage.MultiCostOptions) (*storage.MultiContextCosts, error) { - c.dataSize = new(big.Int).Set(dataSize) - c.refs = append([]storage.ContextCostRef(nil), refs...) +func (c *staticCostCalculator) CalculateMultiContextCosts(_ context.Context, _ common.Address, pieceSizes []uint64, refs []sdkcosts.MultiContextRef, _ *sdkcosts.UploadCostOptions) (*sdkcosts.MultiContextCosts, error) { + c.pieceSizes = append([]uint64(nil), pieceSizes...) + c.refs = append([]sdkcosts.MultiContextRef(nil), refs...) return c.costs, nil } diff --git a/internal/systemtest/filecoin.go b/internal/systemtest/filecoin.go index 4f85a8b..59a8748 100644 --- a/internal/systemtest/filecoin.go +++ b/internal/systemtest/filecoin.go @@ -21,6 +21,7 @@ import ( "github.com/strahe/synaps3/internal/observability" "github.com/strahe/synaps3/internal/synapse" appTypes "github.com/strahe/synaps3/internal/types" + sdkcosts "github.com/strahe/synapse-go/costs" "github.com/strahe/synapse-go/piece" "github.com/strahe/synapse-go/storage" sdktypes "github.com/strahe/synapse-go/types" @@ -91,14 +92,14 @@ func NewMemoryFilecoin() *MemoryFilecoin { } } -func (m *MemoryFilecoin) PrepareUpload(ctx context.Context, _ uint64, targets []synapse.StorageTarget) (*storage.MultiContextCosts, error) { +func (m *MemoryFilecoin) PrepareUpload(ctx context.Context, _ uint64, targets []synapse.StorageTarget) (*sdkcosts.MultiContextCosts, error) { if err := ctx.Err(); err != nil { return nil, err } if len(targets) == 0 { return nil, fmt.Errorf("%w: upload has no contexts", errInvalidFilecoinSequence) } - return &storage.MultiContextCosts{DepositNeeded: new(big.Int), Ready: true}, nil + return &sdkcosts.MultiContextCosts{DepositNeeded: new(big.Int), Ready: true}, nil } func (m *MemoryFilecoin) SelectUploadTargets(ctx context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { diff --git a/internal/testutil/mocks.go b/internal/testutil/mocks.go index d9e4da8..b3f8707 100644 --- a/internal/testutil/mocks.go +++ b/internal/testutil/mocks.go @@ -9,6 +9,7 @@ import ( "github.com/ipfs/go-cid" "github.com/strahe/synaps3/internal/cache" "github.com/strahe/synaps3/internal/synapse" + sdkcosts "github.com/strahe/synapse-go/costs" "github.com/strahe/synapse-go/storage" sdktypes "github.com/strahe/synapse-go/types" ) @@ -26,7 +27,7 @@ var ( type MockStorageClient struct { UploadFunc func(ctx context.Context, r io.Reader, opts *storage.UploadOptions) (*storage.UploadResult, error) DownloadFunc func(ctx context.Context, pieceCID cid.Cid, opts *storage.DownloadOptions) (io.ReadCloser, error) - PrepareUploadFunc func(ctx context.Context, dataSize uint64, targets []synapse.StorageTarget) (*storage.MultiContextCosts, error) + PrepareUploadFunc func(ctx context.Context, dataSize uint64, targets []synapse.StorageTarget) (*sdkcosts.MultiContextCosts, error) SelectUploadTargetsFunc func(ctx context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) OpenProviderTargetFunc func(ctx context.Context, providerID sdktypes.BigInt, opts storage.NewProviderContextOptions) (synapse.ProviderTarget, error) OpenDataSetTargetFunc func(ctx context.Context, dataSetID sdktypes.BigInt, opts storage.NewDataSetContextOptions) (synapse.DataSetTarget, error) @@ -58,11 +59,11 @@ func (m *MockStorageClient) Download(ctx context.Context, pieceCID cid.Cid, opts return nil, errors.New("MockStorageClient.Download not configured") } -func (m *MockStorageClient) PrepareUpload(ctx context.Context, dataSize uint64, targets []synapse.StorageTarget) (*storage.MultiContextCosts, error) { +func (m *MockStorageClient) PrepareUpload(ctx context.Context, dataSize uint64, targets []synapse.StorageTarget) (*sdkcosts.MultiContextCosts, error) { if m.PrepareUploadFunc != nil { return m.PrepareUploadFunc(ctx, dataSize, targets) } - return &storage.MultiContextCosts{Ready: true}, nil + return &sdkcosts.MultiContextCosts{Ready: true}, nil } func (m *MockStorageClient) SelectUploadTargets(ctx context.Context, opts storage.SelectUploadContextsOptions) ([]synapse.StorageTarget, error) { diff --git a/internal/worker/storage_task_handlers.go b/internal/worker/storage_task_handlers.go index 7ab7bc2..27a7b3b 100644 --- a/internal/worker/storage_task_handlers.go +++ b/internal/worker/storage_task_handlers.go @@ -26,6 +26,7 @@ import ( "github.com/strahe/synaps3/internal/synapse" taskengine "github.com/strahe/synaps3/internal/task" idtypes "github.com/strahe/synaps3/internal/types" + sdkcosts "github.com/strahe/synapse-go/costs" "github.com/strahe/synapse-go/piece" "github.com/strahe/synapse-go/storage" sdktypes "github.com/strahe/synapse-go/types" @@ -243,7 +244,7 @@ func (h *TaskHandlers) uploadPlanHandler() taskengine.Handler { return taskHandler{definition: definition, execute: run, recover: run} } -func uploadFundingWaitMessage(costs *storage.MultiContextCosts) string { +func uploadFundingWaitMessage(costs *sdkcosts.MultiContextCosts) string { parts := make([]string, 0, 2) if costs != nil && costs.DepositNeeded != nil && costs.DepositNeeded.Sign() > 0 { parts = append(parts, fmt.Sprintf("deposit %s USDFC base units", costs.DepositNeeded.String())) diff --git a/tests/integration/calibration/calibration_test.go b/tests/integration/calibration/calibration_test.go index e422f97..53d32a5 100644 --- a/tests/integration/calibration/calibration_test.go +++ b/tests/integration/calibration/calibration_test.go @@ -32,9 +32,10 @@ import ( ) const ( - integrationNetwork = "calibration" - integrationCopies = 3 - uploadTaskTimeout = 2 * time.Minute + integrationNetwork = "calibration" + integrationCopies = 3 + bucketProvisionTimeout = 10 * time.Minute + uploadTaskTimeout = 2 * time.Minute adminUsername = "admin" ) @@ -81,6 +82,7 @@ func TestCalibrationBackedGoldenPath(t *testing.T) { if _, err := s3Client.CreateBucket(t.Context(), &awss3.CreateBucketInput{Bucket: aws.String(bucket)}); err != nil { t.Fatalf("CreateBucket: %v\n%s", err, runtime.Diagnostics()) } + waitForBucketReady(t, admin, bucket) content := bytes.Repeat([]byte("synaps3-calibration-e2e\n"), 6000) checksum := sha256.Sum256(content) @@ -101,7 +103,7 @@ func TestCalibrationBackedGoldenPath(t *testing.T) { object := waitForStoredObject(t, admin, bucket, key, walletActions) provenance := waitForCommittedCopies(t, admin, bucket, object.VersionID) assertDataSetMetadata(t, t.Context(), privateKey, bucket, provenance) - waitForCompletedUploadTasks(t, admin, object.VersionID) + waitForCompletedUploadTasks(t, admin) waitForCacheEviction(t, admin, bucket, key) logStep(t, "verifying cold S3 read after cache eviction") waitForS3Object(t, runtime, s3Client, bucket, key, content, checksum) @@ -715,6 +717,52 @@ type storedObject struct { Snapshot string } +type calibrationBucketStatus struct { + Status string `json:"status"` + DataSets []struct { + CopyIndex int `json:"copy_index"` + Provider string `json:"provider_id"` + DataSetID *string `json:"data_set_id,omitempty"` + Status string `json:"status"` + } `json:"data_sets"` +} + +func waitForBucketReady(t *testing.T, admin *e2e.AdminClient, bucket string) { + t.Helper() + progress := newProgressLog() + e2e.Eventually(t, t.Context(), bucketProvisionTimeout, "Calibration bucket storage to become ready", func(ctx context.Context) (string, bool, error) { + var status calibrationBucketStatus + raw, err := admin.GetJSON(ctx, "/api/v1/buckets/"+url.PathEscape(bucket), &status) + if err != nil { + return raw, false, err + } + summary := calibrationBucketStatusSummary(status) + progress.Changed(t, "bucket storage", summary) + switch status.Status { + case "ready": + return summary, true, nil + case "provisioning": + return summary, false, nil + default: + return summary, false, fmt.Errorf("unexpected bucket status %q", status.Status) + } + }, e2e.WithPollInterval(5*time.Second)) +} + +func calibrationBucketStatusSummary(bucket calibrationBucketStatus) string { + lines := []string{fmt.Sprintf("status=%s data_sets=%d", bucket.Status, len(bucket.DataSets))} + for _, dataSet := range bucket.DataSets { + lines = append(lines, fmt.Sprintf( + "- copy=%d provider=%s dataset=%s status=%s", + dataSet.CopyIndex, + dataSet.Provider, + nullableString(dataSet.DataSetID), + dataSet.Status, + )) + } + return strings.Join(lines, "\n") +} + func waitForStoredObject(t *testing.T, admin *e2e.AdminClient, bucket, key string, actions *calibrationWalletActions) storedObject { t.Helper() progress := newProgressLog() @@ -733,26 +781,23 @@ func waitForStoredObject(t *testing.T, admin *e2e.AdminClient, bucket, key strin if item.Status == "warning" || item.State == "failed" { return storedObject{VersionID: item.CurrentVersionID, Snapshot: raw}, false, fmt.Errorf("object entered failed state: %s", e2e.DiagnosticValue(item)) } - resolveUploadDependency(t, ctx, admin, item.CurrentVersionID, actions, taskProgress) + resolveUploadDependency(t, ctx, admin, actions, taskProgress) return storedObject{VersionID: item.CurrentVersionID, Snapshot: raw}, item.CurrentVersionID != "" && item.State == "stored" && item.Location.Filecoin, nil }, e2e.WithPollInterval(5*time.Second)) } -func resolveUploadDependency(t *testing.T, ctx context.Context, admin *e2e.AdminClient, versionID string, actions *calibrationWalletActions, progress *progressLog) { +func resolveUploadDependency(t *testing.T, ctx context.Context, admin *e2e.AdminClient, actions *calibrationWalletActions, progress *progressLog) { t.Helper() - if versionID == "" { - return - } var tasks e2e.TaskListResponse raw, err := admin.GetJSON(ctx, "/api/v1/tasks?type=upload_plan&limit=100", &tasks) if err != nil { t.Fatalf("GET upload tasks: %v; body=%s", err, e2e.Redact(raw)) } if progress != nil { - progress.Changed(t, "upload tasks", uploadTaskSummary(tasks, versionID)) + progress.Changed(t, "upload tasks", uploadTaskSummary(tasks)) } for _, task := range tasks.Tasks { - if !taskHasSubject(task, "object_version", versionID) || task.Status != "pending" || task.WaitReason == nil || *task.WaitReason != "funding" { + if task.Status != "pending" || task.WaitReason == nil || *task.WaitReason != "funding" { continue } message := nullableString(task.StatusMessage) @@ -871,7 +916,7 @@ func assertDataSetMetadata(t *testing.T, ctx context.Context, privateKey, bucket } } -func waitForCompletedUploadTasks(t *testing.T, admin *e2e.AdminClient, versionID string) { +func waitForCompletedUploadTasks(t *testing.T, admin *e2e.AdminClient) { t.Helper() progress := newProgressLog() lastSummary := "none" @@ -881,13 +926,10 @@ func waitForCompletedUploadTasks(t *testing.T, admin *e2e.AdminClient, versionID if err != nil { return lastSummary, false, err } - lastSummary = uploadTaskSummary(tasks, versionID) + lastSummary = uploadTaskSummary(tasks) progress.Changed(t, "upload tasks", lastSummary) seen, active, failed := 0, 0, 0 for _, task := range tasks.Tasks { - if !taskHasSubject(task, "object_version", versionID) { - continue - } seen++ switch task.Status { case "completed": @@ -1150,12 +1192,9 @@ func provenanceSummary(provenance e2e.ProvenanceResponse) string { return strings.Join(lines, "\n") } -func uploadTaskSummary(tasks e2e.TaskListResponse, versionID string) string { +func uploadTaskSummary(tasks e2e.TaskListResponse) string { lines := make([]string, 0, len(tasks.Tasks)) for _, task := range tasks.Tasks { - if !taskHasSubject(task, "object_version", versionID) { - continue - } lines = append(lines, fmt.Sprintf( "- id=%d type=%s status=%s retry=%d available=%s wait=%s message=%s error=%s", task.ID, @@ -1174,11 +1213,6 @@ func uploadTaskSummary(tasks e2e.TaskListResponse, versionID string) string { return strings.Join(lines, "\n") } -func taskHasSubject(task e2e.TaskItem, subjectType, subjectKey string) bool { - return task.SubjectType != nil && task.SubjectKey != nil && - *task.SubjectType == subjectType && *task.SubjectKey == subjectKey -} - func observabilitySnapshotSummary(providers e2e.ProviderObservationPage, dataSets e2e.DataSetObservationPage) string { lines := []string{fmt.Sprintf( "providers total=%d available=%d degraded=%d unavailable=%d unknown=%d", diff --git a/ui/src/lib/risk-confirmation.ts b/ui/src/lib/risk-confirmation.ts index b1b8ffd..86e6ba5 100644 --- a/ui/src/lib/risk-confirmation.ts +++ b/ui/src/lib/risk-confirmation.ts @@ -104,7 +104,7 @@ export function collectSettingsRiskChanges( initial.filecoin.allow_private_networks, next.filecoin.allow_private_networks, 'high', - 'Allows private-network provider URLs for retrieval and diagnostics; enable only in trusted environments.' + 'Allows private-network URLs used for storage provider operations, retrieval, and diagnostics; enable only for trusted infrastructure.' ) } addChanged( diff --git a/ui/test/risk-confirmation.test.ts b/ui/test/risk-confirmation.test.ts index 4382b19..9c1dce3 100644 --- a/ui/test/risk-confirmation.test.ts +++ b/ui/test/risk-confirmation.test.ts @@ -130,7 +130,7 @@ test('settings risk collection classifies high-risk security boundary changes', ] ) const privateNetworks = changes.find((change) => change.field === 'filecoin.allow_private_networks') - assert.match(privateNetworks?.reason ?? '', /diagnostic/i) + assert.match(privateNetworks?.reason ?? '', /storage provider operations, retrieval, and diagnostics/i) assert.equal(settingsRiskNeedsStrongConfirmation(changes), true) }) From bcdf5ca081304524949778ebb0bc10154284790c Mon Sep 17 00:00:00 2001 From: Lee <7932644+strahe@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:00:42 +0800 Subject: [PATCH 2/5] chore(synapse): upgrade SDK to v0.6.0 --- go.mod | 6 +- go.sum | 20 +- .../2026092101_commit_status_url.go | 82 ++++++ .../2026092101_commit_status_url_test.go | 99 ++++++++ .../2026999999_transaction_semantics_test.go | 2 +- .../db/migrations/schema_integrity_test.go | 2 +- .../migrations/schema_model_alignment_test.go | 5 +- internal/db/repository/storage_commit_repo.go | 10 +- .../db/repository/storage_content_repo.go | 2 +- internal/model/storage.go | 2 +- internal/storagecommit/advancer.go | 46 ++-- internal/storagecommit/advancer_test.go | 237 +++++++++++------- internal/storagecommit/attempt.go | 2 +- internal/storagecommit/store.go | 10 +- internal/storagecommit/submission.go | 57 ----- internal/storagecommit/submission_test.go | 93 ------- internal/synapse/interfaces.go | 4 +- internal/synapse/pdp_status.go | 6 + internal/synapse/pdp_status_test.go | 6 +- internal/synapse/storage_client.go | 8 +- internal/systemtest/filecoin.go | 75 +++--- internal/systemtest/filecoin_test.go | 2 +- internal/testutil/mocks.go | 12 +- internal/worker/storage_task_handlers.go | 5 +- internal/worker/task_handlers_test.go | 10 +- 25 files changed, 449 insertions(+), 354 deletions(-) create mode 100644 internal/db/migrations/2026092101_commit_status_url.go create mode 100644 internal/db/migrations/2026092101_commit_status_url_test.go delete mode 100644 internal/storagecommit/submission.go delete mode 100644 internal/storagecommit/submission_test.go diff --git a/go.mod b/go.mod index a328d7b..acb4ce9 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/knadh/koanf/v2 v2.3.4 github.com/multiformats/go-multihash v0.2.3 github.com/prometheus/client_golang v1.23.2 - github.com/strahe/synapse-go v0.6.0-beta.2 + github.com/strahe/synapse-go v0.6.0 github.com/uptrace/bun v1.2.18 github.com/uptrace/bun/dialect/pgdialect v1.2.18 github.com/uptrace/bun/dialect/sqlitedialect v1.2.18 @@ -149,9 +149,9 @@ require ( github.com/valyala/fasthttp v1.71.0 // indirect github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - golang.org/x/sync v0.20.0 + golang.org/x/sync v0.21.0 golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/text v0.39.0 // indirect golang.org/x/time v0.15.0 // indirect modernc.org/libc v1.70.0 // indirect modernc.org/mathutil v1.7.1 // indirect diff --git a/go.sum b/go.sum index 496ae67..1f89f64 100644 --- a/go.sum +++ b/go.sum @@ -388,8 +388,8 @@ github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/strahe/synapse-go v0.6.0-beta.2 h1:EjA7wVfWJIZoWuaBOpcW7+RfufAZMNRsNIBNUQDupvw= -github.com/strahe/synapse-go v0.6.0-beta.2/go.mod h1:B6t1aDkuqfVtcJuOau1n5erDsyiAUAEs1jlHjnmBvsA= +github.com/strahe/synapse-go v0.6.0 h1:dvvcXnM1xw6TN2lXitWXlfwVn9b//NR/6S12EUMsIGg= +github.com/strahe/synapse-go v0.6.0/go.mod h1:hhAaoXJ8O1QUWVeDCwbtjZXog5qhJVyri8bZdvPSYLE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= @@ -481,8 +481,8 @@ golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGb golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY= golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -491,8 +491,8 @@ golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -511,16 +511,16 @@ golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= +golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/db/migrations/2026092101_commit_status_url.go b/internal/db/migrations/2026092101_commit_status_url.go new file mode 100644 index 0000000..fe404cf --- /dev/null +++ b/internal/db/migrations/2026092101_commit_status_url.go @@ -0,0 +1,82 @@ +package migrations + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" +) + +func init() { + Migrations.MustRegister( + transactionalMigration(up2026092101CommitStatusURL), + func(context.Context, *bun.DB) error { + return errors.New("commit status URL schema cannot be rolled back; create a new empty database") + }, + ) +} + +// Both ledgers must be empty because this development migration deliberately +// does not convert persisted submissions or their confirmed projections. +func up2026092101CommitStatusURL(ctx context.Context, db bun.IDB) error { + for _, table := range []string{"storage_commit_attempts", "storage_copies"} { + count, err := db.NewSelect().Table(table).Count(ctx) + if err != nil { + return fmt.Errorf("checking %s before commit ledger rebuild: %w", table, err) + } + if count != 0 { + return fmt.Errorf("%s is not empty: %w", table, incompatibleDatabaseError()) + } + } + if db.Dialect().Name() == dialect.PG { + if _, err := db.ExecContext(ctx, `ALTER TABLE storage_copies DROP CONSTRAINT fk_storage_copies_confirmed_attempt`); err != nil { + return fmt.Errorf("dropping commit projection foreign key: %w", err) + } + } + if _, err := db.ExecContext(ctx, `DROP TABLE storage_commit_attempts`); err != nil { + return fmt.Errorf("dropping empty commit ledger: %w", err) + } + spec := storageCommitAttemptTable2026090101() + spec.model = (*storageCommitAttempt2026092101)(nil) + spec.jsonColumns = nil + for i, constraint := range spec.constraints { + spec.constraints[i] = strings.ReplaceAll(constraint, "submission_json", "status_url") + } + if err := createInitialTable(ctx, db, spec); err != nil { + return err + } + for _, index := range storageIndexes2026090101() { + if index.table != "storage_commit_attempts" { + continue + } + if err := createInitialIndexes(ctx, db, index); err != nil { + return err + } + } + return addForwardForeignKey(ctx, db, "storage_copies", storageCopyConfirmedAttemptForeignKey2026090101()) +} + +type storageCommitAttempt2026092101 struct { + bun.BaseModel `bun:"table:storage_commit_attempts"` + + AttemptID string `bun:"type:text,pk"` + ContentID int64 `bun:",notnull"` + StorageDataSetID int64 `bun:",notnull"` + Status string `bun:"type:text,notnull,default:'reserved'"` + ExtraDataHex *string `bun:"type:text"` + TransactionID *string `bun:"type:text"` + StatusURL *string `bun:"type:text"` + ConfirmedTransactionID *string `bun:"type:text"` + AttentionCode *string `bun:"type:text"` + AttentionAt *time.Time + ReleaseReason *string `bun:"type:text"` + LastError *string `bun:"type:text"` + AttemptedAt *time.Time + ResolvedAt *time.Time + CreatedAt time.Time `bun:",notnull"` + UpdatedAt time.Time `bun:",notnull"` +} diff --git a/internal/db/migrations/2026092101_commit_status_url_test.go b/internal/db/migrations/2026092101_commit_status_url_test.go new file mode 100644 index 0000000..374f52b --- /dev/null +++ b/internal/db/migrations/2026092101_commit_status_url_test.go @@ -0,0 +1,99 @@ +package migrations + +import ( + "errors" + "slices" + "strings" + "testing" + + "github.com/uptrace/bun" +) + +func TestCommitStatusURLMigrationRebuildsEmptyLedger(t *testing.T) { + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatal(err) + } + if err := runMigrationBody(t.Context(), db, up2026092101CommitStatusURL); err != nil { + t.Fatal(err) + } + for _, column := range []string{"status_url", "submission_json"} { + exists, err := columnExists(t.Context(), db, "storage_commit_attempts", column) + if err != nil || exists != (column == "status_url") { + t.Fatalf("column %s exists=%t err=%v", column, exists, err) + } + } + for _, index := range []string{ + "idx_storage_commit_attempts_unresolved_copy", + "idx_storage_commit_attempts_unresolved_data_set", + "idx_storage_commit_attempts_copy_history", + } { + exists, err := indexExists(t.Context(), db, index) + if err != nil || !exists { + t.Fatalf("index %s exists=%t err=%v", index, exists, err) + } + } + constraints := semanticConstraintLines(t, db, applicationSchemaTables(t, db)) + for _, name := range []string{ + "chk_storage_commit_attempts_evidence_shape", + "chk_storage_commit_attempts_resolution", + "chk_storage_commit_attempts_submission", + "uq_storage_commit_attempts_status", + } { + if !slices.Contains(constraints, "constraint|storage_commit_attempts|"+name) { + t.Fatalf("commit ledger constraint %s was not restored", name) + } + } + foreignKeys := semanticForeignKeyLines(t, db, applicationSchemaTables(t, db)) + for _, column := range []string{"confirmed_attempt_id", "confirmed_attempt_status"} { + found := false + for _, line := range foreignKeys { + if strings.HasPrefix(line, "foreign-key|storage_copies|"+column+"|storage_commit_attempts|") { + found = true + break + } + } + if !found { + t.Fatalf("storage copy foreign key column %s was not restored", column) + } + } + if _, err := db.ExecContext(t.Context(), `INSERT INTO storage_commit_attempts + (attempt_id, content_id, storage_data_set_id, status, extra_data_hex, attempted_at, status_url, created_at, updated_at) + VALUES ('invalid-status', 1, 1, 'attempted', 'abcd', CURRENT_TIMESTAMP, + 'https://provider.example/status', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`); err == nil { + t.Fatal("status URL without transaction was accepted") + } + }) +} + +func TestCommitStatusURLMigrationRejectsNonemptyLedgers(t *testing.T) { + for _, populated := range []string{"attempt", "copy"} { + t.Run(populated, func(t *testing.T) { + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { + t.Fatal(err) + } + if populated == "attempt" { + if _, err := db.ExecContext(t.Context(), `INSERT INTO storage_commit_attempts + (attempt_id, content_id, storage_data_set_id, created_at, updated_at) + VALUES ('existing', 1, 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`); err != nil { + t.Fatal(err) + } + } else { + bucketID := insertBaselineTestBucketWithSlots(t, db, "migration-copy", 1) + contentID := insertBaselineTestContent(t, db, bucketID, "migration-copy") + dataSetID := insertBaselineTestDataSet(t, db, bucketID, "provider", 0, 1, true) + insertBaselineTestCopy(t, db, contentID, bucketID, dataSetID, 0, "provider", "ingress") + } + err := runMigrationBody(t.Context(), db, up2026092101CommitStatusURL) + if !errors.Is(err, ErrIncompatibleDatabase) || !strings.Contains(err.Error(), "new empty database") { + t.Fatalf("migration error = %v, want explicit empty-database refusal", err) + } + exists, err := columnExists(t.Context(), db, "storage_commit_attempts", "submission_json") + if err != nil || !exists { + t.Fatalf("rejected migration changed original ledger: exists=%t err=%v", exists, err) + } + }) + }) + } +} diff --git a/internal/db/migrations/2026999999_transaction_semantics_test.go b/internal/db/migrations/2026999999_transaction_semantics_test.go index fbfc288..d60200a 100644 --- a/internal/db/migrations/2026999999_transaction_semantics_test.go +++ b/internal/db/migrations/2026999999_transaction_semantics_test.go @@ -98,7 +98,7 @@ func TestInitialBaselineRepairsMissingMarkerOnlyForCompletePostState(t *testing. if _, err := migrator.Migrate(ctx); err != nil { t.Fatalf("repair baseline marker: %v", err) } - assertAppliedMigrationCount(t, ctx, migrator, 1) + assertAppliedMigrationCount(t, ctx, migrator, 2) }) } diff --git a/internal/db/migrations/schema_integrity_test.go b/internal/db/migrations/schema_integrity_test.go index 075439e..85f1c40 100644 --- a/internal/db/migrations/schema_integrity_test.go +++ b/internal/db/migrations/schema_integrity_test.go @@ -437,7 +437,7 @@ func TestFreshBaselineIsIdempotentAndCannotRollback(t *testing.T) { if err != nil { t.Fatalf("migrate fresh schema: %v", err) } - if len(first.Migrations) != 1 || first.Migrations[0].Name != InitialSchemaName { + if len(first.Migrations) != 2 || first.Migrations[0].Name != InitialSchemaName || first.Migrations[1].Name != "2026092101" { t.Fatalf("first migration group = %#v", first.Migrations) } second, err := migrator.Migrate(ctx) diff --git a/internal/db/migrations/schema_model_alignment_test.go b/internal/db/migrations/schema_model_alignment_test.go index cc02c9f..f1dc146 100644 --- a/internal/db/migrations/schema_model_alignment_test.go +++ b/internal/db/migrations/schema_model_alignment_test.go @@ -23,7 +23,7 @@ import ( "github.com/uptrace/bun/dialect" ) -func TestRuntimeModelsMatchAppliedBaseline(t *testing.T) { +func TestRuntimeModelsMatchAppliedMigrations(t *testing.T) { models := runtimePersistentModels() runtimeTablesFromAST := runtimePersistentModelTablesFromAST(t) @@ -44,6 +44,9 @@ func TestRuntimeModelsMatchAppliedBaseline(t *testing.T) { if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { t.Fatalf("create initial schema: %v", err) } + if err := runMigrationBody(t.Context(), db, up2026092101CommitStatusURL); err != nil { + t.Fatalf("create final commit ledger: %v", err) + } appliedTables := applicationSchemaTables(t, db) if !slices.Equal(appliedTables, registeredTables) { t.Fatalf("applied tables = %v, runtime model tables = %v", appliedTables, registeredTables) diff --git a/internal/db/repository/storage_commit_repo.go b/internal/db/repository/storage_commit_repo.go index d9fa9fb..e88c752 100644 --- a/internal/db/repository/storage_commit_repo.go +++ b/internal/db/repository/storage_commit_repo.go @@ -229,13 +229,13 @@ func (r *BunStorageContentRepo) RecordCommitSubmission(ctx context.Context, inpu res, err := db.NewUpdate(). Model((*storagecommit.Attempt)(nil)). Set("transaction_id = COALESCE(transaction_id, ?)", input.TransactionID). - Set("submission_json = COALESCE(submission_json, ?)", input.SubmissionJSON). + Set("status_url = COALESCE(status_url, ?)", input.StatusURL). Set("updated_at = ?", commitInputTime(input.Now)). Where("attempt_id = ?", input.AttemptID). Where("content_id = ? AND storage_data_set_id = ?", input.Copy.ContentID, input.Copy.StorageDataSetID). Where("status = ? AND resolved_at IS NULL", storagecommit.AttemptStatusAttempted). Where("(transaction_id IS NULL OR transaction_id = ?)", input.TransactionID). - Where("(submission_json IS NULL OR submission_json = ?)", input.SubmissionJSON). + Where("(status_url IS NULL OR status_url = ?)", input.StatusURL). Exec(ctx) if err != nil { return err @@ -323,7 +323,7 @@ func (r *BunStorageContentRepo) ReleaseCommitAttempt(ctx context.Context, input storagecommit.AttemptStatusReserved, storagecommit.AttemptStatusAttempted, })). - Where("transaction_id IS NULL AND submission_json IS NULL") + Where("transaction_id IS NULL AND status_url IS NULL") } else { q = q.Where("status = ?", storagecommit.AttemptStatusReserved) } @@ -676,7 +676,7 @@ func projectActiveCommitAttempt(q *bun.SelectQuery, copyAlias string) { ColumnExpr("active_commit_attempt.attempt_id AS commit_attempt_id"). ColumnExpr("active_commit_attempt.attempted_at AS commit_attempted_at"). ColumnExpr("active_commit_attempt.transaction_id AS commit_transaction_id"). - ColumnExpr("active_commit_attempt.submission_json AS commit_submission_json"). + ColumnExpr("active_commit_attempt.status_url AS commit_status_url"). ColumnExpr("active_commit_attempt.confirmed_transaction_id AS commit_confirmed_transaction_id"). ColumnExpr("active_commit_attempt.attention_code AS commit_attention_code"). ColumnExpr("active_commit_attempt.attention_at AS commit_attention_at"). @@ -746,7 +746,7 @@ func validateCommitEvidenceInput(input storagecommit.EvidenceInput, requireSubmi if err := validateCommitCopyIdentity(input.Copy); err != nil || input.AttemptID == "" || input.TransactionID == "" || - (requireSubmission && input.SubmissionJSON == "") { + (requireSubmission && input.StatusURL == "") { return fmt.Errorf("recording storage commit evidence: %w", ErrInvalidInput) } return nil diff --git a/internal/db/repository/storage_content_repo.go b/internal/db/repository/storage_content_repo.go index 883a830..332ff40 100644 --- a/internal/db/repository/storage_content_repo.go +++ b/internal/db/repository/storage_content_repo.go @@ -244,7 +244,7 @@ func (r *BunStorageContentRepo) ListCopies(ctx context.Context, contentID int64) active_commit_attempt.attempt_id AS commit_attempt_id, active_commit_attempt.attempted_at AS commit_attempted_at, active_commit_attempt.transaction_id AS commit_transaction_id, - active_commit_attempt.submission_json AS commit_submission_json, + active_commit_attempt.status_url AS commit_status_url, active_commit_attempt.confirmed_transaction_id AS commit_confirmed_transaction_id, active_commit_attempt.attention_code AS commit_attention_code, active_commit_attempt.attention_at AS commit_attention_at diff --git a/internal/model/storage.go b/internal/model/storage.go index 94b6b37..a3c2383 100644 --- a/internal/model/storage.go +++ b/internal/model/storage.go @@ -156,7 +156,7 @@ type StorageCopy struct { CommitAttemptID *string `bun:",scanonly"` CommitAttemptedAt *time.Time `bun:",scanonly"` CommitTransactionID *string `bun:",scanonly"` - CommitSubmissionJSON *string `bun:",scanonly"` + CommitStatusURL *string `bun:",scanonly"` CommitConfirmedTransactionID *string `bun:",scanonly"` CommitAttentionCode *string `bun:",scanonly"` CommitAttentionAt *time.Time `bun:",scanonly"` diff --git a/internal/storagecommit/advancer.go b/internal/storagecommit/advancer.go index 9a8e4c7..8b78cbd 100644 --- a/internal/storagecommit/advancer.go +++ b/internal/storagecommit/advancer.go @@ -200,13 +200,14 @@ func (a *Advancer) submitReserved( submission, submitErr := input.Target.SubmitCommit(ctx, storage.CommitRequest{ Pieces: input.Pieces, ExtraData: extraData, - OnSubmitted: func(transactionID string) { + OnSubmitted: func(submission storage.CommitSubmission) { callbackObserved.Store(true) evidenceCtx, cancel := evidenceContext(ctx) - err := a.Store.RecordCommitTransaction(evidenceCtx, EvidenceInput{ + err := a.Store.RecordCommitSubmission(evidenceCtx, EvidenceInput{ Copy: identity, AttemptID: attemptID, - TransactionID: transactionID, + TransactionID: submission.TransactionID, + StatusURL: submission.StatusURL, Now: a.now(), }) cancel() @@ -242,17 +243,13 @@ func (a *Advancer) submitReserved( } return AdvanceResult{State: AdvancePending, AttemptID: attemptID}, nil } - submissionJSON, err := EncodeSubmission(*submission) - if err != nil { - return AdvanceResult{State: AdvancePending, AttemptID: attemptID}, err - } evidenceCtx, cancel := evidenceContext(ctx) err = a.Store.RecordCommitSubmission(evidenceCtx, EvidenceInput{ - Copy: identity, - AttemptID: attemptID, - TransactionID: submission.TransactionID, - SubmissionJSON: submissionJSON, - Now: a.now(), + Copy: identity, + AttemptID: attemptID, + TransactionID: submission.TransactionID, + StatusURL: submission.StatusURL, + Now: a.now(), }) cancel() if err != nil { @@ -275,19 +272,9 @@ func (a *Advancer) observe( if context.Cause(ctx) != nil { return AdvanceResult{State: AdvancePending, AttemptID: attemptID}, nil } - if copyRow.CommitSubmissionJSON != nil && *copyRow.CommitSubmissionJSON != "" { - submission, err := DecodeSubmission(*copyRow.CommitSubmissionJSON) - if err != nil { - // A submission we can no longer read is still confirmable through its - // recorded transaction. That evidence names the exact transaction, so - // it is safer than asking an operator to release the attempt. - if copyRow.CommitTransactionID != nil && *copyRow.CommitTransactionID != "" { - return a.observeTransaction(ctx, input, copyRow) - } - return a.attentionForCopy(ctx, identity, copyRow, attemptID, AttentionInvalidSubmission, false) - } + if copyRow.CommitStatusURL != nil && *copyRow.CommitStatusURL != "" { requestCtx, cancel := context.WithTimeout(ctx, a.requestTimeout()) - status, err := input.Target.GetCommitStatus(requestCtx, submission) + status, err := input.Target.GetCommitStatus(requestCtx, *copyRow.CommitStatusURL) cancel() if err != nil { if context.Cause(ctx) != nil { @@ -303,7 +290,7 @@ func (a *Advancer) observe( } return a.pendingOrAttention(ctx, identity, copyRow, attemptID) } - return a.classifySDKStatus(ctx, identity, copyRow, attemptID, status) + return a.classifySDKStatus(ctx, input, identity, copyRow, attemptID, status) } if copyRow.CommitTransactionID != nil && *copyRow.CommitTransactionID != "" { return a.observeTransaction(ctx, input, copyRow) @@ -395,6 +382,7 @@ func (a *Advancer) observeTransaction( func (a *Advancer) classifySDKStatus( ctx context.Context, + input AdvanceInput, identity CopyIdentity, copyRow model.StorageCopy, attemptID string, @@ -403,11 +391,17 @@ func (a *Advancer) classifySDKStatus( if status == nil { return a.pendingOrAttention(ctx, identity, copyRow, attemptID) } + ref, bound := input.Target.DataSetRef() + if !bound || status.DataSet == nil || status.Kind != storage.CommitKindAddPieces || + copyRow.CommitTransactionID == nil || status.TransactionID != *copyRow.CommitTransactionID || + !status.DataSet.Equal(ref) { + return a.attentionForCopy(ctx, identity, copyRow, attemptID, AttentionSubmissionMismatch, false) + } switch status.State { case storage.CommitStatePending: return a.pendingOrAttention(ctx, identity, copyRow, attemptID) case storage.CommitStateConfirmed: - if status.DataSet == nil { + if len(status.PieceIDs) != len(input.Pieces) { return a.attentionForCopy(ctx, identity, copyRow, attemptID, AttentionSubmissionMismatch, false) } return AdvanceResult{ diff --git a/internal/storagecommit/advancer_test.go b/internal/storagecommit/advancer_test.go index c19cebc..a0535b0 100644 --- a/internal/storagecommit/advancer_test.go +++ b/internal/storagecommit/advancer_test.go @@ -33,6 +33,7 @@ func TestAdvancerPersistsFourSubmissionsBeforeConfirmationAndAdmitsFIFO(t *testi } var presignCalls, submissionCalls, confirmationCalls int + submittedTransactions := make(map[string]string) target.PresignForCommitFunc = func(context.Context, []storage.PieceInput) ([]byte, error) { presignCalls++ return []byte{0xab, 0xcd}, nil @@ -40,24 +41,27 @@ func TestAdvancerPersistsFourSubmissionsBeforeConfirmationAndAdmitsFIFO(t *testi target.SubmitCommitFunc = func(_ context.Context, request storage.CommitRequest) (*storage.CommitSubmission, error) { submissionCalls++ tx := fmt.Sprintf("0x%064x", submissionCalls) - if request.OnSubmitted != nil { - request.OnSubmitted(tx) - } - return &storage.CommitSubmission{ + submission := storage.CommitSubmission{ Kind: storage.CommitKindAddPieces, TransactionID: tx, StatusURL: fmt.Sprintf("https://provider.example/status/%d", submissionCalls), ProviderID: binding.ProviderID.SDK(), DataSet: &dataSetRef, PieceCIDs: []cid.Cid{pieceCID}, - }, nil + } + submittedTransactions[submission.StatusURL] = tx + if request.OnSubmitted != nil { + request.OnSubmitted(submission) + } + return &submission, nil } confirm := false - target.GetCommitStatusFunc = func(_ context.Context, submission storage.CommitSubmission) (*storage.CommitStatus, error) { + target.GetCommitStatusFunc = func(_ context.Context, statusURL string) (*storage.CommitStatus, error) { confirmationCalls++ + tx := submittedTransactions[statusURL] if !confirm { - return &storage.CommitStatus{State: storage.CommitStatePending, TransactionID: submission.TransactionID}, nil + return &storage.CommitStatus{Kind: storage.CommitKindAddPieces, State: storage.CommitStatePending, TransactionID: tx, DataSet: &dataSetRef}, nil } return &storage.CommitStatus{ - State: storage.CommitStateConfirmed, TransactionID: submission.TransactionID, + Kind: storage.CommitKindAddPieces, State: storage.CommitStateConfirmed, TransactionID: tx, DataSet: &dataSetRef, PieceIDs: []sdktypes.BigInt{sdktypes.NewBigInt(5001)}, }, nil @@ -79,7 +83,7 @@ func TestAdvancerPersistsFourSubmissionsBeforeConfirmationAndAdmitsFIFO(t *testi for i := range 4 { persisted := loadAdvancerCopy(t, repos, copies[i].ID) if persisted.CommitAttemptID == nil || persisted.CommitAttemptedAt == nil || - persisted.CommitTransactionID == nil || persisted.CommitSubmissionJSON == nil { + persisted.CommitTransactionID == nil || persisted.CommitStatusURL == nil { t.Fatalf("copy %d submission evidence was not fully persisted: %#v", i, persisted) } } @@ -143,6 +147,126 @@ func TestAdvancerPersistsFourSubmissionsBeforeConfirmationAndAdmitsFIFO(t *testi } } +func TestAdvancerRejectsMismatchedCommitStatus(t *testing.T) { + for _, mismatch := range []string{"transaction", "kind", "data set", "piece count", "rejected transaction"} { + t.Run(mismatch, func(t *testing.T) { + db := testutil.NewTestDB(t) + repos := repository.NewRepositories(db) + binding, copies, pieceCID := seedAdvancerCopies(t, db, 1) + identity := advancerCopyIdentity(copies[0]) + if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ + Copy: identity, AttemptID: "status-mismatch", + }); err != nil { + t.Fatalf("reserve: %v", err) + } + if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ + Copy: identity, AttemptID: "status-mismatch", ExtraDataHex: "abcd", + }); err != nil { + t.Fatalf("mark attempted: %v", err) + } + const statusURL = "https://provider.example/status/commit" + if err := repos.Contents.RecordCommitSubmission(t.Context(), storagecommit.EvidenceInput{ + Copy: identity, AttemptID: "status-mismatch", TransactionID: "0xexpected", StatusURL: statusURL, + }); err != nil { + t.Fatalf("record status URL: %v", err) + } + target := testutil.NewMockDataSetTarget(binding.ProviderID.SDK(), binding.DataSetID.SDK(), nil) + target.ClientDataSetIDValue = sdktypes.NewBigInt(9001) + ref, ok := target.DataSetRef() + if !ok { + t.Fatal("mock target has no data set ref") + } + status := &storage.CommitStatus{ + Kind: storage.CommitKindAddPieces, State: storage.CommitStateConfirmed, + TransactionID: "0xexpected", DataSet: &ref, + PieceIDs: []sdktypes.BigInt{sdktypes.NewBigInt(5001)}, + } + switch mismatch { + case "transaction": + status.TransactionID = "0xother" + case "kind": + status.Kind = storage.CommitKindCreateAndAdd + case "data set": + other, err := storage.NewDataSetRef(binding.ProviderID.SDK(), sdktypes.NewBigInt(9999), sdktypes.NewBigInt(9001)) + if err != nil { + t.Fatal(err) + } + status.DataSet = &other + case "piece count": + status.PieceIDs = nil + case "rejected transaction": + status.State = storage.CommitStateRejected + status.TransactionID = "0xother" + } + target.GetCommitStatusFunc = func(_ context.Context, gotURL string) (*storage.CommitStatus, error) { + if gotURL != statusURL { + t.Fatalf("status URL = %q, want %q", gotURL, statusURL) + } + return status, nil + } + result, err := (&storagecommit.Advancer{Store: repos.Contents}).Advance(t.Context(), storagecommit.AdvanceInput{ + Copy: *loadAdvancerCopy(t, repos, copies[0].ID), Binding: *binding, Target: target, + Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, + }) + if err != nil || result.State != storagecommit.AdvanceNeedsAttention || + result.AttentionCode != storagecommit.AttentionSubmissionMismatch || result.Confirmation != nil { + t.Fatalf("mismatched status = %#v err=%v", result, err) + } + }) + } +} + +func TestCommitEvidenceIsAtomicAndIdempotent(t *testing.T) { + db := testutil.NewTestDB(t) + repos := repository.NewRepositories(db) + _, copies, _ := seedAdvancerCopies(t, db, 1) + identity := advancerCopyIdentity(copies[0]) + if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ + Copy: identity, AttemptID: "atomic-evidence", + }); err != nil { + t.Fatalf("reserve: %v", err) + } + if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ + Copy: identity, AttemptID: "atomic-evidence", ExtraDataHex: "abcd", + }); err != nil { + t.Fatalf("mark attempted: %v", err) + } + evidence := storagecommit.EvidenceInput{ + Copy: identity, AttemptID: "atomic-evidence", + TransactionID: "0xsubmitted", StatusURL: "https://provider.example/status/submitted", + } + withoutURL := evidence + withoutURL.StatusURL = "" + if err := repos.Contents.RecordCommitSubmission(t.Context(), withoutURL); err == nil { + t.Fatal("accepted submission evidence without status URL") + } + before := loadAdvancerCopy(t, repos, copies[0].ID) + if before.CommitTransactionID != nil || before.CommitStatusURL != nil { + t.Fatalf("invalid evidence partially persisted: %#v", before) + } + if err := repos.Contents.RecordCommitSubmission(t.Context(), evidence); err != nil { + t.Fatalf("record evidence: %v", err) + } + if err := repos.Contents.RecordCommitSubmission(t.Context(), evidence); err != nil { + t.Fatalf("idempotent replay: %v", err) + } + conflict := evidence + conflict.StatusURL = "https://provider.example/status/other" + if err := repos.Contents.RecordCommitSubmission(t.Context(), conflict); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("conflicting URL error = %v, want conflict", err) + } + conflict = evidence + conflict.TransactionID = "0xother" + if err := repos.Contents.RecordCommitSubmission(t.Context(), conflict); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("conflicting transaction error = %v, want conflict", err) + } + copyRow := loadAdvancerCopy(t, repos, copies[0].ID) + if copyRow.CommitTransactionID == nil || *copyRow.CommitTransactionID != evidence.TransactionID || + copyRow.CommitStatusURL == nil || *copyRow.CommitStatusURL != evidence.StatusURL { + t.Fatalf("conflicting writes changed evidence: %#v", copyRow) + } +} + func TestAdvancerOwnerTerminalReleasesUnattemptedReservationWithoutSDK(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) @@ -345,7 +469,7 @@ func TestAdvancerSubmissionCallbackPreventsErrorBasedReset(t *testing.T) { return []byte{0xab, 0xcd}, nil } target.SubmitCommitFunc = func(_ context.Context, request storage.CommitRequest) (*storage.CommitSubmission, error) { - request.OnSubmitted("0xcallback") + request.OnSubmitted(storage.CommitSubmission{TransactionID: "0xcallback", StatusURL: "https://provider.example/status/callback"}) return nil, storage.ErrInvalidArgument } @@ -358,7 +482,8 @@ func TestAdvancerSubmissionCallbackPreventsErrorBasedReset(t *testing.T) { } copyRow := loadAdvancerCopy(t, repos, copies[0].ID) if copyRow.CommitAttemptID == nil || copyRow.CommitAttemptedAt == nil || - copyRow.CommitTransactionID == nil || *copyRow.CommitTransactionID != "0xcallback" { + copyRow.CommitTransactionID == nil || *copyRow.CommitTransactionID != "0xcallback" || + copyRow.CommitStatusURL == nil || *copyRow.CommitStatusURL != "https://provider.example/status/callback" { t.Fatalf("callback evidence was reset: %#v", copyRow) } } @@ -377,7 +502,7 @@ func TestAdvancerSurfacesDurableSubmissionEvidenceFailure(t *testing.T) { return []byte{0xab}, nil } target.SubmitCommitFunc = func(_ context.Context, request storage.CommitRequest) (*storage.CommitSubmission, error) { - request.OnSubmitted("0xevidence") + request.OnSubmitted(storage.CommitSubmission{TransactionID: "0xevidence", StatusURL: "https://provider.example/status/evidence"}) return &storage.CommitSubmission{ Kind: storage.CommitKindAddPieces, TransactionID: "0xevidence", StatusURL: "https://provider.example/status/evidence", @@ -397,7 +522,7 @@ func TestAdvancerSurfacesDurableSubmissionEvidenceFailure(t *testing.T) { } persisted := loadAdvancerCopy(t, repos, copies[0].ID) if persisted.CommitAttemptID == nil || persisted.CommitAttemptedAt == nil || - persisted.CommitTransactionID != nil || persisted.CommitSubmissionJSON != nil { + persisted.CommitTransactionID != nil || persisted.CommitStatusURL != nil { t.Fatalf("failed evidence write lost attempt fence or invented evidence: %#v", persisted) } } @@ -418,7 +543,7 @@ func TestAdvancerUnavailableConfirmationWaitsThenRecoversFromAttention(t *testin } target.SubmitCommitFunc = func(_ context.Context, request storage.CommitRequest) (*storage.CommitSubmission, error) { const tx = "0xunavailable" - request.OnSubmitted(tx) + request.OnSubmitted(storage.CommitSubmission{TransactionID: tx, StatusURL: "https://provider.example/status/unavailable"}) return &storage.CommitSubmission{ Kind: storage.CommitKindAddPieces, TransactionID: tx, StatusURL: "https://provider.example/status/unavailable", @@ -427,12 +552,12 @@ func TestAdvancerUnavailableConfirmationWaitsThenRecoversFromAttention(t *testin }, nil } confirmed := false - target.GetCommitStatusFunc = func(context.Context, storage.CommitSubmission) (*storage.CommitStatus, error) { + target.GetCommitStatusFunc = func(context.Context, string) (*storage.CommitStatus, error) { if !confirmed { return nil, storage.ErrDataSetUnavailable } return &storage.CommitStatus{ - State: storage.CommitStateConfirmed, TransactionID: "0xunavailable", + Kind: storage.CommitKindAddPieces, State: storage.CommitStateConfirmed, TransactionID: "0xunavailable", ConfirmedTransactionID: "0xconfirmed", DataSet: &dataSetRef, PieceIDs: []sdktypes.BigInt{sdktypes.NewBigInt(5001)}, }, nil @@ -617,7 +742,7 @@ func TestAdvancerFullSubmissionInvalidStatusKeepsStableAttentionAndRecovers(t *t } target.SubmitCommitFunc = func(_ context.Context, request storage.CommitRequest) (*storage.CommitSubmission, error) { const tx = "0x0000000000000000000000000000000000000000000000000000000000000101" - request.OnSubmitted(tx) + request.OnSubmitted(storage.CommitSubmission{TransactionID: tx, StatusURL: "https://provider.example/status/invalid"}) return &storage.CommitSubmission{ Kind: storage.CommitKindAddPieces, TransactionID: tx, StatusURL: "https://provider.example/status/invalid", @@ -627,21 +752,21 @@ func TestAdvancerFullSubmissionInvalidStatusKeepsStableAttentionAndRecovers(t *t } mode := "invalid" statusCalls := 0 - target.GetCommitStatusFunc = func(_ context.Context, submission storage.CommitSubmission) (*storage.CommitStatus, error) { + target.GetCommitStatusFunc = func(_ context.Context, statusURL string) (*storage.CommitStatus, error) { statusCalls++ switch mode { case "invalid": return nil, fmt.Errorf("provider status identity: %w", pdp.ErrInvalidStatus) case "pending": - return &storage.CommitStatus{State: storage.CommitStatePending, TransactionID: submission.TransactionID}, nil + return &storage.CommitStatus{Kind: storage.CommitKindAddPieces, State: storage.CommitStatePending, TransactionID: "0x0000000000000000000000000000000000000000000000000000000000000101", DataSet: &dataSetRef}, nil case "confirmed": return &storage.CommitStatus{ - State: storage.CommitStateConfirmed, TransactionID: submission.TransactionID, + Kind: storage.CommitKindAddPieces, State: storage.CommitStateConfirmed, TransactionID: "0x0000000000000000000000000000000000000000000000000000000000000101", ConfirmedTransactionID: "0xconfirmed", DataSet: &dataSetRef, PieceIDs: []sdktypes.BigInt{sdktypes.NewBigInt(5001)}, }, nil case "rejected": - return &storage.CommitStatus{State: storage.CommitStateRejected, TransactionID: submission.TransactionID}, nil + return &storage.CommitStatus{Kind: storage.CommitKindAddPieces, State: storage.CommitStateRejected, TransactionID: "0x0000000000000000000000000000000000000000000000000000000000000101", DataSet: &dataSetRef}, nil default: t.Fatalf("unexpected status mode %q", mode) return nil, nil @@ -1278,7 +1403,7 @@ func TestAdvancerWriteBlockedDataSetReleasesAttemptWithCause(t *testing.T) { return []byte{0xab}, nil } target.SubmitCommitFunc = func(_ context.Context, request storage.CommitRequest) (*storage.CommitSubmission, error) { - request.OnSubmitted("0xwriteblocked") + request.OnSubmitted(storage.CommitSubmission{TransactionID: "0xwriteblocked", StatusURL: "https://provider.example/status/writeblocked"}) return nil, writeBlocked() } @@ -1297,71 +1422,3 @@ func TestAdvancerWriteBlockedDataSetReleasesAttemptWithCause(t *testing.T) { } }) } - -func TestAdvancerUnreadableSubmissionFallsBackToTransactionEvidence(t *testing.T) { - seed := func(t *testing.T, repos *repository.Repositories, copyRow model.StorageCopy) { - t.Helper() - identity := advancerCopyIdentity(copyRow) - if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: identity, AttemptID: "unreadable", - }); err != nil { - t.Fatalf("reserve: %v", err) - } - if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ - Copy: identity, AttemptID: "unreadable", ExtraDataHex: "abcd", - }); err != nil { - t.Fatalf("mark attempted: %v", err) - } - if err := repos.Contents.RecordCommitSubmission(t.Context(), storagecommit.EvidenceInput{ - Copy: identity, AttemptID: "unreadable", TransactionID: "0xunreadable", - // A future envelope version stands in for any submission this build can - // no longer decode, such as an SDK that renamed a field. - SubmissionJSON: `{"version":99,"submission":{}}`, - }); err != nil { - t.Fatalf("record submission: %v", err) - } - } - - t.Run("confirms through the recorded transaction", func(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - binding, copies, pieceCID := seedAdvancerCopies(t, db, 1) - seed(t, repos, copies[0]) - target := testutil.NewMockDataSetTarget(binding.ProviderID.SDK(), binding.DataSetID.SDK(), nil) - target.ClientDataSetIDValue = sdktypes.NewBigInt(9001) - target.GetCommitStatusFunc = func(context.Context, storage.CommitSubmission) (*storage.CommitStatus, error) { - t.Fatal("unreadable submission reached GetCommitStatus") - return nil, nil - } - checker := commitStatusCheckerFunc(func(context.Context, synapse.AddPiecesStatusInput) (synapse.PDPStatusResult, error) { - return synapse.PDPStatusResult{ - State: synapse.PDPStatusConfirmed, ConfirmedPieceIDs: []string{"5001"}, - ConfirmedTransactionID: "0xconfirmedtx", - }, nil - }) - - result, err := (&storagecommit.Advancer{Store: repos.Contents, StatusChecker: checker}).Advance( - t.Context(), storagecommit.AdvanceInput{ - Copy: *loadAdvancerCopy(t, repos, copies[0].ID), Binding: *binding, Target: target, - Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, - }) - if err != nil || result.State != storagecommit.AdvanceConfirmed || result.Confirmation == nil || - result.Confirmation.ConfirmedTransactionID != "0xconfirmedtx" { - t.Fatalf("advance = %#v err=%v, want confirmation from transaction evidence", result, err) - } - }) - - t.Run("rejects submission without transaction evidence", func(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - _, copies, _ := seedAdvancerCopies(t, db, 1) - seed(t, repos, copies[0]) - if _, err := db.NewUpdate(). - Model((*storagecommit.Attempt)(nil)). - Set("transaction_id = NULL"). - Where("attempt_id = ?", "unreadable"). - Exec(t.Context()); err == nil { - t.Fatal("commit ledger accepted submission evidence without a transaction") - } - }) -} diff --git a/internal/storagecommit/attempt.go b/internal/storagecommit/attempt.go index ceaf999..fc2b7fb 100644 --- a/internal/storagecommit/attempt.go +++ b/internal/storagecommit/attempt.go @@ -30,7 +30,7 @@ type Attempt struct { Status AttemptStatus `bun:"status,type:text,notnull,default:'reserved'"` ExtraDataHex *string `bun:"extra_data_hex,type:text,nullzero"` TransactionID *string `bun:"transaction_id,type:text,nullzero"` - SubmissionJSON *string `bun:"submission_json,type:text,nullzero"` + StatusURL *string `bun:"status_url,type:text,nullzero"` ConfirmedTransactionID *string `bun:"confirmed_transaction_id,type:text,nullzero"` AttentionCode *string `bun:"attention_code,type:text,nullzero"` AttentionAt *time.Time `bun:"attention_at,nullzero"` diff --git a/internal/storagecommit/store.go b/internal/storagecommit/store.go index 34a8ffb..339b0ae 100644 --- a/internal/storagecommit/store.go +++ b/internal/storagecommit/store.go @@ -53,11 +53,11 @@ type AttemptResult struct { } type EvidenceInput struct { - Copy CopyIdentity - AttemptID string - TransactionID string - SubmissionJSON string - Now time.Time + Copy CopyIdentity + AttemptID string + TransactionID string + StatusURL string + Now time.Time } type AttentionInput struct { diff --git a/internal/storagecommit/submission.go b/internal/storagecommit/submission.go deleted file mode 100644 index befc612..0000000 --- a/internal/storagecommit/submission.go +++ /dev/null @@ -1,57 +0,0 @@ -package storagecommit - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io" - - "github.com/strahe/synapse-go/storage" -) - -const submissionEnvelopeVersion = 1 - -type submissionEnvelope struct { - Version int `json:"version"` - Submission storage.CommitSubmission `json:"submission"` -} - -func EncodeSubmission(submission storage.CommitSubmission) (string, error) { - payload, err := json.Marshal(submissionEnvelope{ - Version: submissionEnvelopeVersion, - Submission: submission, - }) - if err != nil { - return "", fmt.Errorf("encoding storage commit submission: %w", err) - } - return string(payload), nil -} - -func DecodeSubmission(value string) (storage.CommitSubmission, error) { - var envelope submissionEnvelope - decoder := json.NewDecoder(bytes.NewBufferString(value)) - decoder.DisallowUnknownFields() - if err := decoder.Decode(&envelope); err != nil { - return storage.CommitSubmission{}, fmt.Errorf("decoding storage commit submission: %w", err) - } - if err := ensureSubmissionJSONEOF(decoder); err != nil { - return storage.CommitSubmission{}, err - } - if envelope.Version != submissionEnvelopeVersion { - return storage.CommitSubmission{}, fmt.Errorf("unsupported storage commit submission version %d", envelope.Version) - } - return envelope.Submission, nil -} - -func ensureSubmissionJSONEOF(decoder *json.Decoder) error { - var trailing any - err := decoder.Decode(&trailing) - if errors.Is(err, io.EOF) { - return nil - } - if err != nil { - return fmt.Errorf("decoding trailing storage commit submission data: %w", err) - } - return errors.New("storage commit submission contains trailing data") -} diff --git a/internal/storagecommit/submission_test.go b/internal/storagecommit/submission_test.go deleted file mode 100644 index e1536df..0000000 --- a/internal/storagecommit/submission_test.go +++ /dev/null @@ -1,93 +0,0 @@ -package storagecommit - -import ( - "strings" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ipfs/go-cid" - "github.com/multiformats/go-multihash" - "github.com/strahe/synapse-go/storage" - sdktypes "github.com/strahe/synapse-go/types" -) - -func TestSubmissionEnvelopeRoundTrip(t *testing.T) { - submission := testCommitSubmission(t) - - encoded, err := EncodeSubmission(submission) - if err != nil { - t.Fatalf("EncodeSubmission: %v", err) - } - decoded, err := DecodeSubmission(encoded) - if err != nil { - t.Fatalf("DecodeSubmission: %v", err) - } - if decoded.Kind != submission.Kind || decoded.TransactionID != submission.TransactionID || decoded.StatusURL != submission.StatusURL { - t.Fatalf("decoded submission = %#v, want %#v", decoded, submission) - } -} - -func TestSubmissionEnvelopeDecodesBeta1Fixture(t *testing.T) { - const beta1Envelope = `{"version":1,"submission":{"kind":"add-pieces","transactionId":"0x0000000000000000000000000000000000000000000000000000000000000011","statusUrl":"https://provider.example/commit/0x0000000000000000000000000000000000000000000000000000000000000011","providerId":"1","identity":{"payer":"0x0000000000000000000000000000000000001234","chainId":314159,"recordKeeper":"0x0000000000000000000000000000000000005678"},"dataSet":{"providerId":"1","dataSetId":"42","clientDataSetId":"7"},"clientDataSetId":null,"pieceCids":[{"/":"bafkzcibcaaces3nobte6ezpp4wqan2age2s5yxcatzotcvobhgcmv5wi2xh5mbi"}]}}` - - decoded, err := DecodeSubmission(beta1Envelope) - if err != nil { - t.Fatalf("DecodeSubmission beta.1 fixture: %v", err) - } - if decoded.Kind != storage.CommitKindAddPieces || decoded.TransactionID != "0x0000000000000000000000000000000000000000000000000000000000000011" { - t.Fatalf("decoded beta.1 submission = %#v", decoded) - } - if decoded.ProviderID.String() != "1" || decoded.Identity.ChainID != sdktypes.ChainID(314159) || decoded.Identity.Payer != common.HexToAddress("0x1234") { - t.Fatalf("decoded beta.1 identity = provider %s, chain %d", decoded.ProviderID.String(), decoded.Identity.ChainID) - } - if decoded.DataSet == nil || decoded.DataSet.ProviderID().String() != "1" || decoded.DataSet.DataSetID().String() != "42" || decoded.DataSet.ClientDataSetID().String() != "7" { - t.Fatalf("decoded beta.1 data set = %#v", decoded.DataSet) - } - if decoded.ClientDataSetID != nil || len(decoded.PieceCIDs) != 1 || decoded.PieceCIDs[0].String() != "bafkzcibcaaces3nobte6ezpp4wqan2age2s5yxcatzotcvobhgcmv5wi2xh5mbi" { - t.Fatalf("decoded beta.1 recovery fields = %#v", decoded) - } -} - -func TestSubmissionEnvelopeRejectsUnsupportedOrMalformedData(t *testing.T) { - encoded, err := EncodeSubmission(testCommitSubmission(t)) - if err != nil { - t.Fatalf("EncodeSubmission: %v", err) - } - tests := []struct { - name string - value string - want string - }{ - {name: "unsupported version", value: strings.Replace(encoded, `"version":1`, `"version":2`, 1), want: "unsupported storage commit submission version 2"}, - {name: "unknown envelope field", value: strings.TrimSuffix(encoded, "}") + `,"extra":true}`, want: "unknown field"}, - {name: "trailing data", value: encoded + ` {}`, want: "trailing data"}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - _, err := DecodeSubmission(test.value) - if err == nil || !strings.Contains(err.Error(), test.want) { - t.Fatalf("DecodeSubmission error = %v, want %q", err, test.want) - } - }) - } -} - -func testCommitSubmission(t *testing.T) storage.CommitSubmission { - t.Helper() - mh, err := multihash.Sum([]byte("submission-envelope-test"), multihash.SHA2_256, -1) - if err != nil { - t.Fatalf("create multihash: %v", err) - } - return storage.CommitSubmission{ - Kind: storage.CommitKindAddPieces, - TransactionID: "0xcommit", - StatusURL: "https://provider.example/commit/0xcommit", - ProviderID: sdktypes.NewBigInt(1), - Identity: storage.ContextIdentity{ - Payer: common.HexToAddress("0x1234"), - ChainID: sdktypes.ChainID(314159), - RecordKeeper: common.HexToAddress("0x5678"), - }, - PieceCIDs: []cid.Cid{cid.NewCidV1(cid.Raw, mh)}, - } -} diff --git a/internal/synapse/interfaces.go b/internal/synapse/interfaces.go index 5834398..8de034e 100644 --- a/internal/synapse/interfaces.go +++ b/internal/synapse/interfaces.go @@ -28,7 +28,7 @@ type StorageTarget interface { type ProviderTarget interface { StorageTarget CreateDataSet(context.Context, *storage.CreateDataSetOptions) (*storage.CreateDataSetResult, error) - WaitForDataSetCreated(context.Context, storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) + WaitForDataSetCreated(context.Context, string, sdktypes.BigInt) (*storage.CreateDataSetResult, error) // ContextIdentity is the payer, chain, and record keeper the target signs // for. Client data set IDs are only unique within it. ContextIdentity() storage.ContextIdentity @@ -44,7 +44,7 @@ type DataSetTarget interface { PresignForCommit(context.Context, []storage.PieceInput) ([]byte, error) Pull(context.Context, storage.PullRequest) (*storage.PullResult, error) SubmitCommit(context.Context, storage.CommitRequest) (*storage.CommitSubmission, error) - GetCommitStatus(context.Context, storage.CommitSubmission) (*storage.CommitStatus, error) + GetCommitStatus(context.Context, string) (*storage.CommitStatus, error) PieceStatus(context.Context, cid.Cid) (*storage.PieceStatus, error) } diff --git a/internal/synapse/pdp_status.go b/internal/synapse/pdp_status.go index 3bea9c4..79bf921 100644 --- a/internal/synapse/pdp_status.go +++ b/internal/synapse/pdp_status.go @@ -103,6 +103,9 @@ func (c *PDPStatusChecker) CheckDataSetCreationStatus(ctx context.Context, input } status, err := client.GetDataSetCreationStatus(ctx, statusURL) if status == nil { + if errors.Is(err, pdp.ErrInvalidStatus) { + return result.withError(PDPStatusMismatch, err.Error()) + } if err == nil { err = errors.New("empty data set creation status") } @@ -163,6 +166,9 @@ func (c *PDPStatusChecker) GetAddPiecesStatus(ctx context.Context, input AddPiec } status, err := client.GetAddPiecesStatus(ctx, statusURL) if status == nil { + if errors.Is(err, pdp.ErrInvalidStatus) { + result.State = PDPStatusMismatch + } if err == nil { err = errors.New("empty add-pieces status") } diff --git a/internal/synapse/pdp_status_test.go b/internal/synapse/pdp_status_test.go index 29532e9..7f44ba3 100644 --- a/internal/synapse/pdp_status_test.go +++ b/internal/synapse/pdp_status_test.go @@ -76,7 +76,7 @@ func TestPDPStatusCheckerChecksDataSetCreationStatusOnce(t *testing.T) { var requests int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests++ - if r.URL.Path != "/pdp/data-sets/created/0xabc" { + if r.URL.Path != "/pdp/data-sets/created/"+testCreateDataSetTxHash { t.Fatalf("path = %q, want creation status path", r.URL.Path) } _, _ = fmt.Fprintf(w, `{"createMessageHash":%q,"service":"svc","txStatus":"pending","dataSetCreated":false,"ok":null}`, testCreateDataSetTxHash) @@ -85,7 +85,7 @@ func TestPDPStatusCheckerChecksDataSetCreationStatusOnce(t *testing.T) { checker := NewPDPStatusChecker(PDPStatusCheckerOptions{Timeout: time.Second, AllowPrivateNetworks: true}) got := checker.CheckDataSetCreationStatus(t.Context(), DataSetCreationStatusInput{ - StatusURL: server.URL + "/pdp/data-sets/created/0xabc", + StatusURL: server.URL + "/pdp/data-sets/created/" + testCreateDataSetTxHash, TransactionID: testCreateDataSetTxHash, }) @@ -215,7 +215,7 @@ func TestPDPStatusCheckerClassifiesSDKRejectedCreationStatuses(t *testing.T) { checker := NewPDPStatusChecker(PDPStatusCheckerOptions{Timeout: time.Second, AllowPrivateNetworks: true}) got := checker.CheckDataSetCreationStatus(t.Context(), DataSetCreationStatusInput{ - StatusURL: server.URL + "/pdp/data-sets/created/0xabc", + StatusURL: server.URL + "/pdp/data-sets/created/" + testCreateDataSetTxHash, TransactionID: testCreateDataSetTxHash, }) diff --git a/internal/synapse/storage_client.go b/internal/synapse/storage_client.go index 52e58cd..3a3ae07 100644 --- a/internal/synapse/storage_client.go +++ b/internal/synapse/storage_client.go @@ -222,8 +222,8 @@ func (c *providerTargetAdapter) CreateDataSet(ctx context.Context, opts *storage return result, NormalizeProviderOperationError(ctx, err) } -func (c *providerTargetAdapter) WaitForDataSetCreated(ctx context.Context, submission storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) { - result, err := c.provider.WaitForDataSetCreated(ctx, submission) +func (c *providerTargetAdapter) WaitForDataSetCreated(ctx context.Context, statusURL string, clientDataSetID sdktypes.BigInt) (*storage.CreateDataSetResult, error) { + result, err := c.provider.WaitForDataSetCreated(ctx, statusURL, clientDataSetID) return result, NormalizeProviderOperationError(ctx, err) } @@ -272,8 +272,8 @@ func (c *dataSetTargetAdapter) SubmitCommit(ctx context.Context, request storage return result, NormalizeProviderOperationError(ctx, err) } -func (c *dataSetTargetAdapter) GetCommitStatus(ctx context.Context, submission storage.CommitSubmission) (*storage.CommitStatus, error) { - result, err := c.dataSet.GetCommitStatus(ctx, submission) +func (c *dataSetTargetAdapter) GetCommitStatus(ctx context.Context, statusURL string) (*storage.CommitStatus, error) { + result, err := c.dataSet.GetCommitStatus(ctx, statusURL) return result, NormalizeProviderOperationError(ctx, err) } diff --git a/internal/systemtest/filecoin.go b/internal/systemtest/filecoin.go index 59a8748..2260045 100644 --- a/internal/systemtest/filecoin.go +++ b/internal/systemtest/filecoin.go @@ -11,6 +11,7 @@ import ( "maps" "math/big" "slices" + "strings" "sync" "time" @@ -48,13 +49,14 @@ type memoryPiece struct { type MemoryFilecoin struct { mu sync.RWMutex - providers []sdktypes.BigInt - dataSets map[string]*memoryDataSet - pendingDataSets map[string]sdktypes.BigInt - submissions map[string]sdktypes.BigInt - pieces map[string]*memoryPiece - nextDataSet map[string]uint64 - nextPiece uint64 + providers []sdktypes.BigInt + dataSets map[string]*memoryDataSet + pendingDataSets map[string]sdktypes.BigInt + submissions map[string]sdktypes.BigInt + commitSubmissions map[string]storage.CommitSubmission + pieces map[string]*memoryPiece + nextDataSet map[string]uint64 + nextPiece uint64 // terminated records the epoch at which each data set's service ends, and // epoch is the observed chain head. Tests advance the head to prove that // retirement waits for the chain rather than for the call returning. @@ -81,14 +83,15 @@ func NewMemoryFilecoin() *MemoryFilecoin { sdktypes.NewBigInt(103), sdktypes.NewBigInt(104), }, - dataSets: make(map[string]*memoryDataSet), - pendingDataSets: make(map[string]sdktypes.BigInt), - submissions: make(map[string]sdktypes.BigInt), - pieces: make(map[string]*memoryPiece), - nextDataSet: make(map[string]uint64), - nextPiece: 1, - terminated: make(map[string]int64), - epoch: 1000, + dataSets: make(map[string]*memoryDataSet), + pendingDataSets: make(map[string]sdktypes.BigInt), + submissions: make(map[string]sdktypes.BigInt), + commitSubmissions: make(map[string]storage.CommitSubmission), + pieces: make(map[string]*memoryPiece), + nextDataSet: make(map[string]uint64), + nextPiece: 1, + terminated: make(map[string]int64), + epoch: 1000, } } @@ -351,7 +354,7 @@ func (c *memoryProviderTarget) CreateDataSet(ctx context.Context, opts *storage. submission := storage.CreateDataSetSubmission{ ProviderID: c.provider.Copy(), TransactionID: txID, - StatusURL: c.ServiceURL() + "/status/" + txID, ClientDataSetID: copyBigIntPtr(clientID), + StatusURL: c.ServiceURL() + "/status/" + txID, ClientDataSetID: clientID.Copy(), } if opts != nil && opts.OnSubmitted != nil { opts.OnSubmitted(submission) @@ -386,29 +389,26 @@ func (c *memoryProviderTarget) FindDataSetByClientDataSetID(ctx context.Context, return storage.DataSetRef{}, false, nil } -func (c *memoryProviderTarget) WaitForDataSetCreated(ctx context.Context, submission storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) { +func (c *memoryProviderTarget) WaitForDataSetCreated(ctx context.Context, statusURL string, clientDataSetID sdktypes.BigInt) (*storage.CreateDataSetResult, error) { if err := ctx.Err(); err != nil { return nil, err } - if !submission.ProviderID.Equal(c.provider) { - return nil, fmt.Errorf("%w: submission provider %s does not match target provider %s", - errInvalidFilecoinSequence, submission.ProviderID.String(), c.provider.String()) - } - if submission.ClientDataSetID == nil { - return nil, fmt.Errorf("%w: submission is missing client data set ID", errInvalidFilecoinSequence) + txID, ok := strings.CutPrefix(statusURL, c.ServiceURL()+"/status/") + if !ok || txID == "" { + return nil, fmt.Errorf("%w: invalid dataset status URL %q", errInvalidFilecoinSequence, statusURL) } c.filecoin.mu.RLock() - id, ok := c.filecoin.submissions[submission.TransactionID] + id, ok := c.filecoin.submissions[txID] dataSet := c.filecoin.dataSets[id.String()] c.filecoin.mu.RUnlock() - if !ok || dataSet == nil || !dataSet.provider.Equal(c.provider) { - return nil, fmt.Errorf("%w: unknown dataset submission %q", errInvalidFilecoinSequence, submission.TransactionID) + if !ok || dataSet == nil || !dataSet.provider.Equal(c.provider) || !dataSet.clientID.Equal(clientDataSetID) { + return nil, fmt.Errorf("%w: unknown dataset submission %q", errInvalidFilecoinSequence, txID) } ref, err := storage.NewDataSetRef(c.provider, id, dataSet.clientID) if err != nil { return nil, err } - return &storage.CreateDataSetResult{TransactionID: submission.TransactionID, DataSet: ref}, nil + return &storage.CreateDataSetResult{TransactionID: txID, DataSet: ref}, nil } func (c *memoryDataSetTarget) Store(ctx context.Context, reader io.Reader, opts *storage.StoreOptions) (*storage.StoreResult, error) { @@ -534,23 +534,30 @@ func (c *memoryDataSetTarget) SubmitCommit(ctx context.Context, request storage. } txID := fmt.Sprintf("commit-%s-%s", dataSetID.String(), pieceIDs[0].String()) c.filecoin.mu.Unlock() - if request.OnSubmitted != nil { - request.OnSubmitted(txID) - } - return &storage.CommitSubmission{ + submission := storage.CommitSubmission{ Kind: storage.CommitKindAddPieces, TransactionID: txID, StatusURL: c.ServiceURL() + "/status/" + txID, ProviderID: c.provider.Copy(), DataSet: &c.ref, PieceCIDs: append([]cid.Cid(nil), pieceCIDs(request.Pieces)...), - }, nil + } + c.filecoin.mu.Lock() + c.filecoin.commitSubmissions[submission.StatusURL] = submission + c.filecoin.mu.Unlock() + if request.OnSubmitted != nil { + request.OnSubmitted(submission) + } + return &submission, nil } -func (c *memoryDataSetTarget) GetCommitStatus(ctx context.Context, submission storage.CommitSubmission) (*storage.CommitStatus, error) { +func (c *memoryDataSetTarget) GetCommitStatus(ctx context.Context, statusURL string) (*storage.CommitStatus, error) { if err := ctx.Err(); err != nil { return nil, err } + c.filecoin.mu.RLock() + submission, found := c.filecoin.commitSubmissions[statusURL] + c.filecoin.mu.RUnlock() dataSetID := c.DataSetID() - if dataSetID == nil || submission.Kind != storage.CommitKindAddPieces || submission.DataSet == nil || + if !found || dataSetID == nil || submission.Kind != storage.CommitKindAddPieces || submission.DataSet == nil || !submission.ProviderID.Equal(c.provider) || !submission.DataSet.DataSetID().Equal(*dataSetID) || len(submission.PieceCIDs) == 0 { return nil, fmt.Errorf("%w: commit submission does not match target", errInvalidFilecoinSequence) } diff --git a/internal/systemtest/filecoin_test.go b/internal/systemtest/filecoin_test.go index 5ed487e..1ce32ff 100644 --- a/internal/systemtest/filecoin_test.go +++ b/internal/systemtest/filecoin_test.go @@ -89,7 +89,7 @@ func TestMemoryFilecoinLifecycleAndProviderIsolation(t *testing.T) { if err != nil { t.Fatalf("SubmitCommit provider %s: %v", uploadContext.ProviderID().String(), err) } - status, err := uploadContext.GetCommitStatus(ctx, *submission) + status, err := uploadContext.GetCommitStatus(ctx, submission.StatusURL) if err != nil || status == nil || status.State != storage.CommitStateConfirmed || len(status.PieceIDs) != 1 { t.Fatalf("GetCommitStatus provider %s: status=%#v err=%v", uploadContext.ProviderID().String(), status, err) } diff --git a/internal/testutil/mocks.go b/internal/testutil/mocks.go index b3f8707..a3cdd08 100644 --- a/internal/testutil/mocks.go +++ b/internal/testutil/mocks.go @@ -188,7 +188,7 @@ type MockStorageTarget struct { ServiceURLValue string WithCDNValue bool CreateDataSetFunc func(context.Context, *storage.CreateDataSetOptions) (*storage.CreateDataSetResult, error) - WaitDataSetFunc func(context.Context, storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) + WaitDataSetFunc func(context.Context, string, sdktypes.BigInt) (*storage.CreateDataSetResult, error) // ContextIdentityValue overrides DefaultContextIdentity. ContextIdentityValue storage.ContextIdentity FindDataSetByClientIDFunc func(context.Context, sdktypes.BigInt) (storage.DataSetRef, bool, error) @@ -196,7 +196,7 @@ type MockStorageTarget struct { PresignForCommitFunc func(context.Context, []storage.PieceInput) ([]byte, error) PullFunc func(context.Context, storage.PullRequest) (*storage.PullResult, error) SubmitCommitFunc func(context.Context, storage.CommitRequest) (*storage.CommitSubmission, error) - GetCommitStatusFunc func(context.Context, storage.CommitSubmission) (*storage.CommitStatus, error) + GetCommitStatusFunc func(context.Context, string) (*storage.CommitStatus, error) PieceStatusFunc func(context.Context, cid.Cid) (*storage.PieceStatus, error) } @@ -256,9 +256,9 @@ func (m *MockStorageTarget) CreateDataSet(ctx context.Context, opts *storage.Cre return nil, errors.New("MockStorageTarget.CreateDataSet not configured") } -func (m *MockStorageTarget) WaitForDataSetCreated(ctx context.Context, submission storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) { +func (m *MockStorageTarget) WaitForDataSetCreated(ctx context.Context, statusURL string, clientDataSetID sdktypes.BigInt) (*storage.CreateDataSetResult, error) { if m.WaitDataSetFunc != nil { - return m.WaitDataSetFunc(ctx, submission) + return m.WaitDataSetFunc(ctx, statusURL, clientDataSetID) } return nil, errors.New("MockStorageTarget.WaitForDataSetCreated not configured") } @@ -313,9 +313,9 @@ func (m *MockStorageTarget) SubmitCommit(ctx context.Context, request storage.Co return nil, errors.New("MockStorageTarget.SubmitCommit not configured") } -func (m *MockStorageTarget) GetCommitStatus(ctx context.Context, submission storage.CommitSubmission) (*storage.CommitStatus, error) { +func (m *MockStorageTarget) GetCommitStatus(ctx context.Context, statusURL string) (*storage.CommitStatus, error) { if m.GetCommitStatusFunc != nil { - return m.GetCommitStatusFunc(ctx, submission) + return m.GetCommitStatusFunc(ctx, statusURL) } return nil, errors.New("MockStorageTarget.GetCommitStatus not configured") } diff --git a/internal/worker/storage_task_handlers.go b/internal/worker/storage_task_handlers.go index 27a7b3b..1ed76d7 100644 --- a/internal/worker/storage_task_handlers.go +++ b/internal/worker/storage_task_handlers.go @@ -633,10 +633,7 @@ func (h *TaskHandlers) waitDataSetCreation( checkpoint dataSetCreationCheckpoint, clientDataSetID sdktypes.BigInt, ) taskengine.Result { - result, err := provider.WaitForDataSetCreated(ctx, storage.CreateDataSetSubmission{ - ProviderID: binding.ProviderID.SDK(), TransactionID: checkpoint.TransactionID, - StatusURL: checkpoint.StatusURL, ClientDataSetID: &clientDataSetID, - }) + result, err := provider.WaitForDataSetCreated(ctx, checkpoint.StatusURL, clientDataSetID) if err != nil { if errors.Is(err, synapse.ErrProviderTransactionRejected) { if checkpoint.Sends > 1 { diff --git a/internal/worker/task_handlers_test.go b/internal/worker/task_handlers_test.go index fed4864..c727550 100644 --- a/internal/worker/task_handlers_test.go +++ b/internal/worker/task_handlers_test.go @@ -3764,7 +3764,7 @@ func TestDataSetEnsureReleasesTheProviderWhenCreationIsRejected(t *testing.T) { providerID := testOnChainID(t, 26000+sequence) target := &testutil.MockStorageTarget{ ProviderIDValue: providerID.SDK(), - WaitDataSetFunc: func(context.Context, storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) { + WaitDataSetFunc: func(context.Context, string, sdktypes.BigInt) (*storage.CreateDataSetResult, error) { return nil, pdp.ErrTxRejected }, } @@ -3935,13 +3935,13 @@ func TestDataSetCreationWithUnobservedOutcomeResendsTheSameID(t *testing.T) { } opts.OnSubmitted(storage.CreateDataSetSubmission{ ProviderID: providerID.SDK(), TransactionID: "0x" + strings.Repeat("ab", 32), - StatusURL: "https://provider.example/pdp/data-sets/created/2", ClientDataSetID: opts.ClientDataSetID, + StatusURL: "https://provider.example/pdp/data-sets/created/2", ClientDataSetID: opts.ClientDataSetID.Copy(), }) <-ctx.Done() return nil, ctx.Err() }, // The resent request loses to the first one, which did land after all. - WaitDataSetFunc: func(context.Context, storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) { + WaitDataSetFunc: func(context.Context, string, sdktypes.BigInt) (*storage.CreateDataSetResult, error) { landed.Store(true) return nil, pdp.ErrTxRejected }, @@ -4070,7 +4070,7 @@ func TestDataSetCreationRecoveryStopsOnChangedIdentityOrConflict(t *testing.T) { sends.Add(1) return nil, errors.New("unexpected create request") }, - WaitDataSetFunc: func(context.Context, storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) { + WaitDataSetFunc: func(context.Context, string, sdktypes.BigInt) (*storage.CreateDataSetResult, error) { waits.Add(1) // What the provider would report: the data set the original // wallet's request created. @@ -4291,7 +4291,7 @@ func TestDataSetCreationLooksUpARejectedResendInsteadOfGivingUp(t *testing.T) { var lookups atomic.Int64 target := &testutil.MockStorageTarget{ ProviderIDValue: providerID.SDK(), - WaitDataSetFunc: func(context.Context, storage.CreateDataSetSubmission) (*storage.CreateDataSetResult, error) { + WaitDataSetFunc: func(context.Context, string, sdktypes.BigInt) (*storage.CreateDataSetResult, error) { return nil, fmt.Errorf("submission: %w", synapse.ErrProviderTransactionRejected) }, FindDataSetByClientIDFunc: func(context.Context, sdktypes.BigInt) (storage.DataSetRef, bool, error) { From 34dbf311ade3cd97a96db98bbfb4f528629c6c13 Mon Sep 17 00:00:00 2001 From: Lee <7932644+strahe@users.noreply.github.com> Date: Mon, 21 Sep 2026 23:59:07 +0800 Subject: [PATCH 3/5] fix(db): fold commit status URL into baseline --- .../migrations/2026090101_initial_schema.go | 15 ++- .../2026092101_commit_status_url.go | 82 --------------- .../2026092101_commit_status_url_test.go | 99 ------------------- .../2026999999_transaction_semantics_test.go | 2 +- internal/db/migrations/migrations.go | 31 +++++- .../db/migrations/schema_behavior_test.go | 2 +- internal/db/migrations/schema_builder.go | 2 - .../db/migrations/schema_integrity_test.go | 28 +++++- .../migrations/schema_model_alignment_test.go | 5 +- 9 files changed, 64 insertions(+), 202 deletions(-) delete mode 100644 internal/db/migrations/2026092101_commit_status_url.go delete mode 100644 internal/db/migrations/2026092101_commit_status_url_test.go diff --git a/internal/db/migrations/2026090101_initial_schema.go b/internal/db/migrations/2026090101_initial_schema.go index e6f196a..ce47ddb 100644 --- a/internal/db/migrations/2026090101_initial_schema.go +++ b/internal/db/migrations/2026090101_initial_schema.go @@ -571,7 +571,7 @@ type storageCommitAttempt2026090101 struct { Status string `bun:"type:text,notnull,default:'reserved'"` ExtraDataHex *string `bun:"type:text"` TransactionID *string `bun:"type:text"` - SubmissionJSON *string `bun:"type:text"` + StatusURL *string `bun:"type:text"` ConfirmedTransactionID *string `bun:"type:text"` AttentionCode *string `bun:"type:text"` AttentionAt *time.Time @@ -826,23 +826,22 @@ func storageCopyConfirmedAttemptForeignKey2026090101() initialForwardForeignKey func storageCommitAttemptTable2026090101() initialTableSpec { return initialTableSpec{ - name: "storage_commit_attempts", - jsonColumns: initialJSONColumns("storage_commit_attempts"), - model: (*storageCommitAttempt2026090101)(nil), + name: "storage_commit_attempts", + model: (*storageCommitAttempt2026090101)(nil), constraints: []string{ - "CONSTRAINT chk_storage_commit_attempts_identity CHECK (attempt_id <> '' AND (extra_data_hex IS NULL OR extra_data_hex <> '') AND (transaction_id IS NULL OR transaction_id <> '') AND (submission_json IS NULL OR submission_json <> '') AND (confirmed_transaction_id IS NULL OR confirmed_transaction_id <> '') AND (attention_code IS NULL OR attention_code <> '') AND (release_reason IS NULL OR release_reason <> ''))", + "CONSTRAINT chk_storage_commit_attempts_identity CHECK (attempt_id <> '' AND (extra_data_hex IS NULL OR extra_data_hex <> '') AND (transaction_id IS NULL OR transaction_id <> '') AND (status_url IS NULL OR status_url <> '') AND (confirmed_transaction_id IS NULL OR confirmed_transaction_id <> '') AND (attention_code IS NULL OR attention_code <> '') AND (release_reason IS NULL OR release_reason <> ''))", "CONSTRAINT chk_storage_commit_attempts_status CHECK (status IN ('reserved', 'attempted', 'confirmed', 'released', 'rejected'))", // Candidate key for the copy's confirmed-attempt projection. "CONSTRAINT uq_storage_commit_attempts_status UNIQUE (attempt_id, status)", "CONSTRAINT chk_storage_commit_attempts_resolution CHECK ((status IN ('reserved', 'attempted') AND resolved_at IS NULL) OR (status IN ('confirmed', 'released', 'rejected') AND resolved_at IS NOT NULL))", `CONSTRAINT chk_storage_commit_attempts_evidence_shape CHECK ( - (status = 'reserved' AND attempted_at IS NULL AND extra_data_hex IS NULL AND transaction_id IS NULL AND submission_json IS NULL AND confirmed_transaction_id IS NULL AND attention_code IS NULL AND attention_at IS NULL AND last_error IS NULL) + (status = 'reserved' AND attempted_at IS NULL AND extra_data_hex IS NULL AND transaction_id IS NULL AND status_url IS NULL AND confirmed_transaction_id IS NULL AND attention_code IS NULL AND attention_at IS NULL AND last_error IS NULL) OR (status = 'attempted' AND attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL AND confirmed_transaction_id IS NULL AND last_error IS NULL) OR (status = 'confirmed' AND attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL AND transaction_id IS NOT NULL AND confirmed_transaction_id IS NOT NULL AND last_error IS NULL) - OR (status = 'released' AND confirmed_transaction_id IS NULL AND last_error IS NULL AND ((attempted_at IS NULL AND extra_data_hex IS NULL AND transaction_id IS NULL AND submission_json IS NULL AND attention_code IS NULL AND attention_at IS NULL) OR (attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL))) + OR (status = 'released' AND confirmed_transaction_id IS NULL AND last_error IS NULL AND ((attempted_at IS NULL AND extra_data_hex IS NULL AND transaction_id IS NULL AND status_url IS NULL AND attention_code IS NULL AND attention_at IS NULL) OR (attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL))) OR (status = 'rejected' AND attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL AND confirmed_transaction_id IS NULL AND last_error IS NOT NULL AND last_error <> '') )`, - "CONSTRAINT chk_storage_commit_attempts_submission CHECK (submission_json IS NULL OR transaction_id IS NOT NULL)", + "CONSTRAINT chk_storage_commit_attempts_status_url CHECK (status_url IS NULL OR transaction_id IS NOT NULL)", "CONSTRAINT chk_storage_commit_attempts_attention CHECK ((attention_code IS NULL AND attention_at IS NULL) OR (attention_code IS NOT NULL AND attention_at IS NOT NULL AND attempted_at IS NOT NULL))", "CONSTRAINT chk_storage_commit_attempts_release CHECK ((status = 'released' AND release_reason IS NOT NULL) OR (status <> 'released' AND release_reason IS NULL))", }, diff --git a/internal/db/migrations/2026092101_commit_status_url.go b/internal/db/migrations/2026092101_commit_status_url.go deleted file mode 100644 index fe404cf..0000000 --- a/internal/db/migrations/2026092101_commit_status_url.go +++ /dev/null @@ -1,82 +0,0 @@ -package migrations - -import ( - "context" - "errors" - "fmt" - "strings" - "time" - - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect" -) - -func init() { - Migrations.MustRegister( - transactionalMigration(up2026092101CommitStatusURL), - func(context.Context, *bun.DB) error { - return errors.New("commit status URL schema cannot be rolled back; create a new empty database") - }, - ) -} - -// Both ledgers must be empty because this development migration deliberately -// does not convert persisted submissions or their confirmed projections. -func up2026092101CommitStatusURL(ctx context.Context, db bun.IDB) error { - for _, table := range []string{"storage_commit_attempts", "storage_copies"} { - count, err := db.NewSelect().Table(table).Count(ctx) - if err != nil { - return fmt.Errorf("checking %s before commit ledger rebuild: %w", table, err) - } - if count != 0 { - return fmt.Errorf("%s is not empty: %w", table, incompatibleDatabaseError()) - } - } - if db.Dialect().Name() == dialect.PG { - if _, err := db.ExecContext(ctx, `ALTER TABLE storage_copies DROP CONSTRAINT fk_storage_copies_confirmed_attempt`); err != nil { - return fmt.Errorf("dropping commit projection foreign key: %w", err) - } - } - if _, err := db.ExecContext(ctx, `DROP TABLE storage_commit_attempts`); err != nil { - return fmt.Errorf("dropping empty commit ledger: %w", err) - } - spec := storageCommitAttemptTable2026090101() - spec.model = (*storageCommitAttempt2026092101)(nil) - spec.jsonColumns = nil - for i, constraint := range spec.constraints { - spec.constraints[i] = strings.ReplaceAll(constraint, "submission_json", "status_url") - } - if err := createInitialTable(ctx, db, spec); err != nil { - return err - } - for _, index := range storageIndexes2026090101() { - if index.table != "storage_commit_attempts" { - continue - } - if err := createInitialIndexes(ctx, db, index); err != nil { - return err - } - } - return addForwardForeignKey(ctx, db, "storage_copies", storageCopyConfirmedAttemptForeignKey2026090101()) -} - -type storageCommitAttempt2026092101 struct { - bun.BaseModel `bun:"table:storage_commit_attempts"` - - AttemptID string `bun:"type:text,pk"` - ContentID int64 `bun:",notnull"` - StorageDataSetID int64 `bun:",notnull"` - Status string `bun:"type:text,notnull,default:'reserved'"` - ExtraDataHex *string `bun:"type:text"` - TransactionID *string `bun:"type:text"` - StatusURL *string `bun:"type:text"` - ConfirmedTransactionID *string `bun:"type:text"` - AttentionCode *string `bun:"type:text"` - AttentionAt *time.Time - ReleaseReason *string `bun:"type:text"` - LastError *string `bun:"type:text"` - AttemptedAt *time.Time - ResolvedAt *time.Time - CreatedAt time.Time `bun:",notnull"` - UpdatedAt time.Time `bun:",notnull"` -} diff --git a/internal/db/migrations/2026092101_commit_status_url_test.go b/internal/db/migrations/2026092101_commit_status_url_test.go deleted file mode 100644 index 374f52b..0000000 --- a/internal/db/migrations/2026092101_commit_status_url_test.go +++ /dev/null @@ -1,99 +0,0 @@ -package migrations - -import ( - "errors" - "slices" - "strings" - "testing" - - "github.com/uptrace/bun" -) - -func TestCommitStatusURLMigrationRebuildsEmptyLedger(t *testing.T) { - testMigrationDialects(t, func(t *testing.T, db *bun.DB) { - if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { - t.Fatal(err) - } - if err := runMigrationBody(t.Context(), db, up2026092101CommitStatusURL); err != nil { - t.Fatal(err) - } - for _, column := range []string{"status_url", "submission_json"} { - exists, err := columnExists(t.Context(), db, "storage_commit_attempts", column) - if err != nil || exists != (column == "status_url") { - t.Fatalf("column %s exists=%t err=%v", column, exists, err) - } - } - for _, index := range []string{ - "idx_storage_commit_attempts_unresolved_copy", - "idx_storage_commit_attempts_unresolved_data_set", - "idx_storage_commit_attempts_copy_history", - } { - exists, err := indexExists(t.Context(), db, index) - if err != nil || !exists { - t.Fatalf("index %s exists=%t err=%v", index, exists, err) - } - } - constraints := semanticConstraintLines(t, db, applicationSchemaTables(t, db)) - for _, name := range []string{ - "chk_storage_commit_attempts_evidence_shape", - "chk_storage_commit_attempts_resolution", - "chk_storage_commit_attempts_submission", - "uq_storage_commit_attempts_status", - } { - if !slices.Contains(constraints, "constraint|storage_commit_attempts|"+name) { - t.Fatalf("commit ledger constraint %s was not restored", name) - } - } - foreignKeys := semanticForeignKeyLines(t, db, applicationSchemaTables(t, db)) - for _, column := range []string{"confirmed_attempt_id", "confirmed_attempt_status"} { - found := false - for _, line := range foreignKeys { - if strings.HasPrefix(line, "foreign-key|storage_copies|"+column+"|storage_commit_attempts|") { - found = true - break - } - } - if !found { - t.Fatalf("storage copy foreign key column %s was not restored", column) - } - } - if _, err := db.ExecContext(t.Context(), `INSERT INTO storage_commit_attempts - (attempt_id, content_id, storage_data_set_id, status, extra_data_hex, attempted_at, status_url, created_at, updated_at) - VALUES ('invalid-status', 1, 1, 'attempted', 'abcd', CURRENT_TIMESTAMP, - 'https://provider.example/status', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`); err == nil { - t.Fatal("status URL without transaction was accepted") - } - }) -} - -func TestCommitStatusURLMigrationRejectsNonemptyLedgers(t *testing.T) { - for _, populated := range []string{"attempt", "copy"} { - t.Run(populated, func(t *testing.T) { - testMigrationDialects(t, func(t *testing.T, db *bun.DB) { - if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { - t.Fatal(err) - } - if populated == "attempt" { - if _, err := db.ExecContext(t.Context(), `INSERT INTO storage_commit_attempts - (attempt_id, content_id, storage_data_set_id, created_at, updated_at) - VALUES ('existing', 1, 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`); err != nil { - t.Fatal(err) - } - } else { - bucketID := insertBaselineTestBucketWithSlots(t, db, "migration-copy", 1) - contentID := insertBaselineTestContent(t, db, bucketID, "migration-copy") - dataSetID := insertBaselineTestDataSet(t, db, bucketID, "provider", 0, 1, true) - insertBaselineTestCopy(t, db, contentID, bucketID, dataSetID, 0, "provider", "ingress") - } - err := runMigrationBody(t.Context(), db, up2026092101CommitStatusURL) - if !errors.Is(err, ErrIncompatibleDatabase) || !strings.Contains(err.Error(), "new empty database") { - t.Fatalf("migration error = %v, want explicit empty-database refusal", err) - } - exists, err := columnExists(t.Context(), db, "storage_commit_attempts", "submission_json") - if err != nil || !exists { - t.Fatalf("rejected migration changed original ledger: exists=%t err=%v", exists, err) - } - }) - }) - } -} diff --git a/internal/db/migrations/2026999999_transaction_semantics_test.go b/internal/db/migrations/2026999999_transaction_semantics_test.go index d60200a..fbfc288 100644 --- a/internal/db/migrations/2026999999_transaction_semantics_test.go +++ b/internal/db/migrations/2026999999_transaction_semantics_test.go @@ -98,7 +98,7 @@ func TestInitialBaselineRepairsMissingMarkerOnlyForCompletePostState(t *testing. if _, err := migrator.Migrate(ctx); err != nil { t.Fatalf("repair baseline marker: %v", err) } - assertAppliedMigrationCount(t, ctx, migrator, 2) + assertAppliedMigrationCount(t, ctx, migrator, 1) }) } diff --git a/internal/db/migrations/migrations.go b/internal/db/migrations/migrations.go index ba3c37a..38d92e5 100644 --- a/internal/db/migrations/migrations.go +++ b/internal/db/migrations/migrations.go @@ -56,9 +56,34 @@ func NewMigrator(db *bun.DB) *migrate.Migrator { } // ValidateTarget accepts an empty application database or a database whose -// applied migrations are an exact ordered prefix of the current registry. +// applied migrations and schema match the current unreleased baseline. func ValidateTarget(ctx context.Context, db bun.IDB) error { - return validateTarget(ctx, db, Migrations) + if err := validateTarget(ctx, db, Migrations); err != nil { + return err + } + markerExists, err := tableExists(ctx, db, "bun_migrations") + if err != nil || !markerExists { + return err + } + var names []string + if err := db.NewRaw("SELECT name FROM bun_migrations ORDER BY id").Scan(ctx, &names); err != nil { + return err + } + if len(names) == 0 { + return nil + } + statusURLExists, err := columnExists(ctx, db, "storage_commit_attempts", "status_url") + if err != nil { + return fmt.Errorf("checking commit status URL column: %w", err) + } + oldJSONExists, err := columnExists(ctx, db, "storage_commit_attempts", "submission_json") + if err != nil { + return fmt.Errorf("checking obsolete commit submission column: %w", err) + } + if !statusURLExists || oldJSONExists { + return incompatibleDatabaseError() + } + return nil } func validateTarget(ctx context.Context, db bun.IDB, registry *migrate.Migrations) error { @@ -164,6 +189,7 @@ func initialSchemaPostStateComplete(ctx context.Context, db bun.IDB) (bool, erro {"storage_copies", "content_id"}, {"storage_copies", "storage_data_set_id"}, {"storage_commit_attempts", "attempt_id"}, + {"storage_commit_attempts", "status_url"}, {"storage_replacement_items", "target_data_set_id"}, {"storage_cleanup_copies", "bucket_id"}, {"storage_cleanup_copies", "checksum"}, @@ -182,6 +208,7 @@ func initialSchemaPostStateComplete(ctx context.Context, db bun.IDB) (bool, erro {"multipart_uploads", "id"}, {"storage_copies", "commit_attempt_id"}, {"storage_copies", "commit_transaction_id"}, + {"storage_commit_attempts", "submission_json"}, {"storage_copies", "upload_id"}, {"storage_replacement_items", "target_copy_id"}, {"object_versions", "state"}, diff --git a/internal/db/migrations/schema_behavior_test.go b/internal/db/migrations/schema_behavior_test.go index 5391282..99aac52 100644 --- a/internal/db/migrations/schema_behavior_test.go +++ b/internal/db/migrations/schema_behavior_test.go @@ -195,7 +195,7 @@ func TestBaselineStorageIdentityAndLedgerConstraints(t *testing.T) { WHERE attempt_id = 'attempt-2'`) mustRejectStatement(t, db, `UPDATE storage_commit_attempts SET status = 'attempted', attempted_at = current_timestamp, - extra_data_hex = 'abcd', submission_json = '{}' + extra_data_hex = 'abcd', status_url = 'https://provider.example/status' WHERE attempt_id = 'attempt-2'`) target := insertBaselineTestDataSet(t, db, bucketA, "202", 0, 2, false) diff --git a/internal/db/migrations/schema_builder.go b/internal/db/migrations/schema_builder.go index ccd4b0d..805be46 100644 --- a/internal/db/migrations/schema_builder.go +++ b/internal/db/migrations/schema_builder.go @@ -47,8 +47,6 @@ type initialJSONColumnSpec struct { func initialJSONColumns(table string) []initialJSONColumnSpec { switch table { - case "storage_commit_attempts": - return []initialJSONColumnSpec{{name: "submission_json", shape: initialJSONObject, nullable: true, text: true}} case "task_payloads": return []initialJSONColumnSpec{ {name: "input_json", shape: initialJSONObject}, diff --git a/internal/db/migrations/schema_integrity_test.go b/internal/db/migrations/schema_integrity_test.go index 85f1c40..3a31c43 100644 --- a/internal/db/migrations/schema_integrity_test.go +++ b/internal/db/migrations/schema_integrity_test.go @@ -26,7 +26,7 @@ import ( ) const ( - initialPortableSchemaFingerprint = "21723d7d64f621047bd6fd1879851008fdc18c3fd0b173168e063a0ae3d0c7d6" + initialPortableSchemaFingerprint = "ed0bfc2f753f6456b19ea840b2ba90feb8cccaffa249e269aad4260a40d02990" ) func TestMigrationRegistryStartsWithUniqueOrderedBaseline(t *testing.T) { @@ -211,7 +211,7 @@ func TestInitialSchemaContractSQLite(t *testing.T) { {"storage_copies", "commit_attempt_id"}, {"storage_copies", "commit_attempted_at"}, {"storage_copies", "commit_transaction_id"}, - {"storage_copies", "commit_submission_json"}, + {"storage_copies", "commit_status_url"}, {"storage_copies", "commit_confirmed_transaction_id"}, {"storage_copies", "commit_attention_code"}, {"storage_copies", "commit_attention_at"}, @@ -437,7 +437,7 @@ func TestFreshBaselineIsIdempotentAndCannotRollback(t *testing.T) { if err != nil { t.Fatalf("migrate fresh schema: %v", err) } - if len(first.Migrations) != 2 || first.Migrations[0].Name != InitialSchemaName || first.Migrations[1].Name != "2026092101" { + if len(first.Migrations) != 1 || first.Migrations[0].Name != InitialSchemaName { t.Fatalf("first migration group = %#v", first.Migrations) } second, err := migrator.Migrate(ctx) @@ -456,6 +456,28 @@ func TestFreshBaselineIsIdempotentAndCannotRollback(t *testing.T) { }) } +func TestValidateTargetRejectsObsoleteCommitLedgerShape(t *testing.T) { + testMigrationDialects(t, func(t *testing.T, db *bun.DB) { + ctx := t.Context() + migrator := NewMigrator(db) + if err := migrator.Init(ctx); err != nil { + t.Fatal(err) + } + if _, err := migrator.Migrate(ctx); err != nil { + t.Fatal(err) + } + if err := ValidateTarget(ctx, db); err != nil { + t.Fatalf("validate current schema: %v", err) + } + if _, err := db.ExecContext(ctx, `ALTER TABLE storage_commit_attempts RENAME COLUMN status_url TO submission_json`); err != nil { + t.Fatalf("simulate obsolete commit ledger: %v", err) + } + if err := ValidateTarget(ctx, db); !errors.Is(err, ErrIncompatibleDatabase) { + t.Fatalf("validate obsolete commit ledger = %v, want incompatible database", err) + } + }) +} + func runMigrationBody(ctx context.Context, db *bun.DB, body migrationBody) error { return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { return body(ctx, tx) diff --git a/internal/db/migrations/schema_model_alignment_test.go b/internal/db/migrations/schema_model_alignment_test.go index f1dc146..cc02c9f 100644 --- a/internal/db/migrations/schema_model_alignment_test.go +++ b/internal/db/migrations/schema_model_alignment_test.go @@ -23,7 +23,7 @@ import ( "github.com/uptrace/bun/dialect" ) -func TestRuntimeModelsMatchAppliedMigrations(t *testing.T) { +func TestRuntimeModelsMatchAppliedBaseline(t *testing.T) { models := runtimePersistentModels() runtimeTablesFromAST := runtimePersistentModelTablesFromAST(t) @@ -44,9 +44,6 @@ func TestRuntimeModelsMatchAppliedMigrations(t *testing.T) { if err := runMigrationBody(t.Context(), db, up2026090101InitialSchema); err != nil { t.Fatalf("create initial schema: %v", err) } - if err := runMigrationBody(t.Context(), db, up2026092101CommitStatusURL); err != nil { - t.Fatalf("create final commit ledger: %v", err) - } appliedTables := applicationSchemaTables(t, db) if !slices.Equal(appliedTables, registeredTables) { t.Fatalf("applied tables = %v, runtime model tables = %v", appliedTables, registeredTables) From 794144d0c08f23d20db5948c21f702287db797f9 Mon Sep 17 00:00:00 2001 From: Lee <7932644+strahe@users.noreply.github.com> Date: Tue, 22 Sep 2026 11:00:12 +0800 Subject: [PATCH 4/5] fix(storage): require complete submission evidence --- internal/app/runtime.go | 1 - internal/db/db_test.go | 4 +- .../migrations/2026090101_initial_schema.go | 2 +- .../db/migrations/schema_behavior_test.go | 11 + .../db/migrations/schema_integrity_test.go | 2 +- internal/db/repository/interfaces.go | 1 - internal/db/repository/storage_commit_repo.go | 24 -- .../db/repository/storage_content_repo.go | 4 +- internal/storagecommit/advancer.go | 77 ------ internal/storagecommit/advancer_test.go | 177 +----------- internal/storagecommit/codes.go | 2 - internal/storagecommit/store.go | 1 - internal/synapse/pdp_status.go | 161 +---------- internal/synapse/pdp_status_test.go | 254 ++---------------- internal/testutil/storagecommit.go | 2 + internal/worker/storage_task_handlers.go | 4 +- internal/worker/task_handlers.go | 2 - ui/src/api/client.ts | 1 - ui/src/lib/storage-confirmation-attention.ts | 1 - .../storage-confirmation-attention.test.ts | 1 - 20 files changed, 64 insertions(+), 668 deletions(-) diff --git a/internal/app/runtime.go b/internal/app/runtime.go index 719ef8b..7e5d2ea 100644 --- a/internal/app/runtime.go +++ b/internal/app/runtime.go @@ -133,7 +133,6 @@ func NewRuntime(ctx context.Context, opts RuntimeOptions) (_ *Runtime, err error Terminator: opts.Filecoin.Terminator, Epochs: opts.Filecoin.Epochs, Observability: observabilityService, - CommitStatus: pdpStatusChecker, ParkedPieces: pdpStatusChecker, EvictionPolicy: evictionPolicy, MaxCacheBytes: maxCacheBytes, diff --git a/internal/db/db_test.go b/internal/db/db_test.go index 3f1f248..b89e95e 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -440,7 +440,7 @@ func TestRunMigrations_StorageProvenanceConstraints(t *testing.T) { // to exist before the copy can claim it. mustReject(t, db, "expected a committed copy without confirmed evidence to fail", `INSERT INTO storage_copies (content_id, bucket_id, content_size, copy_index, provider_id, piece_id, transfer_method, status, retrieval_url, storage_data_set_id, created_at, updated_at) VALUES (1, 1, 10, 0, '101', '2001', 'ingress', 'committed', 'https://provider.example/piece', 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) mustExec(t, db, `INSERT INTO storage_copies (content_id, bucket_id, content_size, copy_index, provider_id, piece_id, transfer_method, status, retrieval_url, storage_data_set_id, created_at, updated_at) VALUES (1, 1, 10, 0, '101', '2001', 'ingress', 'piece_ready', 'https://provider.example/piece', 1, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) - mustExec(t, db, `INSERT INTO storage_commit_attempts (attempt_id, content_id, storage_data_set_id, status, extra_data_hex, transaction_id, confirmed_transaction_id, attempted_at, resolved_at, created_at, updated_at) VALUES ('attempt-1', 1, 1, 'confirmed', 'abcd', 'tx-1', 'tx-1', current_timestamp, current_timestamp, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO storage_commit_attempts (attempt_id, content_id, storage_data_set_id, status, extra_data_hex, transaction_id, status_url, confirmed_transaction_id, attempted_at, resolved_at, created_at, updated_at) VALUES ('attempt-1', 1, 1, 'confirmed', 'abcd', 'tx-1', 'https://provider.example/status/attempt-1', 'tx-1', current_timestamp, current_timestamp, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) mustReject(t, db, "expected a copy to refuse projecting an unconfirmed status", `UPDATE storage_copies SET status = 'committed', confirmed_attempt_id = 'attempt-1', confirmed_attempt_status = 'attempted' WHERE content_id = 1 AND copy_index = 0`) mustReject(t, db, "expected a copy to refuse projecting a missing attempt", `UPDATE storage_copies SET status = 'committed', confirmed_attempt_id = 'attempt-missing', confirmed_attempt_status = 'confirmed' WHERE content_id = 1 AND copy_index = 0`) mustExec(t, db, `UPDATE storage_copies SET status = 'committed', confirmed_attempt_id = 'attempt-1', confirmed_attempt_status = 'confirmed' WHERE content_id = 1 AND copy_index = 0`) @@ -455,7 +455,7 @@ func TestRunMigrations_StorageProvenanceConstraints(t *testing.T) { mustRejectRequiredColumn(t, db, "expected copy without a data set and provider binding to fail", `INSERT INTO storage_copies (content_id, bucket_id, content_size, copy_index, transfer_method, created_at, updated_at) VALUES (1, 1, 10, 3, 'peer_pull', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) mustExec(t, db, `INSERT INTO storage_copies (content_id, bucket_id, content_size, copy_index, provider_id, transfer_method, storage_data_set_id, created_at, updated_at) VALUES (1, 1, 10, 1, '202', 'peer_pull', 2, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) mustReject(t, db, "expected committed copy without piece identity to fail", `UPDATE storage_copies SET status = 'committed' WHERE content_id = 1 AND copy_index = 1`) - mustExec(t, db, `INSERT INTO storage_commit_attempts (attempt_id, content_id, storage_data_set_id, status, extra_data_hex, transaction_id, confirmed_transaction_id, attempted_at, resolved_at, created_at, updated_at) VALUES ('attempt-2', 1, 2, 'confirmed', 'abcd', 'tx-2', 'tx-2', current_timestamp, current_timestamp, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + mustExec(t, db, `INSERT INTO storage_commit_attempts (attempt_id, content_id, storage_data_set_id, status, extra_data_hex, transaction_id, status_url, confirmed_transaction_id, attempted_at, resolved_at, created_at, updated_at) VALUES ('attempt-2', 1, 2, 'confirmed', 'abcd', 'tx-2', 'https://provider.example/status/attempt-2', 'tx-2', current_timestamp, current_timestamp, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) mustExec(t, db, `UPDATE storage_copies SET status = 'committed', piece_id = '0', retrieval_url = 'https://provider.example/zero-piece', confirmed_attempt_id = 'attempt-2', confirmed_attempt_status = 'confirmed' WHERE content_id = 1 AND copy_index = 1`) mustExec(t, db, `INSERT INTO storage_cleanup_copies (content_id, bucket_id, copy_index, provider_id, storage_data_set_id, piece_id, piece_cid, checksum, created_at, updated_at) VALUES (1, 1, 0, '101', 1, '2001', 'bafk2bzacefake', printf('%064x', 1), CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) mustReject(t, db, "expected duplicate physical cleanup identity to fail", `INSERT INTO storage_cleanup_copies (content_id, bucket_id, copy_index, provider_id, storage_data_set_id, piece_id, piece_cid, checksum, created_at, updated_at) VALUES (1, 1, 0, '101', 1, '2001', 'bafk2bzaceduplicate', printf('%064x', 1), CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) diff --git a/internal/db/migrations/2026090101_initial_schema.go b/internal/db/migrations/2026090101_initial_schema.go index ce47ddb..ebc8c2a 100644 --- a/internal/db/migrations/2026090101_initial_schema.go +++ b/internal/db/migrations/2026090101_initial_schema.go @@ -841,7 +841,7 @@ func storageCommitAttemptTable2026090101() initialTableSpec { OR (status = 'released' AND confirmed_transaction_id IS NULL AND last_error IS NULL AND ((attempted_at IS NULL AND extra_data_hex IS NULL AND transaction_id IS NULL AND status_url IS NULL AND attention_code IS NULL AND attention_at IS NULL) OR (attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL))) OR (status = 'rejected' AND attempted_at IS NOT NULL AND extra_data_hex IS NOT NULL AND confirmed_transaction_id IS NULL AND last_error IS NOT NULL AND last_error <> '') )`, - "CONSTRAINT chk_storage_commit_attempts_status_url CHECK (status_url IS NULL OR transaction_id IS NOT NULL)", + "CONSTRAINT chk_storage_commit_attempts_submission_evidence CHECK ((transaction_id IS NULL AND status_url IS NULL) OR (transaction_id IS NOT NULL AND status_url IS NOT NULL))", "CONSTRAINT chk_storage_commit_attempts_attention CHECK ((attention_code IS NULL AND attention_at IS NULL) OR (attention_code IS NOT NULL AND attention_at IS NOT NULL AND attempted_at IS NOT NULL))", "CONSTRAINT chk_storage_commit_attempts_release CHECK ((status = 'released' AND release_reason IS NOT NULL) OR (status <> 'released' AND release_reason IS NULL))", }, diff --git a/internal/db/migrations/schema_behavior_test.go b/internal/db/migrations/schema_behavior_test.go index 99aac52..9b9b5fc 100644 --- a/internal/db/migrations/schema_behavior_test.go +++ b/internal/db/migrations/schema_behavior_test.go @@ -197,6 +197,17 @@ func TestBaselineStorageIdentityAndLedgerConstraints(t *testing.T) { SET status = 'attempted', attempted_at = current_timestamp, extra_data_hex = 'abcd', status_url = 'https://provider.example/status' WHERE attempt_id = 'attempt-2'`) + mustRejectStatement(t, db, `UPDATE storage_commit_attempts + SET status = 'attempted', attempted_at = current_timestamp, + extra_data_hex = 'abcd', transaction_id = '0xabc' + WHERE attempt_id = 'attempt-2'`) + if _, err := db.Exec(`UPDATE storage_commit_attempts + SET status = 'attempted', attempted_at = current_timestamp, + extra_data_hex = 'abcd', transaction_id = '0xabc', + status_url = 'https://provider.example/status' + WHERE attempt_id = 'attempt-2'`); err != nil { + t.Fatalf("record complete commit submission: %v", err) + } target := insertBaselineTestDataSet(t, db, bucketA, "202", 0, 2, false) var replacementID int64 diff --git a/internal/db/migrations/schema_integrity_test.go b/internal/db/migrations/schema_integrity_test.go index 3a31c43..3047504 100644 --- a/internal/db/migrations/schema_integrity_test.go +++ b/internal/db/migrations/schema_integrity_test.go @@ -26,7 +26,7 @@ import ( ) const ( - initialPortableSchemaFingerprint = "ed0bfc2f753f6456b19ea840b2ba90feb8cccaffa249e269aad4260a40d02990" + initialPortableSchemaFingerprint = "657cbdd208ba7ff26431b90df7f03e40233b0475ab25253508b58922fd828c7e" ) func TestMigrationRegistryStartsWithUniqueOrderedBaseline(t *testing.T) { diff --git a/internal/db/repository/interfaces.go b/internal/db/repository/interfaces.go index f5f0518..e9eb76d 100644 --- a/internal/db/repository/interfaces.go +++ b/internal/db/repository/interfaces.go @@ -521,7 +521,6 @@ type StorageContentRepository interface { ReopenFailedUploadCopy(ctx context.Context, copyID int64) error ReserveCommitAttempt(ctx context.Context, input storagecommit.ReserveInput) (storagecommit.ReserveResult, error) MarkCommitAttempted(ctx context.Context, input storagecommit.AttemptInput) (storagecommit.AttemptResult, error) - RecordCommitTransaction(ctx context.Context, input storagecommit.EvidenceInput) error RecordCommitSubmission(ctx context.Context, input storagecommit.EvidenceInput) error MarkCommitAttention(ctx context.Context, input storagecommit.AttentionInput) error ResetCommitAttempt(ctx context.Context, input storagecommit.ResetInput) error diff --git a/internal/db/repository/storage_commit_repo.go b/internal/db/repository/storage_commit_repo.go index e88c752..81b47ba 100644 --- a/internal/db/repository/storage_commit_repo.go +++ b/internal/db/repository/storage_commit_repo.go @@ -197,30 +197,6 @@ func (r *BunStorageContentRepo) MarkCommitAttempted( return out, err } -func (r *BunStorageContentRepo) RecordCommitTransaction(ctx context.Context, input storagecommit.EvidenceInput) error { - if err := validateCommitEvidenceInput(input, false); err != nil { - return err - } - return r.mutateAttempt(ctx, input.Copy, "recording storage commit transaction", func(db bun.IDB, _ int64) error { - res, err := db.NewUpdate(). - Model((*storagecommit.Attempt)(nil)). - Set("transaction_id = COALESCE(transaction_id, ?)", input.TransactionID). - Set("updated_at = ?", commitInputTime(input.Now)). - Where("attempt_id = ?", input.AttemptID). - Where("content_id = ? AND storage_data_set_id = ?", input.Copy.ContentID, input.Copy.StorageDataSetID). - Where("status = ? AND resolved_at IS NULL", storagecommit.AttemptStatusAttempted). - Where("(transaction_id IS NULL OR transaction_id = ?)", input.TransactionID). - Exec(ctx) - if err != nil { - return err - } - if rows, _ := res.RowsAffected(); rows != 1 { - return ErrConflict - } - return nil - }) -} - func (r *BunStorageContentRepo) RecordCommitSubmission(ctx context.Context, input storagecommit.EvidenceInput) error { if err := validateCommitEvidenceInput(input, true); err != nil { return err diff --git a/internal/db/repository/storage_content_repo.go b/internal/db/repository/storage_content_repo.go index 332ff40..9fd93a7 100644 --- a/internal/db/repository/storage_content_repo.go +++ b/internal/db/repository/storage_content_repo.go @@ -1312,14 +1312,14 @@ func (r *BunStorageContentRepo) MarkUploadCopyCommitted(ctx context.Context, inp attemptQuery := db.NewUpdate(). Model((*storagecommit.Attempt)(nil)). Set("status = ?", storagecommit.AttemptStatusConfirmed). - Set("transaction_id = COALESCE(transaction_id, ?)", input.CommitTransactionID). Set("confirmed_transaction_id = ?", input.CommitConfirmedTransactionID). Set("resolved_at = ?", now). Set("updated_at = ?", now). Where("attempt_id = ?", input.CommitAttemptID). Where("content_id = ? AND storage_data_set_id = ?", input.ContentID, copyIdentity.StorageDataSetID). Where("status = ? AND resolved_at IS NULL", storagecommit.AttemptStatusAttempted). - Where("(transaction_id IS NULL OR transaction_id = ?)", input.CommitTransactionID) + Where("transaction_id = ?", input.CommitTransactionID). + Where("status_url IS NOT NULL") if input.CommitExtraDataHex != "" { attemptQuery = attemptQuery.Where("extra_data_hex = ?", input.CommitExtraDataHex) } diff --git a/internal/storagecommit/advancer.go b/internal/storagecommit/advancer.go index 8b78cbd..a966154 100644 --- a/internal/storagecommit/advancer.go +++ b/internal/storagecommit/advancer.go @@ -12,10 +12,8 @@ import ( "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/synapse" - idtypes "github.com/strahe/synaps3/internal/types" "github.com/strahe/synapse-go/pdp" "github.com/strahe/synapse-go/storage" - sdktypes "github.com/strahe/synapse-go/types" ) const ( @@ -24,13 +22,8 @@ const ( evidenceWriteTimeout = 10 * time.Second ) -type AddPiecesStatusChecker interface { - GetAddPiecesStatus(context.Context, synapse.AddPiecesStatusInput) (synapse.PDPStatusResult, error) -} - type Advancer struct { Store Store - StatusChecker AddPiecesStatusChecker RequestTimeout time.Duration AttentionAfter time.Duration Now func() time.Time @@ -292,9 +285,6 @@ func (a *Advancer) observe( } return a.classifySDKStatus(ctx, input, identity, copyRow, attemptID, status) } - if copyRow.CommitTransactionID != nil && *copyRow.CommitTransactionID != "" { - return a.observeTransaction(ctx, input, copyRow) - } if result, ok := existingAttentionResult( copyRow, attemptID, AttentionAttemptOnlyAmbiguous, false, ); ok { @@ -313,73 +303,6 @@ func (a *Advancer) observe( return a.attentionForCopy(ctx, identity, copyRow, attemptID, code, false) } -func (a *Advancer) observeTransaction( - ctx context.Context, - input AdvanceInput, - copyRow model.StorageCopy, -) (AdvanceResult, error) { - identity := copyIdentity(input, false) - attemptID := *copyRow.CommitAttemptID - transactionID := *copyRow.CommitTransactionID - checker := a.StatusChecker - if checker == nil { - checker = synapse.NewPDPStatusChecker(synapse.PDPStatusCheckerOptions{Timeout: a.requestTimeout()}) - } - requestCtx, cancel := context.WithTimeout(ctx, a.requestTimeout()) - result, err := checker.GetAddPiecesStatus(requestCtx, synapse.AddPiecesStatusInput{ - ServiceURL: input.Target.ServiceURL(), - DataSetID: input.Binding.DataSetID.String(), - TransactionID: transactionID, - ExpectedPieceCount: len(input.Pieces), - }) - cancel() - if err != nil { - if context.Cause(ctx) != nil { - return AdvanceResult{State: AdvancePending, AttemptID: attemptID}, nil - } - if result.State == synapse.PDPStatusMismatch { - return a.attentionForCopy(ctx, identity, copyRow, attemptID, AttentionSubmissionMismatch, false) - } - return a.pendingOrAttention(ctx, identity, copyRow, attemptID) - } - switch result.State { - case synapse.PDPStatusPending: - return a.pendingOrAttention(ctx, identity, copyRow, attemptID) - case synapse.PDPStatusConfirmed: - pieceIDs := make([]sdktypes.BigInt, 0, len(result.ConfirmedPieceIDs)) - for _, raw := range result.ConfirmedPieceIDs { - pieceID, err := idtypes.ParseOnChainID("confirmed piece ID", raw) - if err != nil { - return a.attentionForCopy(ctx, identity, copyRow, attemptID, AttentionSubmissionMismatch, false) - } - pieceIDs = append(pieceIDs, pieceID.SDK()) - } - ref, bound := input.Target.DataSetRef() - if !bound { - return a.attentionForCopy(ctx, identity, copyRow, attemptID, AttentionSubmissionMismatch, false) - } - return AdvanceResult{ - State: AdvanceConfirmed, - AttemptID: attemptID, - Confirmation: &storage.CommitResult{ - TransactionID: transactionID, - ConfirmedTransactionID: confirmedTransactionID(transactionID, result.ConfirmedTransactionID), - DataSet: ref, - PieceIDs: pieceIDs, - }, - }, nil - case synapse.PDPStatusRejected: - if err := a.reset(ctx, identity, attemptID, pdp.ErrTxRejected); err != nil { - return AdvanceResult{}, err - } - return AdvanceResult{State: AdvanceRejected, AttemptID: attemptID}, nil - case synapse.PDPStatusMismatch: - return a.attentionForCopy(ctx, identity, copyRow, attemptID, AttentionSubmissionMismatch, false) - default: - return a.pendingOrAttention(ctx, identity, copyRow, attemptID) - } -} - func (a *Advancer) classifySDKStatus( ctx context.Context, input AdvanceInput, diff --git a/internal/storagecommit/advancer_test.go b/internal/storagecommit/advancer_test.go index a0535b0..172a37b 100644 --- a/internal/storagecommit/advancer_test.go +++ b/internal/storagecommit/advancer_test.go @@ -961,158 +961,6 @@ func TestReleaseCommitAttentionResumesFencedFailedTask(t *testing.T) { } } -func TestAdvancerTxOnlyEvidenceConfirmsWithoutPieceStatus(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - binding, copies, pieceCID := seedAdvancerCopies(t, db, 1) - identity := advancerCopyIdentity(copies[0]) - if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: identity, AttemptID: "tx-only", - }); err != nil { - t.Fatalf("reserve: %v", err) - } - if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ - Copy: identity, AttemptID: "tx-only", ExtraDataHex: "abcd", - }); err != nil { - t.Fatalf("mark attempted: %v", err) - } - if err := repos.Contents.RecordCommitTransaction(t.Context(), storagecommit.EvidenceInput{ - Copy: identity, AttemptID: "tx-only", TransactionID: "0xtxonly", - }); err != nil { - t.Fatalf("record transaction: %v", err) - } - if _, err := db.NewUpdate(). - Model((*storagecommit.Attempt)(nil)). - Set("attention_code = ?", "future_attention_code"). - Set("attention_at = ?", time.Now()). - Where("attempt_id = ?", "tx-only"). - Exec(t.Context()); err != nil { - t.Fatalf("set future tx-only attention: %v", err) - } - copyRow := loadAdvancerCopy(t, repos, copies[0].ID) - target := testutil.NewMockDataSetTarget(binding.ProviderID.SDK(), binding.DataSetID.SDK(), nil) - target.ClientDataSetIDValue = sdktypes.NewBigInt(9001) - target.PieceStatusFunc = func(context.Context, cid.Cid) (*storage.PieceStatus, error) { - t.Fatal("tx-only recovery used PieceStatus") - return nil, nil - } - checkerCalls := 0 - checker := commitStatusCheckerFunc(func(context.Context, synapse.AddPiecesStatusInput) (synapse.PDPStatusResult, error) { - checkerCalls++ - return synapse.PDPStatusResult{ - State: synapse.PDPStatusConfirmed, ConfirmedPieceIDs: []string{"5001"}, - }, nil - }) - result, err := (&storagecommit.Advancer{Store: repos.Contents, StatusChecker: checker}).Advance( - t.Context(), storagecommit.AdvanceInput{ - Copy: *copyRow, Binding: *binding, Target: target, - Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, - }) - if err != nil || result.State != storagecommit.AdvanceConfirmed || result.Confirmation == nil || - result.Confirmation.ConfirmedTransactionID != "0xtxonly" || checkerCalls != 1 { - t.Fatalf("advance = %#v err=%v checkerCalls=%d", result, err, checkerCalls) - } -} - -func TestAdvancerTxOnlyMismatchNeedsAttention(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - binding, copies, pieceCID := seedAdvancerCopies(t, db, 1) - identity := advancerCopyIdentity(copies[0]) - if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: identity, AttemptID: "tx-only-mismatch", - }); err != nil { - t.Fatalf("reserve: %v", err) - } - if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ - Copy: identity, AttemptID: "tx-only-mismatch", ExtraDataHex: "abcd", - }); err != nil { - t.Fatalf("mark attempted: %v", err) - } - if err := repos.Contents.RecordCommitTransaction(t.Context(), storagecommit.EvidenceInput{ - Copy: identity, AttemptID: "tx-only-mismatch", TransactionID: "0xmismatch", - }); err != nil { - t.Fatalf("record transaction: %v", err) - } - copyRow := loadAdvancerCopy(t, repos, copies[0].ID) - target := testutil.NewMockDataSetTarget(binding.ProviderID.SDK(), binding.DataSetID.SDK(), nil) - checker := commitStatusCheckerFunc(func(context.Context, synapse.AddPiecesStatusInput) (synapse.PDPStatusResult, error) { - return synapse.PDPStatusResult{State: synapse.PDPStatusMismatch}, nil - }) - - result, err := (&storagecommit.Advancer{Store: repos.Contents, StatusChecker: checker}).Advance( - t.Context(), storagecommit.AdvanceInput{ - Copy: *copyRow, Binding: *binding, Target: target, - Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, - }) - if err != nil || result.State != storagecommit.AdvanceNeedsAttention || - result.AttentionCode != storagecommit.AttentionSubmissionMismatch { - t.Fatalf("advance = %#v err=%v, want submission mismatch attention", result, err) - } - persisted := loadAdvancerCopy(t, repos, copies[0].ID) - if persisted.CommitAttentionCode == nil || - *persisted.CommitAttentionCode != string(storagecommit.AttentionSubmissionMismatch) { - t.Fatalf("mismatch attention was not persisted: %#v", persisted) - } -} - -func TestAdvancerTxOnlyRequestTimeoutRemainsPending(t *testing.T) { - db := testutil.NewTestDB(t) - repos := repository.NewRepositories(db) - binding, copies, pieceCID := seedAdvancerCopies(t, db, 1) - identity := advancerCopyIdentity(copies[0]) - if _, err := repos.Contents.ReserveCommitAttempt(t.Context(), storagecommit.ReserveInput{ - Copy: identity, AttemptID: "tx-only-timeout", - }); err != nil { - t.Fatalf("reserve: %v", err) - } - if _, err := repos.Contents.MarkCommitAttempted(t.Context(), storagecommit.AttemptInput{ - Copy: identity, AttemptID: "tx-only-timeout", ExtraDataHex: "abcd", - }); err != nil { - t.Fatalf("mark attempted: %v", err) - } - if err := repos.Contents.RecordCommitTransaction(t.Context(), storagecommit.EvidenceInput{ - Copy: identity, AttemptID: "tx-only-timeout", TransactionID: "0xtimeout", - }); err != nil { - t.Fatalf("record transaction: %v", err) - } - copyRow := loadAdvancerCopy(t, repos, copies[0].ID) - target := testutil.NewMockDataSetTarget(binding.ProviderID.SDK(), binding.DataSetID.SDK(), nil) - const requestTimeout = 50 * time.Millisecond - var observedDeadline time.Time - checker := commitStatusCheckerFunc(func(ctx context.Context, _ synapse.AddPiecesStatusInput) (synapse.PDPStatusResult, error) { - var ok bool - observedDeadline, ok = ctx.Deadline() - if !ok { - t.Fatal("status checker context has no deadline") - } - <-ctx.Done() - return synapse.PDPStatusResult{}, ctx.Err() - }) - parentCtx := t.Context() - startedAt := time.Now() - result, err := (&storagecommit.Advancer{ - Store: repos.Contents, StatusChecker: checker, RequestTimeout: requestTimeout, - }).Advance(parentCtx, storagecommit.AdvanceInput{ - Copy: *copyRow, Binding: *binding, Target: target, - Pieces: []storage.PieceInput{{PieceCID: pieceCID}}, - }) - if err != nil || result.State != storagecommit.AdvancePending { - t.Fatalf("advance = %#v err=%v, want pending request timeout", result, err) - } - if context.Cause(parentCtx) != nil { - t.Fatalf("request timeout canceled parent context: %v", context.Cause(parentCtx)) - } - deadlineAfter := observedDeadline.Sub(startedAt) - if deadlineAfter <= 0 || deadlineAfter > 2*requestTimeout { - t.Fatalf("checker deadline after %s, want finite deadline near %s", deadlineAfter, requestTimeout) - } - persisted := loadAdvancerCopy(t, repos, copies[0].ID) - if persisted.CommitAttentionAt != nil || persisted.CommitAttentionCode != nil { - t.Fatalf("recent request timeout wrote attention: %#v", persisted) - } -} - func TestAdvancerCanceledObservationDoesNotWriteAttention(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) @@ -1149,8 +997,6 @@ func TestAdvancerCanceledObservationDoesNotWriteAttention(t *testing.T) { } } -type commitStatusCheckerFunc func(context.Context, synapse.AddPiecesStatusInput) (synapse.PDPStatusResult, error) - type failingCommitEvidenceStore struct { storagecommit.Store err error @@ -1178,21 +1024,10 @@ func (s *failingMarkAttemptedStore) MarkCommitAttempted( return storagecommit.AttemptResult{}, s.err } -func (s *failingCommitEvidenceStore) RecordCommitTransaction(context.Context, storagecommit.EvidenceInput) error { - return s.err -} - func (s *failingCommitEvidenceStore) RecordCommitSubmission(context.Context, storagecommit.EvidenceInput) error { return s.err } -func (f commitStatusCheckerFunc) GetAddPiecesStatus( - ctx context.Context, - input synapse.AddPiecesStatusInput, -) (synapse.PDPStatusResult, error) { - return f(ctx, input) -} - func TestCommitCapacityWakesOnlyTheQueueHead(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) @@ -1265,12 +1100,22 @@ func TestCommitCapacityWakesOnlyTheQueueHead(t *testing.T) { t.Fatalf("mark copy 0 attempted: %v", err) } pieceID := idtypes.OnChainIDFromSDK(sdktypes.NewBigInt(5001)) - if err := repos.Contents.MarkUploadCopyCommitted(t.Context(), repository.MarkUploadCopyCommittedInput{ + confirmation := repository.MarkUploadCopyCommittedInput{ StorageCopyID: copies[0].ID, ContentID: copies[0].ContentID, CopyIndex: 0, PieceCID: pieceCID.String(), PieceID: &pieceID, RetrievalURL: "https://provider.example/piece", CommitExtraDataHex: "abcd", CommitTransactionID: "0x01", CommitAttemptID: "queue-attempt-0", CommitConfirmedTransactionID: "0x01", + } + if err := repos.Contents.MarkUploadCopyCommitted(t.Context(), confirmation); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("confirm copy 0 without submission evidence: %v, want conflict", err) + } + if err := repos.Contents.RecordCommitSubmission(t.Context(), storagecommit.EvidenceInput{ + Copy: advancerCopyIdentity(copies[0]), AttemptID: "queue-attempt-0", + TransactionID: "0x01", StatusURL: "https://provider.example/status/queue-attempt-0", }); err != nil { + t.Fatalf("record copy 0 submission: %v", err) + } + if err := repos.Contents.MarkUploadCopyCommitted(t.Context(), confirmation); err != nil { t.Fatalf("confirm copy 0: %v", err) } runnable("after a confirmation frees one slot", 4) diff --git a/internal/storagecommit/codes.go b/internal/storagecommit/codes.go index 7c8cca2..cc23926 100644 --- a/internal/storagecommit/codes.go +++ b/internal/storagecommit/codes.go @@ -8,7 +8,6 @@ type AttentionCode string const ( AttentionAttemptOnlyAmbiguous AttentionCode = "attempt_only_ambiguous" AttentionUnattributedPiece AttentionCode = "unattributed_piece" - AttentionInvalidSubmission AttentionCode = "invalid_submission" AttentionSubmissionMismatch AttentionCode = "submission_mismatch" AttentionDataSetUnavailable AttentionCode = "data_set_unavailable" AttentionConfirmationTimeout AttentionCode = "confirmation_timeout" @@ -18,7 +17,6 @@ func (c AttentionCode) Valid() bool { switch c { case AttentionAttemptOnlyAmbiguous, AttentionUnattributedPiece, - AttentionInvalidSubmission, AttentionSubmissionMismatch, AttentionDataSetUnavailable, AttentionConfirmationTimeout: diff --git a/internal/storagecommit/store.go b/internal/storagecommit/store.go index 339b0ae..57aa02b 100644 --- a/internal/storagecommit/store.go +++ b/internal/storagecommit/store.go @@ -94,7 +94,6 @@ type ReservationReleaseInput struct { type Store interface { ReserveCommitAttempt(context.Context, ReserveInput) (ReserveResult, error) MarkCommitAttempted(context.Context, AttemptInput) (AttemptResult, error) - RecordCommitTransaction(context.Context, EvidenceInput) error RecordCommitSubmission(context.Context, EvidenceInput) error MarkCommitAttention(context.Context, AttentionInput) error ResetCommitAttempt(context.Context, ResetInput) error diff --git a/internal/synapse/pdp_status.go b/internal/synapse/pdp_status.go index 79bf921..f16c01c 100644 --- a/internal/synapse/pdp_status.go +++ b/internal/synapse/pdp_status.go @@ -11,7 +11,6 @@ import ( "strings" "time" - "github.com/ethereum/go-ethereum/common" "github.com/ipfs/go-cid" "github.com/strahe/synapse-go/pdp" ) @@ -70,25 +69,13 @@ type DataSetCreationStatusInput struct { ExpectedDataSetID string } -type AddPiecesStatusInput struct { - ServiceURL string - StatusURL string - DataSetID string - TransactionID string - ExpectedPieceCount int -} - type PDPStatusResult struct { - State PDPStatusState - StatusURL string - TxStatus string - DataSetID string - DataSetCreated bool - PiecesAdded bool - PieceCount int - ConfirmedPieceIDs []string - ConfirmedTransactionID string - Error string + State PDPStatusState + StatusURL string + TxStatus string + DataSetID string + DataSetCreated bool + Error string } func (c *PDPStatusChecker) CheckDataSetCreationStatus(ctx context.Context, input DataSetCreationStatusInput) PDPStatusResult { @@ -133,76 +120,6 @@ func (c *PDPStatusChecker) CheckDataSetCreationStatus(ctx context.Context, input return result } -func (c *PDPStatusChecker) CheckAddPiecesStatus(ctx context.Context, input AddPiecesStatusInput) PDPStatusResult { - result, err := c.GetAddPiecesStatus(ctx, input) - if err == nil { - return result - } - state := result.State - if state == "" { - state = PDPStatusUnavailable - } - return result.withError(state, err.Error()) -} - -// GetAddPiecesStatus performs one validated status request while preserving -// the underlying error for operational callers that must classify retries. -func (c *PDPStatusChecker) GetAddPiecesStatus(ctx context.Context, input AddPiecesStatusInput) (PDPStatusResult, error) { - statusURL := input.StatusURL - if statusURL == "" { - var err error - statusURL, err = buildAddPiecesStatusURL(input.ServiceURL, input.DataSetID, input.TransactionID) - if err != nil { - return PDPStatusResult{}, err - } - } - result := PDPStatusResult{StatusURL: statusURL} - if input.ExpectedPieceCount <= 0 { - return result, errors.New("missing expected piece count") - } - client, err := c.clientForStatusURL(statusURL) - if err != nil { - return result, err - } - status, err := client.GetAddPiecesStatus(ctx, statusURL) - if status == nil { - if errors.Is(err, pdp.ErrInvalidStatus) { - result.State = PDPStatusMismatch - } - if err == nil { - err = errors.New("empty add-pieces status") - } - return result, err - } - result.TxStatus = status.TxStatus - result.DataSetID = status.DataSetID.String() - result.PieceCount = status.PieceCount - result.PiecesAdded = status.PiecesAdded - if status.ConfirmedTxHash != (common.Hash{}) { - result.ConfirmedTransactionID = status.ConfirmedTxHash.Hex() - } - for _, id := range status.ConfirmedPieceIDs { - result.ConfirmedPieceIDs = append(result.ConfirmedPieceIDs, id.String()) - } - if errors.Is(err, pdp.ErrInvalidStatus) { - result.State = PDPStatusMismatch - return result, err - } - if err != nil && !errors.Is(err, pdp.ErrTxRejected) { - return result, err - } - if err := validateAddPiecesStatusIdentity(input, result, status.TxHash.Hex()); err != nil { - result.State = PDPStatusMismatch - return result, err - } - if errors.Is(err, pdp.ErrTxRejected) { - result.State = PDPStatusRejected - return result, nil - } - result.State = classifyAddPiecesStatus(status.TxStatus, status.PiecesAdded, status.PieceCount, input.ExpectedPieceCount, len(result.ConfirmedPieceIDs)) - return result, nil -} - // FindParkedPiece checks the provider-local piece endpoint once. Its client // uses the same redirect, DNS, and private-network protections as status // polling. @@ -295,19 +212,6 @@ func validateDataSetCreationStatusIdentity(input DataSetCreationStatusInput, res return nil } -func validateAddPiecesStatusIdentity(input AddPiecesStatusInput, result PDPStatusResult, statusTxHash string) error { - if expected := strings.TrimSpace(input.DataSetID); expected != "" && strings.TrimSpace(result.DataSetID) != expected { - return fmt.Errorf("status data set ID mismatch: got %s want %s", result.DataSetID, expected) - } - if expected := normalizeStatusTxHash(input.TransactionID); expected != "" { - got := normalizeStatusTxHash(statusTxHash) - if !strings.EqualFold(got, expected) { - return fmt.Errorf("status transaction ID mismatch: got %s want %s", statusTxHash, expected) - } - } - return nil -} - func normalizeStatusTxHash(value string) string { value = strings.TrimSpace(value) if value == "" { @@ -319,34 +223,6 @@ func normalizeStatusTxHash(value string) string { return strings.ToLower(value) } -func buildAddPiecesStatusURL(serviceURL, dataSetID, transactionID string) (string, error) { - if serviceURL == "" { - return "", fmt.Errorf("missing service URL") - } - if dataSetID == "" { - return "", fmt.Errorf("missing data set ID") - } - if transactionID == "" { - return "", fmt.Errorf("missing transaction ID") - } - base, err := url.Parse(serviceURL) - if err != nil { - return "", fmt.Errorf("parse service URL: %w", err) - } - if base.Scheme != "http" && base.Scheme != "https" { - return "", fmt.Errorf("unsupported service URL scheme %q", base.Scheme) - } - base.Path = strings.TrimRight(base.Path, "/") - path, err := url.JoinPath(base.Path, "pdp", "data-sets", dataSetID, "pieces", "added", transactionID) - if err != nil { - return "", fmt.Errorf("build add-pieces status path: %w", err) - } - base.Path = path - base.RawQuery = "" - base.Fragment = "" - return base.String(), nil -} - func classifyCreationStatus(txStatus string, dataSetCreated bool) PDPStatusState { switch txStatus { case "pending": @@ -363,31 +239,6 @@ func classifyCreationStatus(txStatus string, dataSetCreated bool) PDPStatusState } } -func classifyAddPiecesStatus(txStatus string, piecesAdded bool, pieceCount, expectedPieceCount, confirmedPieceIDCount int) PDPStatusState { - switch txStatus { - case "pending": - return PDPStatusPending - case "confirmed": - if !piecesAdded { - return PDPStatusMismatch - } - if expectedPieceCount <= 0 { - return PDPStatusMismatch - } - if pieceCount != expectedPieceCount { - return PDPStatusMismatch - } - if confirmedPieceIDCount != expectedPieceCount { - return PDPStatusMismatch - } - return PDPStatusConfirmed - case "rejected": - return PDPStatusRejected - default: - return PDPStatusUnknown - } -} - func newPDPStatusHTTPClient(timeout time.Duration, allowPrivate bool) *http.Client { base := &net.Dialer{ Timeout: 30 * time.Second, diff --git a/internal/synapse/pdp_status_test.go b/internal/synapse/pdp_status_test.go index 7f44ba3..b1507b1 100644 --- a/internal/synapse/pdp_status_test.go +++ b/internal/synapse/pdp_status_test.go @@ -13,8 +13,6 @@ import ( const ( testCreateDataSetTxHash = "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" - testAddPiecesTxHash = "0x7890abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456" - testConfirmedTxHash = "0xabcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" ) func TestPDPStatusCheckerFindsProviderParkedPiece(t *testing.T) { @@ -97,104 +95,6 @@ func TestPDPStatusCheckerChecksDataSetCreationStatusOnce(t *testing.T) { } } -func TestPDPStatusCheckerChecksAddPiecesStatus(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/pdp/data-sets/1001/pieces/added/"+testAddPiecesTxHash { - t.Fatalf("path = %q, want add-pieces status path", r.URL.Path) - } - _, _ = fmt.Fprintf(w, `{"txHash":%q,"confirmedTxHash":%q,"txStatus":"confirmed","dataSetId":1001,"pieceCount":1,"addMessageOk":true,"piecesAdded":true,"confirmedPieceIds":[2001]}`, testAddPiecesTxHash, testConfirmedTxHash) - })) - defer server.Close() - - checker := NewPDPStatusChecker(PDPStatusCheckerOptions{Timeout: time.Second, AllowPrivateNetworks: true}) - got := checker.CheckAddPiecesStatus(t.Context(), AddPiecesStatusInput{ - ServiceURL: server.URL, - DataSetID: "1001", - TransactionID: testAddPiecesTxHash, - ExpectedPieceCount: 1, - }) - - if got.State != PDPStatusConfirmed || got.TxStatus != "confirmed" || !got.PiecesAdded || - len(got.ConfirmedPieceIDs) != 1 || got.ConfirmedPieceIDs[0] != "2001" || - got.ConfirmedTransactionID != testConfirmedTxHash { - t.Fatalf("status = %#v, want confirmed add-pieces result", got) - } -} - -func TestPDPStatusCheckerClassifiesAddPiecesMismatch(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = fmt.Fprintf(w, `{"txHash":%q,"txStatus":"confirmed","dataSetId":1001,"pieceCount":1,"addMessageOk":null,"piecesAdded":false}`, testAddPiecesTxHash) - })) - defer server.Close() - - checker := NewPDPStatusChecker(PDPStatusCheckerOptions{Timeout: time.Second, AllowPrivateNetworks: true}) - got := checker.CheckAddPiecesStatus(t.Context(), AddPiecesStatusInput{ - ServiceURL: server.URL, - DataSetID: "1001", - TransactionID: testAddPiecesTxHash, - ExpectedPieceCount: 1, - }) - - if got.State != PDPStatusMismatch { - t.Fatalf("state = %s, want mismatch: %#v", got.State, got) - } -} - -func TestPDPStatusCheckerClassifiesAddPiecesIdentityMismatch(t *testing.T) { - tests := []struct { - name string - response string - dataSetID string - transactionID string - }{ - { - name: "data set mismatch", - response: fmt.Sprintf(`{"txHash":%q,"txStatus":"confirmed","dataSetId":9999,"pieceCount":1,"addMessageOk":true,"piecesAdded":true}`, testAddPiecesTxHash), - dataSetID: "1001", - transactionID: testAddPiecesTxHash, - }, - { - name: "transaction mismatch", - response: `{"txHash":"0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","txStatus":"confirmed","dataSetId":1001,"pieceCount":1,"addMessageOk":true,"piecesAdded":true}`, - dataSetID: "1001", - transactionID: testAddPiecesTxHash, - }, - { - name: "rejected transaction mismatch", - response: `{"txHash":"0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","txStatus":"failed","dataSetId":1001,"pieceCount":0,"addMessageOk":false,"piecesAdded":false}`, - dataSetID: "1001", - transactionID: testAddPiecesTxHash, - }, - { - name: "missing data set id", - response: fmt.Sprintf(`{"txHash":%q,"txStatus":"confirmed","pieceCount":1,"addMessageOk":true,"piecesAdded":true}`, testAddPiecesTxHash), - dataSetID: "1001", - transactionID: testAddPiecesTxHash, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = fmt.Fprint(w, tt.response) - })) - defer server.Close() - - checker := NewPDPStatusChecker(PDPStatusCheckerOptions{Timeout: time.Second, AllowPrivateNetworks: true}) - got := checker.CheckAddPiecesStatus(t.Context(), AddPiecesStatusInput{ - ServiceURL: server.URL, - DataSetID: tt.dataSetID, - TransactionID: tt.transactionID, - ExpectedPieceCount: 1, - }) - - if got.State != PDPStatusMismatch || got.Error == "" { - t.Fatalf("status = %#v, want mismatch with identity error", got) - } - }) - } -} - func TestPDPStatusCheckerClassifiesSDKRejectedCreationStatuses(t *testing.T) { tests := []struct { name string @@ -226,52 +126,17 @@ func TestPDPStatusCheckerClassifiesSDKRejectedCreationStatuses(t *testing.T) { } } -func TestPDPStatusCheckerClassifiesSDKRejectedAddPiecesStatuses(t *testing.T) { - tests := []struct { - name string - response string - }{ - {name: "rejected", response: fmt.Sprintf(`{"txHash":%q,"txStatus":"rejected","dataSetId":1001,"pieceCount":0,"addMessageOk":false,"piecesAdded":false}`, testAddPiecesTxHash)}, - {name: "failed", response: fmt.Sprintf(`{"txHash":%q,"txStatus":"failed","dataSetId":1001,"pieceCount":0,"addMessageOk":false,"piecesAdded":false}`, testAddPiecesTxHash)}, - {name: "reorged", response: fmt.Sprintf(`{"txHash":%q,"txStatus":"reorged","dataSetId":1001,"pieceCount":1,"addMessageOk":true,"piecesAdded":true,"confirmedPieceIds":[2001]}`, testAddPiecesTxHash)}, - {name: "confirmed rejected", response: fmt.Sprintf(`{"txHash":%q,"txStatus":"confirmed","dataSetId":1001,"pieceCount":0,"addMessageOk":false,"piecesAdded":false}`, testAddPiecesTxHash)}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = fmt.Fprint(w, tt.response) - })) - defer server.Close() - - checker := NewPDPStatusChecker(PDPStatusCheckerOptions{Timeout: time.Second, AllowPrivateNetworks: true}) - got := checker.CheckAddPiecesStatus(t.Context(), AddPiecesStatusInput{ - ServiceURL: server.URL, - DataSetID: "1001", - TransactionID: testAddPiecesTxHash, - ExpectedPieceCount: 1, - }) - - if got.State != PDPStatusRejected || got.Error != "" { - t.Fatalf("status = %#v, want rejected without checker error", got) - } - }) - } -} - func TestPDPStatusCheckerClassifiesTimeoutUnavailable(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { time.Sleep(50 * time.Millisecond) - _, _ = fmt.Fprintf(w, `{"txHash":%q,"txStatus":"pending","dataSetId":1001,"pieceCount":1,"piecesAdded":false}`, testAddPiecesTxHash) + _, _ = fmt.Fprintf(w, `{"createMessageHash":%q,"service":"svc","txStatus":"pending","dataSetCreated":false,"ok":null}`, testCreateDataSetTxHash) })) defer server.Close() checker := NewPDPStatusChecker(PDPStatusCheckerOptions{Timeout: 5 * time.Millisecond, AllowPrivateNetworks: true}) - got := checker.CheckAddPiecesStatus(t.Context(), AddPiecesStatusInput{ - ServiceURL: server.URL, - DataSetID: "1001", - TransactionID: testAddPiecesTxHash, - ExpectedPieceCount: 1, + got := checker.CheckDataSetCreationStatus(t.Context(), DataSetCreationStatusInput{ + StatusURL: server.URL + "/pdp/data-sets/created/" + testCreateDataSetTxHash, + TransactionID: testCreateDataSetTxHash, }) if got.State != PDPStatusUnavailable || got.Error == "" { @@ -286,11 +151,9 @@ func TestPDPStatusCheckerClassifiesMalformedResponseUnavailable(t *testing.T) { defer server.Close() checker := NewPDPStatusChecker(PDPStatusCheckerOptions{Timeout: time.Second, AllowPrivateNetworks: true}) - got := checker.CheckAddPiecesStatus(t.Context(), AddPiecesStatusInput{ - ServiceURL: server.URL, - DataSetID: "1001", - TransactionID: testAddPiecesTxHash, - ExpectedPieceCount: 1, + got := checker.CheckDataSetCreationStatus(t.Context(), DataSetCreationStatusInput{ + StatusURL: server.URL + "/pdp/data-sets/created/" + testCreateDataSetTxHash, + TransactionID: testCreateDataSetTxHash, }) if got.State != PDPStatusUnavailable || got.Error == "" { @@ -349,73 +212,16 @@ func TestPDPStatusCheckerClassifiesDataSetCreationIdentityMismatch(t *testing.T) } } -func TestPDPStatusCheckerRejectsConfirmedPieceIDCountMismatch(t *testing.T) { - tests := []struct { - name string - confirmedPieceIDs string - }{ - {name: "missing", confirmedPieceIDs: `[]`}, - {name: "excess", confirmedPieceIDs: `[2001,2002]`}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = fmt.Fprintf(w, `{"txHash":%q,"txStatus":"confirmed","dataSetId":1001,"pieceCount":1,"addMessageOk":true,"piecesAdded":true,"confirmedPieceIds":%s}`, testAddPiecesTxHash, tt.confirmedPieceIDs) - })) - defer server.Close() - - checker := NewPDPStatusChecker(PDPStatusCheckerOptions{Timeout: time.Second, AllowPrivateNetworks: true}) - got := checker.CheckAddPiecesStatus(t.Context(), AddPiecesStatusInput{ - ServiceURL: server.URL, - DataSetID: "1001", - TransactionID: testAddPiecesTxHash, - ExpectedPieceCount: 1, - }) - - if got.State != PDPStatusMismatch { - t.Fatalf("status = %#v, want mismatch for confirmed PieceID count", got) - } - }) - } -} - -func TestPDPStatusCheckerRejectsInvalidExpectedPieceCountBeforeRequest(t *testing.T) { - var requests int - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - requests++ - _, _ = fmt.Fprintf(w, `{"txHash":%q,"txStatus":"confirmed","dataSetId":1001,"pieceCount":0,"addMessageOk":true,"piecesAdded":true,"confirmedPieceIds":[]}`, testAddPiecesTxHash) - })) - defer server.Close() - - checker := NewPDPStatusChecker(PDPStatusCheckerOptions{Timeout: time.Second, AllowPrivateNetworks: true}) - got := checker.CheckAddPiecesStatus(t.Context(), AddPiecesStatusInput{ - ServiceURL: server.URL, - DataSetID: "1001", - TransactionID: testAddPiecesTxHash, - ExpectedPieceCount: 0, - }) - - if requests != 0 { - t.Fatalf("requests = %d, want none for invalid expected piece count", requests) - } - if got.State != PDPStatusUnavailable || !strings.Contains(got.Error, "expected piece count") { - t.Fatalf("status = %#v, want unavailable expected piece count error", got) - } -} - func TestPDPStatusCheckerBlocksPrivateStatusURLByDefault(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = fmt.Fprintf(w, `{"txHash":%q,"txStatus":"pending","dataSetId":1001,"pieceCount":1,"piecesAdded":false}`, testAddPiecesTxHash) + _, _ = fmt.Fprintf(w, `{"createMessageHash":%q,"service":"svc","txStatus":"pending","dataSetCreated":false,"ok":null}`, testCreateDataSetTxHash) })) defer server.Close() checker := NewPDPStatusChecker(PDPStatusCheckerOptions{Timeout: time.Second}) - got := checker.CheckAddPiecesStatus(t.Context(), AddPiecesStatusInput{ - StatusURL: server.URL + "/pdp/data-sets/1001/pieces/added/" + testAddPiecesTxHash, - DataSetID: "1001", - TransactionID: testAddPiecesTxHash, - ExpectedPieceCount: 1, + got := checker.CheckDataSetCreationStatus(t.Context(), DataSetCreationStatusInput{ + StatusURL: server.URL + "/pdp/data-sets/created/" + testCreateDataSetTxHash, + TransactionID: testCreateDataSetTxHash, }) if got.State != PDPStatusUnavailable || !strings.Contains(got.Error, "private network") { @@ -425,16 +231,14 @@ func TestPDPStatusCheckerBlocksPrivateStatusURLByDefault(t *testing.T) { func TestPDPStatusCheckerAllowsPrivateStatusURLWhenConfigured(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = fmt.Fprintf(w, `{"txHash":%q,"txStatus":"pending","dataSetId":1001,"pieceCount":1,"piecesAdded":false}`, testAddPiecesTxHash) + _, _ = fmt.Fprintf(w, `{"createMessageHash":%q,"service":"svc","txStatus":"pending","dataSetCreated":false,"ok":null}`, testCreateDataSetTxHash) })) defer server.Close() checker := NewPDPStatusChecker(PDPStatusCheckerOptions{Timeout: time.Second, AllowPrivateNetworks: true}) - got := checker.CheckAddPiecesStatus(t.Context(), AddPiecesStatusInput{ - StatusURL: server.URL + "/pdp/data-sets/1001/pieces/added/" + testAddPiecesTxHash, - DataSetID: "1001", - TransactionID: testAddPiecesTxHash, - ExpectedPieceCount: 1, + got := checker.CheckDataSetCreationStatus(t.Context(), DataSetCreationStatusInput{ + StatusURL: server.URL + "/pdp/data-sets/created/" + testCreateDataSetTxHash, + TransactionID: testCreateDataSetTxHash, }) if got.State != PDPStatusPending || got.Error != "" { @@ -446,18 +250,16 @@ func TestPDPStatusCheckerDoesNotUseEnvironmentProxy(t *testing.T) { var proxyRequests int proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { proxyRequests++ - _, _ = fmt.Fprintf(w, `{"txHash":%q,"txStatus":"confirmed","dataSetId":1001,"pieceCount":1,"addMessageOk":true,"piecesAdded":true,"confirmedPieceIds":[2001]}`, testAddPiecesTxHash) + _, _ = fmt.Fprintf(w, `{"createMessageHash":%q,"service":"svc","txStatus":"pending","dataSetCreated":false,"ok":null}`, testCreateDataSetTxHash) })) defer proxy.Close() t.Setenv("HTTP_PROXY", proxy.URL) t.Setenv("http_proxy", proxy.URL) checker := NewPDPStatusChecker(PDPStatusCheckerOptions{Timeout: 50 * time.Millisecond, AllowPrivateNetworks: true}) - got := checker.CheckAddPiecesStatus(t.Context(), AddPiecesStatusInput{ - StatusURL: "http://proxy-only.invalid/pdp/data-sets/1001/pieces/added/" + testAddPiecesTxHash, - DataSetID: "1001", - TransactionID: testAddPiecesTxHash, - ExpectedPieceCount: 1, + got := checker.CheckDataSetCreationStatus(t.Context(), DataSetCreationStatusInput{ + StatusURL: "http://proxy-only.invalid/pdp/data-sets/created/" + testCreateDataSetTxHash, + TransactionID: testCreateDataSetTxHash, }) if proxyRequests != 0 { @@ -473,7 +275,7 @@ func TestPDPStatusCheckerDoesNotFollowRedirects(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/redirected" { redirected++ - _, _ = fmt.Fprintf(w, `{"txHash":%q,"txStatus":"confirmed","dataSetId":1001,"pieceCount":1,"addMessageOk":true,"piecesAdded":true,"confirmedPieceIds":[2001]}`, testAddPiecesTxHash) + _, _ = fmt.Fprintf(w, `{"createMessageHash":%q,"service":"svc","txStatus":"pending","dataSetCreated":false,"ok":null}`, testCreateDataSetTxHash) return } http.Redirect(w, r, "/redirected", http.StatusFound) @@ -481,11 +283,9 @@ func TestPDPStatusCheckerDoesNotFollowRedirects(t *testing.T) { defer server.Close() checker := NewPDPStatusChecker(PDPStatusCheckerOptions{Timeout: time.Second, AllowPrivateNetworks: true}) - got := checker.CheckAddPiecesStatus(t.Context(), AddPiecesStatusInput{ - StatusURL: server.URL + "/start", - DataSetID: "1001", - TransactionID: testAddPiecesTxHash, - ExpectedPieceCount: 1, + got := checker.CheckDataSetCreationStatus(t.Context(), DataSetCreationStatusInput{ + StatusURL: server.URL + "/pdp/data-sets/created/" + testCreateDataSetTxHash, + TransactionID: testCreateDataSetTxHash, }) if redirected != 0 { @@ -509,11 +309,9 @@ func TestPDPStatusCheckerBlocksIPv6SpecialUseAddresses(t *testing.T) { for _, value := range tests { t.Run(value, func(t *testing.T) { checker := NewPDPStatusChecker(PDPStatusCheckerOptions{Timeout: time.Second}) - got := checker.CheckAddPiecesStatus(t.Context(), AddPiecesStatusInput{ - StatusURL: "http://[" + value + "]:80/pdp/data-sets/1001/pieces/added/" + testAddPiecesTxHash, - DataSetID: "1001", - TransactionID: testAddPiecesTxHash, - ExpectedPieceCount: 1, + got := checker.CheckDataSetCreationStatus(t.Context(), DataSetCreationStatusInput{ + StatusURL: "http://[" + value + "]:80/pdp/data-sets/created/" + testCreateDataSetTxHash, + TransactionID: testCreateDataSetTxHash, }) if got.State != PDPStatusUnavailable || !strings.Contains(got.Error, "private network") { diff --git a/internal/testutil/storagecommit.go b/internal/testutil/storagecommit.go index 10f6684..f1bfe6f 100644 --- a/internal/testutil/storagecommit.go +++ b/internal/testutil/storagecommit.go @@ -51,10 +51,12 @@ func CommitStorageCopy( input.CommitAttemptID = fmt.Sprintf("attempt-%d-%d", input.ContentID, input.StorageCopyID) } now := time.Now() + statusURL := "https://provider.example/status/" + input.CommitAttemptID attempt := &storagecommit.Attempt{ AttemptID: input.CommitAttemptID, ContentID: copyRow.ContentID, StorageDataSetID: copyRow.StorageDataSetID, Status: storagecommit.AttemptStatusAttempted, ExtraDataHex: &input.CommitExtraDataHex, TransactionID: &input.CommitTransactionID, + StatusURL: &statusURL, AttemptedAt: &now, CreatedAt: now, UpdatedAt: now, } if _, err := db.NewInsert().Model(attempt).Exec(ctx); err != nil { diff --git a/internal/worker/storage_task_handlers.go b/internal/worker/storage_task_handlers.go index 1ed76d7..041cbe0 100644 --- a/internal/worker/storage_task_handlers.go +++ b/internal/worker/storage_task_handlers.go @@ -1288,7 +1288,7 @@ func (h *TaskHandlers) runCommit(ctx context.Context, execution taskengine.Execu target, err := h.openReadyDataSet(ctx, binding) if err != nil { if copyRow.CommitAttemptedAt != nil && copyRow.CommitAttemptID != nil { - advancer := storagecommit.Advancer{Store: h.deps.Repositories.Contents, StatusChecker: h.deps.CommitStatus} + advancer := storagecommit.Advancer{Store: h.deps.Repositories.Contents} advanced, advanceErr := advancer.AdvanceUnavailable(ctx, *copyRow, *binding) if advanceErr != nil { return h.retryCopyTask(execution, input, copyRow, advanceErr, "commit_recovery_failed") @@ -1302,7 +1302,7 @@ func (h *TaskHandlers) runCommit(ctx context.Context, execution taskengine.Execu } return taskengine.Suspend(model.TaskResumeModeRecover, storagePollInterval, "provider_confirmation", "Checking storage registration", nil) } - advancer := storagecommit.Advancer{Store: h.deps.Repositories.Contents, StatusChecker: h.deps.CommitStatus} + advancer := storagecommit.Advancer{Store: h.deps.Repositories.Contents} advance := func(ctx context.Context) (storagecommit.AdvanceResult, error) { return advancer.Advance(ctx, storagecommit.AdvanceInput{ Copy: *copyRow, Binding: *binding, Target: target, diff --git a/internal/worker/task_handlers.go b/internal/worker/task_handlers.go index 1c7af54..99ec45d 100644 --- a/internal/worker/task_handlers.go +++ b/internal/worker/task_handlers.go @@ -12,7 +12,6 @@ import ( "github.com/strahe/synaps3/internal/cacheaccess" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/observability" - "github.com/strahe/synaps3/internal/storagecommit" "github.com/strahe/synaps3/internal/synapse" taskengine "github.com/strahe/synaps3/internal/task" ) @@ -33,7 +32,6 @@ type TaskHandlerDependencies struct { Terminator synapse.ServiceTerminator Epochs synapse.ChainEpochReader Observability *observability.Service - CommitStatus storagecommit.AddPiecesStatusChecker ParkedPieces synapse.ParkedPieceChecker EvictionPolicy cache.EvictionPolicy MaxCacheBytes int64 diff --git a/ui/src/api/client.ts b/ui/src/api/client.ts index f6073fb..5317309 100644 --- a/ui/src/api/client.ts +++ b/ui/src/api/client.ts @@ -553,7 +553,6 @@ export type ObjectUploadCopyStatus = 'pending' | 'piece_ready' | 'committing' | export type StorageCommitAttentionCode = | 'attempt_only_ambiguous' | 'unattributed_piece' - | 'invalid_submission' | 'submission_mismatch' | 'data_set_unavailable' | 'confirmation_timeout' diff --git a/ui/src/lib/storage-confirmation-attention.ts b/ui/src/lib/storage-confirmation-attention.ts index 7daa996..2a58a03 100644 --- a/ui/src/lib/storage-confirmation-attention.ts +++ b/ui/src/lib/storage-confirmation-attention.ts @@ -3,7 +3,6 @@ import type { StorageCommitAttentionCode } from '@/api/client' const knownAttentionLabels = { attempt_only_ambiguous: 'Submission result unknown', unattributed_piece: 'Piece ownership unknown', - invalid_submission: 'Saved submission is invalid', submission_mismatch: 'Confirmation does not match', data_set_unavailable: 'Data set is unavailable', confirmation_timeout: 'Confirmation timed out', diff --git a/ui/test/storage-confirmation-attention.test.ts b/ui/test/storage-confirmation-attention.test.ts index 7488ad2..e2908ba 100644 --- a/ui/test/storage-confirmation-attention.test.ts +++ b/ui/test/storage-confirmation-attention.test.ts @@ -10,7 +10,6 @@ test('known confirmation reasons use operator-facing labels', () => { const cases = [ ['attempt_only_ambiguous', 'Submission result unknown'], ['unattributed_piece', 'Piece ownership unknown'], - ['invalid_submission', 'Saved submission is invalid'], ['submission_mismatch', 'Confirmation does not match'], ['data_set_unavailable', 'Data set is unavailable'], ['confirmation_timeout', 'Confirmation timed out'], From 9a7c6152407898763c37badc46f9887922b3f067 Mon Sep 17 00:00:00 2001 From: Lee <7932644+strahe@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:20:59 +0800 Subject: [PATCH 5/5] refactor(storage): simplify submission validation --- internal/db/repository/storage_commit_repo.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/db/repository/storage_commit_repo.go b/internal/db/repository/storage_commit_repo.go index 81b47ba..4a472b0 100644 --- a/internal/db/repository/storage_commit_repo.go +++ b/internal/db/repository/storage_commit_repo.go @@ -198,7 +198,7 @@ func (r *BunStorageContentRepo) MarkCommitAttempted( } func (r *BunStorageContentRepo) RecordCommitSubmission(ctx context.Context, input storagecommit.EvidenceInput) error { - if err := validateCommitEvidenceInput(input, true); err != nil { + if err := validateCommitSubmissionInput(input); err != nil { return err } return r.mutateAttempt(ctx, input.Copy, "recording storage commit submission", func(db bun.IDB, _ int64) error { @@ -718,11 +718,11 @@ func validateCommitCopyIdentity(identity storagecommit.CopyIdentity) error { return nil } -func validateCommitEvidenceInput(input storagecommit.EvidenceInput, requireSubmission bool) error { +func validateCommitSubmissionInput(input storagecommit.EvidenceInput) error { if err := validateCommitCopyIdentity(input.Copy); err != nil || input.AttemptID == "" || input.TransactionID == "" || - (requireSubmission && input.StatusURL == "") { + input.StatusURL == "" { return fmt.Errorf("recording storage commit evidence: %w", ErrInvalidInput) } return nil