From 950ad24b49d66d4a0a7e9e006339cf64d4e80246 Mon Sep 17 00:00:00 2001 From: Lee <7932644+strahe@users.noreply.github.com> Date: Thu, 24 Sep 2026 11:42:25 +0800 Subject: [PATCH 1/4] fix(s3): honor object ranges and verify compatibility matrix --- .github/workflows/ci.yml | 27 ++ Makefile | 5 +- docs/en/concepts/write-path-cache.md | 2 + docs/en/reference/s3-compatibility.md | 4 +- docs/zh/concepts/write-path-cache.md | 2 + docs/zh/reference/s3-compatibility.md | 4 +- internal/backend/integration_test.go | 74 ++++++ internal/backend/multipart.go | 4 +- internal/backend/object.go | 29 ++- internal/backend/object_range.go | 80 ++++++ internal/backend/object_range_test.go | 62 +++++ internal/backend/object_test.go | 61 +++++ internal/objectreader/reader.go | 3 + tests/system/s3_clients_test.go | 183 ++++++++++++++ tests/system/s3_matrix_test.go | 347 ++++++++++++++++++++++++++ 15 files changed, 879 insertions(+), 8 deletions(-) create mode 100644 internal/backend/object_range.go create mode 100644 internal/backend/object_range_test.go create mode 100644 tests/system/s3_clients_test.go create mode 100644 tests/system/s3_matrix_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad9a630..f6cdb50 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,6 +114,33 @@ jobs: - name: Test system runtime run: make test-system + - name: Install pinned S3 clients + run: | + set -euo pipefail + tools_dir="$(mktemp -d)" + case "$(uname -m)" in + x86_64) aws_arch=x86_64; client_arch=amd64 ;; + aarch64|arm64) aws_arch=aarch64; client_arch=arm64 ;; + *) echo "Unsupported S3 client architecture" >&2; exit 1 ;; + esac + mkdir -p "$RUNNER_TEMP/s3-client-bin" + curl --fail --location --retry 3 --silent --show-error \ + "https://awscli.amazonaws.com/awscli-exe-linux-${aws_arch}-2.31.0.zip" -o "$tools_dir/aws.zip" + unzip -q "$tools_dir/aws.zip" -d "$tools_dir" + "$tools_dir/aws/install" --install-dir "$RUNNER_TEMP/aws-cli" --bin-dir "$RUNNER_TEMP/s3-client-bin" + curl --fail --location --retry 3 --silent --show-error \ + "https://downloads.rclone.org/v1.71.0/rclone-v1.71.0-linux-${client_arch}.zip" -o "$tools_dir/rclone.zip" + unzip -q "$tools_dir/rclone.zip" -d "$tools_dir" + install "$tools_dir/rclone-v1.71.0-linux-${client_arch}/rclone" "$RUNNER_TEMP/s3-client-bin/rclone" + curl --fail --location --retry 3 --silent --show-error \ + "https://github.com/minio/mc/releases/download/RELEASE.2025-08-13T08-35-41Z/mc.linux-${client_arch}.RELEASE.2025-08-13T08-35-41Z" \ + -o "$RUNNER_TEMP/s3-client-bin/mc" + chmod +x "$RUNNER_TEMP/s3-client-bin/mc" + echo "$RUNNER_TEMP/s3-client-bin" >> "$GITHUB_PATH" + + - name: Test AWS CLI, rclone, and MinIO Client + run: make test-s3-clients + - name: Build dashboard systemtest server run: make build-systemtest-server diff --git a/Makefile b/Makefile index 73b93ad..c749ad9 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ LDFLAGS := -X $(MODULE)/internal/buildinfo.Version=$(VERSION) \ -X $(MODULE)/internal/buildinfo.Commit=$(COMMIT) \ -X $(MODULE)/internal/buildinfo.Date=$(DATE) -.PHONY: all build build-go build-systemtest-server build-integration-server docs-build test test-fast test-race test-system test-integration test-ui-e2e test-docker-entrypoint test-docker-deployment lint fmt check verify-e2e verify-fast verify-norace verify-race clean run ui-install ui-build ui-dev ui-e2e-install +.PHONY: all build build-go build-systemtest-server build-integration-server docs-build test test-fast test-race test-system test-s3-clients test-integration test-ui-e2e test-docker-entrypoint test-docker-deployment lint fmt check verify-e2e verify-fast verify-norace verify-race clean run ui-install ui-build ui-dev ui-e2e-install .PHONY: docker-init docker-up docker-verify docker-down docker-status docker-logs docker-password all: build @@ -55,6 +55,9 @@ test-race: test-system: $(CGO) go test $(GOFLAGS) -tags='dev systemtest' -count=1 ./tests/testutil/... ./internal/systemtest ./tests/system +test-s3-clients: + SYNAPS3_TEST_S3_CLIENTS=1 $(CGO) go test $(GOFLAGS) -tags='dev systemtest' -count=1 -run '^TestS3Clients$$' ./tests/system + test-integration: build-integration-server $(CGO) go test -v $(GOFLAGS) -tags=integration -count=1 -timeout=45m ./tests/integration/... diff --git a/docs/en/concepts/write-path-cache.md b/docs/en/concepts/write-path-cache.md index dbd9c11..c5aef4a 100644 --- a/docs/en/concepts/write-path-cache.md +++ b/docs/en/concepts/write-path-cache.md @@ -30,6 +30,8 @@ The S3 response does not wait for Filecoin provider latency. After the write is `GetObject` reads local cache first. If the cache entry is missing and an available remote copy is recorded, SynapS3 can retrieve the object from the storage provider, verify it, serve the response, and restore the local cache when possible. +A single byte-range request returns only the requested bytes. On a cache miss, SynapS3 still downloads the complete remote object to verify it before the range response finishes; a complete read can also restore the local cache. Closing the request early does not leave a partial cache entry. + Successful foreground cache opens refresh the entry's LRU access time. This includes S3 object and range reads, cached CopyObject sources, Admin content downloads, and version restores. Metadata-only operations such as `HeadObject` do not refresh it, and the background Uploader does not make an entry look recently used. A complete remote rehydration starts a new LRU age for the restored entry. Repeated reads of the same version coalesce access-time updates to at most one database write per minute. Access tracking is best effort and never turns a successful read into an error. diff --git a/docs/en/reference/s3-compatibility.md b/docs/en/reference/s3-compatibility.md index 4aa229e..8104b36 100644 --- a/docs/en/reference/s3-compatibility.md +++ b/docs/en/reference/s3-compatibility.md @@ -40,8 +40,8 @@ SynapS3 mainly supports path-style S3 access for writing bucket and object data | Ownership controls | `PutBucketOwnershipControls` | Partial | Accepts only `BucketOwnerPreferred`; rejects other ownership modes. | | Ownership controls | `DeleteBucketOwnershipControls` | Partial | Keeps the ACL-compatible `BucketOwnerPreferred` behavior. | | Object | `PutObject` | Supported | Stores an object through the cache-first write model. | -| Object | `GetObject` | Supported | Reads from cache or committed remote storage. | -| Object | `HeadObject` | Supported | Reads object metadata. | +| Object | `GetObject` | Supported | Reads from cache or committed remote storage, including single byte ranges, version metadata, and `Last-Modified`. A cold remote range still downloads the complete source for integrity verification. | +| Object | `HeadObject` | Supported | Reads object metadata, including custom metadata on the requested version. | | Object | `DeleteObject` | Supported | Creates a delete marker without `versionId`; with `versionId`, deletes an eligible data version or delete marker. | | Object | `DeleteObjects` | Supported | Applies the same version-aware deletion rules to each entry and reports entry-specific failures. | | Object | `CopyObject` | Supported | Source object must be readable from cache or committed remote storage. | diff --git a/docs/zh/concepts/write-path-cache.md b/docs/zh/concepts/write-path-cache.md index 93f9c53..437ba5f 100644 --- a/docs/zh/concepts/write-path-cache.md +++ b/docs/zh/concepts/write-path-cache.md @@ -30,6 +30,8 @@ SynapS3 会校验请求,保存对象及其元数据,再返回 S3 兼容的 E `GetObject` 会先读本地缓存。缓存缺失时,如果记录了可用的远端副本,SynapS3 可以从存储提供方取回并校验对象、返回响应,并在可能时恢复本地缓存。 +单段字节 Range 请求只返回所需字节。缓存缺失时,SynapS3 仍会下载完整远端对象,并在区间响应完成前校验内容;完整读取也可以恢复本地缓存。请求提前关闭不会留下不完整的缓存条目。 + 前台成功打开缓存时,会刷新该条目的 LRU 访问时间。这包括 S3 对象和 Range 读取、命中缓存的 CopyObject 源、Admin 内容下载和版本恢复。`HeadObject` 等只读元数据操作不会刷新,后台 Uploader 读取也不会让条目看起来刚被使用。远端内容完整回填后,会为恢复的缓存重新开始计算 LRU 时间。 同一版本的重复读取会合并访问时间更新,每分钟最多写一次数据库。访问时间按 best-effort 方式记录,更新失败不会让原本成功的读取失败。 diff --git a/docs/zh/reference/s3-compatibility.md b/docs/zh/reference/s3-compatibility.md index cc8d70a..06821f5 100644 --- a/docs/zh/reference/s3-compatibility.md +++ b/docs/zh/reference/s3-compatibility.md @@ -40,8 +40,8 @@ SynapS3 主要支持 path-style S3 访问,负责把存储桶和对象数据写 | Ownership Controls | `PutBucketOwnershipControls` | 部分支持 | 只接受 `BucketOwnerPreferred`,拒绝其他 ownership modes。 | | Ownership Controls | `DeleteBucketOwnershipControls` | 部分支持 | 保持 ACL 兼容的 `BucketOwnerPreferred` 行为。 | | 对象 | `PutObject` | 支持 | 按缓存优先的写入模型存储对象。 | -| 对象 | `GetObject` | 支持 | 从缓存或已提交的远端存储读取。 | -| 对象 | `HeadObject` | 支持 | 读取对象元数据。 | +| 对象 | `GetObject` | 支持 | 从缓存或已提交的远端存储读取,支持单段字节 Range,并返回指定版本的自定义元数据和 `Last-Modified`。冷缓存远端 Range 仍需完整下载来源以校验内容。 | +| 对象 | `HeadObject` | 支持 | 读取对象元数据,包括指定版本的自定义元数据。 | | 对象 | `DeleteObject` | 支持 | 不带 `versionId` 时创建 delete marker;带 `versionId` 时删除符合条件的数据版本或 delete marker。 | | 对象 | `DeleteObjects` | 支持 | 对每个条目应用相同的版本删除规则,并分别返回失败结果。 | | 对象 | `CopyObject` | 支持 | 源对象必须可从缓存或已提交的远端存储读取。 | diff --git a/internal/backend/integration_test.go b/internal/backend/integration_test.go index 2c8d1b4..c8d8267 100644 --- a/internal/backend/integration_test.go +++ b/internal/backend/integration_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/ipfs/go-cid" @@ -259,6 +260,79 @@ func TestIntegration_ColdReadAfterEviction(t *testing.T) { } } +func TestIntegration_ColdRangeRehydratesOnlyAfterCompleteRead(t *testing.T) { + ib := newIntegrationBackend(t) + ctx := t.Context() + bucket := testutil.SeedBucket(t, ib.db, "range-remote-bucket") + content := strings.Repeat("remote range content", 12) + putObject(t, ib.backend, bucket.Name, "range.bin", content) + version, err := ib.repos.Objects.GetCurrentVersionByBucketAndKey(ctx, bucket.ID, "range.bin") + if err != nil || version == nil { + t.Fatalf("version = %#v, %v", version, err) + } + pieceCID := buildDummyCID(t) + acceptBackendVersionUpload(t, ib.db, ib.repos, version.VersionID, pieceCID, "https://provider.example/range") + cacheKey := version.CacheKey() + evict := func() { + t.Helper() + if err := ib.repos.Objects.ClearContentCachePresence(ctx, *version.ContentID); err != nil { + t.Fatal(err) + } + if err := ib.cache.Delete(ctx, bucket.Name, cacheKey); err != nil { + t.Fatal(err) + } + } + evict() + readBytes := 0 + ib.storage.DownloadFunc = func(_ context.Context, _ cid.Cid, _ *storage.DownloadOptions) (io.ReadCloser, error) { + return io.NopCloser(&countingReader{Reader: strings.NewReader(content), count: &readBytes}), nil + } + getRange := func() *s3.GetObjectOutput { + t.Helper() + out, err := ib.backend.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket.Name), Key: aws.String("range.bin"), Range: aws.String("bytes=2-5"), + }) + if err != nil { + t.Fatal(err) + } + return out + } + out := getRange() + got, err := io.ReadAll(out.Body) + if closeErr := out.Body.Close(); closeErr != nil { + t.Fatal(closeErr) + } + if err != nil || string(got) != content[2:6] || readBytes != len(content) { + t.Fatalf("cold range = %q, %v; source read %d/%d bytes", got, err, readBytes, len(content)) + } + for attempt := 0; attempt < 200 && !ib.cache.Exists(ctx, bucket.Name, cacheKey); attempt++ { + time.Sleep(10 * time.Millisecond) + } + if !ib.cache.Exists(ctx, bucket.Name, cacheKey) { + t.Fatal("complete cold range did not rehydrate cache") + } + + evict() + out = getRange() + if err := out.Body.Close(); err != nil { + t.Fatal(err) + } + if ib.cache.Exists(ctx, bucket.Name, cacheKey) { + t.Fatal("early close committed an incomplete cache entry") + } +} + +type countingReader struct { + io.Reader + count *int +} + +func (r *countingReader) Read(p []byte) (int, error) { + n, err := r.Reader.Read(p) + *r.count += n + return n, err +} + func TestIntegration_MultipartUpload_Abort(t *testing.T) { ib := newIntegrationBackend(t) ctx := context.Background() diff --git a/internal/backend/multipart.go b/internal/backend/multipart.go index 70451a8..8dfc3d2 100644 --- a/internal/backend/multipart.go +++ b/internal/backend/multipart.go @@ -136,6 +136,9 @@ func (b *SynapseBackend) UploadPartCopy(ctx context.Context, input *s3.UploadPar if partNum < 1 || partNum > 10000 { return s3response.CopyPartResult{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgPartNumber, fmt.Sprint(*input.PartNumber)) } + if input.CopySourceRange != nil && *input.CopySourceRange != "" { + return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNotImplemented) + } // Parse and validate source object srcBucketName, srcKey, srcVersionID, err := parseCopySource(*input.CopySource) @@ -154,7 +157,6 @@ func (b *SynapseBackend) UploadPartCopy(ctx context.Context, input *s3.UploadPar } defer func() { _ = srcResult.Body.Close() }() - // NOTE: CopySourceRange for partial copies is not yet supported (future enhancement). cacheInfo, err := b.cache.PutPart(ctx, *input.UploadId, partNum, objectlimits.LimitFOCUploadReader(srcResult.Body)) if err != nil { if errors.Is(err, objectlimits.ErrTooLarge) { diff --git a/internal/backend/object.go b/internal/backend/object.go index 153da0f..6e7a0cf 100644 --- a/internal/backend/object.go +++ b/internal/backend/object.go @@ -27,6 +27,7 @@ import ( "github.com/strahe/synaps3/internal/storagecleanup" "github.com/strahe/synaps3/internal/storagepipeline" taskengine "github.com/strahe/synaps3/internal/task" + versitybackend "github.com/versity/versitygw/backend" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3response" ) @@ -200,15 +201,36 @@ func (b *SynapseBackend) GetObject(ctx context.Context, input *s3.GetObjectInput admin.CacheHitsTotal.Inc() } - admin.ObjectOperationsTotal.WithLabelValues("get", "success").Inc() etag := fmt.Sprintf(`"%s"`, out.ETag) contentType := out.ContentType + acceptRanges := "bytes" + length := out.Size + var contentRange *string + if input.Range != nil && *input.Range != "" { + start, count, valid, rangeErr := versitybackend.ParseObjectRange(out.Size, *input.Range) + if rangeErr != nil { + _ = out.Body.Close() + admin.ObjectOperationsTotal.WithLabelValues("get", "failure").Inc() + return nil, rangeErr + } + if valid { + out.Body = newRangeReadCloser(out.Body, start, count, out.Source == objectreader.SourceProvider) + length = count + rangeValue := fmt.Sprintf("bytes %d-%d/%d", start, start+count-1, out.Size) + contentRange = &rangeValue + } + } + admin.ObjectOperationsTotal.WithLabelValues("get", "success").Inc() return &s3.GetObjectOutput{ Body: out.Body, - ContentLength: &out.Size, + ContentLength: &length, + ContentRange: contentRange, + AcceptRanges: &acceptRanges, ETag: &etag, ContentType: &contentType, VersionId: &out.VersionID, + LastModified: &out.LastModified, + Metadata: out.Metadata, }, nil } @@ -239,6 +261,7 @@ func (b *SynapseBackend) HeadObject(ctx context.Context, input *s3.HeadObjectInp ContentType: &meta.ContentType, LastModified: &meta.LastModified, VersionId: &meta.VersionID, + Metadata: meta.Metadata, }, nil } @@ -1074,6 +1097,7 @@ type objectMetadataResult struct { VersionID string MultipartUploadID *string LastModified time.Time + Metadata map[string]string } func (b *SynapseBackend) objectMetadata(ctx context.Context, bucketID int64, key, versionID string) (objectMetadataResult, error) { @@ -1090,6 +1114,7 @@ func (b *SynapseBackend) objectMetadata(ctx context.Context, bucketID int64, key VersionID: version.VersionID, MultipartUploadID: version.MultipartUploadID, LastModified: version.CreatedAt, + Metadata: maps.Clone(version.Metadata), }, nil } diff --git a/internal/backend/object_range.go b/internal/backend/object_range.go new file mode 100644 index 0000000..27a3f36 --- /dev/null +++ b/internal/backend/object_range.go @@ -0,0 +1,80 @@ +package backend + +import ( + "errors" + "io" +) + +// rangeReadCloser keeps the original stream open until the requested response +// finishes. A remote stream must reach EOF to validate its piece CID and finish +// the cache rehydration; its final response bytes are held until that succeeds. +type rangeReadCloser struct { + source io.ReadCloser + skip int64 + remaining int64 + validate bool + finished bool +} + +func newRangeReadCloser(source io.ReadCloser, start, length int64, validate bool) io.ReadCloser { + return &rangeReadCloser{source: source, skip: start, remaining: length, validate: validate} +} + +func (r *rangeReadCloser) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + if r.finished { + return 0, io.EOF + } + if r.skip > 0 { + if _, err := io.CopyN(io.Discard, r.source, r.skip); err != nil { + return 0, err + } + r.skip = 0 + } + if int64(len(p)) > r.remaining { + p = p[:int(r.remaining)] + } + n, err := r.source.Read(p) + r.remaining -= int64(n) + if err != nil && !errors.Is(err, io.EOF) { + if r.validate && r.remaining == 0 { + return 0, err + } + return n, err + } + if r.remaining > 0 { + if errors.Is(err, io.EOF) { + return n, io.ErrUnexpectedEOF + } + return n, nil + } + if r.validate { + var discard [32 * 1024]byte + noProgress := 0 + for { + count, drainErr := r.source.Read(discard[:]) + if errors.Is(drainErr, io.EOF) { + break + } + if drainErr != nil { + return 0, drainErr + } + if count == 0 { + noProgress++ + if noProgress >= 100 { + return 0, io.ErrNoProgress + } + } else { + noProgress = 0 + } + } + } + r.finished = true + return n, nil +} + +func (r *rangeReadCloser) Close() error { + return r.source.Close() +} diff --git a/internal/backend/object_range_test.go b/internal/backend/object_range_test.go new file mode 100644 index 0000000..1db90a8 --- /dev/null +++ b/internal/backend/object_range_test.go @@ -0,0 +1,62 @@ +package backend + +import ( + "bytes" + "errors" + "io" + "testing" +) + +type failingRangeSource struct { + *bytes.Reader + err error +} + +func (s *failingRangeSource) Read(p []byte) (int, error) { + n, err := s.Reader.Read(p) + if errors.Is(err, io.EOF) { + return 0, s.err + } + return n, err +} + +func (*failingRangeSource) Close() error { return nil } + +func TestRemoteRangeWaitsForSourceValidation(t *testing.T) { + sourceErr := errors.New("piece checksum mismatch") + body := newRangeReadCloser(&failingRangeSource{Reader: bytes.NewReader([]byte("abcdefgh")), err: sourceErr}, 2, 3, true) + defer func() { _ = body.Close() }() + buf := make([]byte, 3) + n, err := body.Read(buf) + if n != 0 || !errors.Is(err, sourceErr) { + t.Fatalf("final range read = %d, %v; want withheld bytes and source error", n, err) + } +} + +type finalChunkErrorSource struct{ err error } + +func (s *finalChunkErrorSource) Read(p []byte) (int, error) { + copy(p, "abc") + return 3, s.err +} + +func (*finalChunkErrorSource) Close() error { return nil } + +func TestRemoteRangeWithholdsFinalChunkReturnedWithError(t *testing.T) { + sourceErr := errors.New("piece checksum mismatch") + body := newRangeReadCloser(&finalChunkErrorSource{err: sourceErr}, 0, 3, true) + defer func() { _ = body.Close() }() + n, err := body.Read(make([]byte, 3)) + if n != 0 || !errors.Is(err, sourceErr) { + t.Fatalf("final range read = %d, %v; want withheld bytes and source error", n, err) + } +} + +func TestCachedRangeStopsAtRequestedLength(t *testing.T) { + body := newRangeReadCloser(io.NopCloser(bytes.NewReader([]byte("abcdefgh"))), 2, 3, false) + defer func() { _ = body.Close() }() + got, err := io.ReadAll(body) + if err != nil || string(got) != "cde" { + t.Fatalf("range body = %q, %v; want cde", got, err) + } +} diff --git a/internal/backend/object_test.go b/internal/backend/object_test.go index f7152a2..6ba4baf 100644 --- a/internal/backend/object_test.go +++ b/internal/backend/object_test.go @@ -1218,6 +1218,67 @@ func TestGetObject_FromCache(t *testing.T) { } } +func TestGetObjectRangeAndVersionMetadata(t *testing.T) { + tb := newTestBackend(t) + ctx := t.Context() + seedActiveBucket(t, tb, "range-bucket") + body := validTestObjectBody("range-body") + put, err := tb.backend.PutObject(ctx, s3response.PutObjectInput{ + Bucket: aws.String("range-bucket"), Key: aws.String("file.txt"), Body: strings.NewReader(body), + Metadata: map[string]string{"custom": "first"}, + }) + if err != nil { + t.Fatalf("PutObject: %v", err) + } + putValidTestObject(t, tb, "range-bucket", "file.txt", "new version") + get, err := tb.backend.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String("range-bucket"), Key: aws.String("file.txt"), VersionId: &put.VersionID, + Range: aws.String("bytes=2-5"), + }) + if err != nil { + t.Fatalf("GetObject range: %v", err) + } + data, readErr := io.ReadAll(get.Body) + _ = get.Body.Close() + if readErr != nil || string(data) != body[2:6] || *get.ContentLength != 4 || + get.ContentRange == nil || *get.ContentRange != fmt.Sprintf("bytes 2-5/%d", len(body)) || + get.LastModified == nil || get.Metadata["custom"] != "first" { + t.Fatalf("range output = %#v body=%q readErr=%v", get, data, readErr) + } + head, err := tb.backend.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String("range-bucket"), Key: aws.String("file.txt"), VersionId: &put.VersionID, + }) + if err != nil || head.Metadata["custom"] != "first" { + t.Fatalf("HeadObject metadata = %#v, %v", head, err) + } + for _, tc := range []struct { + name, request, expected string + }{ + {"open-ended", "bytes=3-", body[3:]}, + {"suffix", "bytes=-4", body[len(body)-4:]}, + } { + t.Run(tc.name, func(t *testing.T) { + out, err := tb.backend.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String("range-bucket"), Key: aws.String("file.txt"), VersionId: &put.VersionID, Range: &tc.request, + }) + if err != nil { + t.Fatal(err) + } + got, readErr := io.ReadAll(out.Body) + _ = out.Body.Close() + if readErr != nil || string(got) != tc.expected || out.ContentRange == nil || *out.ContentLength != int64(len(tc.expected)) { + t.Fatalf("range %q = %q, %#v, %v", tc.request, got, out, readErr) + } + }) + } + _, err = tb.backend.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String("range-bucket"), Key: aws.String("file.txt"), Range: aws.String("bytes=999999-"), + }) + if err == nil { + t.Fatal("out-of-range GetObject succeeded") + } +} + func TestGetObject_WithVersionIDReadsSpecifiedVersion(t *testing.T) { tb := newTestBackend(t) ctx := context.Background() diff --git a/internal/objectreader/reader.go b/internal/objectreader/reader.go index bf899a9..4c623b6 100644 --- a/internal/objectreader/reader.go +++ b/internal/objectreader/reader.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "log/slog" + "maps" "os" "sync" "time" @@ -55,6 +56,7 @@ type Result struct { VersionID string ContentType string LastModified time.Time + Metadata map[string]string Source Source CacheMiss bool } @@ -242,6 +244,7 @@ func resultFromVersion(version *model.ObjectVersion, body io.ReadCloser, source VersionID: version.VersionID, ContentType: version.ContentType, LastModified: version.CreatedAt, + Metadata: maps.Clone(version.Metadata), Source: source, CacheMiss: cacheMiss, } diff --git a/tests/system/s3_clients_test.go b/tests/system/s3_clients_test.go new file mode 100644 index 0000000..fff8140 --- /dev/null +++ b/tests/system/s3_clients_test.go @@ -0,0 +1,183 @@ +//go:build systemtest + +package system_test + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "io" + "log/slog" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/strahe/synaps3/internal/systemtest" + "github.com/strahe/synaps3/tests/testutil/e2e" +) + +// TestS3Clients is enabled explicitly in CI so the regular system tests need no external CLIs. +func TestS3Clients(t *testing.T) { + if os.Getenv("SYNAPS3_TEST_S3_CLIENTS") != "1" { + t.Skip("set SYNAPS3_TEST_S3_CLIENTS=1 to run external S3 clients") + } + for _, name := range []string{"aws", "rclone", "mc"} { + if _, err := exec.LookPath(name); err != nil { + t.Fatalf("required S3 client %s is unavailable: %v", name, err) + } + } + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + harness, err := systemtest.NewHarness(t.Context(), logger) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := harness.Close(ctx); err != nil { + t.Error(err) + } + }) + endpoint := startS3LoopbackProxy(t, harness.S3SocketPath()) + client := e2e.NewUnixSocketS3Client(harness.S3SocketPath(), systemtest.OwnerAccess, systemtest.OwnerSecret) + bucket := aws.String("s3-clients") + if _, err := client.CreateBucket(t.Context(), &s3.CreateBucketInput{Bucket: bucket}); err != nil { + t.Fatal(err) + } + e2e.Eventually(t, t.Context(), 10*time.Second, "bucket provisioning", func(ctx context.Context) (*s3.PutObjectOutput, bool, error) { + out, err := client.PutObject(ctx, &s3.PutObjectInput{Bucket: bucket, Key: aws.String("readiness.bin"), Body: bytes.NewReader(bytes.Repeat([]byte("ready"), 26))}) + return out, err == nil, err + }) + + temp := t.TempDir() + payload := bytes.Repeat([]byte("SynapS3 offline CLI compatibility\n"), 8192) + input := filepath.Join(temp, "input.bin") + if err := os.WriteFile(input, payload, 0o600); err != nil { + t.Fatal(err) + } + want := sha256.Sum256(payload) + configFile := filepath.Join(temp, "aws-config") + if err := os.WriteFile(configFile, []byte("[default]\nregion = us-east-1\ns3 =\n addressing_style = path\n"), 0o600); err != nil { + t.Fatal(err) + } + commonEnv := []string{ + "HTTP_PROXY=", "HTTPS_PROXY=", "ALL_PROXY=", + "http_proxy=", "https_proxy=", "all_proxy=", + "NO_PROXY=127.0.0.1,localhost", "no_proxy=127.0.0.1,localhost", + "AWS_ACCESS_KEY_ID=" + systemtest.OwnerAccess, + "AWS_SECRET_ACCESS_KEY=" + systemtest.OwnerSecret, + "AWS_DEFAULT_REGION=us-east-1", + "AWS_PROFILE=default", + "AWS_EC2_METADATA_DISABLED=true", + "AWS_MAX_ATTEMPTS=2", + "AWS_CONFIG_FILE=" + configFile, + "AWS_SHARED_CREDENTIALS_FILE=" + filepath.Join(temp, "no-credentials"), + "RCLONE_CONFIG=" + filepath.Join(temp, "no-rclone-config"), + "RCLONE_CONFIG_MATRIX_TYPE=s3", + "RCLONE_CONFIG_MATRIX_PROVIDER=Other", + "RCLONE_CONFIG_MATRIX_ACCESS_KEY_ID=" + systemtest.OwnerAccess, + "RCLONE_CONFIG_MATRIX_SECRET_ACCESS_KEY=" + systemtest.OwnerSecret, + "RCLONE_CONFIG_MATRIX_ENDPOINT=" + endpoint, + "RCLONE_CONFIG_MATRIX_REGION=us-east-1", + "RCLONE_CONFIG_MATRIX_FORCE_PATH_STYLE=true", + "MC_CONFIG_DIR=" + filepath.Join(temp, "mc"), + "MC_HOST_matrix=" + strings.Replace(endpoint, "http://", "http://"+systemtest.OwnerAccess+":"+systemtest.OwnerSecret+"@", 1), + } + ctx, cancel := context.WithTimeout(t.Context(), 90*time.Second) + defer cancel() + + for _, tc := range []struct { + name string + put []string + get []string + }{ + { + "AWS CLI", + []string{"aws", "--endpoint-url", endpoint, "s3", "cp", "--only-show-errors", input, "s3://s3-clients/aws.bin"}, + []string{"aws", "--endpoint-url", endpoint, "s3", "cp", "--only-show-errors", "s3://s3-clients/aws.bin"}, + }, + { + "rclone", + []string{"rclone", "copyto", input, "matrix:s3-clients/rclone.bin", "--retries", "1", "--low-level-retries", "1"}, + []string{"rclone", "copyto", "matrix:s3-clients/rclone.bin", "--retries", "1", "--low-level-retries", "1"}, + }, + { + "mc", + []string{"mc", "--quiet", "cp", input, "matrix/s3-clients/mc.bin"}, + []string{"mc", "--quiet", "cp", "matrix/s3-clients/mc.bin"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + if err := runS3Client(ctx, commonEnv, tc.put...); err != nil { + t.Fatalf("upload: %v", err) + } + output := filepath.Join(temp, strings.ReplaceAll(tc.name, " ", "-")+"-output.bin") + if err := runS3Client(ctx, commonEnv, append(tc.get, output)...); err != nil { + t.Fatalf("download: %v", err) + } + got, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if digest := sha256.Sum256(got); digest != want { + t.Fatalf("SHA256 mismatch: got %x, want %x", digest, want) + } + }) + } +} + +func runS3Client(ctx context.Context, extraEnv []string, args ...string) error { + cmd := exec.CommandContext(ctx, args[0], args[1:]...) + cmd.Env = append(os.Environ(), extraEnv...) + output, err := cmd.CombinedOutput() + if err != nil { + message := strings.ReplaceAll(string(output), systemtest.OwnerSecret, "[redacted]") + message = strings.ReplaceAll(message, systemtest.OwnerAccess, "[redacted]") + return fmt.Errorf("%s failed: %w: %s", args[0], err, message) + } + return nil +} + +func startS3LoopbackProxy(t *testing.T, socket string) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + go func() { + for { + incoming, err := listener.Accept() + if err != nil { + return + } + go func() { + defer func() { _ = incoming.Close() }() + upstream, err := net.Dial("unix", socket) + if err != nil { + return + } + defer func() { _ = upstream.Close() }() + copyDone := make(chan struct{}) + go func() { + defer close(copyDone) + _, _ = io.Copy(upstream, incoming) + if unix, ok := upstream.(*net.UnixConn); ok { + _ = unix.CloseWrite() + } + }() + _, _ = io.Copy(incoming, upstream) + <-copyDone + }() + } + }() + return "http://" + listener.Addr().String() +} diff --git a/tests/system/s3_matrix_test.go b/tests/system/s3_matrix_test.go new file mode 100644 index 0000000..27f9b61 --- /dev/null +++ b/tests/system/s3_matrix_test.go @@ -0,0 +1,347 @@ +//go:build systemtest + +package system_test + +import ( + "bytes" + "context" + "errors" + "io" + "log/slog" + "os" + "slices" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "github.com/strahe/synaps3/internal/systemtest" + "github.com/strahe/synaps3/tests/testutil/e2e" +) + +// Each subtest names an operation promised by the public S3 compatibility matrix. +// The SDK sends signed HTTP requests to the real gateway over the harness socket. +func TestS3CompatibilityMatrix(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) + harness, err := systemtest.NewHarness(t.Context(), logger) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := harness.Close(ctx); err != nil { + t.Error(err) + } + }) + client := e2e.NewUnixSocketS3Client(harness.S3SocketPath(), systemtest.OwnerAccess, systemtest.OwnerSecret) + ctx := t.Context() + bucket := aws.String("s3-matrix") + key := aws.String("item.bin") + body := bytes.Repeat([]byte("matrix object\n"), 20) + var firstVersion string + var firstCopyMarker string + var batchCopyMarker string + + t.Run("CreateBucket", func(t *testing.T) { + if _, err := client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: bucket}); err != nil { + t.Fatal(err) + } + }) + t.Run("HeadBucket", func(t *testing.T) { + if _, err := client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: bucket}); err != nil { + t.Fatal(err) + } + }) + t.Run("ListBuckets", func(t *testing.T) { + out, err := client.ListBuckets(ctx, &s3.ListBucketsInput{}) + if err != nil || !slices.ContainsFunc(out.Buckets, func(b types.Bucket) bool { return aws.ToString(b.Name) == *bucket }) { + t.Fatalf("bucket absent: %#v, %v", out, err) + } + }) + t.Run("GetBucketVersioning", func(t *testing.T) { + out, err := client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{Bucket: bucket}) + if err != nil || out.Status != types.BucketVersioningStatusEnabled { + t.Fatalf("versioning = %#v, %v", out, err) + } + }) + t.Run("PutBucketVersioning", func(t *testing.T) { + _, err := client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ + Bucket: bucket, VersioningConfiguration: &types.VersioningConfiguration{Status: types.BucketVersioningStatusEnabled}, + }) + if err != nil { + t.Fatalf("Enabled: %v", err) + } + _, err = client.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{ + Bucket: bucket, VersioningConfiguration: &types.VersioningConfiguration{Status: types.BucketVersioningStatusSuspended}, + }) + requireS3ErrorCode(t, err, "InvalidBucketState") + }) + t.Run("GetBucketAcl", func(t *testing.T) { + out, err := client.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: bucket}) + if err != nil || out.Owner == nil { + t.Fatalf("ACL = %#v, %v", out, err) + } + }) + t.Run("PutBucketAcl", func(t *testing.T) { + e2e.Eventually(t, ctx, 10*time.Second, "bucket provisioning for ACL write", func(ctx context.Context) (*s3.PutBucketAclOutput, bool, error) { + out, err := client.PutBucketAcl(ctx, &s3.PutBucketAclInput{Bucket: bucket, ACL: types.BucketCannedACLPublicRead}) + return out, err == nil, err + }) + out, err := client.GetBucketAcl(ctx, &s3.GetBucketAclInput{Bucket: bucket}) + if err != nil || !slices.ContainsFunc(out.Grants, func(grant types.Grant) bool { + return grant.Grantee != nil && grant.Grantee.Type == types.TypeGroup && grant.Permission == types.PermissionRead + }) { + t.Fatalf("public-read ACL was not persisted: grants=%+v, %v", out.Grants, err) + } + }) + t.Run("GetBucketOwnershipControls", func(t *testing.T) { + out, err := client.GetBucketOwnershipControls(ctx, &s3.GetBucketOwnershipControlsInput{Bucket: bucket}) + if err != nil || out.OwnershipControls == nil || len(out.OwnershipControls.Rules) != 1 || + out.OwnershipControls.Rules[0].ObjectOwnership != types.ObjectOwnershipBucketOwnerPreferred { + t.Fatalf("ownership = %#v, %v", out, err) + } + }) + t.Run("PutBucketOwnershipControls", func(t *testing.T) { + _, err := client.PutBucketOwnershipControls(ctx, &s3.PutBucketOwnershipControlsInput{ + Bucket: bucket, + OwnershipControls: &types.OwnershipControls{Rules: []types.OwnershipControlsRule{{ObjectOwnership: types.ObjectOwnershipBucketOwnerPreferred}}}, + }) + if err != nil { + t.Fatalf("BucketOwnerPreferred: %v", err) + } + _, err = client.PutBucketOwnershipControls(ctx, &s3.PutBucketOwnershipControlsInput{ + Bucket: bucket, + OwnershipControls: &types.OwnershipControls{Rules: []types.OwnershipControlsRule{{ObjectOwnership: types.ObjectOwnershipBucketOwnerEnforced}}}, + }) + requireS3ErrorCode(t, err, "InvalidArgument") + }) + t.Run("DeleteBucketOwnershipControls", func(t *testing.T) { + if _, err := client.DeleteBucketOwnershipControls(ctx, &s3.DeleteBucketOwnershipControlsInput{Bucket: bucket}); err != nil { + t.Fatal(err) + } + out, err := client.GetBucketOwnershipControls(ctx, &s3.GetBucketOwnershipControlsInput{Bucket: bucket}) + if err != nil || out.OwnershipControls == nil || len(out.OwnershipControls.Rules) != 1 || + out.OwnershipControls.Rules[0].ObjectOwnership != types.ObjectOwnershipBucketOwnerPreferred { + t.Fatalf("ACL-compatible ownership lost: %#v, %v", out, err) + } + }) + t.Run("PutObject", func(t *testing.T) { + out := e2e.Eventually(t, ctx, 10*time.Second, "bucket provisioning", func(ctx context.Context) (*s3.PutObjectOutput, bool, error) { + got, err := client.PutObject(ctx, &s3.PutObjectInput{Bucket: bucket, Key: key, Body: bytes.NewReader(body), Metadata: map[string]string{"custom": "first"}}) + return got, err == nil, err + }) + firstVersion = aws.ToString(out.VersionId) + if firstVersion == "" { + t.Fatal("missing version ID") + } + second := e2e.Eventually(t, ctx, 10*time.Second, "second object version", func(ctx context.Context) (*s3.PutObjectOutput, bool, error) { + got, err := client.PutObject(ctx, &s3.PutObjectInput{Bucket: bucket, Key: key, Body: bytes.NewReader(body), Metadata: map[string]string{"custom": "second"}}) + return got, err == nil, err + }) + if aws.ToString(second.VersionId) == firstVersion { + t.Fatal("overwrite reused the first version ID") + } + }) + t.Run("GetObject", func(t *testing.T) { + out, err := client.GetObject(ctx, &s3.GetObjectInput{Bucket: bucket, Key: key, VersionId: &firstVersion, Range: aws.String("bytes=2-5")}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = out.Body.Close() }() + got, err := io.ReadAll(out.Body) + if err != nil || !bytes.Equal(got, body[2:6]) || out.ContentRange == nil || out.LastModified == nil || out.Metadata["custom"] != "first" { + t.Fatalf("range GET = %q, %#v, %v", got, out, err) + } + _, err = client.GetObject(ctx, &s3.GetObjectInput{Bucket: bucket, Key: key, Range: aws.String("bytes=999999-")}) + requireS3ErrorCode(t, err, "InvalidRange") + }) + t.Run("HeadObject", func(t *testing.T) { + out, err := client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: bucket, Key: key, VersionId: &firstVersion}) + if err != nil || out.LastModified == nil || out.Metadata["custom"] != "first" { + t.Fatalf("HEAD = %#v, %v", out, err) + } + }) + t.Run("CopyObject", func(t *testing.T) { + out, err := client.CopyObject(ctx, &s3.CopyObjectInput{Bucket: bucket, Key: aws.String("z-copy.bin"), CopySource: aws.String(*bucket + "/" + *key)}) + if err != nil || out.CopyObjectResult == nil { + t.Fatalf("copy = %#v, %v", out, err) + } + }) + for _, operation := range []string{"ListObjects", "ListObjectsV2"} { + t.Run(operation, func(t *testing.T) { + switch operation { + case "ListObjects": + out, err := client.ListObjects(ctx, &s3.ListObjectsInput{Bucket: bucket, MaxKeys: aws.Int32(1)}) + if err != nil || len(out.Contents) != 1 || !aws.ToBool(out.IsTruncated) { + t.Fatalf("ListObjects = %#v, %v", out, err) + } + page, err := client.ListObjects(ctx, &s3.ListObjectsInput{Bucket: bucket, Marker: out.Contents[0].Key}) + if err != nil || len(page.Contents) != 1 || aws.ToString(page.Contents[0].Key) != "z-copy.bin" { + t.Fatalf("ListObjects marker page = %#v, %v", page, err) + } + case "ListObjectsV2": + out, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: bucket, MaxKeys: aws.Int32(1)}) + if err != nil || len(out.Contents) != 1 || out.NextContinuationToken == nil { + t.Fatalf("ListObjectsV2 = %#v, %v", out, err) + } + page, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: bucket, ContinuationToken: out.NextContinuationToken}) + if err != nil || len(page.Contents) != 1 || aws.ToString(page.Contents[0].Key) != "z-copy.bin" { + t.Fatalf("ListObjectsV2 continuation page = %#v, %v", page, err) + } + } + }) + } + t.Run("DeleteObject", func(t *testing.T) { + out, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: bucket, Key: aws.String("z-copy.bin")}) + if err != nil || !aws.ToBool(out.DeleteMarker) { + t.Fatalf("delete = %#v, %v", out, err) + } + firstCopyMarker = aws.ToString(out.VersionId) + if firstCopyMarker == "" { + t.Fatal("delete marker has no version ID") + } + }) + t.Run("ListObjectVersions", func(t *testing.T) { + out, err := client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{Bucket: bucket}) + if err != nil || len(out.Versions) < 3 || len(out.DeleteMarkers) == 0 || !slices.ContainsFunc(out.Versions, func(v types.ObjectVersion) bool { + return aws.ToString(v.VersionId) == firstVersion + }) { + t.Fatalf("versions and delete markers = %#v, %v", out, err) + } + }) + t.Run("DeleteObjects", func(t *testing.T) { + out, err := client.DeleteObjects(ctx, &s3.DeleteObjectsInput{Bucket: bucket, Delete: &types.Delete{Objects: []types.ObjectIdentifier{{Key: key}, {Key: aws.String("z-copy.bin")}}}}) + if err != nil || len(out.Deleted) != 2 || len(out.Errors) != 0 { + t.Fatalf("batch delete = %#v, %v", out, err) + } + for _, deleted := range out.Deleted { + if aws.ToString(deleted.Key) == "z-copy.bin" { + batchCopyMarker = aws.ToString(deleted.DeleteMarkerVersionId) + } + } + if batchCopyMarker == "" { + t.Fatal("batch delete did not return the copy delete marker version") + } + partial, err := client.DeleteObjects(ctx, &s3.DeleteObjectsInput{Bucket: bucket, Delete: &types.Delete{Objects: []types.ObjectIdentifier{ + {Key: aws.String("z-copy.bin"), VersionId: &firstCopyMarker}, + {Key: key, VersionId: aws.String("missing-version")}, + }}}) + if err != nil || len(partial.Deleted) != 1 || len(partial.Errors) != 1 { + t.Fatalf("batch version deletion and entry error = %#v, %v", partial, err) + } + }) + t.Run("DeleteObject/versionId", func(t *testing.T) { + out, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: bucket, Key: aws.String("z-copy.bin"), VersionId: &batchCopyMarker}) + if err != nil || !aws.ToBool(out.DeleteMarker) { + t.Fatalf("delete marker version = %#v, %v", out, err) + } + }) + + partKey := aws.String("multipart.bin") + var uploadID string + var partETags []types.CompletedPart + t.Run("CreateMultipartUpload", func(t *testing.T) { + out, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{Bucket: bucket, Key: partKey, Metadata: map[string]string{"source": "multipart"}}) + if err != nil { + t.Fatal(err) + } + uploadID = aws.ToString(out.UploadId) + }) + t.Run("UploadPart", func(t *testing.T) { + for partNumber, data := range [][]byte{bytes.Repeat([]byte("a"), 5<<20), bytes.Repeat([]byte("b"), 128)} { + out, err := client.UploadPart(ctx, &s3.UploadPartInput{Bucket: bucket, Key: partKey, UploadId: &uploadID, PartNumber: aws.Int32(int32(partNumber + 1)), Body: bytes.NewReader(data)}) + if err != nil { + t.Fatalf("part %d: %v", partNumber+1, err) + } + partETags = append(partETags, types.CompletedPart{PartNumber: aws.Int32(int32(partNumber + 1)), ETag: out.ETag}) + } + }) + t.Run("ListMultipartUploads", func(t *testing.T) { + out, err := client.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{Bucket: bucket}) + if err != nil || len(out.Uploads) == 0 { + t.Fatalf("uploads = %#v, %v", out, err) + } + }) + t.Run("ListParts", func(t *testing.T) { + out, err := client.ListParts(ctx, &s3.ListPartsInput{Bucket: bucket, Key: partKey, UploadId: &uploadID}) + if err != nil || len(out.Parts) != 2 { + t.Fatalf("parts = %#v, %v", out, err) + } + }) + t.Run("CompleteMultipartUpload", func(t *testing.T) { + _, err := client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: bucket, Key: partKey, UploadId: &uploadID, + MultipartUpload: &types.CompletedMultipartUpload{Parts: partETags}, + }) + if err != nil { + t.Fatal(err) + } + }) + t.Run("GetObjectAttributes", func(t *testing.T) { + out, err := client.GetObjectAttributes(ctx, &s3.GetObjectAttributesInput{ + Bucket: bucket, Key: partKey, + ObjectAttributes: []types.ObjectAttributes{types.ObjectAttributesEtag, types.ObjectAttributesObjectParts, types.ObjectAttributesObjectSize}, + }) + if err != nil || out.ObjectParts == nil || len(out.ObjectParts.Parts) != 2 || out.ObjectParts.TotalPartsCount != nil || + aws.ToInt64(out.ObjectSize) != (5<<20)+128 || out.ETag == nil { + t.Fatalf("attributes = %#v, %v", out, err) + } + }) + t.Run("UploadPartCopy", func(t *testing.T) { + out, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{Bucket: bucket, Key: aws.String("copy-part.bin")}) + if err != nil { + t.Fatal(err) + } + copyUploadID := aws.ToString(out.UploadId) + copySource := *bucket + "/" + *partKey + copied, err := client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: bucket, Key: aws.String("copy-part.bin"), UploadId: ©UploadID, + PartNumber: aws.Int32(1), CopySource: ©Source, + }) + if err != nil || copied.CopyPartResult == nil { + t.Fatalf("whole copy = %#v, %v", copied, err) + } + _, err = client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: bucket, Key: aws.String("copy-part.bin"), UploadId: ©UploadID, + PartNumber: aws.Int32(2), CopySource: ©Source, CopySourceRange: aws.String("bytes=0-127"), + }) + requireS3ErrorCode(t, err, "NotImplemented") + _, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: bucket, Key: aws.String("copy-part.bin"), UploadId: ©UploadID, + MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{{PartNumber: aws.Int32(1), ETag: copied.CopyPartResult.ETag}}}, + }) + if err != nil { + t.Fatal(err) + } + get, err := client.GetObject(ctx, &s3.GetObjectInput{Bucket: bucket, Key: aws.String("copy-part.bin")}) + if err != nil { + t.Fatal(err) + } + copiedBody, readErr := io.ReadAll(get.Body) + _ = get.Body.Close() + want := append(bytes.Repeat([]byte("a"), 5<<20), bytes.Repeat([]byte("b"), 128)...) + if readErr != nil || !bytes.Equal(copiedBody, want) { + t.Fatalf("whole-object part copy returned %d bytes, read error %v", len(copiedBody), readErr) + } + }) + t.Run("AbortMultipartUpload", func(t *testing.T) { + out, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{Bucket: bucket, Key: aws.String("abort.bin")}) + if err != nil { + t.Fatal(err) + } + if _, err := client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{Bucket: bucket, Key: aws.String("abort.bin"), UploadId: out.UploadId}); err != nil { + t.Fatal(err) + } + }) +} + +func requireS3ErrorCode(t *testing.T, err error, want string) { + t.Helper() + var apiErr smithy.APIError + if !errors.As(err, &apiErr) || apiErr.ErrorCode() != want { + t.Fatalf("S3 error = %v; want %s", err, want) + } +} From 070180293db9e4eb17a9049917ab4f26889b04f1 Mon Sep 17 00:00:00 2001 From: Lee <7932644+strahe@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:23:13 +0800 Subject: [PATCH 2/4] ci(s3): split compatibility checks and report matrix results --- .github/workflows/ci.yml | 27 ---- .github/workflows/s3-compatibility.yml | 80 ++++++++++ Makefile | 7 +- README.md | 5 +- scripts/s3_compatibility_report.py | 202 ++++++++++++++++++++++++ scripts/test_s3_compatibility_report.py | 101 ++++++++++++ tests/system/s3_clients_test.go | 8 +- tests/system/s3_matrix_test.go | 2 +- 8 files changed, 395 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/s3-compatibility.yml create mode 100644 scripts/s3_compatibility_report.py create mode 100644 scripts/test_s3_compatibility_report.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6cdb50..ad9a630 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,33 +114,6 @@ jobs: - name: Test system runtime run: make test-system - - name: Install pinned S3 clients - run: | - set -euo pipefail - tools_dir="$(mktemp -d)" - case "$(uname -m)" in - x86_64) aws_arch=x86_64; client_arch=amd64 ;; - aarch64|arm64) aws_arch=aarch64; client_arch=arm64 ;; - *) echo "Unsupported S3 client architecture" >&2; exit 1 ;; - esac - mkdir -p "$RUNNER_TEMP/s3-client-bin" - curl --fail --location --retry 3 --silent --show-error \ - "https://awscli.amazonaws.com/awscli-exe-linux-${aws_arch}-2.31.0.zip" -o "$tools_dir/aws.zip" - unzip -q "$tools_dir/aws.zip" -d "$tools_dir" - "$tools_dir/aws/install" --install-dir "$RUNNER_TEMP/aws-cli" --bin-dir "$RUNNER_TEMP/s3-client-bin" - curl --fail --location --retry 3 --silent --show-error \ - "https://downloads.rclone.org/v1.71.0/rclone-v1.71.0-linux-${client_arch}.zip" -o "$tools_dir/rclone.zip" - unzip -q "$tools_dir/rclone.zip" -d "$tools_dir" - install "$tools_dir/rclone-v1.71.0-linux-${client_arch}/rclone" "$RUNNER_TEMP/s3-client-bin/rclone" - curl --fail --location --retry 3 --silent --show-error \ - "https://github.com/minio/mc/releases/download/RELEASE.2025-08-13T08-35-41Z/mc.linux-${client_arch}.RELEASE.2025-08-13T08-35-41Z" \ - -o "$RUNNER_TEMP/s3-client-bin/mc" - chmod +x "$RUNNER_TEMP/s3-client-bin/mc" - echo "$RUNNER_TEMP/s3-client-bin" >> "$GITHUB_PATH" - - - name: Test AWS CLI, rclone, and MinIO Client - run: make test-s3-clients - - name: Build dashboard systemtest server run: make build-systemtest-server diff --git a/.github/workflows/s3-compatibility.yml b/.github/workflows/s3-compatibility.yml new file mode 100644 index 0000000..d5caf11 --- /dev/null +++ b/.github/workflows/s3-compatibility.yml @@ -0,0 +1,80 @@ +name: S3 Compatibility + +on: + push: + branches: + - main + pull_request: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + compatibility: + runs-on: ${{ vars.RUNS_ON || 'ubuntu-latest' }} + timeout-minutes: 25 + env: + GOTOOLCHAIN: auto + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + + - name: Validate report rules + run: python3 -m unittest discover -s scripts -p 'test_s3_compatibility_report.py' + + - name: Install pinned S3 clients + run: | + set -euo pipefail + tools_dir="$(mktemp -d)" + case "$(uname -m)" in + x86_64) aws_arch=x86_64; client_arch=amd64 ;; + aarch64|arm64) aws_arch=aarch64; client_arch=arm64 ;; + *) echo "Unsupported S3 client architecture" >&2; exit 1 ;; + esac + mkdir -p "$RUNNER_TEMP/s3-client-bin" + curl --fail --location --retry 3 --silent --show-error \ + "https://awscli.amazonaws.com/awscli-exe-linux-${aws_arch}-2.31.0.zip" -o "$tools_dir/aws.zip" + unzip -q "$tools_dir/aws.zip" -d "$tools_dir" + "$tools_dir/aws/install" --install-dir "$RUNNER_TEMP/aws-cli" --bin-dir "$RUNNER_TEMP/s3-client-bin" + curl --fail --location --retry 3 --silent --show-error \ + "https://downloads.rclone.org/v1.71.0/rclone-v1.71.0-linux-${client_arch}.zip" -o "$tools_dir/rclone.zip" + unzip -q "$tools_dir/rclone.zip" -d "$tools_dir" + install "$tools_dir/rclone-v1.71.0-linux-${client_arch}/rclone" "$RUNNER_TEMP/s3-client-bin/rclone" + curl --fail --location --retry 3 --silent --show-error \ + "https://github.com/minio/mc/releases/download/RELEASE.2025-08-13T08-35-41Z/mc.linux-${client_arch}.RELEASE.2025-08-13T08-35-41Z" \ + -o "$RUNNER_TEMP/s3-client-bin/mc" + chmod +x "$RUNNER_TEMP/s3-client-bin/mc" + echo "$RUNNER_TEMP/s3-client-bin" >> "$GITHUB_PATH" + + - name: Test documented S3 operations + run: | + go test -json -trimpath -tags='dev systemtest s3compat' -count=1 \ + -run '^TestS3CompatibilityMatrix$' ./tests/system > "$RUNNER_TEMP/s3-matrix.json" + + - name: Test AWS CLI, rclone, and MinIO Client + if: ${{ !cancelled() }} + run: | + go test -json -trimpath -tags='dev systemtest s3compat' -count=1 \ + -run '^TestS3Clients$' ./tests/system > "$RUNNER_TEMP/s3-clients.json" + + - name: Publish compatibility summary + if: ${{ always() }} + env: + SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + python3 scripts/s3_compatibility_report.py \ + --matrix-json "$RUNNER_TEMP/s3-matrix.json" \ + --clients-json "$RUNNER_TEMP/s3-clients.json" \ + --summary "$GITHUB_STEP_SUMMARY" \ + --source-sha "$SOURCE_SHA" \ + --checkout-sha "$(git rev-parse HEAD)" diff --git a/Makefile b/Makefile index c749ad9..fc6b4b8 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ LDFLAGS := -X $(MODULE)/internal/buildinfo.Version=$(VERSION) \ -X $(MODULE)/internal/buildinfo.Commit=$(COMMIT) \ -X $(MODULE)/internal/buildinfo.Date=$(DATE) -.PHONY: all build build-go build-systemtest-server build-integration-server docs-build test test-fast test-race test-system test-s3-clients test-integration test-ui-e2e test-docker-entrypoint test-docker-deployment lint fmt check verify-e2e verify-fast verify-norace verify-race clean run ui-install ui-build ui-dev ui-e2e-install +.PHONY: all build build-go build-systemtest-server build-integration-server docs-build test test-fast test-race test-system test-s3-compatibility test-s3-clients test-integration test-ui-e2e test-docker-entrypoint test-docker-deployment lint fmt check verify-e2e verify-fast verify-norace verify-race clean run ui-install ui-build ui-dev ui-e2e-install .PHONY: docker-init docker-up docker-verify docker-down docker-status docker-logs docker-password all: build @@ -55,8 +55,11 @@ test-race: test-system: $(CGO) go test $(GOFLAGS) -tags='dev systemtest' -count=1 ./tests/testutil/... ./internal/systemtest ./tests/system +test-s3-compatibility: + $(CGO) go test $(GOFLAGS) -tags='dev systemtest s3compat' -count=1 -run '^(TestS3CompatibilityMatrix|TestS3Clients)$$' ./tests/system + test-s3-clients: - SYNAPS3_TEST_S3_CLIENTS=1 $(CGO) go test $(GOFLAGS) -tags='dev systemtest' -count=1 -run '^TestS3Clients$$' ./tests/system + $(CGO) go test $(GOFLAGS) -tags='dev systemtest s3compat' -count=1 -run '^TestS3Clients$$' ./tests/system test-integration: build-integration-server $(CGO) go test -v $(GOFLAGS) -tags=integration -count=1 -timeout=45m ./tests/integration/... diff --git a/README.md b/README.md index 8d77724..5f86054 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # SynapS3 [![CI](https://github.com/strahe/SynapS3/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/strahe/SynapS3/actions/workflows/ci.yml) +[![S3 Compatibility](https://github.com/strahe/SynapS3/actions/workflows/s3-compatibility.yml/badge.svg?branch=main)](https://github.com/strahe/SynapS3/actions/workflows/s3-compatibility.yml) [![Package](https://img.shields.io/badge/package-GHCR-blue?logo=github)](https://github.com/strahe/SynapS3/pkgs/container/synaps3) [![Go Report](https://goreportcard.com/badge/github.com/strahe/synaps3)](https://goreportcard.com/report/github.com/strahe/synaps3) [![Go Version](https://img.shields.io/github/go-mod/go-version/strahe/SynapS3?filename=go.mod)](go.mod) @@ -50,7 +51,7 @@ Writes commit to local cache and metadata before returning success. Reads use lo | Object | `ListObjects` | ✅ | Marker pagination | | Object | `ListObjectsV2` | ✅ | Continuation-token pagination | | Object | `ListObjectVersions` | ✅ | Lists object versions and delete markers | -| Object | `GetObjectAttributes` | ✅ | Reports metadata and multipart `ObjectParts`; `TotalPartsCount` is not emitted | +| Object | `GetObjectAttributes` | ⚠️ | Reports metadata and multipart `ObjectParts`; `TotalPartsCount` is not emitted | | Multipart | `CreateMultipartUpload` | ✅ | Starts an upload | | Multipart | `UploadPart` | ✅ | Uploads one part | | Multipart | `UploadPartCopy` | ⚠️ | Whole-object copy only; range copy is not supported | @@ -59,6 +60,8 @@ Writes commit to local cache and metadata before returning success. Reads use lo | Multipart | `ListMultipartUploads` | ✅ | Lists open uploads | | Multipart | `ListParts` | ✅ | Lists uploaded parts | +The [S3 Compatibility runs](https://github.com/strahe/SynapS3/actions/workflows/s3-compatibility.yml) verify this matrix offline with [operation tests](https://github.com/strahe/SynapS3/blob/main/tests/system/s3_matrix_test.go) and [AWS CLI, rclone, and MinIO Client tests](https://github.com/strahe/SynapS3/blob/main/tests/system/s3_clients_test.go). No Filecoin network or wallet transactions are used. To run them locally, install the three clients and run `make test-s3-compatibility`. + ## License See [LICENSE](LICENSE). diff --git a/scripts/s3_compatibility_report.py b/scripts/s3_compatibility_report.py new file mode 100644 index 0000000..4a8be53 --- /dev/null +++ b/scripts/s3_compatibility_report.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Validate the documented S3 matrix against black-box Go test results.""" + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + + +MATRIX_ROOT = "TestS3CompatibilityMatrix" +CLIENT_ROOT = "TestS3Clients" +CLIENTS = {"AWS_CLI": ("aws", "--version"), "rclone": ("rclone", "version"), "mc": ("mc", "--version")} +STATUS = { + "Supported": "supported", + "Partial": "partial", + "Not supported": "unsupported", + "支持": "supported", + "部分支持": "partial", + "不支持": "unsupported", + "✅": "supported", + "⚠️": "partial", + "❌": "unsupported", +} + + +def matrix_rows(path, heading): + source = Path(path).read_text(encoding="utf-8") + marker = f"## {heading}\n" + if source.count(marker) != 1: + raise ValueError(f"{path}: expected one {heading} section") + section = source.split(marker, 1)[1].split("\n## ", 1)[0] + rows = {} + for line in section.splitlines(): + if not line.startswith("|"): + continue + columns = [column.strip() for column in line.strip().strip("|").split("|")] + if len(columns) != 4: + raise ValueError(f"{path}: expected four columns in {line!r}") + if columns[1] in {"Operation", "操作"} or columns[1].startswith("---"): + continue + area, names, raw_status, _ = columns + if raw_status not in STATUS: + raise ValueError(f"{path}: unknown status {raw_status!r} for {names}") + operations = re.findall(r"`([^`]+)`", names) + if not operations or re.sub(r"`[^`]+`|[,\s]", "", names): + raise ValueError(f"{path}: cannot parse operations in {names!r}") + for operation in operations: + if operation in rows: + raise ValueError(f"{path}: duplicate operation {operation}") + rows[operation] = (STATUS[raw_status], area) + if not rows: + raise ValueError(f"{path}: operation matrix is empty") + return rows + + +def validate_matrices(root): + en = matrix_rows(root / "docs/en/reference/s3-compatibility.md", "Operation Matrix") + zh = matrix_rows(root / "docs/zh/reference/s3-compatibility.md", "操作矩阵") + readme = matrix_rows(root / "README.md", "Core S3 Compatibility") + errors = [] + for operation in sorted(en.keys() | zh.keys()): + if operation not in en or operation not in zh: + errors.append(f"English/Chinese matrix operation differs: {operation}") + elif en[operation][0] != zh[operation][0]: + errors.append(f"English/Chinese matrix status differs: {operation}") + for operation, (status, _) in readme.items(): + if operation not in en: + errors.append(f"README operation absent from full matrix: {operation}") + elif status != en[operation][0]: + errors.append(f"README status differs from full matrix: {operation}") + promised = {name: value for name, value in en.items() if value[0] in {"supported", "partial"}} + if not promised: + errors.append("No supported or partial operations found") + return promised, errors + + +def test_results(path): + results = {} + package_results = [] + with Path(path).open(encoding="utf-8") as stream: + for line_number, line in enumerate(stream, 1): + try: + event = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"{path}:{line_number}: invalid Go test JSON: {exc}") from exc + action = event.get("Action") + if action not in {"pass", "fail", "skip"}: + continue + test = event.get("Test") + if test: + results.setdefault(test, []).append(action) + else: + package_results.append(action) + if not results and not package_results: + raise ValueError(f"{path}: no completed Go tests") + return results, package_results + + +def result_status(results, test): + actions = results.get(test, []) + if len(actions) != 1: + return "missing" if not actions else "duplicate" + return actions[0] + + +def client_versions(): + versions = {} + for name, command in CLIENTS.items(): + try: + output = subprocess.run(command, capture_output=True, text=True, timeout=10, check=True) + versions[name] = (output.stdout or output.stderr).splitlines()[0].strip() + except (OSError, subprocess.SubprocessError, IndexError): + versions[name] = "unavailable" + return versions + + +def cell(value): + return str(value).replace("|", "\\|").replace("\n", " ").replace("\r", " ") + + +def report(root, matrix_json, clients_json, source_sha, checkout_sha, versions): + errors = [] + try: + promised, matrix_errors = validate_matrices(root) + errors.extend(matrix_errors) + except (OSError, ValueError) as exc: + promised = {} + errors.append(str(exc)) + + test_sets = {} + for label, path in (("matrix", matrix_json), ("clients", clients_json)): + try: + test_sets[label] = test_results(path) + except (OSError, ValueError) as exc: + test_sets[label] = ({}, []) + errors.append(str(exc)) + + matrix_tests, matrix_package = test_sets["matrix"] + client_tests, client_package = test_sets["clients"] + for label, tests, package, root_test in ( + ("matrix", matrix_tests, matrix_package, MATRIX_ROOT), + ("clients", client_tests, client_package, CLIENT_ROOT), + ): + if result_status(tests, root_test) != "pass": + errors.append(f"{label} root test did not pass: {root_test}") + if package != ["pass"]: + errors.append(f"{label} Go package did not pass") + + lines = [ + "# S3 Compatibility", + "", + f"PR source commit: `{cell(source_sha)}` ", + f"Tested checkout: `{cell(checkout_sha)}`", + "", + "## S3 operations", + "", + "| Operation | Matrix status | Test result |", + "| --- | --- | --- |", + ] + for operation, (status, _) in promised.items(): + result = result_status(matrix_tests, f"{MATRIX_ROOT}/{operation}") + if result != "pass": + errors.append(f"{operation} test is {result}") + lines.append(f"| `{cell(operation)}` | {status} | {result} |") + + lines.extend(("", "## S3 clients", "", "| Client | Version | Test result |", "| --- | --- | --- |")) + for name in CLIENTS: + result = result_status(client_tests, f"{CLIENT_ROOT}/{name}") + if result != "pass": + errors.append(f"{name} client test is {result}") + if versions[name] == "unavailable": + errors.append(f"{name} version is unavailable") + lines.append(f"| {name} | {cell(versions[name])} | {result} |") + + if errors: + lines.extend(("", "## Validation errors", "")) + lines.extend(f"- {cell(error)}" for error in errors) + return "\n".join(lines) + "\n", errors + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--matrix-json", type=Path, required=True) + parser.add_argument("--clients-json", type=Path, required=True) + parser.add_argument("--summary", type=Path, required=True) + parser.add_argument("--source-sha", required=True) + parser.add_argument("--checkout-sha", required=True) + args = parser.parse_args() + summary, errors = report(args.root, args.matrix_json, args.clients_json, args.source_sha, args.checkout_sha, client_versions()) + with args.summary.open("a", encoding="utf-8") as output: + output.write(summary) + print("S3 compatibility report: " + ("failed" if errors else "passed")) + for error in errors: + print(error, file=sys.stderr) + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_s3_compatibility_report.py b/scripts/test_s3_compatibility_report.py new file mode 100644 index 0000000..9ad393b --- /dev/null +++ b/scripts/test_s3_compatibility_report.py @@ -0,0 +1,101 @@ +"""Failure cases for the S3 compatibility evidence gate.""" + +import json +import tempfile +import unittest +from pathlib import Path + +import s3_compatibility_report as report + + +class CompatibilityReportTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + for relative, heading, status in ( + ("docs/en/reference/s3-compatibility.md", "Operation Matrix", "Supported"), + ("docs/zh/reference/s3-compatibility.md", "操作矩阵", "支持"), + ("README.md", "Core S3 Compatibility", "✅"), + ): + path = self.root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + f"## {heading}\n\n| Area | Operation | Status | Notes |\n" + "| --- | --- | --- | --- |\n" + f"| Object | `GetObject` | {status} | Reads bytes. |\n", + encoding="utf-8", + ) + self.matrix_json = self.root / "matrix.json" + self.clients_json = self.root / "clients.json" + self.write_results(self.matrix_json, report.MATRIX_ROOT, ["GetObject"]) + self.write_results(self.clients_json, report.CLIENT_ROOT, list(report.CLIENTS)) + + @staticmethod + def write_results(path, root, children, changed=None, package="pass"): + changed = changed or {} + events = [{"Action": changed.get(child, "pass"), "Test": f"{root}/{child}"} for child in children] + events += [{"Action": "pass", "Test": root}, {"Action": package}] + path.write_text("".join(json.dumps(event) + "\n" for event in events), encoding="utf-8") + + def run_report(self): + return report.report( + self.root, + self.matrix_json, + self.clients_json, + "source-sha", + "checkout-sha", + {name: "test-version" for name in report.CLIENTS}, + ) + + def test_complete_results_pass(self): + summary, errors = self.run_report() + self.assertEqual(errors, []) + self.assertIn("| `GetObject` | supported | pass |", summary) + self.assertIn("| AWS_CLI | test-version | pass |", summary) + + def test_missing_skipped_and_failed_results_fail(self): + for action in ("missing", "skip", "fail"): + with self.subTest(action=action): + children = [] if action == "missing" else ["GetObject"] + changed = {} if action == "missing" else {"GetObject": action} + self.write_results(self.matrix_json, report.MATRIX_ROOT, children, changed) + _, errors = self.run_report() + self.assertTrue(any(f"GetObject test is {action}" in error for error in errors), errors) + + def test_missing_result_file_fails(self): + self.clients_json.unlink() + _, errors = self.run_report() + self.assertTrue(any("clients.json" in error for error in errors), errors) + + def test_missing_client_fails(self): + self.write_results(self.clients_json, report.CLIENT_ROOT, ["AWS_CLI", "rclone"]) + _, errors = self.run_report() + self.assertIn("mc client test is missing", errors) + + def test_unavailable_client_version_fails(self): + _, errors = report.report( + self.root, + self.matrix_json, + self.clients_json, + "source-sha", + "checkout-sha", + {name: "unavailable" if name == "mc" else "test-version" for name in report.CLIENTS}, + ) + self.assertIn("mc version is unavailable", errors) + + def test_bilingual_status_mismatch_fails(self): + path = self.root / "docs/zh/reference/s3-compatibility.md" + path.write_text(path.read_text(encoding="utf-8").replace("| 支持 |", "| 部分支持 |"), encoding="utf-8") + _, errors = self.run_report() + self.assertIn("English/Chinese matrix status differs: GetObject", errors) + + def test_readme_status_mismatch_fails(self): + path = self.root / "README.md" + path.write_text(path.read_text(encoding="utf-8").replace("| ✅ |", "| ⚠️ |"), encoding="utf-8") + _, errors = self.run_report() + self.assertIn("README status differs from full matrix: GetObject", errors) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/system/s3_clients_test.go b/tests/system/s3_clients_test.go index fff8140..fe47caa 100644 --- a/tests/system/s3_clients_test.go +++ b/tests/system/s3_clients_test.go @@ -1,4 +1,4 @@ -//go:build systemtest +//go:build systemtest && s3compat package system_test @@ -23,11 +23,7 @@ import ( "github.com/strahe/synaps3/tests/testutil/e2e" ) -// TestS3Clients is enabled explicitly in CI so the regular system tests need no external CLIs. func TestS3Clients(t *testing.T) { - if os.Getenv("SYNAPS3_TEST_S3_CLIENTS") != "1" { - t.Skip("set SYNAPS3_TEST_S3_CLIENTS=1 to run external S3 clients") - } for _, name := range []string{"aws", "rclone", "mc"} { if _, err := exec.LookPath(name); err != nil { t.Fatalf("required S3 client %s is unavailable: %v", name, err) @@ -100,7 +96,7 @@ func TestS3Clients(t *testing.T) { get []string }{ { - "AWS CLI", + "AWS_CLI", []string{"aws", "--endpoint-url", endpoint, "s3", "cp", "--only-show-errors", input, "s3://s3-clients/aws.bin"}, []string{"aws", "--endpoint-url", endpoint, "s3", "cp", "--only-show-errors", "s3://s3-clients/aws.bin"}, }, diff --git a/tests/system/s3_matrix_test.go b/tests/system/s3_matrix_test.go index 27f9b61..45e63a8 100644 --- a/tests/system/s3_matrix_test.go +++ b/tests/system/s3_matrix_test.go @@ -1,4 +1,4 @@ -//go:build systemtest +//go:build systemtest && s3compat package system_test From 15d99a05fc299403521dcc780ff9d16449302f52 Mon Sep 17 00:00:00 2001 From: Lee <7932644+strahe@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:31:25 +0800 Subject: [PATCH 3/4] docs(readme): shorten compatibility check links --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f86054..5e029e9 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Writes commit to local cache and metadata before returning success. Reads use lo | Multipart | `ListMultipartUploads` | ✅ | Lists open uploads | | Multipart | `ListParts` | ✅ | Lists uploaded parts | -The [S3 Compatibility runs](https://github.com/strahe/SynapS3/actions/workflows/s3-compatibility.yml) verify this matrix offline with [operation tests](https://github.com/strahe/SynapS3/blob/main/tests/system/s3_matrix_test.go) and [AWS CLI, rclone, and MinIO Client tests](https://github.com/strahe/SynapS3/blob/main/tests/system/s3_clients_test.go). No Filecoin network or wallet transactions are used. To run them locally, install the three clients and run `make test-s3-compatibility`. +Compatibility checks: [runs](https://github.com/strahe/SynapS3/actions/workflows/s3-compatibility.yml) · [matrix tests](https://github.com/strahe/SynapS3/blob/main/tests/system/s3_matrix_test.go) · [client tests](https://github.com/strahe/SynapS3/blob/main/tests/system/s3_clients_test.go). ## License From 22ca29981ca6ac75ef9f9b9d4ca788561247ae4d Mon Sep 17 00:00:00 2001 From: Lee <7932644+strahe@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:52:10 +0800 Subject: [PATCH 4/4] docs(readme): link to S3 compatibility reports --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 5e029e9..c59e097 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ [![CI](https://github.com/strahe/SynapS3/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/strahe/SynapS3/actions/workflows/ci.yml) [![S3 Compatibility](https://github.com/strahe/SynapS3/actions/workflows/s3-compatibility.yml/badge.svg?branch=main)](https://github.com/strahe/SynapS3/actions/workflows/s3-compatibility.yml) [![Package](https://img.shields.io/badge/package-GHCR-blue?logo=github)](https://github.com/strahe/SynapS3/pkgs/container/synaps3) -[![Go Report](https://goreportcard.com/badge/github.com/strahe/synaps3)](https://goreportcard.com/report/github.com/strahe/synaps3) [![Go Version](https://img.shields.io/github/go-mod/go-version/strahe/SynapS3?filename=go.mod)](go.mod) SynapS3 is an open-source, self-hosted S3-compatible gateway for Filecoin storage. @@ -60,7 +59,7 @@ Writes commit to local cache and metadata before returning success. Reads use lo | Multipart | `ListMultipartUploads` | ✅ | Lists open uploads | | Multipart | `ListParts` | ✅ | Lists uploaded parts | -Compatibility checks: [runs](https://github.com/strahe/SynapS3/actions/workflows/s3-compatibility.yml) · [matrix tests](https://github.com/strahe/SynapS3/blob/main/tests/system/s3_matrix_test.go) · [client tests](https://github.com/strahe/SynapS3/blob/main/tests/system/s3_clients_test.go). +See [S3 Compatibility runs](https://github.com/strahe/SynapS3/actions/workflows/s3-compatibility.yml) for individual reports. ## License