diff --git a/CHANGELOG.md b/CHANGELOG.md index dadd1781..846095b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,33 @@ formae agent. ### Added +- `GCP::Storage::ManagedFolder` — an IAM boundary inside a bucket, letting a + policy be attached to a prefix without granting it over the whole bucket. + Requires uniform bucket-level access. +- `GCP::Storage::Folder` — a real directory node, available only in a bucket + created with a hierarchical namespace. Renaming one moves everything beneath + it, where a managed folder only governs who may read a prefix. +- `GCP::Storage::Bucket` models `iamConfiguration.uniformBucketLevelAccess` and + `hierarchicalNamespace`. Neither folder type can exist without them, and + hierarchical namespace is fixed at creation — a bucket is created flat or + hierarchical and cannot convert. + +### Fixed + +- `GCP::Storage::Bucket`'s resolvable names properties the resource actually + has. It pointed at `"Id"`, `"SelfLink"` and `"Name"` — capitalised, matching + nothing — so `bkt.res.name` resolved to no value and **any resource + referencing a bucket reached the plugin with the reference unresolved**. It + went unnoticed because nothing in the repository referenced a bucket until the + folder types did. `selfLink` is removed: the bucket has no such property. +- Storage names containing a slash survive a native-ID round trip. Both folder + types are named with a **trailing slash that is part of the identity** + ("reports/" is not "reports"), and the parser took a single path segment, so + the slash was dropped and the rebuilt URL addressed a folder that does not + exist. The name is now taken whole and escaped when addressed — a no-op for + every pre-existing storage name, none of which contains a slash. + +### Fixed - `GCP::DNS::ResourceRecordSet` — what a managed zone actually serves: one name, one record type, and the data behind it. This completes Cloud DNS. diff --git a/pkg/resources/storage/api.go b/pkg/resources/storage/api.go index f1c56e97..f5fcae3e 100644 --- a/pkg/resources/storage/api.go +++ b/pkg/resources/storage/api.go @@ -6,6 +6,7 @@ package storage import ( "fmt" + "net/url" "strings" "github.com/platform-engineering-labs/formae-plugin-gcp/pkg/resources/base" @@ -77,7 +78,10 @@ func storagePathBuilder(ctx base.PathContext) string { // Bucket-scoped bucketName := bucketInfo if ctx.ResourceName != "" { - return fmt.Sprintf("/b/%s/%s/%s", bucketName, ctx.ResourceType, ctx.ResourceName) + // A name containing a slash - a managed folder or a folder - has to be + // escaped, or the slash reads as another path segment. PathEscape is a + // no-op for every other storage name, which contains no slash. + return fmt.Sprintf("/b/%s/%s/%s", bucketName, ctx.ResourceType, url.PathEscape(ctx.ResourceName)) } return fmt.Sprintf("/b/%s/%s", bucketName, ctx.ResourceType) } @@ -153,7 +157,12 @@ func parseStorageNativeID(nativeID string) (base.PathContext, error) { if len(parts) >= 4 && parts[0] == "b" && parts[2] != "o" { ctx.ParentResource = parts[1] // bucket name ctx.ResourceType = parts[2] - ctx.ResourceName = parts[3] + // Everything after the collection is the name. For an ACL entity or an + // anywhere cache that is a single segment and this is exactly parts[3]; + // a managed folder or a folder is named with a trailing slash that is + // part of its identity ("reports/" is not "reports"), and taking one + // segment would silently truncate it. + ctx.ResourceName = strings.Join(parts[3:], "/") return ctx, nil } diff --git a/pkg/resources/storage/bucket_walking_list.go b/pkg/resources/storage/bucket_walking_list.go new file mode 100644 index 00000000..18efa500 --- /dev/null +++ b/pkg/resources/storage/bucket_walking_list.go @@ -0,0 +1,115 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package storage + +import ( + "context" + "fmt" + + "github.com/platform-engineering-labs/formae-plugin-gcp/pkg/config" + "github.com/platform-engineering-labs/formae-plugin-gcp/pkg/resources/prov" + "github.com/platform-engineering-labs/formae-plugin-gcp/pkg/resources/registry" + "github.com/platform-engineering-labs/formae-plugin-gcp/pkg/transport" + "github.com/platform-engineering-labs/formae-plugin-gcp/pkg/utils" + "github.com/platform-engineering-labs/formae/pkg/plugin/resource" +) + +// Both folder types live in a bucket, and discovery lists with no properties - +// so it can name no bucket, the path builder falls through to the +// project-scoped branch and asks /projects/{p}/managedFolders, which addresses +// nothing. GCS has no wildcard for the bucket segment, so the only way to +// discover either is to walk the buckets. +// +// This is the sixth copy of this shape in the plugin (Service Directory, +// Spanner, Cloud SQL, DNS, Bigtable and now Storage). It belongs in base. +type bucketWalkingListProvisioner struct { + prov.Provisioner + cfg *config.Config + collection string +} + +// registerBucketWalkingLists is called from the package init in resources.go so +// the generic registration is guaranteed to have landed first. +func registerBucketWalkingLists() { + for _, spec := range []struct { + resourceType string + collection string + }{ + {ManagedFolderResourceType, "managedFolders"}, + {FolderResourceType, "folders"}, + } { + spec := spec + registry.Register(spec.resourceType, + []resource.Operation{resource.OperationList}, + func(cfg *config.Config) prov.Provisioner { + return &bucketWalkingListProvisioner{ + Provisioner: storageRegistry.CreateProvisioner(cfg, spec.resourceType), + cfg: cfg, + collection: spec.collection, + } + }) + } +} + +func (p *bucketWalkingListProvisioner) List( + ctx context.Context, request *resource.ListRequest, +) (*resource.ListResult, error) { + // A named bucket is the caller telling us where to look. + if request.AdditionalProperties != nil && request.AdditionalProperties["bucket"] != "" { + return p.Provisioner.List(ctx, request) + } + + cfg := config.PathFromTargetConfig(request.TargetConfig) + if cfg.Project == "" { + return &resource.ListResult{NativeIDs: []string{}}, nil + } + + client, err := transport.NewClient(ctx, p.cfg) + if err != nil { + return nil, fmt.Errorf("failed to create transport client: %w", err) + } + + bucketsURL := fmt.Sprintf("%s/b?project=%s", StorageAPI.BaseURL, cfg.Project) + resp, err := client.SendRequest(ctx, transport.RequestOptions{Method: "GET", URL: bucketsURL}) + if err != nil { + wrapped := transport.WrapError(err, "failed to list storage buckets") + return nil, fmt.Errorf("%s", wrapped.Message) + } + + nativeIDs := []string{} + buckets, _ := resp.Body["items"].([]interface{}) + for _, raw := range buckets { + bucket, ok := raw.(map[string]interface{}) + if !ok { + continue + } + bucketName := utils.GetString(bucket, "name") + if bucketName == "" { + continue + } + itemsResp, listErr := client.SendRequest(ctx, transport.RequestOptions{ + Method: "GET", + URL: fmt.Sprintf("%s/b/%s/%s", StorageAPI.BaseURL, bucketName, p.collection), + }) + if listErr != nil { + // A flat bucket has no folders and a bucket with ACLs has no managed + // folders; neither is a reason to hide the rest. + continue + } + items, _ := itemsResp.Body["items"].([]interface{}) + for _, rawItem := range items { + item, ok := rawItem.(map[string]interface{}) + if !ok { + continue + } + // The name carries its trailing slash, which the native ID keeps. + if name := utils.GetString(item, "name"); name != "" { + nativeIDs = append(nativeIDs, + fmt.Sprintf("b/%s/%s/%s", bucketName, p.collection, name)) + } + } + } + return &resource.ListResult{NativeIDs: nativeIDs}, nil +} diff --git a/pkg/resources/storage/resources.go b/pkg/resources/storage/resources.go index dc22b1a8..47919fb7 100644 --- a/pkg/resources/storage/resources.go +++ b/pkg/resources/storage/resources.go @@ -18,6 +18,8 @@ const ( AnywhereCacheResourceType = "GCP::Storage::AnywhereCache" BucketAccessControlResourceType = "GCP::Storage::BucketAccessControl" DefaultObjectAccessControlResourceType = "GCP::Storage::DefaultObjectAccessControl" + ManagedFolderResourceType = "GCP::Storage::ManagedFolder" + FolderResourceType = "GCP::Storage::Folder" ObjectAccessControlResourceType = "GCP::Storage::ObjectAccessControl" ) @@ -144,6 +146,47 @@ func init() { RequestTransformer: wrapBodyBuilder(aclBodyBuilder), ResponseTransformer: nil, }, + { + // A managed folder is an IAM boundary inside a bucket: it lets a + // policy be attached to a prefix without giving it to the whole + // bucket. It requires uniform bucket-level access - GCS refuses to + // create one where per-object ACLs still apply. + // + // Its name ends with a slash, which is part of its identity and is + // escaped in the URL. + ResourceType: ManagedFolderResourceType, + ResourceConfig: base.ResourceConfig{ + ResourceType: "managedFolders", + ParentResource: &base.ParentResourceConfig{ + ParentType: "bucket", + RequiresParent: true, + ParentPathSegments: []string{"b"}, + }, + // A managed folder carries nothing but its name; there is + // nothing an update could change. + SupportsUpdate: false, + }, + RequestTransformer: base.DropFields("bucket"), + ResponseTransformer: base.ResponseTransformerFunc(bucketScopedResponseTransformer), + }, + { + // A folder is a real directory, available only in a bucket created + // with a hierarchical namespace. Where a managed folder is an IAM + // boundary over a prefix, a folder is an actual node - renaming one + // moves everything beneath it. + ResourceType: FolderResourceType, + ResourceConfig: base.ResourceConfig{ + ResourceType: "folders", + ParentResource: &base.ParentResourceConfig{ + ParentType: "bucket", + RequiresParent: true, + ParentPathSegments: []string{"b"}, + }, + SupportsUpdate: false, + }, + RequestTransformer: base.DropFields("bucket"), + ResponseTransformer: base.ResponseTransformerFunc(bucketScopedResponseTransformer), + }, // NOTE: ObjectAccessControlResourceType requires special handling for object-scoped resources // The base package currently doesn't support resources that need TWO parent properties (bucket + object). // This resource type is commented out pending enhancement to base package's parent extraction mechanism. @@ -172,4 +215,24 @@ func init() { if err != nil { panic(err) } + + registerBucketWalkingLists() +} + +// bucketScopedResponseTransformer puts back the bucket a folder belongs to and +// drops what GCS echoes that describes the request rather than the resource. +// Both folder types report "bucket" themselves, so unlike most nested resources +// here nothing has to be recovered from the URL. +func bucketScopedResponseTransformer( + props map[string]interface{}, _ base.TransformContext, +) map[string]interface{} { + out := make(map[string]interface{}, len(props)) + for k, v := range props { + switch k { + case "kind", "selfLink", "metageneration", "id": + continue + } + out[k] = v + } + return out } diff --git a/pkg/resources/storage/storage_native_id_unit_test.go b/pkg/resources/storage/storage_native_id_unit_test.go new file mode 100644 index 00000000..2a90f7db --- /dev/null +++ b/pkg/resources/storage/storage_native_id_unit_test.go @@ -0,0 +1,92 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +//go:build unit + +package storage + +import ( + "testing" + + "github.com/platform-engineering-labs/formae-plugin-gcp/pkg/resources/base" +) + +// The names of every storage type that existed before managed folders are a +// single segment. These pin that the parser and the path builder still treat +// them exactly as they did - this is the half of the change that must not move. +func TestExistingStorageNamesAreUnchanged(t *testing.T) { + cases := map[string]struct { + nativeID string + wantPath string + }{ + "bucket acl entity": { + "b/my-bucket/acl/user-someone@example.com", + "/b/my-bucket/acl/user-someone@example.com", + }, + "default object acl": { + "b/my-bucket/defaultObjectAcl/allUsers", + "/b/my-bucket/defaultObjectAcl/allUsers", + }, + "anywhere cache": { + "b/my-bucket/anywhereCaches/cache-1", + "/b/my-bucket/anywhereCaches/cache-1", + }, + } + for name, tc := range cases { + ctx, err := parseStorageNativeID(tc.nativeID) + if err != nil { + t.Errorf("%s: parse: %v", name, err) + continue + } + if got := storagePathBuilder(ctx); got != tc.wantPath { + t.Errorf("%s: path = %q, want %q", name, got, tc.wantPath) + } + } +} + +// A managed folder is named with a trailing slash that is part of its identity: +// "reports/" is not "reports". Splitting on "/" and taking one segment dropped +// it, and the rebuilt URL then addressed a folder that does not exist. +func TestSlashTerminatedNamesSurviveAndAreEscaped(t *testing.T) { + ctx, err := parseStorageNativeID("b/my-bucket/managedFolders/reports/") + if err != nil { + t.Fatalf("parse: %v", err) + } + if ctx.ParentResource != "my-bucket" || ctx.ResourceType != "managedFolders" { + t.Errorf("ctx = %+v", ctx) + } + if ctx.ResourceName != "reports/" { + t.Errorf("name = %q, want %q", ctx.ResourceName, "reports/") + } + // The slash has to be escaped or it reads as another path segment. + want := "/b/my-bucket/managedFolders/reports%2F" + if got := storagePathBuilder(ctx); got != want { + t.Errorf("path = %q, want %q", got, want) + } + + // Nested folders keep every segment. + nested, err := parseStorageNativeID("b/my-bucket/folders/a/b/") + if err != nil { + t.Fatalf("parse nested: %v", err) + } + if nested.ResourceName != "a/b/" { + t.Errorf("nested name = %q", nested.ResourceName) + } +} + +// A bucket is still addressed by a bare name, and a project-scoped resource +// still by its project path. +func TestBucketAndProjectScopedShapesStillParse(t *testing.T) { + b, err := parseStorageNativeID("my-bucket") + if err != nil || b.ResourceType != "b" || b.ResourceName != "my-bucket" { + t.Errorf("bucket ctx = %+v err=%v", b, err) + } + p, err := parseStorageNativeID("projects/proj/hmacKeys/GOOG1EXAMPLE") + if err != nil || p.ResourceType != "hmacKeys" || p.ResourceName != "GOOG1EXAMPLE" || p.ParentResource != "" { + t.Errorf("project ctx = %+v err=%v", p, err) + } + if got := storagePathBuilder(base.PathContext{Project: "proj", ResourceType: "hmacKeys", ResourceName: "GOOG1EXAMPLE"}); got != "/projects/proj/hmacKeys/GOOG1EXAMPLE" { + t.Errorf("project path = %q", got) + } +} diff --git a/schema/pkl/storage/bucket.pkl b/schema/pkl/storage/bucket.pkl index 6905cf39..906f7983 100644 --- a/schema/pkl/storage/bucket.pkl +++ b/schema/pkl/storage/bucket.pkl @@ -87,22 +87,57 @@ open class RetentionPolicy extends formae.SubResource { retentionPeriod: Int? } +/// Every entry here names a property this resource actually declares. +/// +/// They previously read "Id", "SelfLink" and "Name" — capitalised, matching +/// nothing — so `bkt.res.name` resolved to no value and any resource +/// referencing a bucket reached the plugin with the reference unresolved. It +/// went unnoticed because nothing in the repository referenced a bucket until +/// the folder types did. `selfLink` is gone with them: the bucket has no such +/// property to point at. open class BucketResolvable extends formae.Resolvable { hidden type = module.type hidden id: BucketResolvable = (this) { - property = "Id" - } - - hidden selfLink: BucketResolvable = (this) { - property = "SelfLink" + property = "id" } hidden name: BucketResolvable = (this) { - property = "Name" + property = "name" } } +/// Whether ACLs are disabled in favour of IAM alone. Managed folders require +/// this to be enabled — GCS refuses to create one in a bucket that still has +/// per-object ACLs. +@gcp.SubResourceHint +open class UniformBucketLevelAccess extends formae.SubResource { + @gcp.FieldHint + enabled: Boolean? + + /// When the setting stops being reversible. Set by GCS. + @gcp.FieldHint { hasProviderDefault = true } + lockedTime: String? +} + +@gcp.SubResourceHint +open class IamConfiguration extends formae.SubResource { + @gcp.FieldHint + uniformBucketLevelAccess: UniformBucketLevelAccess? + + /// GCS fills this in ("inherited" unless set). + @gcp.FieldHint { hasProviderDefault = true } + publicAccessPrevention: String? +} + +/// Gives the bucket real folders rather than name prefixes. Fixed at creation: +/// an existing flat bucket cannot be converted. +@gcp.SubResourceHint +open class HierarchicalNamespace extends formae.SubResource { + @gcp.FieldHint + enabled: Boolean? +} + @gcp.ResourceHint { type = module.type identifier = "name" @@ -149,6 +184,21 @@ open class Bucket extends formae.Resource { } name: String + /// How access is controlled. GCS always reports this, so it is a provider + /// default when a forma says nothing; set + /// `uniformBucketLevelAccess.enabled` to turn ACLs off, which managed + /// folders require. + @gcp.FieldHint { hasProviderDefault = true } + iamConfiguration: IamConfiguration? + + /// Enables real folders. Immutable — a bucket is created flat or + /// hierarchical and cannot change afterwards. + @gcp.FieldHint { + createOnly = true + hasProviderDefault = true + } + hierarchicalNamespace: HierarchicalNamespace? + @gcp.FieldHint owner: Owner? diff --git a/schema/pkl/storage/folder.pkl b/schema/pkl/storage/folder.pkl new file mode 100644 index 00000000..ae3ee775 --- /dev/null +++ b/schema/pkl/storage/folder.pkl @@ -0,0 +1,62 @@ +/* + * © 2025 Platform Engineering Labs Inc. + * + * SPDX-License-Identifier: FSL-1.1-ALv2 + */ + +/// GCP Storage Managed Folder +/// +/// An IAM boundary inside a bucket: it lets a policy be attached to a prefix +/// without granting it over the whole bucket. The objects themselves are +/// unaffected — a managed folder is about who may read them, not where they +/// live. +/// +/// Requires uniform bucket-level access on the bucket; GCS refuses to create +/// one where per-object ACLs still apply. +module gcp.storage.folder + +import "../gcp.pkl" +import "@formae/formae.pkl" + +const type = "GCP::Storage::Folder" + +open class FolderResolvable extends formae.Resolvable { + hidden type = module.type + + hidden name: FolderResolvable = (this) { + property = "name" + } +} + +@gcp.ResourceHint { + type = module.type + identifier = "name" +} +open class Folder extends formae.Resource { + hidden parent = this + + /// The folder's name, which **must end with a slash** — "reports/" is a + /// managed folder, "reports" is not. The trailing slash is part of the + /// identity, not decoration, and is escaped when the resource is addressed. + /// Immutable. + @gcp.FieldHint { createOnly = true } + name: String(endsWith("/")) + + /// Name of the owning bucket. A path component, not a body field — pass + /// `bkt.res.name` so formae creates the bucket first. + @gcp.FieldHint { createOnly = true } + bucket: (String|formae.Resolvable) + + /// When GCS created the folder. + @gcp.FieldHint { hasProviderDefault = true } + createTime: String? + + /// When GCS last updated it. + @gcp.FieldHint { hasProviderDefault = true } + updateTime: String? + + hidden res: FolderResolvable = new { + label = parent.label + stack = parent.stack?.label + } +} diff --git a/schema/pkl/storage/managed_folder.pkl b/schema/pkl/storage/managed_folder.pkl new file mode 100644 index 00000000..0fa06dba --- /dev/null +++ b/schema/pkl/storage/managed_folder.pkl @@ -0,0 +1,62 @@ +/* + * © 2025 Platform Engineering Labs Inc. + * + * SPDX-License-Identifier: FSL-1.1-ALv2 + */ + +/// GCP Storage Managed Folder +/// +/// An IAM boundary inside a bucket: it lets a policy be attached to a prefix +/// without granting it over the whole bucket. The objects themselves are +/// unaffected — a managed folder is about who may read them, not where they +/// live. +/// +/// Requires uniform bucket-level access on the bucket; GCS refuses to create +/// one where per-object ACLs still apply. +module gcp.storage.managed_folder + +import "../gcp.pkl" +import "@formae/formae.pkl" + +const type = "GCP::Storage::ManagedFolder" + +open class ManagedFolderResolvable extends formae.Resolvable { + hidden type = module.type + + hidden name: ManagedFolderResolvable = (this) { + property = "name" + } +} + +@gcp.ResourceHint { + type = module.type + identifier = "name" +} +open class ManagedFolder extends formae.Resource { + hidden parent = this + + /// The folder's name, which **must end with a slash** — "reports/" is a + /// managed folder, "reports" is not. The trailing slash is part of the + /// identity, not decoration, and is escaped when the resource is addressed. + /// Immutable. + @gcp.FieldHint { createOnly = true } + name: String(endsWith("/")) + + /// Name of the owning bucket. A path component, not a body field — pass + /// `bkt.res.name` so formae creates the bucket first. + @gcp.FieldHint { createOnly = true } + bucket: (String|formae.Resolvable) + + /// When GCS created the folder. + @gcp.FieldHint { hasProviderDefault = true } + createTime: String? + + /// When GCS last updated it. + @gcp.FieldHint { hasProviderDefault = true } + updateTime: String? + + hidden res: ManagedFolderResolvable = new { + label = parent.label + stack = parent.stack?.label + } +} diff --git a/testdata/storage-folder.pkl b/testdata/storage-folder.pkl new file mode 100644 index 00000000..173a69cb --- /dev/null +++ b/testdata/storage-folder.pkl @@ -0,0 +1,42 @@ +/* + * © 2025 Platform Engineering Labs Inc. + * + * SPDX-License-Identifier: FSL-1.1-ALv2 + */ + +amends "@formae/forma.pkl" + +import "@gcp/storage/bucket.pkl" +import "@gcp/storage/folder.pkl" as folder +import "./config/vars.pkl" as v + +// A folder is a real directory node, so its bucket must be created with a +// hierarchical namespace - which also requires uniform bucket-level access, and +// cannot be turned on afterwards. +local bkt = new bucket.Bucket { + label = "plugin-sdk-test-folder-bucket" + name = "formae-plugin-sdk-test-fld-\(v.testRunID)" + location = v.gcpRegion + storageClass = "STANDARD" + iamConfiguration = new bucket.IamConfiguration { + uniformBucketLevelAccess = new bucket.UniformBucketLevelAccess { + enabled = true + } + } + hierarchicalNamespace = new bucket.HierarchicalNamespace { + enabled = true + } +} + +forma { + v.stack + v.target + + bkt + + new folder.Folder { + label = "plugin-sdk-test-storage-folder" + name = "formae-data/" + bucket = bkt.res.name + } +} diff --git a/testdata/storage-managed-folder.pkl b/testdata/storage-managed-folder.pkl new file mode 100644 index 00000000..a6347120 --- /dev/null +++ b/testdata/storage-managed-folder.pkl @@ -0,0 +1,40 @@ +/* + * © 2025 Platform Engineering Labs Inc. + * + * SPDX-License-Identifier: FSL-1.1-ALv2 + */ + +amends "@formae/forma.pkl" + +import "@gcp/storage/bucket.pkl" +import "@gcp/storage/managed_folder.pkl" as managedFolder +import "./config/vars.pkl" as v + +// A managed folder needs uniform bucket-level access: GCS refuses to create one +// in a bucket where per-object ACLs still apply. An empty bucket stores nothing +// and costs nothing. +local bkt = new bucket.Bucket { + label = "plugin-sdk-test-mf-bucket" + name = "formae-plugin-sdk-test-mf-\(v.testRunID)" + location = v.gcpRegion + storageClass = "STANDARD" + iamConfiguration = new bucket.IamConfiguration { + uniformBucketLevelAccess = new bucket.UniformBucketLevelAccess { + enabled = true + } + } +} + +forma { + v.stack + v.target + + bkt + + // The trailing slash is part of the name, not decoration. + new managedFolder.ManagedFolder { + label = "plugin-sdk-test-storage-managed-folder" + name = "formae-reports/" + bucket = bkt.res.name + } +}