From 38228ccc42ee387085816d3308d86158802e25d3 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Tue, 10 Mar 2026 22:36:12 -0700 Subject: [PATCH] feat: Spanner storage backend and per-cluster allocators - Add Spanner backend factory and storage decorator integration - Per-cluster AllocatorRegistry for independent ClusterIP/NodePort ranges - Per-cluster AllocatorRepairManager for service allocator repair loops - Spanner smoke test infrastructure (emulator setup, per-test DB lifecycle) - Update go.mod replace directives to point to GitHub repos Co-Authored-By: Claude Opus 4.6 --- cmd/apiserver/app/config.go | 56 +++++ cmd/apiserver/app/options/options.go | 17 ++ go.mod | 32 ++- go.sum | 191 ++++++++++----- .../bootstrap/allocator_registry.go | 223 ++++++++++++++++++ .../bootstrap/allocator_repair.go | 108 +++++++++ pkg/multicluster/bootstrap/crd_controller.go | 17 +- pkg/multicluster/options.go | 11 + pkg/multicluster/storage.go | 63 ++++- pkg/multicluster/typedinformer/adapters.go | 20 +- test/smoke/apiserver_test.go | 68 ++++++ test/smoke/auth_restart_test.go | 4 + test/smoke/core_bootstrap_test.go | 2 +- test/smoke/no_etcd_metadata_test.go | 3 + 14 files changed, 735 insertions(+), 80 deletions(-) create mode 100644 pkg/multicluster/bootstrap/allocator_registry.go create mode 100644 pkg/multicluster/bootstrap/allocator_repair.go diff --git a/cmd/apiserver/app/config.go b/cmd/apiserver/app/config.go index fdeb4c1..520b216 100644 --- a/cmd/apiserver/app/config.go +++ b/cmd/apiserver/app/config.go @@ -32,6 +32,8 @@ import ( genericfilters "k8s.io/apiserver/pkg/endpoints/filters" "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/storage" + storagefeature "k8s.io/apiserver/pkg/storage/feature" serverfilters "k8s.io/apiserver/pkg/server/filters" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/apiserver/pkg/util/webhook" @@ -40,10 +42,12 @@ import ( aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" "k8s.io/kubernetes/pkg/api/legacyscheme" + api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/controlplane" controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver" "k8s.io/kubernetes/pkg/features" generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" + servicestore "k8s.io/kubernetes/pkg/registry/core/service/storage" "github.com/kplane-dev/apiserver/cmd/apiserver/app/options" mc "github.com/kplane-dev/apiserver/pkg/multicluster" @@ -129,6 +133,18 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) { clientPool := mc.NewClientPool(genericConfig.LoopbackClientConfig, mcOpts.PathPrefix, mcOpts.ControlPlaneSegment) informerRegistry := mc.NewInformerRegistry(wait.ContextForChannel(genericConfig.DrainedNotify())) mcOpts.InformerRegistry = informerRegistry + if opts.SpannerProject != "" { + mcOpts.Spanner = &mc.SpannerConfig{ + Project: opts.SpannerProject, + Instance: opts.SpannerInstance, + Database: opts.SpannerDatabase, + EmulatorHost: opts.SpannerEmulatorHost, + } + // Spanner implements RequestWatchProgress natively via its + // broadcaster. Tell the feature checker so the cacher allows + // SendInitialEvents (WatchList) watches. + storagefeature.SetFeatureSupported(storage.RequestWatchProgress, true) + } var crdRuntimeMgr *mcbootstrap.CRDRuntimeManager systemNamespaceBootstrapper := mcbootstrap.NewSystemNamespaceBootstrapper(mcbootstrap.SystemNamespaceOptions{ ClientForCluster: clientPool.KubeClientForCluster, @@ -234,6 +250,40 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) { if c.KubeAPIs.ControlPlane.Generic.RESTOptionsGetter != nil { c.KubeAPIs.ControlPlane.Generic.RESTOptionsGetter = decorateRESTOptionsGetter("controlplane", c.KubeAPIs.ControlPlane.Generic.RESTOptionsGetter, mcOpts) } + + // Per-cluster service allocators: create an AllocatorRegistry so each VCP + // gets its own ClusterIP and NodePort bitmap, bypassing the shared root range. + serviceStorageConfig, err := storageFactory.NewConfig(api.Resource("services"), &api.Service{}) + if err != nil { + return nil, err + } + var allocBackendFactory mcbootstrap.StorageFactory + if mcOpts.Spanner != nil { + sf := mc.NewSpannerBackendFactory(mcOpts.Spanner) + allocBackendFactory = mcbootstrap.StorageFactory(sf) + } + allocatorRegistry := mcbootstrap.NewAllocatorRegistry(mcbootstrap.AllocatorRegistryOptions{ + PrimaryServiceCIDR: opts.PrimaryServiceClusterIPRange, + SecondaryServiceCIDR: opts.SecondaryServiceClusterIPRange, + NodePortRange: opts.ServiceNodePortRange, + ServiceStorageConfig: serviceStorageConfig.ForResource(api.Resource("serviceipallocations")), + BackendFactory: allocBackendFactory, + }) + c.KubeAPIs.Extra.ServiceRESTHook = func(rest *servicestore.REST) { + rest.ClusterAllocators = func(ctx context.Context) *servicestore.Allocators { + cid, _, _ := mc.FromContext(ctx) + if cid == "" || cid == mcOpts.DefaultCluster { + return nil // use root allocators + } + alloc, err := allocatorRegistry.AllocatorsForCluster(cid) + if err != nil { + klog.Errorf("mc.allocatorRegistry failed cluster=%s: %v", cid, err) + return nil // fall back to root allocators + } + return alloc + } + } + targetPort := 443 if opts.SecureServing != nil && opts.SecureServing.BindPort > 0 { targetPort = opts.SecureServing.BindPort @@ -254,6 +304,11 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) { TargetPort: targetPort, NodePort: opts.KubernetesServiceNodePort, }) + allocatorRepairMgr := mcbootstrap.NewAllocatorRepairManager(mcbootstrap.AllocatorRepairOptions{ + ClientForCluster: clientPool.KubeClientForCluster, + StopChForCluster: stopChForCluster, + Registry: allocatorRegistry, + }) mcOpts.OnClusterSelected = func(clusterID string) { // Preserve upstream root bootstrap as-is; only add multicluster bootstrap for non-root VCPs. if clusterID == mcOpts.DefaultCluster { @@ -265,6 +320,7 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) { go serviceCIDRBootstrapper.Ensure(clusterID) go rbacBootstrapper.Ensure(clusterID) go internalControllerMgr.Ensure(clusterID) + go allocatorRepairMgr.Ensure(clusterID) if opts.KubernetesServiceMode == options.KubernetesServiceModePerClusterAutoIP { go kubeServiceControllerMgr.Ensure(clusterID) } diff --git a/cmd/apiserver/app/options/options.go b/cmd/apiserver/app/options/options.go index 85a438a..6d42aec 100644 --- a/cmd/apiserver/app/options/options.go +++ b/cmd/apiserver/app/options/options.go @@ -64,6 +64,13 @@ type Extra struct { EndpointReconcilerType string MasterCount int + + // Spanner backend flags. When SpannerProject is non-empty, storage uses + // Spanner instead of etcd for multicluster resource data. + SpannerProject string + SpannerInstance string + SpannerDatabase string + SpannerEmulatorHost string } const ( @@ -124,6 +131,16 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) { mcfs.StringVar(&s.ServiceCIDRSharingMode, "service-cidr-sharing-mode", s.ServiceCIDRSharingMode, "Service CIDR strategy across control planes: 'shared' keeps root-managed defaults only; 'per-cluster' bootstraps default ServiceCIDR in each virtual control plane.") + spannerfs := fss.FlagSet("spanner") + spannerfs.StringVar(&s.SpannerProject, "spanner-project", s.SpannerProject, + "GCP project ID for Spanner storage backend. When set, multicluster resource storage uses Spanner instead of etcd.") + spannerfs.StringVar(&s.SpannerInstance, "spanner-instance", s.SpannerInstance, + "Spanner instance ID.") + spannerfs.StringVar(&s.SpannerDatabase, "spanner-database", s.SpannerDatabase, + "Spanner database name.") + spannerfs.StringVar(&s.SpannerEmulatorHost, "spanner-emulator-host", s.SpannerEmulatorHost, + "Spanner emulator host:port for local development (e.g. localhost:9010). Disables TLS and authentication.") + fs := fss.FlagSet("misc") fs.BoolVar(&s.AllowPrivileged, "allow-privileged", s.AllowPrivileged, diff --git a/go.mod b/go.mod index 1904141..f0426e5 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.25.0 require ( github.com/kplane-dev/informer v0.0.0-00010101000000-000000000000 + github.com/kplane-dev/spanner v0.0.0 github.com/kplane-dev/storage v0.0.0 github.com/spf13/cobra v1.10.1 go.etcd.io/etcd/client/v3 v3.6.7 @@ -24,6 +25,7 @@ require ( replace ( github.com/kplane-dev/informer => github.com/kplane-dev/informer v0.0.0-20260303050920-e9c86850386e + github.com/kplane-dev/spanner => github.com/kplane-dev/spanner v0.0.0-20260311053139-032c9428fc1a github.com/kplane-dev/storage => github.com/kplane-dev/storage v0.0.0-20260303050750-8ad94e8ce404 k8s.io/api => github.com/kplane-dev/kubernetes/staging/src/k8s.io/api v0.0.0-20260303044756-e9e2a52adaf0 k8s.io/apiextensions-apiserver => github.com/kplane-dev/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20260303044756-e9e2a52adaf0 @@ -61,14 +63,25 @@ replace ( require ( cel.dev/expr v0.24.0 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.18.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.5.3 // indirect + cloud.google.com/go/longrunning v0.8.0 // indirect + cloud.google.com/go/monitoring v1.24.3 // indirect + cloud.google.com/go/spanner v1.88.0 // indirect cyphar.com/go-pathrs v0.2.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.6.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect github.com/coreos/go-oidc v2.5.0+incompatible // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.7.0 // indirect @@ -76,9 +89,12 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect @@ -100,7 +116,10 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/cel-go v0.26.1 // indirect github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect + github.com/googleapis/gax-go/v2 v2.17.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 // indirect @@ -119,6 +138,7 @@ require ( github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/selinux v1.13.1 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/pquerna/cachecontrol v0.2.0 // indirect github.com/prometheus/client_golang v1.23.2 // indirect @@ -127,17 +147,21 @@ require ( github.com/prometheus/procfs v0.19.2 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/spf13/pflag v1.0.10 // indirect + github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stoewer/go-strcase v1.3.1 // indirect github.com/x448/float16 v0.8.4 // indirect go.etcd.io/etcd/api/v3 v3.6.7 // indirect go.etcd.io/etcd/client/pkg/v3 v3.6.7 // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect go.opentelemetry.io/otel/metric v1.40.0 // indirect go.opentelemetry.io/otel/sdk v1.40.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect go.opentelemetry.io/otel/trace v1.40.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect @@ -147,14 +171,16 @@ require ( golang.org/x/crypto v0.47.0 // indirect golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect golang.org/x/net v0.49.0 // indirect - golang.org/x/oauth2 v0.34.0 // indirect + golang.org/x/oauth2 v0.35.0 // indirect golang.org/x/sys v0.40.0 // indirect golang.org/x/term v0.39.0 // indirect golang.org/x/text v0.33.0 // indirect golang.org/x/time v0.14.0 // indirect golang.org/x/tools v0.40.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/api v0.265.0 // indirect + google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect google.golang.org/grpc v1.78.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect diff --git a/go.sum b/go.sum index 36b41cb..662ae7f 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,31 @@ cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs= +cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc= +cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU= +cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= +cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= +cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= +cloud.google.com/go/spanner v1.88.0 h1:HS+5TuEYZOVOXj9K+0EtrbTw7bKBLrMe3vgGsbnehmU= +cloud.google.com/go/spanner v1.88.0/go.mod h1:MzulBwuuYwQUVdkZXBBFapmXee3N+sQrj2T/yup6uEE= cyphar.com/go-pathrs v0.2.2 h1:y9w7hxbkr3zEL78Fjzeg4HEhs2xNy+fbwHiHGJJY2Xo= cyphar.com/go-pathrs v0.2.2/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.6.0 h1:BzsL0qE7LvtTEtXG7Dt5NS1EP0CQwI21HZfj9aGghhw= +github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.6.0/go.mod h1:I7kE2kM3qCr9QPT4cU4cCFYkEpVyVr16YOGUHzy+nR0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -20,8 +42,13 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0= +github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4= github.com/coreos/go-oidc v2.5.0+incompatible h1:6W0vGJR3Tu0r0PwfmjOrRZSlfxeEln8dsejt3ZWIvwo= github.com/coreos/go-oidc v2.5.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= @@ -43,12 +70,26 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM= +github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs= +github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo= +github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -90,6 +131,22 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U= +github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= @@ -98,13 +155,26 @@ github.com/google/cel-go v0.26.1 h1:iPbVVEdkhTX++hpe3lzSk7D3G3QSYqLGoHOcEio+UXQ= github.com/google/cel-go v0.26.1/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao= +github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8= +github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc= +github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 h1:QGLs/O40yoNK9vmy4rhUGBVyMf1lISBGtXRpsu/Qu/o= @@ -125,64 +195,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/kplane-dev/informer v0.0.0-20260303050920-e9c86850386e h1:alhKoCGQhvNyKi+8h0/m3N46/mfxmvzUAK4gFabPsJk= -github.com/kplane-dev/informer v0.0.0-20260303050920-e9c86850386e/go.mod h1:Nd1KQEeObbfGhP3NZsLNrOtNVDl6h4L3l4R52FQrJYI= -github.com/kplane-dev/kubernetes v0.0.0-20260303044756-e9e2a52adaf0 h1:6s12hLrc1qmwLFEhI5OofZ7J07sw8q7pg552jgNlpTU= -github.com/kplane-dev/kubernetes v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:9uEdeMBkUjkGSL2ioy2pKr0+aLNIvmOan7p8F7ENHbk= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/api v0.0.0-20260303044756-e9e2a52adaf0 h1:sQjyTi6O+gAnxvjltaNQk77Q+0gCPopYDH9P1YZkN/k= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/api v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:KOrdwhDi3QHVXr50HAWBhO2r9uMtWbO46CiIHzRVtU8= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20260303044756-e9e2a52adaf0 h1:SC2NAxlp1NfqJo6lfPgGWYDdqlT3A39UpmvlOlznvKM= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:wZyuUfbqWsRKO42GvZVHZSJ0JttMO7YrXnu/bv/GqPg= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20260303044756-e9e2a52adaf0 h1:2TJwa1qB8fIP5+SxnZydicT9uxKFnVx92afDO8O1tVE= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:7mgr/dli8ofwAbcIQXetFVX1fbOYsOYojq3AUbybVmQ= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20260303044756-e9e2a52adaf0 h1:ylj6Fc6O3iXH6pLyfYEFhXjNjgEm9v3mVt8jWsdZeYY= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:GNWcUSRqjpm4i1hrLaGA7EQrl60YdahMic4aS+WUQVI= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20260303044756-e9e2a52adaf0 h1:RM/DJi1rJcAb7RQI+Zt5NxHJNk2cAM+KDVl9Xl9lxIU= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/client-go v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:7IM9p4c8CafSxF7ZY0F46WHylFn3o4mLVW5T1VZbaY8= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20260303044756-e9e2a52adaf0 h1:2PkgzWXTPKwkcuzEcxsMTUf8llfcOl5rzR3r1UbSLVw= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:bGWrOCyYWe9G/1zpB0hSLAO+BnZUNbNVWXcHAzfaF+E= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20260303044756-e9e2a52adaf0 h1:dt9wd4bwSNXxi8Ma/PdECxhVAotI8LVUusSPMzSf740= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:CxFNOU1/U4gfM4SU7hRkzCRhTfUweuTLRWKWh6yitcE= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260303044756-e9e2a52adaf0 h1:azL0YSPSlc/id0oPDs4hXeWSiv8HO9vrNhxEl6LdbVY= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:R6vYa1XRfX3PdQEGNkCaL3pt7NvLU2ti7FPzsEsA6GQ= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20260303044756-e9e2a52adaf0 h1:c3DFoxrOUhhzUIyLOjGRvFGABKKPF1aDgq5xYHpaOEI= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:axBpPWJYMZFstDUXj45ooTW3njLMa7B8Kz8o4QY5xfk= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20260303044756-e9e2a52adaf0 h1:FUKLrOqS04/6uSpEVD0cLCT9+IRCuQpgEceDpJ9PoNs= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:gjjSpBn/ZS8VRB+BMH7Ttd2kk+Hr+IYvm4LwBwQAtaU= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20260303044756-e9e2a52adaf0 h1:Q3mSnGo3lyBFi83ztys5jItqPuqqGDIi0HPmlWCclb8= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:sez47HjW1useDMcQeVkW7kQNjwzaLJlaJoMcWYU6WVE= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/cri-client v0.0.0-20260303044756-e9e2a52adaf0 h1:PNPWPY/kn7vuwIYFeXCWdc0A06S8Mm08jzmErUisus4= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/cri-client v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:rVFUCMkvRUmEXFCLoBSsAbnSarFL/9tYTe6kE+nh/a0= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20260303044756-e9e2a52adaf0 h1:QOHDI3guNsPh6OlJqVMOu/wgwYLlVtMhm4U05oNDa8c= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:h4+rRn/HN1k7Uln9nnxykSWPI5ZBQCBJyHosyW4F7nk= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v0.0.0-20260303044756-e9e2a52adaf0 h1:LUI/X6tSzQknb1Sj6NTdwZlqjPcpu1i5Ncyz8SQVlGo= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/dynamic-resource-allocation v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:2UvGX2oAJmCnqKSEBlBMnCEWisTXl2ZXQK/quJy61Q4= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/endpointslice v0.0.0-20260303044756-e9e2a52adaf0 h1:7ow1+bA4PhrMBdYxnOwjHrSRHYZJrIeNXjjmGH3d0C4= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/endpointslice v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:fdIUBx+W5Ypy0TBvfW8qonHr4QV3iXw+N+ADcGYf8eI= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/externaljwt v0.0.0-20260303044756-e9e2a52adaf0 h1:Q+5yNIJcssUqBYZLqrDOB4SukPAutQDtzYdowPDsfUg= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/externaljwt v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:+8tzBBizhTus+bM0JPY/4h2O26NxIQgcEBdK5Xdfehc= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/kms v0.0.0-20260303044756-e9e2a52adaf0 h1:No3QOZxohl0wJ9y8FIeS+40LQaKYsKi/9JDNSN64u/s= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/kms v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:rhO9JHsWibaRLx12/ViuMQdeZi43hsiXCl8KEEihjaw= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20260303044756-e9e2a52adaf0 h1:aa+2RkO/nOKISlkzNxnHsGRSlqYgdN3f0K9hmUDryP0= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:KXJ9MlZYxlzOWKhHqfdxOTDdmfk/+ftdOwN//JGQs0g= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20260303044756-e9e2a52adaf0 h1:M93LWZ6iNDKr5lKTKc727Idty1Yb28cqgSfM9iBS+F0= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:fvSj40FltpPVU4KLda+I/WhFTwwiGuPpzpy+5UeiOaM= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/kube-proxy v0.0.0-20260303044756-e9e2a52adaf0 h1:3irDg3wS8FC2Ad2etU4zIuvPOrvqZLLn/uAL+zarO7c= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/kube-proxy v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:1nDymEUiZa0nIsVZkeSAFISoJsp0Nnd40vWuipDGLv0= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20260303044756-e9e2a52adaf0 h1:oTGF1xkyClEHgE6tPJRea+nr+qj87xhk7spFpczhCUg= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:5ZwsfUdHnj/nEAAdk8utKGDG6c4eZTyZg/7BHPE4uaQ= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20260303044756-e9e2a52adaf0 h1:UMxEse6cLj5UVoPrnUvCLXejOy1Q0NQjZ8f22mYuOXI= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:ZmlXPtm3wJFF6x/7QU8q/op7GHIyBvnUjsbkud1NeNQ= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20260303044756-e9e2a52adaf0 h1:zUmxHL3TG3Ou7SN/zU0I5ykFaTOuterfjqAPIJWgb7c= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:h59njVygVWeHIhmMqMoocQNjUyC3q8D51fZNSuMEAis= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/metrics v0.0.0-20260303044756-e9e2a52adaf0 h1:FrczvZ9dkyIrBdL9dMjeFy4bAe1mdWZOeJklFJUxmpE= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/metrics v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:5vBQ1SE0Ed1m5R6BkCKRXDboEjmKbbO17EyrF6XXiqA= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20260303044756-e9e2a52adaf0 h1:JuZNurkMdNL7PrkddsBKsbJuEtCGaC6lB5fvGsVHwsU= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:0tRkeDDp5zQDR/JKhcttJMwq3mTWqNm5GE2DgjfRvsY= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20260303044756-e9e2a52adaf0 h1:VD7WwvMpgBTi++h1LpLszj9fZXBwMhPtb6jIjlpLeGo= -github.com/kplane-dev/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20260303044756-e9e2a52adaf0/go.mod h1:2Hv3+Ga3SCWHO9E8+Ym58CioDvWe7wW21gy3nVetVNw= -github.com/kplane-dev/storage v0.0.0-20260303050750-8ad94e8ce404 h1:WBSxvdd1pjj2BesniHdRIBklW84P8KQfjvc/RRCvNJg= -github.com/kplane-dev/storage v0.0.0-20260303050750-8ad94e8ce404/go.mod h1:CApTI1DlyUgXF/p8uWvPVQ0pyPoeY7Yt+enSu57gPoo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -215,6 +227,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/selinux v1.13.1 h1:A8nNeceYngH9Ow++M+VVEwJVpdFmrlxsN22F+ISDCJE= github.com/opencontainers/selinux v1.13.1/go.mod h1:S10WXZ/osk2kWOYKy1x2f/eXF5ZHJoUs8UU/2caNRbg= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -222,6 +236,7 @@ github.com/pquerna/cachecontrol v0.2.0 h1:vBXSNuE5MYP9IJ5kjsdo8uq+w41jSPgvba2DEn github.com/pquerna/cachecontrol v0.2.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= @@ -242,6 +257,8 @@ github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4 github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= +github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -278,8 +295,12 @@ go.etcd.io/etcd/server/v3 v3.6.7 h1:8dEGQ877tj0cQJFEfD2bDoZDA76qbS2OkvCNjwAyrSo= go.etcd.io/etcd/server/v3 v3.6.7/go.mod h1:LEM328bPA2uVMhN0+Ht/vAsADW127QS1oM7EuHrOTy0= go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs= +go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 h1:XmiuHzgJt067+a6kwyAzkhXooYVv3/TOw9cM2VfJgUM= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0/go.mod h1:KDgtbWKTQs4bM+VPUr6WlL9m/WXcmkCcBlIzqxPGzmI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= @@ -315,25 +336,38 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0= golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= -golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= -golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= +golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -349,6 +383,10 @@ golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -360,12 +398,35 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= -google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/api v0.265.0 h1:FZvfUdI8nfmuNrE34aOWFPmLC+qRBEiNm3JdivTvAAU= +google.golang.org/api v0.265.0/go.mod h1:uAvfEl3SLUj/7n6k+lJutcswVojHPp2Sp08jWCu8hLY= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM= +google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM= +google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0= +google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -382,6 +443,8 @@ gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYs gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20260127142750-a19766b6e2d4 h1:HhDfevmPS+OalTjQRKbTHppRIz01AWi8s45TMXStgYY= diff --git a/pkg/multicluster/bootstrap/allocator_registry.go b/pkg/multicluster/bootstrap/allocator_registry.go new file mode 100644 index 0000000..1f20210 --- /dev/null +++ b/pkg/multicluster/bootstrap/allocator_registry.go @@ -0,0 +1,223 @@ +package bootstrap + +import ( + "fmt" + "net" + "sync" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/storage" + "k8s.io/apiserver/pkg/storage/storagebackend" + "k8s.io/apiserver/pkg/storage/storagebackend/factory" + "k8s.io/klog/v2" + api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/registry/core/rangeallocation" + "k8s.io/kubernetes/pkg/registry/core/service/allocator" + serviceallocator "k8s.io/kubernetes/pkg/registry/core/service/allocator/storage" + "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" + "k8s.io/kubernetes/pkg/registry/core/service/portallocator" + svcstorage "k8s.io/kubernetes/pkg/registry/core/service/storage" + utilnet "k8s.io/apimachinery/pkg/util/net" + netutils "k8s.io/utils/net" +) + +// StorageFactory creates a raw storage.Interface for a given resource prefix. +type StorageFactory func( + config *storagebackend.ConfigForResource, + newFunc, newListFunc func() runtime.Object, + resourcePrefix string, +) (storage.Interface, factory.DestroyFunc, error) + +// AllocatorRegistryOptions configures the per-cluster allocator registry. +type AllocatorRegistryOptions struct { + PrimaryServiceCIDR net.IPNet + SecondaryServiceCIDR net.IPNet + NodePortRange utilnet.PortRange + // ServiceStorageConfig is the base storage config for the "services" resource. + ServiceStorageConfig *storagebackend.ConfigForResource + // BackendFactory creates raw storage.Interface for allocator bitmap persistence. + // When nil, factory.Create (etcd) is used. + BackendFactory StorageFactory +} + +// clusterRegistries holds the range registries and allocators for one cluster. +type clusterRegistries struct { + alloc *svcstorage.Allocators + clusterIP rangeallocation.RangeRegistry + secondaryClusterIP rangeallocation.RangeRegistry + nodePort rangeallocation.RangeRegistry +} + +// AllocatorRegistry lazily creates per-cluster service allocators. +type AllocatorRegistry struct { + opts AllocatorRegistryOptions + + mu sync.Mutex + cache map[string]*clusterRegistries +} + +// NewAllocatorRegistry creates a new per-cluster allocator registry. +func NewAllocatorRegistry(opts AllocatorRegistryOptions) *AllocatorRegistry { + return &AllocatorRegistry{ + opts: opts, + cache: map[string]*clusterRegistries{}, + } +} + +// AllocatorsForCluster returns (lazily creating) allocators for the given cluster. +func (r *AllocatorRegistry) AllocatorsForCluster(clusterID string) (*svcstorage.Allocators, error) { + cr, err := r.ensureCluster(clusterID) + if err != nil { + return nil, err + } + return cr.alloc, nil +} + +func (r *AllocatorRegistry) ensureCluster(clusterID string) (*clusterRegistries, error) { + r.mu.Lock() + defer r.mu.Unlock() + if cr, ok := r.cache[clusterID]; ok { + return cr, nil + } + cr, err := r.newClusterRegistries(clusterID) + if err != nil { + return nil, err + } + r.cache[clusterID] = cr + return cr, nil +} + +func (r *AllocatorRegistry) newClusterRegistries(clusterID string) (*clusterRegistries, error) { + cr := &clusterRegistries{} + ipAllocs := map[api.IPFamily]ipallocator.Interface{} + + primaryAlloc, primaryReg, err := r.newIPAllocator(clusterID, &r.opts.PrimaryServiceCIDR, "/ranges/serviceips") + if err != nil { + return nil, fmt.Errorf("primary IP allocator for cluster %s: %w", clusterID, err) + } + ipAllocs[primaryAlloc.IPFamily()] = primaryAlloc + cr.clusterIP = primaryReg + + if r.opts.SecondaryServiceCIDR.IP != nil { + secondaryAlloc, secondaryReg, err := r.newIPAllocator(clusterID, &r.opts.SecondaryServiceCIDR, "/ranges/secondaryserviceips") + if err != nil { + return nil, fmt.Errorf("secondary IP allocator for cluster %s: %w", clusterID, err) + } + ipAllocs[secondaryAlloc.IPFamily()] = secondaryAlloc + cr.secondaryClusterIP = secondaryReg + } + + portAlloc, portReg, err := r.newPortAllocator(clusterID) + if err != nil { + return nil, fmt.Errorf("port allocator for cluster %s: %w", clusterID, err) + } + cr.nodePort = portReg + + defaultFamily := api.IPv4Protocol + if netutils.IsIPv6CIDR(&r.opts.PrimaryServiceCIDR) { + defaultFamily = api.IPv6Protocol + } + + a := svcstorage.MakeAllocators(defaultFamily, ipAllocs, portAlloc) + cr.alloc = &a + klog.Infof("mc.allocatorRegistry created allocators for cluster=%s defaultFamily=%s families=%d", clusterID, defaultFamily, len(ipAllocs)) + return cr, nil +} + +func (r *AllocatorRegistry) newIPAllocator(clusterID string, cidr *net.IPNet, baseKey string) (*ipallocator.Range, rangeallocation.RangeRegistry, error) { + clusterBaseKey := clusterRangeKey(clusterID, baseKey) + var reg rangeallocation.RangeRegistry + alloc, err := ipallocator.New(cidr, func(max int, rangeSpec string, offset int) (allocator.Interface, error) { + mem := allocator.NewAllocationMapWithOffset(max, rangeSpec, offset) + store, err := r.createStorage(clusterBaseKey) + if err != nil { + return nil, err + } + etcd := serviceallocator.NewWithStorage(mem, clusterBaseKey, store, api.Resource("serviceipallocations")) + // Initialize the bitmap if it doesn't exist yet. + rangeRegistry, err := etcd.Get() + if err != nil { + return nil, err + } + rangeRegistry.Range = rangeSpec + if len(rangeRegistry.ResourceVersion) == 0 { + if err := etcd.CreateOrUpdate(rangeRegistry); err != nil { + return nil, err + } + } + reg = etcd + return etcd, nil + }) + return alloc, reg, err +} + +func (r *AllocatorRegistry) newPortAllocator(clusterID string) (*portallocator.PortAllocator, rangeallocation.RangeRegistry, error) { + clusterBaseKey := clusterRangeKey(clusterID, "/ranges/servicenodeports") + var reg rangeallocation.RangeRegistry + alloc, err := portallocator.New(r.opts.NodePortRange, func(max int, rangeSpec string, offset int) (allocator.Interface, error) { + mem := allocator.NewAllocationMapWithOffset(max, rangeSpec, offset) + store, err := r.createStorage(clusterBaseKey) + if err != nil { + return nil, err + } + etcd := serviceallocator.NewWithStorage(mem, clusterBaseKey, store, api.Resource("servicenodeportallocations")) + reg = etcd + return etcd, nil + }) + return alloc, reg, err +} + +// clusterIPRegistry returns the RangeRegistry for primary ClusterIP, or nil. +func (r *AllocatorRegistry) clusterIPRegistry(clusterID string) rangeallocation.RangeRegistry { + r.mu.Lock() + defer r.mu.Unlock() + if cr, ok := r.cache[clusterID]; ok { + return cr.clusterIP + } + return nil +} + +// secondaryClusterIPRegistry returns the RangeRegistry for secondary ClusterIP, or nil. +func (r *AllocatorRegistry) secondaryClusterIPRegistry(clusterID string) rangeallocation.RangeRegistry { + r.mu.Lock() + defer r.mu.Unlock() + if cr, ok := r.cache[clusterID]; ok { + return cr.secondaryClusterIP + } + return nil +} + +// secondaryCIDR returns a pointer to the secondary CIDR, or nil if not configured. +func (r *AllocatorRegistry) secondaryCIDR() *net.IPNet { + if r.opts.SecondaryServiceCIDR.IP == nil { + return &net.IPNet{} + } + return &r.opts.SecondaryServiceCIDR +} + +// nodePortRegistry returns the RangeRegistry for NodePorts, or nil. +func (r *AllocatorRegistry) nodePortRegistry(clusterID string) rangeallocation.RangeRegistry { + r.mu.Lock() + defer r.mu.Unlock() + if cr, ok := r.cache[clusterID]; ok { + return cr.nodePort + } + return nil +} + +func (r *AllocatorRegistry) createStorage(baseKey string) (storage.Interface, error) { + cfg := *r.opts.ServiceStorageConfig + newFunc := func() runtime.Object { return &api.RangeAllocation{} } + if r.opts.BackendFactory != nil { + s, _, err := r.opts.BackendFactory(&cfg, newFunc, nil, baseKey) + return s, err + } + s, _, err := factory.Create(cfg, newFunc, nil, baseKey) + return s, err +} + +// clusterRangeKey prefixes a range key with the cluster path. +// e.g. "/ranges/serviceips" → "/ranges/clusters//serviceips" +func clusterRangeKey(clusterID, baseKey string) string { + return "/ranges/clusters/" + clusterID + baseKey[len("/ranges"):] +} diff --git a/pkg/multicluster/bootstrap/allocator_repair.go b/pkg/multicluster/bootstrap/allocator_repair.go new file mode 100644 index 0000000..eabef9e --- /dev/null +++ b/pkg/multicluster/bootstrap/allocator_repair.go @@ -0,0 +1,108 @@ +package bootstrap + +import ( + "sync" + "time" + + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + serviceipallocatorcontroller "k8s.io/kubernetes/pkg/registry/core/service/ipallocator/controller" + portallocatorcontroller "k8s.io/kubernetes/pkg/registry/core/service/portallocator/controller" +) + +// AllocatorRepairOptions configures per-cluster allocator repair controllers. +type AllocatorRepairOptions struct { + ClientForCluster func(clusterID string) (kubernetes.Interface, error) + StopChForCluster func(clusterID string) (<-chan struct{}, error) + Registry *AllocatorRegistry + RepairInterval time.Duration +} + +// AllocatorRepairManager lazily starts per-cluster service allocator repair loops. +type AllocatorRepairManager struct { + opts AllocatorRepairOptions + mu sync.Mutex + run map[string]struct{} +} + +// NewAllocatorRepairManager creates a new per-cluster allocator repair manager. +func NewAllocatorRepairManager(opts AllocatorRepairOptions) *AllocatorRepairManager { + if opts.RepairInterval <= 0 { + opts.RepairInterval = 3 * time.Minute + } + return &AllocatorRepairManager{ + opts: opts, + run: map[string]struct{}{}, + } +} + +// Ensure starts the repair loops for the given cluster if not already running. +func (m *AllocatorRepairManager) Ensure(clusterID string) { + if m == nil || clusterID == "" || m.opts.Registry == nil { + return + } + m.mu.Lock() + if _, ok := m.run[clusterID]; ok { + m.mu.Unlock() + return + } + m.run[clusterID] = struct{}{} + m.mu.Unlock() + + go m.startRepair(clusterID) +} + +func (m *AllocatorRepairManager) startRepair(clusterID string) { + cs, err := m.opts.ClientForCluster(clusterID) + if err != nil { + klog.Errorf("mc.allocatorRepair client failed cluster=%s: %v", clusterID, err) + return + } + stopCh, err := m.opts.StopChForCluster(clusterID) + if err != nil { + klog.Errorf("mc.allocatorRepair stop channel failed cluster=%s: %v", clusterID, err) + return + } + + // Ensure allocators are initialized before starting repair. + _, err = m.opts.Registry.AllocatorsForCluster(clusterID) + if err != nil { + klog.Errorf("mc.allocatorRepair allocator init failed cluster=%s: %v", clusterID, err) + return + } + + regOpts := m.opts.Registry.opts + + // Convert read-only stop channel to a writable one for RunUntil. + portStopCh := make(chan struct{}) + ipStopCh := make(chan struct{}) + go func() { + <-stopCh + close(portStopCh) + close(ipStopCh) + }() + + // Node port repair + portRepair := portallocatorcontroller.NewRepair( + m.opts.RepairInterval, + cs.CoreV1(), + cs.EventsV1(), + regOpts.NodePortRange, + m.opts.Registry.nodePortRegistry(clusterID), + ) + go portRepair.RunUntil(func() {}, portStopCh) + + // ClusterIP repair + ipRepair := serviceipallocatorcontroller.NewRepair( + m.opts.RepairInterval, + cs.CoreV1(), + cs.EventsV1(), + ®Opts.PrimaryServiceCIDR, + m.opts.Registry.clusterIPRegistry(clusterID), + m.opts.Registry.secondaryCIDR(), + m.opts.Registry.secondaryClusterIPRegistry(clusterID), + ) + go ipRepair.RunUntil(func() {}, ipStopCh) + + klog.Infof("mc.allocatorRepair started repair controllers for cluster=%s", clusterID) +} diff --git a/pkg/multicluster/bootstrap/crd_controller.go b/pkg/multicluster/bootstrap/crd_controller.go index 3632452..7041335 100644 --- a/pkg/multicluster/bootstrap/crd_controller.go +++ b/pkg/multicluster/bootstrap/crd_controller.go @@ -2,6 +2,7 @@ package bootstrap import ( "sync" + "time" "k8s.io/klog/v2" ) @@ -79,7 +80,21 @@ func (c *MulticlusterCRDController) run() { case clusterID := <-c.queue: if c.runtimeManager != nil { if err := c.runtimeManager.EnsureCluster(clusterID, c.stopCh); err != nil { - klog.Errorf("mc.crdController ensure cluster failed cluster=%s err=%v", clusterID, err) + klog.Warningf("mc.crdController ensure cluster failed cluster=%s err=%v (will retry)", clusterID, err) + // Re-enqueue after a short delay. Storage may not be + // registered yet if the apiextensions server hasn't + // finished construction. + go func() { + select { + case <-c.stopCh: + case <-time.After(500 * time.Millisecond): + c.mu.Lock() + delete(c.enqueued, clusterID) + c.mu.Unlock() + c.EnsureCluster(clusterID) + } + }() + continue } } diff --git a/pkg/multicluster/options.go b/pkg/multicluster/options.go index a69259f..38fefb1 100644 --- a/pkg/multicluster/options.go +++ b/pkg/multicluster/options.go @@ -18,6 +18,15 @@ import ( // // These defaults are safe and aim for lowest watch cardinality. +// SpannerConfig holds connection details for a Spanner backend. +// When set on Options, storage uses Spanner instead of etcd. +type SpannerConfig struct { + Project string + Instance string + Database string + EmulatorHost string // for development/testing +} + type Options struct { EtcdPrefix string DefaultCluster string @@ -34,6 +43,8 @@ type Options struct { // InformerRegistry provides MultiClusterInformer instances for resource types. // Set by config.go and used by storage.go to register clusteredStorage instances. InformerRegistry *InformerRegistry + // Spanner, when non-nil, selects Spanner as the storage backend instead of etcd. + Spanner *SpannerConfig } // ResourceScope defines which keyspace view a request should use. diff --git a/pkg/multicluster/storage.go b/pkg/multicluster/storage.go index f78328d..f97abd1 100644 --- a/pkg/multicluster/storage.go +++ b/pkg/multicluster/storage.go @@ -27,7 +27,10 @@ import ( "github.com/kplane-dev/apiserver/pkg/multicluster/internalcap" mcstorage "github.com/kplane-dev/apiserver/pkg/multicluster/storage" + spanner "cloud.google.com/go/spanner" + spannerstore "github.com/kplane-dev/spanner" extstorage "github.com/kplane-dev/storage" + "k8s.io/apiserver/pkg/storage/value/encrypt/identity" ) // RESTOptionsDecorator wraps the underlying getter to inject a decorator that @@ -71,10 +74,14 @@ func (w RESTOptionsDecorator) GetRESTOptions(resource schema.GroupResource, exam // identity hooks (IdentityFromKey, WrapWatchObject, UnwrapObject) and // sets WrapDecodedObject on the etcd3 storage config so that decoded // objects carry their storage key through the watch.Event boundary. - base := extstorage.StorageWithClusterIdentity(extstorage.DecoratorConfig{ + decoratorCfg := extstorage.DecoratorConfig{ KeyLayout: extstorage.DefaultKeyLayout(), GroupResource: resource, - }) + } + if w.Options.Spanner != nil { + decoratorCfg.BackendFactory = NewSpannerBackendFactory(w.Options.Spanner) + } + base := extstorage.StorageWithClusterIdentity(decoratorCfg) base = wrapBaseDecorator(base, w.Options) opts.Decorator = func( config *storagebackend.ConfigForResource, @@ -440,3 +447,55 @@ func (c *clusteredStorage) ensureStore() (storage.Interface, error) { c.destroyFn = destroy return c.store, nil } + +// NewSpannerBackendFactory creates a BackendFactory from a SpannerConfig. +// The factory creates Spanner-backed storage.Interface instances using the +// same transformer pipeline as etcd. A single Spanner client is shared across +// all resource stores to avoid session pool explosion. +func NewSpannerBackendFactory(cfg *SpannerConfig) extstorage.BackendFactory { + var ( + sharedClient *spanner.Client + clientOnce sync.Once + clientErr error + ) + + return func( + config *storagebackend.ConfigForResource, + newFunc, newListFunc func() runtime.Object, + resourcePrefix string, + ) (storage.Interface, factory.DestroyFunc, error) { + clientOnce.Do(func() { + spannerCfg := spannerstore.SpannerConfig{ + Project: cfg.Project, + Instance: cfg.Instance, + Database: cfg.Database, + EmulatorHost: cfg.EmulatorHost, + } + sharedClient, clientErr = spannerCfg.NewClient(context.Background()) + }) + if clientErr != nil { + return nil, nil, fmt.Errorf("creating spanner client: %w", clientErr) + } + + transformer := config.Transformer + if transformer == nil { + transformer = identity.NewEncryptCheckTransformer() + } + + s := spannerstore.NewStore( + sharedClient, + config.Codec, + newFunc, + newListFunc, + config.Prefix, + resourcePrefix, + transformer, + config.WrapDecodedObject, + ) + + // Individual stores don't close the shared client. + destroyFunc := func() {} + + return s, destroyFunc, nil + } +} diff --git a/pkg/multicluster/typedinformer/adapters.go b/pkg/multicluster/typedinformer/adapters.go index 12629cf..61d9159 100644 --- a/pkg/multicluster/typedinformer/adapters.go +++ b/pkg/multicluster/typedinformer/adapters.go @@ -6,8 +6,10 @@ import ( admissionregistrationv1 "k8s.io/api/admissionregistration/v1" corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" admissionregistrationlisters "k8s.io/client-go/listers/admissionregistration/v1" corelisters "k8s.io/client-go/listers/core/v1" discoverylisters "k8s.io/client-go/listers/discovery/v1" @@ -63,7 +65,7 @@ func (l *namespaceLister) List(sel labels.Selector) (ret []*corev1.Namespace, er func (l *namespaceLister) Get(name string) (*corev1.Namespace, error) { obj, ok := l.mci.Get(l.cluster, "", name) if !ok { - return nil, fmt.Errorf("namespace %q not found", name) + return nil, apierrors.NewNotFound(schema.GroupResource{Resource: "namespaces"}, name) } return convert[corev1.Namespace](obj) } @@ -139,7 +141,7 @@ func (l *secretNamespaceLister) List(sel labels.Selector) (ret []*corev1.Secret, func (l *secretNamespaceLister) Get(name string) (*corev1.Secret, error) { obj, ok := l.parent.mci.Get(l.parent.cluster, l.namespace, name) if !ok { - return nil, fmt.Errorf("secret %s/%s not found", l.namespace, name) + return nil, apierrors.NewNotFound(schema.GroupResource{Resource: "secrets"}, name) } return convert[corev1.Secret](obj) } @@ -193,7 +195,7 @@ func (l *serviceAccountNamespaceLister) List(sel labels.Selector) (ret []*corev1 func (l *serviceAccountNamespaceLister) Get(name string) (*corev1.ServiceAccount, error) { obj, ok := l.parent.mci.Get(l.parent.cluster, l.namespace, name) if !ok { - return nil, fmt.Errorf("serviceaccount %s/%s not found", l.namespace, name) + return nil, apierrors.NewNotFound(schema.GroupResource{Resource: "serviceaccounts"}, name) } return convert[corev1.ServiceAccount](obj) } @@ -247,7 +249,7 @@ func (l *podNamespaceLister) List(sel labels.Selector) (ret []*corev1.Pod, err e func (l *podNamespaceLister) Get(name string) (*corev1.Pod, error) { obj, ok := l.parent.mci.Get(l.parent.cluster, l.namespace, name) if !ok { - return nil, fmt.Errorf("pod %s/%s not found", l.namespace, name) + return nil, apierrors.NewNotFound(schema.GroupResource{Resource: "pods"}, name) } return convert[corev1.Pod](obj) } @@ -282,7 +284,7 @@ func (l *nodeLister) List(sel labels.Selector) (ret []*corev1.Node, err error) { func (l *nodeLister) Get(name string) (*corev1.Node, error) { obj, ok := l.mci.Get(l.cluster, "", name) if !ok { - return nil, fmt.Errorf("node %q not found", name) + return nil, apierrors.NewNotFound(schema.GroupResource{Resource: "nodes"}, name) } return convert[corev1.Node](obj) } @@ -336,7 +338,7 @@ func (l *serviceNamespaceLister) List(sel labels.Selector) (ret []*corev1.Servic func (l *serviceNamespaceLister) Get(name string) (*corev1.Service, error) { obj, ok := l.parent.mci.Get(l.parent.cluster, l.namespace, name) if !ok { - return nil, fmt.Errorf("service %s/%s not found", l.namespace, name) + return nil, apierrors.NewNotFound(schema.GroupResource{Resource: "services"}, name) } return convert[corev1.Service](obj) } @@ -390,7 +392,7 @@ func (l *endpointSliceNamespaceLister) List(sel labels.Selector) (ret []*discove func (l *endpointSliceNamespaceLister) Get(name string) (*discoveryv1.EndpointSlice, error) { obj, ok := l.parent.mci.Get(l.parent.cluster, l.namespace, name) if !ok { - return nil, fmt.Errorf("endpointslice %s/%s not found", l.namespace, name) + return nil, apierrors.NewNotFound(schema.GroupResource{Group: "discovery.k8s.io", Resource: "endpointslices"}, name) } return convert[discoveryv1.EndpointSlice](obj) } @@ -425,7 +427,7 @@ func (l *mutatingWebhookConfigurationLister) List(sel labels.Selector) (ret []*a func (l *mutatingWebhookConfigurationLister) Get(name string) (*admissionregistrationv1.MutatingWebhookConfiguration, error) { obj, ok := l.mci.Get(l.cluster, "", name) if !ok { - return nil, fmt.Errorf("mutatingwebhookconfiguration %q not found", name) + return nil, apierrors.NewNotFound(schema.GroupResource{Group: "admissionregistration.k8s.io", Resource: "mutatingwebhookconfigurations"}, name) } return convert[admissionregistrationv1.MutatingWebhookConfiguration](obj) } @@ -460,7 +462,7 @@ func (l *validatingWebhookConfigurationLister) List(sel labels.Selector) (ret [] func (l *validatingWebhookConfigurationLister) Get(name string) (*admissionregistrationv1.ValidatingWebhookConfiguration, error) { obj, ok := l.mci.Get(l.cluster, "", name) if !ok { - return nil, fmt.Errorf("validatingwebhookconfiguration %q not found", name) + return nil, apierrors.NewNotFound(schema.GroupResource{Group: "admissionregistration.k8s.io", Resource: "validatingwebhookconfigurations"}, name) } return convert[admissionregistrationv1.ValidatingWebhookConfiguration](obj) } diff --git a/test/smoke/apiserver_test.go b/test/smoke/apiserver_test.go index 972e04e..43e7940 100644 --- a/test/smoke/apiserver_test.go +++ b/test/smoke/apiserver_test.go @@ -23,6 +23,7 @@ import ( "time" mc "github.com/kplane-dev/apiserver/pkg/multicluster" + spannerstore "github.com/kplane-dev/spanner" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -116,6 +117,60 @@ type apiserverOptions struct { etcdPrefix string port int extraArgs []string + // spannerDB, when non-empty, adds --spanner-* flags to use Spanner storage. + spannerDB string +} + +func storageBackend() string { + return os.Getenv("KPLANE_STORAGE_BACKEND") // "" or "spanner" +} + +func spannerEmulatorHost() string { + if h := os.Getenv("SPANNER_EMULATOR_HOST"); h != "" { + return h + } + return "localhost:9010" +} + +var spannerInstanceOnce sync.Once + +func ensureSpannerInstance(t *testing.T) { + t.Helper() + spannerInstanceOnce.Do(func() { + cfg := spannerstore.SpannerConfig{ + Project: "test-project", + Instance: "test-instance", + EmulatorHost: spannerEmulatorHost(), + } + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + if err := spannerstore.EnsureInstance(ctx, cfg); err != nil { + t.Fatalf("EnsureInstance: %v", err) + } + }) +} + +func setupSpannerDB(t *testing.T) string { + t.Helper() + ensureSpannerInstance(t) + dbName := fmt.Sprintf("smoke_%d", time.Now().UnixNano()) + cfg := spannerstore.SpannerConfig{ + Project: "test-project", + Instance: "test-instance", + Database: dbName, + EmulatorHost: spannerEmulatorHost(), + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := spannerstore.EnsureSchema(ctx, cfg); err != nil { + t.Fatalf("EnsureSchema for smoke test: %v", err) + } + t.Cleanup(func() { + dropCtx, dropCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer dropCancel() + _ = spannerstore.DropDatabase(dropCtx, cfg) + }) + return dbName } func startAPIServer(t *testing.T, etcdEndpoints string) *testAPIServer { @@ -124,6 +179,11 @@ func startAPIServer(t *testing.T, etcdEndpoints string) *testAPIServer { func startAPIServerWithOptions(t *testing.T, etcdEndpoints string, opts apiserverOptions) *testAPIServer { t.Helper() + if storageBackend() == "spanner" { + if opts.spannerDB == "" { + opts.spannerDB = setupSpannerDB(t) + } + } if strings.TrimSpace(etcdEndpoints) == "" { t.Skip("ETCD_ENDPOINTS is not set; skipping integration smoke tests") } @@ -176,6 +236,14 @@ func startAPIServerWithOptions(t *testing.T, etcdEndpoints string, opts apiserve if opts.rootCluster != mc.DefaultClusterName { args = append(args, "--root-control-plane-name="+opts.rootCluster) } + if opts.spannerDB != "" { + args = append(args, + "--spanner-project=test-project", + "--spanner-instance=test-instance", + "--spanner-database="+opts.spannerDB, + "--spanner-emulator-host="+spannerEmulatorHost(), + ) + } if len(opts.extraArgs) > 0 { args = append(args, opts.extraArgs...) } diff --git a/test/smoke/auth_restart_test.go b/test/smoke/auth_restart_test.go index 81df342..e2f1818 100644 --- a/test/smoke/auth_restart_test.go +++ b/test/smoke/auth_restart_test.go @@ -25,6 +25,10 @@ func TestRBACIsolationAfterAPIServerRestart(t *testing.T) { port := mustFreePort(t) prefix := fmt.Sprintf("/registry-smoke-restart-rbac-%d", time.Now().UnixNano()) opts := apiserverOptions{etcdPrefix: prefix, port: port} + // Pre-create the Spanner database so both apiserver instances share it. + if storageBackend() == "spanner" { + opts.spannerDB = setupSpannerDB(t) + } s1 := startAPIServerWithOptions(t, etcd, opts) clusterA := "c-" + randSuffix(3) diff --git a/test/smoke/core_bootstrap_test.go b/test/smoke/core_bootstrap_test.go index 6ed8dd2..47e23db 100644 --- a/test/smoke/core_bootstrap_test.go +++ b/test/smoke/core_bootstrap_test.go @@ -23,7 +23,7 @@ func TestKubernetesServiceBootstrapPerCluster(t *testing.T) { t.Fatalf("trigger request failed: %v", err) } - deadline := time.Now().Add(10 * time.Second) + deadline := time.Now().Add(30 * time.Second) for { svc, err := cs.CoreV1().Services(metav1.NamespaceDefault).Get(t.Context(), "kubernetes", metav1.GetOptions{}) if err == nil { diff --git a/test/smoke/no_etcd_metadata_test.go b/test/smoke/no_etcd_metadata_test.go index 53e7e88..5825cba 100644 --- a/test/smoke/no_etcd_metadata_test.go +++ b/test/smoke/no_etcd_metadata_test.go @@ -18,6 +18,9 @@ import ( // raw etcd values to verify that cluster identity is NOT stored as labels or // annotations on the object. Identity is derived entirely from the etcd key path. func TestNoIdentityMetadataInEtcd(t *testing.T) { + if storageBackend() == "spanner" { + t.Skip("test reads raw etcd data; not applicable to Spanner backend") + } etcdEndpoints := os.Getenv("ETCD_ENDPOINTS") if etcdEndpoints == "" { t.Skip("ETCD_ENDPOINTS not set")