feat: Spanner storage backend and per-cluster allocators

cmd/apiserver/app/config.go

@@ -32,6 +32,8 @@ import (
 	genericfilters "k8s.io/apiserver/pkg/endpoints/filters"
 	"k8s.io/apiserver/pkg/registry/generic"
 	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/storage"
+	storagefeature "k8s.io/apiserver/pkg/storage/feature"
 	serverfilters "k8s.io/apiserver/pkg/server/filters"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/pkg/util/webhook"
@@ -40,10 +42,12 @@ import (
 	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
 
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	api "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/controlplane"
 	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
 	"k8s.io/kubernetes/pkg/features"
 	generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
+	servicestore "k8s.io/kubernetes/pkg/registry/core/service/storage"
 
 	"github.com/kplane-dev/apiserver/cmd/apiserver/app/options"
 	mc "github.com/kplane-dev/apiserver/pkg/multicluster"
@@ -129,6 +133,18 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) {
 	clientPool := mc.NewClientPool(genericConfig.LoopbackClientConfig, mcOpts.PathPrefix, mcOpts.ControlPlaneSegment)
 	informerRegistry := mc.NewInformerRegistry(wait.ContextForChannel(genericConfig.DrainedNotify()))
 	mcOpts.InformerRegistry = informerRegistry
+	if opts.SpannerProject != "" {
+		mcOpts.Spanner = &mc.SpannerConfig{
+			Project:      opts.SpannerProject,
+			Instance:     opts.SpannerInstance,
+			Database:     opts.SpannerDatabase,
+			EmulatorHost: opts.SpannerEmulatorHost,
+		}
+		// Spanner implements RequestWatchProgress natively via its
+		// broadcaster. Tell the feature checker so the cacher allows
+		// SendInitialEvents (WatchList) watches.
+		storagefeature.SetFeatureSupported(storage.RequestWatchProgress, true)
+	}
 	var crdRuntimeMgr *mcbootstrap.CRDRuntimeManager
 	systemNamespaceBootstrapper := mcbootstrap.NewSystemNamespaceBootstrapper(mcbootstrap.SystemNamespaceOptions{
 		ClientForCluster: clientPool.KubeClientForCluster,
@@ -234,6 +250,40 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) {
 	if c.KubeAPIs.ControlPlane.Generic.RESTOptionsGetter != nil {
 		c.KubeAPIs.ControlPlane.Generic.RESTOptionsGetter = decorateRESTOptionsGetter("controlplane", c.KubeAPIs.ControlPlane.Generic.RESTOptionsGetter, mcOpts)
 	}
+
+	// Per-cluster service allocators: create an AllocatorRegistry so each VCP
+	// gets its own ClusterIP and NodePort bitmap, bypassing the shared root range.
+	serviceStorageConfig, err := storageFactory.NewConfig(api.Resource("services"), &api.Service{})
+	if err != nil {
+		return nil, err
+	}
+	var allocBackendFactory mcbootstrap.StorageFactory
+	if mcOpts.Spanner != nil {
+		sf := mc.NewSpannerBackendFactory(mcOpts.Spanner)
+		allocBackendFactory = mcbootstrap.StorageFactory(sf)
+	}
+	allocatorRegistry := mcbootstrap.NewAllocatorRegistry(mcbootstrap.AllocatorRegistryOptions{
+		PrimaryServiceCIDR:   opts.PrimaryServiceClusterIPRange,
+		SecondaryServiceCIDR: opts.SecondaryServiceClusterIPRange,
+		NodePortRange:        opts.ServiceNodePortRange,
+		ServiceStorageConfig: serviceStorageConfig.ForResource(api.Resource("serviceipallocations")),
+		BackendFactory:       allocBackendFactory,
+	})
+	c.KubeAPIs.Extra.ServiceRESTHook = func(rest *servicestore.REST) {
+		rest.ClusterAllocators = func(ctx context.Context) *servicestore.Allocators {
+			cid, _, _ := mc.FromContext(ctx)
+			if cid == "" || cid == mcOpts.DefaultCluster {
+				return nil // use root allocators
+			}
+			alloc, err := allocatorRegistry.AllocatorsForCluster(cid)
+			if err != nil {
+				klog.Errorf("mc.allocatorRegistry failed cluster=%s: %v", cid, err)
+				return nil // fall back to root allocators
+			}
+			return alloc
+		}
+	}
+
 	targetPort := 443
 	if opts.SecureServing != nil && opts.SecureServing.BindPort > 0 {
 		targetPort = opts.SecureServing.BindPort
@@ -254,6 +304,11 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) {
 		TargetPort:       targetPort,
 		NodePort:         opts.KubernetesServiceNodePort,
 	})
+	allocatorRepairMgr := mcbootstrap.NewAllocatorRepairManager(mcbootstrap.AllocatorRepairOptions{
+		ClientForCluster: clientPool.KubeClientForCluster,
+		StopChForCluster: stopChForCluster,
+		Registry:         allocatorRegistry,
+	})
 	mcOpts.OnClusterSelected = func(clusterID string) {
 		// Preserve upstream root bootstrap as-is; only add multicluster bootstrap for non-root VCPs.
 		if clusterID == mcOpts.DefaultCluster {
@@ -265,6 +320,7 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) {
 		go serviceCIDRBootstrapper.Ensure(clusterID)
 		go rbacBootstrapper.Ensure(clusterID)
 		go internalControllerMgr.Ensure(clusterID)
+		go allocatorRepairMgr.Ensure(clusterID)
 		if opts.KubernetesServiceMode == options.KubernetesServiceModePerClusterAutoIP {
 			go kubeServiceControllerMgr.Ensure(clusterID)
 		}

cmd/apiserver/app/options/options.go

@@ -64,6 +64,13 @@ type Extra struct {
 	EndpointReconcilerType string
 
 	MasterCount int
+
+	// Spanner backend flags. When SpannerProject is non-empty, storage uses
+	// Spanner instead of etcd for multicluster resource data.
+	SpannerProject      string
+	SpannerInstance     string
+	SpannerDatabase     string
+	SpannerEmulatorHost string
 }
 
 const (
@@ -124,6 +131,16 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
 	mcfs.StringVar(&s.ServiceCIDRSharingMode, "service-cidr-sharing-mode", s.ServiceCIDRSharingMode,
 		"Service CIDR strategy across control planes: 'shared' keeps root-managed defaults only; 'per-cluster' bootstraps default ServiceCIDR in each virtual control plane.")
 
+	spannerfs := fss.FlagSet("spanner")
+	spannerfs.StringVar(&s.SpannerProject, "spanner-project", s.SpannerProject,
+		"GCP project ID for Spanner storage backend. When set, multicluster resource storage uses Spanner instead of etcd.")
+	spannerfs.StringVar(&s.SpannerInstance, "spanner-instance", s.SpannerInstance,
+		"Spanner instance ID.")
+	spannerfs.StringVar(&s.SpannerDatabase, "spanner-database", s.SpannerDatabase,
+		"Spanner database name.")
+	spannerfs.StringVar(&s.SpannerEmulatorHost, "spanner-emulator-host", s.SpannerEmulatorHost,
+		"Spanner emulator host:port for local development (e.g. localhost:9010). Disables TLS and authentication.")
+
 	fs := fss.FlagSet("misc")
 
 	fs.BoolVar(&s.AllowPrivileged, "allow-privileged", s.AllowPrivileged,

go.mod

@@ -4,6 +4,7 @@ go 1.25.0
 
 require (
 	github.com/kplane-dev/informer v0.0.0-00010101000000-000000000000
+	github.com/kplane-dev/spanner v0.0.0
 	github.com/kplane-dev/storage v0.0.0
 	github.com/spf13/cobra v1.10.1
 	go.etcd.io/etcd/client/v3 v3.6.7
@@ -24,6 +25,7 @@ require (
 
 replace (
 	github.com/kplane-dev/informer => github.com/kplane-dev/informer v0.0.0-20260303050920-e9c86850386e
+	github.com/kplane-dev/spanner => github.com/kplane-dev/spanner v0.0.0-20260311053139-032c9428fc1a
 	github.com/kplane-dev/storage => github.com/kplane-dev/storage v0.0.0-20260303050750-8ad94e8ce404
 	k8s.io/api => github.com/kplane-dev/kubernetes/staging/src/k8s.io/api v0.0.0-20260303044756-e9e2a52adaf0
 	k8s.io/apiextensions-apiserver => github.com/kplane-dev/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20260303044756-e9e2a52adaf0
@@ -61,24 +63,38 @@ replace (
 
 require (
 	cel.dev/expr v0.24.0 // indirect
+	cloud.google.com/go v0.123.0 // indirect
+	cloud.google.com/go/auth v0.18.1 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
+	cloud.google.com/go/compute/metadata v0.9.0 // indirect
+	cloud.google.com/go/iam v1.5.3 // indirect
+	cloud.google.com/go/longrunning v0.8.0 // indirect
+	cloud.google.com/go/monitoring v1.24.3 // indirect
+	cloud.google.com/go/spanner v1.88.0 // indirect
 	cyphar.com/go-pathrs v0.2.2 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
+	github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.6.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
 	github.com/coreos/go-oidc v2.5.0+incompatible // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.7.0 // indirect
 	github.com/cyphar/filepath-securejoin v0.6.1 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/distribution/reference v0.6.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.13.0 // indirect
+	github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
+	github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zapr v1.3.0 // indirect
@@ -100,7 +116,10 @@ require (
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/cel-go v0.26.1 // indirect
 	github.com/google/gnostic-models v0.7.0 // indirect
+	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
+	github.com/googleapis/gax-go/v2 v2.17.0 // indirect
 	github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 // indirect
@@ -119,6 +138,7 @@ require (
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/selinux v1.13.1 // indirect
+	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/pquerna/cachecontrol v0.2.0 // indirect
 	github.com/prometheus/client_golang v1.23.2 // indirect
@@ -127,17 +147,21 @@ require (
 	github.com/prometheus/procfs v0.19.2 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
+	github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
 	github.com/stoewer/go-strcase v1.3.1 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.etcd.io/etcd/api/v3 v3.6.7 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.6.7 // indirect
+	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect
 	go.opentelemetry.io/otel/metric v1.40.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.40.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect
 	go.opentelemetry.io/otel/trace v1.40.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
@@ -147,14 +171,16 @@ require (
 	golang.org/x/crypto v0.47.0 // indirect
 	golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect
 	golang.org/x/net v0.49.0 // indirect
-	golang.org/x/oauth2 v0.34.0 // indirect
+	golang.org/x/oauth2 v0.35.0 // indirect
 	golang.org/x/sys v0.40.0 // indirect
 	golang.org/x/term v0.39.0 // indirect
 	golang.org/x/text v0.33.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
 	golang.org/x/tools v0.40.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect
+	google.golang.org/api v0.265.0 // indirect
+	google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
 	google.golang.org/grpc v1.78.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect