From 929c7ce08cfc123c27300e544beca03066bd5cd1 Mon Sep 17 00:00:00 2001 From: ystaticy Date: Wed, 29 Apr 2026 09:54:59 +0800 Subject: [PATCH 01/21] support default RU group Signed-off-by: ystaticy --- pkg/domain/BUILD.bazel | 6 +- pkg/domain/runaway.go | 40 ++++++++++- pkg/domain/runaway_test.go | 132 +++++++++++++++++++++++++++++++++++++ 3 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 pkg/domain/runaway_test.go diff --git a/pkg/domain/BUILD.bazel b/pkg/domain/BUILD.bazel index 3971bd44c7d60..f86bc3d6bd21a 100644 --- a/pkg/domain/BUILD.bazel +++ b/pkg/domain/BUILD.bazel @@ -116,6 +116,7 @@ go_library( "@com_github_pingcap_failpoint//:failpoint", "@com_github_pingcap_kvproto//pkg/metapb", "@com_github_pingcap_kvproto//pkg/pdpb", + "@com_github_pingcap_kvproto//pkg/resource_manager", "@com_github_pingcap_log//:log", "@com_github_pingcap_metering_sdk//config", "@com_github_stretchr_testify//require", @@ -148,12 +149,13 @@ go_test( "plan_replayer_slow_log_test.go", "plan_replayer_test.go", "ru_stats_test.go", + "runaway_test.go", "schema_checker_test.go", "topn_slow_query_test.go", ], embed = [":domain"], flaky = True, - shard_count = 30, + shard_count = 31, deps = [ "//pkg/config", "//pkg/ddl", @@ -173,6 +175,7 @@ go_test( "//pkg/parser/mysql", "//pkg/parser/terror", "//pkg/planner/extstore", + "//pkg/resourcegroup/runaway", "//pkg/server", "//pkg/session", "//pkg/sessionctx/vardef", @@ -190,6 +193,7 @@ go_test( "@com_github_ngaut_pools//:pools", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", + "@com_github_pingcap_kvproto//pkg/meta_storagepb", "@com_github_pingcap_kvproto//pkg/metapb", "@com_github_pingcap_kvproto//pkg/resource_manager", "@com_github_prometheus_client_model//go", diff --git a/pkg/domain/runaway.go b/pkg/domain/runaway.go index ae1f2d6d233c2..e3f9639fc8460 100644 --- a/pkg/domain/runaway.go +++ b/pkg/domain/runaway.go @@ -17,8 +17,12 @@ package domain import ( "context" "net" + "os" "strconv" + "strings" + "time" + rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/resourcegroup/runaway" "github.com/pingcap/tidb/pkg/util/logutil" @@ -28,6 +32,40 @@ import ( rmclient "github.com/tikv/pd/client/resource_group/controller" ) +const ( + defaultDegradedRUFillRate = 2_000_000 + defaultDegradedRUBurstLimit = 50_000_000_000 + defaultDegradedModeWaitTimeout = 3 * time.Second / 2 + vipWaitRetryInterval = 100 * time.Millisecond + vipWaitRetryTimes = 20 +) + +func newDefaultDegradedRUSettings() *rmpb.GroupRequestUnitSettings { + return &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: defaultDegradedRUFillRate, + BurstLimit: defaultDegradedRUBurstLimit, + }, + }, + } +} + +func newResourceGroupsControllerOptions() []rmclient.ResourceControlCreateOption { + opts := []rmclient.ResourceControlCreateOption{ + rmclient.WithMaxWaitDuration(runaway.MaxWaitDuration), + rmclient.WithDegradedModeWaitDuration(defaultDegradedModeWaitTimeout), + rmclient.WithDegradedRUSettings(newDefaultDegradedRUSettings()), + } + if strings.Contains(os.Getenv("NAMESPACE"), "vip") { + opts = append(opts, + rmclient.WithWaitRetryInterval(vipWaitRetryInterval), + rmclient.WithWaitRetryTimes(vipWaitRetryTimes), + ) + } + return opts +} + func (do *Domain) initResourceGroupsController(ctx context.Context, pdClient pd.Client, uniqueID uint64) error { if pdClient == nil { logutil.BgLogger().Warn("cannot setup up resource controller, not using tikv storage") @@ -39,7 +77,7 @@ func (do *Domain) initResourceGroupsController(ctx context.Context, pdClient pd. if codec := do.Store().GetCodec(); codec != nil { keyspaceID = uint32(codec.GetKeyspaceID()) } - control, err := rmclient.NewResourceGroupController(ctx, uniqueID, pdClient, nil, keyspaceID, rmclient.WithMaxWaitDuration(runaway.MaxWaitDuration)) + control, err := rmclient.NewResourceGroupController(ctx, uniqueID, pdClient, nil, keyspaceID, newResourceGroupsControllerOptions()...) if err != nil { return err } diff --git a/pkg/domain/runaway_test.go b/pkg/domain/runaway_test.go new file mode 100644 index 0000000000000..7f39a3bb350b2 --- /dev/null +++ b/pkg/domain/runaway_test.go @@ -0,0 +1,132 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package domain + +import ( + "context" + "errors" + "testing" + + "github.com/pingcap/kvproto/pkg/meta_storagepb" + rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/pingcap/tidb/pkg/resourcegroup/runaway" + "github.com/stretchr/testify/require" + pd "github.com/tikv/pd/client" + "github.com/tikv/pd/client/opt" + rmclient "github.com/tikv/pd/client/resource_group/controller" +) + +type resourceGroupProviderStub struct { + resourceGroup *rmpb.ResourceGroup + resourceErr error +} + +func (s *resourceGroupProviderStub) GetResourceGroup(context.Context, string, ...pd.GetResourceGroupOption) (*rmpb.ResourceGroup, error) { + return s.resourceGroup, s.resourceErr +} + +func (s *resourceGroupProviderStub) ListResourceGroups(context.Context, ...pd.GetResourceGroupOption) ([]*rmpb.ResourceGroup, error) { + return nil, nil +} + +func (s *resourceGroupProviderStub) AddResourceGroup(context.Context, *rmpb.ResourceGroup) (string, error) { + return "", nil +} + +func (s *resourceGroupProviderStub) ModifyResourceGroup(context.Context, *rmpb.ResourceGroup) (string, error) { + return "", nil +} + +func (s *resourceGroupProviderStub) DeleteResourceGroup(context.Context, string) (string, error) { + return "", nil +} + +func (s *resourceGroupProviderStub) AcquireTokenBuckets(context.Context, *rmpb.TokenBucketsRequest) ([]*rmpb.TokenBucketResponse, error) { + return nil, nil +} + +func (s *resourceGroupProviderStub) LoadResourceGroups(context.Context) ([]*rmpb.ResourceGroup, int64, error) { + return nil, 0, nil +} + +func (s *resourceGroupProviderStub) Watch(context.Context, []byte, ...opt.MetaStorageOption) (chan []*meta_storagepb.Event, error) { + return make(chan []*meta_storagepb.Event), nil +} + +func (s *resourceGroupProviderStub) Get(context.Context, []byte, ...opt.MetaStorageOption) (*meta_storagepb.GetResponse, error) { + return &meta_storagepb.GetResponse{Header: &meta_storagepb.ResponseHeader{}}, nil +} + +func (s *resourceGroupProviderStub) Put(context.Context, []byte, []byte, ...opt.MetaStorageOption) (*meta_storagepb.PutResponse, error) { + return &meta_storagepb.PutResponse{Header: &meta_storagepb.ResponseHeader{}}, nil +} + +func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + provider := &resourceGroupProviderStub{ + resourceGroup: &rmpb.ResourceGroup{ + Name: "test-group", + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{FillRate: 1}, + }, + }, + }, + resourceErr: errors.New("resource group unavailable"), + } + + controllerWithFallback, err := rmclient.NewResourceGroupController( + ctx, + 1, + provider, + nil, + 0, + newResourceGroupsControllerOptions()..., + ) + require.NoError(t, err) + controllerWithFallback.Start(ctx) + defer func() { + require.NoError(t, controllerWithFallback.Stop()) + }() + + group, err := controllerWithFallback.GetResourceGroup("test-group") + require.NoError(t, err) + require.Equal(t, &rmpb.ResourceGroup{ + Name: "test-group", + Mode: rmpb.GroupMode_RUMode, + RUSettings: newDefaultDegradedRUSettings(), + }, group) + + controllerWithoutFallback, err := rmclient.NewResourceGroupController( + ctx, + 2, + provider, + nil, + 0, + rmclient.WithMaxWaitDuration(runaway.MaxWaitDuration), + ) + require.NoError(t, err) + controllerWithoutFallback.Start(ctx) + defer func() { + require.NoError(t, controllerWithoutFallback.Stop()) + }() + + group, err = controllerWithoutFallback.GetResourceGroup("test-group") + require.Error(t, err) + require.Nil(t, group) +} From 40d3deabfa85902a443c0536035d0e5ccd5c698e Mon Sep 17 00:00:00 2001 From: ystaticy Date: Sun, 5 Jul 2026 15:28:31 +0800 Subject: [PATCH 02/21] only in starter Signed-off-by: ystaticy --- pkg/domain/BUILD.bazel | 3 ++- pkg/domain/runaway.go | 13 +++++++++++-- pkg/domain/runaway_test.go | 11 +++++++++-- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/pkg/domain/BUILD.bazel b/pkg/domain/BUILD.bazel index f86bc3d6bd21a..b9c86014f1942 100644 --- a/pkg/domain/BUILD.bazel +++ b/pkg/domain/BUILD.bazel @@ -108,6 +108,7 @@ go_library( "//pkg/util/sqlexec", "//pkg/util/sqlkiller", "//pkg/util/syncutil", + "//pkg/util/versioninfo", "//pkg/workloadlearning", "@com_github_burntsushi_toml//:toml", "@com_github_docker_go_units//:go-units", @@ -175,7 +176,6 @@ go_test( "//pkg/parser/mysql", "//pkg/parser/terror", "//pkg/planner/extstore", - "//pkg/resourcegroup/runaway", "//pkg/server", "//pkg/session", "//pkg/sessionctx/vardef", @@ -190,6 +190,7 @@ go_test( "//pkg/util/mock", "//pkg/util/replayer", "//pkg/util/stmtsummary/v2:stmtsummary", + "//pkg/util/versioninfo", "@com_github_ngaut_pools//:pools", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", diff --git a/pkg/domain/runaway.go b/pkg/domain/runaway.go index e3f9639fc8460..4727b6de6b68e 100644 --- a/pkg/domain/runaway.go +++ b/pkg/domain/runaway.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/resourcegroup/runaway" "github.com/pingcap/tidb/pkg/util/logutil" + "github.com/pingcap/tidb/pkg/util/versioninfo" "github.com/tikv/client-go/v2/tikv" pd "github.com/tikv/pd/client" "github.com/tikv/pd/client/constants" @@ -40,6 +41,10 @@ const ( vipWaitRetryTimes = 20 ) +func isStarter() bool { + return versioninfo.TiDBEdition == "Starter" +} + func newDefaultDegradedRUSettings() *rmpb.GroupRequestUnitSettings { return &rmpb.GroupRequestUnitSettings{ RU: &rmpb.TokenBucket{ @@ -54,8 +59,12 @@ func newDefaultDegradedRUSettings() *rmpb.GroupRequestUnitSettings { func newResourceGroupsControllerOptions() []rmclient.ResourceControlCreateOption { opts := []rmclient.ResourceControlCreateOption{ rmclient.WithMaxWaitDuration(runaway.MaxWaitDuration), - rmclient.WithDegradedModeWaitDuration(defaultDegradedModeWaitTimeout), - rmclient.WithDegradedRUSettings(newDefaultDegradedRUSettings()), + } + if isStarter() { + opts = append(opts, + rmclient.WithDegradedModeWaitDuration(defaultDegradedModeWaitTimeout), + rmclient.WithDegradedRUSettings(newDefaultDegradedRUSettings()), + ) } if strings.Contains(os.Getenv("NAMESPACE"), "vip") { opts = append(opts, diff --git a/pkg/domain/runaway_test.go b/pkg/domain/runaway_test.go index 7f39a3bb350b2..6aef80ead2047 100644 --- a/pkg/domain/runaway_test.go +++ b/pkg/domain/runaway_test.go @@ -21,7 +21,7 @@ import ( "github.com/pingcap/kvproto/pkg/meta_storagepb" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" - "github.com/pingcap/tidb/pkg/resourcegroup/runaway" + "github.com/pingcap/tidb/pkg/util/versioninfo" "github.com/stretchr/testify/require" pd "github.com/tikv/pd/client" "github.com/tikv/pd/client/opt" @@ -74,6 +74,11 @@ func (s *resourceGroupProviderStub) Put(context.Context, []byte, []byte, ...opt. } func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { + originalEdition := versioninfo.TiDBEdition + t.Cleanup(func() { + versioninfo.TiDBEdition = originalEdition + }) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -90,6 +95,7 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { resourceErr: errors.New("resource group unavailable"), } + versioninfo.TiDBEdition = "Starter" controllerWithFallback, err := rmclient.NewResourceGroupController( ctx, 1, @@ -112,13 +118,14 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { RUSettings: newDefaultDegradedRUSettings(), }, group) + versioninfo.TiDBEdition = versioninfo.CommunityEdition controllerWithoutFallback, err := rmclient.NewResourceGroupController( ctx, 2, provider, nil, 0, - rmclient.WithMaxWaitDuration(runaway.MaxWaitDuration), + newResourceGroupsControllerOptions()..., ) require.NoError(t, err) controllerWithoutFallback.Start(ctx) From df69722b5064f6568940a8767f80c4caaa6d9c6c Mon Sep 17 00:00:00 2001 From: ystaticy Date: Sun, 5 Jul 2026 16:48:19 +0800 Subject: [PATCH 03/21] fix name Signed-off-by: ystaticy --- pkg/domain/runaway.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/domain/runaway.go b/pkg/domain/runaway.go index 4727b6de6b68e..8014e51ee86d5 100644 --- a/pkg/domain/runaway.go +++ b/pkg/domain/runaway.go @@ -37,8 +37,8 @@ const ( defaultDegradedRUFillRate = 2_000_000 defaultDegradedRUBurstLimit = 50_000_000_000 defaultDegradedModeWaitTimeout = 3 * time.Second / 2 - vipWaitRetryInterval = 100 * time.Millisecond - vipWaitRetryTimes = 20 + tokenWaitRetryInterval = 100 * time.Millisecond + tokenWaitRetryTimes = 20 ) func isStarter() bool { @@ -68,8 +68,8 @@ func newResourceGroupsControllerOptions() []rmclient.ResourceControlCreateOption } if strings.Contains(os.Getenv("NAMESPACE"), "vip") { opts = append(opts, - rmclient.WithWaitRetryInterval(vipWaitRetryInterval), - rmclient.WithWaitRetryTimes(vipWaitRetryTimes), + rmclient.WithWaitRetryInterval(tokenWaitRetryInterval), + rmclient.WithWaitRetryTimes(tokenWaitRetryTimes), ) } return opts From 6d1ebffc70a594af1e1bdffb50ef3697d94f0348 Mon Sep 17 00:00:00 2001 From: ystaticy Date: Sun, 5 Jul 2026 18:07:06 +0800 Subject: [PATCH 04/21] domain: use deploy mode for Starter RU fallback --- pkg/domain/BUILD.bazel | 4 +-- pkg/domain/runaway.go | 8 ++---- pkg/domain/runaway_test.go | 58 +++++++++++++++++++++++++++++++++----- 3 files changed, 55 insertions(+), 15 deletions(-) diff --git a/pkg/domain/BUILD.bazel b/pkg/domain/BUILD.bazel index 29a808390b8f2..f008e43432cb8 100644 --- a/pkg/domain/BUILD.bazel +++ b/pkg/domain/BUILD.bazel @@ -108,7 +108,6 @@ go_library( "//pkg/util/sqlexec", "//pkg/util/sqlkiller", "//pkg/util/syncutil", - "//pkg/util/versioninfo", "//pkg/workloadlearning", "@com_github_burntsushi_toml//:toml", "@com_github_docker_go_units//:go-units", @@ -160,6 +159,8 @@ go_test( shard_count = 31, deps = [ "//pkg/config", + "//pkg/config/deploymode", + "//pkg/config/kerneltype", "//pkg/ddl", "//pkg/domain/infosync", "//pkg/domain/serverinfo", @@ -191,7 +192,6 @@ go_test( "//pkg/util/mock", "//pkg/util/replayer", "//pkg/util/stmtsummary/v2:stmtsummary", - "//pkg/util/versioninfo", "@com_github_ngaut_pools//:pools", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", diff --git a/pkg/domain/runaway.go b/pkg/domain/runaway.go index 8014e51ee86d5..486c38ac7c53d 100644 --- a/pkg/domain/runaway.go +++ b/pkg/domain/runaway.go @@ -23,10 +23,10 @@ import ( "time" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/pingcap/tidb/pkg/config/deploymode" "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/resourcegroup/runaway" "github.com/pingcap/tidb/pkg/util/logutil" - "github.com/pingcap/tidb/pkg/util/versioninfo" "github.com/tikv/client-go/v2/tikv" pd "github.com/tikv/pd/client" "github.com/tikv/pd/client/constants" @@ -41,10 +41,6 @@ const ( tokenWaitRetryTimes = 20 ) -func isStarter() bool { - return versioninfo.TiDBEdition == "Starter" -} - func newDefaultDegradedRUSettings() *rmpb.GroupRequestUnitSettings { return &rmpb.GroupRequestUnitSettings{ RU: &rmpb.TokenBucket{ @@ -60,7 +56,7 @@ func newResourceGroupsControllerOptions() []rmclient.ResourceControlCreateOption opts := []rmclient.ResourceControlCreateOption{ rmclient.WithMaxWaitDuration(runaway.MaxWaitDuration), } - if isStarter() { + if deploymode.IsStarter() { opts = append(opts, rmclient.WithDegradedModeWaitDuration(defaultDegradedModeWaitTimeout), rmclient.WithDegradedRUSettings(newDefaultDegradedRUSettings()), diff --git a/pkg/domain/runaway_test.go b/pkg/domain/runaway_test.go index 6aef80ead2047..cebd66530a5cf 100644 --- a/pkg/domain/runaway_test.go +++ b/pkg/domain/runaway_test.go @@ -21,7 +21,8 @@ import ( "github.com/pingcap/kvproto/pkg/meta_storagepb" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" - "github.com/pingcap/tidb/pkg/util/versioninfo" + "github.com/pingcap/tidb/pkg/config/deploymode" + "github.com/pingcap/tidb/pkg/config/kerneltype" "github.com/stretchr/testify/require" pd "github.com/tikv/pd/client" "github.com/tikv/pd/client/opt" @@ -33,10 +34,15 @@ type resourceGroupProviderStub struct { resourceErr error } +// GetResourceGroup returns both the mocked resource group and the mocked error. +// This lets the test verify whether the controller uses the degraded fallback +// only for the editions that enable it. func (s *resourceGroupProviderStub) GetResourceGroup(context.Context, string, ...pd.GetResourceGroupOption) (*rmpb.ResourceGroup, error) { return s.resourceGroup, s.resourceErr } +// The remaining methods only satisfy the controller's provider dependencies. +// The fallback test does not exercise their behavior. func (s *resourceGroupProviderStub) ListResourceGroups(context.Context, ...pd.GetResourceGroupOption) ([]*rmpb.ResourceGroup, error) { return nil, nil } @@ -74,14 +80,21 @@ func (s *resourceGroupProviderStub) Put(context.Context, []byte, []byte, ...opt. } func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { - originalEdition := versioninfo.TiDBEdition - t.Cleanup(func() { - versioninfo.TiDBEdition = originalEdition - }) + if kerneltype.IsNextGen() { + // Preserve the process-wide deploy mode because deploymode.IsStarter reads + // it directly when newResourceGroupsControllerOptions builds controller options. + originalDeployMode := deploymode.Get() + t.Cleanup(func() { + require.NoError(t, deploymode.Set(originalDeployMode)) + }) + } + // Use one cancelable context for the short-lived controllers created below. ctx, cancel := context.WithCancel(context.Background()) defer cancel() + // Return a normal resource group together with an error. The error path is + // what should trigger the degraded RU fallback in Starter mode. provider := &resourceGroupProviderStub{ resourceGroup: &rmpb.ResourceGroup{ Name: "test-group", @@ -95,7 +108,32 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { resourceErr: errors.New("resource group unavailable"), } - versioninfo.TiDBEdition = "Starter" + if !kerneltype.IsNextGen() { + // In classic builds deploymode.IsStarter is always false. This still guards + // against accidentally enabling degraded fallback outside NextGen Starter. + controller, err := rmclient.NewResourceGroupController( + ctx, + 1, + provider, + nil, + 0, + newResourceGroupsControllerOptions()..., + ) + require.NoError(t, err) + controller.Start(ctx) + defer func() { + require.NoError(t, controller.Stop()) + }() + + group, err := controller.GetResourceGroup("test-group") + require.Error(t, err) + require.Nil(t, group) + return + } + + // Step 1: build a controller in Starter mode so default options include + // degraded RU settings. + require.NoError(t, deploymode.Set(deploymode.Starter)) controllerWithFallback, err := rmclient.NewResourceGroupController( ctx, 1, @@ -110,6 +148,8 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { require.NoError(t, controllerWithFallback.Stop()) }() + // Step 2: when loading the group fails, Starter should synthesize a degraded + // fallback group instead of returning the provider error. group, err := controllerWithFallback.GetResourceGroup("test-group") require.NoError(t, err) require.Equal(t, &rmpb.ResourceGroup{ @@ -118,7 +158,9 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { RUSettings: newDefaultDegradedRUSettings(), }, group) - versioninfo.TiDBEdition = versioninfo.CommunityEdition + // Step 3: rebuild the controller as a non-Starter edition with the same + // provider, so the only behavior change is the edition-gated option. + require.NoError(t, deploymode.Set(deploymode.Premium)) controllerWithoutFallback, err := rmclient.NewResourceGroupController( ctx, 2, @@ -133,6 +175,8 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { require.NoError(t, controllerWithoutFallback.Stop()) }() + // Step 4: non-Starter editions must not install degraded RU fallback, so the + // provider error is returned and no group is synthesized. group, err = controllerWithoutFallback.GetResourceGroup("test-group") require.Error(t, err) require.Nil(t, group) From 25cacf46f06ae5999a3bf6c6db43b997f8997c5a Mon Sep 17 00:00:00 2001 From: ystaticy Date: Fri, 10 Jul 2026 17:27:11 +0800 Subject: [PATCH 05/21] fix error Signed-off-by: ystaticy --- pkg/domain/BUILD.bazel | 8 +- pkg/domain/resource_group_provider.go | 117 ++++++++++++++++ pkg/domain/resource_group_provider_test.go | 92 +++++++++++++ pkg/domain/runaway.go | 7 +- pkg/domain/runaway_test.go | 147 ++++++++++++++++----- 5 files changed, 333 insertions(+), 38 deletions(-) create mode 100644 pkg/domain/resource_group_provider.go create mode 100644 pkg/domain/resource_group_provider_test.go diff --git a/pkg/domain/BUILD.bazel b/pkg/domain/BUILD.bazel index f008e43432cb8..d625af6d5df72 100644 --- a/pkg/domain/BUILD.bazel +++ b/pkg/domain/BUILD.bazel @@ -12,6 +12,7 @@ go_library( "plan_replayer.go", "plan_replayer_dump.go", "ru_stats.go", + "resource_group_provider.go", "runaway.go", "schema_checker.go", "sysvar_cache.go", @@ -123,6 +124,7 @@ go_library( "@com_github_tikv_client_go_v2//tikv", "@com_github_tikv_client_go_v2//txnkv/transaction", "@com_github_tikv_pd_client//:client", + "@com_github_tikv_pd_client//errs", "@com_github_tikv_pd_client//constants", "@com_github_tikv_pd_client//http", "@com_github_tikv_pd_client//opt", @@ -150,13 +152,14 @@ go_test( "plan_replayer_slow_log_test.go", "plan_replayer_test.go", "ru_stats_test.go", + "resource_group_provider_test.go", "runaway_test.go", "schema_checker_test.go", "topn_slow_query_test.go", ], embed = [":domain"], flaky = True, - shard_count = 31, + shard_count = 34, deps = [ "//pkg/config", "//pkg/config/deploymode", @@ -178,6 +181,7 @@ go_test( "//pkg/parser/mysql", "//pkg/parser/terror", "//pkg/planner/extstore", + "//pkg/resourcegroup/runaway", "//pkg/server", "//pkg/session", "//pkg/sessionctx/vardef", @@ -200,8 +204,10 @@ go_test( "@com_github_pingcap_kvproto//pkg/resource_manager", "@com_github_prometheus_client_model//go", "@com_github_stretchr_testify//require", + "@com_github_tikv_client_go_v2//tikvrpc", "@com_github_tikv_client_go_v2//txnkv/transaction", "@com_github_tikv_pd_client//:client", + "@com_github_tikv_pd_client//errs", "@com_github_tikv_pd_client//opt", "@com_github_tikv_pd_client//resource_group/controller", "@io_etcd_go_etcd_tests_v3//integration", diff --git a/pkg/domain/resource_group_provider.go b/pkg/domain/resource_group_provider.go new file mode 100644 index 0000000000000..1b9135c471ff3 --- /dev/null +++ b/pkg/domain/resource_group_provider.go @@ -0,0 +1,117 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package domain + +import ( + "context" + stderrors "errors" + "strings" + + rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/pingcap/errors" + pd "github.com/tikv/pd/client" + pderr "github.com/tikv/pd/client/errs" + rmclient "github.com/tikv/pd/client/resource_group/controller" +) + +// starterResourceGroupProvider wraps a resource group provider and synthesizes +// degraded resource groups only when resource-manager lookup fails due to +// unavailability. +type starterResourceGroupProvider struct { + rmclient.ResourceGroupProvider + degradedRUSettings *rmpb.GroupRequestUnitSettings +} + +func newStarterResourceGroupProvider(provider rmclient.ResourceGroupProvider) *starterResourceGroupProvider { + return &starterResourceGroupProvider{ + ResourceGroupProvider: provider, + degradedRUSettings: newDefaultDegradedRUSettings(), + } +} + +func (p *starterResourceGroupProvider) GetResourceGroup( + ctx context.Context, resourceGroupName string, opts ...pd.GetResourceGroupOption, +) (*rmpb.ResourceGroup, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + group, err := p.ResourceGroupProvider.GetResourceGroup(ctx, resourceGroupName, opts...) + if err == nil { + return group, nil + } + if !shouldUseDegradedResourceGroupFallback(ctx, err) { + return nil, err + } + return newDegradedResourceGroup(resourceGroupName, p.degradedRUSettings), nil +} + +func newDegradedResourceGroup(name string, ruSettings *rmpb.GroupRequestUnitSettings) *rmpb.ResourceGroup { + return &rmpb.ResourceGroup{ + Name: name, + Mode: rmpb.GroupMode_RUMode, + RUSettings: ruSettings, + } +} + +func shouldUseDegradedResourceGroupFallback(ctx context.Context, err error) bool { + if err == nil { + return false + } + if ctx.Err() != nil { + return false + } + if stderrors.Is(err, context.Canceled) || stderrors.Is(err, context.DeadlineExceeded) { + return false + } + if isResourceGroupNotExistError(err) { + return false + } + if errors.Is(err, pderr.ErrClientResourceGroupConfigUnavailable) { + return true + } + var getRGErr *pderr.ErrClientGetResourceGroup + if errors.As(err, &getRGErr) { + return isResourceManagerUnavailableCause(getRGErr.Cause) + } + return isResourceManagerUnavailableCause(err.Error()) +} + +func isResourceGroupNotExistError(err error) bool { + var getRGErr *pderr.ErrClientGetResourceGroup + if !errors.As(err, &getRGErr) { + return false + } + return isResourceGroupNotExistCause(getRGErr.Cause) +} + +func isResourceGroupNotExistCause(cause string) bool { + lower := strings.ToLower(cause) + return strings.Contains(lower, "does not exist") || + strings.Contains(lower, "not found") || + strings.Contains(lower, "not exist") +} + +func isResourceManagerUnavailableCause(cause string) bool { + if isResourceGroupNotExistCause(cause) { + return false + } + lower := strings.ToLower(cause) + return strings.Contains(lower, "unavailable") || + strings.Contains(lower, "connection refused") || + strings.Contains(lower, "connection reset") || + strings.Contains(lower, "deadline exceeded") || + strings.Contains(lower, "transport is closing") || + strings.Contains(lower, "rpc error") +} diff --git a/pkg/domain/resource_group_provider_test.go b/pkg/domain/resource_group_provider_test.go new file mode 100644 index 0000000000000..1f75de1da8ba7 --- /dev/null +++ b/pkg/domain/resource_group_provider_test.go @@ -0,0 +1,92 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package domain + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + pderr "github.com/tikv/pd/client/errs" +) + +func TestShouldUseDegradedResourceGroupFallback(t *testing.T) { + t.Parallel() + + ctx := context.Background() + canceledCtx, cancel := context.WithCancel(context.Background()) + cancel() + + tests := []struct { + name string + ctx context.Context + err error + want bool + }{ + { + name: "resource manager unavailable", + ctx: ctx, + err: &pderr.ErrClientGetResourceGroup{ + ResourceGroupName: "rg1", + Cause: "resource group unavailable", + }, + want: true, + }, + { + name: "resource group not found", + ctx: ctx, + err: &pderr.ErrClientGetResourceGroup{ + ResourceGroupName: "rg1", + Cause: "resource group does not exist", + }, + want: false, + }, + { + name: "context canceled", + ctx: canceledCtx, + err: &pderr.ErrClientGetResourceGroup{ + ResourceGroupName: "rg1", + Cause: "resource group unavailable", + }, + want: false, + }, + { + name: "lookup canceled", + ctx: ctx, + err: context.Canceled, + want: false, + }, + { + name: "config unavailable", + ctx: ctx, + err: pderr.ErrClientResourceGroupConfigUnavailable.FastGenByArgs("not configured"), + want: true, + }, + { + name: "generic non retryable", + ctx: ctx, + err: errors.New("invalid resource group lookup"), + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + require.Equal(t, tt.want, shouldUseDegradedResourceGroupFallback(tt.ctx, tt.err)) + }) + } +} diff --git a/pkg/domain/runaway.go b/pkg/domain/runaway.go index 486c38ac7c53d..fbbb011c864e7 100644 --- a/pkg/domain/runaway.go +++ b/pkg/domain/runaway.go @@ -59,7 +59,6 @@ func newResourceGroupsControllerOptions() []rmclient.ResourceControlCreateOption if deploymode.IsStarter() { opts = append(opts, rmclient.WithDegradedModeWaitDuration(defaultDegradedModeWaitTimeout), - rmclient.WithDegradedRUSettings(newDefaultDegradedRUSettings()), ) } if strings.Contains(os.Getenv("NAMESPACE"), "vip") { @@ -82,7 +81,11 @@ func (do *Domain) initResourceGroupsController(ctx context.Context, pdClient pd. if codec := do.Store().GetCodec(); codec != nil { keyspaceID = uint32(codec.GetKeyspaceID()) } - control, err := rmclient.NewResourceGroupController(ctx, uniqueID, pdClient, nil, keyspaceID, newResourceGroupsControllerOptions()...) + var provider rmclient.ResourceGroupProvider = pdClient + if deploymode.IsStarter() { + provider = newStarterResourceGroupProvider(pdClient) + } + control, err := rmclient.NewResourceGroupController(ctx, uniqueID, provider, nil, keyspaceID, newResourceGroupsControllerOptions()...) if err != nil { return err } diff --git a/pkg/domain/runaway_test.go b/pkg/domain/runaway_test.go index cebd66530a5cf..19ad5b2f557c0 100644 --- a/pkg/domain/runaway_test.go +++ b/pkg/domain/runaway_test.go @@ -18,13 +18,17 @@ import ( "context" "errors" "testing" + "time" "github.com/pingcap/kvproto/pkg/meta_storagepb" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/tidb/pkg/config/deploymode" "github.com/pingcap/tidb/pkg/config/kerneltype" + "github.com/pingcap/tidb/pkg/resourcegroup/runaway" "github.com/stretchr/testify/require" + "github.com/tikv/client-go/v2/tikvrpc" pd "github.com/tikv/pd/client" + pderr "github.com/tikv/pd/client/errs" "github.com/tikv/pd/client/opt" rmclient "github.com/tikv/pd/client/resource_group/controller" ) @@ -79,6 +83,28 @@ func (s *resourceGroupProviderStub) Put(context.Context, []byte, []byte, ...opt. return &meta_storagepb.PutResponse{Header: &meta_storagepb.ResponseHeader{}}, nil } +func newStarterControllerForTest(t *testing.T, provider rmclient.ResourceGroupProvider) *rmclient.ResourceGroupsController { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + require.NoError(t, deploymode.Set(deploymode.Starter)) + controller, err := rmclient.NewResourceGroupController( + ctx, + 1, + newStarterResourceGroupProvider(provider), + nil, + 0, + newResourceGroupsControllerOptions()..., + ) + require.NoError(t, err) + controller.Start(ctx) + t.Cleanup(func() { + require.NoError(t, controller.Stop()) + }) + return controller +} + func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { if kerneltype.IsNextGen() { // Preserve the process-wide deploy mode because deploymode.IsStarter reads @@ -89,12 +115,9 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { }) } - // Use one cancelable context for the short-lived controllers created below. ctx, cancel := context.WithCancel(context.Background()) defer cancel() - // Return a normal resource group together with an error. The error path is - // what should trigger the degraded RU fallback in Starter mode. provider := &resourceGroupProviderStub{ resourceGroup: &rmpb.ResourceGroup{ Name: "test-group", @@ -105,12 +128,13 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { }, }, }, - resourceErr: errors.New("resource group unavailable"), + resourceErr: &pderr.ErrClientGetResourceGroup{ + ResourceGroupName: "test-group", + Cause: "resource group unavailable", + }, } if !kerneltype.IsNextGen() { - // In classic builds deploymode.IsStarter is always false. This still guards - // against accidentally enabling degraded fallback outside NextGen Starter. controller, err := rmclient.NewResourceGroupController( ctx, 1, @@ -131,35 +155,11 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { return } - // Step 1: build a controller in Starter mode so default options include - // degraded RU settings. - require.NoError(t, deploymode.Set(deploymode.Starter)) - controllerWithFallback, err := rmclient.NewResourceGroupController( - ctx, - 1, - provider, - nil, - 0, - newResourceGroupsControllerOptions()..., - ) - require.NoError(t, err) - controllerWithFallback.Start(ctx) - defer func() { - require.NoError(t, controllerWithFallback.Stop()) - }() - - // Step 2: when loading the group fails, Starter should synthesize a degraded - // fallback group instead of returning the provider error. + controllerWithFallback := newStarterControllerForTest(t, provider) group, err := controllerWithFallback.GetResourceGroup("test-group") require.NoError(t, err) - require.Equal(t, &rmpb.ResourceGroup{ - Name: "test-group", - Mode: rmpb.GroupMode_RUMode, - RUSettings: newDefaultDegradedRUSettings(), - }, group) - - // Step 3: rebuild the controller as a non-Starter edition with the same - // provider, so the only behavior change is the edition-gated option. + require.Equal(t, newDegradedResourceGroup("test-group", newDefaultDegradedRUSettings()), group) + require.NoError(t, deploymode.Set(deploymode.Premium)) controllerWithoutFallback, err := rmclient.NewResourceGroupController( ctx, @@ -175,9 +175,86 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { require.NoError(t, controllerWithoutFallback.Stop()) }() - // Step 4: non-Starter editions must not install degraded RU fallback, so the - // provider error is returned and no group is synthesized. group, err = controllerWithoutFallback.GetResourceGroup("test-group") require.Error(t, err) require.Nil(t, group) } + +func TestStarterResourceGroupProviderNonRetryableErrors(t *testing.T) { + if !kerneltype.IsNextGen() { + t.Skip("Starter deploy mode is only available in NextGen builds") + } + originalDeployMode := deploymode.Get() + t.Cleanup(func() { + require.NoError(t, deploymode.Set(originalDeployMode)) + }) + + t.Run("NotFound", func(t *testing.T) { + provider := &resourceGroupProviderStub{ + resourceErr: rmclient.NewResourceGroupNotExistErr("missing-group"), + } + controller := newStarterControllerForTest(t, provider) + + group, err := controller.GetResourceGroup("missing-group") + require.Error(t, err) + require.Nil(t, group) + }) + + t.Run("ContextCanceled", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + provider := &resourceGroupProviderStub{ + resourceErr: context.Canceled, + } + wrapped := newStarterResourceGroupProvider(provider) + group, err := wrapped.GetResourceGroup(ctx, "test-group") + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, group) + }) + + t.Run("GenericNonRetryableError", func(t *testing.T) { + provider := &resourceGroupProviderStub{ + resourceErr: errors.New("invalid resource group lookup"), + } + controller := newStarterControllerForTest(t, provider) + + group, err := controller.GetResourceGroup("test-group") + require.Error(t, err) + require.Nil(t, group) + }) +} + +func TestStarterSwitchGroupRejectsMissingGroup(t *testing.T) { + if !kerneltype.IsNextGen() { + t.Skip("Starter deploy mode is only available in NextGen builds") + } + originalDeployMode := deploymode.Get() + t.Cleanup(func() { + require.NoError(t, deploymode.Set(originalDeployMode)) + }) + + provider := &resourceGroupProviderStub{ + resourceErr: rmclient.NewResourceGroupNotExistErr("missing-switch-group"), + } + controller := newStarterControllerForTest(t, provider) + manager := runaway.NewRunawayManager(controller, "127.0.0.1:4000", nil, make(chan struct{}), nil, nil) + checker := runaway.NewChecker( + manager, + "source-group", + &rmpb.RunawaySettings{ + Action: rmpb.RunawayAction_SwitchGroup, + SwitchGroupName: "missing-switch-group", + Rule: &rmpb.RunawayRule{ProcessedKeys: 1}, + }, + "SELECT 1", + "sql_digest", + "plan_digest", + time.Now(), + ) + + require.NoError(t, checker.CheckThresholds(nil, 10, nil)) + req := &tikvrpc.Request{} + require.NoError(t, checker.BeforeCopRequest(req)) + require.Empty(t, req.ResourceControlContext.ResourceGroupName) +} From 29ff847779ecb0e8c4209cf58f7b26502326ec93 Mon Sep 17 00:00:00 2001 From: ystaticy Date: Mon, 13 Jul 2026 15:49:19 +0800 Subject: [PATCH 06/21] fix comments Signed-off-by: ystaticy --- cmd/tidb-server/main.go | 5 +++++ cmd/tidb-server/main_test.go | 2 ++ pkg/config/config.go | 3 +++ pkg/domain/BUILD.bazel | 8 ++++---- pkg/domain/resource_group_provider.go | 9 ++++----- pkg/domain/runaway.go | 4 ++-- pkg/domain/runaway_test.go | 21 +++++++++++++++++++++ 7 files changed, 41 insertions(+), 11 deletions(-) diff --git a/cmd/tidb-server/main.go b/cmd/tidb-server/main.go index e117e3e138629..516971d745e07 100644 --- a/cmd/tidb-server/main.go +++ b/cmd/tidb-server/main.go @@ -710,6 +710,11 @@ func overrideConfig(cfg *config.Config, fset *flag.FlagSet) { fset.Visit(func(f *flag.Flag) { actualFlags[f.Name] = true }) + if actualFlags[nmStarterParams] && cfg.DeployMode == deploymode.Starter { + params, err := parseStarterAdditionalParams(getStarterAdditionalParams()) + terror.MustNil(err) + cfg.StarterParams.PodNamespace = params.podNamespace + } // Base if actualFlags[nmHost] { diff --git a/cmd/tidb-server/main_test.go b/cmd/tidb-server/main_test.go index 5808ccc14bdf9..9650261a94e54 100644 --- a/cmd/tidb-server/main_test.go +++ b/cmd/tidb-server/main_test.go @@ -89,8 +89,10 @@ func TestOverrideConfigKeyspaceActivateMode(t *testing.T) { })) cfg := config.NewConfig() + cfg.DeployMode = deploymode.Starter overrideConfig(cfg, fset) require.True(t, cfg.KeyspaceActivateMode) + require.Equal(t, "ns-1", cfg.StarterParams.PodNamespace) require.Equal(t, "pod-name=pod-1,pod-ip=10.0.0.1,pod-namespace=ns-1", *starterAdditionalParams) } diff --git a/pkg/config/config.go b/pkg/config/config.go index a7cbf232abf5f..32a61be17b6f0 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1094,6 +1094,9 @@ type StarterParams struct { // ManagerAddr is the TiDB manager address used by the shutdown notifier. // When empty and EnableManagerNotifier is true, the Starter path derives the service address from starter additional params. ManagerAddr string `toml:"manager-addr" json:"manager-addr,omitempty"` + // PodNamespace captures the runtime pod namespace from --starter-additional-params. + // It is command-line derived runtime metadata rather than file-backed config. + PodNamespace string `toml:"-" json:"-"` // MaxImportDataSize is the maximum total real source data size allowed for IMPORT INTO. // Zero means unlimited. MaxImportDataSize configtypes.ByteSize `toml:"max-import-data-size" json:"max-import-data-size,omitempty"` diff --git a/pkg/domain/BUILD.bazel b/pkg/domain/BUILD.bazel index d625af6d5df72..7cf5bf7b3f08e 100644 --- a/pkg/domain/BUILD.bazel +++ b/pkg/domain/BUILD.bazel @@ -11,8 +11,8 @@ go_library( "optimize_trace.go", "plan_replayer.go", "plan_replayer_dump.go", - "ru_stats.go", "resource_group_provider.go", + "ru_stats.go", "runaway.go", "schema_checker.go", "sysvar_cache.go", @@ -124,8 +124,8 @@ go_library( "@com_github_tikv_client_go_v2//tikv", "@com_github_tikv_client_go_v2//txnkv/transaction", "@com_github_tikv_pd_client//:client", - "@com_github_tikv_pd_client//errs", "@com_github_tikv_pd_client//constants", + "@com_github_tikv_pd_client//errs", "@com_github_tikv_pd_client//http", "@com_github_tikv_pd_client//opt", "@com_github_tikv_pd_client//pkg/caller", @@ -151,15 +151,15 @@ go_test( "plan_replayer_handle_test.go", "plan_replayer_slow_log_test.go", "plan_replayer_test.go", - "ru_stats_test.go", "resource_group_provider_test.go", + "ru_stats_test.go", "runaway_test.go", "schema_checker_test.go", "topn_slow_query_test.go", ], embed = [":domain"], flaky = True, - shard_count = 34, + shard_count = 35, deps = [ "//pkg/config", "//pkg/config/deploymode", diff --git a/pkg/domain/resource_group_provider.go b/pkg/domain/resource_group_provider.go index 1b9135c471ff3..91d5c1570acd8 100644 --- a/pkg/domain/resource_group_provider.go +++ b/pkg/domain/resource_group_provider.go @@ -20,7 +20,6 @@ import ( "strings" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" - "github.com/pingcap/errors" pd "github.com/tikv/pd/client" pderr "github.com/tikv/pd/client/errs" rmclient "github.com/tikv/pd/client/resource_group/controller" @@ -37,7 +36,7 @@ type starterResourceGroupProvider struct { func newStarterResourceGroupProvider(provider rmclient.ResourceGroupProvider) *starterResourceGroupProvider { return &starterResourceGroupProvider{ ResourceGroupProvider: provider, - degradedRUSettings: newDefaultDegradedRUSettings(), + degradedRUSettings: newDefaultDegradedRUSettings(), } } @@ -78,11 +77,11 @@ func shouldUseDegradedResourceGroupFallback(ctx context.Context, err error) bool if isResourceGroupNotExistError(err) { return false } - if errors.Is(err, pderr.ErrClientResourceGroupConfigUnavailable) { + if stderrors.Is(err, pderr.ErrClientResourceGroupConfigUnavailable) { return true } var getRGErr *pderr.ErrClientGetResourceGroup - if errors.As(err, &getRGErr) { + if stderrors.As(err, &getRGErr) { return isResourceManagerUnavailableCause(getRGErr.Cause) } return isResourceManagerUnavailableCause(err.Error()) @@ -90,7 +89,7 @@ func shouldUseDegradedResourceGroupFallback(ctx context.Context, err error) bool func isResourceGroupNotExistError(err error) bool { var getRGErr *pderr.ErrClientGetResourceGroup - if !errors.As(err, &getRGErr) { + if !stderrors.As(err, &getRGErr) { return false } return isResourceGroupNotExistCause(getRGErr.Cause) diff --git a/pkg/domain/runaway.go b/pkg/domain/runaway.go index fbbb011c864e7..4f6cc7710a39a 100644 --- a/pkg/domain/runaway.go +++ b/pkg/domain/runaway.go @@ -17,12 +17,12 @@ package domain import ( "context" "net" - "os" "strconv" "strings" "time" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/config/deploymode" "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/resourcegroup/runaway" @@ -61,7 +61,7 @@ func newResourceGroupsControllerOptions() []rmclient.ResourceControlCreateOption rmclient.WithDegradedModeWaitDuration(defaultDegradedModeWaitTimeout), ) } - if strings.Contains(os.Getenv("NAMESPACE"), "vip") { + if strings.Contains(config.GetGlobalConfig().StarterParams.PodNamespace, "vip") { opts = append(opts, rmclient.WithWaitRetryInterval(tokenWaitRetryInterval), rmclient.WithWaitRetryTimes(tokenWaitRetryTimes), diff --git a/pkg/domain/runaway_test.go b/pkg/domain/runaway_test.go index 19ad5b2f557c0..365168233af50 100644 --- a/pkg/domain/runaway_test.go +++ b/pkg/domain/runaway_test.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/kvproto/pkg/meta_storagepb" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" + "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/config/deploymode" "github.com/pingcap/tidb/pkg/config/kerneltype" "github.com/pingcap/tidb/pkg/resourcegroup/runaway" @@ -258,3 +259,23 @@ func TestStarterSwitchGroupRejectsMissingGroup(t *testing.T) { require.NoError(t, checker.BeforeCopRequest(req)) require.Empty(t, req.ResourceControlContext.ResourceGroupName) } + +func TestResourceGroupsControllerOptionsUseStarterPodNamespaceForVIPWaitRetry(t *testing.T) { + restoreConfig := config.RestoreFunc() + t.Cleanup(restoreConfig) + originalDeployMode := deploymode.Get() + t.Cleanup(func() { + require.NoError(t, deploymode.Set(originalDeployMode)) + }) + require.NoError(t, deploymode.Set(deploymode.Starter)) + + config.UpdateGlobal(func(conf *config.Config) { + conf.StarterParams.PodNamespace = "starter-vip-ns" + }) + require.Len(t, newResourceGroupsControllerOptions(), 4) + + config.UpdateGlobal(func(conf *config.Config) { + conf.StarterParams.PodNamespace = "starter-standard-ns" + }) + require.Len(t, newResourceGroupsControllerOptions(), 2) +} From 8054a7f59251471ddf120722940c905a599a9c2f Mon Sep 17 00:00:00 2001 From: ystaticy Date: Mon, 13 Jul 2026 17:30:22 +0800 Subject: [PATCH 07/21] fix comments Signed-off-by: ystaticy --- pkg/domain/resource_group_provider_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/domain/resource_group_provider_test.go b/pkg/domain/resource_group_provider_test.go index 1f75de1da8ba7..d51dcdd73bc96 100644 --- a/pkg/domain/resource_group_provider_test.go +++ b/pkg/domain/resource_group_provider_test.go @@ -24,8 +24,6 @@ import ( ) func TestShouldUseDegradedResourceGroupFallback(t *testing.T) { - t.Parallel() - ctx := context.Background() canceledCtx, cancel := context.WithCancel(context.Background()) cancel() @@ -85,7 +83,6 @@ func TestShouldUseDegradedResourceGroupFallback(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - t.Parallel() require.Equal(t, tt.want, shouldUseDegradedResourceGroupFallback(tt.ctx, tt.err)) }) } From 0c6d89dea1e2bc402bb0ceeea690360d2f37d0d6 Mon Sep 17 00:00:00 2001 From: ystaticy Date: Tue, 14 Jul 2026 10:20:55 +0800 Subject: [PATCH 08/21] fix build Signed-off-by: ystaticy --- pkg/domain/BUILD.bazel | 1 + pkg/domain/runaway_test.go | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/domain/BUILD.bazel b/pkg/domain/BUILD.bazel index 7cf5bf7b3f08e..1c1be2dd6b8d0 100644 --- a/pkg/domain/BUILD.bazel +++ b/pkg/domain/BUILD.bazel @@ -207,6 +207,7 @@ go_test( "@com_github_tikv_client_go_v2//tikvrpc", "@com_github_tikv_client_go_v2//txnkv/transaction", "@com_github_tikv_pd_client//:client", + "@com_github_tikv_pd_client//clients/metastorage", "@com_github_tikv_pd_client//errs", "@com_github_tikv_pd_client//opt", "@com_github_tikv_pd_client//resource_group/controller", diff --git a/pkg/domain/runaway_test.go b/pkg/domain/runaway_test.go index 365168233af50..f63cae4234fe1 100644 --- a/pkg/domain/runaway_test.go +++ b/pkg/domain/runaway_test.go @@ -29,6 +29,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/tikvrpc" pd "github.com/tikv/pd/client" + "github.com/tikv/pd/client/clients/metastorage" pderr "github.com/tikv/pd/client/errs" "github.com/tikv/pd/client/opt" rmclient "github.com/tikv/pd/client/resource_group/controller" @@ -72,8 +73,8 @@ func (s *resourceGroupProviderStub) LoadResourceGroups(context.Context) ([]*rmpb return nil, 0, nil } -func (s *resourceGroupProviderStub) Watch(context.Context, []byte, ...opt.MetaStorageOption) (chan []*meta_storagepb.Event, error) { - return make(chan []*meta_storagepb.Event), nil +func (s *resourceGroupProviderStub) Watch(context.Context, []byte, ...opt.MetaStorageOption) (chan *metastorage.WatchResponse, error) { + return make(chan *metastorage.WatchResponse), nil } func (s *resourceGroupProviderStub) Get(context.Context, []byte, ...opt.MetaStorageOption) (*meta_storagepb.GetResponse, error) { From 73f9e5abec81dd1349243ab6e916035719f58480 Mon Sep 17 00:00:00 2001 From: ystaticy Date: Tue, 14 Jul 2026 11:10:19 +0800 Subject: [PATCH 09/21] fix ut Signed-off-by: ystaticy --- pkg/domain/runaway_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/domain/runaway_test.go b/pkg/domain/runaway_test.go index f63cae4234fe1..1deec47293744 100644 --- a/pkg/domain/runaway_test.go +++ b/pkg/domain/runaway_test.go @@ -40,6 +40,8 @@ type resourceGroupProviderStub struct { resourceErr error } +var _ metastorage.Client = (*resourceGroupProviderStub)(nil) + // GetResourceGroup returns both the mocked resource group and the mocked error. // This lets the test verify whether the controller uses the degraded fallback // only for the editions that enable it. @@ -73,8 +75,8 @@ func (s *resourceGroupProviderStub) LoadResourceGroups(context.Context) ([]*rmpb return nil, 0, nil } -func (s *resourceGroupProviderStub) Watch(context.Context, []byte, ...opt.MetaStorageOption) (chan *metastorage.WatchResponse, error) { - return make(chan *metastorage.WatchResponse), nil +func (s *resourceGroupProviderStub) Watch(context.Context, []byte, ...opt.MetaStorageOption) (chan []*meta_storagepb.Event, error) { + return make(chan []*meta_storagepb.Event), nil } func (s *resourceGroupProviderStub) Get(context.Context, []byte, ...opt.MetaStorageOption) (*meta_storagepb.GetResponse, error) { @@ -241,6 +243,7 @@ func TestStarterSwitchGroupRejectsMissingGroup(t *testing.T) { } controller := newStarterControllerForTest(t, provider) manager := runaway.NewRunawayManager(controller, "127.0.0.1:4000", nil, make(chan struct{}), nil, nil) + t.Cleanup(manager.Stop) checker := runaway.NewChecker( manager, "source-group", @@ -258,7 +261,7 @@ func TestStarterSwitchGroupRejectsMissingGroup(t *testing.T) { require.NoError(t, checker.CheckThresholds(nil, 10, nil)) req := &tikvrpc.Request{} require.NoError(t, checker.BeforeCopRequest(req)) - require.Empty(t, req.ResourceControlContext.ResourceGroupName) + require.Empty(t, req.GetResourceControlContext().GetResourceGroupName()) } func TestResourceGroupsControllerOptionsUseStarterPodNamespaceForVIPWaitRetry(t *testing.T) { From 636fcd9e0eb8494efa336b4a6b237737820f2323 Mon Sep 17 00:00:00 2001 From: ystaticy Date: Tue, 14 Jul 2026 14:01:02 +0800 Subject: [PATCH 10/21] fix build Signed-off-by: ystaticy --- pkg/domain/BUILD.bazel | 2 - .../infosync/resource_manager_client.go | 8 ++ pkg/domain/runaway_test.go | 93 ++++++------------- pkg/executor/BUILD.bazel | 2 - pkg/executor/adapter_internal_test.go | 63 +++---------- 5 files changed, 48 insertions(+), 120 deletions(-) diff --git a/pkg/domain/BUILD.bazel b/pkg/domain/BUILD.bazel index 1c1be2dd6b8d0..dc84cdd7d002e 100644 --- a/pkg/domain/BUILD.bazel +++ b/pkg/domain/BUILD.bazel @@ -199,7 +199,6 @@ go_test( "@com_github_ngaut_pools//:pools", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", - "@com_github_pingcap_kvproto//pkg/meta_storagepb", "@com_github_pingcap_kvproto//pkg/metapb", "@com_github_pingcap_kvproto//pkg/resource_manager", "@com_github_prometheus_client_model//go", @@ -207,7 +206,6 @@ go_test( "@com_github_tikv_client_go_v2//tikvrpc", "@com_github_tikv_client_go_v2//txnkv/transaction", "@com_github_tikv_pd_client//:client", - "@com_github_tikv_pd_client//clients/metastorage", "@com_github_tikv_pd_client//errs", "@com_github_tikv_pd_client//opt", "@com_github_tikv_pd_client//resource_group/controller", diff --git a/pkg/domain/infosync/resource_manager_client.go b/pkg/domain/infosync/resource_manager_client.go index fbad3d822a948..3f44500b57dbd 100644 --- a/pkg/domain/infosync/resource_manager_client.go +++ b/pkg/domain/infosync/resource_manager_client.go @@ -147,6 +147,14 @@ func (*mockResourceManagerClient) LoadResourceGroups(context.Context) ([]*rmpb.R return nil, 0, nil } +func (*mockResourceManagerClient) Get(context.Context, []byte, ...opt.MetaStorageOption) (*meta_storagepb.GetResponse, error) { + return &meta_storagepb.GetResponse{Header: &meta_storagepb.ResponseHeader{}}, nil +} + +func (*mockResourceManagerClient) Put(context.Context, []byte, []byte, ...opt.MetaStorageOption) (*meta_storagepb.PutResponse, error) { + return &meta_storagepb.PutResponse{Header: &meta_storagepb.ResponseHeader{}}, nil +} + func (m *mockResourceManagerClient) Watch(_ context.Context, key []byte, _ ...opt.MetaStorageOption) (chan []*meta_storagepb.Event, error) { if bytes.Equal(pd.GroupSettingsPathPrefixBytes(m.keyspaceID), key) { return m.eventCh, nil diff --git a/pkg/domain/runaway_test.go b/pkg/domain/runaway_test.go index 1deec47293744..3184f44c2ffd3 100644 --- a/pkg/domain/runaway_test.go +++ b/pkg/domain/runaway_test.go @@ -20,27 +20,35 @@ import ( "testing" "time" - "github.com/pingcap/kvproto/pkg/meta_storagepb" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/config/deploymode" "github.com/pingcap/tidb/pkg/config/kerneltype" + "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/resourcegroup/runaway" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/tikvrpc" pd "github.com/tikv/pd/client" - "github.com/tikv/pd/client/clients/metastorage" pderr "github.com/tikv/pd/client/errs" - "github.com/tikv/pd/client/opt" rmclient "github.com/tikv/pd/client/resource_group/controller" ) type resourceGroupProviderStub struct { + rmclient.ResourceGroupProvider resourceGroup *rmpb.ResourceGroup resourceErr error } -var _ metastorage.Client = (*resourceGroupProviderStub)(nil) +func newResourceGroupProviderStub(t *testing.T, resourceGroup *rmpb.ResourceGroup, resourceErr error) *resourceGroupProviderStub { + t.Helper() + baseProvider, ok := infosync.NewMockResourceManagerClient(0).(rmclient.ResourceGroupProvider) + require.True(t, ok) + return &resourceGroupProviderStub{ + ResourceGroupProvider: baseProvider, + resourceGroup: resourceGroup, + resourceErr: resourceErr, + } +} // GetResourceGroup returns both the mocked resource group and the mocked error. // This lets the test verify whether the controller uses the degraded fallback @@ -49,44 +57,6 @@ func (s *resourceGroupProviderStub) GetResourceGroup(context.Context, string, .. return s.resourceGroup, s.resourceErr } -// The remaining methods only satisfy the controller's provider dependencies. -// The fallback test does not exercise their behavior. -func (s *resourceGroupProviderStub) ListResourceGroups(context.Context, ...pd.GetResourceGroupOption) ([]*rmpb.ResourceGroup, error) { - return nil, nil -} - -func (s *resourceGroupProviderStub) AddResourceGroup(context.Context, *rmpb.ResourceGroup) (string, error) { - return "", nil -} - -func (s *resourceGroupProviderStub) ModifyResourceGroup(context.Context, *rmpb.ResourceGroup) (string, error) { - return "", nil -} - -func (s *resourceGroupProviderStub) DeleteResourceGroup(context.Context, string) (string, error) { - return "", nil -} - -func (s *resourceGroupProviderStub) AcquireTokenBuckets(context.Context, *rmpb.TokenBucketsRequest) ([]*rmpb.TokenBucketResponse, error) { - return nil, nil -} - -func (s *resourceGroupProviderStub) LoadResourceGroups(context.Context) ([]*rmpb.ResourceGroup, int64, error) { - return nil, 0, nil -} - -func (s *resourceGroupProviderStub) Watch(context.Context, []byte, ...opt.MetaStorageOption) (chan []*meta_storagepb.Event, error) { - return make(chan []*meta_storagepb.Event), nil -} - -func (s *resourceGroupProviderStub) Get(context.Context, []byte, ...opt.MetaStorageOption) (*meta_storagepb.GetResponse, error) { - return &meta_storagepb.GetResponse{Header: &meta_storagepb.ResponseHeader{}}, nil -} - -func (s *resourceGroupProviderStub) Put(context.Context, []byte, []byte, ...opt.MetaStorageOption) (*meta_storagepb.PutResponse, error) { - return &meta_storagepb.PutResponse{Header: &meta_storagepb.ResponseHeader{}}, nil -} - func newStarterControllerForTest(t *testing.T, provider rmclient.ResourceGroupProvider) *rmclient.ResourceGroupsController { t.Helper() ctx, cancel := context.WithCancel(context.Background()) @@ -122,21 +92,18 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - provider := &resourceGroupProviderStub{ - resourceGroup: &rmpb.ResourceGroup{ - Name: "test-group", - Mode: rmpb.GroupMode_RUMode, - RUSettings: &rmpb.GroupRequestUnitSettings{ - RU: &rmpb.TokenBucket{ - Settings: &rmpb.TokenLimitSettings{FillRate: 1}, - }, + provider := newResourceGroupProviderStub(t, &rmpb.ResourceGroup{ + Name: "test-group", + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{FillRate: 1}, }, }, - resourceErr: &pderr.ErrClientGetResourceGroup{ - ResourceGroupName: "test-group", - Cause: "resource group unavailable", - }, - } + }, &pderr.ErrClientGetResourceGroup{ + ResourceGroupName: "test-group", + Cause: "resource group unavailable", + }) if !kerneltype.IsNextGen() { controller, err := rmclient.NewResourceGroupController( @@ -194,9 +161,7 @@ func TestStarterResourceGroupProviderNonRetryableErrors(t *testing.T) { }) t.Run("NotFound", func(t *testing.T) { - provider := &resourceGroupProviderStub{ - resourceErr: rmclient.NewResourceGroupNotExistErr("missing-group"), - } + provider := newResourceGroupProviderStub(t, nil, rmclient.NewResourceGroupNotExistErr("missing-group")) controller := newStarterControllerForTest(t, provider) group, err := controller.GetResourceGroup("missing-group") @@ -208,9 +173,7 @@ func TestStarterResourceGroupProviderNonRetryableErrors(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - provider := &resourceGroupProviderStub{ - resourceErr: context.Canceled, - } + provider := newResourceGroupProviderStub(t, nil, context.Canceled) wrapped := newStarterResourceGroupProvider(provider) group, err := wrapped.GetResourceGroup(ctx, "test-group") require.ErrorIs(t, err, context.Canceled) @@ -218,9 +181,7 @@ func TestStarterResourceGroupProviderNonRetryableErrors(t *testing.T) { }) t.Run("GenericNonRetryableError", func(t *testing.T) { - provider := &resourceGroupProviderStub{ - resourceErr: errors.New("invalid resource group lookup"), - } + provider := newResourceGroupProviderStub(t, nil, errors.New("invalid resource group lookup")) controller := newStarterControllerForTest(t, provider) group, err := controller.GetResourceGroup("test-group") @@ -238,9 +199,7 @@ func TestStarterSwitchGroupRejectsMissingGroup(t *testing.T) { require.NoError(t, deploymode.Set(originalDeployMode)) }) - provider := &resourceGroupProviderStub{ - resourceErr: rmclient.NewResourceGroupNotExistErr("missing-switch-group"), - } + provider := newResourceGroupProviderStub(t, nil, rmclient.NewResourceGroupNotExistErr("missing-switch-group")) controller := newStarterControllerForTest(t, provider) manager := runaway.NewRunawayManager(controller, "127.0.0.1:4000", nil, make(chan struct{}), nil, nil) t.Cleanup(manager.Stop) diff --git a/pkg/executor/BUILD.bazel b/pkg/executor/BUILD.bazel index 9b26968e75bc8..562f07bfd6b2c 100644 --- a/pkg/executor/BUILD.bazel +++ b/pkg/executor/BUILD.bazel @@ -562,8 +562,6 @@ go_test( "@com_github_tikv_client_go_v2//tikv", "@com_github_tikv_client_go_v2//tikvrpc", "@com_github_tikv_client_go_v2//util", - "@com_github_tikv_pd_client//:client", - "@com_github_tikv_pd_client//clients/metastorage", "@com_github_tikv_pd_client//http", "@com_github_tikv_pd_client//opt", "@com_github_tikv_pd_client//resource_group/controller", diff --git a/pkg/executor/adapter_internal_test.go b/pkg/executor/adapter_internal_test.go index 7aa2237ece5e4..5a5d1263f73ac 100644 --- a/pkg/executor/adapter_internal_test.go +++ b/pkg/executor/adapter_internal_test.go @@ -22,8 +22,8 @@ import ( "time" "github.com/pingcap/kvproto/pkg/meta_storagepb" - rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/tidb/pkg/domain" + "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/expression" "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/parser" @@ -40,8 +40,6 @@ import ( "github.com/pingcap/tidb/pkg/util/topsql/stmtstats" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/util" - pd "github.com/tikv/pd/client" - metastorage "github.com/tikv/pd/client/clients/metastorage" "github.com/tikv/pd/client/opt" rmclient "github.com/tikv/pd/client/resource_group/controller" ) @@ -452,9 +450,20 @@ func TestObserveStmtFinishedOnTopProfilingIgnores(t *testing.T) { } type mockResourceGroupProvider struct { + rmclient.ResourceGroupProvider config *rmclient.Config } +func newMockResourceGroupProvider(t *testing.T, config *rmclient.Config) *mockResourceGroupProvider { + t.Helper() + baseProvider, ok := infosync.NewMockResourceManagerClient(1).(rmclient.ResourceGroupProvider) + require.True(t, ok) + return &mockResourceGroupProvider{ + ResourceGroupProvider: baseProvider, + config: config, + } +} + func newMockDomainWithRUVersion(t *testing.T, version rmclient.RUVersion) *domain.Domain { t.Helper() ctx, cancel := context.WithCancel(context.Background()) @@ -462,7 +471,7 @@ func newMockDomainWithRUVersion(t *testing.T, version rmclient.RUVersion) *domai cfg := rmclient.DefaultConfig() cfg.RUVersionPolicy = &rmclient.RUVersionPolicy{Default: version} - provider := &mockResourceGroupProvider{config: cfg} + provider := newMockResourceGroupProvider(t, cfg) controller, err := rmclient.NewResourceGroupController(ctx, 1, provider, nil, 1) require.NoError(t, err) @@ -481,48 +490,4 @@ func (m *mockResourceGroupProvider) Get(ctx context.Context, key []byte, opts .. }, nil } -func (*mockResourceGroupProvider) Watch(ctx context.Context, key []byte, opts ...opt.MetaStorageOption) (chan []*meta_storagepb.Event, error) { - ch := make(chan []*meta_storagepb.Event) - go func() { - <-ctx.Done() - close(ch) - }() - return ch, nil -} - -func (*mockResourceGroupProvider) Put(context.Context, []byte, []byte, ...opt.MetaStorageOption) (*meta_storagepb.PutResponse, error) { - return &meta_storagepb.PutResponse{}, nil -} - -func (*mockResourceGroupProvider) GetResourceGroup(context.Context, string, ...pd.GetResourceGroupOption) (*rmpb.ResourceGroup, error) { - return nil, nil -} - -func (*mockResourceGroupProvider) ListResourceGroups(context.Context, ...pd.GetResourceGroupOption) ([]*rmpb.ResourceGroup, error) { - return nil, nil -} - -func (*mockResourceGroupProvider) AddResourceGroup(context.Context, *rmpb.ResourceGroup) (string, error) { - return "", nil -} - -func (*mockResourceGroupProvider) ModifyResourceGroup(context.Context, *rmpb.ResourceGroup) (string, error) { - return "", nil -} - -func (*mockResourceGroupProvider) DeleteResourceGroup(context.Context, string) (string, error) { - return "", nil -} - -func (*mockResourceGroupProvider) AcquireTokenBuckets(context.Context, *rmpb.TokenBucketsRequest) ([]*rmpb.TokenBucketResponse, error) { - return nil, nil -} - -func (*mockResourceGroupProvider) LoadResourceGroups(context.Context) ([]*rmpb.ResourceGroup, int64, error) { - return nil, 0, nil -} - -var ( - _ metastorage.Client = (*mockResourceGroupProvider)(nil) - _ rmclient.ResourceGroupProvider = (*mockResourceGroupProvider)(nil) -) +var _ rmclient.ResourceGroupProvider = (*mockResourceGroupProvider)(nil) From a633842009a7061fc41ac579e0eed81264ba2bff Mon Sep 17 00:00:00 2001 From: ystaticy Date: Tue, 14 Jul 2026 15:16:00 +0800 Subject: [PATCH 11/21] fix ut Signed-off-by: ystaticy --- pkg/domain/runaway_test.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pkg/domain/runaway_test.go b/pkg/domain/runaway_test.go index 3184f44c2ffd3..198f448e36f5b 100644 --- a/pkg/domain/runaway_test.go +++ b/pkg/domain/runaway_test.go @@ -226,19 +226,25 @@ func TestStarterSwitchGroupRejectsMissingGroup(t *testing.T) { func TestResourceGroupsControllerOptionsUseStarterPodNamespaceForVIPWaitRetry(t *testing.T) { restoreConfig := config.RestoreFunc() t.Cleanup(restoreConfig) - originalDeployMode := deploymode.Get() - t.Cleanup(func() { - require.NoError(t, deploymode.Set(originalDeployMode)) - }) - require.NoError(t, deploymode.Set(deploymode.Starter)) + vipOptionCount := 3 + standardOptionCount := 1 + if kerneltype.IsNextGen() { + originalDeployMode := deploymode.Get() + t.Cleanup(func() { + require.NoError(t, deploymode.Set(originalDeployMode)) + }) + require.NoError(t, deploymode.Set(deploymode.Starter)) + vipOptionCount = 4 + standardOptionCount = 2 + } config.UpdateGlobal(func(conf *config.Config) { conf.StarterParams.PodNamespace = "starter-vip-ns" }) - require.Len(t, newResourceGroupsControllerOptions(), 4) + require.Len(t, newResourceGroupsControllerOptions(), vipOptionCount) config.UpdateGlobal(func(conf *config.Config) { conf.StarterParams.PodNamespace = "starter-standard-ns" }) - require.Len(t, newResourceGroupsControllerOptions(), 2) + require.Len(t, newResourceGroupsControllerOptions(), standardOptionCount) } From cf2bf7f8aa71284bc9b47ecb12bb53d941d7743f Mon Sep 17 00:00:00 2001 From: ystaticy Date: Tue, 14 Jul 2026 17:12:47 +0800 Subject: [PATCH 12/21] fix code format Signed-off-by: ystaticy --- pkg/domain/BUILD.bazel | 1 + .../resource_group_controller_options.go | 49 +++++++++++++++++++ pkg/domain/resource_group_provider.go | 16 ++++++ pkg/domain/runaway.go | 41 ---------------- 4 files changed, 66 insertions(+), 41 deletions(-) create mode 100644 pkg/domain/resource_group_controller_options.go diff --git a/pkg/domain/BUILD.bazel b/pkg/domain/BUILD.bazel index dc84cdd7d002e..42c0d4fde0065 100644 --- a/pkg/domain/BUILD.bazel +++ b/pkg/domain/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "optimize_trace.go", "plan_replayer.go", "plan_replayer_dump.go", + "resource_group_controller_options.go", "resource_group_provider.go", "ru_stats.go", "runaway.go", diff --git a/pkg/domain/resource_group_controller_options.go b/pkg/domain/resource_group_controller_options.go new file mode 100644 index 0000000000000..dcb7a277346bf --- /dev/null +++ b/pkg/domain/resource_group_controller_options.go @@ -0,0 +1,49 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package domain + +import ( + "strings" + "time" + + "github.com/pingcap/tidb/pkg/config" + "github.com/pingcap/tidb/pkg/config/deploymode" + "github.com/pingcap/tidb/pkg/resourcegroup/runaway" + rmclient "github.com/tikv/pd/client/resource_group/controller" +) + +const ( + defaultDegradedModeWaitTimeout = 3 * time.Second / 2 + tokenWaitRetryInterval = 100 * time.Millisecond + tokenWaitRetryTimes = 20 +) + +func newResourceGroupsControllerOptions() []rmclient.ResourceControlCreateOption { + opts := []rmclient.ResourceControlCreateOption{ + rmclient.WithMaxWaitDuration(runaway.MaxWaitDuration), + } + if deploymode.IsStarter() { + opts = append(opts, + rmclient.WithDegradedModeWaitDuration(defaultDegradedModeWaitTimeout), + ) + } + if strings.Contains(config.GetGlobalConfig().StarterParams.PodNamespace, "vip") { + opts = append(opts, + rmclient.WithWaitRetryInterval(tokenWaitRetryInterval), + rmclient.WithWaitRetryTimes(tokenWaitRetryTimes), + ) + } + return opts +} diff --git a/pkg/domain/resource_group_provider.go b/pkg/domain/resource_group_provider.go index 91d5c1570acd8..0f6b5c44e4823 100644 --- a/pkg/domain/resource_group_provider.go +++ b/pkg/domain/resource_group_provider.go @@ -25,6 +25,22 @@ import ( rmclient "github.com/tikv/pd/client/resource_group/controller" ) +const ( + defaultDegradedRUFillRate = 2_000_000 + defaultDegradedRUBurstLimit = 50_000_000_000 +) + +func newDefaultDegradedRUSettings() *rmpb.GroupRequestUnitSettings { + return &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: defaultDegradedRUFillRate, + BurstLimit: defaultDegradedRUBurstLimit, + }, + }, + } +} + // starterResourceGroupProvider wraps a resource group provider and synthesizes // degraded resource groups only when resource-manager lookup fails due to // unavailability. diff --git a/pkg/domain/runaway.go b/pkg/domain/runaway.go index 4f6cc7710a39a..8be3f00e11590 100644 --- a/pkg/domain/runaway.go +++ b/pkg/domain/runaway.go @@ -18,11 +18,7 @@ import ( "context" "net" "strconv" - "strings" - "time" - rmpb "github.com/pingcap/kvproto/pkg/resource_manager" - "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/config/deploymode" "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/resourcegroup/runaway" @@ -33,43 +29,6 @@ import ( rmclient "github.com/tikv/pd/client/resource_group/controller" ) -const ( - defaultDegradedRUFillRate = 2_000_000 - defaultDegradedRUBurstLimit = 50_000_000_000 - defaultDegradedModeWaitTimeout = 3 * time.Second / 2 - tokenWaitRetryInterval = 100 * time.Millisecond - tokenWaitRetryTimes = 20 -) - -func newDefaultDegradedRUSettings() *rmpb.GroupRequestUnitSettings { - return &rmpb.GroupRequestUnitSettings{ - RU: &rmpb.TokenBucket{ - Settings: &rmpb.TokenLimitSettings{ - FillRate: defaultDegradedRUFillRate, - BurstLimit: defaultDegradedRUBurstLimit, - }, - }, - } -} - -func newResourceGroupsControllerOptions() []rmclient.ResourceControlCreateOption { - opts := []rmclient.ResourceControlCreateOption{ - rmclient.WithMaxWaitDuration(runaway.MaxWaitDuration), - } - if deploymode.IsStarter() { - opts = append(opts, - rmclient.WithDegradedModeWaitDuration(defaultDegradedModeWaitTimeout), - ) - } - if strings.Contains(config.GetGlobalConfig().StarterParams.PodNamespace, "vip") { - opts = append(opts, - rmclient.WithWaitRetryInterval(tokenWaitRetryInterval), - rmclient.WithWaitRetryTimes(tokenWaitRetryTimes), - ) - } - return opts -} - func (do *Domain) initResourceGroupsController(ctx context.Context, pdClient pd.Client, uniqueID uint64) error { if pdClient == nil { logutil.BgLogger().Warn("cannot setup up resource controller, not using tikv storage") From 7a97d4e38f6e1b77c582c78bc76a1ac8126a68d0 Mon Sep 17 00:00:00 2001 From: ystaticy Date: Tue, 14 Jul 2026 19:49:54 +0800 Subject: [PATCH 13/21] use with degradedResourceGroup Signed-off-by: ystaticy --- pkg/domain/BUILD.bazel | 6 +- .../resource_group_controller_options.go | 17 +++ pkg/domain/resource_group_provider.go | 132 ------------------ pkg/domain/resource_group_provider_test.go | 89 ------------ pkg/domain/runaway.go | 7 +- pkg/domain/runaway_test.go | 73 +++++++--- 6 files changed, 74 insertions(+), 250 deletions(-) delete mode 100644 pkg/domain/resource_group_provider.go delete mode 100644 pkg/domain/resource_group_provider_test.go diff --git a/pkg/domain/BUILD.bazel b/pkg/domain/BUILD.bazel index 42c0d4fde0065..e3d3d26942457 100644 --- a/pkg/domain/BUILD.bazel +++ b/pkg/domain/BUILD.bazel @@ -12,7 +12,6 @@ go_library( "plan_replayer.go", "plan_replayer_dump.go", "resource_group_controller_options.go", - "resource_group_provider.go", "ru_stats.go", "runaway.go", "schema_checker.go", @@ -126,7 +125,6 @@ go_library( "@com_github_tikv_client_go_v2//txnkv/transaction", "@com_github_tikv_pd_client//:client", "@com_github_tikv_pd_client//constants", - "@com_github_tikv_pd_client//errs", "@com_github_tikv_pd_client//http", "@com_github_tikv_pd_client//opt", "@com_github_tikv_pd_client//pkg/caller", @@ -152,7 +150,6 @@ go_test( "plan_replayer_handle_test.go", "plan_replayer_slow_log_test.go", "plan_replayer_test.go", - "resource_group_provider_test.go", "ru_stats_test.go", "runaway_test.go", "schema_checker_test.go", @@ -160,7 +157,7 @@ go_test( ], embed = [":domain"], flaky = True, - shard_count = 35, + shard_count = 34, deps = [ "//pkg/config", "//pkg/config/deploymode", @@ -200,6 +197,7 @@ go_test( "@com_github_ngaut_pools//:pools", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", + "@com_github_pingcap_kvproto//pkg/kvrpcpb", "@com_github_pingcap_kvproto//pkg/metapb", "@com_github_pingcap_kvproto//pkg/resource_manager", "@com_github_prometheus_client_model//go", diff --git a/pkg/domain/resource_group_controller_options.go b/pkg/domain/resource_group_controller_options.go index dcb7a277346bf..5df7a378f75a8 100644 --- a/pkg/domain/resource_group_controller_options.go +++ b/pkg/domain/resource_group_controller_options.go @@ -18,6 +18,7 @@ import ( "strings" "time" + rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/config/deploymode" "github.com/pingcap/tidb/pkg/resourcegroup/runaway" @@ -25,17 +26,33 @@ import ( ) const ( + defaultDegradedRUFillRate = 2_000_000 + defaultDegradedRUBurstLimit = 50_000_000_000 defaultDegradedModeWaitTimeout = 3 * time.Second / 2 tokenWaitRetryInterval = 100 * time.Millisecond tokenWaitRetryTimes = 20 ) +func newDefaultDegradedRUSettings() *rmpb.GroupRequestUnitSettings { + return &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{ + FillRate: defaultDegradedRUFillRate, + BurstLimit: defaultDegradedRUBurstLimit, + }, + }, + } +} + func newResourceGroupsControllerOptions() []rmclient.ResourceControlCreateOption { opts := []rmclient.ResourceControlCreateOption{ rmclient.WithMaxWaitDuration(runaway.MaxWaitDuration), } if deploymode.IsStarter() { opts = append(opts, + // Keep degraded-group synthesis inside the controller so degraded + // resource groups are not inserted into the controller cache. + rmclient.WithDegradedRUSettings(newDefaultDegradedRUSettings()), rmclient.WithDegradedModeWaitDuration(defaultDegradedModeWaitTimeout), ) } diff --git a/pkg/domain/resource_group_provider.go b/pkg/domain/resource_group_provider.go deleted file mode 100644 index 0f6b5c44e4823..0000000000000 --- a/pkg/domain/resource_group_provider.go +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package domain - -import ( - "context" - stderrors "errors" - "strings" - - rmpb "github.com/pingcap/kvproto/pkg/resource_manager" - pd "github.com/tikv/pd/client" - pderr "github.com/tikv/pd/client/errs" - rmclient "github.com/tikv/pd/client/resource_group/controller" -) - -const ( - defaultDegradedRUFillRate = 2_000_000 - defaultDegradedRUBurstLimit = 50_000_000_000 -) - -func newDefaultDegradedRUSettings() *rmpb.GroupRequestUnitSettings { - return &rmpb.GroupRequestUnitSettings{ - RU: &rmpb.TokenBucket{ - Settings: &rmpb.TokenLimitSettings{ - FillRate: defaultDegradedRUFillRate, - BurstLimit: defaultDegradedRUBurstLimit, - }, - }, - } -} - -// starterResourceGroupProvider wraps a resource group provider and synthesizes -// degraded resource groups only when resource-manager lookup fails due to -// unavailability. -type starterResourceGroupProvider struct { - rmclient.ResourceGroupProvider - degradedRUSettings *rmpb.GroupRequestUnitSettings -} - -func newStarterResourceGroupProvider(provider rmclient.ResourceGroupProvider) *starterResourceGroupProvider { - return &starterResourceGroupProvider{ - ResourceGroupProvider: provider, - degradedRUSettings: newDefaultDegradedRUSettings(), - } -} - -func (p *starterResourceGroupProvider) GetResourceGroup( - ctx context.Context, resourceGroupName string, opts ...pd.GetResourceGroupOption, -) (*rmpb.ResourceGroup, error) { - if err := ctx.Err(); err != nil { - return nil, err - } - group, err := p.ResourceGroupProvider.GetResourceGroup(ctx, resourceGroupName, opts...) - if err == nil { - return group, nil - } - if !shouldUseDegradedResourceGroupFallback(ctx, err) { - return nil, err - } - return newDegradedResourceGroup(resourceGroupName, p.degradedRUSettings), nil -} - -func newDegradedResourceGroup(name string, ruSettings *rmpb.GroupRequestUnitSettings) *rmpb.ResourceGroup { - return &rmpb.ResourceGroup{ - Name: name, - Mode: rmpb.GroupMode_RUMode, - RUSettings: ruSettings, - } -} - -func shouldUseDegradedResourceGroupFallback(ctx context.Context, err error) bool { - if err == nil { - return false - } - if ctx.Err() != nil { - return false - } - if stderrors.Is(err, context.Canceled) || stderrors.Is(err, context.DeadlineExceeded) { - return false - } - if isResourceGroupNotExistError(err) { - return false - } - if stderrors.Is(err, pderr.ErrClientResourceGroupConfigUnavailable) { - return true - } - var getRGErr *pderr.ErrClientGetResourceGroup - if stderrors.As(err, &getRGErr) { - return isResourceManagerUnavailableCause(getRGErr.Cause) - } - return isResourceManagerUnavailableCause(err.Error()) -} - -func isResourceGroupNotExistError(err error) bool { - var getRGErr *pderr.ErrClientGetResourceGroup - if !stderrors.As(err, &getRGErr) { - return false - } - return isResourceGroupNotExistCause(getRGErr.Cause) -} - -func isResourceGroupNotExistCause(cause string) bool { - lower := strings.ToLower(cause) - return strings.Contains(lower, "does not exist") || - strings.Contains(lower, "not found") || - strings.Contains(lower, "not exist") -} - -func isResourceManagerUnavailableCause(cause string) bool { - if isResourceGroupNotExistCause(cause) { - return false - } - lower := strings.ToLower(cause) - return strings.Contains(lower, "unavailable") || - strings.Contains(lower, "connection refused") || - strings.Contains(lower, "connection reset") || - strings.Contains(lower, "deadline exceeded") || - strings.Contains(lower, "transport is closing") || - strings.Contains(lower, "rpc error") -} diff --git a/pkg/domain/resource_group_provider_test.go b/pkg/domain/resource_group_provider_test.go deleted file mode 100644 index d51dcdd73bc96..0000000000000 --- a/pkg/domain/resource_group_provider_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package domain - -import ( - "context" - "errors" - "testing" - - "github.com/stretchr/testify/require" - pderr "github.com/tikv/pd/client/errs" -) - -func TestShouldUseDegradedResourceGroupFallback(t *testing.T) { - ctx := context.Background() - canceledCtx, cancel := context.WithCancel(context.Background()) - cancel() - - tests := []struct { - name string - ctx context.Context - err error - want bool - }{ - { - name: "resource manager unavailable", - ctx: ctx, - err: &pderr.ErrClientGetResourceGroup{ - ResourceGroupName: "rg1", - Cause: "resource group unavailable", - }, - want: true, - }, - { - name: "resource group not found", - ctx: ctx, - err: &pderr.ErrClientGetResourceGroup{ - ResourceGroupName: "rg1", - Cause: "resource group does not exist", - }, - want: false, - }, - { - name: "context canceled", - ctx: canceledCtx, - err: &pderr.ErrClientGetResourceGroup{ - ResourceGroupName: "rg1", - Cause: "resource group unavailable", - }, - want: false, - }, - { - name: "lookup canceled", - ctx: ctx, - err: context.Canceled, - want: false, - }, - { - name: "config unavailable", - ctx: ctx, - err: pderr.ErrClientResourceGroupConfigUnavailable.FastGenByArgs("not configured"), - want: true, - }, - { - name: "generic non retryable", - ctx: ctx, - err: errors.New("invalid resource group lookup"), - want: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - require.Equal(t, tt.want, shouldUseDegradedResourceGroupFallback(tt.ctx, tt.err)) - }) - } -} diff --git a/pkg/domain/runaway.go b/pkg/domain/runaway.go index 8be3f00e11590..439369bc2536f 100644 --- a/pkg/domain/runaway.go +++ b/pkg/domain/runaway.go @@ -19,7 +19,6 @@ import ( "net" "strconv" - "github.com/pingcap/tidb/pkg/config/deploymode" "github.com/pingcap/tidb/pkg/domain/infosync" "github.com/pingcap/tidb/pkg/resourcegroup/runaway" "github.com/pingcap/tidb/pkg/util/logutil" @@ -40,11 +39,7 @@ func (do *Domain) initResourceGroupsController(ctx context.Context, pdClient pd. if codec := do.Store().GetCodec(); codec != nil { keyspaceID = uint32(codec.GetKeyspaceID()) } - var provider rmclient.ResourceGroupProvider = pdClient - if deploymode.IsStarter() { - provider = newStarterResourceGroupProvider(pdClient) - } - control, err := rmclient.NewResourceGroupController(ctx, uniqueID, provider, nil, keyspaceID, newResourceGroupsControllerOptions()...) + control, err := rmclient.NewResourceGroupController(ctx, uniqueID, pdClient, nil, keyspaceID, newResourceGroupsControllerOptions()...) if err != nil { return err } diff --git a/pkg/domain/runaway_test.go b/pkg/domain/runaway_test.go index 198f448e36f5b..ce0b57d74d265 100644 --- a/pkg/domain/runaway_test.go +++ b/pkg/domain/runaway_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + "github.com/pingcap/kvproto/pkg/kvrpcpb" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/config/deploymode" @@ -66,7 +67,7 @@ func newStarterControllerForTest(t *testing.T, provider rmclient.ResourceGroupPr controller, err := rmclient.NewResourceGroupController( ctx, 1, - newStarterResourceGroupProvider(provider), + provider, nil, 0, newResourceGroupsControllerOptions()..., @@ -129,9 +130,33 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { controllerWithFallback := newStarterControllerForTest(t, provider) group, err := controllerWithFallback.GetResourceGroup("test-group") require.NoError(t, err) - require.Equal(t, newDegradedResourceGroup("test-group", newDefaultDegradedRUSettings()), group) + require.Equal(t, &rmpb.ResourceGroup{ + Name: "test-group", + Mode: rmpb.GroupMode_RUMode, + RUSettings: newDefaultDegradedRUSettings(), + }, group) + + // The degraded group should not be cached. Once the provider recovers, the + // next lookup should observe the real resource-group metadata. + provider.resourceGroup = &rmpb.ResourceGroup{ + Name: "test-group", + Mode: rmpb.GroupMode_RUMode, + RUSettings: &rmpb.GroupRequestUnitSettings{ + RU: &rmpb.TokenBucket{ + Settings: &rmpb.TokenLimitSettings{FillRate: 1}, + }, + }, + } + provider.resourceErr = nil + group, err = controllerWithFallback.GetResourceGroup("test-group") + require.NoError(t, err) + require.Equal(t, provider.resourceGroup, group) require.NoError(t, deploymode.Set(deploymode.Premium)) + provider = newResourceGroupProviderStub(t, nil, &pderr.ErrClientGetResourceGroup{ + ResourceGroupName: "test-group", + Cause: "resource group unavailable", + }) controllerWithoutFallback, err := rmclient.NewResourceGroupController( ctx, 2, @@ -151,7 +176,7 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { require.Nil(t, group) } -func TestStarterResourceGroupProviderNonRetryableErrors(t *testing.T) { +func TestStarterControllerGetResourceGroupUsesDegradedSettingsOnErrors(t *testing.T) { if !kerneltype.IsNextGen() { t.Skip("Starter deploy mode is only available in NextGen builds") } @@ -160,24 +185,30 @@ func TestStarterResourceGroupProviderNonRetryableErrors(t *testing.T) { require.NoError(t, deploymode.Set(originalDeployMode)) }) + expectedGroup := func(name string) *rmpb.ResourceGroup { + return &rmpb.ResourceGroup{ + Name: name, + Mode: rmpb.GroupMode_RUMode, + RUSettings: newDefaultDegradedRUSettings(), + } + } + t.Run("NotFound", func(t *testing.T) { provider := newResourceGroupProviderStub(t, nil, rmclient.NewResourceGroupNotExistErr("missing-group")) controller := newStarterControllerForTest(t, provider) group, err := controller.GetResourceGroup("missing-group") - require.Error(t, err) - require.Nil(t, group) + require.NoError(t, err) + require.Equal(t, expectedGroup("missing-group"), group) }) t.Run("ContextCanceled", func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - provider := newResourceGroupProviderStub(t, nil, context.Canceled) - wrapped := newStarterResourceGroupProvider(provider) - group, err := wrapped.GetResourceGroup(ctx, "test-group") - require.ErrorIs(t, err, context.Canceled) - require.Nil(t, group) + controller := newStarterControllerForTest(t, provider) + + group, err := controller.GetResourceGroup("test-group") + require.NoError(t, err) + require.Equal(t, expectedGroup("test-group"), group) }) t.Run("GenericNonRetryableError", func(t *testing.T) { @@ -185,12 +216,12 @@ func TestStarterResourceGroupProviderNonRetryableErrors(t *testing.T) { controller := newStarterControllerForTest(t, provider) group, err := controller.GetResourceGroup("test-group") - require.Error(t, err) - require.Nil(t, group) + require.NoError(t, err) + require.Equal(t, expectedGroup("test-group"), group) }) } -func TestStarterSwitchGroupRejectsMissingGroup(t *testing.T) { +func TestStarterSwitchGroupUsesDegradedGroupOnLookupError(t *testing.T) { if !kerneltype.IsNextGen() { t.Skip("Starter deploy mode is only available in NextGen builds") } @@ -218,9 +249,13 @@ func TestStarterSwitchGroupRejectsMissingGroup(t *testing.T) { ) require.NoError(t, checker.CheckThresholds(nil, 10, nil)) - req := &tikvrpc.Request{} + req := &tikvrpc.Request{ + Context: kvrpcpb.Context{ + ResourceControlContext: &kvrpcpb.ResourceControlContext{}, + }, + } require.NoError(t, checker.BeforeCopRequest(req)) - require.Empty(t, req.GetResourceControlContext().GetResourceGroupName()) + require.Equal(t, "missing-switch-group", req.GetResourceControlContext().GetResourceGroupName()) } func TestResourceGroupsControllerOptionsUseStarterPodNamespaceForVIPWaitRetry(t *testing.T) { @@ -234,8 +269,8 @@ func TestResourceGroupsControllerOptionsUseStarterPodNamespaceForVIPWaitRetry(t require.NoError(t, deploymode.Set(originalDeployMode)) }) require.NoError(t, deploymode.Set(deploymode.Starter)) - vipOptionCount = 4 - standardOptionCount = 2 + vipOptionCount = 5 + standardOptionCount = 3 } config.UpdateGlobal(func(conf *config.Config) { From d1989e5733d061380afd38fb87150482d98492c6 Mon Sep 17 00:00:00 2001 From: ystaticy Date: Tue, 14 Jul 2026 22:12:47 +0800 Subject: [PATCH 14/21] fix config Signed-off-by: ystaticy --- cmd/tidb-server/main.go | 29 +++++++++++++++++------------ cmd/tidb-server/main_test.go | 22 ++++++++++++++++------ pkg/config/config.go | 9 +++++++++ 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/cmd/tidb-server/main.go b/cmd/tidb-server/main.go index 516971d745e07..5102accf8c6f3 100644 --- a/cmd/tidb-server/main.go +++ b/cmd/tidb-server/main.go @@ -711,9 +711,7 @@ func overrideConfig(cfg *config.Config, fset *flag.FlagSet) { actualFlags[f.Name] = true }) if actualFlags[nmStarterParams] && cfg.DeployMode == deploymode.Starter { - params, err := parseStarterAdditionalParams(getStarterAdditionalParams()) - terror.MustNil(err) - cfg.StarterParams.PodNamespace = params.podNamespace + terror.MustNil(applyStarterAdditionalParams(cfg, getStarterAdditionalParams())) } // Base @@ -1406,6 +1404,18 @@ func parseStarterAdditionalParams(raw string) (starterParams, error) { return params, nil } +func applyStarterAdditionalParams(cfg *config.Config, raw string) error { + params, err := parseStarterAdditionalParams(raw) + if err != nil { + return err + } + cfg.StarterParams.ManagerNamespace = params.managerNamespace + cfg.StarterParams.PodName = params.podName + cfg.StarterParams.PodIP = params.podIP + cfg.StarterParams.PodNamespace = params.podNamespace + return nil +} + func getStarterAdditionalParams() string { if starterAdditionalParams == nil { return "" @@ -1429,23 +1439,18 @@ func createMgrClientForStarter() (tidbmanager.Client, error) { return nil, err } - params, err := parseStarterAdditionalParams(getStarterAdditionalParams()) - if err != nil { - return nil, err - } - managerAddr := cfg.StarterParams.ManagerAddr if managerAddr == "" { - managerNs := params.managerNamespace + managerNs := cfg.StarterParams.ManagerNamespace if managerNs == "" { return nil, fmt.Errorf("manager notifier requires manager-addr config or manager-namespace in --starter-additional-params") } managerAddr = fmt.Sprintf("manager-server.%s.svc:8000", managerNs) } - podName := params.podName - podIP := params.podIP - namespace := params.podNamespace + podName := cfg.StarterParams.PodName + podIP := cfg.StarterParams.PodIP + namespace := cfg.StarterParams.PodNamespace if podName == "" || podIP == "" || namespace == "" { return nil, fmt.Errorf("manager notifier requires --starter-additional-params with pod-name, pod-ip and pod-namespace: pod-name=%q, pod-ip=%q, pod-namespace=%q", podName, podIP, namespace) diff --git a/cmd/tidb-server/main_test.go b/cmd/tidb-server/main_test.go index 9650261a94e54..50f5e0d5bf8de 100644 --- a/cmd/tidb-server/main_test.go +++ b/cmd/tidb-server/main_test.go @@ -92,6 +92,8 @@ func TestOverrideConfigKeyspaceActivateMode(t *testing.T) { cfg.DeployMode = deploymode.Starter overrideConfig(cfg, fset) require.True(t, cfg.KeyspaceActivateMode) + require.Equal(t, "pod-1", cfg.StarterParams.PodName) + require.Equal(t, "10.0.0.1", cfg.StarterParams.PodIP) require.Equal(t, "ns-1", cfg.StarterParams.PodNamespace) require.Equal(t, "pod-name=pod-1,pod-ip=10.0.0.1,pod-namespace=ns-1", *starterAdditionalParams) } @@ -210,25 +212,33 @@ func TestCreateMgrClientRequiresPodIdentityInStarter(t *testing.T) { require.ErrorContains(t, err, "manager notifier requires --starter-additional-params") duplicatedParam := "pod-name=pod-1,pod-name=pod-2,pod-ip=10.0.0.1,pod-namespace=ns-1" - starterAdditionalParams = &duplicatedParam - _, err = createMgrClientForStarter() + config.UpdateGlobal(func(conf *config.Config) { + conf.StarterParams.ManagerNamespace = "" + conf.StarterParams.PodName = "" + conf.StarterParams.PodIP = "" + conf.StarterParams.PodNamespace = "" + }) + err = applyStarterAdditionalParams(config.GetGlobalConfig(), duplicatedParam) require.ErrorContains(t, err, `starter additional param "pod-name" is duplicated`) unknownParam := "pod-name=pod-1,pod-ip=10.0.0.1,pod-namespace=ns-1,unknown=value" - starterAdditionalParams = &unknownParam - _, err = createMgrClientForStarter() + err = applyStarterAdditionalParams(config.GetGlobalConfig(), unknownParam) require.ErrorContains(t, err, `unknown starter additional param "unknown"`) config.UpdateGlobal(func(conf *config.Config) { conf.StarterParams.ManagerAddr = "" + conf.StarterParams.ManagerNamespace = "" + conf.StarterParams.PodName = "" + conf.StarterParams.PodIP = "" + conf.StarterParams.PodNamespace = "" }) missingManagerNamespace := "pod-name=pod-1,pod-ip=10.0.0.1,pod-namespace=ns-1" - starterAdditionalParams = &missingManagerNamespace + require.NoError(t, applyStarterAdditionalParams(config.GetGlobalConfig(), missingManagerNamespace)) _, err = createMgrClientForStarter() require.ErrorContains(t, err, "manager notifier requires manager-addr config or manager-namespace in --starter-additional-params") validParams := "manager-namespace=manager-ns,pod-name=pod-1,pod-ip=10.0.0.1,pod-namespace=ns-1" - starterAdditionalParams = &validParams + require.NoError(t, applyStarterAdditionalParams(config.GetGlobalConfig(), validParams)) cli, err := createMgrClientForStarter() require.NoError(t, err) require.NotNil(t, cli) diff --git a/pkg/config/config.go b/pkg/config/config.go index 32a61be17b6f0..8d3236bbf3895 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1094,6 +1094,15 @@ type StarterParams struct { // ManagerAddr is the TiDB manager address used by the shutdown notifier. // When empty and EnableManagerNotifier is true, the Starter path derives the service address from starter additional params. ManagerAddr string `toml:"manager-addr" json:"manager-addr,omitempty"` + // ManagerNamespace captures the runtime manager namespace from --starter-additional-params. + // It is command-line derived runtime metadata rather than file-backed config. + ManagerNamespace string `toml:"-" json:"-"` + // PodName captures the runtime pod name from --starter-additional-params. + // It is command-line derived runtime metadata rather than file-backed config. + PodName string `toml:"-" json:"-"` + // PodIP captures the runtime pod IP from --starter-additional-params. + // It is command-line derived runtime metadata rather than file-backed config. + PodIP string `toml:"-" json:"-"` // PodNamespace captures the runtime pod namespace from --starter-additional-params. // It is command-line derived runtime metadata rather than file-backed config. PodNamespace string `toml:"-" json:"-"` From 0ea629f60ac274adc532c0d0424ce2e18c11ec63 Mon Sep 17 00:00:00 2001 From: ystaticy Date: Wed, 15 Jul 2026 22:34:40 +0800 Subject: [PATCH 15/21] fix comments Signed-off-by: ystaticy --- pkg/domain/runaway_test.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pkg/domain/runaway_test.go b/pkg/domain/runaway_test.go index ce0b57d74d265..dec2fe7a54e04 100644 --- a/pkg/domain/runaway_test.go +++ b/pkg/domain/runaway_test.go @@ -80,6 +80,18 @@ func newStarterControllerForTest(t *testing.T, provider rmclient.ResourceGroupPr return controller } +func requireDegradedResourceGroup(t *testing.T, group *rmpb.ResourceGroup, name string) { + t.Helper() + require.NotNil(t, group) + require.Equal(t, name, group.Name) + require.Equal(t, rmpb.GroupMode_RUMode, group.Mode) + require.NotNil(t, group.RUSettings) + require.NotNil(t, group.RUSettings.RU) + require.NotNil(t, group.RUSettings.RU.Settings) + require.EqualValues(t, 2_000_000, group.RUSettings.RU.Settings.FillRate) + require.EqualValues(t, 50_000_000_000, group.RUSettings.RU.Settings.BurstLimit) +} + func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { if kerneltype.IsNextGen() { // Preserve the process-wide deploy mode because deploymode.IsStarter reads @@ -130,11 +142,7 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { controllerWithFallback := newStarterControllerForTest(t, provider) group, err := controllerWithFallback.GetResourceGroup("test-group") require.NoError(t, err) - require.Equal(t, &rmpb.ResourceGroup{ - Name: "test-group", - Mode: rmpb.GroupMode_RUMode, - RUSettings: newDefaultDegradedRUSettings(), - }, group) + requireDegradedResourceGroup(t, group, "test-group") // The degraded group should not be cached. Once the provider recovers, the // next lookup should observe the real resource-group metadata. @@ -185,21 +193,13 @@ func TestStarterControllerGetResourceGroupUsesDegradedSettingsOnErrors(t *testin require.NoError(t, deploymode.Set(originalDeployMode)) }) - expectedGroup := func(name string) *rmpb.ResourceGroup { - return &rmpb.ResourceGroup{ - Name: name, - Mode: rmpb.GroupMode_RUMode, - RUSettings: newDefaultDegradedRUSettings(), - } - } - t.Run("NotFound", func(t *testing.T) { provider := newResourceGroupProviderStub(t, nil, rmclient.NewResourceGroupNotExistErr("missing-group")) controller := newStarterControllerForTest(t, provider) group, err := controller.GetResourceGroup("missing-group") require.NoError(t, err) - require.Equal(t, expectedGroup("missing-group"), group) + requireDegradedResourceGroup(t, group, "missing-group") }) t.Run("ContextCanceled", func(t *testing.T) { @@ -208,7 +208,7 @@ func TestStarterControllerGetResourceGroupUsesDegradedSettingsOnErrors(t *testin group, err := controller.GetResourceGroup("test-group") require.NoError(t, err) - require.Equal(t, expectedGroup("test-group"), group) + requireDegradedResourceGroup(t, group, "test-group") }) t.Run("GenericNonRetryableError", func(t *testing.T) { @@ -217,7 +217,7 @@ func TestStarterControllerGetResourceGroupUsesDegradedSettingsOnErrors(t *testin group, err := controller.GetResourceGroup("test-group") require.NoError(t, err) - require.Equal(t, expectedGroup("test-group"), group) + requireDegradedResourceGroup(t, group, "test-group") }) } From 59c011f16151db01b2527ae6dcb15915f7cfdb89 Mon Sep 17 00:00:00 2001 From: ystaticy Date: Mon, 20 Jul 2026 16:44:01 +0800 Subject: [PATCH 16/21] update pd client Signed-off-by: ystaticy --- DEPS.bzl | 4 +-- go.mod | 2 +- go.sum | 4 +-- pkg/domain/BUILD.bazel | 4 ++- pkg/domain/runaway_test.go | 71 ++++++++++---------------------------- 5 files changed, 27 insertions(+), 58 deletions(-) diff --git a/DEPS.bzl b/DEPS.bzl index 381bd2e9a9f28..30682100fc3cc 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -4207,8 +4207,8 @@ def go_deps(): build_tags = ["nextgen", "intest"], build_file_proto_mode = "disable_global", importpath = "github.com/tikv/pd/client", - sum = "h1:OoBvgoeWmdNEXtS+eOlhysz/OvhA4GS0OdPVhTXteGA=", - version = "v0.0.0-20260708075407-4e05b9d2c2d3", + sum = "h1:BiPn4oLXVqB9ptPNmyo3rh/2IH1dTouXrrnneH86Z3M=", + version = "v0.0.0-20260720043438-0b37df9a48ed", ) go_repository( name = "com_github_timakin_bodyclose", diff --git a/go.mod b/go.mod index e590aed6c5c42..179607ad82d4e 100644 --- a/go.mod +++ b/go.mod @@ -123,7 +123,7 @@ require ( github.com/stretchr/testify v1.11.1 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 github.com/tikv/client-go/v2 v2.0.8-0.20260708122311-01bd8f99f4da - github.com/tikv/pd/client v0.0.0-20260708075407-4e05b9d2c2d3 + github.com/tikv/pd/client v0.0.0-20260720043438-0b37df9a48ed github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67 github.com/twmb/murmur3 v1.1.6 github.com/uber/jaeger-client-go v2.22.1+incompatible diff --git a/go.sum b/go.sum index a69dcfc6c875c..0440b05504932 100644 --- a/go.sum +++ b/go.sum @@ -902,8 +902,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tikv/client-go/v2 v2.0.8-0.20260708122311-01bd8f99f4da h1:Ju9uUKu3M5gPl7Z90e9dcGSe8LtXxGqvSetbPcr6tyc= github.com/tikv/client-go/v2 v2.0.8-0.20260708122311-01bd8f99f4da/go.mod h1:MRhIujZMMkYcI49Euif4+A3+SHSGwTpyUUadNz48R4g= -github.com/tikv/pd/client v0.0.0-20260708075407-4e05b9d2c2d3 h1:OoBvgoeWmdNEXtS+eOlhysz/OvhA4GS0OdPVhTXteGA= -github.com/tikv/pd/client v0.0.0-20260708075407-4e05b9d2c2d3/go.mod h1:3/Bu91CJONgkDA+Y0v/cnbROSJnu5tQ09vv7JGybUBA= +github.com/tikv/pd/client v0.0.0-20260720043438-0b37df9a48ed h1:BiPn4oLXVqB9ptPNmyo3rh/2IH1dTouXrrnneH86Z3M= +github.com/tikv/pd/client v0.0.0-20260720043438-0b37df9a48ed/go.mod h1:3/Bu91CJONgkDA+Y0v/cnbROSJnu5tQ09vv7JGybUBA= github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67 h1:9LPGD+jzxMlnk5r6+hJnar67cgpDIz/iyD+rfl5r2Vk= github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67/go.mod h1:mkjARE7Yr8qU23YcGMSALbIxTQ9r9QBVahQOBRfU460= github.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w= diff --git a/pkg/domain/BUILD.bazel b/pkg/domain/BUILD.bazel index e3d3d26942457..8fbbc1674c6b2 100644 --- a/pkg/domain/BUILD.bazel +++ b/pkg/domain/BUILD.bazel @@ -157,7 +157,7 @@ go_test( ], embed = [":domain"], flaky = True, - shard_count = 34, + shard_count = 33, deps = [ "//pkg/config", "//pkg/config/deploymode", @@ -209,6 +209,8 @@ go_test( "@com_github_tikv_pd_client//opt", "@com_github_tikv_pd_client//resource_group/controller", "@io_etcd_go_etcd_tests_v3//integration", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//status", "@org_uber_go_goleak//:goleak", ], ) diff --git a/pkg/domain/runaway_test.go b/pkg/domain/runaway_test.go index dec2fe7a54e04..0b270489269f4 100644 --- a/pkg/domain/runaway_test.go +++ b/pkg/domain/runaway_test.go @@ -16,7 +16,6 @@ package domain import ( "context" - "errors" "testing" "time" @@ -32,6 +31,8 @@ import ( pd "github.com/tikv/pd/client" pderr "github.com/tikv/pd/client/errs" rmclient "github.com/tikv/pd/client/resource_group/controller" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type resourceGroupProviderStub struct { @@ -88,8 +89,17 @@ func requireDegradedResourceGroup(t *testing.T, group *rmpb.ResourceGroup, name require.NotNil(t, group.RUSettings) require.NotNil(t, group.RUSettings.RU) require.NotNil(t, group.RUSettings.RU.Settings) - require.EqualValues(t, 2_000_000, group.RUSettings.RU.Settings.FillRate) - require.EqualValues(t, 50_000_000_000, group.RUSettings.RU.Settings.BurstLimit) + require.EqualValues(t, defaultDegradedRUFillRate, group.RUSettings.RU.Settings.FillRate) + require.EqualValues(t, defaultDegradedRUBurstLimit, group.RUSettings.RU.Settings.BurstLimit) +} + +func newTransientGetResourceGroupErr(name string) error { + err := status.Error(codes.Unavailable, "resource manager unavailable") + return &pderr.ErrClientGetResourceGroup{ + ResourceGroupName: name, + Cause: err.Error(), + Err: err, + } } func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { @@ -113,10 +123,7 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { Settings: &rmpb.TokenLimitSettings{FillRate: 1}, }, }, - }, &pderr.ErrClientGetResourceGroup{ - ResourceGroupName: "test-group", - Cause: "resource group unavailable", - }) + }, newTransientGetResourceGroupErr("test-group")) if !kerneltype.IsNextGen() { controller, err := rmclient.NewResourceGroupController( @@ -161,10 +168,7 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { require.Equal(t, provider.resourceGroup, group) require.NoError(t, deploymode.Set(deploymode.Premium)) - provider = newResourceGroupProviderStub(t, nil, &pderr.ErrClientGetResourceGroup{ - ResourceGroupName: "test-group", - Cause: "resource group unavailable", - }) + provider = newResourceGroupProviderStub(t, nil, newTransientGetResourceGroupErr("test-group")) controllerWithoutFallback, err := rmclient.NewResourceGroupController( ctx, 2, @@ -184,44 +188,7 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { require.Nil(t, group) } -func TestStarterControllerGetResourceGroupUsesDegradedSettingsOnErrors(t *testing.T) { - if !kerneltype.IsNextGen() { - t.Skip("Starter deploy mode is only available in NextGen builds") - } - originalDeployMode := deploymode.Get() - t.Cleanup(func() { - require.NoError(t, deploymode.Set(originalDeployMode)) - }) - - t.Run("NotFound", func(t *testing.T) { - provider := newResourceGroupProviderStub(t, nil, rmclient.NewResourceGroupNotExistErr("missing-group")) - controller := newStarterControllerForTest(t, provider) - - group, err := controller.GetResourceGroup("missing-group") - require.NoError(t, err) - requireDegradedResourceGroup(t, group, "missing-group") - }) - - t.Run("ContextCanceled", func(t *testing.T) { - provider := newResourceGroupProviderStub(t, nil, context.Canceled) - controller := newStarterControllerForTest(t, provider) - - group, err := controller.GetResourceGroup("test-group") - require.NoError(t, err) - requireDegradedResourceGroup(t, group, "test-group") - }) - - t.Run("GenericNonRetryableError", func(t *testing.T) { - provider := newResourceGroupProviderStub(t, nil, errors.New("invalid resource group lookup")) - controller := newStarterControllerForTest(t, provider) - - group, err := controller.GetResourceGroup("test-group") - require.NoError(t, err) - requireDegradedResourceGroup(t, group, "test-group") - }) -} - -func TestStarterSwitchGroupUsesDegradedGroupOnLookupError(t *testing.T) { +func TestStarterSwitchGroupUsesDegradedGroupOnTransientLookupError(t *testing.T) { if !kerneltype.IsNextGen() { t.Skip("Starter deploy mode is only available in NextGen builds") } @@ -230,7 +197,7 @@ func TestStarterSwitchGroupUsesDegradedGroupOnLookupError(t *testing.T) { require.NoError(t, deploymode.Set(originalDeployMode)) }) - provider := newResourceGroupProviderStub(t, nil, rmclient.NewResourceGroupNotExistErr("missing-switch-group")) + provider := newResourceGroupProviderStub(t, nil, newTransientGetResourceGroupErr("target-switch-group")) controller := newStarterControllerForTest(t, provider) manager := runaway.NewRunawayManager(controller, "127.0.0.1:4000", nil, make(chan struct{}), nil, nil) t.Cleanup(manager.Stop) @@ -239,7 +206,7 @@ func TestStarterSwitchGroupUsesDegradedGroupOnLookupError(t *testing.T) { "source-group", &rmpb.RunawaySettings{ Action: rmpb.RunawayAction_SwitchGroup, - SwitchGroupName: "missing-switch-group", + SwitchGroupName: "target-switch-group", Rule: &rmpb.RunawayRule{ProcessedKeys: 1}, }, "SELECT 1", @@ -255,7 +222,7 @@ func TestStarterSwitchGroupUsesDegradedGroupOnLookupError(t *testing.T) { }, } require.NoError(t, checker.BeforeCopRequest(req)) - require.Equal(t, "missing-switch-group", req.GetResourceControlContext().GetResourceGroupName()) + require.Equal(t, "target-switch-group", req.GetResourceControlContext().GetResourceGroupName()) } func TestResourceGroupsControllerOptionsUseStarterPodNamespaceForVIPWaitRetry(t *testing.T) { From 5e8359b3fd9f2565c681529109b0890875ed6fcf Mon Sep 17 00:00:00 2001 From: ystaticy Date: Mon, 20 Jul 2026 17:50:18 +0800 Subject: [PATCH 17/21] add config Signed-off-by: ystaticy --- cmd/tidb-server/main.go | 16 ++- cmd/tidb-server/main_test.go | 9 +- pkg/config/config.go | 3 + pkg/domain/BUILD.bazel | 1 + .../resource_group_controller_options.go | 7 +- pkg/domain/runaway_test.go | 110 +++++++++++++++--- 6 files changed, 116 insertions(+), 30 deletions(-) diff --git a/cmd/tidb-server/main.go b/cmd/tidb-server/main.go index 5102accf8c6f3..8cafdf07a8092 100644 --- a/cmd/tidb-server/main.go +++ b/cmd/tidb-server/main.go @@ -1351,10 +1351,11 @@ func prepareKeyspaceObservabilityForStarter(metadata map[string]string) error { } type starterParams struct { - managerNamespace string - podName string - podIP string - podNamespace string + managerNamespace string + podName string + podIP string + podNamespace string + enableGetResourceGroupDegraded bool } func parseStarterAdditionalParams(raw string) (starterParams, error) { @@ -1397,6 +1398,12 @@ func parseStarterAdditionalParams(raw string) (starterParams, error) { params.podIP = value case "pod-namespace": params.podNamespace = value + case "enable-get-resource-group-degraded": + enable, err := strconv.ParseBool(value) + if err != nil { + return params, fmt.Errorf("starter additional param %q must be a bool: %w", key, err) + } + params.enableGetResourceGroupDegraded = enable default: return params, fmt.Errorf("unknown starter additional param %q", key) } @@ -1413,6 +1420,7 @@ func applyStarterAdditionalParams(cfg *config.Config, raw string) error { cfg.StarterParams.PodName = params.podName cfg.StarterParams.PodIP = params.podIP cfg.StarterParams.PodNamespace = params.podNamespace + cfg.StarterParams.EnableGetResourceGroupDegraded = params.enableGetResourceGroupDegraded return nil } diff --git a/cmd/tidb-server/main_test.go b/cmd/tidb-server/main_test.go index 50f5e0d5bf8de..317e2ac8d75ab 100644 --- a/cmd/tidb-server/main_test.go +++ b/cmd/tidb-server/main_test.go @@ -85,7 +85,7 @@ func TestOverrideConfigKeyspaceActivateMode(t *testing.T) { fset := initFlagSet() require.NoError(t, fset.Parse([]string{ "--keyspace-activate=true", - "--starter-additional-params=pod-name=pod-1,pod-ip=10.0.0.1,pod-namespace=ns-1", + "--starter-additional-params=pod-name=pod-1,pod-ip=10.0.0.1,pod-namespace=ns-1,enable-get-resource-group-degraded=true", })) cfg := config.NewConfig() @@ -95,7 +95,8 @@ func TestOverrideConfigKeyspaceActivateMode(t *testing.T) { require.Equal(t, "pod-1", cfg.StarterParams.PodName) require.Equal(t, "10.0.0.1", cfg.StarterParams.PodIP) require.Equal(t, "ns-1", cfg.StarterParams.PodNamespace) - require.Equal(t, "pod-name=pod-1,pod-ip=10.0.0.1,pod-namespace=ns-1", *starterAdditionalParams) + require.True(t, cfg.StarterParams.EnableGetResourceGroupDegraded) + require.Equal(t, "pod-name=pod-1,pod-ip=10.0.0.1,pod-namespace=ns-1,enable-get-resource-group-degraded=true", *starterAdditionalParams) } func TestSetGlobalVars(t *testing.T) { @@ -225,6 +226,10 @@ func TestCreateMgrClientRequiresPodIdentityInStarter(t *testing.T) { err = applyStarterAdditionalParams(config.GetGlobalConfig(), unknownParam) require.ErrorContains(t, err, `unknown starter additional param "unknown"`) + invalidBoolParam := "enable-get-resource-group-degraded=definitely" + err = applyStarterAdditionalParams(config.GetGlobalConfig(), invalidBoolParam) + require.ErrorContains(t, err, `starter additional param "enable-get-resource-group-degraded" must be a bool`) + config.UpdateGlobal(func(conf *config.Config) { conf.StarterParams.ManagerAddr = "" conf.StarterParams.ManagerNamespace = "" diff --git a/pkg/config/config.go b/pkg/config/config.go index 8d3236bbf3895..63c3fb917c588 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1106,6 +1106,9 @@ type StarterParams struct { // PodNamespace captures the runtime pod namespace from --starter-additional-params. // It is command-line derived runtime metadata rather than file-backed config. PodNamespace string `toml:"-" json:"-"` + // EnableGetResourceGroupDegraded enables degraded GetResourceGroup handling for resource control. + // It is command-line derived runtime metadata rather than file-backed config. + EnableGetResourceGroupDegraded bool `toml:"-" json:"-"` // MaxImportDataSize is the maximum total real source data size allowed for IMPORT INTO. // Zero means unlimited. MaxImportDataSize configtypes.ByteSize `toml:"max-import-data-size" json:"max-import-data-size,omitempty"` diff --git a/pkg/domain/BUILD.bazel b/pkg/domain/BUILD.bazel index 8fbbc1674c6b2..f6804d4b65a2d 100644 --- a/pkg/domain/BUILD.bazel +++ b/pkg/domain/BUILD.bazel @@ -198,6 +198,7 @@ go_test( "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", "@com_github_pingcap_kvproto//pkg/kvrpcpb", + "@com_github_pingcap_kvproto//pkg/meta_storagepb", "@com_github_pingcap_kvproto//pkg/metapb", "@com_github_pingcap_kvproto//pkg/resource_manager", "@com_github_prometheus_client_model//go", diff --git a/pkg/domain/resource_group_controller_options.go b/pkg/domain/resource_group_controller_options.go index 5df7a378f75a8..ada7a0640cdcb 100644 --- a/pkg/domain/resource_group_controller_options.go +++ b/pkg/domain/resource_group_controller_options.go @@ -15,7 +15,6 @@ package domain import ( - "strings" "time" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" @@ -48,16 +47,12 @@ func newResourceGroupsControllerOptions() []rmclient.ResourceControlCreateOption opts := []rmclient.ResourceControlCreateOption{ rmclient.WithMaxWaitDuration(runaway.MaxWaitDuration), } - if deploymode.IsStarter() { + if deploymode.IsStarter() && config.GetGlobalConfig().StarterParams.EnableGetResourceGroupDegraded { opts = append(opts, // Keep degraded-group synthesis inside the controller so degraded // resource groups are not inserted into the controller cache. rmclient.WithDegradedRUSettings(newDefaultDegradedRUSettings()), rmclient.WithDegradedModeWaitDuration(defaultDegradedModeWaitTimeout), - ) - } - if strings.Contains(config.GetGlobalConfig().StarterParams.PodNamespace, "vip") { - opts = append(opts, rmclient.WithWaitRetryInterval(tokenWaitRetryInterval), rmclient.WithWaitRetryTimes(tokenWaitRetryTimes), ) diff --git a/pkg/domain/runaway_test.go b/pkg/domain/runaway_test.go index 0b270489269f4..1859791a55fa4 100644 --- a/pkg/domain/runaway_test.go +++ b/pkg/domain/runaway_test.go @@ -16,10 +16,12 @@ package domain import ( "context" + "encoding/json" "testing" "time" "github.com/pingcap/kvproto/pkg/kvrpcpb" + "github.com/pingcap/kvproto/pkg/meta_storagepb" rmpb "github.com/pingcap/kvproto/pkg/resource_manager" "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/config/deploymode" @@ -30,6 +32,7 @@ import ( "github.com/tikv/client-go/v2/tikvrpc" pd "github.com/tikv/pd/client" pderr "github.com/tikv/pd/client/errs" + "github.com/tikv/pd/client/opt" rmclient "github.com/tikv/pd/client/resource_group/controller" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -37,8 +40,9 @@ import ( type resourceGroupProviderStub struct { rmclient.ResourceGroupProvider - resourceGroup *rmpb.ResourceGroup - resourceErr error + resourceGroup *rmpb.ResourceGroup + resourceErr error + controllerConfig *rmclient.Config } func newResourceGroupProviderStub(t *testing.T, resourceGroup *rmpb.ResourceGroup, resourceErr error) *resourceGroupProviderStub { @@ -59,6 +63,22 @@ func (s *resourceGroupProviderStub) GetResourceGroup(context.Context, string, .. return s.resourceGroup, s.resourceErr } +func (s *resourceGroupProviderStub) Get(ctx context.Context, key []byte, opts ...opt.MetaStorageOption) (*meta_storagepb.GetResponse, error) { + if s.controllerConfig == nil { + return s.ResourceGroupProvider.Get(ctx, key, opts...) + } + value, err := json.Marshal(s.controllerConfig) + if err != nil { + return nil, err + } + return &meta_storagepb.GetResponse{ + Kvs: []*meta_storagepb.KeyValue{{ + Key: key, + Value: value, + }}, + }, nil +} + func newStarterControllerForTest(t *testing.T, provider rmclient.ResourceGroupProvider) *rmclient.ResourceGroupsController { t.Helper() ctx, cancel := context.WithCancel(context.Background()) @@ -103,6 +123,8 @@ func newTransientGetResourceGroupErr(name string) error { } func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { + restoreConfig := config.RestoreFunc() + t.Cleanup(restoreConfig) if kerneltype.IsNextGen() { // Preserve the process-wide deploy mode because deploymode.IsStarter reads // it directly when newResourceGroupsControllerOptions builds controller options. @@ -146,6 +168,9 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { return } + config.UpdateGlobal(func(conf *config.Config) { + conf.StarterParams.EnableGetResourceGroupDegraded = true + }) controllerWithFallback := newStarterControllerForTest(t, provider) group, err := controllerWithFallback.GetResourceGroup("test-group") require.NoError(t, err) @@ -192,11 +217,16 @@ func TestStarterSwitchGroupUsesDegradedGroupOnTransientLookupError(t *testing.T) if !kerneltype.IsNextGen() { t.Skip("Starter deploy mode is only available in NextGen builds") } + restoreConfig := config.RestoreFunc() + t.Cleanup(restoreConfig) originalDeployMode := deploymode.Get() t.Cleanup(func() { require.NoError(t, deploymode.Set(originalDeployMode)) }) + config.UpdateGlobal(func(conf *config.Config) { + conf.StarterParams.EnableGetResourceGroupDegraded = true + }) provider := newResourceGroupProviderStub(t, nil, newTransientGetResourceGroupErr("target-switch-group")) controller := newStarterControllerForTest(t, provider) manager := runaway.NewRunawayManager(controller, "127.0.0.1:4000", nil, make(chan struct{}), nil, nil) @@ -225,28 +255,72 @@ func TestStarterSwitchGroupUsesDegradedGroupOnTransientLookupError(t *testing.T) require.Equal(t, "target-switch-group", req.GetResourceControlContext().GetResourceGroupName()) } -func TestResourceGroupsControllerOptionsUseStarterPodNamespaceForVIPWaitRetry(t *testing.T) { +func TestResourceGroupsControllerOptionsUseExplicitGetResourceGroupDegraded(t *testing.T) { + if !kerneltype.IsNextGen() { + t.Skip("Starter deploy mode is only available in NextGen builds") + } restoreConfig := config.RestoreFunc() t.Cleanup(restoreConfig) - vipOptionCount := 3 - standardOptionCount := 1 - if kerneltype.IsNextGen() { - originalDeployMode := deploymode.Get() - t.Cleanup(func() { - require.NoError(t, deploymode.Set(originalDeployMode)) + originalDeployMode := deploymode.Get() + t.Cleanup(func() { + require.NoError(t, deploymode.Set(originalDeployMode)) + }) + + newController := func(t *testing.T) *rmclient.ResourceGroupsController { + t.Helper() + provider := newResourceGroupProviderStub(t, nil, nil) + provider.controllerConfig = rmclient.DefaultConfig() + provider.controllerConfig.WaitRetryInterval = rmclient.NewDuration(250 * time.Millisecond) + provider.controllerConfig.WaitRetryTimes = 4 + provider.controllerConfig.LTBTokenRPCMaxDelay = rmclient.NewDuration(time.Second) + + controller, err := rmclient.NewResourceGroupController( + context.Background(), + 1, + provider, + nil, + 0, + newResourceGroupsControllerOptions()..., + ) + require.NoError(t, err) + return controller + } + + t.Run("enabled explicitly in starter", func(t *testing.T) { + require.NoError(t, deploymode.Set(deploymode.Starter)) + config.UpdateGlobal(func(conf *config.Config) { + conf.StarterParams.PodNamespace = "starter-standard-ns" + conf.StarterParams.EnableGetResourceGroupDegraded = true }) + + ruConfig := newController(t).GetConfig() + require.Equal(t, tokenWaitRetryInterval, ruConfig.WaitRetryInterval) + require.Equal(t, tokenWaitRetryTimes, ruConfig.WaitRetryTimes) + require.Equal(t, defaultDegradedModeWaitTimeout, ruConfig.DegradedModeWaitDuration) + }) + + t.Run("namespace alone does not enable degraded mode", func(t *testing.T) { require.NoError(t, deploymode.Set(deploymode.Starter)) - vipOptionCount = 5 - standardOptionCount = 3 - } + config.UpdateGlobal(func(conf *config.Config) { + conf.StarterParams.PodNamespace = "starter-vip-ns" + conf.StarterParams.EnableGetResourceGroupDegraded = false + }) - config.UpdateGlobal(func(conf *config.Config) { - conf.StarterParams.PodNamespace = "starter-vip-ns" + ruConfig := newController(t).GetConfig() + require.Equal(t, 250*time.Millisecond, ruConfig.WaitRetryInterval) + require.Equal(t, 4, ruConfig.WaitRetryTimes) + require.Zero(t, ruConfig.DegradedModeWaitDuration) }) - require.Len(t, newResourceGroupsControllerOptions(), vipOptionCount) - config.UpdateGlobal(func(conf *config.Config) { - conf.StarterParams.PodNamespace = "starter-standard-ns" + t.Run("ignored outside starter", func(t *testing.T) { + require.NoError(t, deploymode.Set(deploymode.Premium)) + config.UpdateGlobal(func(conf *config.Config) { + conf.StarterParams.EnableGetResourceGroupDegraded = true + }) + + ruConfig := newController(t).GetConfig() + require.Equal(t, 250*time.Millisecond, ruConfig.WaitRetryInterval) + require.Equal(t, 4, ruConfig.WaitRetryTimes) + require.Zero(t, ruConfig.DegradedModeWaitDuration) }) - require.Len(t, newResourceGroupsControllerOptions(), standardOptionCount) } From 87fec4d3bfdb737c63407ada9aeffaaa23153ec6 Mon Sep 17 00:00:00 2001 From: ystaticy Date: Mon, 20 Jul 2026 18:07:26 +0800 Subject: [PATCH 18/21] fix discussion_r3586340548 Signed-off-by: ystaticy --- pkg/domain/runaway_test.go | 204 ++++++++++++++++--------------------- 1 file changed, 85 insertions(+), 119 deletions(-) diff --git a/pkg/domain/runaway_test.go b/pkg/domain/runaway_test.go index 1859791a55fa4..e0cf6861a5462 100644 --- a/pkg/domain/runaway_test.go +++ b/pkg/domain/runaway_test.go @@ -122,64 +122,24 @@ func newTransientGetResourceGroupErr(name string) error { } } -func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { +func restoreResourceGroupControllerTestState(t *testing.T) { + t.Helper() restoreConfig := config.RestoreFunc() t.Cleanup(restoreConfig) - if kerneltype.IsNextGen() { - // Preserve the process-wide deploy mode because deploymode.IsStarter reads - // it directly when newResourceGroupsControllerOptions builds controller options. - originalDeployMode := deploymode.Get() - t.Cleanup(func() { - require.NoError(t, deploymode.Set(originalDeployMode)) - }) - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - provider := newResourceGroupProviderStub(t, &rmpb.ResourceGroup{ - Name: "test-group", - Mode: rmpb.GroupMode_RUMode, - RUSettings: &rmpb.GroupRequestUnitSettings{ - RU: &rmpb.TokenBucket{ - Settings: &rmpb.TokenLimitSettings{FillRate: 1}, - }, - }, - }, newTransientGetResourceGroupErr("test-group")) - if !kerneltype.IsNextGen() { - controller, err := rmclient.NewResourceGroupController( - ctx, - 1, - provider, - nil, - 0, - newResourceGroupsControllerOptions()..., - ) - require.NoError(t, err) - controller.Start(ctx) - defer func() { - require.NoError(t, controller.Stop()) - }() - - group, err := controller.GetResourceGroup("test-group") - require.Error(t, err) - require.Nil(t, group) return } - - config.UpdateGlobal(func(conf *config.Config) { - conf.StarterParams.EnableGetResourceGroupDegraded = true + // Preserve the process-wide deploy mode because deploymode.IsStarter reads + // it directly when newResourceGroupsControllerOptions builds controller options. + originalDeployMode := deploymode.Get() + t.Cleanup(func() { + require.NoError(t, deploymode.Set(originalDeployMode)) }) - controllerWithFallback := newStarterControllerForTest(t, provider) - group, err := controllerWithFallback.GetResourceGroup("test-group") - require.NoError(t, err) - requireDegradedResourceGroup(t, group, "test-group") +} - // The degraded group should not be cached. Once the provider recovers, the - // next lookup should observe the real resource-group metadata. - provider.resourceGroup = &rmpb.ResourceGroup{ - Name: "test-group", +func newTestResourceGroup(name string) *rmpb.ResourceGroup { + return &rmpb.ResourceGroup{ + Name: name, Mode: rmpb.GroupMode_RUMode, RUSettings: &rmpb.GroupRequestUnitSettings{ RU: &rmpb.TokenBucket{ @@ -187,84 +147,50 @@ func TestResourceGroupsControllerOptionsProvideDegradedFallback(t *testing.T) { }, }, } - provider.resourceErr = nil - group, err = controllerWithFallback.GetResourceGroup("test-group") - require.NoError(t, err) - require.Equal(t, provider.resourceGroup, group) - - require.NoError(t, deploymode.Set(deploymode.Premium)) - provider = newResourceGroupProviderStub(t, nil, newTransientGetResourceGroupErr("test-group")) - controllerWithoutFallback, err := rmclient.NewResourceGroupController( - ctx, - 2, - provider, - nil, - 0, - newResourceGroupsControllerOptions()..., - ) - require.NoError(t, err) - controllerWithoutFallback.Start(ctx) - defer func() { - require.NoError(t, controllerWithoutFallback.Stop()) - }() - - group, err = controllerWithoutFallback.GetResourceGroup("test-group") - require.Error(t, err) - require.Nil(t, group) } -func TestStarterSwitchGroupUsesDegradedGroupOnTransientLookupError(t *testing.T) { +func TestStarterDegradedResourceGroup(t *testing.T) { if !kerneltype.IsNextGen() { t.Skip("Starter deploy mode is only available in NextGen builds") } - restoreConfig := config.RestoreFunc() - t.Cleanup(restoreConfig) - originalDeployMode := deploymode.Get() - t.Cleanup(func() { - require.NoError(t, deploymode.Set(originalDeployMode)) - }) - config.UpdateGlobal(func(conf *config.Config) { - conf.StarterParams.EnableGetResourceGroupDegraded = true + t.Run("fallback", func(t *testing.T) { + restoreResourceGroupControllerTestState(t) + config.UpdateGlobal(func(conf *config.Config) { + conf.StarterParams.EnableGetResourceGroupDegraded = true + }) + + provider := newResourceGroupProviderStub(t, nil, newTransientGetResourceGroupErr("test-group")) + controller := newStarterControllerForTest(t, provider) + group, err := controller.GetResourceGroup("test-group") + require.NoError(t, err) + requireDegradedResourceGroup(t, group, "test-group") }) - provider := newResourceGroupProviderStub(t, nil, newTransientGetResourceGroupErr("target-switch-group")) - controller := newStarterControllerForTest(t, provider) - manager := runaway.NewRunawayManager(controller, "127.0.0.1:4000", nil, make(chan struct{}), nil, nil) - t.Cleanup(manager.Stop) - checker := runaway.NewChecker( - manager, - "source-group", - &rmpb.RunawaySettings{ - Action: rmpb.RunawayAction_SwitchGroup, - SwitchGroupName: "target-switch-group", - Rule: &rmpb.RunawayRule{ProcessedKeys: 1}, - }, - "SELECT 1", - "sql_digest", - "plan_digest", - time.Now(), - ) - require.NoError(t, checker.CheckThresholds(nil, 10, nil)) - req := &tikvrpc.Request{ - Context: kvrpcpb.Context{ - ResourceControlContext: &kvrpcpb.ResourceControlContext{}, - }, - } - require.NoError(t, checker.BeforeCopRequest(req)) - require.Equal(t, "target-switch-group", req.GetResourceControlContext().GetResourceGroupName()) + t.Run("recovery does not cache degraded group", func(t *testing.T) { + restoreResourceGroupControllerTestState(t) + config.UpdateGlobal(func(conf *config.Config) { + conf.StarterParams.EnableGetResourceGroupDegraded = true + }) + + provider := newResourceGroupProviderStub(t, nil, newTransientGetResourceGroupErr("test-group")) + controller := newStarterControllerForTest(t, provider) + group, err := controller.GetResourceGroup("test-group") + require.NoError(t, err) + requireDegradedResourceGroup(t, group, "test-group") + + provider.resourceGroup = newTestResourceGroup("test-group") + provider.resourceErr = nil + group, err = controller.GetResourceGroup("test-group") + require.NoError(t, err) + require.Equal(t, provider.resourceGroup, group) + }) } -func TestResourceGroupsControllerOptionsUseExplicitGetResourceGroupDegraded(t *testing.T) { +func TestResourceGroupsControllerOptions(t *testing.T) { if !kerneltype.IsNextGen() { t.Skip("Starter deploy mode is only available in NextGen builds") } - restoreConfig := config.RestoreFunc() - t.Cleanup(restoreConfig) - originalDeployMode := deploymode.Get() - t.Cleanup(func() { - require.NoError(t, deploymode.Set(originalDeployMode)) - }) newController := func(t *testing.T) *rmclient.ResourceGroupsController { t.Helper() @@ -286,7 +212,8 @@ func TestResourceGroupsControllerOptionsUseExplicitGetResourceGroupDegraded(t *t return controller } - t.Run("enabled explicitly in starter", func(t *testing.T) { + t.Run("starter enables degraded mode explicitly", func(t *testing.T) { + restoreResourceGroupControllerTestState(t) require.NoError(t, deploymode.Set(deploymode.Starter)) config.UpdateGlobal(func(conf *config.Config) { conf.StarterParams.PodNamespace = "starter-standard-ns" @@ -299,7 +226,8 @@ func TestResourceGroupsControllerOptionsUseExplicitGetResourceGroupDegraded(t *t require.Equal(t, defaultDegradedModeWaitTimeout, ruConfig.DegradedModeWaitDuration) }) - t.Run("namespace alone does not enable degraded mode", func(t *testing.T) { + t.Run("starter namespace alone does not enable degraded mode", func(t *testing.T) { + restoreResourceGroupControllerTestState(t) require.NoError(t, deploymode.Set(deploymode.Starter)) config.UpdateGlobal(func(conf *config.Config) { conf.StarterParams.PodNamespace = "starter-vip-ns" @@ -312,7 +240,8 @@ func TestResourceGroupsControllerOptionsUseExplicitGetResourceGroupDegraded(t *t require.Zero(t, ruConfig.DegradedModeWaitDuration) }) - t.Run("ignored outside starter", func(t *testing.T) { + t.Run("non starter ignores degraded flag", func(t *testing.T) { + restoreResourceGroupControllerTestState(t) require.NoError(t, deploymode.Set(deploymode.Premium)) config.UpdateGlobal(func(conf *config.Config) { conf.StarterParams.EnableGetResourceGroupDegraded = true @@ -324,3 +253,40 @@ func TestResourceGroupsControllerOptionsUseExplicitGetResourceGroupDegraded(t *t require.Zero(t, ruConfig.DegradedModeWaitDuration) }) } + +func TestStarterRunawaySwitchGroup(t *testing.T) { + if !kerneltype.IsNextGen() { + t.Skip("Starter deploy mode is only available in NextGen builds") + } + restoreResourceGroupControllerTestState(t) + + config.UpdateGlobal(func(conf *config.Config) { + conf.StarterParams.EnableGetResourceGroupDegraded = true + }) + provider := newResourceGroupProviderStub(t, nil, newTransientGetResourceGroupErr("target-switch-group")) + controller := newStarterControllerForTest(t, provider) + manager := runaway.NewRunawayManager(controller, "127.0.0.1:4000", nil, make(chan struct{}), nil, nil) + t.Cleanup(manager.Stop) + checker := runaway.NewChecker( + manager, + "source-group", + &rmpb.RunawaySettings{ + Action: rmpb.RunawayAction_SwitchGroup, + SwitchGroupName: "target-switch-group", + Rule: &rmpb.RunawayRule{ProcessedKeys: 1}, + }, + "SELECT 1", + "sql_digest", + "plan_digest", + time.Now(), + ) + + require.NoError(t, checker.CheckThresholds(nil, 10, nil)) + req := &tikvrpc.Request{ + Context: kvrpcpb.Context{ + ResourceControlContext: &kvrpcpb.ResourceControlContext{}, + }, + } + require.NoError(t, checker.BeforeCopRequest(req)) + require.Equal(t, "target-switch-group", req.GetResourceControlContext().GetResourceGroupName()) +} From 16506f75a5dc38ca29de6e562c52d1bbfb708a3c Mon Sep 17 00:00:00 2001 From: ystaticy Date: Tue, 21 Jul 2026 15:14:29 +0800 Subject: [PATCH 19/21] add code comments Signed-off-by: ystaticy --- pkg/domain/resource_group_controller_options.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/domain/resource_group_controller_options.go b/pkg/domain/resource_group_controller_options.go index ada7a0640cdcb..8cf38c2c2644e 100644 --- a/pkg/domain/resource_group_controller_options.go +++ b/pkg/domain/resource_group_controller_options.go @@ -49,8 +49,13 @@ func newResourceGroupsControllerOptions() []rmclient.ResourceControlCreateOption } if deploymode.IsStarter() && config.GetGlobalConfig().StarterParams.EnableGetResourceGroupDegraded { opts = append(opts, - // Keep degraded-group synthesis inside the controller so degraded - // resource groups are not inserted into the controller cache. + // This Starter-only degraded path is a best-effort UX fallback for + // temporary GetResourceGroup failures. It provides a permissive + // group so user requests do not fail immediately; it is not intended + // to define a precise cross-RPC RU limit or response-side accounting + // contract while resource manager is unavailable. Keep synthesis + // inside the controller so degraded groups are not inserted into the + // normal metadata cache. rmclient.WithDegradedRUSettings(newDefaultDegradedRUSettings()), rmclient.WithDegradedModeWaitDuration(defaultDegradedModeWaitTimeout), rmclient.WithWaitRetryInterval(tokenWaitRetryInterval), From 606f1bf76b942cc62062df5aada2988af8b6f8ea Mon Sep 17 00:00:00 2001 From: ystaticy Date: Tue, 21 Jul 2026 15:39:46 +0800 Subject: [PATCH 20/21] remove unused code Signed-off-by: ystaticy --- cmd/tidb-server/main.go | 17 +++++++++-------- cmd/tidb-server/main_test.go | 26 ++++++++------------------ pkg/config/config.go | 14 +------------- pkg/domain/runaway_test.go | 4 +--- 4 files changed, 19 insertions(+), 42 deletions(-) diff --git a/cmd/tidb-server/main.go b/cmd/tidb-server/main.go index 8cafdf07a8092..b9476df7fdbf7 100644 --- a/cmd/tidb-server/main.go +++ b/cmd/tidb-server/main.go @@ -1416,10 +1416,6 @@ func applyStarterAdditionalParams(cfg *config.Config, raw string) error { if err != nil { return err } - cfg.StarterParams.ManagerNamespace = params.managerNamespace - cfg.StarterParams.PodName = params.podName - cfg.StarterParams.PodIP = params.podIP - cfg.StarterParams.PodNamespace = params.podNamespace cfg.StarterParams.EnableGetResourceGroupDegraded = params.enableGetResourceGroupDegraded return nil } @@ -1447,18 +1443,23 @@ func createMgrClientForStarter() (tidbmanager.Client, error) { return nil, err } + params, err := parseStarterAdditionalParams(getStarterAdditionalParams()) + if err != nil { + return nil, err + } + managerAddr := cfg.StarterParams.ManagerAddr if managerAddr == "" { - managerNs := cfg.StarterParams.ManagerNamespace + managerNs := params.managerNamespace if managerNs == "" { return nil, fmt.Errorf("manager notifier requires manager-addr config or manager-namespace in --starter-additional-params") } managerAddr = fmt.Sprintf("manager-server.%s.svc:8000", managerNs) } - podName := cfg.StarterParams.PodName - podIP := cfg.StarterParams.PodIP - namespace := cfg.StarterParams.PodNamespace + podName := params.podName + podIP := params.podIP + namespace := params.podNamespace if podName == "" || podIP == "" || namespace == "" { return nil, fmt.Errorf("manager notifier requires --starter-additional-params with pod-name, pod-ip and pod-namespace: pod-name=%q, pod-ip=%q, pod-namespace=%q", podName, podIP, namespace) diff --git a/cmd/tidb-server/main_test.go b/cmd/tidb-server/main_test.go index 317e2ac8d75ab..67b6ff9201039 100644 --- a/cmd/tidb-server/main_test.go +++ b/cmd/tidb-server/main_test.go @@ -92,9 +92,6 @@ func TestOverrideConfigKeyspaceActivateMode(t *testing.T) { cfg.DeployMode = deploymode.Starter overrideConfig(cfg, fset) require.True(t, cfg.KeyspaceActivateMode) - require.Equal(t, "pod-1", cfg.StarterParams.PodName) - require.Equal(t, "10.0.0.1", cfg.StarterParams.PodIP) - require.Equal(t, "ns-1", cfg.StarterParams.PodNamespace) require.True(t, cfg.StarterParams.EnableGetResourceGroupDegraded) require.Equal(t, "pod-name=pod-1,pod-ip=10.0.0.1,pod-namespace=ns-1,enable-get-resource-group-degraded=true", *starterAdditionalParams) } @@ -213,37 +210,30 @@ func TestCreateMgrClientRequiresPodIdentityInStarter(t *testing.T) { require.ErrorContains(t, err, "manager notifier requires --starter-additional-params") duplicatedParam := "pod-name=pod-1,pod-name=pod-2,pod-ip=10.0.0.1,pod-namespace=ns-1" - config.UpdateGlobal(func(conf *config.Config) { - conf.StarterParams.ManagerNamespace = "" - conf.StarterParams.PodName = "" - conf.StarterParams.PodIP = "" - conf.StarterParams.PodNamespace = "" - }) - err = applyStarterAdditionalParams(config.GetGlobalConfig(), duplicatedParam) + starterAdditionalParams = &duplicatedParam + _, err = createMgrClientForStarter() require.ErrorContains(t, err, `starter additional param "pod-name" is duplicated`) unknownParam := "pod-name=pod-1,pod-ip=10.0.0.1,pod-namespace=ns-1,unknown=value" - err = applyStarterAdditionalParams(config.GetGlobalConfig(), unknownParam) + starterAdditionalParams = &unknownParam + _, err = createMgrClientForStarter() require.ErrorContains(t, err, `unknown starter additional param "unknown"`) invalidBoolParam := "enable-get-resource-group-degraded=definitely" - err = applyStarterAdditionalParams(config.GetGlobalConfig(), invalidBoolParam) + starterAdditionalParams = &invalidBoolParam + _, err = createMgrClientForStarter() require.ErrorContains(t, err, `starter additional param "enable-get-resource-group-degraded" must be a bool`) config.UpdateGlobal(func(conf *config.Config) { conf.StarterParams.ManagerAddr = "" - conf.StarterParams.ManagerNamespace = "" - conf.StarterParams.PodName = "" - conf.StarterParams.PodIP = "" - conf.StarterParams.PodNamespace = "" }) missingManagerNamespace := "pod-name=pod-1,pod-ip=10.0.0.1,pod-namespace=ns-1" - require.NoError(t, applyStarterAdditionalParams(config.GetGlobalConfig(), missingManagerNamespace)) + starterAdditionalParams = &missingManagerNamespace _, err = createMgrClientForStarter() require.ErrorContains(t, err, "manager notifier requires manager-addr config or manager-namespace in --starter-additional-params") validParams := "manager-namespace=manager-ns,pod-name=pod-1,pod-ip=10.0.0.1,pod-namespace=ns-1" - require.NoError(t, applyStarterAdditionalParams(config.GetGlobalConfig(), validParams)) + starterAdditionalParams = &validParams cli, err := createMgrClientForStarter() require.NoError(t, err) require.NotNil(t, cli) diff --git a/pkg/config/config.go b/pkg/config/config.go index 63c3fb917c588..a979be95951f2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1094,20 +1094,8 @@ type StarterParams struct { // ManagerAddr is the TiDB manager address used by the shutdown notifier. // When empty and EnableManagerNotifier is true, the Starter path derives the service address from starter additional params. ManagerAddr string `toml:"manager-addr" json:"manager-addr,omitempty"` - // ManagerNamespace captures the runtime manager namespace from --starter-additional-params. - // It is command-line derived runtime metadata rather than file-backed config. - ManagerNamespace string `toml:"-" json:"-"` - // PodName captures the runtime pod name from --starter-additional-params. - // It is command-line derived runtime metadata rather than file-backed config. - PodName string `toml:"-" json:"-"` - // PodIP captures the runtime pod IP from --starter-additional-params. - // It is command-line derived runtime metadata rather than file-backed config. - PodIP string `toml:"-" json:"-"` - // PodNamespace captures the runtime pod namespace from --starter-additional-params. - // It is command-line derived runtime metadata rather than file-backed config. - PodNamespace string `toml:"-" json:"-"` // EnableGetResourceGroupDegraded enables degraded GetResourceGroup handling for resource control. - // It is command-line derived runtime metadata rather than file-backed config. + // It is populated from --starter-additional-params and is not file-backed config. EnableGetResourceGroupDegraded bool `toml:"-" json:"-"` // MaxImportDataSize is the maximum total real source data size allowed for IMPORT INTO. // Zero means unlimited. diff --git a/pkg/domain/runaway_test.go b/pkg/domain/runaway_test.go index e0cf6861a5462..d1ae572df5b93 100644 --- a/pkg/domain/runaway_test.go +++ b/pkg/domain/runaway_test.go @@ -216,7 +216,6 @@ func TestResourceGroupsControllerOptions(t *testing.T) { restoreResourceGroupControllerTestState(t) require.NoError(t, deploymode.Set(deploymode.Starter)) config.UpdateGlobal(func(conf *config.Config) { - conf.StarterParams.PodNamespace = "starter-standard-ns" conf.StarterParams.EnableGetResourceGroupDegraded = true }) @@ -226,11 +225,10 @@ func TestResourceGroupsControllerOptions(t *testing.T) { require.Equal(t, defaultDegradedModeWaitTimeout, ruConfig.DegradedModeWaitDuration) }) - t.Run("starter namespace alone does not enable degraded mode", func(t *testing.T) { + t.Run("starter without degraded flag keeps default retry settings", func(t *testing.T) { restoreResourceGroupControllerTestState(t) require.NoError(t, deploymode.Set(deploymode.Starter)) config.UpdateGlobal(func(conf *config.Config) { - conf.StarterParams.PodNamespace = "starter-vip-ns" conf.StarterParams.EnableGetResourceGroupDegraded = false }) From 8b9cf6d39d777d01993e762d681c48d173e7941d Mon Sep 17 00:00:00 2001 From: ystaticy Date: Tue, 21 Jul 2026 15:50:19 +0800 Subject: [PATCH 21/21] fix config name Signed-off-by: ystaticy --- cmd/tidb-server/main.go | 16 ++++++++-------- cmd/tidb-server/main_test.go | 10 +++++----- pkg/config/config.go | 4 ++-- pkg/domain/resource_group_controller_options.go | 4 ++-- pkg/domain/runaway_test.go | 12 ++++++------ 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/cmd/tidb-server/main.go b/cmd/tidb-server/main.go index b9476df7fdbf7..98891d4aec575 100644 --- a/cmd/tidb-server/main.go +++ b/cmd/tidb-server/main.go @@ -1351,11 +1351,11 @@ func prepareKeyspaceObservabilityForStarter(metadata map[string]string) error { } type starterParams struct { - managerNamespace string - podName string - podIP string - podNamespace string - enableGetResourceGroupDegraded bool + managerNamespace string + podName string + podIP string + podNamespace string + enableRGFallback bool } func parseStarterAdditionalParams(raw string) (starterParams, error) { @@ -1398,12 +1398,12 @@ func parseStarterAdditionalParams(raw string) (starterParams, error) { params.podIP = value case "pod-namespace": params.podNamespace = value - case "enable-get-resource-group-degraded": + case "enable-rg-fallback": enable, err := strconv.ParseBool(value) if err != nil { return params, fmt.Errorf("starter additional param %q must be a bool: %w", key, err) } - params.enableGetResourceGroupDegraded = enable + params.enableRGFallback = enable default: return params, fmt.Errorf("unknown starter additional param %q", key) } @@ -1416,7 +1416,7 @@ func applyStarterAdditionalParams(cfg *config.Config, raw string) error { if err != nil { return err } - cfg.StarterParams.EnableGetResourceGroupDegraded = params.enableGetResourceGroupDegraded + cfg.StarterParams.EnableRGFallback = params.enableRGFallback return nil } diff --git a/cmd/tidb-server/main_test.go b/cmd/tidb-server/main_test.go index 67b6ff9201039..307444e36eb9e 100644 --- a/cmd/tidb-server/main_test.go +++ b/cmd/tidb-server/main_test.go @@ -85,15 +85,15 @@ func TestOverrideConfigKeyspaceActivateMode(t *testing.T) { fset := initFlagSet() require.NoError(t, fset.Parse([]string{ "--keyspace-activate=true", - "--starter-additional-params=pod-name=pod-1,pod-ip=10.0.0.1,pod-namespace=ns-1,enable-get-resource-group-degraded=true", + "--starter-additional-params=pod-name=pod-1,pod-ip=10.0.0.1,pod-namespace=ns-1,enable-rg-fallback=true", })) cfg := config.NewConfig() cfg.DeployMode = deploymode.Starter overrideConfig(cfg, fset) require.True(t, cfg.KeyspaceActivateMode) - require.True(t, cfg.StarterParams.EnableGetResourceGroupDegraded) - require.Equal(t, "pod-name=pod-1,pod-ip=10.0.0.1,pod-namespace=ns-1,enable-get-resource-group-degraded=true", *starterAdditionalParams) + require.True(t, cfg.StarterParams.EnableRGFallback) + require.Equal(t, "pod-name=pod-1,pod-ip=10.0.0.1,pod-namespace=ns-1,enable-rg-fallback=true", *starterAdditionalParams) } func TestSetGlobalVars(t *testing.T) { @@ -219,10 +219,10 @@ func TestCreateMgrClientRequiresPodIdentityInStarter(t *testing.T) { _, err = createMgrClientForStarter() require.ErrorContains(t, err, `unknown starter additional param "unknown"`) - invalidBoolParam := "enable-get-resource-group-degraded=definitely" + invalidBoolParam := "enable-rg-fallback=definitely" starterAdditionalParams = &invalidBoolParam _, err = createMgrClientForStarter() - require.ErrorContains(t, err, `starter additional param "enable-get-resource-group-degraded" must be a bool`) + require.ErrorContains(t, err, `starter additional param "enable-rg-fallback" must be a bool`) config.UpdateGlobal(func(conf *config.Config) { conf.StarterParams.ManagerAddr = "" diff --git a/pkg/config/config.go b/pkg/config/config.go index a979be95951f2..07e0c58d7c460 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1094,9 +1094,9 @@ type StarterParams struct { // ManagerAddr is the TiDB manager address used by the shutdown notifier. // When empty and EnableManagerNotifier is true, the Starter path derives the service address from starter additional params. ManagerAddr string `toml:"manager-addr" json:"manager-addr,omitempty"` - // EnableGetResourceGroupDegraded enables degraded GetResourceGroup handling for resource control. + // EnableRGFallback enables resource group lookup fallback for resource control. // It is populated from --starter-additional-params and is not file-backed config. - EnableGetResourceGroupDegraded bool `toml:"-" json:"-"` + EnableRGFallback bool `toml:"-" json:"-"` // MaxImportDataSize is the maximum total real source data size allowed for IMPORT INTO. // Zero means unlimited. MaxImportDataSize configtypes.ByteSize `toml:"max-import-data-size" json:"max-import-data-size,omitempty"` diff --git a/pkg/domain/resource_group_controller_options.go b/pkg/domain/resource_group_controller_options.go index 8cf38c2c2644e..abb9c0f1ef749 100644 --- a/pkg/domain/resource_group_controller_options.go +++ b/pkg/domain/resource_group_controller_options.go @@ -47,9 +47,9 @@ func newResourceGroupsControllerOptions() []rmclient.ResourceControlCreateOption opts := []rmclient.ResourceControlCreateOption{ rmclient.WithMaxWaitDuration(runaway.MaxWaitDuration), } - if deploymode.IsStarter() && config.GetGlobalConfig().StarterParams.EnableGetResourceGroupDegraded { + if deploymode.IsStarter() && config.GetGlobalConfig().StarterParams.EnableRGFallback { opts = append(opts, - // This Starter-only degraded path is a best-effort UX fallback for + // This Starter-only fallback path is a best-effort UX fallback for // temporary GetResourceGroup failures. It provides a permissive // group so user requests do not fail immediately; it is not intended // to define a precise cross-RPC RU limit or response-side accounting diff --git a/pkg/domain/runaway_test.go b/pkg/domain/runaway_test.go index d1ae572df5b93..dcec631574306 100644 --- a/pkg/domain/runaway_test.go +++ b/pkg/domain/runaway_test.go @@ -157,7 +157,7 @@ func TestStarterDegradedResourceGroup(t *testing.T) { t.Run("fallback", func(t *testing.T) { restoreResourceGroupControllerTestState(t) config.UpdateGlobal(func(conf *config.Config) { - conf.StarterParams.EnableGetResourceGroupDegraded = true + conf.StarterParams.EnableRGFallback = true }) provider := newResourceGroupProviderStub(t, nil, newTransientGetResourceGroupErr("test-group")) @@ -170,7 +170,7 @@ func TestStarterDegradedResourceGroup(t *testing.T) { t.Run("recovery does not cache degraded group", func(t *testing.T) { restoreResourceGroupControllerTestState(t) config.UpdateGlobal(func(conf *config.Config) { - conf.StarterParams.EnableGetResourceGroupDegraded = true + conf.StarterParams.EnableRGFallback = true }) provider := newResourceGroupProviderStub(t, nil, newTransientGetResourceGroupErr("test-group")) @@ -216,7 +216,7 @@ func TestResourceGroupsControllerOptions(t *testing.T) { restoreResourceGroupControllerTestState(t) require.NoError(t, deploymode.Set(deploymode.Starter)) config.UpdateGlobal(func(conf *config.Config) { - conf.StarterParams.EnableGetResourceGroupDegraded = true + conf.StarterParams.EnableRGFallback = true }) ruConfig := newController(t).GetConfig() @@ -229,7 +229,7 @@ func TestResourceGroupsControllerOptions(t *testing.T) { restoreResourceGroupControllerTestState(t) require.NoError(t, deploymode.Set(deploymode.Starter)) config.UpdateGlobal(func(conf *config.Config) { - conf.StarterParams.EnableGetResourceGroupDegraded = false + conf.StarterParams.EnableRGFallback = false }) ruConfig := newController(t).GetConfig() @@ -242,7 +242,7 @@ func TestResourceGroupsControllerOptions(t *testing.T) { restoreResourceGroupControllerTestState(t) require.NoError(t, deploymode.Set(deploymode.Premium)) config.UpdateGlobal(func(conf *config.Config) { - conf.StarterParams.EnableGetResourceGroupDegraded = true + conf.StarterParams.EnableRGFallback = true }) ruConfig := newController(t).GetConfig() @@ -259,7 +259,7 @@ func TestStarterRunawaySwitchGroup(t *testing.T) { restoreResourceGroupControllerTestState(t) config.UpdateGlobal(func(conf *config.Config) { - conf.StarterParams.EnableGetResourceGroupDegraded = true + conf.StarterParams.EnableRGFallback = true }) provider := newResourceGroupProviderStub(t, nil, newTransientGetResourceGroupErr("target-switch-group")) controller := newStarterControllerForTest(t, provider)